diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java index 12aa46c7d..b21b168ca 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java @@ -2549,7 +2549,8 @@ public boolean resettingNodeCursor() { windowEventInterpreter.addListener(uiChangeEventInterpreter); windowEventInterpreter.addListener(imageCaptioner); directTouchController = - new DirectTouchController(this, prefs, pipeline.getFeedbackReturner()); + new DirectTouchController( + this, prefs, pipeline.getFeedbackReturner(), windowEventInterpreter); windowEventInterpreter.addListener(directTouchController); ringerModeAndScreenMonitor.addScreenChangedListener(directTouchController); displayMonitor.addDisplayStateChangedListener(directTouchController); diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java index d2f8258d9..3e016fcb7 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java @@ -18,6 +18,7 @@ import static androidx.core.view.accessibility.AccessibilityNodeInfoCompat.FOCUS_INPUT; import static androidx.core.view.accessibility.AccessibilityWindowInfoCompat.TYPE_INPUT_METHOD; +import static com.google.android.accessibility.utils.AccessibilityWindowInfoUtils.WINDOW_ID_NONE; import static com.google.android.accessibility.utils.traversal.TraversalStrategy.SEARCH_FOCUS_FORWARD; import android.accessibilityservice.AccessibilityService; @@ -135,6 +136,17 @@ private boolean isOnTopWindow( return (currentActive.getLayer() > lastWindow.getLayer()) && activeRect.intersect(lastRect); } + /** + * Whether the node is still in the window, as {@link AccessibilityNodeInfoUtils#isInWindow} + * checks, with the window's root fetched already. + */ + private static boolean isInWindow( + AccessibilityNodeInfoCompat node, int windowId, AccessibilityNodeInfoCompat windowRoot) { + int nodeWindowId = node.getWindowId(); + return (nodeWindowId == WINDOW_ID_NONE || nodeWindowId == windowId) + && AccessibilityNodeInfoUtils.hasDescendant(windowRoot, node); + } + /** Restores last focus from {@link AccessibilityFocusActionHistory} to the active window. */ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) { @@ -175,12 +187,6 @@ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) return false; } - AccessibilityNodeInfoCompat root = - AccessibilityWindowInfoUtils.getRootCompat(currentActiveWindow); - if (root == null) { - return false; - } - int windowId = currentActiveWindow.getId(); int windowType = AccessibilityWindowInfoUtils.getType(currentActiveWindow); WindowIdentifier windowIdentifier = WindowIdentifier.create(windowId, screenState); @@ -202,6 +208,14 @@ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) return false; } + // Fetching the root waits for the window's app, which is drawing the new screen, so it is + // fetched only when there is a focus to restore, and a single time. + AccessibilityNodeInfoCompat root = + AccessibilityWindowInfoUtils.getRootCompat(currentActiveWindow); + if (root == null) { + return false; + } + long startTime = primesController.getTime(); AccessibilityNodeInfoCompat nodeToRestoreFocus = @@ -221,7 +235,7 @@ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) // b/365017291: Only check native nodes because sometimes the WebView is gone in the node // tree. || (!WebInterfaceUtils.isWebContainer(nodeToRestoreFocus) - && !AccessibilityNodeInfoUtils.isInWindow(nodeToRestoreFocus, currentActiveWindow))) { + && !isInWindow(nodeToRestoreFocus, windowId, root))) { LogUtils.e(TAG, "Do not restore focus from the invalid node " + nodeToRestoreFocus); return false; } diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt index e3981e7a9..82e9376cc 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt @@ -67,6 +67,8 @@ class DirectTouchController( private val service: AccessibilityService, private val prefs: SharedPreferences, private val feedback: Pipeline.FeedbackReturner, + /** Keeps the package of each window's root, which every look at the screen needs. */ + private val windowEventInterpreter: WindowEventInterpreter, ) : WindowEventInterpreter.WindowEventHandler, RingerModeAndScreenMonitor.ScreenChangedListener, @@ -222,8 +224,7 @@ class DirectTouchController( return } val windows = service.windows.orEmpty() - val mainPackage = - windows.firstOrNull { it.id == mainWindowId }?.root?.packageName?.toString() + val mainPackage = windows.firstOrNull { it.id == mainWindowId }?.let { rootPackage(it) } val directTyping = mainPackage != null && DirectTouchSettings.isDirectTyping(prefs, mainPackage) val shouldBeActive = mainPackage != null && shouldBeActive(windows, mainPackage) val navBarDirect = isNavBarDirect() @@ -349,9 +350,12 @@ class DirectTouchController( } } + /** The package of the window's root, which the window's app is asked for once per window. */ + private fun rootPackage(window: AccessibilityWindowInfo): String? = + windowEventInterpreter.getRootPackageName(window)?.toString() + private fun isSystemUiWindow(window: AccessibilityWindowInfo): Boolean = - window.type == AccessibilityWindowInfo.TYPE_SYSTEM && - window.root?.packageName?.toString() == SYSTEM_UI + window.type == AccessibilityWindowInfo.TYPE_SYSTEM && rootPackage(window) == SYSTEM_UI private fun shouldBeActive( windows: List, @@ -360,8 +364,11 @@ class DirectTouchController( if (mainPackage in IGNORED_PACKAGES) { return false } - val capable = DirectTouchCapability.declaresDirectTouch(service.packageManager, mainPackage) - DirectTouchSettings.onFirstSight(prefs, mainPackage, capable) + // Only the first sight of an app uses its manifest, which the package manager has to read. + if (!DirectTouchSettings.isSeen(prefs, mainPackage)) { + val capable = DirectTouchCapability.declaresDirectTouch(service.packageManager, mainPackage) + DirectTouchSettings.onFirstSight(prefs, mainPackage, capable) + } // A dialog is a window of its own, so any other application window is either a dialog from the // same app or another app. val otherApplicationWindows = @@ -371,11 +378,10 @@ class DirectTouchController( masterEnabled = DirectTouchSettings.isMasterEnabled(prefs), appEnabled = DirectTouchSettings.isAppEnabled(prefs, mainPackage), screenInteractive = screenInteractive && displayOn, - foreignAppWindow = - otherApplicationWindows.any { it.root?.packageName?.toString() != mainPackage }, - systemUiCoversHalf = systemUiCoversHalf(windows), dialogShowing = otherApplicationWindows.isNotEmpty(), textFieldFocused = textFieldFocused, + foreignAppWindow = { otherApplicationWindows.any { rootPackage(it) != mainPackage } }, + systemUiCoversHalf = { systemUiCoversHalf(windows) }, ) return DirectTouchPolicy.shouldBeActive(input) } @@ -384,11 +390,10 @@ class DirectTouchController( val displayHeight = displayBounds().height() val bounds = Rect() return windows.any { window -> - if (!isSystemUiWindow(window)) { - return@any false - } window.getBoundsInScreen(bounds) - DirectTouchPolicy.coversHalfScreen(window.isActive, bounds.height(), displayHeight) + // Telling System UI apart asks the window's app for its root, so the shape comes first. + DirectTouchPolicy.coversHalfScreen(window.isActive, bounds.height(), displayHeight) && + isSystemUiWindow(window) } } diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt index c4b211bb6..3d59922ff 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt @@ -16,18 +16,22 @@ package com.google.android.accessibility.talkback.directtouch -/** What is on screen and how the user set things up, as far as direct touch cares. */ +/** + * What is on screen and how the user set things up, as far as direct touch cares. The two checks + * that ask apps for their windows' roots are functions, so that they run only when the rest leave + * the answer open. + */ data class DirectTouchInput( val masterEnabled: Boolean, /** The app in front is turned on in the direct touch list. */ val appEnabled: Boolean, val screenInteractive: Boolean, - /** Another app's window is on top of the game. */ - val foreignAppWindow: Boolean, - /** The notification shade or quick settings cover at least half the screen. */ - val systemUiCoversHalf: Boolean, val dialogShowing: Boolean, val textFieldFocused: Boolean, + /** Another app's window is on top of the game. */ + val foreignAppWindow: () -> Boolean, + /** The notification shade or quick settings cover at least half the screen. */ + val systemUiCoversHalf: () -> Boolean, ) object DirectTouchPolicy { @@ -45,8 +49,8 @@ object DirectTouchPolicy { input.masterEnabled && input.appEnabled && input.screenInteractive && - !input.foreignAppWindow && - !input.systemUiCoversHalf && !input.dialogShowing && - !input.textFieldFocused + !input.textFieldFocused && + !input.foreignAppWindow() && + !input.systemUiCoversHalf() } diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt index 5ed8dda9c..e89934a52 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt @@ -94,6 +94,9 @@ object DirectTouchSettings { prefs.edit().putBoolean(PREF_TYPING_PREFIX + pkg, enabled).apply() } + /** Whether [pkg] has been seen, after which [onFirstSight] changes nothing for it. */ + fun isSeen(prefs: SharedPreferences, pkg: String): Boolean = pkg in stringSet(prefs, PREF_SEEN) + /** * Records that [pkg] has been seen. The first time, an app that [declaresCapability] is turned * on. Later calls change nothing, so an app the user turned off stays off. diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt new file mode 100644 index 000000000..63cb24161 --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt @@ -0,0 +1,218 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback.actor + +import android.accessibilityservice.AccessibilityService +import android.graphics.Rect +import android.os.SystemClock +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat +import com.google.android.accessibility.talkback.ActorState +import com.google.android.accessibility.talkback.ActorStateWritable +import com.google.android.accessibility.talkback.Pipeline +import com.google.android.accessibility.talkback.PrimesController +import com.google.android.accessibility.talkback.focusmanagement.interpreter.ScreenState +import com.google.android.accessibility.talkback.focusmanagement.record.AccessibilityFocusActionHistory +import com.google.android.accessibility.talkback.focusmanagement.record.AccessibilityFocusActionHistory.WindowIdentifier +import com.google.android.accessibility.talkback.focusmanagement.record.FocusActionInfo +import com.google.android.accessibility.talkback.monitor.InputMethodMonitor +import com.google.android.accessibility.utils.FocusFinder +import com.google.android.accessibility.utils.FormFactorUtils +import com.google.android.accessibility.utils.Performance.EVENT_ID_UNTRACKED +import com.google.android.accessibility.utils.input.WindowsDelegate +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.Implementation +import org.robolectric.annotation.Implements +import org.robolectric.shadows.ShadowAccessibilityWindowInfo + +/** + * Restoring the last focus when the screen changes, and what it costs: fetching a window's root + * waits for the window's app, which is busy drawing the new screen. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34], shadows = [FocusActorForScreenStateChangeTest.RootsShadow::class]) +class FocusActorForScreenStateChangeTest { + + class TestService : AccessibilityService() { + override fun onAccessibilityEvent(event: AccessibilityEvent) {} + + override fun onInterrupt() {} + } + + /** Counts how often a window's root is fetched. */ + @Implements(AccessibilityWindowInfo::class) + class RootsShadow : ShadowAccessibilityWindowInfo() { + @Implementation + override fun getRoot(): AccessibilityNodeInfo? { + rootRequests++ + return super.getRoot() + } + + companion object { + var rootRequests = 0 + } + } + + /** Names each window by its id, as the window interpreter would. */ + private val windows = + object : WindowsDelegate { + override fun getWindowTitle(windowId: Int): CharSequence = "Window $windowId" + + override fun getAccessibilityPaneTitle(windowId: Int): CharSequence = "" + + override fun isSplitScreenMode(displayId: Int): Boolean = false + + override fun hasAccessibilityPane(windowId: Int): Boolean = false + } + + private lateinit var service: TestService + private lateinit var history: AccessibilityFocusActionHistory + private lateinit var actor: FocusActorForScreenStateChange + private val focused = mutableListOf() + + @Before + fun setUp() { + service = Robolectric.setupService(TestService::class.java) + FormFactorUtils.initialize(service) + history = AccessibilityFocusActionHistory(service) + actor = + FocusActorForScreenStateChange( + service, + InputMethodMonitor(service), + FocusFinder(service), + PrimesController(), + ) + actor.setActorState( + ActorState( + ActorStateWritable( + null, null, null, null, history.reader, null, null, null, null, null, null, null, null, + null, + ) + ) + ) + actor.setPipeline( + Pipeline.FeedbackReturner { feedback -> + focused += feedback.failovers().first().focus()?.target() + true + } + ) + } + + @Test + fun newScreenWithoutAFocusToRestoreLeavesItsRootAlone() { + val app = window(APP, AccessibilityWindowInfo.TYPE_APPLICATION, root()) + shadowOf(service).setWindows(listOf(app)) + + assertFalse(restore(app)) + assertEquals(0, RootsShadow.rootRequests) + } + + @Test + fun systemWindowGetsNoFocusLastPutInAnApp() { + // The history has a focus for the shade's window id, whose node reports an application + // window, and the node is still there. Only the rule that keeps a focus from an app out of + // System UI stops it from being restored. + val root = root() + val app = window(SHADE, AccessibilityWindowInfo.TYPE_APPLICATION, root) + history.onAccessibilityFocusAction(button(app, root), focusInfo(), 1, screen(app)) + val shade = window(SHADE, AccessibilityWindowInfo.TYPE_SYSTEM, root) + shadowOf(service).setWindows(listOf(shade)) + assertNotNull( + history.reader.getLastFocusActionRecordInWindow(WindowIdentifier.create(SHADE, screen(shade))) + ) + + assertFalse(restore(shade)) + assertEquals(0, RootsShadow.rootRequests) + } + + @Test + fun focusLastPutInTheWindowIsRestored() { + val root = root() + val app = window(APP, AccessibilityWindowInfo.TYPE_APPLICATION, root) + history.onAccessibilityFocusAction(button(app, root), focusInfo(), 1, screen(app)) + shadowOf(service).setWindows(listOf(app)) + + assertTrue(restore(app)) + assertEquals(listOf("Play"), focused.map { it?.text?.toString() }) + // One root serves to find the node and to check that the node is still in the window. + assertEquals(1, RootsShadow.rootRequests) + } + + /** Restores the last focus into [active], counting only the roots that fetches. */ + private fun restore(active: AccessibilityWindowInfo): Boolean { + RootsShadow.rootRequests = 0 + return actor.restoreLastFocusedNode(screen(active), EVENT_ID_UNTRACKED) + } + + private fun screen(active: AccessibilityWindowInfo) = + ScreenState(windows, active, SystemClock.uptimeMillis()) + + private fun root() = AccessibilityNodeInfo().apply { packageName = APP_PACKAGE } + + private fun window(id: Int, type: Int, root: AccessibilityNodeInfo): AccessibilityWindowInfo { + val window = AccessibilityWindowInfo() + val shadow = shadowOf(window) + shadow.setId(id) + shadow.setType(type) + shadow.setActive(true) + shadow.setFocused(true) + shadow.setBoundsInScreen(Rect(0, 0, 1080, 2340)) + shadow.setRoot(root) + return window + } + + /** A button in the window, which can still be found there when its focus is restored. */ + private fun button( + window: AccessibilityWindowInfo, + root: AccessibilityNodeInfo, + ): AccessibilityNodeInfoCompat { + val node = AccessibilityNodeInfo() + node.packageName = APP_PACKAGE + node.className = "android.widget.Button" + node.text = "Play" + node.isVisibleToUser = true + node.isEnabled = true + node.isClickable = true + node.isFocusable = true + node.setBoundsInScreen(Rect(100, 1000, 980, 1150)) + shadowOf(node).setAccessibilityWindowInfo(window) + shadowOf(node).setRefreshReturnValue(true) + shadowOf(root).addChild(node) + return AccessibilityNodeInfoCompat.wrap(node) + } + + private fun focusInfo() = + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build() + + private companion object { + const val APP = 10 + const val SHADE = 20 + const val APP_PACKAGE = "com.example.app" + } +} diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchControllerTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchControllerTest.kt new file mode 100644 index 000000000..da999234b --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchControllerTest.kt @@ -0,0 +1,329 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback.directtouch + +import android.accessibilityservice.AccessibilityService +import android.content.pm.ApplicationInfo +import android.content.pm.PackageInfo +import android.content.pm.PackageManager +import android.graphics.Rect +import android.os.Bundle +import android.os.Looper +import android.os.SystemClock +import android.util.SparseArray +import android.view.Display +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ADDED +import android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_REMOVED +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import com.google.android.accessibility.talkback.Feedback +import com.google.android.accessibility.talkback.Pipeline +import com.google.android.accessibility.utils.input.WindowEventInterpreter +import com.google.android.accessibility.utils.monitor.DisplayMonitor +import java.time.Duration +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.Implementation +import org.robolectric.annotation.Implements +import org.robolectric.shadow.api.Shadow +import org.robolectric.shadows.ShadowAccessibilityService +import org.robolectric.shadows.ShadowAccessibilityWindowInfo +import org.robolectric.shadows.ShadowApplicationPackageManager +import org.robolectric.util.ReflectionHelpers +import org.robolectric.util.ReflectionHelpers.ClassParameter + +/** + * Feeds window changes through the window interpreter to direct touch, as the service does, and + * records what direct touch says and which passthrough regions it sends. It also counts what each + * look at the screen costs: fetching a window's root waits for the window's app, and reading an + * app's manifest asks the package manager. + */ +@RunWith(RobolectricTestRunner::class) +@Config( + sdk = [34], + qualifiers = "w360dp-h780dp-xxhdpi", + shadows = + [ + DirectTouchControllerTest.WindowsShadow::class, + DirectTouchControllerTest.RootsShadow::class, + DirectTouchControllerTest.ManifestsShadow::class, + ], +) +class DirectTouchControllerTest { + + class TestService : AccessibilityService() { + override fun onAccessibilityEvent(event: AccessibilityEvent) {} + + override fun onInterrupt() {} + } + + /** Lists the windows a test puts on screen, in a new list each time as Android does. */ + @Implements(AccessibilityService::class) + class WindowsShadow : ShadowAccessibilityService() { + var onScreen: List = emptyList() + + @Implementation + override fun getWindows(): List = ArrayList(onScreen) + + @Implementation + override fun getWindowsOnAllDisplays(): SparseArray> = + SparseArray>().apply { + put(Display.DEFAULT_DISPLAY, ArrayList(onScreen)) + } + } + + /** Counts how often a window's root is fetched. */ + @Implements(AccessibilityWindowInfo::class) + class RootsShadow : ShadowAccessibilityWindowInfo() { + @Implementation + override fun getRoot(): AccessibilityNodeInfo? { + rootRequests++ + return super.getRoot() + } + + companion object { + var rootRequests = 0 + } + } + + /** Counts how often the package manager is asked for an app's manifest, with its meta-data. */ + @Implements(className = "android.app.ApplicationPackageManager", isInAndroidSdk = false) + class ManifestsShadow : ShadowApplicationPackageManager() { + @Implementation + override fun getApplicationInfo(packageName: String, flags: Int): ApplicationInfo { + if (flags and PackageManager.GET_META_DATA != 0) { + manifestRequests++ + } + return super.getApplicationInfo(packageName, flags) + } + + companion object { + var manifestRequests = 0 + } + } + + private lateinit var service: TestService + private lateinit var serviceWindows: WindowsShadow + private lateinit var interpreter: WindowEventInterpreter + private lateinit var controller: DirectTouchController + private val prefs = FakeSharedPreferences() + private val transcript = mutableListOf() + + @Before + fun setUp() { + service = Robolectric.setupService(TestService::class.java) + serviceWindows = Shadow.extract(service) + interpreter = WindowEventInterpreter(service, DisplayMonitor(service)) + controller = + DirectTouchController( + service, + prefs, + Pipeline.FeedbackReturner { feedback -> + transcript += describe(feedback) + true + }, + interpreter, + ) + interpreter.addListener(controller) + installApp(GAME_PACKAGE, asksForDirectTouch = false) + installApp(GAME2_PACKAGE, asksForDirectTouch = true) + RootsShadow.rootRequests = 0 + ManifestsShadow.manifestRequests = 0 + } + + @Test + fun gameNotTurnedOnStaysOff() { + windowsChange(GAME, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game()) + // A toast comes and goes over the game. + windowsChange(TOAST, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game(), toast()) + windowsChange(TOAST, WINDOWS_CHANGE_REMOVED, statusBar(), navigationBar(), game()) + + assertTranscript() + // Direct touch looks at the screen several times for each window change. The game's root is + // fetched once, and its manifest is read once, the first time the game is seen. + assertEquals(1, RootsShadow.rootRequests) + assertEquals(1, ManifestsShadow.manifestRequests) + } + + @Test + fun gameTurnedOnHandsTouchBackWhileTheShadeIsOpen() { + DirectTouchSettings.setAppEnabled(prefs, GAME_PACKAGE, true) + + windowsChange(GAME, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game()) + val underShade = game(inactive = true) + windowsChange(SHADE, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), underShade, shade()) + windowsChange(SHADE, WINDOWS_CHANGE_REMOVED, statusBar(), navigationBar(), game()) + + assertTranscript( + "region [0,80][1080,2220] complex=false", + "say \"Direct touch on\"", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region none", + "say \"Direct touch off\"", + "region [0,80][1080,2220] complex=false", + "say \"Direct touch on\"", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + ) + // Only the game's root and the shade's are fetched, once each. The bars are too small to be the + // shade, so they are never asked whether they belong to System UI. + assertEquals(2, RootsShadow.rootRequests) + assertEquals(1, ManifestsShadow.manifestRequests) + } + + @Test + fun gameThatAsksForDirectTouchGetsItTheFirstTimeItIsSeen() { + windowsChange(GAME2, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game2()) + windowsChange(TOAST, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game2(), toast()) + windowsChange(TOAST, WINDOWS_CHANGE_REMOVED, statusBar(), navigationBar(), game2()) + + assertTranscript( + "region [0,80][1080,2220] complex=false", + "say \"Direct touch on\"", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + ) + assertEquals(1, RootsShadow.rootRequests) + assertEquals(1, ManifestsShadow.manifestRequests) + } + + // Screens + + private fun statusBar() = + window(STATUS_BAR, SYSTEM, "Status bar", Rect(0, 0, 1080, 80), SYSTEM_UI) + + private fun navigationBar() = + window(NAVIGATION_BAR, SYSTEM, "Navigation bar", Rect(0, 2220, 1080, 2340), SYSTEM_UI) + + private fun game(inactive: Boolean = false) = + window(GAME, APPLICATION, "Game", SCREEN, GAME_PACKAGE, active = !inactive) + + private fun game2() = window(GAME2, APPLICATION, "Game 2", SCREEN, GAME2_PACKAGE, active = true) + + private fun toast() = window(TOAST, SYSTEM, "Saved", Rect(200, 1900, 880, 2050), SYSTEM_UI) + + private fun shade() = + window(SHADE, SYSTEM, "Notification shade", SCREEN, SYSTEM_UI, active = true, layer = 9) + + /** A window as Android lists it, with a title, so that the interpreter needn't name it. */ + private fun window( + id: Int, + type: Int, + title: String, + bounds: Rect, + rootPackage: String, + active: Boolean = false, + layer: Int = if (type == SYSTEM) 5 else 1, + ): AccessibilityWindowInfo { + val window = AccessibilityWindowInfo() + val shadow = shadowOf(window) + shadow.setId(id) + shadow.setType(type) + shadow.setTitle(title) + shadow.setBoundsInScreen(bounds) + shadow.setLayer(layer) + shadow.setActive(active) + shadow.setFocused(active) + shadow.setDisplayId(Display.DEFAULT_DISPLAY) + shadow.setRoot(AccessibilityNodeInfo().apply { packageName = rootPackage }) + return window + } + + private fun installApp(packageName: String, asksForDirectTouch: Boolean) { + val info = PackageInfo() + info.packageName = packageName + info.applicationInfo = + ApplicationInfo().apply { + this.packageName = packageName + if (asksForDirectTouch) { + metaData = Bundle().apply { putBoolean(DirectTouchSettings.CAPABILITY_KEY, true) } + } + } + shadowOf(service.packageManager).installPackage(info) + } + + // Events + + /** The windows change to [windows], Android reports it, and things settle for a second. */ + private fun windowsChange(windowId: Int, changes: Int, vararg windows: AccessibilityWindowInfo) { + serviceWindows.onScreen = windows.toList() + val event = AccessibilityEvent(AccessibilityEvent.TYPE_WINDOWS_CHANGED) + ReflectionHelpers.callInstanceMethod( + event, + "setWindowId", + ClassParameter.from(Int::class.javaPrimitiveType, windowId), + ) + ReflectionHelpers.callInstanceMethod( + event, + "setWindowChanges", + ClassParameter.from(Int::class.javaPrimitiveType, changes), + ) + event.eventTime = SystemClock.uptimeMillis() + interpreter.interpret(event, null) + controller.onAccessibilityEvent(event) + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofSeconds(1)) + } + + // Transcript + + private fun assertTranscript(vararg expected: String) { + assertEquals(expected.joinToString("\n"), transcript.joinToString("\n")) + } + + private fun describe(feedback: Feedback): String { + val part = feedback.failovers().first() + part.speech()?.let { + return "say \"${it.text()}\"" + } + part.passThroughMode()?.let { + val region = it.region() + return if (region.isEmpty) { + "region none" + } else { + "region ${region.bounds.toShortString()} complex=${region.isComplex}" + } + } + return part.toString() + } + + private companion object { + const val STATUS_BAR = 1 + const val NAVIGATION_BAR = 2 + const val SHADE = 3 + const val TOAST = 4 + const val GAME = 10 + const val GAME2 = 11 + const val GAME_PACKAGE = "com.example.game" + const val GAME2_PACKAGE = "com.example.game2" + const val SYSTEM_UI = "com.android.systemui" + const val SYSTEM = AccessibilityWindowInfo.TYPE_SYSTEM + const val APPLICATION = AccessibilityWindowInfo.TYPE_APPLICATION + val SCREEN = Rect(0, 0, 1080, 2340) + } +} diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt index 37449f843..2f9e4ea6d 100644 --- a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt @@ -16,6 +16,7 @@ package com.google.android.accessibility.talkback.directtouch +import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -26,10 +27,10 @@ class DirectTouchPolicyTest { masterEnabled = true, appEnabled = true, screenInteractive = true, - foreignAppWindow = false, - systemUiCoversHalf = false, dialogShowing = false, textFieldFocused = false, + foreignAppWindow = { false }, + systemUiCoversHalf = { false }, ) @Test @@ -39,8 +40,8 @@ class DirectTouchPolicyTest { @Test fun eachHandBackConditionTurnsItOff() { - assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(foreignAppWindow = true))) - assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(systemUiCoversHalf = true))) + assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(foreignAppWindow = { true }))) + assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(systemUiCoversHalf = { true }))) assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(dialogShowing = true))) assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(textFieldFocused = true))) assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(screenInteractive = false))) @@ -48,6 +49,35 @@ class DirectTouchPolicyTest { assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(appEnabled = false))) } + @Test + fun windowRootsAreAskedForOnlyWhenTheyCanChangeTheAnswer() { + for (inputs in 0 until 128) { + fun set(bit: Int) = inputs and (1 shl bit) != 0 + var rootChecks = 0 + val input = + DirectTouchInput( + masterEnabled = set(0), + appEnabled = set(1), + screenInteractive = set(2), + dialogShowing = set(3), + textFieldFocused = set(4), + foreignAppWindow = { + rootChecks++ + set(5) + }, + systemUiCoversHalf = { + rootChecks++ + set(6) + }, + ) + val rest = set(0) && set(1) && set(2) && !set(3) && !set(4) + val expected = rest && !set(5) && !set(6) + assertEquals("inputs $inputs", expected, DirectTouchPolicy.shouldBeActive(input)) + val expectedRootChecks = if (!rest) 0 else if (set(5)) 1 else 2 + assertEquals("root checks for inputs $inputs", expectedRootChecks, rootChecks) + } + } + @Test fun inactiveSystemBarIsNotTheShade() { // A landscape navigation bar is 730 px tall on a 1080 px display, but inactive until touched. diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt index b408af89e..5895d7dbb 100644 --- a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt @@ -43,6 +43,14 @@ class DirectTouchSettingsTest { assertFalse(DirectTouchSettings.isAppEnabled(prefs, "a.b")) } + @Test + fun appIsSeenOnceFirstSightRecordsIt() { + assertFalse(DirectTouchSettings.isSeen(prefs, "a.b")) + DirectTouchSettings.onFirstSight(prefs, "a.b", declaresCapability = false) + assertTrue(DirectTouchSettings.isSeen(prefs, "a.b")) + assertFalse(DirectTouchSettings.isSeen(prefs, "c.d")) + } + @Test fun firstSightIgnoresIncapableApp() { DirectTouchSettings.onFirstSight(prefs, "a.b", declaresCapability = false) diff --git a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java index 379462a89..2193b45d7 100644 --- a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java +++ b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java @@ -304,6 +304,13 @@ public String toString() { private long cachedWindowsTime; // How long the windows are kept, in case a change came without a windows event. private static final long WINDOWS_CACHE_MAX_AGE_MS = 2_000; + // Whether a window event or a check is being interpreted. All its lookups use the windows it + // fetched, which are current, so a window missing from them doesn't make them be fetched again. + private boolean interpreting; + // The package of each listed window's root, by window id, since fetching a root waits for the + // window's app. A window keeps its id until it closes, and no other window gets it. A root that + // couldn't be fetched isn't kept. + private final SparseArray rootPackageNames = new SparseArray<>(); private final boolean isSplitScreenModeAvailable; private final HashMap windowIdToData = new HashMap<>(); // Caches the window roles from last window transition for comparison. @@ -364,6 +371,7 @@ public void onResumeInfrastructure() { public void onSuspendInfrastructure() { displayMonitor.removeDisplayStateChangedListener(this); + rootPackageNames.clear(); } private void clearScreenState() { @@ -404,14 +412,24 @@ public CharSequence getAccessibilityPaneTitle(int windowId) { return null; } + @Nullable AccessibilityWindowInfo window = findWindow(windowId); + return (window == null) ? null : AccessibilityWindowInfoUtils.getTitle(window); + } + + private @Nullable AccessibilityWindowInfo findWindow(int windowId) { + // A role without a window has none to find, and not finding one would fetch the windows again. + if (windowId == WINDOW_ID_NONE) { + return null; + } + long fetchedBefore = cachedWindowsTime; @Nullable AccessibilityWindowInfo window = findWindow(windowsOnAllDisplays(), windowId); - if (window == null && cachedWindowsTime == fetchedBefore) { + if (window == null && !interpreting && cachedWindowsTime == fetchedBefore) { // The window may have appeared since the windows were fetched, before its windows event. cachedWindowsOnAllDisplays = null; window = findWindow(windowsOnAllDisplays(), windowId); } - return (window == null) ? null : AccessibilityWindowInfoUtils.getTitle(window); + return window; } private static @Nullable AccessibilityWindowInfo findWindow( @@ -429,7 +447,8 @@ public CharSequence getAccessibilityPaneTitle(int windowId) { /** * Returns the windows on every display, kept until the windows change. Asking the system for them * each time held up the announcement of every swipe, which looks up the window's title and - * whether the screen is split. + * whether the screen is split. Each window event, and each check while the windows settle, + * fetches them once for all its lookups. */ private SparseArray> windowsOnAllDisplays() { long now = SystemClock.uptimeMillis(); @@ -589,7 +608,13 @@ public void interpret(AccessibilityEvent event, @Nullable EventId eventId, boole if (screenTransitionStartTime == 0) { screenTransitionStartTime = event.getEventTime(); } - InterpretationAndRoles interpAndRoles = interpretInternal(event, depth); + InterpretationAndRoles interpAndRoles; + interpreting = true; + try { + interpAndRoles = interpretInternal(event, depth); + } finally { + interpreting = false; + } EventInterpretation interpretation = interpAndRoles.interpretation; WindowRoles newWindowRoles = interpAndRoles.roles; @@ -845,10 +870,18 @@ void delayedInterpret(EventInterpretation interpretation, @Nullable EventId even // Map windows to roles, detect window role changes. WindowRoles latestRoles = (pendingWindowRoles == null) ? windowRoles : pendingWindowRoles; WindowRoles newWindowRoles = new WindowRoles(latestRoles); - updateWindowRoles(interpretation, service, newWindowRoles, depth + 1); - setWindowTitles(newWindowRoles); - detectWindowChanges(newWindowRoles, interpretation, depth + 1); - detectInputMethodChanged(newWindowRoles, interpretation, /* checkDuplicate= */ true, depth + 1); + // Look at the windows as they are now, since they can change before their windows event. + cachedWindowsOnAllDisplays = null; + interpreting = true; + try { + updateWindowRoles(interpretation, service, newWindowRoles, depth + 1); + setWindowTitles(newWindowRoles); + detectWindowChanges(newWindowRoles, interpretation, depth + 1); + detectInputMethodChanged( + newWindowRoles, interpretation, /* checkDuplicate= */ true, depth + 1); + } finally { + interpreting = false; + } LogUtils.v(TAG, "END delayedInterpret() interpretation=%s", interpretation); // Don't send delayed interpretation if it's a dialog opened event to avoid announcing the @@ -1024,10 +1057,10 @@ private void updateWindowTitlesImp( } case AccessibilityEvent.TYPE_WINDOWS_CHANGED -> { { + forgetRootPackagesOfClosedWindows(); HashSet windowIdsToBeRemoved = new HashSet<>(windowIdToData.keySet()); int displayId = interpretation.getDisplayId(); - List windows = - AccessibilityServiceCompatUtils.getWindowsOnAllDisplays(service).get(displayId); + List windows = windowsOnAllDisplays().get(displayId); if (windows == null) { return; } @@ -1195,8 +1228,7 @@ private void updateWindowRoles( AccessibilityWindowInfo inputMethodWindow = null; int displayId = interpretation.getDisplayId(); - List windows = - AccessibilityServiceCompatUtils.getWindowsOnAllDisplays(service).get(displayId); + List windows = windowsOnAllDisplays().get(displayId); // If there are no windows available, clear the cached IDs. if (windows == null || windows.isEmpty()) { @@ -1376,7 +1408,7 @@ private void detectInputMethodChanged( // source or not. If so, use the announcement first. If not, they are supposed to be // different transitions and should keep both. int inputMethodWindowId = interpretation.getInputMethod().id; - CharSequence inputMethodPackageName = getWindowPackageName(service, inputMethodWindowId); + CharSequence inputMethodPackageName = getWindowPackageName(inputMethodWindowId); CharSequence announcementPackageName = announcement.packageName(); if (inputMethodWindowId == WINDOW_ID_NONE || (inputMethodPackageName != null @@ -1410,7 +1442,7 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque @Nullable CharSequence packageName = (window == null) ? null : window.eventPackageName; // Try to get package name from accessibility window info if it's not in the map. if (packageName == null) { - packageName = getWindowPackageName(service, windowId); + packageName = getWindowPackageName(windowId); } if (packageName != null) { title = getApplicationLabel(packageName); @@ -1423,24 +1455,37 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque return title; } - private static @Nullable CharSequence getWindowPackageName( - AccessibilityService service, int windowId) { - @Nullable CharSequence packageName = null; + private @Nullable CharSequence getWindowPackageName(int windowId) { + @Nullable AccessibilityWindowInfo window = findWindow(windowId); + return (window == null) ? null : getRootPackageName(window); + } - List windows = getAllWindows(service); - for (AccessibilityWindowInfo accessibilityWindowInfo : windows) { - if (accessibilityWindowInfo.getId() == windowId) { - AccessibilityNodeInfo rootNode = - AccessibilityWindowInfoUtils.getRoot(accessibilityWindowInfo); - if (rootNode != null) { - packageName = rootNode.getPackageName(); - } - break; + /** + * Returns the package of the root of a window that is on screen. The window's app is asked the + * first time, which waits for its main thread, and the answer is kept until the window closes. + */ + public @Nullable CharSequence getRootPackageName(AccessibilityWindowInfo window) { + int windowId = window.getId(); + @Nullable CharSequence packageName = rootPackageNames.get(windowId); + if (packageName == null) { + AccessibilityNodeInfo rootNode = AccessibilityWindowInfoUtils.getRoot(window); + packageName = (rootNode == null) ? null : rootNode.getPackageName(); + if (packageName != null) { + rootPackageNames.put(windowId, packageName); } } return packageName; } + private void forgetRootPackagesOfClosedWindows() { + SparseArray> windows = windowsOnAllDisplays(); + for (int i = rootPackageNames.size() - 1; i >= 0; i--) { + if (findWindow(windows, rootPackageNames.keyAt(i)) == null) { + rootPackageNames.removeAt(i); + } + } + } + private @Nullable CharSequence getApplicationLabel(CharSequence packageName) { PackageManager packageManager = service.getPackageManager(); if (packageManager == null) { @@ -1457,17 +1502,6 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque return packageManager.getApplicationLabel(applicationInfo); } - private static List getAllWindows(AccessibilityService service) { - List windows = new ArrayList<>(); - SparseArray> windowsOnAllDisplays = - AccessibilityServiceCompatUtils.getWindowsOnAllDisplays(service); - final int displaySize = windowsOnAllDisplays.size(); - for (int i = 0; i < displaySize; i++) { - windows.addAll(windowsOnAllDisplays.valueAt(i)); - } - return windows; - } - // ///////////////////////////////////////////////////////////////////////////////////// // Inner classes: event interpretation diff --git a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java new file mode 100644 index 000000000..e01fbfc43 --- /dev/null +++ b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java @@ -0,0 +1,779 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.utils.input; + +import static android.view.accessibility.AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_APPEARED; +import static android.view.accessibility.AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_DISAPPEARED; +import static android.view.accessibility.AccessibilityEvent.TYPE_WINDOWS_CHANGED; +import static android.view.accessibility.AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED; +import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ADDED; +import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_PIP; +import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_REMOVED; +import static com.google.android.accessibility.utils.AccessibilityWindowInfoUtils.WINDOW_ID_NONE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.robolectric.Shadows.shadowOf; + +import android.accessibilityservice.AccessibilityService; +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageInfo; +import android.graphics.Rect; +import android.os.Looper; +import android.os.SystemClock; +import android.text.TextUtils; +import android.util.SparseArray; +import android.view.Display; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityWindowInfo; +import androidx.annotation.Nullable; +import com.google.android.accessibility.utils.input.WindowEventInterpreter.Announcement; +import com.google.android.accessibility.utils.input.WindowEventInterpreter.EventInterpretation; +import com.google.android.accessibility.utils.input.WindowEventInterpreter.WindowInterpretation; +import com.google.android.accessibility.utils.monitor.DisplayMonitor; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.Robolectric; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.annotation.Config; +import org.robolectric.annotation.Implementation; +import org.robolectric.annotation.Implements; +import org.robolectric.shadow.api.Shadow; +import org.robolectric.shadows.ShadowAccessibilityRecord; +import org.robolectric.shadows.ShadowAccessibilityService; +import org.robolectric.shadows.ShadowAccessibilityWindowInfo; +import org.robolectric.util.ReflectionHelpers; +import org.robolectric.util.ReflectionHelpers.ClassParameter; + +/** + * Feeds the window interpreter the window events of common screen changes, with the windows the + * system would list at each moment, and records every interpretation that reaches its listeners. + * Each test pins down all that a listener can read, so that changes to how the interpreter looks up + * windows can show that what it tells the rest of Backtalk stays the same. + * + *

The main looper is paused, so time passes only when a test waits, and the 10 ms checks of a + * window change run one by one. + */ +@RunWith(RobolectricTestRunner.class) +@Config( + sdk = 34, + shadows = { + WindowEventInterpreterTest.WindowsShadow.class, + WindowEventInterpreterTest.RootsShadow.class + }) +public class WindowEventInterpreterTest { + + /** The service the interpreter belongs to. */ + public static class TestService extends AccessibilityService { + @Override + public void onAccessibilityEvent(AccessibilityEvent event) {} + + @Override + public void onInterrupt() {} + } + + /** Lists the windows a test puts on screen, and counts how often the service is asked. */ + @Implements(AccessibilityService.class) + public static class WindowsShadow extends ShadowAccessibilityService { + private SparseArray> windows = new SparseArray<>(); + int windowListRequests; + + void setWindows(SparseArray> windows) { + this.windows = windows; + } + + // Like Android, hands out a new list each time, so that a list kept by the interpreter doesn't + // change when the test puts other windows on screen. + @Implementation + @Override + protected SparseArray> getWindowsOnAllDisplays() { + windowListRequests++; + SparseArray> copy = new SparseArray<>(); + for (int i = 0; i < windows.size(); i++) { + copy.put(windows.keyAt(i), new ArrayList<>(windows.valueAt(i))); + } + return copy; + } + + @Implementation + @Override + protected List getWindows() { + windowListRequests++; + return new ArrayList<>(windows.get(Display.DEFAULT_DISPLAY, new ArrayList<>())); + } + } + + /** Counts how often a window's root is fetched, which waits for the window's app. */ + @Implements(AccessibilityWindowInfo.class) + public static class RootsShadow extends ShadowAccessibilityWindowInfo { + static int rootRequests; + + @Implementation + @Override + protected AccessibilityNodeInfo getRoot() { + rootRequests++; + return super.getRoot(); + } + } + + private static final int STATUS_BAR = 1; + private static final int NAVIGATION_BAR = 2; + private static final int SETTINGS = 10; + private static final int WIFI = 11; + private static final int TIP = 12; + private static final int KEYBOARD = 20; + private static final int DIVIDER = 30; + private static final int CALCULATOR = 31; + private static final int VIDEO = 40; + private static final int NOTES = 50; + private static final int VOLUME = 60; + private static final int DOWNLOADS = 70; + + private static final String SETTINGS_PACKAGE = "com.android.settings"; + private static final String NOTES_PACKAGE = "com.example.notes"; + private static final String SYSTEM_UI_PACKAGE = "com.android.systemui"; + private static final String VOLUME_DIALOG_CLASS = + "com.android.systemui.volume.VolumeDialogImpl$CustomDialog"; + + private static final Rect SCREEN = new Rect(0, 0, 1080, 2400); + private static final Rect TOP_HALF = new Rect(0, 0, 1080, 1180); + private static final Rect BOTTOM_HALF = new Rect(0, 1220, 1080, 2400); + + private TestService service; + private WindowsShadow serviceWindows; + private WindowEventInterpreter interpreter; + private final List transcript = new ArrayList<>(); + private long startTime; + + @Before + public void setUp() { + service = Robolectric.setupService(TestService.class); + serviceWindows = Shadow.extract(service); + RootsShadow.rootRequests = 0; + interpreter = new WindowEventInterpreter(service, new DisplayMonitor(service)); + interpreter.addListener((interpretation, eventId) -> transcript.add(describe(interpretation))); + } + + @After + public void tearDown() { + WindowEventInterpreter.setReduceWindowDelay(false); + } + + @Test + public void screenNamedByItsStateChangeSettlesAtTheFirstCheck() { + openSettings(); + + setWindows(statusBar(), navigationBar(), app(WIFI, "Wi-Fi").active().build()); + interpret(stateChanged(WIFI, "Wi-Fi", SETTINGS_PACKAGE)); + waitMs(5); + interpret(windowsChanged(WIFI, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms state from=11 original start=0 delay=10/550 main settled | src 11\"Wi-Fi\" settled" + + " was 10\"Settings\" | A 11\"Wi-Fi\" settled was 10\"Settings\"", + "5ms windows from=11 changes=0x1 original start=0 delay=10/550 main settled | src" + + " 11\"Wi-Fi\" settled was 10\"Settings\" | A 11\"Wi-Fi\" settled was 10\"Settings\"", + "5ms windows from=10 changes=0x2 original start=0 delay=10/550 main settled | src 10" + + " feedback \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" settled was" + + " 10\"Settings\"", + "15ms windows from=10 changes=0x2 stable start=0 delay=10/550 main settled | src 10" + + " feedback \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" settled was" + + " 10\"Settings\""); + } + + @Test + public void screenWithoutAStateChangeTitleWaitsForTheLongestDelay() { + openSettings(); + + setWindows(statusBar(), navigationBar(), app(WIFI, "Wi-Fi").active().build()); + interpret(windowsChanged(WIFI, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=11 changes=0x1 original start=0 delay=10/550 main | src 11\"Wi-Fi\" was" + + " 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", + "0ms windows from=10 changes=0x2 original start=0 delay=10/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", + "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\""); + // Each event and each of the 55 checks asks for the windows once, though each looks up the + // Settings window, which has closed. + assertEquals(2 + 55, serviceWindows.windowListRequests); + } + + @Test + public void reducedDelayEndsTheWaitSooner() { + WindowEventInterpreter.setReduceWindowDelay(true); + openSettings(); + + setWindows(statusBar(), navigationBar(), app(WIFI, "Wi-Fi").active().build()); + interpret(windowsChanged(WIFI, WINDOWS_CHANGE_ADDED)); + waitMs(600); + + assertTranscript( + "0ms windows from=11 changes=0x1 original start=0 delay=10/200 main | src 11\"Wi-Fi\" was" + + " 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", + "200ms windows from=11 changes=0x1 stable start=0 delay=200/200 main | src 11\"Wi-Fi\"" + + " was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\""); + // The event and each of the 20 checks ask for the windows once, for all their lookups. + assertEquals(1 + 20, serviceWindows.windowListRequests); + } + + @Test + public void windowsThatChangeWithoutAnEventAreSeenWhileWaiting() { + openSettings(); + + // A window opens over Settings, but closes again before the system reports that it closed. + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").inactive().build(), + app(TIP, "Tip").layer(2).bounds(new Rect(100, 800, 980, 1600)).active().build()); + interpret(windowsChanged(TIP, WINDOWS_CHANGE_ADDED)); + waitMs(35); + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + waitMs(20); + interpret(windowsChanged(TIP, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=12 changes=0x1 original start=0 delay=10/550 main | src 12\"Tip\" was" + + " 10\"Settings\" | A 12\"Tip\" was 10\"Settings\"", + "40ms windows from=12 changes=0x1 stable start=0 delay=40/550 settled | src 12\"Tip\" was" + + " 10\"Settings\" | A 10\"Settings\"", + "55ms windows from=12 changes=0x2 original stable start=55 delay=0/0 settled | src 12" + + " feedback \"untitled\" was 12\"Tip\" | A 10\"Settings\""); + } + + @Test + public void splitScreenNamesBothApps() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").bounds(TOP_HALF).active().build(), + window(DIVIDER, AccessibilityWindowInfo.TYPE_SPLIT_SCREEN_DIVIDER, "Divider") + .layer(3) + .bounds(new Rect(0, 1180, 1080, 1220)) + .build(), + app(CALCULATOR, "Calculator").bounds(BOTTOM_HALF).inactive().build()); + interpret(windowsChanged(CALCULATOR, WINDOWS_CHANGE_ADDED)); + waitMs(600); + + assertTranscript( + "0ms windows from=31 changes=0x1 original start=0 delay=10/550 main | src 31\"Calculator\"" + + " was 10\"Settings\" | A 10\"Settings\" | B 31\"Calculator\" was -", + "550ms windows from=31 changes=0x1 stable start=0 delay=550/550 main | src" + + " 31\"Calculator\" was 10\"Settings\" | A 10\"Settings\" | B 31\"Calculator\" was -"); + assertTrue(interpreter.isSplitScreenMode(Display.DEFAULT_DISPLAY)); + } + + @Test + public void keyboardIsReportedWhenShownAndHidden() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").active().build(), + window(KEYBOARD, AccessibilityWindowInfo.TYPE_INPUT_METHOD, "Gboard") + .layer(3) + .bounds(new Rect(0, 1500, 1080, 2280)) + .build()); + interpret(windowsChanged(KEYBOARD, WINDOWS_CHANGE_ADDED)); + waitMs(600); + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(KEYBOARD, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=20 changes=0x1 original start=0 delay=10/550 imeChanged announceIme |" + + " src 20\"Gboard\" was 10\"Settings\" | A 10\"Settings\" | ime 20\"Gboard\" was -", + "550ms windows from=20 changes=0x1 stable start=0 delay=550/550 imeChanged announceIme |" + + " src 20\"Gboard\" was 10\"Settings\" | A 10\"Settings\" | ime 20\"Gboard\" was -", + "600ms windows from=20 changes=0x2 original start=600 delay=10/550 imeChanged announceIme" + + " | src 20 feedback \"untitled\" was 20\"Gboard\" | A 10\"Settings\" | ime -" + + " feedback \"untitled\" was 20\"Gboard\"", + "1150ms windows from=20 changes=0x2 stable start=600 delay=550/550 imeChanged announceIme" + + " | src 20 feedback \"untitled\" was 20\"Gboard\" | A 10\"Settings\" | ime -" + + " feedback \"untitled\" was 20\"Gboard\""); + } + + @Test + public void pictureInPictureWindowIsReported() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").active().build(), + app(VIDEO, "Video") + .layer(4) + .bounds(new Rect(600, 1600, 1000, 1900)) + .pictureInPicture() + .build()); + interpret(windowsChanged(VIDEO, WINDOWS_CHANGE_ADDED | WINDOWS_CHANGE_PIP)); + waitMs(600); + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(VIDEO, WINDOWS_CHANGE_REMOVED | WINDOWS_CHANGE_PIP)); + waitMs(600); + + assertTranscript( + "0ms windows from=40 changes=0x401 original stable start=0 delay=0/0 pipChanged | src" + + " 40\"Video\" was 10\"Settings\" | A 10\"Settings\" | pip 40\"Video\" was -", + "600ms windows from=40 changes=0x402 original stable start=600 delay=0/0 pipChanged | src" + + " 40 feedback \"untitled\" was 40\"Video\" | A 10\"Settings\" | pip - feedback" + + " \"untitled\" was 40\"Video\""); + } + + @Test + public void windowWithoutATitleIsNamedAfterItsApp() { + installApp(NOTES_PACKAGE, "Notes"); + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(NOTES, null).rootPackage(NOTES_PACKAGE).active().build()); + interpret(windowsChanged(NOTES, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + assertEquals("Notes", interpreter.getWindowTitle(NOTES).toString()); + // Notes closes, and is no longer named after its app. + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(NOTES, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=50 changes=0x1 original start=0 delay=10/550 main | src 50 feedback" + + " \"Notes\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", + "0ms windows from=10 changes=0x2 original start=0 delay=10/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", + "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", + "600ms windows from=10 changes=0x1 original start=600 delay=10/550 main | src" + + " 10\"Settings\" was 10 | A 10\"Settings\" was 50", + "600ms windows from=50 changes=0x2 original start=600 delay=10/550 main | src 50 feedback" + + " \"untitled\" was 10 | A 10\"Settings\" was 50", + "1150ms windows from=50 changes=0x2 stable start=600 delay=550/550 main | src 50 feedback" + + " \"untitled\" was 10 | A 10\"Settings\" was 50"); + // The app is asked for the window's root once, though the window is named at every check. + assertEquals(1, RootsShadow.rootRequests); + } + + @Test + public void appOfAWindowWithoutATitleIsAskedForAgainUntilItAnswers() { + installApp(NOTES_PACKAGE, "Notes"); + openSettings(); + + // At first the window's root can't be fetched, as before its app has drawn it. + setWindows(statusBar(), navigationBar(), app(NOTES, null).active().build()); + interpret(windowsChanged(NOTES, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(100); + setWindows( + statusBar(), + navigationBar(), + app(NOTES, null).rootPackage(NOTES_PACKAGE).active().build()); + int rootRequestsBefore = RootsShadow.rootRequests; + waitMs(500); + + assertTranscript( + "0ms windows from=50 changes=0x1 original start=0 delay=10/550 main | src 50 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"untitled\" was 10\"Settings\"", + "0ms windows from=10 changes=0x2 original start=0 delay=10/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"untitled\" was 10\"Settings\"", + "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\""); + // Once the root can be fetched, it is fetched once, and its package is kept. + assertEquals(1, RootsShadow.rootRequests - rootRequestsBefore); + } + + @Test + public void rootPackageIsKeptForOthersToo() { + AccessibilityWindowInfo notes = app(NOTES, null).rootPackage(NOTES_PACKAGE).active().build(); + setWindows(statusBar(), navigationBar(), notes); + + assertEquals(NOTES_PACKAGE, interpreter.getRootPackageName(notes).toString()); + assertEquals(NOTES_PACKAGE, interpreter.getRootPackageName(notes).toString()); + assertEquals(1, RootsShadow.rootRequests); + } + + @Test + public void paneTitleNamesTheWindowUntilThePaneCloses() { + openSettings(); + + interpret(pane(SETTINGS, CONTENT_CHANGE_TYPE_PANE_APPEARED, "Filters")); + waitMs(600); + assertEquals("Filters", interpreter.getAccessibilityPaneTitle(SETTINGS).toString()); + interpret(pane(SETTINGS, CONTENT_CHANGE_TYPE_PANE_DISAPPEARED, "Filters")); + waitMs(600); + + assertTranscript( + "0ms state from=10 content=0x10 original source start=0 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Filters\" settled was 10\"Settings\" | A 10\"Filters\"" + + " settled was 10\"Settings\"", + "10ms state from=10 content=0x10 stable source start=0 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Filters\" settled was 10\"Settings\" | A 10\"Filters\"" + + " settled was 10\"Settings\"", + "600ms state from=10 content=0x20 original source start=600 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Settings\" settled was 10\"Filters\" | A" + + " 10\"Settings\" settled was 10\"Filters\"", + "610ms state from=10 content=0x20 stable source start=600 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Settings\" settled was 10\"Filters\" | A" + + " 10\"Settings\" settled was 10\"Filters\""); + assertFalse(interpreter.hasAccessibilityPane(SETTINGS)); + } + + @Test + public void screenOffForgetsTheWindowsAndTheFirstChangeAfterwardIsMarked() { + openSettings(); + + interpreter.onDisplayStateChanged(false); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + waitMs(600); + interpreter.onDisplayStateChanged(true); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + waitMs(600); + + assertTranscript( + "600ms windows from=10 changes=0x1 original wakeUp start=600 delay=10/550 main | src" + + " 10\"Settings\" was - | A 10\"Settings\" was -", + "1150ms windows from=10 changes=0x1 stable wakeUp start=600 delay=550/550 main | src" + + " 10\"Settings\" was - | A 10\"Settings\" was -"); + } + + @Test + public void volumePanelIsAnnounced() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").active().build(), + window(VOLUME, AccessibilityWindowInfo.TYPE_SYSTEM, "Volume") + .layer(6) + .bounds(new Rect(900, 600, 1080, 1600)) + .build()); + AccessibilityEvent volume = stateChanged(VOLUME, "Volume", SYSTEM_UI_PACKAGE); + volume.setClassName(VOLUME_DIALOG_CLASS); + interpret(volume); + waitMs(600); + + assertTranscript( + "0ms state from=60 original start=0 delay=10/550 settled announcement=\"Volume\" from" + + " com.android.systemui volume | src 60\"Volume\" was 10\"Settings\" | A" + + " 10\"Settings\"", + "10ms state from=60 stable start=0 delay=10/550 settled announcement=\"Volume\" from" + + " com.android.systemui volume | src 60\"Volume\" was 10\"Settings\" | A" + + " 10\"Settings\""); + } + + @Test + public void titleOfAWindowListedBeforeItsEventIsFound() { + openSettings(); + assertEquals("Settings", interpreter.getWindowTitle(SETTINGS).toString()); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").inactive().build(), + app(DOWNLOADS, "Downloads").layer(2).active().build()); + + assertEquals("Downloads", interpreter.getWindowTitle(DOWNLOADS).toString()); + } + + // Screens + + /** Puts Settings on screen, lets the interpreter take it in, and starts the transcript there. */ + private void openSettings() { + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + waitMs(600); + transcript.clear(); + serviceWindows.windowListRequests = 0; + RootsShadow.rootRequests = 0; + startTime = SystemClock.uptimeMillis(); + } + + private static AccessibilityWindowInfo statusBar() { + return window(STATUS_BAR, AccessibilityWindowInfo.TYPE_SYSTEM, "Status bar") + .layer(5) + .bounds(new Rect(0, 0, 1080, 80)) + .build(); + } + + private static AccessibilityWindowInfo navigationBar() { + return window(NAVIGATION_BAR, AccessibilityWindowInfo.TYPE_SYSTEM, "Navigation bar") + .layer(5) + .bounds(new Rect(0, 2280, 1080, 2400)) + .build(); + } + + private static WindowBuilder app(int id, @Nullable String title) { + return window(id, AccessibilityWindowInfo.TYPE_APPLICATION, title); + } + + private static WindowBuilder window(int id, int type, @Nullable String title) { + return new WindowBuilder(id, type, title); + } + + /** Makes a new window each time, as the windows Android lists are copies. */ + private static class WindowBuilder { + private final int id; + private final int type; + private final @Nullable String title; + private int layer = 1; + private Rect bounds = SCREEN; + private boolean active; + private boolean pictureInPicture; + private @Nullable String rootPackage; + + WindowBuilder(int id, int type, @Nullable String title) { + this.id = id; + this.type = type; + this.title = title; + } + + WindowBuilder layer(int layer) { + this.layer = layer; + return this; + } + + WindowBuilder bounds(Rect bounds) { + this.bounds = bounds; + return this; + } + + /** Active and focused, as the window the user is in is. */ + WindowBuilder active() { + active = true; + return this; + } + + WindowBuilder inactive() { + active = false; + return this; + } + + WindowBuilder pictureInPicture() { + pictureInPicture = true; + return this; + } + + WindowBuilder rootPackage(String packageName) { + rootPackage = packageName; + return this; + } + + AccessibilityWindowInfo build() { + AccessibilityWindowInfo window = new AccessibilityWindowInfo(); + var shadow = shadowOf(window); + shadow.setId(id); + shadow.setType(type); + shadow.setTitle(title); + shadow.setLayer(layer); + shadow.setBoundsInScreen(bounds); + shadow.setActive(active); + shadow.setFocused(active); + shadow.setPictureInPicture(pictureInPicture); + shadow.setDisplayId(Display.DEFAULT_DISPLAY); + if (rootPackage != null) { + AccessibilityNodeInfo root = new AccessibilityNodeInfo(); + root.setPackageName(rootPackage); + shadow.setRoot(root); + } + return window; + } + } + + private void setWindows(AccessibilityWindowInfo... windows) { + SparseArray> onAllDisplays = new SparseArray<>(); + onAllDisplays.put(Display.DEFAULT_DISPLAY, new ArrayList<>(Arrays.asList(windows))); + serviceWindows.setWindows(onAllDisplays); + } + + private void installApp(String packageName, String label) { + PackageInfo packageInfo = new PackageInfo(); + packageInfo.packageName = packageName; + packageInfo.applicationInfo = new ApplicationInfo(); + packageInfo.applicationInfo.packageName = packageName; + packageInfo.applicationInfo.nonLocalizedLabel = label; + shadowOf(service.getPackageManager()).installPackage(packageInfo); + } + + // Events + + private void interpret(AccessibilityEvent event) { + interpreter.interpret(event, /* eventId= */ null); + } + + private static void waitMs(long ms) { + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(ms)); + } + + private static AccessibilityEvent windowsChanged(int windowId, int changes) { + AccessibilityEvent event = event(TYPE_WINDOWS_CHANGED, windowId); + ReflectionHelpers.callInstanceMethod( + event, "setWindowChanges", ClassParameter.from(int.class, changes)); + return event; + } + + private static AccessibilityEvent stateChanged( + int windowId, CharSequence text, String packageName) { + AccessibilityEvent event = event(TYPE_WINDOW_STATE_CHANGED, windowId); + event.getText().add(text); + event.setPackageName(packageName); + return event; + } + + /** A pane opening or closing in the window, as Android reports it with the pane as source. */ + private static AccessibilityEvent pane(int windowId, int contentChangeType, String paneTitle) { + AccessibilityEvent event = event(TYPE_WINDOW_STATE_CHANGED, windowId); + event.setContentChangeTypes(contentChangeType); + event.setPackageName(SETTINGS_PACKAGE); + AccessibilityNodeInfo source = new AccessibilityNodeInfo(); + source.setPaneTitle(paneTitle); + source.setPackageName(SETTINGS_PACKAGE); + ShadowAccessibilityRecord shadow = Shadow.extract(event); + shadow.setSourceNode(source); + return event; + } + + private static AccessibilityEvent event(int type, int windowId) { + AccessibilityEvent event = new AccessibilityEvent(type); + ReflectionHelpers.callInstanceMethod( + event, "setWindowId", ClassParameter.from(int.class, windowId)); + event.setEventTime(SystemClock.uptimeMillis()); + return event; + } + + // Transcript + + private void assertTranscript(String... expected) { + assertEquals(String.join("\n", expected), String.join("\n", transcript)); + } + + /** Everything a listener can read from the interpretation, leaving out what is unset. */ + private String describe(EventInterpretation in) { + StringBuilder line = new StringBuilder(); + line.append(SystemClock.uptimeMillis() - startTime).append("ms "); + line.append(in.getEventType() == TYPE_WINDOWS_CHANGED ? "windows" : "state"); + line.append(" from=").append(windowId(in.getWindowIdFromEvent())); + if (in.getDisplayId() != Display.DEFAULT_DISPLAY) { + line.append(" display=").append(in.getDisplayId()); + } + if (in.getWindowChangeTypes() != 0) { + line.append(" changes=0x").append(Integer.toHexString(in.getWindowChangeTypes())); + } + if (in.getContentChangeTypes() != 0) { + line.append(" content=0x").append(Integer.toHexString(in.getContentChangeTypes())); + } + flag(line, in.isOriginalEvent(), "original"); + flag(line, in.areWindowsStable(), "stable"); + flag(line, !in.isAllowAnnounce(), "silent"); + flag(line, in.isInterpretFirstTimeWhenWakeUp(), "wakeUp"); + flag(line, in.isWebDialogOpenedEvent(), "webDialog"); + flag(line, in.getSourceNode() != null, "source"); + long start = in.getEventStartTime(); + line.append(" start=").append((start == 0) ? 0 : start - startTime); + line.append(" delay=").append(in.getTotalDelayMs()).append('/').append(in.getMaxDelayMs()); + flag(line, in.getMainWindowsChanged(), "main"); + flag(line, in.getPicInPicChanged(), "pipChanged"); + flag(line, in.getInputMethodChanged(), "imeChanged"); + flag(line, in.getShouldAnnounceInputMethodChange(), "announceIme"); + flag(line, in.getHorizontalPlacement(), "sideBySide"); + flag(line, in.hasTitlesFromStateChange(), "settled"); + @Nullable Announcement announcement = in.getAnnouncement(); + if (announcement != null) { + line.append(" announcement=\"").append(announcement.text()).append("\" from "); + line.append(announcement.packageName()); + flag(line, announcement.isFromVolumeControlPanel(), "volume"); + flag(line, announcement.isFromInputMethodEditor(), "keyboard"); + } + if (in.getAnchorNodeRole() != 0) { + line.append(" anchor=").append(in.getAnchorNodeRole()); + } + if (in.getSourceBoundsInScreen() != null) { + line.append(" bounds=").append(in.getSourceBoundsInScreen().toShortString()); + } + window(line, "src", in.getEventSourceWindow()); + window(line, "A", in.getWindowA()); + window(line, "B", in.getWindowB()); + window(line, "other", in.getOtherActiveWindow()); + window(line, "pip", in.getPicInPic()); + window(line, "ime", in.getInputMethod()); + return line.toString(); + } + + /** + * Describes the window in one role. A role with no window, which a listener sees as untitled, is + * left out unless it just lost its window. + */ + private void window(StringBuilder line, String role, WindowInterpretation window) { + boolean changed = window.idOrTitleChanged(); + if (window.getId() == WINDOW_ID_NONE + && window.getTitle() == null + && !changed + && TextUtils.equals(window.getTitleForFeedback(), untitled())) { + return; + } + line.append(" | ").append(role).append(' ').append(windowId(window.getId())); + title(line, window.getTitle()); + if (!TextUtils.equals(window.getTitleForFeedback(), window.getTitle())) { + line.append(" feedback "); + title(line, window.getTitleForFeedback()); + } + if (changed) { + flag(line, window.hasTitleFromStateChange(), "settled"); + line.append(" was ").append(windowId(window.getOldId())); + title(line, window.getOldTitle()); + } + } + + private String untitled() { + return service.getString(com.google.android.accessibility.utils.R.string.untitled_window); + } + + private static void title(StringBuilder line, @Nullable CharSequence title) { + if (title != null) { + line.append('"').append(title).append('"'); + } + } + + private static String windowId(int id) { + return (id == WINDOW_ID_NONE) ? "-" : String.valueOf(id); + } + + private static void flag(StringBuilder line, boolean set, String name) { + if (set) { + line.append(' ').append(name); + } + } +}