From 489a755619623c06e5dfc3b3d1de92a6d034c3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Mon, 12 Oct 2026 10:31:10 +0200 Subject: [PATCH 1/7] Add tests that record what the window interpreter reports WindowEventInterpreterTest feeds the window interpreter the window events of common screen changes, with the windows Android would list at each moment, and records everything a listener can read from each interpretation, and when it came. The screen changes are a new screen named by its state change, one without such a name, the shorter wait when animations are off, a window that closes before the system reports it, split screen, the keyboard, picture-in-picture, a window without a title, a pane, the screen turning off and on, and the volume panel. The tests pin down what the interpreter reports today, so that changes to how it looks up windows can show that the rest of Backtalk is told the same. The main looper is paused, so each 10 ms check runs at a known time, and the service counts how often it is asked for the window list. A screen change that waits the full 550 ms asks for it 913 times. --- .../input/WindowEventInterpreterTest.java | 702 ++++++++++++++++++ 1 file changed, 702 insertions(+) create mode 100644 utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java diff --git a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java new file mode 100644 index 000000000..d7fced568 --- /dev/null +++ b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java @@ -0,0 +1,702 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.utils.input; + +import static android.view.accessibility.AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_APPEARED; +import static android.view.accessibility.AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_DISAPPEARED; +import static android.view.accessibility.AccessibilityEvent.TYPE_WINDOWS_CHANGED; +import static android.view.accessibility.AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED; +import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ADDED; +import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_PIP; +import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_REMOVED; +import static com.google.android.accessibility.utils.AccessibilityWindowInfoUtils.WINDOW_ID_NONE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.robolectric.Shadows.shadowOf; + +import android.accessibilityservice.AccessibilityService; +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageInfo; +import android.graphics.Rect; +import android.os.Looper; +import android.os.SystemClock; +import android.text.TextUtils; +import android.util.SparseArray; +import android.view.Display; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityWindowInfo; +import androidx.annotation.Nullable; +import com.google.android.accessibility.utils.input.WindowEventInterpreter.Announcement; +import com.google.android.accessibility.utils.input.WindowEventInterpreter.EventInterpretation; +import com.google.android.accessibility.utils.input.WindowEventInterpreter.WindowInterpretation; +import com.google.android.accessibility.utils.monitor.DisplayMonitor; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.Robolectric; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.annotation.Config; +import org.robolectric.annotation.Implementation; +import org.robolectric.annotation.Implements; +import org.robolectric.shadow.api.Shadow; +import org.robolectric.shadows.ShadowAccessibilityRecord; +import org.robolectric.shadows.ShadowAccessibilityService; +import org.robolectric.util.ReflectionHelpers; +import org.robolectric.util.ReflectionHelpers.ClassParameter; + +/** + * Feeds the window interpreter the window events of common screen changes, with the windows the + * system would list at each moment, and records every interpretation that reaches its listeners. + * Each test pins down all that a listener can read, so that changes to how the interpreter looks up + * windows can show that what it tells the rest of Backtalk stays the same. + * + *

The main looper is paused, so time passes only when a test waits, and the 10 ms checks of a + * window change run one by one. + */ +@RunWith(RobolectricTestRunner.class) +@Config(sdk = 34, shadows = WindowEventInterpreterTest.WindowsShadow.class) +public class WindowEventInterpreterTest { + + /** The service the interpreter belongs to. */ + public static class TestService extends AccessibilityService { + @Override + public void onAccessibilityEvent(AccessibilityEvent event) {} + + @Override + public void onInterrupt() {} + } + + /** Lists the windows a test puts on screen, and counts how often the service is asked. */ + @Implements(AccessibilityService.class) + public static class WindowsShadow extends ShadowAccessibilityService { + private SparseArray> windows = new SparseArray<>(); + int windowListRequests; + + void setWindows(SparseArray> windows) { + this.windows = windows; + } + + // Like Android, hands out a new list each time, so that a list kept by the interpreter doesn't + // change when the test puts other windows on screen. + @Implementation + @Override + protected SparseArray> getWindowsOnAllDisplays() { + windowListRequests++; + SparseArray> copy = new SparseArray<>(); + for (int i = 0; i < windows.size(); i++) { + copy.put(windows.keyAt(i), new ArrayList<>(windows.valueAt(i))); + } + return copy; + } + + @Implementation + @Override + protected List getWindows() { + windowListRequests++; + return new ArrayList<>(windows.get(Display.DEFAULT_DISPLAY, new ArrayList<>())); + } + } + + private static final int STATUS_BAR = 1; + private static final int NAVIGATION_BAR = 2; + private static final int SETTINGS = 10; + private static final int WIFI = 11; + private static final int TIP = 12; + private static final int KEYBOARD = 20; + private static final int DIVIDER = 30; + private static final int CALCULATOR = 31; + private static final int VIDEO = 40; + private static final int NOTES = 50; + private static final int VOLUME = 60; + private static final int DOWNLOADS = 70; + + private static final String SETTINGS_PACKAGE = "com.android.settings"; + private static final String NOTES_PACKAGE = "com.example.notes"; + private static final String SYSTEM_UI_PACKAGE = "com.android.systemui"; + private static final String VOLUME_DIALOG_CLASS = + "com.android.systemui.volume.VolumeDialogImpl$CustomDialog"; + + private static final Rect SCREEN = new Rect(0, 0, 1080, 2400); + private static final Rect TOP_HALF = new Rect(0, 0, 1080, 1180); + private static final Rect BOTTOM_HALF = new Rect(0, 1220, 1080, 2400); + + private TestService service; + private WindowsShadow serviceWindows; + private WindowEventInterpreter interpreter; + private final List transcript = new ArrayList<>(); + private long startTime; + + @Before + public void setUp() { + service = Robolectric.setupService(TestService.class); + serviceWindows = Shadow.extract(service); + interpreter = new WindowEventInterpreter(service, new DisplayMonitor(service)); + interpreter.addListener((interpretation, eventId) -> transcript.add(describe(interpretation))); + } + + @After + public void tearDown() { + WindowEventInterpreter.setReduceWindowDelay(false); + } + + @Test + public void screenNamedByItsStateChangeSettlesAtTheFirstCheck() { + openSettings(); + + setWindows(statusBar(), navigationBar(), app(WIFI, "Wi-Fi").active().build()); + interpret(stateChanged(WIFI, "Wi-Fi", SETTINGS_PACKAGE)); + waitMs(5); + interpret(windowsChanged(WIFI, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms state from=11 original start=0 delay=10/550 main settled | src 11\"Wi-Fi\" settled" + + " was 10\"Settings\" | A 11\"Wi-Fi\" settled was 10\"Settings\"", + "5ms windows from=11 changes=0x1 original start=0 delay=10/550 main settled | src" + + " 11\"Wi-Fi\" settled was 10\"Settings\" | A 11\"Wi-Fi\" settled was 10\"Settings\"", + "5ms windows from=10 changes=0x2 original start=0 delay=10/550 main settled | src 10" + + " feedback \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" settled was" + + " 10\"Settings\"", + "15ms windows from=10 changes=0x2 stable start=0 delay=10/550 main settled | src 10" + + " feedback \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" settled was" + + " 10\"Settings\""); + } + + @Test + public void screenWithoutAStateChangeTitleWaitsForTheLongestDelay() { + openSettings(); + + setWindows(statusBar(), navigationBar(), app(WIFI, "Wi-Fi").active().build()); + interpret(windowsChanged(WIFI, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=11 changes=0x1 original start=0 delay=10/550 main | src 11\"Wi-Fi\" was" + + " 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", + "0ms windows from=10 changes=0x2 original start=0 delay=10/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", + "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\""); + } + + @Test + public void reducedDelayEndsTheWaitSooner() { + WindowEventInterpreter.setReduceWindowDelay(true); + openSettings(); + + setWindows(statusBar(), navigationBar(), app(WIFI, "Wi-Fi").active().build()); + interpret(windowsChanged(WIFI, WINDOWS_CHANGE_ADDED)); + waitMs(600); + + assertTranscript( + "0ms windows from=11 changes=0x1 original start=0 delay=10/200 main | src 11\"Wi-Fi\" was" + + " 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", + "200ms windows from=11 changes=0x1 stable start=0 delay=200/200 main | src 11\"Wi-Fi\"" + + " was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\""); + } + + @Test + public void windowsThatChangeWithoutAnEventAreSeenWhileWaiting() { + openSettings(); + + // A window opens over Settings, but closes again before the system reports that it closed. + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").inactive().build(), + app(TIP, "Tip").layer(2).bounds(new Rect(100, 800, 980, 1600)).active().build()); + interpret(windowsChanged(TIP, WINDOWS_CHANGE_ADDED)); + waitMs(35); + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + waitMs(20); + interpret(windowsChanged(TIP, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=12 changes=0x1 original start=0 delay=10/550 main | src 12\"Tip\" was" + + " 10\"Settings\" | A 12\"Tip\" was 10\"Settings\"", + "40ms windows from=12 changes=0x1 stable start=0 delay=40/550 settled | src 12\"Tip\" was" + + " 10\"Settings\" | A 10\"Settings\"", + "55ms windows from=12 changes=0x2 original stable start=55 delay=0/0 settled | src 12" + + " feedback \"untitled\" was 12\"Tip\" | A 10\"Settings\""); + } + + @Test + public void splitScreenNamesBothApps() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").bounds(TOP_HALF).active().build(), + window(DIVIDER, AccessibilityWindowInfo.TYPE_SPLIT_SCREEN_DIVIDER, "Divider") + .layer(3) + .bounds(new Rect(0, 1180, 1080, 1220)) + .build(), + app(CALCULATOR, "Calculator").bounds(BOTTOM_HALF).inactive().build()); + interpret(windowsChanged(CALCULATOR, WINDOWS_CHANGE_ADDED)); + waitMs(600); + + assertTranscript( + "0ms windows from=31 changes=0x1 original start=0 delay=10/550 main | src 31\"Calculator\"" + + " was 10\"Settings\" | A 10\"Settings\" | B 31\"Calculator\" was -", + "550ms windows from=31 changes=0x1 stable start=0 delay=550/550 main | src" + + " 31\"Calculator\" was 10\"Settings\" | A 10\"Settings\" | B 31\"Calculator\" was -"); + assertTrue(interpreter.isSplitScreenMode(Display.DEFAULT_DISPLAY)); + } + + @Test + public void keyboardIsReportedWhenShownAndHidden() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").active().build(), + window(KEYBOARD, AccessibilityWindowInfo.TYPE_INPUT_METHOD, "Gboard") + .layer(3) + .bounds(new Rect(0, 1500, 1080, 2280)) + .build()); + interpret(windowsChanged(KEYBOARD, WINDOWS_CHANGE_ADDED)); + waitMs(600); + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(KEYBOARD, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=20 changes=0x1 original start=0 delay=10/550 imeChanged announceIme |" + + " src 20\"Gboard\" was 10\"Settings\" | A 10\"Settings\" | ime 20\"Gboard\" was -", + "550ms windows from=20 changes=0x1 stable start=0 delay=550/550 imeChanged announceIme |" + + " src 20\"Gboard\" was 10\"Settings\" | A 10\"Settings\" | ime 20\"Gboard\" was -", + "600ms windows from=20 changes=0x2 original start=600 delay=10/550 imeChanged announceIme" + + " | src 20 feedback \"untitled\" was 20\"Gboard\" | A 10\"Settings\" | ime -" + + " feedback \"untitled\" was 20\"Gboard\"", + "1150ms windows from=20 changes=0x2 stable start=600 delay=550/550 imeChanged announceIme" + + " | src 20 feedback \"untitled\" was 20\"Gboard\" | A 10\"Settings\" | ime -" + + " feedback \"untitled\" was 20\"Gboard\""); + } + + @Test + public void pictureInPictureWindowIsReported() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").active().build(), + app(VIDEO, "Video") + .layer(4) + .bounds(new Rect(600, 1600, 1000, 1900)) + .pictureInPicture() + .build()); + interpret(windowsChanged(VIDEO, WINDOWS_CHANGE_ADDED | WINDOWS_CHANGE_PIP)); + waitMs(600); + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(VIDEO, WINDOWS_CHANGE_REMOVED | WINDOWS_CHANGE_PIP)); + waitMs(600); + + assertTranscript( + "0ms windows from=40 changes=0x401 original stable start=0 delay=0/0 pipChanged | src" + + " 40\"Video\" was 10\"Settings\" | A 10\"Settings\" | pip 40\"Video\" was -", + "600ms windows from=40 changes=0x402 original stable start=600 delay=0/0 pipChanged | src" + + " 40 feedback \"untitled\" was 40\"Video\" | A 10\"Settings\" | pip - feedback" + + " \"untitled\" was 40\"Video\""); + } + + @Test + public void windowWithoutATitleIsNamedAfterItsApp() { + installApp(NOTES_PACKAGE, "Notes"); + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(NOTES, null).rootPackage(NOTES_PACKAGE).active().build()); + interpret(windowsChanged(NOTES, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(600); + + assertTranscript( + "0ms windows from=50 changes=0x1 original start=0 delay=10/550 main | src 50 feedback" + + " \"Notes\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", + "0ms windows from=10 changes=0x2 original start=0 delay=10/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", + "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\""); + assertEquals("Notes", interpreter.getWindowTitle(NOTES).toString()); + } + + @Test + public void paneTitleNamesTheWindowUntilThePaneCloses() { + openSettings(); + + interpret(pane(SETTINGS, CONTENT_CHANGE_TYPE_PANE_APPEARED, "Filters")); + waitMs(600); + assertEquals("Filters", interpreter.getAccessibilityPaneTitle(SETTINGS).toString()); + interpret(pane(SETTINGS, CONTENT_CHANGE_TYPE_PANE_DISAPPEARED, "Filters")); + waitMs(600); + + assertTranscript( + "0ms state from=10 content=0x10 original source start=0 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Filters\" settled was 10\"Settings\" | A 10\"Filters\"" + + " settled was 10\"Settings\"", + "10ms state from=10 content=0x10 stable source start=0 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Filters\" settled was 10\"Settings\" | A 10\"Filters\"" + + " settled was 10\"Settings\"", + "600ms state from=10 content=0x20 original source start=600 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Settings\" settled was 10\"Filters\" | A" + + " 10\"Settings\" settled was 10\"Filters\"", + "610ms state from=10 content=0x20 stable source start=600 delay=10/550 main settled" + + " bounds=[0,0][0,0] | src 10\"Settings\" settled was 10\"Filters\" | A" + + " 10\"Settings\" settled was 10\"Filters\""); + assertFalse(interpreter.hasAccessibilityPane(SETTINGS)); + } + + @Test + public void screenOffForgetsTheWindowsAndTheFirstChangeAfterwardIsMarked() { + openSettings(); + + interpreter.onDisplayStateChanged(false); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + waitMs(600); + interpreter.onDisplayStateChanged(true); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + waitMs(600); + + assertTranscript( + "600ms windows from=10 changes=0x1 original wakeUp start=600 delay=10/550 main | src" + + " 10\"Settings\" was - | A 10\"Settings\" was -", + "1150ms windows from=10 changes=0x1 stable wakeUp start=600 delay=550/550 main | src" + + " 10\"Settings\" was - | A 10\"Settings\" was -"); + } + + @Test + public void volumePanelIsAnnounced() { + openSettings(); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").active().build(), + window(VOLUME, AccessibilityWindowInfo.TYPE_SYSTEM, "Volume") + .layer(6) + .bounds(new Rect(900, 600, 1080, 1600)) + .build()); + AccessibilityEvent volume = stateChanged(VOLUME, "Volume", SYSTEM_UI_PACKAGE); + volume.setClassName(VOLUME_DIALOG_CLASS); + interpret(volume); + waitMs(600); + + assertTranscript( + "0ms state from=60 original start=0 delay=10/550 settled announcement=\"Volume\" from" + + " com.android.systemui volume | src 60\"Volume\" was 10\"Settings\" | A" + + " 10\"Settings\"", + "10ms state from=60 stable start=0 delay=10/550 settled announcement=\"Volume\" from" + + " com.android.systemui volume | src 60\"Volume\" was 10\"Settings\" | A" + + " 10\"Settings\""); + } + + @Test + public void titleOfAWindowListedBeforeItsEventIsFound() { + openSettings(); + assertEquals("Settings", interpreter.getWindowTitle(SETTINGS).toString()); + + setWindows( + statusBar(), + navigationBar(), + app(SETTINGS, "Settings").inactive().build(), + app(DOWNLOADS, "Downloads").layer(2).active().build()); + + assertEquals("Downloads", interpreter.getWindowTitle(DOWNLOADS).toString()); + } + + // Screens + + /** Puts Settings on screen, lets the interpreter take it in, and starts the transcript there. */ + private void openSettings() { + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + waitMs(600); + transcript.clear(); + serviceWindows.windowListRequests = 0; + startTime = SystemClock.uptimeMillis(); + } + + private static AccessibilityWindowInfo statusBar() { + return window(STATUS_BAR, AccessibilityWindowInfo.TYPE_SYSTEM, "Status bar") + .layer(5) + .bounds(new Rect(0, 0, 1080, 80)) + .build(); + } + + private static AccessibilityWindowInfo navigationBar() { + return window(NAVIGATION_BAR, AccessibilityWindowInfo.TYPE_SYSTEM, "Navigation bar") + .layer(5) + .bounds(new Rect(0, 2280, 1080, 2400)) + .build(); + } + + private static WindowBuilder app(int id, @Nullable String title) { + return window(id, AccessibilityWindowInfo.TYPE_APPLICATION, title); + } + + private static WindowBuilder window(int id, int type, @Nullable String title) { + return new WindowBuilder(id, type, title); + } + + /** Makes a new window each time, as the windows Android lists are copies. */ + private static class WindowBuilder { + private final int id; + private final int type; + private final @Nullable String title; + private int layer = 1; + private Rect bounds = SCREEN; + private boolean active; + private boolean pictureInPicture; + private @Nullable String rootPackage; + + WindowBuilder(int id, int type, @Nullable String title) { + this.id = id; + this.type = type; + this.title = title; + } + + WindowBuilder layer(int layer) { + this.layer = layer; + return this; + } + + WindowBuilder bounds(Rect bounds) { + this.bounds = bounds; + return this; + } + + /** Active and focused, as the window the user is in is. */ + WindowBuilder active() { + active = true; + return this; + } + + WindowBuilder inactive() { + active = false; + return this; + } + + WindowBuilder pictureInPicture() { + pictureInPicture = true; + return this; + } + + WindowBuilder rootPackage(String packageName) { + rootPackage = packageName; + return this; + } + + AccessibilityWindowInfo build() { + AccessibilityWindowInfo window = new AccessibilityWindowInfo(); + var shadow = shadowOf(window); + shadow.setId(id); + shadow.setType(type); + shadow.setTitle(title); + shadow.setLayer(layer); + shadow.setBoundsInScreen(bounds); + shadow.setActive(active); + shadow.setFocused(active); + shadow.setPictureInPicture(pictureInPicture); + shadow.setDisplayId(Display.DEFAULT_DISPLAY); + if (rootPackage != null) { + AccessibilityNodeInfo root = new AccessibilityNodeInfo(); + root.setPackageName(rootPackage); + shadow.setRoot(root); + } + return window; + } + } + + private void setWindows(AccessibilityWindowInfo... windows) { + SparseArray> onAllDisplays = new SparseArray<>(); + onAllDisplays.put(Display.DEFAULT_DISPLAY, new ArrayList<>(Arrays.asList(windows))); + serviceWindows.setWindows(onAllDisplays); + } + + private void installApp(String packageName, String label) { + PackageInfo packageInfo = new PackageInfo(); + packageInfo.packageName = packageName; + packageInfo.applicationInfo = new ApplicationInfo(); + packageInfo.applicationInfo.packageName = packageName; + packageInfo.applicationInfo.nonLocalizedLabel = label; + shadowOf(service.getPackageManager()).installPackage(packageInfo); + } + + // Events + + private void interpret(AccessibilityEvent event) { + interpreter.interpret(event, /* eventId= */ null); + } + + private static void waitMs(long ms) { + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(ms)); + } + + private static AccessibilityEvent windowsChanged(int windowId, int changes) { + AccessibilityEvent event = event(TYPE_WINDOWS_CHANGED, windowId); + ReflectionHelpers.callInstanceMethod( + event, "setWindowChanges", ClassParameter.from(int.class, changes)); + return event; + } + + private static AccessibilityEvent stateChanged( + int windowId, CharSequence text, String packageName) { + AccessibilityEvent event = event(TYPE_WINDOW_STATE_CHANGED, windowId); + event.getText().add(text); + event.setPackageName(packageName); + return event; + } + + /** A pane opening or closing in the window, as Android reports it with the pane as source. */ + private static AccessibilityEvent pane(int windowId, int contentChangeType, String paneTitle) { + AccessibilityEvent event = event(TYPE_WINDOW_STATE_CHANGED, windowId); + event.setContentChangeTypes(contentChangeType); + event.setPackageName(SETTINGS_PACKAGE); + AccessibilityNodeInfo source = new AccessibilityNodeInfo(); + source.setPaneTitle(paneTitle); + source.setPackageName(SETTINGS_PACKAGE); + ShadowAccessibilityRecord shadow = Shadow.extract(event); + shadow.setSourceNode(source); + return event; + } + + private static AccessibilityEvent event(int type, int windowId) { + AccessibilityEvent event = new AccessibilityEvent(type); + ReflectionHelpers.callInstanceMethod( + event, "setWindowId", ClassParameter.from(int.class, windowId)); + event.setEventTime(SystemClock.uptimeMillis()); + return event; + } + + // Transcript + + private void assertTranscript(String... expected) { + assertEquals(String.join("\n", expected), String.join("\n", transcript)); + } + + /** Everything a listener can read from the interpretation, leaving out what is unset. */ + private String describe(EventInterpretation in) { + StringBuilder line = new StringBuilder(); + line.append(SystemClock.uptimeMillis() - startTime).append("ms "); + line.append(in.getEventType() == TYPE_WINDOWS_CHANGED ? "windows" : "state"); + line.append(" from=").append(windowId(in.getWindowIdFromEvent())); + if (in.getDisplayId() != Display.DEFAULT_DISPLAY) { + line.append(" display=").append(in.getDisplayId()); + } + if (in.getWindowChangeTypes() != 0) { + line.append(" changes=0x").append(Integer.toHexString(in.getWindowChangeTypes())); + } + if (in.getContentChangeTypes() != 0) { + line.append(" content=0x").append(Integer.toHexString(in.getContentChangeTypes())); + } + flag(line, in.isOriginalEvent(), "original"); + flag(line, in.areWindowsStable(), "stable"); + flag(line, !in.isAllowAnnounce(), "silent"); + flag(line, in.isInterpretFirstTimeWhenWakeUp(), "wakeUp"); + flag(line, in.isWebDialogOpenedEvent(), "webDialog"); + flag(line, in.getSourceNode() != null, "source"); + long start = in.getEventStartTime(); + line.append(" start=").append((start == 0) ? 0 : start - startTime); + line.append(" delay=").append(in.getTotalDelayMs()).append('/').append(in.getMaxDelayMs()); + flag(line, in.getMainWindowsChanged(), "main"); + flag(line, in.getPicInPicChanged(), "pipChanged"); + flag(line, in.getInputMethodChanged(), "imeChanged"); + flag(line, in.getShouldAnnounceInputMethodChange(), "announceIme"); + flag(line, in.getHorizontalPlacement(), "sideBySide"); + flag(line, in.hasTitlesFromStateChange(), "settled"); + @Nullable Announcement announcement = in.getAnnouncement(); + if (announcement != null) { + line.append(" announcement=\"").append(announcement.text()).append("\" from "); + line.append(announcement.packageName()); + flag(line, announcement.isFromVolumeControlPanel(), "volume"); + flag(line, announcement.isFromInputMethodEditor(), "keyboard"); + } + if (in.getAnchorNodeRole() != 0) { + line.append(" anchor=").append(in.getAnchorNodeRole()); + } + if (in.getSourceBoundsInScreen() != null) { + line.append(" bounds=").append(in.getSourceBoundsInScreen().toShortString()); + } + window(line, "src", in.getEventSourceWindow()); + window(line, "A", in.getWindowA()); + window(line, "B", in.getWindowB()); + window(line, "other", in.getOtherActiveWindow()); + window(line, "pip", in.getPicInPic()); + window(line, "ime", in.getInputMethod()); + return line.toString(); + } + + /** + * Describes the window in one role. A role with no window, which a listener sees as untitled, is + * left out unless it just lost its window. + */ + private void window(StringBuilder line, String role, WindowInterpretation window) { + boolean changed = window.idOrTitleChanged(); + if (window.getId() == WINDOW_ID_NONE + && window.getTitle() == null + && !changed + && TextUtils.equals(window.getTitleForFeedback(), untitled())) { + return; + } + line.append(" | ").append(role).append(' ').append(windowId(window.getId())); + title(line, window.getTitle()); + if (!TextUtils.equals(window.getTitleForFeedback(), window.getTitle())) { + line.append(" feedback "); + title(line, window.getTitleForFeedback()); + } + if (changed) { + flag(line, window.hasTitleFromStateChange(), "settled"); + line.append(" was ").append(windowId(window.getOldId())); + title(line, window.getOldTitle()); + } + } + + private String untitled() { + return service.getString(com.google.android.accessibility.utils.R.string.untitled_window); + } + + private static void title(StringBuilder line, @Nullable CharSequence title) { + if (title != null) { + line.append('"').append(title).append('"'); + } + } + + private static String windowId(int id) { + return (id == WINDOW_ID_NONE) ? "-" : String.valueOf(id); + } + + private static void flag(StringBuilder line, boolean set, String name) { + if (set) { + line.append(' ').append(name); + } + } +} From f9aa16052933e93f25a6933a190373348cb38da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Mon, 12 Oct 2026 11:10:47 +0200 Subject: [PATCH 2/7] Don't look up windows for roles that have no window At a window event, and at each 10 ms check while it waits for the windows to settle, the window interpreter looks up the title of the window in each of its six roles, and the app of each window without a title. A role without a window, such as the second app when the screen isn't split, has the id -1, which no window has. Looking up its title missed the kept windows, so the interpreter fetched them again, and looking up its app fetched them once more. On a screen with one app, and no split screen, dialog, picture-in-picture window or keyboard, each check asked Android for the windows 13 times. These lookups now return at once, with the same result, so such a check asks once, to map the windows to roles. The failed lookups also refreshed the kept windows at each check. The titles now come from the windows kept since the last window event, which Android sends for every change to the windows, and a window missing from them still makes the interpreter fetch them again. A later commit gives each check a list of its own. In the tests, a screen change that waits the full 550 ms asks for the window list 229 times instead of 913, and one that waits 200 ms with animations off asks 23 times instead of 275. What the interpreter reports is unchanged. --- .../accessibility/utils/input/WindowEventInterpreter.java | 6 +++++- .../utils/input/WindowEventInterpreterTest.java | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java index 379462a89..0222e1cf9 100644 --- a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java +++ b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java @@ -400,7 +400,8 @@ public CharSequence getAccessibilityPaneTitle(int windowId) { } private @Nullable CharSequence getWindowTitleFromWindowInfo(int windowId) { - if (!FeatureSupport.supportGetTitleFromWindows()) { + // A role without a window has none to find, and not finding one would fetch the windows again. + if (!FeatureSupport.supportGetTitleFromWindows() || windowId == WINDOW_ID_NONE) { return null; } @@ -1425,6 +1426,9 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque private static @Nullable CharSequence getWindowPackageName( AccessibilityService service, int windowId) { + if (windowId == WINDOW_ID_NONE) { + return null; + } @Nullable CharSequence packageName = null; List windows = getAllWindows(service); diff --git a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java index d7fced568..d6108890e 100644 --- a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java +++ b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java @@ -216,6 +216,10 @@ public void reducedDelayEndsTheWaitSooner() { + " 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", "200ms windows from=11 changes=0x1 stable start=0 delay=200/200 main | src 11\"Wi-Fi\"" + " was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\""); + // The event asks for the windows three times: to note their titles, to map them to roles, and + // to look up the titles of the windows in those roles. Each of the 20 checks asks once, to map + // them to roles again, and the roles that have no window don't ask. + assertEquals(3 + 20, serviceWindows.windowListRequests); } @Test From 72d4bc9e4b501c74fbaf10e83acb2e35d501cb21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Mon, 12 Oct 2026 11:49:52 +0200 Subject: [PATCH 3/7] Use one window list for each window event and each check At a window event, the window interpreter asked Android for the windows up to three times: to note their titles, to map them to roles, and for the titles of the windows in those roles. Each check while it waited for the windows to settle asked again to map them to roles. The window an event names is often gone, as when the event reports that it closed. It was missing from the kept windows at each lookup, so each check fetched them twice more for its title, and once more for its app, since looking up a window's app always fetched the windows anew. Now each window event and each check fetches the windows once, and all its lookups use that list: titles, roles and apps. A window missing from it isn't looked for in a list fetched a moment later. Each check still fetches its own list, so windows that change before their windows event are noticed as before, and the titles now come from the same list as the roles. Lookups from the rest of Backtalk use the list of the last event or check, and fetch the windows again for a window missing from it, as title lookups did. Looking up a window's app now works that way too. The focus history names the window at every focus change, and in a window without a title that looked up its app, fetching the windows each time. In the tests, each event and each check asks for the window list once. A screen change that waits the full 550 ms asks 57 times, against 229 after the previous commit and 913 before it, and one that waits 200 ms asks 21 times. What the interpreter reports is unchanged. --- .../utils/input/WindowEventInterpreter.java | 88 ++++++++++--------- .../input/WindowEventInterpreterTest.java | 9 +- 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java index 0222e1cf9..777c747e5 100644 --- a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java +++ b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java @@ -304,6 +304,9 @@ public String toString() { private long cachedWindowsTime; // How long the windows are kept, in case a change came without a windows event. private static final long WINDOWS_CACHE_MAX_AGE_MS = 2_000; + // Whether a window event or a check is being interpreted. All its lookups use the windows it + // fetched, which are current, so a window missing from them doesn't make them be fetched again. + private boolean interpreting; private final boolean isSplitScreenModeAvailable; private final HashMap windowIdToData = new HashMap<>(); // Caches the window roles from last window transition for comparison. @@ -400,19 +403,28 @@ public CharSequence getAccessibilityPaneTitle(int windowId) { } private @Nullable CharSequence getWindowTitleFromWindowInfo(int windowId) { + if (!FeatureSupport.supportGetTitleFromWindows()) { + return null; + } + + @Nullable AccessibilityWindowInfo window = findWindow(windowId); + return (window == null) ? null : AccessibilityWindowInfoUtils.getTitle(window); + } + + private @Nullable AccessibilityWindowInfo findWindow(int windowId) { // A role without a window has none to find, and not finding one would fetch the windows again. - if (!FeatureSupport.supportGetTitleFromWindows() || windowId == WINDOW_ID_NONE) { + if (windowId == WINDOW_ID_NONE) { return null; } long fetchedBefore = cachedWindowsTime; @Nullable AccessibilityWindowInfo window = findWindow(windowsOnAllDisplays(), windowId); - if (window == null && cachedWindowsTime == fetchedBefore) { + if (window == null && !interpreting && cachedWindowsTime == fetchedBefore) { // The window may have appeared since the windows were fetched, before its windows event. cachedWindowsOnAllDisplays = null; window = findWindow(windowsOnAllDisplays(), windowId); } - return (window == null) ? null : AccessibilityWindowInfoUtils.getTitle(window); + return window; } private static @Nullable AccessibilityWindowInfo findWindow( @@ -430,7 +442,8 @@ public CharSequence getAccessibilityPaneTitle(int windowId) { /** * Returns the windows on every display, kept until the windows change. Asking the system for them * each time held up the announcement of every swipe, which looks up the window's title and - * whether the screen is split. + * whether the screen is split. Each window event, and each check while the windows settle, + * fetches them once for all its lookups. */ private SparseArray> windowsOnAllDisplays() { long now = SystemClock.uptimeMillis(); @@ -590,7 +603,13 @@ public void interpret(AccessibilityEvent event, @Nullable EventId eventId, boole if (screenTransitionStartTime == 0) { screenTransitionStartTime = event.getEventTime(); } - InterpretationAndRoles interpAndRoles = interpretInternal(event, depth); + InterpretationAndRoles interpAndRoles; + interpreting = true; + try { + interpAndRoles = interpretInternal(event, depth); + } finally { + interpreting = false; + } EventInterpretation interpretation = interpAndRoles.interpretation; WindowRoles newWindowRoles = interpAndRoles.roles; @@ -846,10 +865,18 @@ void delayedInterpret(EventInterpretation interpretation, @Nullable EventId even // Map windows to roles, detect window role changes. WindowRoles latestRoles = (pendingWindowRoles == null) ? windowRoles : pendingWindowRoles; WindowRoles newWindowRoles = new WindowRoles(latestRoles); - updateWindowRoles(interpretation, service, newWindowRoles, depth + 1); - setWindowTitles(newWindowRoles); - detectWindowChanges(newWindowRoles, interpretation, depth + 1); - detectInputMethodChanged(newWindowRoles, interpretation, /* checkDuplicate= */ true, depth + 1); + // Look at the windows as they are now, since they can change before their windows event. + cachedWindowsOnAllDisplays = null; + interpreting = true; + try { + updateWindowRoles(interpretation, service, newWindowRoles, depth + 1); + setWindowTitles(newWindowRoles); + detectWindowChanges(newWindowRoles, interpretation, depth + 1); + detectInputMethodChanged( + newWindowRoles, interpretation, /* checkDuplicate= */ true, depth + 1); + } finally { + interpreting = false; + } LogUtils.v(TAG, "END delayedInterpret() interpretation=%s", interpretation); // Don't send delayed interpretation if it's a dialog opened event to avoid announcing the @@ -1027,8 +1054,7 @@ private void updateWindowTitlesImp( { HashSet windowIdsToBeRemoved = new HashSet<>(windowIdToData.keySet()); int displayId = interpretation.getDisplayId(); - List windows = - AccessibilityServiceCompatUtils.getWindowsOnAllDisplays(service).get(displayId); + List windows = windowsOnAllDisplays().get(displayId); if (windows == null) { return; } @@ -1196,8 +1222,7 @@ private void updateWindowRoles( AccessibilityWindowInfo inputMethodWindow = null; int displayId = interpretation.getDisplayId(); - List windows = - AccessibilityServiceCompatUtils.getWindowsOnAllDisplays(service).get(displayId); + List windows = windowsOnAllDisplays().get(displayId); // If there are no windows available, clear the cached IDs. if (windows == null || windows.isEmpty()) { @@ -1377,7 +1402,7 @@ private void detectInputMethodChanged( // source or not. If so, use the announcement first. If not, they are supposed to be // different transitions and should keep both. int inputMethodWindowId = interpretation.getInputMethod().id; - CharSequence inputMethodPackageName = getWindowPackageName(service, inputMethodWindowId); + CharSequence inputMethodPackageName = getWindowPackageName(inputMethodWindowId); CharSequence announcementPackageName = announcement.packageName(); if (inputMethodWindowId == WINDOW_ID_NONE || (inputMethodPackageName != null @@ -1411,7 +1436,7 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque @Nullable CharSequence packageName = (window == null) ? null : window.eventPackageName; // Try to get package name from accessibility window info if it's not in the map. if (packageName == null) { - packageName = getWindowPackageName(service, windowId); + packageName = getWindowPackageName(windowId); } if (packageName != null) { title = getApplicationLabel(packageName); @@ -1424,25 +1449,13 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque return title; } - private static @Nullable CharSequence getWindowPackageName( - AccessibilityService service, int windowId) { - if (windowId == WINDOW_ID_NONE) { + private @Nullable CharSequence getWindowPackageName(int windowId) { + @Nullable AccessibilityWindowInfo window = findWindow(windowId); + if (window == null) { return null; } - @Nullable CharSequence packageName = null; - - List windows = getAllWindows(service); - for (AccessibilityWindowInfo accessibilityWindowInfo : windows) { - if (accessibilityWindowInfo.getId() == windowId) { - AccessibilityNodeInfo rootNode = - AccessibilityWindowInfoUtils.getRoot(accessibilityWindowInfo); - if (rootNode != null) { - packageName = rootNode.getPackageName(); - } - break; - } - } - return packageName; + AccessibilityNodeInfo rootNode = AccessibilityWindowInfoUtils.getRoot(window); + return (rootNode == null) ? null : rootNode.getPackageName(); } private @Nullable CharSequence getApplicationLabel(CharSequence packageName) { @@ -1461,17 +1474,6 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque return packageManager.getApplicationLabel(applicationInfo); } - private static List getAllWindows(AccessibilityService service) { - List windows = new ArrayList<>(); - SparseArray> windowsOnAllDisplays = - AccessibilityServiceCompatUtils.getWindowsOnAllDisplays(service); - final int displaySize = windowsOnAllDisplays.size(); - for (int i = 0; i < displaySize; i++) { - windows.addAll(windowsOnAllDisplays.valueAt(i)); - } - return windows; - } - // ///////////////////////////////////////////////////////////////////////////////////// // Inner classes: event interpretation diff --git a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java index d6108890e..a61ecc57a 100644 --- a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java +++ b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java @@ -200,6 +200,9 @@ public void screenWithoutAStateChangeTitleWaitsForTheLongestDelay() { + " \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + " \"untitled\" was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\""); + // Each event and each of the 55 checks asks for the windows once, though each looks up the + // Settings window, which has closed. + assertEquals(2 + 55, serviceWindows.windowListRequests); } @Test @@ -216,10 +219,8 @@ public void reducedDelayEndsTheWaitSooner() { + " 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\"", "200ms windows from=11 changes=0x1 stable start=0 delay=200/200 main | src 11\"Wi-Fi\"" + " was 10\"Settings\" | A 11\"Wi-Fi\" was 10\"Settings\""); - // The event asks for the windows three times: to note their titles, to map them to roles, and - // to look up the titles of the windows in those roles. Each of the 20 checks asks once, to map - // them to roles again, and the roles that have no window don't ask. - assertEquals(3 + 20, serviceWindows.windowListRequests); + // The event and each of the 20 checks ask for the windows once, for all their lookups. + assertEquals(1 + 20, serviceWindows.windowListRequests); } @Test From 60024a92cd09a1fa409bf247f379fcd32100eb56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Mon, 12 Oct 2026 13:36:04 +0200 Subject: [PATCH 4/7] Keep the package of each window's root by window id A window without a title is named after its app, which the window interpreter found by fetching the window's root and reading its package. Fetching a root waits for the app's main thread, which is busy drawing while the screen changes, and the interpreter fetched it at every check while it waited for the windows to settle, and again whenever the rest of Backtalk asked for the window's title. The focus history asks at every focus change. The package of each window's root is now kept by window id, since a window keeps its id until it closes and no other window gets it. It is used only while the window is listed, so a window that has closed is untitled as before, and the packages of windows that are gone are dropped at each windows event. A root that can't be fetched, as before its app has drawn it, isn't kept, so it is asked for again at the next lookup. In the tests, a window without a title has its root fetched once while the screen changes to it and back, instead of 59 times. A window whose root can't be fetched at first is named after its app at the first check after it can be, with one fetch instead of one at every check. What the interpreter reports is unchanged. --- .../utils/input/WindowEventInterpreter.java | 26 +++++++- .../input/WindowEventInterpreterTest.java | 66 ++++++++++++++++++- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java index 777c747e5..91b52ce76 100644 --- a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java +++ b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java @@ -307,6 +307,10 @@ public String toString() { // Whether a window event or a check is being interpreted. All its lookups use the windows it // fetched, which are current, so a window missing from them doesn't make them be fetched again. private boolean interpreting; + // The package of each listed window's root, by window id, since fetching a root waits for the + // window's app. A window keeps its id until it closes, and no other window gets it. A root that + // couldn't be fetched isn't kept. + private final SparseArray rootPackageNames = new SparseArray<>(); private final boolean isSplitScreenModeAvailable; private final HashMap windowIdToData = new HashMap<>(); // Caches the window roles from last window transition for comparison. @@ -367,6 +371,7 @@ public void onResumeInfrastructure() { public void onSuspendInfrastructure() { displayMonitor.removeDisplayStateChangedListener(this); + rootPackageNames.clear(); } private void clearScreenState() { @@ -1052,6 +1057,7 @@ private void updateWindowTitlesImp( } case AccessibilityEvent.TYPE_WINDOWS_CHANGED -> { { + forgetRootPackagesOfClosedWindows(); HashSet windowIdsToBeRemoved = new HashSet<>(windowIdToData.keySet()); int displayId = interpretation.getDisplayId(); List windows = windowsOnAllDisplays().get(displayId); @@ -1454,8 +1460,24 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque if (window == null) { return null; } - AccessibilityNodeInfo rootNode = AccessibilityWindowInfoUtils.getRoot(window); - return (rootNode == null) ? null : rootNode.getPackageName(); + @Nullable CharSequence packageName = rootPackageNames.get(windowId); + if (packageName == null) { + AccessibilityNodeInfo rootNode = AccessibilityWindowInfoUtils.getRoot(window); + packageName = (rootNode == null) ? null : rootNode.getPackageName(); + if (packageName != null) { + rootPackageNames.put(windowId, packageName); + } + } + return packageName; + } + + private void forgetRootPackagesOfClosedWindows() { + SparseArray> windows = windowsOnAllDisplays(); + for (int i = rootPackageNames.size() - 1; i >= 0; i--) { + if (findWindow(windows, rootPackageNames.keyAt(i)) == null) { + rootPackageNames.removeAt(i); + } + } } private @Nullable CharSequence getApplicationLabel(CharSequence packageName) { diff --git a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java index a61ecc57a..e3ac69f3c 100644 --- a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java +++ b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java @@ -62,6 +62,7 @@ import org.robolectric.shadow.api.Shadow; import org.robolectric.shadows.ShadowAccessibilityRecord; import org.robolectric.shadows.ShadowAccessibilityService; +import org.robolectric.shadows.ShadowAccessibilityWindowInfo; import org.robolectric.util.ReflectionHelpers; import org.robolectric.util.ReflectionHelpers.ClassParameter; @@ -75,7 +76,12 @@ * window change run one by one. */ @RunWith(RobolectricTestRunner.class) -@Config(sdk = 34, shadows = WindowEventInterpreterTest.WindowsShadow.class) +@Config( + sdk = 34, + shadows = { + WindowEventInterpreterTest.WindowsShadow.class, + WindowEventInterpreterTest.RootsShadow.class + }) public class WindowEventInterpreterTest { /** The service the interpreter belongs to. */ @@ -118,6 +124,19 @@ protected List getWindows() { } } + /** Counts how often a window's root is fetched, which waits for the window's app. */ + @Implements(AccessibilityWindowInfo.class) + public static class RootsShadow extends ShadowAccessibilityWindowInfo { + static int rootRequests; + + @Implementation + @Override + protected AccessibilityNodeInfo getRoot() { + rootRequests++; + return super.getRoot(); + } + } + private static final int STATUS_BAR = 1; private static final int NAVIGATION_BAR = 2; private static final int SETTINGS = 10; @@ -151,6 +170,7 @@ protected List getWindows() { public void setUp() { service = Robolectric.setupService(TestService.class); serviceWindows = Shadow.extract(service); + RootsShadow.rootRequests = 0; interpreter = new WindowEventInterpreter(service, new DisplayMonitor(service)); interpreter.addListener((interpretation, eventId) -> transcript.add(describe(interpretation))); } @@ -343,15 +363,56 @@ public void windowWithoutATitleIsNamedAfterItsApp() { interpret(windowsChanged(NOTES, WINDOWS_CHANGE_ADDED)); interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); waitMs(600); + assertEquals("Notes", interpreter.getWindowTitle(NOTES).toString()); + // Notes closes, and is no longer named after its app. + setWindows(statusBar(), navigationBar(), app(SETTINGS, "Settings").active().build()); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(NOTES, WINDOWS_CHANGE_REMOVED)); + waitMs(600); assertTranscript( "0ms windows from=50 changes=0x1 original start=0 delay=10/550 main | src 50 feedback" + " \"Notes\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", "0ms windows from=10 changes=0x2 original start=0 delay=10/550 main | src 10 feedback" + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", + "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\"", + "600ms windows from=10 changes=0x1 original start=600 delay=10/550 main | src" + + " 10\"Settings\" was 10 | A 10\"Settings\" was 50", + "600ms windows from=50 changes=0x2 original start=600 delay=10/550 main | src 50 feedback" + + " \"untitled\" was 10 | A 10\"Settings\" was 50", + "1150ms windows from=50 changes=0x2 stable start=600 delay=550/550 main | src 50 feedback" + + " \"untitled\" was 10 | A 10\"Settings\" was 50"); + // The app is asked for the window's root once, though the window is named at every check. + assertEquals(1, RootsShadow.rootRequests); + } + + @Test + public void appOfAWindowWithoutATitleIsAskedForAgainUntilItAnswers() { + installApp(NOTES_PACKAGE, "Notes"); + openSettings(); + + // At first the window's root can't be fetched, as before its app has drawn it. + setWindows(statusBar(), navigationBar(), app(NOTES, null).active().build()); + interpret(windowsChanged(NOTES, WINDOWS_CHANGE_ADDED)); + interpret(windowsChanged(SETTINGS, WINDOWS_CHANGE_REMOVED)); + waitMs(100); + setWindows( + statusBar(), + navigationBar(), + app(NOTES, null).rootPackage(NOTES_PACKAGE).active().build()); + int rootRequestsBefore = RootsShadow.rootRequests; + waitMs(500); + + assertTranscript( + "0ms windows from=50 changes=0x1 original start=0 delay=10/550 main | src 50 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"untitled\" was 10\"Settings\"", + "0ms windows from=10 changes=0x2 original start=0 delay=10/550 main | src 10 feedback" + + " \"untitled\" was 10\"Settings\" | A 50 feedback \"untitled\" was 10\"Settings\"", "550ms windows from=10 changes=0x2 stable start=0 delay=550/550 main | src 10 feedback" + " \"untitled\" was 10\"Settings\" | A 50 feedback \"Notes\" was 10\"Settings\""); - assertEquals("Notes", interpreter.getWindowTitle(NOTES).toString()); + // Once the root can be fetched, it is fetched once, and its package is kept. + assertEquals(1, RootsShadow.rootRequests - rootRequestsBefore); } @Test @@ -447,6 +508,7 @@ private void openSettings() { waitMs(600); transcript.clear(); serviceWindows.windowListRequests = 0; + RootsShadow.rootRequests = 0; startTime = SystemClock.uptimeMillis(); } From 5b04401adf720d25edf38b317d0fccc308653325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Mon, 12 Oct 2026 13:59:45 +0200 Subject: [PATCH 5/7] Let direct touch check its switches before asking apps and the package manager Direct touch looks at the screen 150 ms after every window change and every window interpretation, even when it isn't turned on for any app. Each look fetched the root of the app window in front, of every other application window and of every system window, which waits for each window's app, and read the app's manifest with up to three package manager queries, before it checked a single switch. The main thread stalls on a watch in #152 included this look, run then by screen changes, which 299eab35 dealt with. Window changes still ran it in full. Now each look asks only for what it needs: - The package of a window's root comes from the window interpreter, which keeps it by window id, so the app in front is asked once per window. - The manifest is read only the first time an app is seen, the only time it is used. - The checks that need other windows' roots run only when the switches, the window types and the focus leave the answer open. Another app's window can only be found when an application window other than the game is open, and that already turns direct touch off as a dialog, so those roots aren't fetched at all now. A system window is asked whether it belongs to System UI only when it is active and covers half the screen, as the notification shade does. What direct touch decides, says and sends as the passthrough region is unchanged. A new test feeds window changes through the window interpreter to direct touch, as the service does. Three window changes over a game that isn't turned on fetched 23 window roots and read the manifest 7 times, and now fetch one root and read the manifest once. With the game turned on, and the notification shade opening and closing over it, 30 roots and 9 manifest reads became 2 and 1. DirectTouchPolicyTest goes through all 128 combinations of inputs: the answer is the same as before, and the checks that need roots run only when they can change it. --- .../talkback/TalkBackService.java | 3 +- .../directtouch/DirectTouchController.kt | 31 +- .../talkback/directtouch/DirectTouchPolicy.kt | 20 +- .../directtouch/DirectTouchSettings.kt | 3 + .../directtouch/DirectTouchControllerTest.kt | 329 ++++++++++++++++++ .../directtouch/DirectTouchPolicyTest.kt | 38 +- .../directtouch/DirectTouchSettingsTest.kt | 8 + .../utils/input/WindowEventInterpreter.java | 12 +- .../input/WindowEventInterpreterTest.java | 10 + 9 files changed, 425 insertions(+), 29 deletions(-) create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchControllerTest.kt diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java index 12aa46c7d..b21b168ca 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java @@ -2549,7 +2549,8 @@ public boolean resettingNodeCursor() { windowEventInterpreter.addListener(uiChangeEventInterpreter); windowEventInterpreter.addListener(imageCaptioner); directTouchController = - new DirectTouchController(this, prefs, pipeline.getFeedbackReturner()); + new DirectTouchController( + this, prefs, pipeline.getFeedbackReturner(), windowEventInterpreter); windowEventInterpreter.addListener(directTouchController); ringerModeAndScreenMonitor.addScreenChangedListener(directTouchController); displayMonitor.addDisplayStateChangedListener(directTouchController); diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt index e3981e7a9..2af92d612 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt @@ -67,6 +67,8 @@ class DirectTouchController( private val service: AccessibilityService, private val prefs: SharedPreferences, private val feedback: Pipeline.FeedbackReturner, + /** Keeps the package of each window's root, which direct touch looks up on every look. */ + private val windowEventInterpreter: WindowEventInterpreter, ) : WindowEventInterpreter.WindowEventHandler, RingerModeAndScreenMonitor.ScreenChangedListener, @@ -222,8 +224,7 @@ class DirectTouchController( return } val windows = service.windows.orEmpty() - val mainPackage = - windows.firstOrNull { it.id == mainWindowId }?.root?.packageName?.toString() + val mainPackage = windows.firstOrNull { it.id == mainWindowId }?.let { rootPackage(it) } val directTyping = mainPackage != null && DirectTouchSettings.isDirectTyping(prefs, mainPackage) val shouldBeActive = mainPackage != null && shouldBeActive(windows, mainPackage) val navBarDirect = isNavBarDirect() @@ -349,9 +350,12 @@ class DirectTouchController( } } + /** The package of the window's root, which the window's app is asked for once per window. */ + private fun rootPackage(window: AccessibilityWindowInfo): String? = + windowEventInterpreter.getRootPackageName(window)?.toString() + private fun isSystemUiWindow(window: AccessibilityWindowInfo): Boolean = - window.type == AccessibilityWindowInfo.TYPE_SYSTEM && - window.root?.packageName?.toString() == SYSTEM_UI + window.type == AccessibilityWindowInfo.TYPE_SYSTEM && rootPackage(window) == SYSTEM_UI private fun shouldBeActive( windows: List, @@ -360,8 +364,11 @@ class DirectTouchController( if (mainPackage in IGNORED_PACKAGES) { return false } - val capable = DirectTouchCapability.declaresDirectTouch(service.packageManager, mainPackage) - DirectTouchSettings.onFirstSight(prefs, mainPackage, capable) + // Only the first sight of an app uses its manifest, which the package manager has to read. + if (!DirectTouchSettings.isSeen(prefs, mainPackage)) { + val capable = DirectTouchCapability.declaresDirectTouch(service.packageManager, mainPackage) + DirectTouchSettings.onFirstSight(prefs, mainPackage, capable) + } // A dialog is a window of its own, so any other application window is either a dialog from the // same app or another app. val otherApplicationWindows = @@ -371,11 +378,10 @@ class DirectTouchController( masterEnabled = DirectTouchSettings.isMasterEnabled(prefs), appEnabled = DirectTouchSettings.isAppEnabled(prefs, mainPackage), screenInteractive = screenInteractive && displayOn, - foreignAppWindow = - otherApplicationWindows.any { it.root?.packageName?.toString() != mainPackage }, - systemUiCoversHalf = systemUiCoversHalf(windows), dialogShowing = otherApplicationWindows.isNotEmpty(), textFieldFocused = textFieldFocused, + foreignAppWindow = { otherApplicationWindows.any { rootPackage(it) != mainPackage } }, + systemUiCoversHalf = { systemUiCoversHalf(windows) }, ) return DirectTouchPolicy.shouldBeActive(input) } @@ -384,11 +390,10 @@ class DirectTouchController( val displayHeight = displayBounds().height() val bounds = Rect() return windows.any { window -> - if (!isSystemUiWindow(window)) { - return@any false - } window.getBoundsInScreen(bounds) - DirectTouchPolicy.coversHalfScreen(window.isActive, bounds.height(), displayHeight) + // Telling System UI apart asks the window's app for its root, so the shape comes first. + DirectTouchPolicy.coversHalfScreen(window.isActive, bounds.height(), displayHeight) && + isSystemUiWindow(window) } } diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt index c4b211bb6..3d59922ff 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicy.kt @@ -16,18 +16,22 @@ package com.google.android.accessibility.talkback.directtouch -/** What is on screen and how the user set things up, as far as direct touch cares. */ +/** + * What is on screen and how the user set things up, as far as direct touch cares. The two checks + * that ask apps for their windows' roots are functions, so that they run only when the rest leave + * the answer open. + */ data class DirectTouchInput( val masterEnabled: Boolean, /** The app in front is turned on in the direct touch list. */ val appEnabled: Boolean, val screenInteractive: Boolean, - /** Another app's window is on top of the game. */ - val foreignAppWindow: Boolean, - /** The notification shade or quick settings cover at least half the screen. */ - val systemUiCoversHalf: Boolean, val dialogShowing: Boolean, val textFieldFocused: Boolean, + /** Another app's window is on top of the game. */ + val foreignAppWindow: () -> Boolean, + /** The notification shade or quick settings cover at least half the screen. */ + val systemUiCoversHalf: () -> Boolean, ) object DirectTouchPolicy { @@ -45,8 +49,8 @@ object DirectTouchPolicy { input.masterEnabled && input.appEnabled && input.screenInteractive && - !input.foreignAppWindow && - !input.systemUiCoversHalf && !input.dialogShowing && - !input.textFieldFocused + !input.textFieldFocused && + !input.foreignAppWindow() && + !input.systemUiCoversHalf() } diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt index 5ed8dda9c..e89934a52 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettings.kt @@ -94,6 +94,9 @@ object DirectTouchSettings { prefs.edit().putBoolean(PREF_TYPING_PREFIX + pkg, enabled).apply() } + /** Whether [pkg] has been seen, after which [onFirstSight] changes nothing for it. */ + fun isSeen(prefs: SharedPreferences, pkg: String): Boolean = pkg in stringSet(prefs, PREF_SEEN) + /** * Records that [pkg] has been seen. The first time, an app that [declaresCapability] is turned * on. Later calls change nothing, so an app the user turned off stays off. diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchControllerTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchControllerTest.kt new file mode 100644 index 000000000..da999234b --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchControllerTest.kt @@ -0,0 +1,329 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback.directtouch + +import android.accessibilityservice.AccessibilityService +import android.content.pm.ApplicationInfo +import android.content.pm.PackageInfo +import android.content.pm.PackageManager +import android.graphics.Rect +import android.os.Bundle +import android.os.Looper +import android.os.SystemClock +import android.util.SparseArray +import android.view.Display +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ADDED +import android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_REMOVED +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import com.google.android.accessibility.talkback.Feedback +import com.google.android.accessibility.talkback.Pipeline +import com.google.android.accessibility.utils.input.WindowEventInterpreter +import com.google.android.accessibility.utils.monitor.DisplayMonitor +import java.time.Duration +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.Implementation +import org.robolectric.annotation.Implements +import org.robolectric.shadow.api.Shadow +import org.robolectric.shadows.ShadowAccessibilityService +import org.robolectric.shadows.ShadowAccessibilityWindowInfo +import org.robolectric.shadows.ShadowApplicationPackageManager +import org.robolectric.util.ReflectionHelpers +import org.robolectric.util.ReflectionHelpers.ClassParameter + +/** + * Feeds window changes through the window interpreter to direct touch, as the service does, and + * records what direct touch says and which passthrough regions it sends. It also counts what each + * look at the screen costs: fetching a window's root waits for the window's app, and reading an + * app's manifest asks the package manager. + */ +@RunWith(RobolectricTestRunner::class) +@Config( + sdk = [34], + qualifiers = "w360dp-h780dp-xxhdpi", + shadows = + [ + DirectTouchControllerTest.WindowsShadow::class, + DirectTouchControllerTest.RootsShadow::class, + DirectTouchControllerTest.ManifestsShadow::class, + ], +) +class DirectTouchControllerTest { + + class TestService : AccessibilityService() { + override fun onAccessibilityEvent(event: AccessibilityEvent) {} + + override fun onInterrupt() {} + } + + /** Lists the windows a test puts on screen, in a new list each time as Android does. */ + @Implements(AccessibilityService::class) + class WindowsShadow : ShadowAccessibilityService() { + var onScreen: List = emptyList() + + @Implementation + override fun getWindows(): List = ArrayList(onScreen) + + @Implementation + override fun getWindowsOnAllDisplays(): SparseArray> = + SparseArray>().apply { + put(Display.DEFAULT_DISPLAY, ArrayList(onScreen)) + } + } + + /** Counts how often a window's root is fetched. */ + @Implements(AccessibilityWindowInfo::class) + class RootsShadow : ShadowAccessibilityWindowInfo() { + @Implementation + override fun getRoot(): AccessibilityNodeInfo? { + rootRequests++ + return super.getRoot() + } + + companion object { + var rootRequests = 0 + } + } + + /** Counts how often the package manager is asked for an app's manifest, with its meta-data. */ + @Implements(className = "android.app.ApplicationPackageManager", isInAndroidSdk = false) + class ManifestsShadow : ShadowApplicationPackageManager() { + @Implementation + override fun getApplicationInfo(packageName: String, flags: Int): ApplicationInfo { + if (flags and PackageManager.GET_META_DATA != 0) { + manifestRequests++ + } + return super.getApplicationInfo(packageName, flags) + } + + companion object { + var manifestRequests = 0 + } + } + + private lateinit var service: TestService + private lateinit var serviceWindows: WindowsShadow + private lateinit var interpreter: WindowEventInterpreter + private lateinit var controller: DirectTouchController + private val prefs = FakeSharedPreferences() + private val transcript = mutableListOf() + + @Before + fun setUp() { + service = Robolectric.setupService(TestService::class.java) + serviceWindows = Shadow.extract(service) + interpreter = WindowEventInterpreter(service, DisplayMonitor(service)) + controller = + DirectTouchController( + service, + prefs, + Pipeline.FeedbackReturner { feedback -> + transcript += describe(feedback) + true + }, + interpreter, + ) + interpreter.addListener(controller) + installApp(GAME_PACKAGE, asksForDirectTouch = false) + installApp(GAME2_PACKAGE, asksForDirectTouch = true) + RootsShadow.rootRequests = 0 + ManifestsShadow.manifestRequests = 0 + } + + @Test + fun gameNotTurnedOnStaysOff() { + windowsChange(GAME, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game()) + // A toast comes and goes over the game. + windowsChange(TOAST, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game(), toast()) + windowsChange(TOAST, WINDOWS_CHANGE_REMOVED, statusBar(), navigationBar(), game()) + + assertTranscript() + // Direct touch looks at the screen several times for each window change. The game's root is + // fetched once, and its manifest is read once, the first time the game is seen. + assertEquals(1, RootsShadow.rootRequests) + assertEquals(1, ManifestsShadow.manifestRequests) + } + + @Test + fun gameTurnedOnHandsTouchBackWhileTheShadeIsOpen() { + DirectTouchSettings.setAppEnabled(prefs, GAME_PACKAGE, true) + + windowsChange(GAME, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game()) + val underShade = game(inactive = true) + windowsChange(SHADE, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), underShade, shade()) + windowsChange(SHADE, WINDOWS_CHANGE_REMOVED, statusBar(), navigationBar(), game()) + + assertTranscript( + "region [0,80][1080,2220] complex=false", + "say \"Direct touch on\"", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region none", + "say \"Direct touch off\"", + "region [0,80][1080,2220] complex=false", + "say \"Direct touch on\"", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + ) + // Only the game's root and the shade's are fetched, once each. The bars are too small to be the + // shade, so they are never asked whether they belong to System UI. + assertEquals(2, RootsShadow.rootRequests) + assertEquals(1, ManifestsShadow.manifestRequests) + } + + @Test + fun gameThatAsksForDirectTouchGetsItTheFirstTimeItIsSeen() { + windowsChange(GAME2, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game2()) + windowsChange(TOAST, WINDOWS_CHANGE_ADDED, statusBar(), navigationBar(), game2(), toast()) + windowsChange(TOAST, WINDOWS_CHANGE_REMOVED, statusBar(), navigationBar(), game2()) + + assertTranscript( + "region [0,80][1080,2220] complex=false", + "say \"Direct touch on\"", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + "region [0,80][1080,2220] complex=false", + ) + assertEquals(1, RootsShadow.rootRequests) + assertEquals(1, ManifestsShadow.manifestRequests) + } + + // Screens + + private fun statusBar() = + window(STATUS_BAR, SYSTEM, "Status bar", Rect(0, 0, 1080, 80), SYSTEM_UI) + + private fun navigationBar() = + window(NAVIGATION_BAR, SYSTEM, "Navigation bar", Rect(0, 2220, 1080, 2340), SYSTEM_UI) + + private fun game(inactive: Boolean = false) = + window(GAME, APPLICATION, "Game", SCREEN, GAME_PACKAGE, active = !inactive) + + private fun game2() = window(GAME2, APPLICATION, "Game 2", SCREEN, GAME2_PACKAGE, active = true) + + private fun toast() = window(TOAST, SYSTEM, "Saved", Rect(200, 1900, 880, 2050), SYSTEM_UI) + + private fun shade() = + window(SHADE, SYSTEM, "Notification shade", SCREEN, SYSTEM_UI, active = true, layer = 9) + + /** A window as Android lists it, with a title, so that the interpreter needn't name it. */ + private fun window( + id: Int, + type: Int, + title: String, + bounds: Rect, + rootPackage: String, + active: Boolean = false, + layer: Int = if (type == SYSTEM) 5 else 1, + ): AccessibilityWindowInfo { + val window = AccessibilityWindowInfo() + val shadow = shadowOf(window) + shadow.setId(id) + shadow.setType(type) + shadow.setTitle(title) + shadow.setBoundsInScreen(bounds) + shadow.setLayer(layer) + shadow.setActive(active) + shadow.setFocused(active) + shadow.setDisplayId(Display.DEFAULT_DISPLAY) + shadow.setRoot(AccessibilityNodeInfo().apply { packageName = rootPackage }) + return window + } + + private fun installApp(packageName: String, asksForDirectTouch: Boolean) { + val info = PackageInfo() + info.packageName = packageName + info.applicationInfo = + ApplicationInfo().apply { + this.packageName = packageName + if (asksForDirectTouch) { + metaData = Bundle().apply { putBoolean(DirectTouchSettings.CAPABILITY_KEY, true) } + } + } + shadowOf(service.packageManager).installPackage(info) + } + + // Events + + /** The windows change to [windows], Android reports it, and things settle for a second. */ + private fun windowsChange(windowId: Int, changes: Int, vararg windows: AccessibilityWindowInfo) { + serviceWindows.onScreen = windows.toList() + val event = AccessibilityEvent(AccessibilityEvent.TYPE_WINDOWS_CHANGED) + ReflectionHelpers.callInstanceMethod( + event, + "setWindowId", + ClassParameter.from(Int::class.javaPrimitiveType, windowId), + ) + ReflectionHelpers.callInstanceMethod( + event, + "setWindowChanges", + ClassParameter.from(Int::class.javaPrimitiveType, changes), + ) + event.eventTime = SystemClock.uptimeMillis() + interpreter.interpret(event, null) + controller.onAccessibilityEvent(event) + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofSeconds(1)) + } + + // Transcript + + private fun assertTranscript(vararg expected: String) { + assertEquals(expected.joinToString("\n"), transcript.joinToString("\n")) + } + + private fun describe(feedback: Feedback): String { + val part = feedback.failovers().first() + part.speech()?.let { + return "say \"${it.text()}\"" + } + part.passThroughMode()?.let { + val region = it.region() + return if (region.isEmpty) { + "region none" + } else { + "region ${region.bounds.toShortString()} complex=${region.isComplex}" + } + } + return part.toString() + } + + private companion object { + const val STATUS_BAR = 1 + const val NAVIGATION_BAR = 2 + const val SHADE = 3 + const val TOAST = 4 + const val GAME = 10 + const val GAME2 = 11 + const val GAME_PACKAGE = "com.example.game" + const val GAME2_PACKAGE = "com.example.game2" + const val SYSTEM_UI = "com.android.systemui" + const val SYSTEM = AccessibilityWindowInfo.TYPE_SYSTEM + const val APPLICATION = AccessibilityWindowInfo.TYPE_APPLICATION + val SCREEN = Rect(0, 0, 1080, 2340) + } +} diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt index 37449f843..2f9e4ea6d 100644 --- a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchPolicyTest.kt @@ -16,6 +16,7 @@ package com.google.android.accessibility.talkback.directtouch +import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -26,10 +27,10 @@ class DirectTouchPolicyTest { masterEnabled = true, appEnabled = true, screenInteractive = true, - foreignAppWindow = false, - systemUiCoversHalf = false, dialogShowing = false, textFieldFocused = false, + foreignAppWindow = { false }, + systemUiCoversHalf = { false }, ) @Test @@ -39,8 +40,8 @@ class DirectTouchPolicyTest { @Test fun eachHandBackConditionTurnsItOff() { - assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(foreignAppWindow = true))) - assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(systemUiCoversHalf = true))) + assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(foreignAppWindow = { true }))) + assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(systemUiCoversHalf = { true }))) assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(dialogShowing = true))) assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(textFieldFocused = true))) assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(screenInteractive = false))) @@ -48,6 +49,35 @@ class DirectTouchPolicyTest { assertFalse(DirectTouchPolicy.shouldBeActive(base().copy(appEnabled = false))) } + @Test + fun windowRootsAreAskedForOnlyWhenTheyCanChangeTheAnswer() { + for (inputs in 0 until 128) { + fun set(bit: Int) = inputs and (1 shl bit) != 0 + var rootChecks = 0 + val input = + DirectTouchInput( + masterEnabled = set(0), + appEnabled = set(1), + screenInteractive = set(2), + dialogShowing = set(3), + textFieldFocused = set(4), + foreignAppWindow = { + rootChecks++ + set(5) + }, + systemUiCoversHalf = { + rootChecks++ + set(6) + }, + ) + val rest = set(0) && set(1) && set(2) && !set(3) && !set(4) + val expected = rest && !set(5) && !set(6) + assertEquals("inputs $inputs", expected, DirectTouchPolicy.shouldBeActive(input)) + val expectedRootChecks = if (!rest) 0 else if (set(5)) 1 else 2 + assertEquals("root checks for inputs $inputs", expectedRootChecks, rootChecks) + } + } + @Test fun inactiveSystemBarIsNotTheShade() { // A landscape navigation bar is 730 px tall on a 1080 px display, but inactive until touched. diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt index b408af89e..5895d7dbb 100644 --- a/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/directtouch/DirectTouchSettingsTest.kt @@ -43,6 +43,14 @@ class DirectTouchSettingsTest { assertFalse(DirectTouchSettings.isAppEnabled(prefs, "a.b")) } + @Test + fun appIsSeenOnceFirstSightRecordsIt() { + assertFalse(DirectTouchSettings.isSeen(prefs, "a.b")) + DirectTouchSettings.onFirstSight(prefs, "a.b", declaresCapability = false) + assertTrue(DirectTouchSettings.isSeen(prefs, "a.b")) + assertFalse(DirectTouchSettings.isSeen(prefs, "c.d")) + } + @Test fun firstSightIgnoresIncapableApp() { DirectTouchSettings.onFirstSight(prefs, "a.b", declaresCapability = false) diff --git a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java index 91b52ce76..2193b45d7 100644 --- a/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java +++ b/utils/src/main/java/com/google/android/accessibility/utils/input/WindowEventInterpreter.java @@ -1457,9 +1457,15 @@ private CharSequence getWindowTitleForFeedback(int windowId, @Nullable CharSeque private @Nullable CharSequence getWindowPackageName(int windowId) { @Nullable AccessibilityWindowInfo window = findWindow(windowId); - if (window == null) { - return null; - } + return (window == null) ? null : getRootPackageName(window); + } + + /** + * Returns the package of the root of a window that is on screen. The window's app is asked the + * first time, which waits for its main thread, and the answer is kept until the window closes. + */ + public @Nullable CharSequence getRootPackageName(AccessibilityWindowInfo window) { + int windowId = window.getId(); @Nullable CharSequence packageName = rootPackageNames.get(windowId); if (packageName == null) { AccessibilityNodeInfo rootNode = AccessibilityWindowInfoUtils.getRoot(window); diff --git a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java index e3ac69f3c..e01fbfc43 100644 --- a/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java +++ b/utils/src/test/java/com/google/android/accessibility/utils/input/WindowEventInterpreterTest.java @@ -415,6 +415,16 @@ public void appOfAWindowWithoutATitleIsAskedForAgainUntilItAnswers() { assertEquals(1, RootsShadow.rootRequests - rootRequestsBefore); } + @Test + public void rootPackageIsKeptForOthersToo() { + AccessibilityWindowInfo notes = app(NOTES, null).rootPackage(NOTES_PACKAGE).active().build(); + setWindows(statusBar(), navigationBar(), notes); + + assertEquals(NOTES_PACKAGE, interpreter.getRootPackageName(notes).toString()); + assertEquals(NOTES_PACKAGE, interpreter.getRootPackageName(notes).toString()); + assertEquals(1, RootsShadow.rootRequests); + } + @Test public void paneTitleNamesTheWindowUntilThePaneCloses() { openSettings(); From 113b2868fbbcb40fa724b9eeee1dc9f5688d37fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Mon, 12 Oct 2026 14:55:36 +0200 Subject: [PATCH 6/7] Fetch the window's root only when there is a focus to restore When the screen changes, Backtalk first tries to restore the focus the user last had in the window that is now active. restoreLastFocusedNode fetched that window's root before it looked in the focus history, so a new screen, which has no focus to restore, had its root fetched for nothing, and a later step that focuses the first item fetches it again. The fetch waits for the window's app, which is busy drawing the new screen, and on a watch in #152 it was among the main thread stalls after the screen turned on. The root is now fetched only once the history has a focus for the window, and after the check that keeps a focus from an app out of a System UI window. Restoring a focus also fetched the root a second time, to check that the node is still in the window. That check now uses the root fetched already. What gets focused is unchanged. In a new test, a screen without a focus to restore fetches no root instead of one, and so does a System UI window whose last focus came from an app, while a focus that is restored fetches the root once instead of twice. --- .../actor/FocusActorForScreenStateChange.java | 28 ++- .../FocusActorForScreenStateChangeTest.kt | 213 ++++++++++++++++++ 2 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java index d2f8258d9..4af15e40f 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java @@ -18,6 +18,7 @@ import static androidx.core.view.accessibility.AccessibilityNodeInfoCompat.FOCUS_INPUT; import static androidx.core.view.accessibility.AccessibilityWindowInfoCompat.TYPE_INPUT_METHOD; +import static com.google.android.accessibility.utils.AccessibilityWindowInfoUtils.WINDOW_ID_NONE; import static com.google.android.accessibility.utils.traversal.TraversalStrategy.SEARCH_FOCUS_FORWARD; import android.accessibilityservice.AccessibilityService; @@ -135,6 +136,17 @@ private boolean isOnTopWindow( return (currentActive.getLayer() > lastWindow.getLayer()) && activeRect.intersect(lastRect); } + /** + * Whether the node is still in the window, as {@link AccessibilityNodeInfoUtils#isInWindow} + * checks, with the window's root fetched already. + */ + private static boolean isInWindow( + AccessibilityNodeInfoCompat node, int windowId, AccessibilityNodeInfoCompat windowRoot) { + int nodeWindowId = node.getWindowId(); + return (nodeWindowId == WINDOW_ID_NONE || nodeWindowId == windowId) + && AccessibilityNodeInfoUtils.hasDescendant(windowRoot, node); + } + /** Restores last focus from {@link AccessibilityFocusActionHistory} to the active window. */ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) { @@ -175,12 +187,6 @@ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) return false; } - AccessibilityNodeInfoCompat root = - AccessibilityWindowInfoUtils.getRootCompat(currentActiveWindow); - if (root == null) { - return false; - } - int windowId = currentActiveWindow.getId(); int windowType = AccessibilityWindowInfoUtils.getType(currentActiveWindow); WindowIdentifier windowIdentifier = WindowIdentifier.create(windowId, screenState); @@ -202,6 +208,14 @@ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) return false; } + // Fetching the root waits for the window's app, which is drawing the new screen, so it is + // fetched only once there is a focus to restore, and only once. + AccessibilityNodeInfoCompat root = + AccessibilityWindowInfoUtils.getRootCompat(currentActiveWindow); + if (root == null) { + return false; + } + long startTime = primesController.getTime(); AccessibilityNodeInfoCompat nodeToRestoreFocus = @@ -221,7 +235,7 @@ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) // b/365017291: Only check native nodes because sometimes the WebView is gone in the node // tree. || (!WebInterfaceUtils.isWebContainer(nodeToRestoreFocus) - && !AccessibilityNodeInfoUtils.isInWindow(nodeToRestoreFocus, currentActiveWindow))) { + && !isInWindow(nodeToRestoreFocus, windowId, root))) { LogUtils.e(TAG, "Do not restore focus from the invalid node " + nodeToRestoreFocus); return false; } diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt new file mode 100644 index 000000000..97ce56c45 --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt @@ -0,0 +1,213 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback.actor + +import android.accessibilityservice.AccessibilityService +import android.graphics.Rect +import android.os.SystemClock +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat +import com.google.android.accessibility.talkback.ActorState +import com.google.android.accessibility.talkback.ActorStateWritable +import com.google.android.accessibility.talkback.Feedback +import com.google.android.accessibility.talkback.Pipeline +import com.google.android.accessibility.talkback.PrimesController +import com.google.android.accessibility.talkback.focusmanagement.interpreter.ScreenState +import com.google.android.accessibility.talkback.focusmanagement.record.AccessibilityFocusActionHistory +import com.google.android.accessibility.talkback.focusmanagement.record.FocusActionInfo +import com.google.android.accessibility.talkback.monitor.InputMethodMonitor +import com.google.android.accessibility.utils.FocusFinder +import com.google.android.accessibility.utils.FormFactorUtils +import com.google.android.accessibility.utils.Performance.EVENT_ID_UNTRACKED +import com.google.android.accessibility.utils.input.WindowsDelegate +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.Implementation +import org.robolectric.annotation.Implements +import org.robolectric.shadows.ShadowAccessibilityWindowInfo + +/** + * Restoring the last focus when the screen changes, and what it costs: fetching a window's root + * waits for the window's app, which is busy drawing the new screen. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34], shadows = [FocusActorForScreenStateChangeTest.RootsShadow::class]) +class FocusActorForScreenStateChangeTest { + + class TestService : AccessibilityService() { + override fun onAccessibilityEvent(event: AccessibilityEvent) {} + + override fun onInterrupt() {} + } + + /** Counts how often a window's root is fetched. */ + @Implements(AccessibilityWindowInfo::class) + class RootsShadow : ShadowAccessibilityWindowInfo() { + @Implementation + override fun getRoot(): AccessibilityNodeInfo? { + rootRequests++ + return super.getRoot() + } + + companion object { + var rootRequests = 0 + } + } + + /** Names each window by its id, as the window interpreter would. */ + private val windows = + object : WindowsDelegate { + override fun getWindowTitle(windowId: Int): CharSequence = "Window $windowId" + + override fun getAccessibilityPaneTitle(windowId: Int): CharSequence = "" + + override fun isSplitScreenMode(displayId: Int): Boolean = false + + override fun hasAccessibilityPane(windowId: Int): Boolean = false + } + + private lateinit var service: TestService + private lateinit var history: AccessibilityFocusActionHistory + private lateinit var actor: FocusActorForScreenStateChange + private val focused = mutableListOf() + + @Before + fun setUp() { + service = Robolectric.setupService(TestService::class.java) + FormFactorUtils.initialize(service) + history = AccessibilityFocusActionHistory(service) + actor = + FocusActorForScreenStateChange( + service, + InputMethodMonitor(service), + FocusFinder(service), + PrimesController(), + ) + actor.setActorState( + ActorState( + ActorStateWritable( + null, null, null, null, history.reader, null, null, null, null, null, null, null, null, + null, + ) + ) + ) + actor.setPipeline( + Pipeline.FeedbackReturner { feedback -> + focused += feedback.failovers().first().focus()?.target() + true + } + ) + } + + @Test + fun newScreenWithoutAFocusToRestoreLeavesItsRootAlone() { + val app = window(APP, AccessibilityWindowInfo.TYPE_APPLICATION, root()) + shadowOf(service).setWindows(listOf(app)) + + assertFalse(restore(app)) + assertEquals(0, RootsShadow.rootRequests) + } + + @Test + fun systemWindowGetsNoFocusLastPutInAnApp() { + // The history has a focus for the shade's window id, whose node reports an application + // window. Backtalk doesn't restore a focus from an app into System UI. + val appRoot = root() + val app = window(SHADE, AccessibilityWindowInfo.TYPE_APPLICATION, appRoot) + history.onAccessibilityFocusAction(button(app, appRoot), focusInfo(), 1, screen(app)) + val shade = window(SHADE, AccessibilityWindowInfo.TYPE_SYSTEM, root()) + shadowOf(service).setWindows(listOf(shade)) + + assertFalse(restore(shade)) + assertEquals(0, RootsShadow.rootRequests) + } + + @Test + fun focusLastPutInTheWindowIsRestored() { + val root = root() + val app = window(APP, AccessibilityWindowInfo.TYPE_APPLICATION, root) + history.onAccessibilityFocusAction(button(app, root), focusInfo(), 1, screen(app)) + shadowOf(service).setWindows(listOf(app)) + + assertTrue(restore(app)) + assertEquals(listOf("Play"), focused.map { it?.text?.toString() }) + // One root serves to find the node and to check that the node is still in the window. + assertEquals(1, RootsShadow.rootRequests) + } + + /** Restores the last focus into [active], counting only the roots that fetches. */ + private fun restore(active: AccessibilityWindowInfo): Boolean { + RootsShadow.rootRequests = 0 + return actor.restoreLastFocusedNode(screen(active), EVENT_ID_UNTRACKED) + } + + private fun screen(active: AccessibilityWindowInfo) = + ScreenState(windows, active, SystemClock.uptimeMillis()) + + private fun root() = AccessibilityNodeInfo().apply { packageName = APP_PACKAGE } + + private fun window(id: Int, type: Int, root: AccessibilityNodeInfo): AccessibilityWindowInfo { + val window = AccessibilityWindowInfo() + val shadow = shadowOf(window) + shadow.setId(id) + shadow.setType(type) + shadow.setActive(true) + shadow.setFocused(true) + shadow.setBoundsInScreen(Rect(0, 0, 1080, 2340)) + shadow.setRoot(root) + return window + } + + /** A button in the window, which can still be found there when its focus is restored. */ + private fun button( + window: AccessibilityWindowInfo, + root: AccessibilityNodeInfo, + ): AccessibilityNodeInfoCompat { + val node = AccessibilityNodeInfo() + node.packageName = APP_PACKAGE + node.className = "android.widget.Button" + node.text = "Play" + node.isVisibleToUser = true + node.isEnabled = true + node.isClickable = true + node.isFocusable = true + node.setBoundsInScreen(Rect(100, 1000, 980, 1150)) + shadowOf(node).setAccessibilityWindowInfo(window) + shadowOf(node).setRefreshReturnValue(true) + shadowOf(root).addChild(node) + return AccessibilityNodeInfoCompat.wrap(node) + } + + private fun focusInfo() = + FocusActionInfo.builder().setSourceAction(FocusActionInfo.LOGICAL_NAVIGATION).build() + + private companion object { + const val APP = 10 + const val SHADE = 20 + const val APP_PACKAGE = "com.example.app" + } +} From 1e8f195f0aef3a68c85b1b0e8ebd5e2578cbffe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BBywek?= Date: Mon, 12 Oct 2026 16:01:56 +0200 Subject: [PATCH 7/7] Make the System UI focus test fail without its rule, and fix two comments FocusActorForScreenStateChangeTest checked that a focus from an app isn't restored into a System UI window by seeing the restore fail without fetching the root. The node it recorded wasn't in the shade's tree, though, so without that rule the restore would have failed anyway, only later. Both windows now share one root that holds the node, and the test checks that the history has a focus for the shade's window id, so only the rule stops the restore. With the rule taken out, the test fails. An unused import is gone. The comment on fetching the root in restoreLastFocusedNode used "only once" in two senses, and the comment on direct touch's window interpreter read oddly. Both now say plainly what they mean. --- .../actor/FocusActorForScreenStateChange.java | 2 +- .../directtouch/DirectTouchController.kt | 2 +- .../actor/FocusActorForScreenStateChangeTest.kt | 17 +++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java index 4af15e40f..3e016fcb7 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChange.java @@ -209,7 +209,7 @@ public boolean restoreLastFocusedNode(ScreenState screenState, EventId eventId) } // Fetching the root waits for the window's app, which is drawing the new screen, so it is - // fetched only once there is a focus to restore, and only once. + // fetched only when there is a focus to restore, and a single time. AccessibilityNodeInfoCompat root = AccessibilityWindowInfoUtils.getRootCompat(currentActiveWindow); if (root == null) { diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt index 2af92d612..82e9376cc 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/directtouch/DirectTouchController.kt @@ -67,7 +67,7 @@ class DirectTouchController( private val service: AccessibilityService, private val prefs: SharedPreferences, private val feedback: Pipeline.FeedbackReturner, - /** Keeps the package of each window's root, which direct touch looks up on every look. */ + /** Keeps the package of each window's root, which every look at the screen needs. */ private val windowEventInterpreter: WindowEventInterpreter, ) : WindowEventInterpreter.WindowEventHandler, diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt index 97ce56c45..63cb24161 100644 --- a/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/FocusActorForScreenStateChangeTest.kt @@ -25,11 +25,11 @@ import android.view.accessibility.AccessibilityWindowInfo import androidx.core.view.accessibility.AccessibilityNodeInfoCompat import com.google.android.accessibility.talkback.ActorState import com.google.android.accessibility.talkback.ActorStateWritable -import com.google.android.accessibility.talkback.Feedback import com.google.android.accessibility.talkback.Pipeline import com.google.android.accessibility.talkback.PrimesController import com.google.android.accessibility.talkback.focusmanagement.interpreter.ScreenState import com.google.android.accessibility.talkback.focusmanagement.record.AccessibilityFocusActionHistory +import com.google.android.accessibility.talkback.focusmanagement.record.AccessibilityFocusActionHistory.WindowIdentifier import com.google.android.accessibility.talkback.focusmanagement.record.FocusActionInfo import com.google.android.accessibility.talkback.monitor.InputMethodMonitor import com.google.android.accessibility.utils.FocusFinder @@ -38,6 +38,7 @@ import com.google.android.accessibility.utils.Performance.EVENT_ID_UNTRACKED import com.google.android.accessibility.utils.input.WindowsDelegate import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test @@ -135,12 +136,16 @@ class FocusActorForScreenStateChangeTest { @Test fun systemWindowGetsNoFocusLastPutInAnApp() { // The history has a focus for the shade's window id, whose node reports an application - // window. Backtalk doesn't restore a focus from an app into System UI. - val appRoot = root() - val app = window(SHADE, AccessibilityWindowInfo.TYPE_APPLICATION, appRoot) - history.onAccessibilityFocusAction(button(app, appRoot), focusInfo(), 1, screen(app)) - val shade = window(SHADE, AccessibilityWindowInfo.TYPE_SYSTEM, root()) + // window, and the node is still there. Only the rule that keeps a focus from an app out of + // System UI stops it from being restored. + val root = root() + val app = window(SHADE, AccessibilityWindowInfo.TYPE_APPLICATION, root) + history.onAccessibilityFocusAction(button(app, root), focusInfo(), 1, screen(app)) + val shade = window(SHADE, AccessibilityWindowInfo.TYPE_SYSTEM, root) shadowOf(service).setWindows(listOf(shade)) + assertNotNull( + history.reader.getLastFocusActionRecordInWindow(WindowIdentifier.create(SHADE, screen(shade))) + ) assertFalse(restore(shade)) assertEquals(0, RootsShadow.rootRequests)