From 21fee727428a022df43dcb006343082f28a337d8 Mon Sep 17 00:00:00 2001 From: makhlwf Date: Sat, 11 Jul 2026 22:35:40 +0200 Subject: [PATCH 1/4] fix(editor): sync TalkBack focus with the text cursor The editor wraps Rosemoe's CodeEditor, a custom-drawn View. Its accessibility node reported the whole document as a single generic editable node (class name CodeEditor) and advertised no text-movement granularities, ACTION_SET_SELECTION or granularity actions. TalkBack therefore had no way to move the real text caret with the reading position: double-tapping a word/line placed the caret at the node's centre and subsequent typing landed on the wrong line. Make IDEEditor behave like a standard EditText for screen readers: - Report as android.widget.EditText and advertise CHARACTER/WORD/LINE movement granularities plus ACTION_SET_SELECTION and the next/previous-at-granularity actions. - Handle those actions by converting absolute char indices to (line, column) via the editor's Indexer and moving the real caret, then emit TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY so the spoken segment and caret stay in sync. - Track the last touch-exploration hover point and, on ACTION_CLICK, place the caret there so a double-tap lands on the explored word/line (only active while touch exploration is on, so sighted use is unaffected). Text-segment iteration lives in EditorAccessibilitySegments (character and word mirror the framework's AccessibilityIterators; line walks logical '\n'-separated lines) and is covered by unit tests. --- .../itsaky/androidide/editor/ui/IDEEditor.kt | 205 ++++++++++++++++++ .../utils/EditorAccessibilitySegments.kt | 192 ++++++++++++++++ .../utils/EditorAccessibilitySegmentsTest.kt | 175 +++++++++++++++ 3 files changed, 572 insertions(+) create mode 100644 editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt create mode 100644 editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt diff --git a/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt b/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt index 5b8742f22f..4d39006f5c 100644 --- a/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt +++ b/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt @@ -24,7 +24,11 @@ import android.os.Handler import android.os.Looper import android.util.AttributeSet import android.view.MotionEvent +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityManager +import android.view.accessibility.AccessibilityNodeInfo import android.view.inputmethod.EditorInfo +import android.widget.EditText import androidx.annotation.StringRes import androidx.annotation.VisibleForTesting import com.blankj.utilcode.util.FileUtils @@ -48,6 +52,7 @@ import com.itsaky.androidide.editor.schemes.IDEColorSchemeProvider import com.itsaky.androidide.editor.snippets.AbstractSnippetVariableResolver import com.itsaky.androidide.editor.snippets.FileVariableResolver import com.itsaky.androidide.editor.snippets.WorkspaceVariableResolver +import com.itsaky.androidide.editor.utils.EditorAccessibilitySegments import com.itsaky.androidide.eventbus.events.editor.ChangeType import com.itsaky.androidide.eventbus.events.editor.ColorSchemeInvalidatedEvent import com.itsaky.androidide.eventbus.events.editor.DocumentChangeEvent @@ -521,6 +526,206 @@ constructor( } } + // region Accessibility (screen reader) cursor synchronization + // + // The base CodeEditor exposes its whole content to accessibility services as a single + // editable node, but does not advertise text-movement granularities, ACTION_SET_SELECTION + // or the granularity actions. As a result TalkBack cannot keep the real text cursor in sync + // with the reading position: navigating or double-tapping placed the cursor in the wrong + // location and typing landed on the wrong line. The overrides below make the editor behave + // like a standard EditText for screen readers. + + private val accessibilityManager: AccessibilityManager? by lazy { + context.getSystemService(Context.ACCESSIBILITY_SERVICE) as? AccessibilityManager + } + + // Last touch-exploration (hover) position, in view coordinates, used to place the cursor + // where the user double-taps under TalkBack. NaN means "no recent exploration". + private var lastHoverX = Float.NaN + private var lastHoverY = Float.NaN + + private val isTouchExplorationEnabled: Boolean + get() = accessibilityManager?.isTouchExplorationEnabled == true + + override fun getAccessibilityClassName(): CharSequence = EditText::class.java.name + + override fun createAccessibilityNodeInfo(): AccessibilityNodeInfo? { + val info = super.createAccessibilityNodeInfo() ?: return null + if (!isReleased && isEnabled && isEditable) { + // Report as an EditText so TalkBack engages its text-editing affordances and keeps + // the caret in sync with granular navigation. + info.className = EditText::class.java.name + info.movementGranularities = EditorAccessibilitySegments.SUPPORTED_GRANULARITIES + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_SELECTION) + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_NEXT_AT_MOVEMENT_GRANULARITY) + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY) + } + return info + } + + override fun dispatchHoverEvent(event: MotionEvent): Boolean { + // Remember where the user is exploring so ACTION_CLICK can place the caret there. + if (isTouchExplorationEnabled) { + lastHoverX = event.x + lastHoverY = event.y + } + return super.dispatchHoverEvent(event) + } + + override fun performAccessibilityAction( + action: Int, + arguments: Bundle?, + ): Boolean { + when (action) { + AccessibilityNodeInfo.ACTION_SET_SELECTION -> + if (handleAccessibilitySetSelection(arguments)) return true + + AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY -> + if (handleAccessibilityGranularityMove(action, arguments, forward = true)) return true + + AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY -> + if (handleAccessibilityGranularityMove(action, arguments, forward = false)) return true + + AccessibilityNodeInfo.ACTION_CLICK -> + if (handleAccessibilityClick()) return true + } + return super.performAccessibilityAction(action, arguments) + } + + /** + * Places the caret where the user last explored by touch. This makes a TalkBack double-tap + * on a word or line move the real text cursor to that spot instead of the node's center. + */ + private fun handleAccessibilityClick(): Boolean { + if (isReleased || !isEditable || !isTouchExplorationEnabled) return false + val x = lastHoverX + val y = lastHoverY + if (x.isNaN() || y.isNaN()) return false + return try { + if (!isFocused) requestFocus() + val packed = getPointPositionOnScreen(x, y) + val line = (packed ushr 32).toInt() + val column = (packed and 0xffffffffL).toInt() + if (line < 0 || column < 0) return false + setSelection(line, column) + true + } catch (e: Exception) { + log.error("Error placing cursor from accessibility click", e) + false + } + } + + /** Handles [AccessibilityNodeInfo.ACTION_SET_SELECTION] using absolute character indices. */ + private fun handleAccessibilitySetSelection(arguments: Bundle?): Boolean { + if (isReleased || !isEditable || arguments == null) return false + if (!arguments.containsKey(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT) || + !arguments.containsKey(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT) + ) { + return false + } + val content = text + val length = content.length + val start = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT) + val end = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT) + if (start < 0 || end < 0 || start > length || end > length) return false + return applySelectionFromIndices(start, end) + } + + /** + * Handles the move-by-granularity actions. Moves the real text cursor to the next/previous + * character, word or line and reports the traversed segment so the screen reader reads it, + * mirroring [EditText] behaviour. + */ + private fun handleAccessibilityGranularityMove( + action: Int, + arguments: Bundle?, + forward: Boolean, + ): Boolean { + if (isReleased || !isEditable || arguments == null) return false + val granularity = arguments.getInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT, + ) + val extendSelection = arguments.getBoolean( + AccessibilityNodeInfo.ACTION_ARGUMENT_EXTEND_SELECTION_BOOLEAN, + false, + ) + val content = text + val cur = cursor ?: return false + val selStart = cur.left + val selEnd = cur.right + + val fromIndex = if (forward) selEnd else selStart + val segment = + if (forward) { + EditorAccessibilitySegments.following(content, granularity, fromIndex) + } else { + EditorAccessibilitySegments.preceding(content, granularity, fromIndex) + } ?: return false + + val segStart = segment[0] + val segEnd = segment[1] + val newCaret = if (forward) segEnd else segStart + + val applied = + if (extendSelection) { + val anchor = if (forward) selStart else selEnd + applySelectionFromIndices(minOf(anchor, newCaret), maxOf(anchor, newCaret)) + } else { + applySelectionFromIndices(newCaret, newCaret) + } + if (!applied) return false + + sendTextTraversedEvent(segStart, segEnd, action, granularity) + return true + } + + /** Converts absolute char indices to (line, column) positions and moves the selection. */ + private fun applySelectionFromIndices(start: Int, end: Int): Boolean { + val content = text + return try { + val indexer = content.indexer + val startPos = indexer.getCharPosition(start) + if (start == end) { + setSelection(startPos.line, startPos.column) + } else { + val endPos = indexer.getCharPosition(end) + setSelectionRegion(startPos.line, startPos.column, endPos.line, endPos.column) + } + true + } catch (e: Exception) { + log.error("Error applying accessibility selection [$start, $end]", e) + false + } + } + + /** + * Emits a [AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY] event so the + * screen reader announces the character/word/line the caret just traversed. + */ + @Suppress("DEPRECATION") // AccessibilityEvent(int) requires API 30; obtain() works on minSdk 28. + private fun sendTextTraversedEvent( + fromIndex: Int, + toIndex: Int, + action: Int, + granularity: Int, + ) { + val manager = accessibilityManager ?: return + if (!manager.isEnabled) return + val event = AccessibilityEvent.obtain( + AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY, + ) + onInitializeAccessibilityEvent(event) + event.className = EditText::class.java.name + event.fromIndex = fromIndex + event.toIndex = toIndex + event.action = action + event.movementGranularity = granularity + // The reader substrings [fromIndex, toIndex] out of the event text. + event.text.add(text.toString()) + sendAccessibilityEventUnchecked(event) + } + // endregion Accessibility + override fun copyTextToClipboard( text: CharSequence, start: Int, diff --git a/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt new file mode 100644 index 0000000000..4c4a352e16 --- /dev/null +++ b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt @@ -0,0 +1,192 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.editor.utils + +import android.view.accessibility.AccessibilityNodeInfo +import java.text.BreakIterator + +/** + * Pure text-segment iteration used to keep the editor's text cursor in sync with a + * screen reader's "move by granularity" navigation, so the editor behaves like a + * standard [android.widget.EditText] under TalkBack. + * + * The editor exposes its whole content to accessibility services as a single flat + * [CharSequence]. Movement granularity actions therefore operate on absolute character + * indices into that flat text; the caller is responsible for mapping the returned + * indices back to (line, column) positions in the editor. + * + * The character and word implementations mirror the semantics of the framework's + * (hidden) `android.view.AccessibilityIterators`, so navigation matches what users + * already expect from an [android.widget.EditText]. Line navigation walks logical + * lines (separated by `'\n'`), which for source code is the natural unit. + * + * @author Code On The Go + */ +object EditorAccessibilitySegments { + + private const val DONE = BreakIterator.DONE + + /** + * The set of movement granularities that [following] and [preceding] can handle. + * Intended to be advertised via [AccessibilityNodeInfo.setMovementGranularities]. + */ + const val SUPPORTED_GRANULARITIES = + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER or + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD or + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE + + /** + * Returns the `[start, end]` range (end-exclusive) of the segment that follows + * [offset] for the given [granularity], or `null` when there is nothing after it. + */ + fun following( + text: CharSequence, + granularity: Int, + offset: Int, + ): IntArray? = + when (granularity) { + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER -> characterFollowing(text, offset) + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD -> wordFollowing(text, offset) + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE -> lineFollowing(text, offset) + else -> null + } + + /** + * Returns the `[start, end]` range (end-exclusive) of the segment that precedes + * [offset] for the given [granularity], or `null` when there is nothing before it. + */ + fun preceding( + text: CharSequence, + granularity: Int, + offset: Int, + ): IntArray? = + when (granularity) { + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER -> characterPreceding(text, offset) + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD -> wordPreceding(text, offset) + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE -> linePreceding(text, offset) + else -> null + } + + // --------------------------------------------------------------------------------------------- + // Character + // --------------------------------------------------------------------------------------------- + + private fun characterFollowing(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset >= length) return null + val iterator = BreakIterator.getCharacterInstance().apply { setText(text.toString()) } + var start = if (offset < 0) 0 else offset + if (!iterator.isBoundary(start)) { + start = iterator.following(start) + if (start == DONE) return null + } + val end = iterator.following(start) + if (end == DONE) return null + return intArrayOf(start, end) + } + + private fun characterPreceding(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset <= 0) return null + val iterator = BreakIterator.getCharacterInstance().apply { setText(text.toString()) } + var end = if (offset > length) length else offset + if (!iterator.isBoundary(end)) { + end = iterator.preceding(end) + if (end == DONE) return null + } + val start = iterator.preceding(end) + if (start == DONE) return null + return intArrayOf(start, end) + } + + // --------------------------------------------------------------------------------------------- + // Word + // --------------------------------------------------------------------------------------------- + + private fun wordFollowing(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset >= length) return null + val iterator = BreakIterator.getWordInstance().apply { setText(text.toString()) } + var start = if (offset < 0) 0 else offset + // Skip past any non-word characters (whitespace, punctuation) to the start of a word. + while (!isLetterOrDigit(text, start) && !isWordStart(text, start)) { + start = iterator.following(start) + if (start == DONE) return null + } + val end = iterator.following(start) + if (end == DONE || !isWordEnd(text, end)) return null + return intArrayOf(start, end) + } + + private fun wordPreceding(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset <= 0) return null + val iterator = BreakIterator.getWordInstance().apply { setText(text.toString()) } + var end = if (offset > length) length else offset + while (!isLetterOrDigit(text, end - 1) && !isWordEnd(text, end)) { + end = iterator.preceding(end) + if (end == DONE) return null + } + val start = iterator.preceding(end) + if (start == DONE || !isWordStart(text, start)) return null + return intArrayOf(start, end) + } + + private fun isWordStart(text: CharSequence, index: Int): Boolean = + isLetterOrDigit(text, index) && (index == 0 || !isLetterOrDigit(text, index - 1)) + + private fun isWordEnd(text: CharSequence, index: Int): Boolean = + (index > 0 && isLetterOrDigit(text, index - 1)) && + (index == text.length || !isLetterOrDigit(text, index)) + + private fun isLetterOrDigit(text: CharSequence, index: Int): Boolean = + index in 0 until text.length && Character.isLetterOrDigit(text[index]) + + // --------------------------------------------------------------------------------------------- + // Line (logical, '\n'-separated) + // --------------------------------------------------------------------------------------------- + + private fun lineFollowing(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset >= length) return null + var start = if (offset < 0) 0 else offset + // When not already at a line start, advance to the beginning of the next line. + if (start > 0 && text[start - 1] != '\n') { + while (start < length && text[start] != '\n') start++ + if (start < length) start++ // step over the newline onto the next line + if (start >= length) return null + } + var end = start + while (end < length && text[end] != '\n') end++ + return intArrayOf(start, end) + } + + private fun linePreceding(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset <= 0) return null + var end = if (offset > length) length else offset + // If sitting on a line start, step back onto the previous line's newline. + if (text[end - 1] == '\n') end-- + if (end <= 0) return null + var start = end + while (start > 0 && text[start - 1] != '\n') start-- + var lineEnd = start + while (lineEnd < length && text[lineEnd] != '\n') lineEnd++ + return intArrayOf(start, lineEnd) + } +} diff --git a/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt new file mode 100644 index 0000000000..e2e697ac65 --- /dev/null +++ b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt @@ -0,0 +1,175 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.editor.utils + +import android.view.accessibility.AccessibilityNodeInfo +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * Unit tests for [EditorAccessibilitySegments], which drives screen-reader cursor + * synchronization. The returned ranges are absolute char indices into the flat text. + */ +@RunWith(RobolectricTestRunner::class) +class EditorAccessibilitySegmentsTest { + + private val char = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER + private val word = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD + private val line = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE + + private fun following(text: String, granularity: Int, offset: Int) = + EditorAccessibilitySegments.following(text, granularity, offset)?.toList() + + private fun preceding(text: String, granularity: Int, offset: Int) = + EditorAccessibilitySegments.preceding(text, granularity, offset)?.toList() + + // --------------------------------------------------------------------------------------------- + // Character + // --------------------------------------------------------------------------------------------- + + @Test + fun `character following returns the next single character`() { + assertThat(following("abc", char, 0)).isEqualTo(listOf(0, 1)) + assertThat(following("abc", char, 1)).isEqualTo(listOf(1, 2)) + } + + @Test + fun `character following at end returns null`() { + assertThat(following("abc", char, 3)).isNull() + } + + @Test + fun `character preceding returns the previous single character`() { + assertThat(preceding("abc", char, 3)).isEqualTo(listOf(2, 3)) + assertThat(preceding("abc", char, 1)).isEqualTo(listOf(0, 1)) + } + + @Test + fun `character preceding at start returns null`() { + assertThat(preceding("abc", char, 0)).isNull() + } + + // --------------------------------------------------------------------------------------------- + // Word + // --------------------------------------------------------------------------------------------- + + @Test + fun `word following skips whitespace and returns whole words`() { + val text = "foo bar" + assertThat(following(text, word, 0)).isEqualTo(listOf(0, 3)) // foo + assertThat(following(text, word, 3)).isEqualTo(listOf(4, 7)) // bar + } + + @Test + fun `word following past last word returns null`() { + assertThat(following("foo bar", word, 7)).isNull() + } + + @Test + fun `word preceding returns previous word`() { + val text = "foo bar" + assertThat(preceding(text, word, 7)).isEqualTo(listOf(4, 7)) // bar + assertThat(preceding(text, word, 4)).isEqualTo(listOf(0, 3)) // foo + } + + @Test + fun `word navigation skips separator punctuation between identifiers`() { + // Simulates code like "x=y" — the identifiers are words, '=' is skipped. + val text = "x=y" + assertThat(following(text, word, 0)).isEqualTo(listOf(0, 1)) // x + assertThat(following(text, word, 1)).isEqualTo(listOf(2, 3)) // y + } + + @Test + fun `word navigation treats a dotted identifier as a single word`() { + // Matches EditText/BreakIterator semantics: "a.b" is one word, not three. + val text = "a.b" + assertThat(following(text, word, 0)).isEqualTo(listOf(0, 3)) + } + + // --------------------------------------------------------------------------------------------- + // Line + // --------------------------------------------------------------------------------------------- + + @Test + fun `line following returns the current line when at its start`() { + val text = "abc\ndef\nghi" + assertThat(following(text, line, 0)).isEqualTo(listOf(0, 3)) // abc + } + + @Test + fun `line following from mid-line advances to the next line`() { + val text = "abc\ndef\nghi" + // offset 5 is inside "def"; next line is "ghi". + assertThat(following(text, line, 5)).isEqualTo(listOf(8, 11)) + } + + @Test + fun `line following from end of a line moves to the next line`() { + val text = "abc\ndef" + // offset 3 is the '\n' after abc -> caret sits at end of line 0. + assertThat(following(text, line, 3)).isEqualTo(listOf(4, 7)) // def + } + + @Test + fun `line preceding returns the line the caret sits at the end of`() { + val text = "abc\ndef" + assertThat(preceding(text, line, 7)).isEqualTo(listOf(4, 7)) // def + } + + @Test + fun `line preceding from a line start moves to the previous line`() { + val text = "abc\ndef" + // offset 4 is the start of "def"; previous line is "abc". + assertThat(preceding(text, line, 4)).isEqualTo(listOf(0, 3)) + } + + @Test + fun `line handles empty lines`() { + val text = "abc\n\ndef" + // index 4 is the empty line between the two newlines. + assertThat(following(text, line, 4)).isEqualTo(listOf(4, 4)) + } + + // --------------------------------------------------------------------------------------------- + // Guards + // --------------------------------------------------------------------------------------------- + + @Test + fun `empty text yields no segments`() { + assertThat(following("", char, 0)).isNull() + assertThat(preceding("", word, 0)).isNull() + assertThat(following("", line, 0)).isNull() + } + + @Test + fun `unsupported granularity yields null`() { + assertThat(following("abc", 999, 0)).isNull() + assertThat(preceding("abc", 999, 3)).isNull() + } + + @Test + fun `supported granularities bitmask includes character word and line`() { + val mask = EditorAccessibilitySegments.SUPPORTED_GRANULARITIES + assertThat(mask and char).isEqualTo(char) + assertThat(mask and word).isEqualTo(word) + assertThat(mask and line).isEqualTo(line) + } +} From e88685aefeb3797f6e5ad697f27740adceb7731f Mon Sep 17 00:00:00 2001 From: makhlwf Date: Sat, 11 Jul 2026 23:46:32 +0200 Subject: [PATCH 2/4] fix(editor): address review feedback on a11y cursor sync - getAccessibilityClassName() now mirrors the !isReleased && isEnabled && isEditable guard used by createAccessibilityNodeInfo(), so a read-only or released editor is no longer announced as an "Edit box". - Word granularity navigation snaps a mid-word offset to the enclosing word's start/end, so navigation traverses the whole word (e.g. "hello"@2 -> [0,5]) instead of a truncated range. Adds regression tests for mid-word offsets. - Narrow the accessibility catch blocks to IndexOutOfBoundsException / IllegalArgumentException instead of a broad Exception, per project convention. --- .../itsaky/androidide/editor/ui/IDEEditor.kt | 19 ++++++++++++++++--- .../utils/EditorAccessibilitySegments.kt | 10 ++++++++++ .../utils/EditorAccessibilitySegmentsTest.kt | 11 +++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt b/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt index 4d39006f5c..8435069c15 100644 --- a/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt +++ b/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt @@ -547,7 +547,14 @@ constructor( private val isTouchExplorationEnabled: Boolean get() = accessibilityManager?.isTouchExplorationEnabled == true - override fun getAccessibilityClassName(): CharSequence = EditText::class.java.name + override fun getAccessibilityClassName(): CharSequence = + // Only advertise EditText semantics in the same states createAccessibilityNodeInfo() + // does, so a read-only or released editor is not announced as an "Edit box". + if (!isReleased && isEnabled && isEditable) { + EditText::class.java.name + } else { + super.getAccessibilityClassName() + } override fun createAccessibilityNodeInfo(): AccessibilityNodeInfo? { val info = super.createAccessibilityNodeInfo() ?: return null @@ -609,7 +616,10 @@ constructor( if (line < 0 || column < 0) return false setSelection(line, column) true - } catch (e: Exception) { + } catch (e: IndexOutOfBoundsException) { + log.error("Error placing cursor from accessibility click", e) + false + } catch (e: IllegalArgumentException) { log.error("Error placing cursor from accessibility click", e) false } @@ -692,7 +702,10 @@ constructor( setSelectionRegion(startPos.line, startPos.column, endPos.line, endPos.column) } true - } catch (e: Exception) { + } catch (e: IndexOutOfBoundsException) { + log.error("Error applying accessibility selection [$start, $end]", e) + false + } catch (e: IllegalArgumentException) { log.error("Error applying accessibility selection [$start, $end]", e) false } diff --git a/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt index 4c4a352e16..e733e52877 100644 --- a/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt +++ b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt @@ -123,6 +123,11 @@ object EditorAccessibilitySegments { if (length <= 0 || offset >= length) return null val iterator = BreakIterator.getWordInstance().apply { setText(text.toString()) } var start = if (offset < 0) 0 else offset + // If the offset lands inside a word, snap back to that word's start so the whole + // enclosing word is traversed instead of a truncated tail (e.g. "hello"@2 -> [0,5]). + if (isLetterOrDigit(text, start)) { + while (start > 0 && isLetterOrDigit(text, start - 1)) start-- + } // Skip past any non-word characters (whitespace, punctuation) to the start of a word. while (!isLetterOrDigit(text, start) && !isWordStart(text, start)) { start = iterator.following(start) @@ -138,6 +143,11 @@ object EditorAccessibilitySegments { if (length <= 0 || offset <= 0) return null val iterator = BreakIterator.getWordInstance().apply { setText(text.toString()) } var end = if (offset > length) length else offset + // If the offset lands inside a word, snap forward to that word's end so the whole + // enclosing word is traversed instead of a truncated head (e.g. "hello"@2 -> [0,5]). + if (isLetterOrDigit(text, end - 1) && isLetterOrDigit(text, end)) { + while (end < length && isLetterOrDigit(text, end)) end++ + } while (!isLetterOrDigit(text, end - 1) && !isWordEnd(text, end)) { end = iterator.preceding(end) if (end == DONE) return null diff --git a/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt index e2e697ac65..4189db3a75 100644 --- a/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt +++ b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt @@ -82,6 +82,17 @@ class EditorAccessibilitySegmentsTest { assertThat(following("foo bar", word, 7)).isNull() } + @Test + fun `word following from inside a word snaps to the whole enclosing word`() { + // Offset 2 is inside "hello"; the whole word must be traversed, not a truncated tail. + assertThat(following("hello", word, 2)).isEqualTo(listOf(0, 5)) + } + + @Test + fun `word preceding from inside a word snaps to the whole enclosing word`() { + assertThat(preceding("hello", word, 2)).isEqualTo(listOf(0, 5)) + } + @Test fun `word preceding returns previous word`() { val text = "foo bar" From c3d531346266b1b5d2cc26c7c1dd90339fe2a3be Mon Sep 17 00:00:00 2001 From: makhlwf Date: Tue, 14 Jul 2026 00:01:08 +0200 Subject: [PATCH 3/4] fix(editor): add paragraph a11y granularity and text change/selection events Two TalkBack gaps surfaced in on-device testing: 1. Line and paragraph navigation didn't work. PARAGRAPH granularity was never advertised, and the editor never emitted TYPE_VIEW_TEXT_SELECTION_CHANGED, so the screen reader could not track the caret across the larger granularities. Advertise + implement PARAGRAPH (non-empty logical lines, blank lines skipped) and fire a selection-changed event on every selection change. 2. Inserted/deleted characters were not announced. Emit TYPE_VIEW_TEXT_CHANGED on content changes with fromIndex, added/removed counts and beforeText reconstructed from the change, so TalkBack speaks typed and deleted text like a standard EditText. Both event senders are no-ops when no accessibility service is enabled. Adds unit tests for paragraph navigation. --- .../itsaky/androidide/editor/ui/IDEEditor.kt | 74 +++++++++++++++++++ .../utils/EditorAccessibilitySegments.kt | 35 ++++++++- .../utils/EditorAccessibilitySegmentsTest.kt | 32 ++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt b/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt index 8435069c15..3968d012c1 100644 --- a/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt +++ b/editor/src/main/java/com/itsaky/androidide/editor/ui/IDEEditor.kt @@ -737,6 +737,74 @@ constructor( event.text.add(text.toString()) sendAccessibilityEventUnchecked(event) } + + /** + * Emits a [AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED] event so the screen reader + * tracks the caret/selection as it moves — required for reliable movement-by-granularity + * navigation (lines/paragraphs in particular) and to mirror standard [EditText] behaviour. + * Called on every selection change; cheap no-op when no accessibility service is running. + */ + @Suppress("DEPRECATION") + fun sendSelectionChangedAccessibilityEvent() { + val manager = accessibilityManager ?: return + if (!manager.isEnabled) return + val cur = cursor ?: return + val event = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED) + onInitializeAccessibilityEvent(event) + event.className = EditText::class.java.name + val content = text.toString() + event.fromIndex = cur.left + event.toIndex = cur.right + event.itemCount = content.length + event.text.add(content) + sendAccessibilityEventUnchecked(event) + } + + /** + * Emits a [AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED] event so the screen reader announces + * inserted and deleted characters, like a standard [EditText]. TalkBack reconstructs the + * change from [AccessibilityEvent.getFromIndex], added/removed counts, the current text and + * [AccessibilityEvent.getBeforeText]. Cheap no-op when no accessibility service is running. + */ + @Suppress("DEPRECATION") + fun sendTextChangedAccessibilityEvent(changeEvent: ContentChangeEvent) { + val manager = accessibilityManager ?: return + if (!manager.isEnabled) return + val current = text.toString() + val length = current.length + val fromIndex = changeEvent.changeStart.index.coerceIn(0, length) + val changed = changeEvent.changedText + val changedLen = changed.length + + val event = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED) + onInitializeAccessibilityEvent(event) + event.className = EditText::class.java.name + event.fromIndex = fromIndex + when (changeEvent.action) { + ContentChangeEvent.ACTION_INSERT -> { + event.addedCount = changedLen + event.removedCount = 0 + // beforeText = current text with the just-inserted range removed. + val insertEnd = (fromIndex + changedLen).coerceIn(0, length) + event.beforeText = current.substring(0, fromIndex) + current.substring(insertEnd) + } + + ContentChangeEvent.ACTION_DELETE -> { + event.addedCount = 0 + event.removedCount = changedLen + // beforeText = current text with the just-deleted text put back. + event.beforeText = current.substring(0, fromIndex) + changed + current.substring(fromIndex) + } + + else -> { // ACTION_SET_NEW_TEXT and any other wholesale replacement + event.addedCount = length + event.removedCount = 0 + event.beforeText = "" + } + } + event.text.add(current) + sendAccessibilityEventUnchecked(event) + } // endregion Accessibility override fun copyTextToClipboard( @@ -1017,6 +1085,9 @@ constructor( return@subscribeEvent } + // Announce inserted/deleted characters to screen readers, like a standard EditText. + sendTextChangedAccessibilityEvent(event) + markModified() file ?: return@subscribeEvent @@ -1032,6 +1103,9 @@ constructor( return@subscribeEvent } + // Keep the screen reader's caret in sync as the selection moves, like an EditText. + sendSelectionChangedAccessibilityEvent() + if (_diagnosticWindow?.isShowing == true) { _diagnosticWindow?.dismiss() } diff --git a/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt index e733e52877..ea8620d5d0 100644 --- a/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt +++ b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt @@ -48,7 +48,8 @@ object EditorAccessibilitySegments { const val SUPPORTED_GRANULARITIES = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER or AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD or - AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE or + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH /** * Returns the `[start, end]` range (end-exclusive) of the segment that follows @@ -63,6 +64,7 @@ object EditorAccessibilitySegments { AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER -> characterFollowing(text, offset) AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD -> wordFollowing(text, offset) AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE -> lineFollowing(text, offset) + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH -> paragraphFollowing(text, offset) else -> null } @@ -79,6 +81,7 @@ object EditorAccessibilitySegments { AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER -> characterPreceding(text, offset) AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD -> wordPreceding(text, offset) AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE -> linePreceding(text, offset) + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH -> paragraphPreceding(text, offset) else -> null } @@ -199,4 +202,34 @@ object EditorAccessibilitySegments { while (lineEnd < length && text[lineEnd] != '\n') lineEnd++ return intArrayOf(start, lineEnd) } + + // --------------------------------------------------------------------------------------------- + // Paragraph (non-empty '\n'-separated lines; blank lines are skipped, as EditText does) + // --------------------------------------------------------------------------------------------- + + private fun paragraphFollowing(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset >= length) return null + var start = if (offset < 0) 0 else offset + // Skip blank lines so navigation lands on the next non-empty line. + while (start < length && text[start] == '\n') start++ + if (start >= length) return null + // Snap to the start of the enclosing line. + while (start > 0 && text[start - 1] != '\n') start-- + var end = start + while (end < length && text[end] != '\n') end++ + return intArrayOf(start, end) + } + + private fun paragraphPreceding(text: CharSequence, offset: Int): IntArray? { + val length = text.length + if (length <= 0 || offset <= 0) return null + var end = if (offset > length) length else offset + // Skip blank lines (trailing newlines) so we land on the previous non-empty line's end. + while (end > 0 && text[end - 1] == '\n') end-- + if (end <= 0) return null + var start = end + while (start > 0 && text[start - 1] != '\n') start-- + return intArrayOf(start, end) + } } diff --git a/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt index 4189db3a75..db82300ba8 100644 --- a/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt +++ b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt @@ -33,6 +33,7 @@ class EditorAccessibilitySegmentsTest { private val char = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER private val word = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD private val line = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE + private val paragraph = AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH private fun following(text: String, granularity: Int, offset: Int) = EditorAccessibilitySegments.following(text, granularity, offset)?.toList() @@ -159,6 +160,36 @@ class EditorAccessibilitySegmentsTest { assertThat(following(text, line, 4)).isEqualTo(listOf(4, 4)) } + // --------------------------------------------------------------------------------------------- + // Paragraph + // --------------------------------------------------------------------------------------------- + + @Test + fun `paragraph following returns the current non-empty line`() { + val text = "abc\ndef\nghi" + assertThat(following(text, paragraph, 0)).isEqualTo(listOf(0, 3)) // abc + } + + @Test + fun `paragraph following skips blank lines`() { + val text = "abc\n\ndef" + // From end of "abc" (index 3), the next paragraph is "def" — the blank line is skipped. + assertThat(following(text, paragraph, 3)).isEqualTo(listOf(5, 8)) + } + + @Test + fun `paragraph preceding skips blank lines`() { + val text = "abc\n\ndef" + // From the start of "def" (index 5), the previous paragraph is "abc". + assertThat(preceding(text, paragraph, 5)).isEqualTo(listOf(0, 3)) + } + + @Test + fun `paragraph preceding returns the line the caret sits at the end of`() { + val text = "abc\ndef" + assertThat(preceding(text, paragraph, 7)).isEqualTo(listOf(4, 7)) // def + } + // --------------------------------------------------------------------------------------------- // Guards // --------------------------------------------------------------------------------------------- @@ -182,5 +213,6 @@ class EditorAccessibilitySegmentsTest { assertThat(mask and char).isEqualTo(char) assertThat(mask and word).isEqualTo(word) assertThat(mask and line).isEqualTo(line) + assertThat(mask and paragraph).isEqualTo(paragraph) } } From c40c40f254e74629d28233763d3d4d2aa3f5f9dc Mon Sep 17 00:00:00 2001 From: makhlwf Date: Tue, 14 Jul 2026 01:09:37 +0200 Subject: [PATCH 4/4] fix(editor): correct mid-paragraph a11y navigation paragraphPreceding returned the raw offset as its end bound, so a mid-paragraph offset yielded a truncated range (e.g. "abc\ndef\nghi"@5 gave [4,5] instead of [4,7]). paragraphFollowing also snapped to the current line instead of advancing. Rework both to mirror lineFollowing/linePreceding (advance to the next line forward; return the whole enclosing line backward) while still skipping blank lines, and add mid-paragraph regression tests. --- .../editor/utils/EditorAccessibilitySegments.kt | 16 ++++++++++++---- .../utils/EditorAccessibilitySegmentsTest.kt | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt index ea8620d5d0..cd4a327305 100644 --- a/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt +++ b/editor/src/main/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegments.kt @@ -211,11 +211,15 @@ object EditorAccessibilitySegments { val length = text.length if (length <= 0 || offset >= length) return null var start = if (offset < 0) 0 else offset + // Mirror lineFollowing: when not already at a line start, advance to the next line. + if (start > 0 && text[start - 1] != '\n') { + while (start < length && text[start] != '\n') start++ + if (start < length) start++ // step over the newline onto the next line + if (start >= length) return null + } // Skip blank lines so navigation lands on the next non-empty line. while (start < length && text[start] == '\n') start++ if (start >= length) return null - // Snap to the start of the enclosing line. - while (start > 0 && text[start - 1] != '\n') start-- var end = start while (end < length && text[end] != '\n') end++ return intArrayOf(start, end) @@ -225,11 +229,15 @@ object EditorAccessibilitySegments { val length = text.length if (length <= 0 || offset <= 0) return null var end = if (offset > length) length else offset - // Skip blank lines (trailing newlines) so we land on the previous non-empty line's end. + // Skip blank lines (trailing newlines) so we land on the previous non-empty line. while (end > 0 && text[end - 1] == '\n') end-- if (end <= 0) return null var start = end while (start > 0 && text[start - 1] != '\n') start-- - return intArrayOf(start, end) + // Mirror linePreceding: scan forward to the line end so a mid-paragraph offset returns + // the whole line instead of a truncated head. + var lineEnd = start + while (lineEnd < length && text[lineEnd] != '\n') lineEnd++ + return intArrayOf(start, lineEnd) } } diff --git a/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt index db82300ba8..c3163e7f22 100644 --- a/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt +++ b/editor/src/test/java/com/itsaky/androidide/editor/utils/EditorAccessibilitySegmentsTest.kt @@ -190,6 +190,20 @@ class EditorAccessibilitySegmentsTest { assertThat(preceding(text, paragraph, 7)).isEqualTo(listOf(4, 7)) // def } + @Test + fun `paragraph following from a mid-paragraph offset advances to the next paragraph`() { + val text = "abc\ndef\nghi" + // offset 5 is inside "def"; next paragraph is "ghi". + assertThat(following(text, paragraph, 5)).isEqualTo(listOf(8, 11)) + } + + @Test + fun `paragraph preceding from a mid-paragraph offset returns the whole enclosing paragraph`() { + val text = "abc\ndef\nghi" + // offset 5 is inside "def"; the whole line must be returned, not a truncated head. + assertThat(preceding(text, paragraph, 5)).isEqualTo(listOf(4, 7)) + } + // --------------------------------------------------------------------------------------------- // Guards // ---------------------------------------------------------------------------------------------