From 60e4524d0bd1b5901b271948acedced62224f45f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 15:59:08 +0200 Subject: [PATCH 1/3] fix(runner): refuse an iOS coordinate fill whose point names no text input On the penalized coordinate route the replacement has no element, so its clear and its typed text reach the app-level first responder. Aimed at empty space while an earlier field still holds focus, that rewrote the earlier field. The runner now answers NO_TEXT_INPUT_AT_POINT before the focus tap, and the host publishes details.reason no_text_input_at_point with dispatched: no. Fixes #1721 --- .../RunnerTests+TypeExecution.swift | 22 +++++++++++ .../RunnerTests.swift | 2 + ...RunnerTests+CoordinateTextEntryTests.swift | 27 ++++++++++++++ ...unnerTests+SynthesizedTextEntryTests.swift | 13 ++++--- contracts/fixtures/dispatch-disclosure.json | 6 +++ .../index-dispatch-disclosure.test.ts | 1 + .../runner-error-classification.test.ts | 21 +++++++++++ .../src/runner/runner-contract.ts | 37 +++++++++++++++---- website/docs/docs/commands.md | 1 + 9 files changed, 118 insertions(+), 12 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index c49336a82e..d827ed0f5b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -29,6 +29,12 @@ extension RunnerTests { ), let x = command.x, let y = command.y { // Resolved before the tap: the focus it causes can move the field away from this point. let inputAtPoint = coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) + // No input at the point means no target to bind the replacement to. Dispatching it anyway + // clears and types through the app-level first responder, so a fill aimed at empty space + // rewrites whichever field an earlier focus left active. + guard let inputAtPoint else { + return noTextInputAtPointResponse() + } let policyKind = SynthesizedGesturePolicyKind.coordinateTap let context = synthesizedCoordinateContext( app: activeApp, @@ -149,4 +155,20 @@ extension RunnerTests { } return Response(ok: true, data: data) } + + /// The refusal of a coordinate `fill` whose point names no text input, answered before the focus + /// tap is dispatched so nothing runs. The synthesized first-responder replacement has no element + /// and re-resolves by the point on every read, so without the input the point named, its clear and + /// its typed text reach the app-level first responder: while an earlier field still holds focus, + /// that earlier field. + private func noTextInputAtPointResponse() -> Response { + Response( + ok: false, + error: ErrorPayload( + code: RunnerWireErrorCode.noTextInputAtPoint, + message: "no text input at the provided coordinates", + hint: "A coordinate fill replaces the text of the input under the point, so the point must sit inside one. Run snapshot -i, then fill the field by ref or selector, or aim at a point inside its rect." + ) + ) + } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 3d8b886140..b1dc17d935 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -33,9 +33,11 @@ final class RunnerTests: XCTestCase { /// gate; `MAIN_THREAD_TIMEOUT` is emitted by the transport when a command trips the execution /// watchdog, so the daemon can tell "the main thread is now occupied" from a generic failure. /// `APP_NOT_RUNNING` refuses a read whose session app is not running. + /// `NO_TEXT_INPUT_AT_POINT` refuses a coordinate fill whose point names no text input. enum RunnerWireErrorCode { static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT" static let appNotRunning = "APP_NOT_RUNNING" + static let noTextInputAtPoint = "NO_TEXT_INPUT_AT_POINT" } static let springboardBundleId = "com.apple.springboard" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 61029f357f..9c19426aa2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -52,5 +52,32 @@ extension RunnerTests { XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertEqual(String(describing: textField.value ?? ""), "") } + + /// A penalized coordinate `fill` whose point names no text input cannot bind its replacement to a + /// target, so it refuses with the typed reason instead of clearing and typing through the app-level + /// first responder — which, while an earlier field still holds focus, destroys that field's text. + @MainActor + func testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesAndKeepsTheFocusedField() throws { + let field = try focusSynthesizedReplacementField() + defer { tearDownSynthesizedReplacementField() } + let seeded = try replaceSynthesizedFieldText(field, text: "stale", commandId: "seed-non-text-point") + XCTAssertTrue(seeded.ok, String(describing: seeded.error)) + + let nonTextTarget = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout)) + let nonTextFrame = nonTextTarget.frame + XCTAssertFalse(nonTextFrame.isEmpty) + + let fillCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"fill-penalized-non-text-point","text":"must-not-replace","textEntryMode":"replace","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY)}"# + ) + let failureCountBefore = currentXCTestFailureCount() + let fillResponse = executeTypeCommand(activeApp: app, command: fillCommand) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(fillResponse.ok, String(describing: fillResponse.error)) + XCTAssertEqual(fillResponse.error?.code, "NO_TEXT_INPUT_AT_POINT") + XCTAssertEqual(String(describing: field.value ?? ""), "stale") + } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index bd78542216..0c3cd641fb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -420,10 +420,12 @@ extension RunnerTests { } /// The pre-tap lookup runs on a channel already penalized for failing XCTest reads. A lookup that - /// fails leaves the fill to the point; it used to record the failure, which fails the command - /// and ends the runner. + /// fails used to record the failure, which fails the command and ends the runner, and left the + /// fill to the point. Containment stays: no recorded failure. What changed is what the fill then + /// does with a point it cannot name an input for: it refuses rather than typing through the + /// first responder (#1721). @MainActor - func testSynthesizedReplacementContainsAFailedLookupOfTheInputUnderThePoint() throws { + func testSynthesizedReplacementFailsClosedOnAFailedLookupOfTheInputUnderThePoint() throws { let textField = try focusSynthesizedReplacementField() defer { textInputProbeIssueForTesting = nil @@ -437,8 +439,9 @@ extension RunnerTests { let response = try replaceSynthesizedFieldText(textField, text: "fresh", commandId: "fill-lookup-failed") XCTAssertNil(textInputProbeIssueForTesting, "the fill made no lookup") - XCTAssertTrue(response.ok, String(describing: response.error)) - XCTAssertEqual(String(describing: textField.value ?? ""), "fresh") + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "NO_TEXT_INPUT_AT_POINT") + XCTAssertEqual(String(describing: textField.value ?? ""), "") } #endif } diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index dae5d97f60..53f2729ea8 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -193,6 +193,12 @@ "trigger": "structured runner reply with code ALERT_NOT_FOUND: a refusal that ran nothing", "dispatched": "no" }, + { + "id": "ios-runner.reply.NO_TEXT_INPUT_AT_POINT", + "producer": "ios-runner", + "trigger": "structured runner reply with code NO_TEXT_INPUT_AT_POINT: a coordinate fill refused before its focus tap because the point named no text input", + "dispatched": "no" + }, { "id": "ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED", "producer": "ios-runner", diff --git a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts index bef310980b..6cdce8cf61 100644 --- a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts @@ -168,6 +168,7 @@ const DRIVERS: Record Promise> = { 'ios-runner.reply.SCROLL_KEYBOARD_OCCLUDES_SURFACE': () => replyFailure('SCROLL_KEYBOARD_OCCLUDES_SURFACE'), 'ios-runner.reply.ALERT_NOT_FOUND': () => replyFailure('ALERT_NOT_FOUND'), + 'ios-runner.reply.NO_TEXT_INPUT_AT_POINT': () => replyFailure('NO_TEXT_INPUT_AT_POINT'), 'ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED': () => replyFailure('APP_SCREEN_WINDOW_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), diff --git a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts index f68e403129..57205d8155 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { + NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES, classifyRunnerReportedError, } from '../runner-contract.ts'; @@ -96,6 +97,26 @@ test('the main-thread runner codes publish details.reason beside details.runnerE assert.equal(classifyRunnerReportedError('APP_NOT_RUNNING').details.reason, undefined); }); +// The literal is what the Swift runner encodes, so it is the contract and not the constant name: +// a rename on one side has to fail here rather than silently split the pair. +test('a coordinate fill refused for a point with no input keeps its code and publishes its reason', () => { + assert.equal(NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'NO_TEXT_INPUT_AT_POINT'); + const refusal = classifyRunnerReportedError(NO_TEXT_INPUT_AT_POINT_RUNNER_CODE); + assert.equal(refusal.code, 'NO_TEXT_INPUT_AT_POINT'); + assert.equal(refusal.details.reason, 'no_text_input_at_point'); + assert.equal(refusal.details.dispatched, 'no'); + assert.equal(refusal.details.retriable, undefined); + // The refusal ran nothing, so it is a definite answer and never a resend. + const refusalError = new AppError(refusal.code, refusal.details.reason ?? '', refusal.details); + assert.equal(isRetryableRunnerError(refusalError), false); + assert.equal(isRunnerBusyError(refusalError), false); + // The closest negative: the same words from an unclassified failure earn no reason, so a + // consumer keyed on `reason` cannot mistake a reply that may have landed for this refusal. + const untyped = commandFailed('no text input at the provided coordinates'); + assert.equal(untyped.details?.reason, undefined); + assert.equal(isRetryableRunnerError(untyped), false); +}); + test('retryable requires an AppError with COMMAND_FAILED', () => { assert.equal(isRetryableRunnerError(new Error('fetch failed')), false); assert.equal(isRetryableRunnerError(new AppError('DEVICE_NOT_FOUND', 'fetch failed')), false); diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 3dae04e8f0..ad8713473a 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -256,6 +256,14 @@ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = n RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE, ]); +/** + * The runner's own code refusing a coordinate `fill` whose point names no text input (#1721). The + * synthesized first-responder replacement this route would otherwise take has no element, so its + * clear and its typed text reach whichever field already holds focus. The runner refuses before + * dispatching the focus tap, so the reply proves nothing ran. + */ +export const NO_TEXT_INPUT_AT_POINT_RUNNER_CODE = 'NO_TEXT_INPUT_AT_POINT'; + /** * Runner codes that classify a failure for the host without renaming it on the wire. They stay * `COMMAND_FAILED` and survive as `details.runnerErrorCode`, which is what family policy reads: @@ -263,18 +271,26 @@ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = n * the scroll keyboard refusal for a surface the runner declined to swipe under the keys, and the * retriable `APP_NOT_RUNNING` for a read the runner refused rather than launch the session app. */ -const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlyMap< - string, - { retriable?: true; reason?: RunnerReportedErrorReason } -> = new Map([ - [RUNNER_BUSY_RUNNER_CODE, { retriable: true, reason: 'runner_busy' }], - [MAIN_THREAD_TIMEOUT_RUNNER_CODE, { reason: 'runner_main_thread_timeout' }], +const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlyMap = new Map([ + [RUNNER_BUSY_RUNNER_CODE, { retriable: true }], + [MAIN_THREAD_TIMEOUT_RUNNER_CODE, {}], [APP_NOT_RUNNING_RUNNER_CODE, { retriable: true }], [ALERT_NOT_FOUND_RUNNER_CODE, {}], [SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, {}], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, {}] as const), ]); +/** + * `details.reason` for the runner codes a consumer acts on, so it reads one field instead of + * `details.runnerErrorCode`, which stays for the runner's own vocabulary. Both the codes that stay + * off the wire and the codes that travel under their own name are answered from this one table. + */ +const RUNNER_REPORTED_ERROR_REASONS: ReadonlyMap = new Map([ + [RUNNER_BUSY_RUNNER_CODE, 'runner_busy'], + [MAIN_THREAD_TIMEOUT_RUNNER_CODE, 'runner_main_thread_timeout'], + [NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'no_text_input_at_point'], +]); + /** * Runner codes whose reply proves the command never reached the device. The refusals answer before * the command runs, the selector refusals included: the runner resolves the element and refuses @@ -293,6 +309,7 @@ const RUNNER_ERROR_CODE_DISPATCH: ReadonlyMap = new ['OBSERVATION_UNAVAILABLE', 'no'], [SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, 'no'], [ALERT_NOT_FOUND_RUNNER_CODE, 'no'], + [NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'no'], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, 'no'] as const), ]); @@ -300,7 +317,10 @@ const RUNNER_ERROR_CODE_DISPATCH: ReadonlyMap = new * `details.reason` for the runner codes a consumer acts on, so it reads one field instead of * `details.runnerErrorCode`, which stays for the runner's own vocabulary. */ -type RunnerReportedErrorReason = 'runner_busy' | 'runner_main_thread_timeout'; +type RunnerReportedErrorReason = + | 'runner_busy' + | 'runner_main_thread_timeout' + | 'no_text_input_at_point'; /** Wire code plus the details every path must publish for one runner-reported error code. */ export type RunnerReportedErrorClass = Readonly<{ @@ -326,11 +346,14 @@ export function classifyRunnerReportedError( runnerErrorCode === undefined ? undefined : DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES.get(runnerErrorCode); + const reason = + runnerErrorCode === undefined ? undefined : RUNNER_REPORTED_ERROR_REASONS.get(runnerErrorCode); return Object.freeze({ code: diagnosticOnly ? 'COMMAND_FAILED' : toAppErrorCode(runnerErrorCode), details: Object.freeze({ runnerErrorCode, ...diagnosticOnly, + ...(reason === undefined ? {} : { reason }), dispatched: (runnerErrorCode === undefined ? undefined diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f2099449d5..42dce1d0d5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -438,6 +438,7 @@ A `click` or `press` whose target the daemon resolves to a point (`@ref` or sele Text-entry errors and what to do: - `TEXT_INPUT_NOT_FOCUSED` (`type`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. +- `NO_TEXT_INPUT_AT_POINT` (iOS `fill `): no text input sits under the point, so the runner refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`) and no field changed. Run `snapshot -i` and `fill` the field by ref or selector, or aim at a point inside its rect. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry. - `TEXT_INPUT_COMMIT_NOT_OBSERVED` (iOS `fill`): the runner could not confirm the text landed before its deadline, or the text equals the field's placeholder. The field may hold none, part, or all of it. Run `snapshot -i`; if it doesn't match, retry with the full text quoted and `fill --delay-ms 80`. - `TEXT_INPUT_CLEAR_NOT_OBSERVED` (iOS `fill`): the old text was not seen cleared, so nothing was typed. Inspect with `snapshot -i`, then retry `fill`. From 3a164d1f45ccbfd7e7004bc61779a513394a1e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 16:54:33 +0200 Subject: [PATCH 2/3] test(runner): prove the coordinate fill refusal lands before its focus tap The refusal's dispatched: no claim needs device-side evidence: the fill aimed at a button that counts presses, so a refusal that tapped first moves the counter even when the field comes through unchanged. Also say that the runner identified no input at the point, since a lookup that cannot answer reports the same code as an empty point. Fixes #1721 --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 41 +++++++++++++++++++ .../RunnerTests+TypeExecution.swift | 11 +++-- ...RunnerTests+CoordinateTextEntryTests.swift | 18 ++++++-- contracts/fixtures/dispatch-disclosure.json | 2 +- .../runner-error-classification.test.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 64 insertions(+), 12 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 6a9612eecc..109c610949 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -91,6 +91,10 @@ @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, assign) NSUInteger replacementAlertActions; @property(nonatomic, strong) UILabel *textEntryWriteBackStatus; @property(nonatomic, strong) UILabel *textEntryDigitSlots; +// Counts presses a gesture landed on a control that is not a text input, so a test can tell a +// refusal that ran nothing from one that dispatched its gesture first and refused afterwards. +@property(nonatomic, strong) UILabel *tapWitnessStatus; +@property(nonatomic, assign) NSUInteger tapWitnessPresses; // Filled from the fixture field's own delegate callbacks so a test can tell that a gesture really // took focus from an input that has stopped answering the accessibility query that found it. @property(nonatomic, strong) UILabel *textEntryFocusWitness; @@ -529,6 +533,17 @@ - (void)agentDeviceMaskedToggleDidPress:(UIButton *)sender { [self updateMaskedToggleStatus]; } +- (void)agentDeviceTapWitnessDidPress:(UIButton *)sender { + (void)sender; + self.tapWitnessPresses += 1; + [self updateTapWitnessStatus]; +} + +- (void)updateTapWitnessStatus { + self.tapWitnessStatus.text = + [NSString stringWithFormat:@"presses=%lu", (unsigned long)self.tapWitnessPresses]; +} + - (void)updateMaskedToggleStatus { self.maskedToggleStatus.text = [NSString stringWithFormat:@"presses=%lu", (unsigned long)self.maskedTogglePresses]; @@ -660,6 +675,32 @@ - (void)viewDidLoad { ((AgentDeviceSubmitClearingTextField *)textField).entryStatus = entryStatus; } } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-tap-witness"]) { + // A control that is not a text input, sitting below the field. Its press count is the only + // witness a test has that a coordinate gesture reached this screen: a refusal that dispatched + // its focus tap here would move the counter, and one that refused first would not. + UIButton *witness = [UIButton buttonWithType:UIButtonTypeSystem]; + witness.accessibilityIdentifier = @"agent-device-tap-witness"; + [witness setTitle:@"Tap witness" forState:UIControlStateNormal]; + witness.translatesAutoresizingMaskIntoConstraints = NO; + [witness addTarget:self + action:@selector(agentDeviceTapWitnessDidPress:) + forControlEvents:UIControlEventTouchUpInside]; + [self.view addSubview:witness]; + self.tapWitnessStatus = [[UILabel alloc] init]; + self.tapWitnessStatus.accessibilityIdentifier = @"agent-device-tap-witness-presses"; + self.tapWitnessStatus.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:self.tapWitnessStatus]; + [NSLayoutConstraint activateConstraints:@[ + [witness.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [witness.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:24], + [witness.widthAnchor constraintEqualToConstant:200], + [witness.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], + [self.tapWitnessStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [self.tapWitnessStatus.topAnchor constraintEqualToAnchor:witness.bottomAnchor constant:12], + ]]; + [self updateTapWitnessStatus]; + } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unqueryable-on-focus"]) { self.textEntryFocusWitness = [[UILabel alloc] init]; self.textEntryFocusWitness.accessibilityIdentifier = @"agent-device-text-entry-focus"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index d827ed0f5b..52ed2f7801 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -157,17 +157,16 @@ extension RunnerTests { } /// The refusal of a coordinate `fill` whose point names no text input, answered before the focus - /// tap is dispatched so nothing runs. The synthesized first-responder replacement has no element - /// and re-resolves by the point on every read, so without the input the point named, its clear and - /// its typed text reach the app-level first responder: while an earlier field still holds focus, - /// that earlier field. + /// tap is dispatched so nothing runs. Without an input at the point the synthesized replacement has + /// no element to bind to, so its clear and its typed text would reach the app-level first + /// responder and overwrite whatever field currently holds focus. private func noTextInputAtPointResponse() -> Response { Response( ok: false, error: ErrorPayload( code: RunnerWireErrorCode.noTextInputAtPoint, - message: "no text input at the provided coordinates", - hint: "A coordinate fill replaces the text of the input under the point, so the point must sit inside one. Run snapshot -i, then fill the field by ref or selector, or aim at a point inside its rect." + message: "no text input identified at the provided coordinates", + hint: "A coordinate fill replaces the text of the input under the point, so the point must sit inside one. Run snapshot -i, then fill the field by ref or selector, or aim at a point inside its rect. The runner also reports this when its input lookup cannot answer, so re-run snapshot before concluding the point is empty." ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 9c19426aa2..974b951ac3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -56,17 +56,24 @@ extension RunnerTests { /// A penalized coordinate `fill` whose point names no text input cannot bind its replacement to a /// target, so it refuses with the typed reason instead of clearing and typing through the app-level /// first responder — which, while an earlier field still holds focus, destroys that field's text. + /// The aim point is a button that counts its presses: a refusal that dispatched its focus tap first + /// would move that counter even though the field came through, so the counter is what proves the + /// refusal happened before the tap rather than after it. @MainActor - func testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesAndKeepsTheFocusedField() throws { - let field = try focusSynthesizedReplacementField() + func testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesBeforeItsTapAndKeepsTheFocusedField() throws { + let field = try focusSynthesizedReplacementField(extraLaunchArguments: [ + "--agent-device-text-entry-tap-witness", + ]) defer { tearDownSynthesizedReplacementField() } let seeded = try replaceSynthesizedFieldText(field, text: "stale", commandId: "seed-non-text-point") XCTAssertTrue(seeded.ok, String(describing: seeded.error)) - let nonTextTarget = app.staticTexts["Agent Device Runner"] + let nonTextTarget = app.buttons["agent-device-tap-witness"] XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout)) let nonTextFrame = nonTextTarget.frame XCTAssertFalse(nonTextFrame.isEmpty) + let pressCount = app.staticTexts["agent-device-tap-witness-presses"] + XCTAssertTrue(pressCount.waitForExistence(timeout: appExistenceTimeout)) let fillCommand = try runnerCommandFixture( #"{"command":"type","commandId":"fill-penalized-non-text-point","text":"must-not-replace","textEntryMode":"replace","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY)}"# @@ -78,6 +85,11 @@ extension RunnerTests { XCTAssertFalse(fillResponse.ok, String(describing: fillResponse.error)) XCTAssertEqual(fillResponse.error?.code, "NO_TEXT_INPUT_AT_POINT") XCTAssertEqual(String(describing: field.value ?? ""), "stale") + XCTAssertEqual( + pressCount.label, + "presses=0", + "the refusal dispatched its focus tap: the point's control took the press" + ) } #endif } diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 53f2729ea8..f0cff5830f 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -196,7 +196,7 @@ { "id": "ios-runner.reply.NO_TEXT_INPUT_AT_POINT", "producer": "ios-runner", - "trigger": "structured runner reply with code NO_TEXT_INPUT_AT_POINT: a coordinate fill refused before its focus tap because the point named no text input", + "trigger": "structured runner reply with code NO_TEXT_INPUT_AT_POINT: a coordinate fill refused before its focus tap because the runner identified no text input under the point", "dispatched": "no" }, { diff --git a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts index 57205d8155..45a8824ad9 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts @@ -112,7 +112,7 @@ test('a coordinate fill refused for a point with no input keeps its code and pub assert.equal(isRunnerBusyError(refusalError), false); // The closest negative: the same words from an unclassified failure earn no reason, so a // consumer keyed on `reason` cannot mistake a reply that may have landed for this refusal. - const untyped = commandFailed('no text input at the provided coordinates'); + const untyped = commandFailed('no text input identified at the provided coordinates'); assert.equal(untyped.details?.reason, undefined); assert.equal(isRetryableRunnerError(untyped), false); }); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 42dce1d0d5..c5358112eb 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -438,7 +438,7 @@ A `click` or `press` whose target the daemon resolves to a point (`@ref` or sele Text-entry errors and what to do: - `TEXT_INPUT_NOT_FOCUSED` (`type`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. -- `NO_TEXT_INPUT_AT_POINT` (iOS `fill `): no text input sits under the point, so the runner refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`) and no field changed. Run `snapshot -i` and `fill` the field by ref or selector, or aim at a point inside its rect. +- `NO_TEXT_INPUT_AT_POINT` (iOS `fill `): the runner identified no text input under the point, so it refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`) and no field changed. The same code covers a lookup that could not answer, so an empty point is not proven — run `snapshot -i` again, then `fill` the field by ref or selector, or aim at a point inside its rect. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry. - `TEXT_INPUT_COMMIT_NOT_OBSERVED` (iOS `fill`): the runner could not confirm the text landed before its deadline, or the text equals the field's placeholder. The field may hold none, part, or all of it. Run `snapshot -i`; if it doesn't match, retry with the full text quoted and `fill --delay-ms 80`. - `TEXT_INPUT_CLEAR_NOT_OBSERVED` (iOS `fill`): the old text was not seen cleared, so nothing was typed. Inspect with `snapshot -i`, then retry `fill`. From 28c6cf1a639b95a0d5556089d4e780a086fc82c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 17:22:51 +0200 Subject: [PATCH 3/3] fix(runner): tell an empty fill point from a blind one before the tap The pre-tap lookup returned nil for both, so one refusal carried two claims. coordinateTapTextInputIdentityAt now answers input, absent, or unavailable; absent refuses with NO_TEXT_INPUT_AT_POINT and unavailable refuses with TEXT_INPUT_PROBE_UNAVAILABLE, both before the focus tap. The hint and docs name every route that reaches the guard, including ref, selector and replay fills, whose aim point is the centre of the rect they resolved. Fixes #1721 --- CHANGELOG.md | 1 + .../RunnerTests+TextInputProbe.swift | 43 +++++++++++++++---- .../RunnerTests+TypeExecution.swift | 39 ++++++++++++----- .../RunnerTests.swift | 5 ++- ...RunnerTests+CoordinateTextEntryTests.swift | 20 +++++++++ ...unnerTests+SynthesizedTextEntryTests.swift | 8 ++-- .../RunnerTests+TextEntryPolicyTests.swift | 2 +- contracts/fixtures/dispatch-disclosure.json | 6 +++ .../index-dispatch-disclosure.test.ts | 2 + .../runner-error-classification.test.ts | 16 +++++++ .../src/runner/runner-contract.ts | 16 +++++-- website/docs/docs/commands.md | 3 +- 12 files changed, 132 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7f2a7068..41be908fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Add `settings mobile-data on|off` for adb-backed Android targets. It switches mobile data through `svc data` and reports the `mobileData` state stored in the global `mobile_data` row; multi-SIM read-back is not covered yet. Every other target refuses it before anything runs, and `--app` is refused because the switch is device-wide. +- On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect. - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`. - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state `). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 6f8547cf25..8b8ff0acc7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -38,19 +38,44 @@ extension RunnerTests { } } + /// What the point's text-input lookup can say before a coordinate gesture is dispatched: the + /// input it found, no input at the point, or nothing. A lookup that cannot answer is neither of the + /// first two, and must not be read as one of them. + enum CoordinateTextInputResolution { + case input(TextInputAtPoint) + case absent + case unavailable + } + /// The input `coordinateTapTextInputAt` finds, with the identity it carried then, read inside the - /// same issue containment. Nil when the probe finds none or cannot answer. - func coordinateTapTextInputIdentityAt(app: XCUIApplication, x: Double, y: Double) -> TextInputAtPoint? { - let probed = containingTextInputProbeIssues(fallback: nil) { shouldStop -> TextInputAtPoint? in - guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first, - case .input(let identity) = probeTextEntryInput(element) + /// same issue containment, split into what the lookup can name so a refusal can tell an empty + /// point from a blind one. + func coordinateTapTextInputIdentityAt( + app: XCUIApplication, + x: Double, + y: Double + ) -> CoordinateTextInputResolution { + let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { + shouldStop -> CoordinateTextInputResolution in + guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first else { - return nil + return .absent } - return TextInputAtPoint(element: element, identity: identity) + switch probeTextEntryInput(element) { + case .input(let identity): + return .input(TextInputAtPoint(element: element, identity: identity)) + case .noMatch: + return .absent + case .unavailable: + return .unavailable + } + } + switch probed { + case .success(let resolution): + return resolution + case .failure: + return .unavailable } - guard case .success(let input) = probed else { return nil } - return input } /// Whether `input`'s handle still resolves to the input it named when found, read inside the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 52ed2f7801..acbb62ef54 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -28,12 +28,17 @@ extension RunnerTests { xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { // Resolved before the tap: the focus it causes can move the field away from this point. - let inputAtPoint = coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) - // No input at the point means no target to bind the replacement to. Dispatching it anyway - // clears and types through the app-level first responder, so a fill aimed at empty space - // rewrites whichever field an earlier focus left active. - guard let inputAtPoint else { + // A replacement with no input at the point has nothing to bind to: its clear and its typed + // text would reach the app-level first responder, so an empty point and a lookup that cannot + // answer both refuse here, each under its own reason. + let inputAtPoint: TextInputAtPoint + switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) { + case .input(let input): + inputAtPoint = input + case .absent: return noTextInputAtPointResponse() + case .unavailable: + return textInputProbeUnavailableResponse() } let policyKind = SynthesizedGesturePolicyKind.coordinateTap let context = synthesizedCoordinateContext( @@ -156,17 +161,31 @@ extension RunnerTests { return Response(ok: true, data: data) } - /// The refusal of a coordinate `fill` whose point names no text input, answered before the focus - /// tap is dispatched so nothing runs. Without an input at the point the synthesized replacement has - /// no element to bind to, so its clear and its typed text would reach the app-level first - /// responder and overwrite whatever field currently holds focus. + /// The refusal of a coordinate `fill` whose point identified no text input, answered before the + /// focus tap is dispatched so nothing runs. Without an input at the point the synthesized + /// replacement has no element to bind to, so its clear and its typed text would reach the app-level + /// first responder and overwrite whatever field currently holds focus. private func noTextInputAtPointResponse() -> Response { Response( ok: false, error: ErrorPayload( code: RunnerWireErrorCode.noTextInputAtPoint, message: "no text input identified at the provided coordinates", - hint: "A coordinate fill replaces the text of the input under the point, so the point must sit inside one. Run snapshot -i, then fill the field by ref or selector, or aim at a point inside its rect. The runner also reports this when its input lookup cannot answer, so re-run snapshot before concluding the point is empty." + hint: "A fill replaces the text of the input under its point, so the point must sit inside one — for a ref or selector fill, that point is the centre of the rect it resolved to, which an outer container or a non-input node does not give you. Run snapshot -i, pick the input node itself, and aim at a point inside its rect." + ) + ) + } + + /// The refusal of a coordinate `fill` whose point the input lookup could not read, answered before + /// the focus tap so nothing runs. The channel is degraded, so the runner cannot tell whether an + /// input is there; typing would reach the app-level first responder, which is what #1721 refuses. + private func textInputProbeUnavailableResponse() -> Response { + Response( + ok: false, + error: ErrorPayload( + code: RunnerWireErrorCode.textInputProbeUnavailable, + message: "the runner could not read the text input at the provided coordinates", + hint: "The accessibility lookup for the point could not answer on the degraded channel, so the runner dispatched nothing and no field changed — this does not mean the point has no input. Retry the fill, or run snapshot -i and fill the input node by ref once its capture reads back." ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b1dc17d935..fcb0d942cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -33,11 +33,14 @@ final class RunnerTests: XCTestCase { /// gate; `MAIN_THREAD_TIMEOUT` is emitted by the transport when a command trips the execution /// watchdog, so the daemon can tell "the main thread is now occupied" from a generic failure. /// `APP_NOT_RUNNING` refuses a read whose session app is not running. - /// `NO_TEXT_INPUT_AT_POINT` refuses a coordinate fill whose point names no text input. + /// `NO_TEXT_INPUT_AT_POINT` refuses a coordinate fill whose point identified no text input. + /// `TEXT_INPUT_PROBE_UNAVAILABLE` refuses one whose input lookup could not answer on the degraded + /// channel, so the runner cannot tell whether an input is there. enum RunnerWireErrorCode { static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT" static let appNotRunning = "APP_NOT_RUNNING" static let noTextInputAtPoint = "NO_TEXT_INPUT_AT_POINT" + static let textInputProbeUnavailable = "TEXT_INPUT_PROBE_UNAVAILABLE" } static let springboardBundleId = "com.apple.springboard" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 974b951ac3..eb8c68072d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -2,6 +2,26 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// The input the coordinate probe finds at the point, failing the test when it finds none or + /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. + @MainActor + func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y) { + case .input(let input): + return input + case .absent: + return try XCTUnwrap( + nil as TextInputAtPoint?, + "the probe found no input at the fixture's own field" + ) + case .unavailable: + return try XCTUnwrap( + nil as TextInputAtPoint?, + "the probe could not read the fixture's own field" + ) + } + } + @MainActor func testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType() throws { app.launchArguments = ["--agent-device-text-entry-regression"] diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 0c3cd641fb..ac0ac0e54d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -222,7 +222,7 @@ extension RunnerTests { let field = try focusSynthesizedReplacementField() defer { tearDownSynthesizedReplacementField() } let frame = field.frame - let found = try XCTUnwrap(coordinateTapTextInputIdentityAt(app: app, x: frame.midX, y: frame.midY)) + let found = try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY) let cases: [(dropped: Int, failure: TextEntryFailure?)] = [ (2, nil), (SynthesizedTextPlan.Step.maxClearPassCount, .clearNotObserved), @@ -422,8 +422,8 @@ extension RunnerTests { /// The pre-tap lookup runs on a channel already penalized for failing XCTest reads. A lookup that /// fails used to record the failure, which fails the command and ends the runner, and left the /// fill to the point. Containment stays: no recorded failure. What changed is what the fill then - /// does with a point it cannot name an input for: it refuses rather than typing through the - /// first responder (#1721). + /// does with a point it cannot read: it refuses under the probe-unavailable reason, which is not a + /// claim that the point has no input (#1721). @MainActor func testSynthesizedReplacementFailsClosedOnAFailedLookupOfTheInputUnderThePoint() throws { let textField = try focusSynthesizedReplacementField() @@ -440,7 +440,7 @@ extension RunnerTests { XCTAssertNil(textInputProbeIssueForTesting, "the fill made no lookup") XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "NO_TEXT_INPUT_AT_POINT") + XCTAssertEqual(response.error?.code, "TEXT_INPUT_PROBE_UNAVAILABLE") XCTAssertEqual(String(describing: textField.value ?? ""), "") } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index e6b76fad08..5881482c12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -702,7 +702,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, - inputAtRefreshPoint: try XCTUnwrap(coordinateTapTextInputIdentityAt(app: app, x: frame.midX, y: frame.midY)) + inputAtRefreshPoint: try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY) ), text: "abc", delaySeconds: 0.001, diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index f0cff5830f..4fc748c1fd 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -199,6 +199,12 @@ "trigger": "structured runner reply with code NO_TEXT_INPUT_AT_POINT: a coordinate fill refused before its focus tap because the runner identified no text input under the point", "dispatched": "no" }, + { + "id": "ios-runner.reply.TEXT_INPUT_PROBE_UNAVAILABLE", + "producer": "ios-runner", + "trigger": "structured runner reply with code TEXT_INPUT_PROBE_UNAVAILABLE: a coordinate fill refused before its focus tap because the pre-tap input lookup could not answer", + "dispatched": "no" + }, { "id": "ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED", "producer": "ios-runner", diff --git a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts index 6cdce8cf61..7e66b4dfb9 100644 --- a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts @@ -169,6 +169,8 @@ const DRIVERS: Record Promise> = { replyFailure('SCROLL_KEYBOARD_OCCLUDES_SURFACE'), 'ios-runner.reply.ALERT_NOT_FOUND': () => replyFailure('ALERT_NOT_FOUND'), 'ios-runner.reply.NO_TEXT_INPUT_AT_POINT': () => replyFailure('NO_TEXT_INPUT_AT_POINT'), + 'ios-runner.reply.TEXT_INPUT_PROBE_UNAVAILABLE': () => + replyFailure('TEXT_INPUT_PROBE_UNAVAILABLE'), 'ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED': () => replyFailure('APP_SCREEN_WINDOW_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), diff --git a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts index 45a8824ad9..f6ceb35127 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts @@ -4,6 +4,7 @@ import { AppError, createRequestCanceledError } from '@agent-device/kernel/error import { NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES, + TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, classifyRunnerReportedError, } from '../runner-contract.ts'; import { @@ -117,6 +118,21 @@ test('a coordinate fill refused for a point with no input keeps its code and pub assert.equal(isRetryableRunnerError(untyped), false); }); +// The two coordinate-fill refusals answer different questions and must never collapse into one +// another: an empty point is a fact about the screen, a blind lookup is a fact about the channel. +test('a coordinate fill refused on an unreadable point reports its own code, not no-input', () => { + assert.equal(TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, 'TEXT_INPUT_PROBE_UNAVAILABLE'); + const refusal = classifyRunnerReportedError(TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE); + assert.equal(refusal.code, 'TEXT_INPUT_PROBE_UNAVAILABLE'); + assert.equal(refusal.details.reason, 'text_input_probe_unavailable'); + assert.equal(refusal.details.dispatched, 'no'); + assert.notEqual( + refusal.details.reason, + classifyRunnerReportedError(NO_TEXT_INPUT_AT_POINT_RUNNER_CODE).details.reason, + ); + assert.equal(isRetryableRunnerError(new AppError(refusal.code, '', refusal.details)), false); +}); + test('retryable requires an AppError with COMMAND_FAILED', () => { assert.equal(isRetryableRunnerError(new Error('fetch failed')), false); assert.equal(isRetryableRunnerError(new AppError('DEVICE_NOT_FOUND', 'fetch failed')), false); diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index ad8713473a..f29118d7ee 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -257,13 +257,20 @@ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = n ]); /** - * The runner's own code refusing a coordinate `fill` whose point names no text input (#1721). The - * synthesized first-responder replacement this route would otherwise take has no element, so its + * The runner's own code refusing a coordinate `fill` whose point identified no text input (#1721). + * The synthesized first-responder replacement this route would otherwise take has no element, so its * clear and its typed text reach whichever field already holds focus. The runner refuses before * dispatching the focus tap, so the reply proves nothing ran. */ export const NO_TEXT_INPUT_AT_POINT_RUNNER_CODE = 'NO_TEXT_INPUT_AT_POINT'; +/** + * The runner code refusing a coordinate `fill` whose pre-tap input lookup could not answer on the + * degraded channel. Like NO_TEXT_INPUT_AT_POINT it answers before the focus tap, and it is not a + * claim that the point has no input — only that the runner could not tell. + */ +export const TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE = 'TEXT_INPUT_PROBE_UNAVAILABLE'; + /** * Runner codes that classify a failure for the host without renaming it on the wire. They stay * `COMMAND_FAILED` and survive as `details.runnerErrorCode`, which is what family policy reads: @@ -289,6 +296,7 @@ const RUNNER_REPORTED_ERROR_REASONS: ReadonlyMap = new [SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, 'no'], [ALERT_NOT_FOUND_RUNNER_CODE, 'no'], [NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'no'], + [TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, 'no'], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, 'no'] as const), ]); @@ -320,7 +329,8 @@ const RUNNER_ERROR_CODE_DISPATCH: ReadonlyMap = new type RunnerReportedErrorReason = | 'runner_busy' | 'runner_main_thread_timeout' - | 'no_text_input_at_point'; + | 'no_text_input_at_point' + | 'text_input_probe_unavailable'; /** Wire code plus the details every path must publish for one runner-reported error code. */ export type RunnerReportedErrorClass = Readonly<{ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index c5358112eb..aaf2069bda 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -438,7 +438,8 @@ A `click` or `press` whose target the daemon resolves to a point (`@ref` or sele Text-entry errors and what to do: - `TEXT_INPUT_NOT_FOCUSED` (`type`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. -- `NO_TEXT_INPUT_AT_POINT` (iOS `fill `): the runner identified no text input under the point, so it refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`) and no field changed. The same code covers a lookup that could not answer, so an empty point is not proven — run `snapshot -i` again, then `fill` the field by ref or selector, or aim at a point inside its rect. +- `NO_TEXT_INPUT_AT_POINT` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner identified no text input under that point, so it refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`) and no field changed. A ref or selector fill aims at the centre of the rect it resolved to, so an outer container or a non-input node can land outside the real field: run `snapshot -i`, pick the input node itself, and aim inside its rect. +- `TEXT_INPUT_PROBE_UNAVAILABLE` (iOS `fill` under a degraded XCTest channel): the runner's input lookup for the point could not answer, so it refused before dispatching anything (`error.details.reason` is `text_input_probe_unavailable`, `dispatched: no`). Unlike the refusal above this does not mean the point has no input — retry the fill, or run `snapshot -i` once its capture reads back. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry. - `TEXT_INPUT_COMMIT_NOT_OBSERVED` (iOS `fill`): the runner could not confirm the text landed before its deadline, or the text equals the field's placeholder. The field may hold none, part, or all of it. Run `snapshot -i`; if it doesn't match, retry with the full text quoted and `fill --delay-ms 80`. - `TEXT_INPUT_CLEAR_NOT_OBSERVED` (iOS `fill`): the old text was not seen cleared, so nothing was typed. Inspect with `snapshot -i`, then retry `fill`.