diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7f2a7068..41be908fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Add `settings mobile-data on|off` for adb-backed Android targets. It switches mobile data through `svc data` and reports the `mobileData` state stored in the global `mobile_data` row; multi-SIM read-back is not covered yet. Every other target refuses it before anything runs, and `--app` is refused because the switch is device-wide. +- On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect. - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`. - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state `). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 6a9612eecc..109c610949 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -91,6 +91,10 @@ @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, assign) NSUInteger replacementAlertActions; @property(nonatomic, strong) UILabel *textEntryWriteBackStatus; @property(nonatomic, strong) UILabel *textEntryDigitSlots; +// Counts presses a gesture landed on a control that is not a text input, so a test can tell a +// refusal that ran nothing from one that dispatched its gesture first and refused afterwards. +@property(nonatomic, strong) UILabel *tapWitnessStatus; +@property(nonatomic, assign) NSUInteger tapWitnessPresses; // Filled from the fixture field's own delegate callbacks so a test can tell that a gesture really // took focus from an input that has stopped answering the accessibility query that found it. @property(nonatomic, strong) UILabel *textEntryFocusWitness; @@ -529,6 +533,17 @@ - (void)agentDeviceMaskedToggleDidPress:(UIButton *)sender { [self updateMaskedToggleStatus]; } +- (void)agentDeviceTapWitnessDidPress:(UIButton *)sender { + (void)sender; + self.tapWitnessPresses += 1; + [self updateTapWitnessStatus]; +} + +- (void)updateTapWitnessStatus { + self.tapWitnessStatus.text = + [NSString stringWithFormat:@"presses=%lu", (unsigned long)self.tapWitnessPresses]; +} + - (void)updateMaskedToggleStatus { self.maskedToggleStatus.text = [NSString stringWithFormat:@"presses=%lu", (unsigned long)self.maskedTogglePresses]; @@ -660,6 +675,32 @@ - (void)viewDidLoad { ((AgentDeviceSubmitClearingTextField *)textField).entryStatus = entryStatus; } } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-tap-witness"]) { + // A control that is not a text input, sitting below the field. Its press count is the only + // witness a test has that a coordinate gesture reached this screen: a refusal that dispatched + // its focus tap here would move the counter, and one that refused first would not. + UIButton *witness = [UIButton buttonWithType:UIButtonTypeSystem]; + witness.accessibilityIdentifier = @"agent-device-tap-witness"; + [witness setTitle:@"Tap witness" forState:UIControlStateNormal]; + witness.translatesAutoresizingMaskIntoConstraints = NO; + [witness addTarget:self + action:@selector(agentDeviceTapWitnessDidPress:) + forControlEvents:UIControlEventTouchUpInside]; + [self.view addSubview:witness]; + self.tapWitnessStatus = [[UILabel alloc] init]; + self.tapWitnessStatus.accessibilityIdentifier = @"agent-device-tap-witness-presses"; + self.tapWitnessStatus.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:self.tapWitnessStatus]; + [NSLayoutConstraint activateConstraints:@[ + [witness.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [witness.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:24], + [witness.widthAnchor constraintEqualToConstant:200], + [witness.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], + [self.tapWitnessStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [self.tapWitnessStatus.topAnchor constraintEqualToAnchor:witness.bottomAnchor constant:12], + ]]; + [self updateTapWitnessStatus]; + } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unqueryable-on-focus"]) { self.textEntryFocusWitness = [[UILabel alloc] init]; self.textEntryFocusWitness.accessibilityIdentifier = @"agent-device-text-entry-focus"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 6f8547cf25..8b8ff0acc7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -38,19 +38,44 @@ extension RunnerTests { } } + /// What the point's text-input lookup can say before a coordinate gesture is dispatched: the + /// input it found, no input at the point, or nothing. A lookup that cannot answer is neither of the + /// first two, and must not be read as one of them. + enum CoordinateTextInputResolution { + case input(TextInputAtPoint) + case absent + case unavailable + } + /// The input `coordinateTapTextInputAt` finds, with the identity it carried then, read inside the - /// same issue containment. Nil when the probe finds none or cannot answer. - func coordinateTapTextInputIdentityAt(app: XCUIApplication, x: Double, y: Double) -> TextInputAtPoint? { - let probed = containingTextInputProbeIssues(fallback: nil) { shouldStop -> TextInputAtPoint? in - guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first, - case .input(let identity) = probeTextEntryInput(element) + /// same issue containment, split into what the lookup can name so a refusal can tell an empty + /// point from a blind one. + func coordinateTapTextInputIdentityAt( + app: XCUIApplication, + x: Double, + y: Double + ) -> CoordinateTextInputResolution { + let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { + shouldStop -> CoordinateTextInputResolution in + guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first else { - return nil + return .absent } - return TextInputAtPoint(element: element, identity: identity) + switch probeTextEntryInput(element) { + case .input(let identity): + return .input(TextInputAtPoint(element: element, identity: identity)) + case .noMatch: + return .absent + case .unavailable: + return .unavailable + } + } + switch probed { + case .success(let resolution): + return resolution + case .failure: + return .unavailable } - guard case .success(let input) = probed else { return nil } - return input } /// Whether `input`'s handle still resolves to the input it named when found, read inside the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index c49336a82e..acbb62ef54 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -28,7 +28,18 @@ extension RunnerTests { xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { // Resolved before the tap: the focus it causes can move the field away from this point. - let inputAtPoint = coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) + // A replacement with no input at the point has nothing to bind to: its clear and its typed + // text would reach the app-level first responder, so an empty point and a lookup that cannot + // answer both refuse here, each under its own reason. + let inputAtPoint: TextInputAtPoint + switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) { + case .input(let input): + inputAtPoint = input + case .absent: + return noTextInputAtPointResponse() + case .unavailable: + return textInputProbeUnavailableResponse() + } let policyKind = SynthesizedGesturePolicyKind.coordinateTap let context = synthesizedCoordinateContext( app: activeApp, @@ -149,4 +160,33 @@ extension RunnerTests { } return Response(ok: true, data: data) } + + /// The refusal of a coordinate `fill` whose point identified no text input, answered before the + /// focus tap is dispatched so nothing runs. Without an input at the point the synthesized + /// replacement has no element to bind to, so its clear and its typed text would reach the app-level + /// first responder and overwrite whatever field currently holds focus. + private func noTextInputAtPointResponse() -> Response { + Response( + ok: false, + error: ErrorPayload( + code: RunnerWireErrorCode.noTextInputAtPoint, + message: "no text input identified at the provided coordinates", + hint: "A fill replaces the text of the input under its point, so the point must sit inside one — for a ref or selector fill, that point is the centre of the rect it resolved to, which an outer container or a non-input node does not give you. Run snapshot -i, pick the input node itself, and aim at a point inside its rect." + ) + ) + } + + /// The refusal of a coordinate `fill` whose point the input lookup could not read, answered before + /// the focus tap so nothing runs. The channel is degraded, so the runner cannot tell whether an + /// input is there; typing would reach the app-level first responder, which is what #1721 refuses. + private func textInputProbeUnavailableResponse() -> Response { + Response( + ok: false, + error: ErrorPayload( + code: RunnerWireErrorCode.textInputProbeUnavailable, + message: "the runner could not read the text input at the provided coordinates", + hint: "The accessibility lookup for the point could not answer on the degraded channel, so the runner dispatched nothing and no field changed — this does not mean the point has no input. Retry the fill, or run snapshot -i and fill the input node by ref once its capture reads back." + ) + ) + } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 3d8b886140..fcb0d942cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -33,9 +33,14 @@ final class RunnerTests: XCTestCase { /// gate; `MAIN_THREAD_TIMEOUT` is emitted by the transport when a command trips the execution /// watchdog, so the daemon can tell "the main thread is now occupied" from a generic failure. /// `APP_NOT_RUNNING` refuses a read whose session app is not running. + /// `NO_TEXT_INPUT_AT_POINT` refuses a coordinate fill whose point identified no text input. + /// `TEXT_INPUT_PROBE_UNAVAILABLE` refuses one whose input lookup could not answer on the degraded + /// channel, so the runner cannot tell whether an input is there. enum RunnerWireErrorCode { static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT" static let appNotRunning = "APP_NOT_RUNNING" + static let noTextInputAtPoint = "NO_TEXT_INPUT_AT_POINT" + static let textInputProbeUnavailable = "TEXT_INPUT_PROBE_UNAVAILABLE" } static let springboardBundleId = "com.apple.springboard" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 61029f357f..eb8c68072d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -2,6 +2,26 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// The input the coordinate probe finds at the point, failing the test when it finds none or + /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. + @MainActor + func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y) { + case .input(let input): + return input + case .absent: + return try XCTUnwrap( + nil as TextInputAtPoint?, + "the probe found no input at the fixture's own field" + ) + case .unavailable: + return try XCTUnwrap( + nil as TextInputAtPoint?, + "the probe could not read the fixture's own field" + ) + } + } + @MainActor func testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType() throws { app.launchArguments = ["--agent-device-text-entry-regression"] @@ -52,5 +72,44 @@ extension RunnerTests { XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertEqual(String(describing: textField.value ?? ""), "") } + + /// A penalized coordinate `fill` whose point names no text input cannot bind its replacement to a + /// target, so it refuses with the typed reason instead of clearing and typing through the app-level + /// first responder — which, while an earlier field still holds focus, destroys that field's text. + /// The aim point is a button that counts its presses: a refusal that dispatched its focus tap first + /// would move that counter even though the field came through, so the counter is what proves the + /// refusal happened before the tap rather than after it. + @MainActor + func testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesBeforeItsTapAndKeepsTheFocusedField() throws { + let field = try focusSynthesizedReplacementField(extraLaunchArguments: [ + "--agent-device-text-entry-tap-witness", + ]) + defer { tearDownSynthesizedReplacementField() } + let seeded = try replaceSynthesizedFieldText(field, text: "stale", commandId: "seed-non-text-point") + XCTAssertTrue(seeded.ok, String(describing: seeded.error)) + + let nonTextTarget = app.buttons["agent-device-tap-witness"] + XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout)) + let nonTextFrame = nonTextTarget.frame + XCTAssertFalse(nonTextFrame.isEmpty) + let pressCount = app.staticTexts["agent-device-tap-witness-presses"] + XCTAssertTrue(pressCount.waitForExistence(timeout: appExistenceTimeout)) + + let fillCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"fill-penalized-non-text-point","text":"must-not-replace","textEntryMode":"replace","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY)}"# + ) + let failureCountBefore = currentXCTestFailureCount() + let fillResponse = executeTypeCommand(activeApp: app, command: fillCommand) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(fillResponse.ok, String(describing: fillResponse.error)) + XCTAssertEqual(fillResponse.error?.code, "NO_TEXT_INPUT_AT_POINT") + XCTAssertEqual(String(describing: field.value ?? ""), "stale") + XCTAssertEqual( + pressCount.label, + "presses=0", + "the refusal dispatched its focus tap: the point's control took the press" + ) + } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index bd78542216..ac0ac0e54d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -222,7 +222,7 @@ extension RunnerTests { let field = try focusSynthesizedReplacementField() defer { tearDownSynthesizedReplacementField() } let frame = field.frame - let found = try XCTUnwrap(coordinateTapTextInputIdentityAt(app: app, x: frame.midX, y: frame.midY)) + let found = try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY) let cases: [(dropped: Int, failure: TextEntryFailure?)] = [ (2, nil), (SynthesizedTextPlan.Step.maxClearPassCount, .clearNotObserved), @@ -420,10 +420,12 @@ extension RunnerTests { } /// The pre-tap lookup runs on a channel already penalized for failing XCTest reads. A lookup that - /// fails leaves the fill to the point; it used to record the failure, which fails the command - /// and ends the runner. + /// fails used to record the failure, which fails the command and ends the runner, and left the + /// fill to the point. Containment stays: no recorded failure. What changed is what the fill then + /// does with a point it cannot read: it refuses under the probe-unavailable reason, which is not a + /// claim that the point has no input (#1721). @MainActor - func testSynthesizedReplacementContainsAFailedLookupOfTheInputUnderThePoint() throws { + func testSynthesizedReplacementFailsClosedOnAFailedLookupOfTheInputUnderThePoint() throws { let textField = try focusSynthesizedReplacementField() defer { textInputProbeIssueForTesting = nil @@ -437,8 +439,9 @@ extension RunnerTests { let response = try replaceSynthesizedFieldText(textField, text: "fresh", commandId: "fill-lookup-failed") XCTAssertNil(textInputProbeIssueForTesting, "the fill made no lookup") - XCTAssertTrue(response.ok, String(describing: response.error)) - XCTAssertEqual(String(describing: textField.value ?? ""), "fresh") + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_PROBE_UNAVAILABLE") + XCTAssertEqual(String(describing: textField.value ?? ""), "") } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index e6b76fad08..5881482c12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -702,7 +702,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, - inputAtRefreshPoint: try XCTUnwrap(coordinateTapTextInputIdentityAt(app: app, x: frame.midX, y: frame.midY)) + inputAtRefreshPoint: try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY) ), text: "abc", delaySeconds: 0.001, diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index dae5d97f60..4fc748c1fd 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -193,6 +193,18 @@ "trigger": "structured runner reply with code ALERT_NOT_FOUND: a refusal that ran nothing", "dispatched": "no" }, + { + "id": "ios-runner.reply.NO_TEXT_INPUT_AT_POINT", + "producer": "ios-runner", + "trigger": "structured runner reply with code NO_TEXT_INPUT_AT_POINT: a coordinate fill refused before its focus tap because the runner identified no text input under the point", + "dispatched": "no" + }, + { + "id": "ios-runner.reply.TEXT_INPUT_PROBE_UNAVAILABLE", + "producer": "ios-runner", + "trigger": "structured runner reply with code TEXT_INPUT_PROBE_UNAVAILABLE: a coordinate fill refused before its focus tap because the pre-tap input lookup could not answer", + "dispatched": "no" + }, { "id": "ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED", "producer": "ios-runner", diff --git a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts index bef310980b..7e66b4dfb9 100644 --- a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts @@ -168,6 +168,9 @@ const DRIVERS: Record Promise> = { 'ios-runner.reply.SCROLL_KEYBOARD_OCCLUDES_SURFACE': () => replyFailure('SCROLL_KEYBOARD_OCCLUDES_SURFACE'), 'ios-runner.reply.ALERT_NOT_FOUND': () => replyFailure('ALERT_NOT_FOUND'), + 'ios-runner.reply.NO_TEXT_INPUT_AT_POINT': () => replyFailure('NO_TEXT_INPUT_AT_POINT'), + 'ios-runner.reply.TEXT_INPUT_PROBE_UNAVAILABLE': () => + replyFailure('TEXT_INPUT_PROBE_UNAVAILABLE'), 'ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED': () => replyFailure('APP_SCREEN_WINDOW_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), diff --git a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts index f68e403129..f6ceb35127 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts @@ -2,7 +2,9 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { + NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES, + TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, classifyRunnerReportedError, } from '../runner-contract.ts'; import { @@ -96,6 +98,41 @@ test('the main-thread runner codes publish details.reason beside details.runnerE assert.equal(classifyRunnerReportedError('APP_NOT_RUNNING').details.reason, undefined); }); +// The literal is what the Swift runner encodes, so it is the contract and not the constant name: +// a rename on one side has to fail here rather than silently split the pair. +test('a coordinate fill refused for a point with no input keeps its code and publishes its reason', () => { + assert.equal(NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'NO_TEXT_INPUT_AT_POINT'); + const refusal = classifyRunnerReportedError(NO_TEXT_INPUT_AT_POINT_RUNNER_CODE); + assert.equal(refusal.code, 'NO_TEXT_INPUT_AT_POINT'); + assert.equal(refusal.details.reason, 'no_text_input_at_point'); + assert.equal(refusal.details.dispatched, 'no'); + assert.equal(refusal.details.retriable, undefined); + // The refusal ran nothing, so it is a definite answer and never a resend. + const refusalError = new AppError(refusal.code, refusal.details.reason ?? '', refusal.details); + assert.equal(isRetryableRunnerError(refusalError), false); + assert.equal(isRunnerBusyError(refusalError), false); + // The closest negative: the same words from an unclassified failure earn no reason, so a + // consumer keyed on `reason` cannot mistake a reply that may have landed for this refusal. + const untyped = commandFailed('no text input identified at the provided coordinates'); + assert.equal(untyped.details?.reason, undefined); + assert.equal(isRetryableRunnerError(untyped), false); +}); + +// The two coordinate-fill refusals answer different questions and must never collapse into one +// another: an empty point is a fact about the screen, a blind lookup is a fact about the channel. +test('a coordinate fill refused on an unreadable point reports its own code, not no-input', () => { + assert.equal(TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, 'TEXT_INPUT_PROBE_UNAVAILABLE'); + const refusal = classifyRunnerReportedError(TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE); + assert.equal(refusal.code, 'TEXT_INPUT_PROBE_UNAVAILABLE'); + assert.equal(refusal.details.reason, 'text_input_probe_unavailable'); + assert.equal(refusal.details.dispatched, 'no'); + assert.notEqual( + refusal.details.reason, + classifyRunnerReportedError(NO_TEXT_INPUT_AT_POINT_RUNNER_CODE).details.reason, + ); + assert.equal(isRetryableRunnerError(new AppError(refusal.code, '', refusal.details)), false); +}); + test('retryable requires an AppError with COMMAND_FAILED', () => { assert.equal(isRetryableRunnerError(new Error('fetch failed')), false); assert.equal(isRetryableRunnerError(new AppError('DEVICE_NOT_FOUND', 'fetch failed')), false); diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 3dae04e8f0..f29118d7ee 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -256,6 +256,21 @@ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = n RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE, ]); +/** + * The runner's own code refusing a coordinate `fill` whose point identified no text input (#1721). + * The synthesized first-responder replacement this route would otherwise take has no element, so its + * clear and its typed text reach whichever field already holds focus. The runner refuses before + * dispatching the focus tap, so the reply proves nothing ran. + */ +export const NO_TEXT_INPUT_AT_POINT_RUNNER_CODE = 'NO_TEXT_INPUT_AT_POINT'; + +/** + * The runner code refusing a coordinate `fill` whose pre-tap input lookup could not answer on the + * degraded channel. Like NO_TEXT_INPUT_AT_POINT it answers before the focus tap, and it is not a + * claim that the point has no input — only that the runner could not tell. + */ +export const TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE = 'TEXT_INPUT_PROBE_UNAVAILABLE'; + /** * Runner codes that classify a failure for the host without renaming it on the wire. They stay * `COMMAND_FAILED` and survive as `details.runnerErrorCode`, which is what family policy reads: @@ -263,18 +278,27 @@ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = n * the scroll keyboard refusal for a surface the runner declined to swipe under the keys, and the * retriable `APP_NOT_RUNNING` for a read the runner refused rather than launch the session app. */ -const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlyMap< - string, - { retriable?: true; reason?: RunnerReportedErrorReason } -> = new Map([ - [RUNNER_BUSY_RUNNER_CODE, { retriable: true, reason: 'runner_busy' }], - [MAIN_THREAD_TIMEOUT_RUNNER_CODE, { reason: 'runner_main_thread_timeout' }], +const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlyMap = new Map([ + [RUNNER_BUSY_RUNNER_CODE, { retriable: true }], + [MAIN_THREAD_TIMEOUT_RUNNER_CODE, {}], [APP_NOT_RUNNING_RUNNER_CODE, { retriable: true }], [ALERT_NOT_FOUND_RUNNER_CODE, {}], [SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, {}], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, {}] as const), ]); +/** + * `details.reason` for the runner codes a consumer acts on, so it reads one field instead of + * `details.runnerErrorCode`, which stays for the runner's own vocabulary. Both the codes that stay + * off the wire and the codes that travel under their own name are answered from this one table. + */ +const RUNNER_REPORTED_ERROR_REASONS: ReadonlyMap = new Map([ + [RUNNER_BUSY_RUNNER_CODE, 'runner_busy'], + [MAIN_THREAD_TIMEOUT_RUNNER_CODE, 'runner_main_thread_timeout'], + [NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'no_text_input_at_point'], + [TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, 'text_input_probe_unavailable'], +]); + /** * Runner codes whose reply proves the command never reached the device. The refusals answer before * the command runs, the selector refusals included: the runner resolves the element and refuses @@ -293,6 +317,8 @@ const RUNNER_ERROR_CODE_DISPATCH: ReadonlyMap = new ['OBSERVATION_UNAVAILABLE', 'no'], [SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, 'no'], [ALERT_NOT_FOUND_RUNNER_CODE, 'no'], + [NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'no'], + [TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, 'no'], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, 'no'] as const), ]); @@ -300,7 +326,11 @@ const RUNNER_ERROR_CODE_DISPATCH: ReadonlyMap = new * `details.reason` for the runner codes a consumer acts on, so it reads one field instead of * `details.runnerErrorCode`, which stays for the runner's own vocabulary. */ -type RunnerReportedErrorReason = 'runner_busy' | 'runner_main_thread_timeout'; +type RunnerReportedErrorReason = + | 'runner_busy' + | 'runner_main_thread_timeout' + | 'no_text_input_at_point' + | 'text_input_probe_unavailable'; /** Wire code plus the details every path must publish for one runner-reported error code. */ export type RunnerReportedErrorClass = Readonly<{ @@ -326,11 +356,14 @@ export function classifyRunnerReportedError( runnerErrorCode === undefined ? undefined : DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES.get(runnerErrorCode); + const reason = + runnerErrorCode === undefined ? undefined : RUNNER_REPORTED_ERROR_REASONS.get(runnerErrorCode); return Object.freeze({ code: diagnosticOnly ? 'COMMAND_FAILED' : toAppErrorCode(runnerErrorCode), details: Object.freeze({ runnerErrorCode, ...diagnosticOnly, + ...(reason === undefined ? {} : { reason }), dispatched: (runnerErrorCode === undefined ? undefined diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f2099449d5..aaf2069bda 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -438,6 +438,8 @@ A `click` or `press` whose target the daemon resolves to a point (`@ref` or sele Text-entry errors and what to do: - `TEXT_INPUT_NOT_FOCUSED` (`type`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. +- `NO_TEXT_INPUT_AT_POINT` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner identified no text input under that point, so it refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`) and no field changed. A ref or selector fill aims at the centre of the rect it resolved to, so an outer container or a non-input node can land outside the real field: run `snapshot -i`, pick the input node itself, and aim inside its rect. +- `TEXT_INPUT_PROBE_UNAVAILABLE` (iOS `fill` under a degraded XCTest channel): the runner's input lookup for the point could not answer, so it refused before dispatching anything (`error.details.reason` is `text_input_probe_unavailable`, `dispatched: no`). Unlike the refusal above this does not mean the point has no input — retry the fill, or run `snapshot -i` once its capture reads back. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry. - `TEXT_INPUT_COMMIT_NOT_OBSERVED` (iOS `fill`): the runner could not confirm the text landed before its deadline, or the text equals the field's placeholder. The field may hold none, part, or all of it. Run `snapshot -i`; if it doesn't match, retry with the full text quoted and `fill --delay-ms 80`. - `TEXT_INPUT_CLEAR_NOT_OBSERVED` (iOS `fill`): the old text was not seen cleared, so nothing was typed. Inspect with `snapshot -i`, then retry `fill`.