fix(runner): refuse an iOS coordinate fill whose point names no text input

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Add `settings mobile-data on|off` for adb-backed Android targets. It switches mobile data through `svc data` and reports the `mobileData` state stored in the global `mobile_data` row; multi-SIM read-back is not covered yet. Every other target refuses it before anything runs, and `--app` is refused because the switch is device-wide.
+- On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect.
 - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one.
 - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings <setting> is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`.
 - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid <setting> state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state <app> <extra>`). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace.

apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m

@@ -91,6 +91,10 @@ @interface AgentDeviceRunnerViewController : UIViewController
 @property(nonatomic, assign) NSUInteger replacementAlertActions;
 @property(nonatomic, strong) UILabel *textEntryWriteBackStatus;
 @property(nonatomic, strong) UILabel *textEntryDigitSlots;
+// Counts presses a gesture landed on a control that is not a text input, so a test can tell a
+// refusal that ran nothing from one that dispatched its gesture first and refused afterwards.
+@property(nonatomic, strong) UILabel *tapWitnessStatus;
+@property(nonatomic, assign) NSUInteger tapWitnessPresses;
 // Filled from the fixture field's own delegate callbacks so a test can tell that a gesture really
 // took focus from an input that has stopped answering the accessibility query that found it.
 @property(nonatomic, strong) UILabel *textEntryFocusWitness;
@@ -529,6 +533,17 @@ - (void)agentDeviceMaskedToggleDidPress:(UIButton *)sender {
   [self updateMaskedToggleStatus];
 }
 
+- (void)agentDeviceTapWitnessDidPress:(UIButton *)sender {
+  (void)sender;
+  self.tapWitnessPresses += 1;
+  [self updateTapWitnessStatus];
+}
+
+- (void)updateTapWitnessStatus {
+  self.tapWitnessStatus.text =
+      [NSString stringWithFormat:@"presses=%lu", (unsigned long)self.tapWitnessPresses];
+}
+
 - (void)updateMaskedToggleStatus {
   self.maskedToggleStatus.text =
       [NSString stringWithFormat:@"presses=%lu", (unsigned long)self.maskedTogglePresses];
@@ -660,6 +675,32 @@ - (void)viewDidLoad {
         ((AgentDeviceSubmitClearingTextField *)textField).entryStatus = entryStatus;
       }
     }
+    if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-tap-witness"]) {
+      // A control that is not a text input, sitting below the field. Its press count is the only
+      // witness a test has that a coordinate gesture reached this screen: a refusal that dispatched
+      // its focus tap here would move the counter, and one that refused first would not.
+      UIButton *witness = [UIButton buttonWithType:UIButtonTypeSystem];
+      witness.accessibilityIdentifier = @"agent-device-tap-witness";
+      [witness setTitle:@"Tap witness" forState:UIControlStateNormal];
+      witness.translatesAutoresizingMaskIntoConstraints = NO;
+      [witness addTarget:self
+                  action:@selector(agentDeviceTapWitnessDidPress:)
+        forControlEvents:UIControlEventTouchUpInside];
+      [self.view addSubview:witness];
+      self.tapWitnessStatus = [[UILabel alloc] init];
+      self.tapWitnessStatus.accessibilityIdentifier = @"agent-device-tap-witness-presses";
+      self.tapWitnessStatus.translatesAutoresizingMaskIntoConstraints = NO;
+      [self.view addSubview:self.tapWitnessStatus];
+      [NSLayoutConstraint activateConstraints:@[
+        [witness.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
+        [witness.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:24],
+        [witness.widthAnchor constraintEqualToConstant:200],
+        [witness.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight],
+        [self.tapWitnessStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
+        [self.tapWitnessStatus.topAnchor constraintEqualToAnchor:witness.bottomAnchor constant:12],
+      ]];
+      [self updateTapWitnessStatus];
+    }
     if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unqueryable-on-focus"]) {
       self.textEntryFocusWitness = [[UILabel alloc] init];
       self.textEntryFocusWitness.accessibilityIdentifier = @"agent-device-text-entry-focus";

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift

@@ -38,19 +38,44 @@ extension RunnerTests {
     }
   }
 
+  /// What the point's text-input lookup can say before a coordinate gesture is dispatched: the
+  /// input it found, no input at the point, or nothing. A lookup that cannot answer is neither of the
+  /// first two, and must not be read as one of them.
+  enum CoordinateTextInputResolution {
+    case input(TextInputAtPoint)
+    case absent
+    case unavailable
+  }
+
   /// The input `coordinateTapTextInputAt` finds, with the identity it carried then, read inside the
-  /// same issue containment. Nil when the probe finds none or cannot answer.
-  func coordinateTapTextInputIdentityAt(app: XCUIApplication, x: Double, y: Double) -> TextInputAtPoint? {
-    let probed = containingTextInputProbeIssues(fallback: nil) { shouldStop -> TextInputAtPoint? in
-      guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first,
-            case .input(let identity) = probeTextEntryInput(element)
+  /// same issue containment, split into what the lookup can name so a refusal can tell an empty
+  /// point from a blind one.
+  func coordinateTapTextInputIdentityAt(
+    app: XCUIApplication,
+    x: Double,
+    y: Double
+  ) -> CoordinateTextInputResolution {
+    let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) {
+      shouldStop -> CoordinateTextInputResolution in
+      guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first
       else {
-        return nil
+        return .absent
       }
-      return TextInputAtPoint(element: element, identity: identity)
+      switch probeTextEntryInput(element) {
+      case .input(let identity):
+        return .input(TextInputAtPoint(element: element, identity: identity))
+      case .noMatch:
+        return .absent
+      case .unavailable:
+        return .unavailable
+      }
+    }
+    switch probed {
+    case .success(let resolution):
+      return resolution
+    case .failure:
+      return .unavailable
     }
-    guard case .success(let input) = probed else { return nil }
-    return input
   }
 
   /// Whether `input`'s handle still resolves to the input it named when found, read inside the

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift

@@ -28,7 +28,18 @@ extension RunnerTests {
       xCTestChannelPenalized: xCTestChannelPenalized
     ), let x = command.x, let y = command.y {
       // Resolved before the tap: the focus it causes can move the field away from this point.
-      let inputAtPoint = coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y)
+      // A replacement with no input at the point has nothing to bind to: its clear and its typed
+      // text would reach the app-level first responder, so an empty point and a lookup that cannot
+      // answer both refuse here, each under its own reason.
+      let inputAtPoint: TextInputAtPoint
+      switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) {
+      case .input(let input):
+        inputAtPoint = input
+      case .absent:
+        return noTextInputAtPointResponse()
+      case .unavailable:
+        return textInputProbeUnavailableResponse()
+      }
       let policyKind = SynthesizedGesturePolicyKind.coordinateTap
       let context = synthesizedCoordinateContext(
         app: activeApp,
@@ -149,4 +160,33 @@ extension RunnerTests {
     }
     return Response(ok: true, data: data)
   }
+
+  /// The refusal of a coordinate `fill` whose point identified no text input, answered before the
+  /// focus tap is dispatched so nothing runs. Without an input at the point the synthesized
+  /// replacement has no element to bind to, so its clear and its typed text would reach the app-level
+  /// first responder and overwrite whatever field currently holds focus.
+  private func noTextInputAtPointResponse() -> Response {
+    Response(
+      ok: false,
+      error: ErrorPayload(
+        code: RunnerWireErrorCode.noTextInputAtPoint,
+        message: "no text input identified at the provided coordinates",
+        hint: "A fill replaces the text of the input under its point, so the point must sit inside one — for a ref or selector fill, that point is the centre of the rect it resolved to, which an outer container or a non-input node does not give you. Run snapshot -i, pick the input node itself, and aim at a point inside its rect."
+      )
+    )
+  }
+
+  /// The refusal of a coordinate `fill` whose point the input lookup could not read, answered before
+  /// the focus tap so nothing runs. The channel is degraded, so the runner cannot tell whether an
+  /// input is there; typing would reach the app-level first responder, which is what #1721 refuses.
+  private func textInputProbeUnavailableResponse() -> Response {
+    Response(
+      ok: false,
+      error: ErrorPayload(
+        code: RunnerWireErrorCode.textInputProbeUnavailable,
+        message: "the runner could not read the text input at the provided coordinates",
+        hint: "The accessibility lookup for the point could not answer on the degraded channel, so the runner dispatched nothing and no field changed — this does not mean the point has no input. Retry the fill, or run snapshot -i and fill the input node by ref once its capture reads back."
+      )
+    )
+  }
 }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -33,9 +33,14 @@ final class RunnerTests: XCTestCase {
   /// gate; `MAIN_THREAD_TIMEOUT` is emitted by the transport when a command trips the execution
   /// watchdog, so the daemon can tell "the main thread is now occupied" from a generic failure.
   /// `APP_NOT_RUNNING` refuses a read whose session app is not running.
+  /// `NO_TEXT_INPUT_AT_POINT` refuses a coordinate fill whose point identified no text input.
+  /// `TEXT_INPUT_PROBE_UNAVAILABLE` refuses one whose input lookup could not answer on the degraded
+  /// channel, so the runner cannot tell whether an input is there.
   enum RunnerWireErrorCode {
     static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT"
     static let appNotRunning = "APP_NOT_RUNNING"
+    static let noTextInputAtPoint = "NO_TEXT_INPUT_AT_POINT"
+    static let textInputProbeUnavailable = "TEXT_INPUT_PROBE_UNAVAILABLE"
   }
 
   static let springboardBundleId = "com.apple.springboard"

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift

@@ -2,6 +2,26 @@ import XCTest
 
 extension RunnerTests {
 #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
+  /// The input the coordinate probe finds at the point, failing the test when it finds none or
+  /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal.
+  @MainActor
+  func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint {
+    switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y) {
+    case .input(let input):
+      return input
+    case .absent:
+      return try XCTUnwrap(
+        nil as TextInputAtPoint?,
+        "the probe found no input at the fixture's own field"
+      )
+    case .unavailable:
+      return try XCTUnwrap(
+        nil as TextInputAtPoint?,
+        "the probe could not read the fixture's own field"
+      )
+    }
+  }
+
   @MainActor
   func testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType() throws {
     app.launchArguments = ["--agent-device-text-entry-regression"]
@@ -52,5 +72,44 @@ extension RunnerTests {
     XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore))
     XCTAssertEqual(String(describing: textField.value ?? ""), "")
   }
+
+  /// A penalized coordinate `fill` whose point names no text input cannot bind its replacement to a
+  /// target, so it refuses with the typed reason instead of clearing and typing through the app-level
+  /// first responder — which, while an earlier field still holds focus, destroys that field's text.
+  /// The aim point is a button that counts its presses: a refusal that dispatched its focus tap first
+  /// would move that counter even though the field came through, so the counter is what proves the
+  /// refusal happened before the tap rather than after it.
+  @MainActor
+  func testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesBeforeItsTapAndKeepsTheFocusedField() throws {
+    let field = try focusSynthesizedReplacementField(extraLaunchArguments: [
+      "--agent-device-text-entry-tap-witness",
+    ])
+    defer { tearDownSynthesizedReplacementField() }
+    let seeded = try replaceSynthesizedFieldText(field, text: "stale", commandId: "seed-non-text-point")
+    XCTAssertTrue(seeded.ok, String(describing: seeded.error))
+
+    let nonTextTarget = app.buttons["agent-device-tap-witness"]
+    XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout))
+    let nonTextFrame = nonTextTarget.frame
+    XCTAssertFalse(nonTextFrame.isEmpty)
+    let pressCount = app.staticTexts["agent-device-tap-witness-presses"]
+    XCTAssertTrue(pressCount.waitForExistence(timeout: appExistenceTimeout))
+
+    let fillCommand = try runnerCommandFixture(
+      #"{"command":"type","commandId":"fill-penalized-non-text-point","text":"must-not-replace","textEntryMode":"replace","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY)}"#
+    )
+    let failureCountBefore = currentXCTestFailureCount()
+    let fillResponse = executeTypeCommand(activeApp: app, command: fillCommand)
+
+    XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore))
+    XCTAssertFalse(fillResponse.ok, String(describing: fillResponse.error))
+    XCTAssertEqual(fillResponse.error?.code, "NO_TEXT_INPUT_AT_POINT")
+    XCTAssertEqual(String(describing: field.value ?? ""), "stale")
+    XCTAssertEqual(
+      pressCount.label,
+      "presses=0",
+      "the refusal dispatched its focus tap: the point's control took the press"
+    )
+  }
 #endif
 }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift

@@ -222,7 +222,7 @@ extension RunnerTests {
     let field = try focusSynthesizedReplacementField()
     defer { tearDownSynthesizedReplacementField() }
     let frame = field.frame
-    let found = try XCTUnwrap(coordinateTapTextInputIdentityAt(app: app, x: frame.midX, y: frame.midY))
+    let found = try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY)
     let cases: [(dropped: Int, failure: TextEntryFailure?)] = [
       (2, nil),
       (SynthesizedTextPlan.Step.maxClearPassCount, .clearNotObserved),
@@ -420,10 +420,12 @@ extension RunnerTests {
   }
 
   /// The pre-tap lookup runs on a channel already penalized for failing XCTest reads. A lookup that
-  /// fails leaves the fill to the point; it used to record the failure, which fails the command
-  /// and ends the runner.
+  /// fails used to record the failure, which fails the command and ends the runner, and left the
+  /// fill to the point. Containment stays: no recorded failure. What changed is what the fill then
+  /// does with a point it cannot read: it refuses under the probe-unavailable reason, which is not a
+  /// claim that the point has no input (#1721).
   @MainActor
-  func testSynthesizedReplacementContainsAFailedLookupOfTheInputUnderThePoint() throws {
+  func testSynthesizedReplacementFailsClosedOnAFailedLookupOfTheInputUnderThePoint() throws {
     let textField = try focusSynthesizedReplacementField()
     defer {
       textInputProbeIssueForTesting = nil
@@ -437,8 +439,9 @@ extension RunnerTests {
     let response = try replaceSynthesizedFieldText(textField, text: "fresh", commandId: "fill-lookup-failed")
 
     XCTAssertNil(textInputProbeIssueForTesting, "the fill made no lookup")
-    XCTAssertTrue(response.ok, String(describing: response.error))
-    XCTAssertEqual(String(describing: textField.value ?? ""), "fresh")
+    XCTAssertFalse(response.ok)
+    XCTAssertEqual(response.error?.code, "TEXT_INPUT_PROBE_UNAVAILABLE")
+    XCTAssertEqual(String(describing: textField.value ?? ""), "")
   }
 #endif
 }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift

@@ -702,7 +702,7 @@ extension RunnerTests {
         element: nil,
         refreshPoint: CGPoint(x: frame.midX, y: frame.midY),
         prefersFocusedElement: false,
-        inputAtRefreshPoint: try XCTUnwrap(coordinateTapTextInputIdentityAt(app: app, x: frame.midX, y: frame.midY))
+        inputAtRefreshPoint: try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY)
       ),
       text: "abc",
       delaySeconds: 0.001,

contracts/fixtures/dispatch-disclosure.json

@@ -193,6 +193,18 @@
     "trigger": "structured runner reply with code ALERT_NOT_FOUND: a refusal that ran nothing",
     "dispatched": "no"
   },
+  {
+    "id": "ios-runner.reply.NO_TEXT_INPUT_AT_POINT",
+    "producer": "ios-runner",
+    "trigger": "structured runner reply with code NO_TEXT_INPUT_AT_POINT: a coordinate fill refused before its focus tap because the runner identified no text input under the point",
+    "dispatched": "no"
+  },
+  {
+    "id": "ios-runner.reply.TEXT_INPUT_PROBE_UNAVAILABLE",
+    "producer": "ios-runner",
+    "trigger": "structured runner reply with code TEXT_INPUT_PROBE_UNAVAILABLE: a coordinate fill refused before its focus tap because the pre-tap input lookup could not answer",
+    "dispatched": "no"
+  },
   {
     "id": "ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED",
     "producer": "ios-runner",

packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts

@@ -168,6 +168,9 @@ const DRIVERS: Record<string, () => Promise<unknown>> = {
   'ios-runner.reply.SCROLL_KEYBOARD_OCCLUDES_SURFACE': () =>
     replyFailure('SCROLL_KEYBOARD_OCCLUDES_SURFACE'),
   'ios-runner.reply.ALERT_NOT_FOUND': () => replyFailure('ALERT_NOT_FOUND'),
+  'ios-runner.reply.NO_TEXT_INPUT_AT_POINT': () => replyFailure('NO_TEXT_INPUT_AT_POINT'),
+  'ios-runner.reply.TEXT_INPUT_PROBE_UNAVAILABLE': () =>
+    replyFailure('TEXT_INPUT_PROBE_UNAVAILABLE'),
   'ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED': () =>
     replyFailure('APP_SCREEN_WINDOW_UNRESOLVED'),
   'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'),