perf(ios): reduce capture probes and text entry waits

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift

@@ -467,9 +467,9 @@ extension RunnerTests {
     repairMode: TextTypingRepairMode,
     text: String,
     fromTapWitness: Bool,
-    softwareKeyboardVisible: Bool
+    softwareKeyboardVisible: @autoclosure () -> Bool
   ) -> Bool {
-    repairMode == .none && text == "\n" && fromTapWitness && !softwareKeyboardVisible
+    repairMode == .none && text == "\n" && fromTapWitness && !softwareKeyboardVisible()
   }
 
   /// Length of the shared prefix of two strings. Feeds value-free commit-wait logging: the

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -452,6 +452,14 @@ extension RunnerTests {
     case unreadable
     /// The read could not answer, or no input resolves.
     case unavailable
+
+    func warmupReadIsPending(expectedText: String) -> Bool {
+      switch self {
+      case .text(let value): return value != expectedText
+      case .unreadable: return false
+      case .unavailable: return true
+      }
+    }
   }
 
   /// What a text entry element's accessibility value says about its text. The one owner of the

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift

@@ -252,7 +252,9 @@ extension RunnerTests {
       }
       let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout)
       while Date() < deadline {
-        if editableTextValue(for: resolveTextEntryElement(app: app, target: activeTarget)) == warmupExpectedText {
+        let observed = textValueRead(for: resolveTextEntryElement(app: app, target: activeTarget))
+        if !observed.warmupReadIsPending(expectedText: warmupExpectedText) {
+          if observed == .unreadable { sleepFor(TextEntryTiming.pollInterval) }
           return
         }
         sleepFor(TextEntryTiming.pollInterval)

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift

@@ -52,28 +52,34 @@ extension RunnerTests {
   }
 
   func testSynthesizedFirstResponderTypeAdmitsOnlyTheBareSubmitKeyAfterAHiddenKeyboardTap() {
-    let cases: [(TextTypingRepairMode, String, Bool, Bool, Bool)] = [
-      (.none, "\n", true, false, true),
-      (.none, "\n", true, true, false),
-      (.none, "\n", false, false, false),
-      (.none, "hardware-keyboard", true, false, false),
-      (.none, "\r", true, false, false),
-      (.none, "search\n", true, false, false),
-      (.append, "\n", true, false, false),
-      (.append, "hardware-keyboard", true, false, false),
-      (.replacement, "\n", true, false, false),
+    let cases: [(TextTypingRepairMode, String, Bool, Bool, Bool, Int)] = [
+      (.none, "\n", true, false, true, 1),
+      (.none, "\n", true, true, false, 1),
+      (.none, "\n", false, false, false, 0),
+      (.none, "hardware-keyboard", true, false, false, 0),
+      (.none, "\r", true, false, false, 0),
+      (.none, "search\n", true, false, false, 0),
+      (.append, "\n", true, false, false, 0),
+      (.append, "hardware-keyboard", true, false, false, 0),
+      (.replacement, "\n", true, false, false, 0),
     ]
-    for (mode, text, fromTapWitness, softwareKeyboardVisible, expected) in cases {
+    for (mode, text, fromTapWitness, softwareKeyboardVisible, expected, expectedReads) in cases {
+      var keyboardReads = 0
+      func readKeyboardVisibility() -> Bool {
+        keyboardReads += 1
+        return softwareKeyboardVisible
+      }
       XCTAssertEqual(
         Self.shouldUseSynthesizedFirstResponderType(
           repairMode: mode,
           text: text,
           fromTapWitness: fromTapWitness,
-          softwareKeyboardVisible: softwareKeyboardVisible
+          softwareKeyboardVisible: readKeyboardVisibility()
         ),
         expected,
         "mode: \(mode), text: \(text.debugDescription)"
       )
+      XCTAssertEqual(keyboardReads, expectedReads)
     }
   }
 
@@ -132,6 +138,19 @@ extension RunnerTests {
     XCTAssertEqual(Self.textValueRead(elementType: .button, value: bullets), .unavailable)
   }
 
+  func testWarmupWaitsForPendingTextButNotAnUnreadableValue() {
+    let cases: [(TextValueRead, Bool)] = [
+      (.text("a"), false),
+      (.text(""), true),
+      (.text("old"), true),
+      (.unavailable, true),
+      (.unreadable, false),
+    ]
+    for (read, pending) in cases {
+      XCTAssertEqual(read.warmupReadIsPending(expectedText: "a"), pending)
+    }
+  }
+
   // The penalized route's clear reads through the same rule, so a pre-filled masked plain field
   // gets the unread passes a secure field gets and is then typed into unverified, instead of
   // ending the clear as a field that could not be read.

docs/evidence/ios-e2e-engine-pr1236-2026-10-19.md

@@ -0,0 +1,114 @@
+# iOS engine PR 1236: performance and safety investigation
+
+Date: 2026-10-19.
+
+Agent-device is already competitive on this host. There is a credible route to faster observations and screenshots without weakening its interaction guarantees. This investigation does **not** establish a safe, general end-to-end win: the full runs had different failures, and the 39 mutually passing cases differed by only 2% in one run. No production behavior was changed.
+
+## What they are building
+
+[PR 1236](https://github.com/tester-army/e2e/pull/1236), examined at `71ee26c7c7b01eb8646d7acc456edf5ebb181205`, is the Swift proof of a language-independent engine protocol. Its parent [PR 1232](https://github.com/tester-army/e2e/pull/1232) defines the protocol; [PR 1234](https://github.com/tester-army/e2e/pull/1234) supplies the Java/Android proof.
+
+The existing route is e2e locators → mobile adapter → agent-device SDK/daemon → platform acquisition or XCTest runner. The new route is e2e `connectEngine` → framed JSON-RPC over loopback TCP → a Swift server inside one XCUITest test → app. `xcodebuild test-without-building` starts that server. Network work has its own queue; device commands execute serially on the main thread. Abort notifications can reach the queue while an operation runs.
+
+Swift builds its semantic tree from one `XCUIApplication.snapshot()`. Locator resolution still runs in TypeScript through `resolveLocators: true`; this is not a native selector-index optimization. Observed identities and frames support later gestures. Unique identifiers or type/label addresses use native element activation. The client handles restart/reset through `simctl`. Screenshots are masked from secure-field observations before and after capture.
+
+The engine is deliberately private and experimental. It has one simulator, a device fixture limited to back, and no video or general device-control parity. The 41 executed benchmark tests do not establish agent-device's complete interaction contract.
+
+Primary implementation references: [Swift device](https://github.com/tester-army/e2e/blob/71ee26c7c7b01eb8646d7acc456edf5ebb181205/packages/ios/server/Server/Device.swift), [server](https://github.com/tester-army/e2e/blob/71ee26c7c7b01eb8646d7acc456edf5ebb181205/packages/ios/server/Server/EngineServer.swift), [client/transport](https://github.com/tester-army/e2e/tree/71ee26c7c7b01eb8646d7acc456edf5ebb181205/packages/ios/src), [existing mobile adapter](https://github.com/tester-army/e2e/blob/71ee26c7c7b01eb8646d7acc456edf5ebb181205/packages/mobile/src/surface.ts).
+
+## Experiment
+
+- Agent-device source: `8f3eb3d2efde3654a560fddb39127d556e068352`, `0.21.29-dev`. The timed client imported this checkout's built SDK. The daemon reported `codeOrigin: checkout`. T3's installed `0.21.25` CLI was used only for installation/manual warmup.
+- Hardware: Apple M4 Pro, 48 GiB RAM; Xcode 27.1 (`27A9275`); Node 26.11.1.
+- Device: iPhone 17 Pro simulator, iOS 27.0, UDID `BC54AC11-CD86-4C10-8C4E-5101C773FFEB`.
+- App: their `apps/mobile-benchmark`, `dev.e2e.benchmark`, bundled Release build. Tracked screens and original tests remained unchanged.
+- Both engines used a 1,500 ms transition budget, one worker, and the same installed app. Swift retained its default idle waiting. Runs were sequential, with builds completed before measurement.
+- The generated Expo 57 native host needed a `UIScene` delegate/manifest to launch with this SDK. This adaptation is included in the evidence bundle. [Apple documents the iOS 27 requirement](https://developer.apple.com/documentation/uikit/transitioning-to-the-uikit-scene-based-life-cycle). This is not an exact reproduction of the PR's iOS 26.5 environment.
+- The local agent-device runner used its normal Debug scheme; their server build uses Release. This was a comparison of their normal build commands, not a matched compiler-optimization experiment.
+- Microbenchmarks used five rounds, nearest-rank p50, and an additional untimed value check after every fill. All ten measured fills across both engines had correct values. Startup had one sample and is excluded from conclusions.
+
+The original microbenchmark gives Swift a default 500 ms transition budget while setting agent-device to 1,500 ms. This investigation corrected that mismatch. It also retained per-operation failures in JSON; the upstream harness can exit zero despite recorded operation failures.
+
+## Results
+
+Full original selection, one run per engine:
+
+| Measurement | Agent-device | Swift |
+| --- | ---: | ---: |
+| Total CLI duration | 525.43 s | 535.12 s |
+| Passed / failed / skipped | 39 / 2 / 1 | 40 / 1 / 1 |
+| Sum of attempt durations for the same 39 passing tests | 489.889 s | 500.006 s |
+| Controls file | 179.92 s | 166.92 s |
+| Lists file, including failed WebView case | 213.82 s | 259.15 s |
+| Row-512 virtualized-list test | 116.16 s | 152.14 s |
+| Login-form file | 46.57 s | 30.43 s |
+
+Agent-device full-run ID: `01a153a9-89c7-7eb5-84b9-c402ed0c144e`, exit 3. Swift full-run ID: `01a153b2-bc5b-730e-bb09-7095373a8255`, exit 1. The common-case subtotal excludes startup and includes only tests that passed in both runs. It is not a passing full-suite speed claim.
+
+Engine-call p50, milliseconds:
+
+| Operation | Agent-device | Swift |
+| --- | ---: | ---: |
+| Observe home | 92.6 | 21.6 |
+| Observe form | 90.3 | 14.2 |
+| Locate test ID | 84.5 | 13.3 |
+| Tap field | 521.6 | 980.8 |
+| Fill 19 characters | 3,139.8 | 1,988.3 |
+| Type 10 characters | 1,221.3 | 442.7 |
+| Screenshot | 703.8 | 118.8 |
+| Observe with pixels | 682.8 | 115.8 |
+| Swipe, successful calls only | 544.6 | 1,451.3 |
+| Back until list visible | 895.4 | 1,213.3 |
+
+One agent-device swipe was refused as having no effect, leaving four successful samples. Repeated Enter-on-node succeeded once and failed four times on agent-device; Swift succeeded five times. Those rows are not equivalent successful workloads. The mobile adapter's Enter branch sends a keyboard command without refocusing the node; Swift taps the input first.
+
+Tap samples varied substantially on Swift (396–1,687 ms). The first fill into an empty field took 1,992 ms / 1,406 ms; subsequent samples replaced existing text. Repeated replacement therefore overstates clearing's importance in an ordinary fresh login.
+
+## Diagnosed failures
+
+**Shared WebView selector failure.** Both original runs failed `getByText('TA-BENCH-50')`. On iOS 27 the captured text is `Your coupon code is TA-BENCH-50.` and e2e defaults to exact matching. A separate test changed only that locator to the complete sentence, then performed the original fill, checkbox, apply, and success assertions. It passed on agent-device in 10.79 s and Swift in 11.59 s. The original suite was not edited or relabeled as passing.
+
+**Agent-device Bottom Tabs false occlusion.** After switching to Actions, the visibly unobstructed `complete-action-button` was refused with typed reason `target_covered`, `dispatched: no`. The reported cover was a transparent `_UIFloatingBarContainerView` toolbar at `(0, 0, 402, 791)`.
+
+This reproduced after a simulator restart. The regular bridge capture had 9 nodes and lost the content-branch ancestry; the raw capture had 41 nodes. A pure replay removed existing covered annotations, restored native hittability from the raw capture, and applied the same `annotateCoveredSnapshotNodes` function to each tree. The full tree left the target actionable; the regular projection marked it covered. An XCTest fallback capture also allowed the ref tap. A raw-ref tap reached the visible `Action completed` state, but raw mode deliberately skips normal occlusion annotation and is not a proposed fix.
+
+The owning problem is loss of structural evidence before occlusion classification. Investigate `packages/capture-kit/src/snapshot-state.ts:75`, the iOS regular-presentation pipeline, and `snapshot-occlusion.ts`. The existing sibling-sized chrome tests do not exercise this complete projection-to-ref-dispatch path. Preserve real modal/toolbar-child occlusion while fixing this ordering/provenance issue; do not exempt all toolbars.
+
+## Where safe performance work should go
+
+The agent-device suite issued 831 snapshot commands, spending 178.55 s inside them. It spent 77.24 s in 81 scroll commands and 83.39 s in 42 opens. Three screenshot commands, including failure captures, spent only 2.09 s. These command totals include native/recovery/stabilization work and exclude harness waits.
+
+1. **Consolidate fresh process facts.** After a fresh boot, a warm profiled observation took 103 ms median. Its two `pgrep` and three `ps` probes spent 66 ms median. The system-surface probe, target resolver, and bridge manager own these reads. A persistent native reader could provide fresh device-scoped surface and process-birth facts with fewer subprocesses. Keep unknown-versus-absent behavior, generation checks before publication, cancellation, and system-sheet fallback. Caching absence or dropping identity checks is not the optimization. This is the largest measured opportunity for this suite; no replacement implementation was benchmarked here.
+2. **Keep the occlusion fix ahead of a win claim.** Preserve the structural evidence needed by the existing chrome rule across projection. Add a live-fixture regression through snapshot publication and ref admission. A faster false refusal is not a useful win.
+3. **Optimize bound replacement/focus work.** Runner logs attributed about 539 ms to focus, 105 ms to initial resolution, 1,205 ms to clearing a nonempty field, and 307 ms to final verification, across the repeated 19-character fills. Investigate atomic, identity-bound clearing and fewer duplicate resolutions while retaining final read-back and replaced-input refusal. Do not globally raise synthesis speed: the existing controlled-input evidence records dropped characters at faster rates.
+4. **Use the ready runner for pixels where its capture contract can be preserved.** In the warmed post-login profile, `simctl io ... screenshot` cost 417 ms median and the fresh display query 94 ms. Swift's native route shows that pixels need not cost 700 ms. Preserve active-panel identity, app/screen scope, rotation, density, and secure-field masking. A dynamic foldable panel choice must remain fresh. The images also differ in size: agent-device defaults to 402×874; Swift returns 1206×2622. This would help vision-driven agents far more than this mostly semantic suite.
+5. **Reuse the existing fast PNG encoder when appropriate.** A standalone experiment compared general `PNG.sync.write` with existing `encodePngPixels`, using actual captures and checking decoded RGBA equality every time. At 402×874, p50 was 10.74 → 3.81 ms and output 29,881 → 18,929 bytes. At 1206×2622, it was 101.24 → 24.45 ms and 106,219 → 67,169 bytes. These are encode-only results, ten samples after two warmups. They are useful but cannot explain the screenshot gap. No encoder production path was changed.
+
+The Swift engine does not provide equivalent guarantees merely by passing its selected tests: fill deletes from an earlier observed value without agent-device's final verification; unique addressing is based on the observation; an action does not retire all observed references. Its optional idle-wait switch replaces private XCTest methods. Agent-device already has synthesized gesture/text paths, so adding synthesis is not a new architectural lever by itself. These are design differences to account for, not evidence that every Swift action is unsafe.
+
+## Reproduction and evidence
+
+Initial setup completed with `pnpm install --frozen-lockfile && pnpm build` in this checkout. The agent-device runner was built with `pnpm build:xcuitest`. In the external clone, the e2e, mobile, iOS, and GitHub packages were built, followed by `pnpm --filter @e2e-dev/ios build:server`. The app used `pnpm exec expo run:ios --configuration Release --device generic --no-bundler --output .build/ios`.
+
+The evidence bundle contains original/modified microbenchmark scripts, the current-SDK engine config, native-host adaptation, full JSON reports, terminal logs, raw samples, screenshot experiments, and the regular/raw occlusion captures. It excludes host connection files and daemon credentials.
+
+Representative commands from the external clone:
+
+```sh
+# packages/ios; investigate.ts is included in the bundle
+E2E_IOS_UDID=<udid> AGENT_DEVICE_STATE_DIR=<isolated-state-dir> \
+  BENCH_ONLY=current node bench/investigate.ts --rounds 5
+E2E_IOS_UDID=<udid> BENCH_ONLY=direct node bench/investigate.ts --rounds 5
+
+# apps/mobile-benchmark; custom config is included in the bundle
+E2E_IOS_UDID=<udid> E2E_MOBILE_BENCHMARK_IOS_DEVICE=<udid> \
+  AGENT_DEVICE_STATE_DIR=<isolated-state-dir> BENCH_ENGINE=current \
+  node node_modules/e2e/dist/cli/bin.js run --config e2e.investigate.config.ts
+# Repeat with BENCH_ENGINE=direct, after the first run has closed.
+```
+
+Adjust the bundled SDK import/derived-data paths to the checkout being tested. The upstream microbenchmark remains available as `packages/ios/bench/bench.ts`. The bundled config and native adaptation are patch inputs; Git's tracked-file `dirty: false` field in the upstream report does not describe them.
+
+All manually created sessions were closed. The isolated daemon stopped gracefully and its retained runner cleanup completed. The test simulator was shut down. No changes were pushed and no upstream messages were posted.
+
+Before a product speed claim, fix the regular-projection refusal, use a portable WebView locator, implement the selected optimization, and alternate repeated passing runs with matched build settings. Physical iOS devices, Android, and a global idle-wait change were not validated in this investigation.