diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index d6bfb51342..9bbca8cca2 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -1,6 +1,7 @@ name: Perf Nightly -# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3). +# Nightly iOS and Android command performance against the fixture app, compared night over night +# per platform (#1781 A3). # - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts # failing. PR lanes check behavior, not time. # - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians @@ -8,9 +9,9 @@ name: Perf Nightly # earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. A night # runs 3 rounds, the minimum for a timing verdict, so one failed sample withholds that step's # timing verdict. -# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30. +# - Cost: one macos-26 job of about 45 minutes and one ubuntu emulator job a night; summaries are +# kept 90 days, raw logs 30. # - Delete or park when: no flagged change has been acted on for 60 nights. -# Android stays report-only and runs only on request. on: schedule: @@ -26,7 +27,7 @@ on: required: false default: '1' scenarios: - description: 'Comma-separated iOS scenario ids (empty runs all)' + description: 'Comma-separated scenario ids (empty runs all)' required: false default: '' baseline-nights: @@ -37,8 +38,13 @@ on: description: 'Branch whose runs form the baseline' required: false default: 'main' + ios: + description: 'Run the iOS suite' + type: boolean + required: false + default: true android: - description: 'Also run the report-only Android Settings tour' + description: 'Run the Android suite' type: boolean required: false default: true @@ -58,6 +64,7 @@ env: jobs: perf-ios: name: iOS Command Perf + if: ${{ github.event_name == 'schedule' || github.event.inputs.ios == 'true' }} runs-on: macos-26 timeout-minutes: 100 env: @@ -133,7 +140,7 @@ jobs: BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }} BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }} run: | - pnpm perf:compare --to perf-results \ + pnpm perf:compare --platform ios --to perf-results \ --branch "$BASELINE_BRANCH" \ --nights "$BASELINE_NIGHTS" \ --exclude-run "$GITHUB_RUN_ID" \ @@ -164,26 +171,44 @@ jobs: perf-android: name: Android Command Perf - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }} + if: ${{ github.event_name == 'schedule' || github.event.inputs.android == 'true' }} runs-on: ubuntu-latest - timeout-minutes: 80 - env: - AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' + timeout-minutes: 100 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + + - name: Build the shared CLI + uses: ./.github/actions/run-gate + with: + gate: build - name: Setup Android replay host - id: android-replay-host uses: ./.github/actions/setup-android-replay-host with: package-helpers: 'true' - - name: Run Android command perf benchmark + - name: Restore the fixture APK + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + platform: android + install: 'false' + + # The emulator action runs each script line on its own, so the script is one folded line. + - name: Run Android command perf suite uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 + env: + APP_ID: ${{ steps.fixture-app.outputs.app-id }} + APP_PATH: ${{ steps.fixture-app.outputs.apk-path }} + SCENARIOS: ${{ github.event.inputs.scenarios }} with: api-level: 36 arch: x86_64 @@ -192,15 +217,45 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics script: >- set -e; - adb -s emulator-5554 shell settings put global window_animation_scale 0 || true; - adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true; - adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true; - pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results" + adb -s emulator-5554 install -r "$APP_PATH"; + pnpm perf --platform android --serial emulator-5554 + --app-id "$APP_ID" --app-path "$APP_PATH" + --rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" + ${SCENARIOS:+--scenarios "$SCENARIOS"} + --out-dir "$GITHUB_WORKSPACE/perf-results" + + - name: Compare with earlier nights + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} + env: + GH_TOKEN: ${{ github.token }} + BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }} + BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }} + run: | + pnpm perf:compare --platform android --to perf-results \ + --branch "$BASELINE_BRANCH" \ + --nights "$BASELINE_NIGHTS" \ + --exclude-run "$GITHUB_RUN_ID" \ + --out perf-results --github + echo "Full report: the perf-android-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Android perf summary + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: perf-android-summary + path: | + perf-results/summary.json + perf-results/report.md + perf-results/comparison.json + perf-results/comparison.md + retention-days: 90 + if-no-files-found: error - - name: Upload Android perf report - if: always() + - name: Upload Android perf raw logs + if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: perf-android - path: perf-results/ + name: perf-android-raw + path: perf-results/raw/ + retention-days: 30 if-no-files-found: warn diff --git a/packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts new file mode 100644 index 0000000000..7f314f6b7b --- /dev/null +++ b/packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts @@ -0,0 +1,74 @@ +import { afterEach, beforeEach, test } from 'vitest'; +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { withDiagnosticsScope } from '@agent-device/host-kit/diagnostics'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE as ONE_SHOT_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE as SESSION_PHASE, + snapshotAndroid, +} from '../snapshot.ts'; +import { resetAndroidSnapshotHelperInstallCache } from '../snapshot-helper-install.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; +import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts'; +import { createPersistentSnapshotHelperProvider } from './snapshot-helper-session.fixtures.ts'; +import { mkdtempForTest } from './test-utils/tmp-dir.ts'; + +const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +}; + +beforeEach(async () => { + await resetAndroidSnapshotHelperSessions(); + resetAndroidSnapshotHelperInstallCache(); +}); + +afterEach(async () => { + await resetAndroidSnapshotHelperSessions(); +}); + +async function captureTimers(sessionResponseMode?: 'malformed') { + const provider = createPersistentSnapshotHelperProvider({ + calls: [], + spawnArgs: [], + processes: [], + sessionResponseMode, + oneShotXml: '', + }); + const tmpDir = await mkdtempForTest('agent-device-android-capture-timers-'); + const logPath = path.join(tmpDir, 'diag.ndjson'); + try { + await withDiagnosticsScope( + { debug: true, logPath, session: 'android-test', requestId: 'req-1', command: 'snapshot' }, + async () => + await snapshotAndroid(device, { + helperAdb: provider, + helperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + }), + ); + return (await fs.readFile(logPath, 'utf8')) + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { phase: string; data?: Record }) + .filter(({ phase }) => phase === SESSION_PHASE || phase === ONE_SHOT_PHASE) + .map(({ phase, data }) => ({ phase, transport: data?.transport, answered: data?.answered })); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +} + +test('each helper capture timer records its transport and whether the session answered', async () => { + assert.deepEqual(await captureTimers(), [ + { phase: SESSION_PHASE, transport: 'persistent-session', answered: true }, + ]); + await resetAndroidSnapshotHelperSessions(); + assert.deepEqual(await captureTimers('malformed'), [ + { phase: SESSION_PHASE, transport: 'persistent-session', answered: false }, + { phase: ONE_SHOT_PHASE, transport: 'instrumentation', answered: undefined }, + ]); +}); diff --git a/packages/platform-android/src/mechanics.ts b/packages/platform-android/src/mechanics.ts index 993c017204..f4098ec4a6 100644 --- a/packages/platform-android/src/mechanics.ts +++ b/packages/platform-android/src/mechanics.ts @@ -308,7 +308,12 @@ export { resetAndroidSnapshotHelperRuntime, retireAndroidSnapshotHelperAfterContentFailure, } from './snapshot-helper-runtime.ts'; -export { snapshotAndroid, type AndroidSnapshotOptions } from './snapshot.ts'; +export { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + snapshotAndroid, + type AndroidSnapshotOptions, +} from './snapshot.ts'; export { screenshotAndroid, type AndroidScreenshotOptions } from './screenshot.ts'; export { ANDROID_SNAPSHOT_PRESENTATION_DEFAULT_DEADLINE_MS, @@ -337,6 +342,7 @@ export { ensureAndroidSdkPathConfigured } from './sdk.ts'; export { fillAndroid, typeAndroid } from './text-input.ts'; export { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; export { + ANDROID_TOUCH_HELPER_GESTURE_PHASE, ANDROID_TOUCH_PLAN_PROTOCOL, executeAndroidTouchHelperPlan, normalizeAndroidTouchHelperGestureRequest, diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index ddbf8f47ec..31f9e3d75e 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -36,6 +36,7 @@ import { ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, type AndroidHelperSessionScope, type AndroidSnapshotHelperCaptureOptions, + type AndroidSnapshotHelperTransport, } from './snapshot-helper-types.ts'; import { captureAndroidSnapshotWithHelper, @@ -102,6 +103,19 @@ const HELPER_CONTENT_RECAPTURE_DELAY_MS = 250; * a busy screen run a snapshot past the daemon request envelope. */ const HELPER_CONTENT_RECAPTURE_WINDOW_MS = 10_000; +/** + * The diagnostic phase of a capture through the snapshot helper's persistent session. Its + * `data.transport` names the session transport, and `data.answered` is false when the session + * returned no capture (off, backed off, failed to start, or retired) and a one-shot capture follows. + */ +export const ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE = + 'android_snapshot_helper_session_capture'; +/** + * The diagnostic phase of a one-shot `am instrument` snapshot helper capture. Its `data.transport` + * names the instrumentation transport. + */ +export const ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE = 'android_snapshot_helper_capture'; + export type AndroidSnapshotOptions = SnapshotOptions & { idleWait?: AndroidSnapshotHelperCaptureOptions['idleWait']; appBundleId?: string; @@ -463,15 +477,18 @@ async function captureAndroidUiHierarchyFromHelper(params: { signal, }); captureOptions.idleWait = params.idleWait; + let sessionCapture: AndroidSnapshotHelperOutput | undefined; try { - const sessionCapture = await withDiagnosticTimer( - 'android_snapshot_helper_session_capture', - async () => await captureAndroidSnapshotWithHelperSession(captureOptions), - { + sessionCapture = await withDiagnosticTimer( + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + async () => (sessionCapture = await captureAndroidSnapshotWithHelperSession(captureOptions)), + () => ({ packageName: artifact.manifest.packageName, version: artifact.manifest.version, timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, - }, + transport: 'persistent-session' satisfies AndroidSnapshotHelperTransport, + answered: sessionCapture !== undefined, + }), ); if (sessionCapture) return sessionCapture; } catch (error) { @@ -487,13 +504,14 @@ async function captureAndroidUiHierarchyFromHelper(params: { await resetAndroidSnapshotHelperRuntime(adb, artifact.manifest.packageName); } return await withDiagnosticTimer( - 'android_snapshot_helper_capture', + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, async () => await captureAndroidSnapshotWithHelper(captureOptions), { packageName: artifact.manifest.packageName, version: artifact.manifest.version, timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, commandTimeoutMs: ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, + transport: 'instrumentation' satisfies AndroidSnapshotHelperTransport, }, ); } diff --git a/packages/platform-android/src/touch-helper.ts b/packages/platform-android/src/touch-helper.ts index 3ad1c7b616..196d0e136a 100644 --- a/packages/platform-android/src/touch-helper.ts +++ b/packages/platform-android/src/touch-helper.ts @@ -33,6 +33,8 @@ import { } from './snapshot-helper-types.ts'; export const ANDROID_TOUCH_PLAN_PROTOCOL = 'android-touch-plan-v1'; +/** The diagnostic phase of one gesture that the snapshot helper injects. */ +export const ANDROID_TOUCH_HELPER_GESTURE_PHASE = 'android_touch_helper_gesture'; const HELPER_GESTURE_TIMEOUT_MS = 45_000; const HELPER_GESTURE_TIMEOUT_OVERHEAD_MS = 15_000; const HELPER_VIEWPORT_TIMEOUT_MS = 45_000; @@ -67,7 +69,7 @@ export async function executeAndroidTouchHelperPlan( JSON.stringify({ protocol: ANDROID_TOUCH_PLAN_PROTOCOL, ...request }), ).toString('base64'); const output = await withDiagnosticTimer( - 'android_touch_helper_gesture', + ANDROID_TOUCH_HELPER_GESTURE_PHASE, async () => { // A live snapshot session already owns UiAutomation; injecting through it avoids the // instrumentation stop/restart that previously taxed every gesture. Gesture session errors diff --git a/scripts/perf/README.md b/scripts/perf/README.md index b53c2e1725..5ea3867642 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -1,27 +1,74 @@ # Command performance suite -`pnpm perf --platform ios` times every interaction command against the fixture app (`examples/test-app`). -The steps are in `ios-suite.ts`. Each step belongs to one scenario. +`pnpm perf` times every interaction command against the fixture app (`examples/test-app`) on iOS and on Android. +The steps are in `suite.ts`. Each step belongs to one scenario. Both platforms run the same scenarios. +A step runs on both platforms unless it names one platform. A step can replace its arguments, waits, or expectations on one platform. Each scenario starts from an untimed setup in every round. -The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels. -The suite records a reason for each catalog command that it does not time. `ios-suite.test.ts` keeps that list complete. -`pnpm perf --platform android` runs the older report-only Settings tour. +The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels on iOS and 60 levels on Android. +For each platform, the suite records a reason for each catalog command that it does not time. `suite.test.ts` keeps those lists complete. ## Run locally 1. Run `pnpm build`. The suite drives the built CLI and the built SDK. -2. Use your own booted simulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`. -3. Set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree. -4. Run the suite: +2. Use your own booted simulator or emulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`. +3. For iOS, set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree. +4. For Android, run `pnpm build:android` once. The snapshot helper APK must be in `android/snapshot-helper/dist/`. +5. Run the suite: ```sh -pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/local +pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/ios +pnpm perf --platform android --serial --rounds 3 --warmup 1 --out-dir .tmp/perf/android ``` -`--rounds` is at least 3. `--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. +`--rounds` is at least 3. `--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id or package name. +To install the cached Android Release fixture, use an empty download directory: + +```sh +pnpm test-app:install +NAME="$(sh .github/actions/setup-fixture-app/resolve-artifact-name.sh android)" +RUN="$(gh api "repos/callstack/agent-device/actions/artifacts?name=$NAME" --jq '.artifacts[0].workflow_run.id')" +gh run download "$RUN" --repo callstack/agent-device --name "$NAME" --dir +tar -xzf /binary.tar.gz -C +ANDROID_HOME= bash .github/actions/setup-fixture-app/repack-android-apk.sh /app-release.apk /fixture-app.apk +adb -s install -r /fixture-app.apk +``` + +The repack puts the current JavaScript into the cached native build. +If no artifact has the current fingerprint, build the APK as `.github/workflows/test-app-build-cache.yml` does. + The suite uses its own temporary state directory and daemon. It stops them at the end. At the end, or on SIGINT or SIGTERM, the suite saves the logs to `raw/state/`, stops the session and daemon within a deadline, and saves the logs again. +## Snapshot paths + +Each platform times the normal snapshot path and one forced path. + +- iOS: the simulator accessibility bridge answers normal captures. `home-reads/snapshot-forced-tree` pins the XCTest tree backend through the SDK, because the CLI has no flag for it. Only a regular capture honors the pin; a raw capture runs its own backend ladder. The response must name the `tree` backend with the `requested-backend` reason code. The XCTest tree rejects the heavy feed, so the step runs on the home screen. +- Android: the snapshot helper APK is the only snapshot engine (ADR 0002). Its persistent session answers normal captures. `snapshot-raw-one-shot` restarts the daemon with `AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION=0`, so a one-shot `am instrument` capture answers. The harness restarts the daemon without the variable after the step. Both raw captures must reach depth 56. The forced capture must report the `instrumentation` transport. + +## Platform parity + +These commands differ between the platforms. All other catalog commands run the same steps on both platforms. +Some shared steps use other screen coordinates or waits on Android. The `on.android` fields in `suite.ts` list them. + +| Command | iOS | Android | +| --- | --- | --- | +| `action-button` | lifecycle | Not timed: Android has no key event for the Action button. | +| `prepare` | lifecycle (`prepare ios-runner`) | Not timed: `prepare` has only the `ios-runner` action. | +| `clipboard` | lifecycle ×2 | Not timed: the API 35 and 36 emulator images have no shell clipboard service. | +| `hardware-button` | Not timed: the simulator refuses hardware-button presses. | lifecycle (`hardware-button wake`) | +| `push` | Not timed: the fixture has no notification authorization. | lifecycle (adb broadcast) | +| `telephony` | Not timed: the simulator refuses calls and SMS. | lifecycle (`telephony sms`) | +| `snapshot` | home-reads: `snapshot-forced-tree` | heavy-feed: `snapshot-raw-one-shot` | +| `audio` | Not timed: hosted runners grant no macOS audio permission. | Not timed: the Linux runner has no ScreenCaptureKit. | +| `fold` | Not timed: iPhone Duo only. | Not timed: foldable emulators only. | +| `tv-remote` | Not timed: tvOS only. | Not timed: Android TV only. | +| `hover`, `viewport` | Not timed: web targets only. | Not timed: web targets only. | +| `boot`, `shutdown`, `install`, `reinstall`, `uninstall`, `install-from-source` | Not timed: provisioning time dominates, or the command ends the session under test. | Same reason. | +| `replay`, `test` | Not timed: the sum of the commands that they run. | Same reason. | + +Each run lists the measured commands and every reason in the Coverage section of `report.md`. + ## Output - `summary.json`: the machine-readable run (`agent-device-perf-summary/v1`). It holds the metadata, the coverage, and every sample. @@ -39,39 +86,43 @@ All times are milliseconds. Statistics use only the successful samples of a step | `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture (`home-reads/snapshot-forced-tree`, marked SDK in the report), the span of its request log. Comparisons use this metric. | | `mean ± sd`, `cv` | The mean, the sample standard deviation, and the standard deviation divided by the mean. | | `median`, `p95`, `min`, `max` | The median (the mean of the two middle values for an even count), the nearest-rank 95th percentile, and the extremes. | -| `runner` | The median sum of XCTest runner command times in one request. | -| `spawns` | The median count and time of host processes that one request started. | -| `snapshot source` | The median time of host accessibility-bridge acquisitions in one request. | +| `runner` | The median sum of XCTest runner command times (iOS) or snapshot-helper gesture times (Android) in one request. | +| `spawns` | The median count and time of host processes, for example `xcrun` or `adb`, that one request started. | +| `snapshot source` | The median time of host accessibility-bridge acquisitions (iOS) or snapshot-helper captures (Android) in one request. | | `settle wait` | The median `settle.waitedMs` that a `--settle` response reports. | | `stabilization` | The median time of post-gesture stabilization phases in one request. | | `retries` | The failed attempts of retried operations in all successful samples. | | `top phases` | The five phases with the largest median time per request. | -| `producer` | The number of samples that each snapshot producer answered (`simulator-ax-bridge`, `apple-runner`). | -| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | +| `producer` | The number of samples that each snapshot producer answered: `simulator-ax-bridge` and `apple-runner` on iOS; `android-helper/persistent-session` and `android-helper/instrumentation` (one-shot `am instrument`) on Android. | +| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected` or `android-helper/healthy`. | | fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | -| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that a backend other than the pinned one served the pinned capture. | +| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `restore_failed` means that the daemon restart after a forced-path step could not reopen the session. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that the forced path did not serve the capture. | ## Nightly results -`.github/workflows/perf-nightly.yml` runs the iOS suite every night on `main`. -Each run uploads `perf-ios-summary` (summary, report, comparison; 90 days) and `perf-ios-raw` (logs; 30 days). -The job compares the run with up to seven earlier nights and writes the result to the job summary. +`.github/workflows/perf-nightly.yml` runs the iOS suite and the Android suite every night on `main`. +iOS runs on a `macos-26` simulator. Android runs on an API 36 `pixel_7` emulator on a Linux KVM runner. +For each platform, a run uploads `perf--summary` (summary, report, comparison; 90 days) and `perf--raw` (logs; 30 days). +Each job compares its run with up to seven earlier nights of the same platform and writes the result to the job summary. A night is the newest scheduled run of one UTC day that ran every scenario of the judged run. Manual dispatches are never part of the rolling baseline. To fetch and compare runs on your machine, use `gh` with access to the repository: ```sh -pnpm perf:compare # latest night against the 7 nights before it +pnpm perf:compare # latest iOS night against the 7 nights before it +pnpm perf:compare --platform android # the same for Android pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it pnpm perf:compare --to latest --from 2026-10-21 # latest against the newest run of that day pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare # every run in the range -pnpm perf:compare --to .tmp/perf/local # a local run against recent nights +pnpm perf:compare --to .tmp/perf/ios # a local run against recent nights +pnpm perf:compare --platform android --to .tmp/perf/android ``` A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. An explicit `--to` or `--from` reference accepts runs of any trigger, so it can compare dispatches and local runs. -Downloads stay in `.tmp/perf-nightly//`. +`--platform` (default `ios`) selects the artifact. Every compared run must be of that platform. +Downloads stay in `.tmp/perf-nightly///`. Do not judge a local run against CI nights. The comparison warns when the device, Xcode, or CPU differs. ## Comparison rule diff --git a/scripts/perf/cli.ts b/scripts/perf/cli.ts index cada28867d..e7ad1889b3 100644 --- a/scripts/perf/cli.ts +++ b/scripts/perf/cli.ts @@ -1,7 +1,6 @@ import { performance } from 'node:perf_hooks'; import { runCmdSync } from '@agent-device/host-kit/command'; import { resolveCliArgv, REPO_ROOT } from './config.ts'; -import type { BatchStepSpec } from './scenario.ts'; import type { CliResult } from './types.ts'; const MAX_BUFFER = 64 * 1024 * 1024; @@ -36,7 +35,7 @@ function jsonOk(json: unknown): boolean { export function invokeCli( args: string[], baseFlags: string[], - options: { timeoutMs?: number } = {}, + options: { timeoutMs?: number; env?: Readonly> } = {}, ): CliResult { const full = [...CLI_ARGV, ...args, ...baseFlags, '--json']; const t0 = performance.now(); @@ -51,6 +50,7 @@ export function invokeCli( maxBuffer: MAX_BUFFER, allowFailure: true, timeoutMs: options.timeoutMs, + env: options.env ? { ...process.env, ...options.env } : undefined, }); stdout = r.stdout; stderr = r.stderr; @@ -63,42 +63,3 @@ export function invokeCli( const json = tryParseJson(stdout); return { exitCode, wallClockMs, stdout, stderr, json, ok: exitCode === 0 && jsonOk(json) }; } - -// Wrap a single command in its own `batch` invocation to read per-step durationMs. -export function invokeBatchStep(spec: BatchStepSpec, baseFlags: string[]): CliResult { - const result = invokeCli(['batch', '--steps', JSON.stringify([spec])], baseFlags); - // Defensive: today's stop-only batch surfaces a failed step as a top-level non-zero/ok:false - // (already caught by invokeCli). But if a future on-error mode keeps the batch ok while a step - // fails, don't silently count that step as a success — downgrade ok from the step's own ok. - const stepOk = firstBatchResult(result.json)?.ok; - if (result.ok && stepOk === false) { - return { ...result, ok: false }; - } - return result; -} - -function firstBatchResult(json: unknown): Record | undefined { - const data = (json as { data?: { results?: unknown[] } } | undefined)?.data; - const first = data?.results?.[0]; - return first && typeof first === 'object' ? (first as Record) : undefined; -} - -export function readBatchStepDurationMs(result: CliResult): number | undefined { - const v = firstBatchResult(result.json)?.durationMs; - return typeof v === 'number' ? v : undefined; -} - -export function readBatchStepError(result: CliResult): { code?: string; message?: string } { - const err = (result.json as { error?: { code?: string; message?: string } } | undefined)?.error; - return { code: err?.code, message: err?.message }; -} - -// Proxy for a11y-tree size: snapshot node count (falls back to distinct @eN refs). -export function countElements(result: CliResult): number | undefined { - const stepData = firstBatchResult(result.json)?.data; - if (stepData === undefined || typeof stepData !== 'object') return undefined; - const nodes = (stepData as { nodes?: unknown }).nodes; - if (Array.isArray(nodes)) return nodes.length; - const matches = JSON.stringify(stepData).match(/@e\d+/g); - return matches ? new Set(matches).size : 0; -} diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts index 3b3593acfe..a638c97ce9 100644 --- a/scripts/perf/compare-cli.ts +++ b/scripts/perf/compare-cli.ts @@ -10,19 +10,21 @@ import { resolveRunRef, selectNightlyBaseline, selectRange, - SUMMARY_ARTIFACT, + summaryArtifact, type NightlySource, type WorkflowRun, } from './nightly-artifacts.ts'; -import type { PerfSummary } from './types.ts'; +import type { PerfSummary, Platform } from './types.ts'; -const USAGE = `Usage: pnpm perf:compare [--to ] [--from |..] [--nights ] [options] +const USAGE = `Usage: pnpm perf:compare [--platform ios|android] [--to ] [--from |..] [--nights ] [options] Compares one perf run with earlier runs and prints a Markdown verdict per step. A is latest, a workflow run id, a YYYY-MM-DD date (the newest run that UTC day), or a local path to a summary.json or a \`pnpm perf\` output directory. + --platform ios or android (default: ios). Nightly runs are read from + perf--summary; every compared run must be of this platform. --to Run to judge (default: latest). --from Baseline run, or an inclusive range .. of runs of any trigger. Without it the baseline is the --nights scheduled nights before --to. @@ -36,6 +38,7 @@ path to a summary.json or a \`pnpm perf\` output directory. --github Append the Markdown to $GITHUB_STEP_SUMMARY and annotate flagged steps.`; type Options = { + platform: Platform; to: string; from?: string; nights: number; @@ -47,6 +50,7 @@ type Options = { function parse(argv: string[]): Options { const options: Options = { + platform: 'ios', to: 'latest', nights: 7, github: false, @@ -55,7 +59,7 @@ function parse(argv: string[]): Options { repo: process.env.GITHUB_REPOSITORY ?? 'callstack/agent-device', workflow: 'perf-nightly.yml', branch: 'main', - artifact: SUMMARY_ARTIFACT, + artifact: summaryArtifact('ios'), cacheDir: path.join(REPO_ROOT, '.tmp', 'perf-nightly'), }, }; @@ -67,6 +71,14 @@ function parse(argv: string[]): Options { return next; }; switch (flag) { + case '--platform': { + const platform = value(); + if (platform !== 'ios' && platform !== 'android') + throw new Error('--platform must be ios or android'); + options.platform = platform; + options.source.artifact = summaryArtifact(platform); + break; + } case '--to': options.to = value(); break; @@ -122,14 +134,17 @@ function main(): void { const load = (ref: string): Loaded => { if (fs.existsSync(ref)) return { summary: readLocalSummary(ref), label: path.basename(path.resolve(ref)) }; - const run = resolveRunRef(ref, allRuns()); + const summaries = new Map(); + const run = resolveRunRef(ref, allRuns(), (candidate) => { + const summary = fetchSummary(options.source, candidate); + if (summary) summaries.set(candidate.id, summary); + return summary !== undefined; + }); if (!run) throw new Error( - `No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref}.`, + `No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref} with a ${options.source.artifact} artifact.`, ); - const summary = fetchSummary(options.source, run); - if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`); - return { summary, label: runLabel(run), run }; + return { summary: summaries.get(run.id)!, label: runLabel(run), run }; }; const loadRuns = (candidates: WorkflowRun[]): Loaded[] => candidates.flatMap((run) => { @@ -138,6 +153,11 @@ function main(): void { }); const target = load(options.to); + if (target.summary.platform !== options.platform) { + throw new Error( + `${target.label} is an ${target.summary.platform} run; pass --platform ${target.summary.platform}.`, + ); + } const exclude = new Set([...options.exclude, ...(target.run ? [target.run.id] : [])]); let baseline: Loaded[]; if (options.from?.includes('..')) { diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts index 79aec9562d..f9c8a4ccad 100644 --- a/scripts/perf/compare.test.ts +++ b/scripts/perf/compare.test.ts @@ -128,3 +128,19 @@ test('warns when the baseline ran on another toolchain', () => { assert.equal(comparison.warnings.length, 1); assert.match(comparison.warnings[0]!, /xcode/); }); + +test('refuses a baseline run from another platform', () => { + const android = summaryFixture(flat(1000), { platform: 'android' }); + assert.throws( + () => + compareRuns({ summary: summaryFixture(flat(1000)), label: 'tonight' }, [ + ...nights(flat(1000), flat(1000)), + { summary: android, label: 'android night' }, + ]), + /An ios run cannot be compared with android night \(android\)/, + ); + const comparison = compareRuns({ summary: android, label: 'tonight' }, [ + { summary: summaryFixture(flat(1000), { platform: 'android' }), label: 'night' }, + ]); + assert.equal(comparison.platform, 'android'); +}); diff --git a/scripts/perf/compare.ts b/scripts/perf/compare.ts index 22b1f28a61..4d9289e5a7 100644 --- a/scripts/perf/compare.ts +++ b/scripts/perf/compare.ts @@ -1,6 +1,6 @@ import { MIN_MEASURED_ROUNDS } from './config.ts'; import { median, scaledMad } from './stats.ts'; -import type { Measurement, PerfSummary, Stat } from './types.ts'; +import type { Measurement, PerfSummary, Platform, Stat } from './types.ts'; export type ComparisonThresholds = { // A change must exceed every one of these to be flagged. @@ -78,6 +78,7 @@ const COMPARISON_SCHEMA = 'agent-device-perf-comparison/v1'; export type Comparison = { schema: typeof COMPARISON_SCHEMA; + platform: Platform; target: RunRef; baseline: RunRef[]; thresholds: ComparisonThresholds; @@ -216,13 +217,21 @@ function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): stri * median of its nightly medians, and its noise is the scaled MAD of those medians. A step is flagged * only when its change exceeds the relative floor, the night-to-night spread, tonight's own * sampling error, and an absolute floor together. Tonight and every night that counts need - * `minSamples` successful samples. Failures are judged apart from timing. + * `minSamples` successful samples. Failures are judged apart from timing. Every baseline run must + * come from the platform of the target. */ export function compareRuns( target: { summary: PerfSummary; label: string }, baseline: { summary: PerfSummary; label: string }[], thresholds: ComparisonThresholds = DEFAULT_THRESHOLDS, ): Comparison { + const platform = target.summary.platform; + const foreign = baseline.filter((b) => b.summary.platform !== platform); + if (foreign.length > 0) { + throw new Error( + `An ${platform} run cannot be compared with ${foreign.map((b) => `${b.label} (${b.summary.platform})`).join(', ')}.`, + ); + } const targetById = new Map(target.summary.measurements.map((m) => [m.id, m])); const ids = [ ...target.summary.measurements.map((m) => m.id), @@ -243,6 +252,7 @@ export function compareRuns( } return { schema: COMPARISON_SCHEMA, + platform, target: describeRun(target.summary, target.label), baseline: baseline.map((b) => describeRun(b.summary, b.label)), thresholds, @@ -297,7 +307,7 @@ export function renderComparison(comparison: Comparison): string { .filter((row) => flagRank(row) >= 0) .sort((a, b) => flagRank(a) - flagRank(b)); const lines = [ - `## agent-device perf comparison`, + `## agent-device perf comparison — ${comparison.platform}`, '', `- **Tonight**: ${comparison.target.label} · ${comparison.target.commit.slice(0, 12)} · ${comparison.target.startedAt} · ${comparison.target.device}`, `- **Baseline**: ${comparison.baseline.length === 0 ? 'none' : comparison.baseline.map((b) => `${b.label} (${b.commit.slice(0, 7)})`).join(', ')}`, diff --git a/scripts/perf/config.test.ts b/scripts/perf/config.test.ts index d8bb3b363a..4e14d554bc 100644 --- a/scripts/perf/config.test.ts +++ b/scripts/perf/config.test.ts @@ -5,5 +5,17 @@ import { parseConfig } from './config.ts'; test('a run measures at least three rounds, so every step can get a timing verdict', () => { assert.equal(parseConfig(['--udid', 'sim', '--rounds', '3']).rounds, 3); assert.throws(() => parseConfig(['--udid', 'sim', '--rounds', '2']), /--rounds must be .* >= 3/); - assert.throws(() => parseConfig(['--platform', 'android', '--n', '1']), /--n must be .* >= 3/); + assert.throws( + () => parseConfig(['--platform', 'android', '--serial', 'emulator-5554', '--rounds', '1']), + /--rounds must be .* >= 3/, + ); +}); + +test('each platform requires its device selector', () => { + assert.throws(() => parseConfig(['--platform', 'ios']), /--udid is required/); + assert.throws(() => parseConfig(['--platform', 'android']), /--serial is required/); + assert.equal( + parseConfig(['--platform', 'android', '--serial', 'emulator-5554']).serial, + 'emulator-5554', + ); }); diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index 32623a8ebf..7c2bf3de3e 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -11,35 +11,28 @@ export type PerfConfig = { platform: Platform; rounds: number; // measured rounds (samples per command) warmup: number; // leading rounds dropped from stats - keepArtifacts: boolean; // keep temp state dir + leave device booted + keepArtifacts: boolean; // keep the temporary state dir outDir: string; udid?: string; // iOS simulator UDID (required for iOS) - device?: string; // Android AVD name for boot - serial?: string; // Android device override - appId: string; // iOS fixture bundle id - appPath?: string; // iOS fixture bundle path, recorded in the run metadata - scenarios?: string[]; // iOS scenario ids; all when omitted + serial?: string; // Android emulator serial (required for Android) + appId: string; // fixture bundle id or package name + appPath?: string; // fixture .app or .apk path, recorded in the run metadata + scenarios?: string[]; // scenario ids; all when omitted }; -export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab'; +export const DEFAULT_APP_ID = 'com.callstack.agentdevicelab'; /** The fewest measured rounds that a run accepts; a comparison needs as many successful samples. */ export const MIN_MEASURED_ROUNDS = 3; // How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs). -// Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI: -// AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts" -// (matches the device workflows, which run from source and skip the dist build). +// AGENT_DEVICE_PERF_CLI replaces it, for example with a fake CLI in tests. export function resolveCliArgv(): string[] { const override = process.env.AGENT_DEVICE_PERF_CLI?.trim(); if (override) return override.split(/\s+/); return [CLI_BIN]; } -export function usesSourceCli(): boolean { - return Boolean(process.env.AGENT_DEVICE_PERF_CLI?.trim()); -} - function readValue(argv: string[], i: number, flag: string): string { const v = argv[i + 1]; if (v === undefined) throw new Error(`Missing value for ${flag}`); @@ -62,7 +55,7 @@ export function parseConfig(argv: string[]): PerfConfig { warmup: 1, keepArtifacts: false, outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')), - appId: DEFAULT_IOS_APP_ID, + appId: DEFAULT_APP_ID, }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -73,7 +66,6 @@ export function parseConfig(argv: string[]): PerfConfig { cfg.platform = v; break; } - case '--n': case '--rounds': cfg.rounds = readIntValue(argv, i++, a, MIN_MEASURED_ROUNDS); break; @@ -89,9 +81,6 @@ export function parseConfig(argv: string[]): PerfConfig { case '--udid': cfg.udid = readValue(argv, i++, a); break; - case '--device': - cfg.device = readValue(argv, i++, a); - break; case '--serial': cfg.serial = readValue(argv, i++, a); break; @@ -112,5 +101,7 @@ export function parseConfig(argv: string[]): PerfConfig { } } if (cfg.platform === 'ios' && !cfg.udid) throw new Error('--udid is required for --platform ios'); + if (cfg.platform === 'android' && !cfg.serial) + throw new Error('--serial is required for --platform android'); return cfg; } diff --git a/scripts/perf/harness.ts b/scripts/perf/harness.ts deleted file mode 100644 index 09063440a7..0000000000 --- a/scripts/perf/harness.ts +++ /dev/null @@ -1,193 +0,0 @@ -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { - countElements, - invokeBatchStep, - invokeCli, - readBatchStepDurationMs, - readBatchStepError, -} from './cli.ts'; -import type { PerfConfig } from './config.ts'; -import { resolveProfile, type ResolvedProfile } from './platform-profiles.ts'; -import { buildMeasurement } from './measurement.ts'; -import { buildSettingsTour, type ScenarioStep } from './scenario.ts'; -import type { CliResult, Measurement, Sample } from './types.ts'; - -export type IsolationContext = { - stateDir: string; - artifactsDir: string; - baseFlags: string[]; - profile: ResolvedProfile; -}; - -function log(msg: string): void { - process.stderr.write(`[perf] ${msg}\n`); -} - -export function setupIsolation(cfg: PerfConfig): IsolationContext { - const profile = resolveProfile(cfg); - const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-perf-')); - const artifactsDir = path.join(stateDir, 'artifacts'); - fs.mkdirSync(artifactsDir, { recursive: true }); - const baseFlags = ['--state-dir', stateDir, '--session', 'perf', ...profile.platformFlags]; - log(`state-dir: ${stateDir}`); - log(`device: ${profile.deviceName} (${profile.serial ?? 'default'})`); - return { stateDir, artifactsDir, baseFlags, profile }; -} - -export function teardownIsolation(ctx: IsolationContext, cfg: PerfConfig): void { - log('teardown: closing session'); - try { - const args = ['close']; - if (!cfg.keepArtifacts) args.push('--shutdown'); - invokeCli(args, ctx.baseFlags); - } catch { - /* best-effort */ - } - if (cfg.keepArtifacts) { - log(`keep-artifacts: leaving ${ctx.stateDir} and device in place`); - return; - } - try { - fs.rmSync(ctx.stateDir, { recursive: true, force: true }); - log('teardown: removed temp state dir'); - } catch { - /* best-effort */ - } -} - -function sampleError(r: CliResult): Pick { - const err = readBatchStepError(r); - return { - errorCode: err.code ?? `exit:${r.exitCode}`, - errorMessage: (err.message ?? r.stderr.trim().split('\n').pop() ?? '').slice(0, 200), - }; -} - -// Base sample (timing + ok + error note on failure) shared by every measured invocation. -function toSample(r: CliResult, round: number): Sample { - const sample: Sample = { round, wallClockMs: r.wallClockMs, ok: r.ok }; - if (!r.ok) Object.assign(sample, sampleError(r)); - return sample; -} - -// The first interaction after open/relaunch pays the Android helper startup and a per-relaunch -// first-query settle cost. Run an untimed throwaway interaction so that -// cost is never attributed to a measured command. -function warmRunner(ctx: IsolationContext): void { - invokeCli(['snapshot', '-i'], ctx.baseFlags); -} - -function runStep(step: ScenarioStep, ctx: IsolationContext, round: number): Sample { - // Untimed reset to root for steps whose precondition is a clean, top-of-list root. - if (step.freshRoot) { - invokeCli(['open', ctx.profile.appTarget, '--relaunch'], ctx.baseFlags); - warmRunner(ctx); - } - const r = - step.execMode === 'standalone' - ? invokeCli(step.args, ctx.baseFlags) - : invokeBatchStep(step.step, ctx.baseFlags); - const sample = toSample(r, round); - if (step.execMode === 'batch') { - sample.daemonMs = readBatchStepDurationMs(r); - if (step.isSnapshot) sample.elementCount = countElements(r); - } - return sample; -} - -const TOUR_SCENARIO = 'settings-tour'; - -function tourMeasurement( - step: Pick, - platform: ResolvedProfile['platform'], - samples: Sample[], - warmupDropped: number, -): Measurement { - return buildMeasurement({ - id: `${TOUR_SCENARIO}/${step.label}`, - scenario: TOUR_SCENARIO, - label: step.label, - command: step.command, - args: [], - platform, - execMode: step.execMode, - samples, - warmupDropped, - }); -} - -function setupMeasurementFailed(measurement: Measurement): boolean { - return measurement.samples.some((sample) => !sample.ok); -} - -// Boot the device once and time it. Runs WITHOUT --session so no session lock policy -// applies and the device selectors are honored (selectors are rejected on locked sessions). -function bootOnce(ctx: IsolationContext): Measurement { - log('booting device (no session lock; sampled once)'); - const bootFlags = ['--state-dir', ctx.stateDir, ...ctx.profile.platformFlags]; - const r = invokeCli(['boot', ...ctx.profile.selectorFlags], bootFlags); - const sample = toSample(r, 0); - return tourMeasurement( - { command: 'boot', label: 'boot device', execMode: 'standalone' }, - ctx.profile.platform, - [sample], - 0, - ); -} - -// Establish the session by opening Settings WITH device selectors (open is the only -// interaction command allowed to carry selectors on a fresh session). Locks the session -// to our device so every later call targets it via --session alone. -function establishSession(ctx: IsolationContext): Measurement { - log('establishing session (open with device selectors)'); - const r = invokeCli(['open', ctx.profile.appTarget, ...ctx.profile.selectorFlags], ctx.baseFlags); - const sample = toSample(r, 0); - return tourMeasurement( - { command: 'open', label: 'open (establish + cold)', execMode: 'standalone' }, - ctx.profile.platform, - [sample], - 0, - ); -} - -export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement[] { - const steps = buildSettingsTour(ctx.profile, { artifactsDir: ctx.artifactsDir }); - const acc = new Map(); - for (const step of steps) acc.set(step.label, []); - - const boot = bootOnce(ctx); - const establish = establishSession(ctx); - if (setupMeasurementFailed(boot) || setupMeasurementFailed(establish)) { - log('setup failed; skipping timed tour'); - return [boot, establish]; - } - // Absorb the one-time runner startup before any round so it isn't charged to a measurement. - warmRunner(ctx); - - // Android accessibility dumps time out while the UI is animating; disable animations - // up front (untimed) so snapshot/get/is/fill can read an idle hierarchy. - log('disabling animations'); - invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); - - const totalRounds = cfg.warmup + cfg.rounds; - for (let round = 0; round < totalRounds; round++) { - const measured = round >= cfg.warmup; - log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`); - for (const step of steps) { - const sample = runStep(step, ctx, round); - if (measured) acc.get(step.label)!.push(sample); - // After the round's reset-open relaunch, warm the runner (untimed) so the first measured - // read (snapshot -i) doesn't pay the post-relaunch first-AX-query cost. - if (step.command === 'open' && step.execMode === 'standalone') { - warmRunner(ctx); - } - } - } - - const tourMeasurements = steps.map((step) => - tourMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), - ); - return [boot, establish, ...tourMeasurements]; -} diff --git a/scripts/perf/ios-harness.test.ts b/scripts/perf/ios-harness.test.ts deleted file mode 100644 index e8e88f5129..0000000000 --- a/scripts/perf/ios-harness.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; -import { mkdtempForTestSync } from '@agent-device/test-kit'; -import { test, vi } from 'vitest'; - -// Stands in for the CLI: records each teardown call, and whether the logs were already saved, -// then appends to the daemon log and fails like a teardown that went wrong. -const FAKE_CLI = ` -import fs from 'node:fs'; -const args = process.argv.slice(2); -const stateDir = args[args.indexOf('--state-dir') + 1]; -const command = args[0] === 'daemon' ? 'daemon stop' : args[0]; -const saved = fs.existsSync(process.env.PERF_FAKE_SAVED_LOG); -fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' saved=' + saved + '\\n'); -fs.appendFileSync(stateDir + '/daemon.log', command + '\\n'); -process.exit(1); -`; - -test('finalizing a run saves the logs before and after a failing teardown, once', async () => { - const root = mkdtempForTestSync('perf-finalize-'); - const fakeCli = path.join(root, 'fake-cli.mjs'); - fs.writeFileSync(fakeCli, FAKE_CLI); - const stateDir = path.join(root, 'state'); - const rawDir = path.join(root, 'out', 'raw'); - fs.mkdirSync(path.join(stateDir, 'sessions', 'perf'), { recursive: true }); - fs.writeFileSync(path.join(stateDir, 'daemon.log'), 'suite\n'); - fs.writeFileSync(path.join(stateDir, 'sessions', 'perf', 'runner.log'), 'runner\n'); - fs.writeFileSync(path.join(stateDir, 'daemon.json'), '{"token":"secret"}'); - const calls = path.join(root, 'calls.log'); - vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); - vi.stubEnv('PERF_FAKE_CALLS', calls); - vi.stubEnv('PERF_FAKE_SAVED_LOG', path.join(rawDir, 'state', 'daemon.log')); - // The CLI command is read when the harness loads, so load it after the stub. - const { iosRunFinalizer } = await import('./ios-harness.ts'); - - const finalize = iosRunFinalizer( - { stateDir, udid: 'sim', appId: 'app', artifactsDir: path.join(rawDir, 'artifacts'), rawDir }, - false, - ); - finalize(); - // The interrupt handler and the normal finish both call it; the second call does nothing. - finalize(); - - assert.equal(fs.readFileSync(calls, 'utf8'), 'close saved=true\ndaemon stop saved=true\n'); - const saved = path.join(rawDir, 'state'); - assert.equal( - fs.readFileSync(path.join(saved, 'daemon.log'), 'utf8'), - 'suite\nclose\ndaemon stop\n', - ); - assert.equal( - fs.readFileSync(path.join(saved, 'sessions', 'perf', 'runner.log'), 'utf8'), - 'runner\n', - ); - assert.equal(fs.existsSync(path.join(saved, 'daemon.json')), false); - assert.equal(fs.existsSync(stateDir), false); -}); diff --git a/scripts/perf/ios-suite.test.ts b/scripts/perf/ios-suite.test.ts deleted file mode 100644 index e9d58ece1f..0000000000 --- a/scripts/perf/ios-suite.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import assert from 'node:assert/strict'; -import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { test } from 'vitest'; -import { - describeCoverage, - expectationViolation, - IOS_SUITE, - IOS_SUITE_EXCLUSIONS, -} from './ios-suite.ts'; - -test('every public command is either measured by the iOS suite or excluded with a reason', () => { - const { measured, excluded } = describeCoverage(IOS_SUITE, IOS_SUITE_EXCLUSIONS); - const classified = [...measured, ...excluded.map((e) => e.command)].sort(); - assert.deepEqual(classified, Object.values(PUBLIC_COMMANDS).sort()); - for (const { reason } of excluded) assert.ok(reason.length > 0); -}); - -test('step ids are unique, so comparisons key each step once', () => { - const ids = IOS_SUITE.flatMap((scenario) => scenario.steps.map((s) => `${scenario.id}/${s.id}`)); - assert.equal(new Set(ids).size, ids.length); -}); - -test('a heavy-feed capture breaks its expectation when the tree flattens', () => { - const depth = { maxDepthAtLeast: 64 }; - assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); - assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); - assert.match(expectationViolation(depth, undefined)!, /unknown/); -}); - -test('a pinned capture breaks its expectation unless the pinned backend served it', () => { - const pinned = { pinnedBackend: 'tree' } as const; - const served = { backend: 'tree', state: 'healthy', reasonCode: 'requested-backend' }; - assert.equal(expectationViolation(pinned, served), undefined); - // The raw plan ignores the pin and recovers through private AX on the heavy feed. - assert.match( - expectationViolation(pinned, { - backend: 'private-ax', - state: 'recovered', - reasonCode: 'ax-rejected', - })!, - /private-ax backend \(ax-rejected\)/, - ); - // An unpinned capture can still be tree-first on a light screen. - assert.match( - expectationViolation(pinned, { backend: 'tree', state: 'healthy' })!, - /no reason code/, - ); - assert.match(expectationViolation(pinned, undefined)!, /unknown backend/); -}); diff --git a/scripts/perf/measurement.ts b/scripts/perf/measurement.ts index 1641361a3a..b06229527c 100644 --- a/scripts/perf/measurement.ts +++ b/scripts/perf/measurement.ts @@ -24,7 +24,7 @@ export function buildMeasurement(input: MeasurementInput): Measurement { ...input, wallClock: summarize(ok.map((s) => s.wallClockMs)), daemon: summarize(numbers(ok.map((s) => s.daemonMs))), - elementCount: summarize(numbers(ok.map((s) => s.elementCount ?? s.snapshot?.nodeCount))), + elementCount: summarize(numbers(ok.map((s) => s.snapshot?.nodeCount))), profile: aggregateProfiles(input.samples), failures, notes, diff --git a/scripts/perf/metadata.ts b/scripts/perf/metadata.ts index 47f25b9098..ab6719fa47 100644 --- a/scripts/perf/metadata.ts +++ b/scripts/perf/metadata.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; import { readGitRevision, readTarget, readToolchain } from '../lib/bench-host.ts'; import { REPO_ROOT, type PerfConfig } from './config.ts'; import type { RunMetadata } from './types.ts'; @@ -14,8 +15,28 @@ function readVersion(): string { } } +function adb(serial: string, args: string[]): string | undefined { + try { + const result = runCmdSync('adb', ['-s', serial, ...args], { allowFailure: true }); + return result.exitCode === 0 ? result.stdout.trim() : undefined; + } catch { + return undefined; + } +} + +function emulator(serial: string): RunMetadata['device'] { + const name = adb(serial, ['emu', 'avd', 'name'])?.split('\n')[0]?.trim(); + const release = adb(serial, ['shell', 'getprop', 'ro.build.version.release']); + const sdk = adb(serial, ['shell', 'getprop', 'ro.build.version.sdk']); + return { + name: name || serial, + serial, + runtime: release ? `Android ${release}${sdk ? ` (API ${sdk})` : ''}` : undefined, + }; +} + /** Records what the numbers depend on: source, toolchain, host, and device. */ -export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadata { +export function collectMetadata(cfg: PerfConfig): RunMetadata { const revision = readGitRevision(REPO_ROOT); const toolchain = readToolchain(); const target = cfg.platform === 'ios' ? readTarget(cfg.udid!, cfg.appId, cfg.appPath) : undefined; @@ -38,7 +59,7 @@ export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadat node: toolchain.node, device: target ? { name: target.name, udid: target.udid, runtime: target.runtime.split('.').pop() } - : { name: deviceName, serial: cfg.serial }, - fixture: target ? { appId: target.appId, appPath: target.appPath } : undefined, + : emulator(cfg.serial!), + fixture: { appId: cfg.appId, appPath: cfg.appPath }, }; } diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts index 34a1aac17a..bced580207 100644 --- a/scripts/perf/nightly-artifacts.test.ts +++ b/scripts/perf/nightly-artifacts.test.ts @@ -30,6 +30,11 @@ test('resolves latest, run ids, and dates to the newest matching run', () => { assert.equal(resolveRunRef('2026-10-19', RUNS)?.id, '103'); assert.equal(resolveRunRef('2026-10-18', RUNS), undefined); assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/); + // A run without the platform artifact gives way to the newest run that has it. + const withArtifact = (run: WorkflowRun) => run.id !== '104' && run.id !== '103'; + assert.equal(resolveRunRef('latest', RUNS, withArtifact)?.id, '102'); + assert.equal(resolveRunRef('2026-10-19', RUNS, withArtifact)?.id, '102'); + assert.equal(resolveRunRef('104', RUNS, withArtifact), undefined); }); const baselineIds = ( @@ -110,4 +115,12 @@ test('a rolling baseline accepts only runs that measured every target scenario', assert.equal(coversTarget(full, full), true); assert.equal(coversTarget(partial, full), false); assert.equal(coversTarget(full, partial), true); + const android = summaryFixture( + [ + { id: 'a/x', daemonMs: [1] }, + { id: 'b/y', daemonMs: [1] }, + ], + { platform: 'android' }, + ); + assert.equal(coversTarget(android, full), false); }); diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts index 8f0525f077..7daaf517f9 100644 --- a/scripts/perf/nightly-artifacts.ts +++ b/scripts/perf/nightly-artifacts.ts @@ -3,10 +3,12 @@ import path from 'node:path'; import { runCmdSync } from '@agent-device/host-kit/command'; import { SUMMARY_FILE } from './report.ts'; import { parsePerfSummary } from './summary.ts'; -import type { PerfSummary } from './types.ts'; +import type { PerfSummary, Platform } from './types.ts'; -/** The stable name of the summary artifact that the nightly workflow uploads for every run. */ -export const SUMMARY_ARTIFACT = 'perf-ios-summary'; +/** The stable name of the summary artifact that the nightly workflow uploads for each platform. */ +export function summaryArtifact(platform: Platform): string { + return `perf-${platform}-summary`; +} export type WorkflowRun = { id: string; @@ -79,13 +81,20 @@ function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] { /** * Resolves one run reference: `latest`, a numeric run id, or a `YYYY-MM-DD` date (the newest run - * created that UTC day). Returns undefined when no listed run matches. + * created that UTC day). Only runs that `accept` takes count, so a run that skipped a platform + * does not hide an older run of that platform. Returns undefined when no listed run matches. */ -export function resolveRunRef(ref: string, runs: WorkflowRun[]): WorkflowRun | undefined { +export function resolveRunRef( + ref: string, + runs: WorkflowRun[], + accept: (run: WorkflowRun) => boolean = () => true, +): WorkflowRun | undefined { const newestFirst = sortNewestFirst(runs); - if (ref === 'latest') return newestFirst[0]; - if (/^\d+$/.test(ref)) return newestFirst.find((run) => run.id === ref); - if (DATE_REF.test(ref)) return newestFirst.find((run) => run.createdAt.slice(0, 10) === ref); + const first = (match: (run: WorkflowRun) => boolean) => + newestFirst.find((run) => match(run) && accept(run)); + if (ref === 'latest') return first(() => true); + if (/^\d+$/.test(ref)) return first((run) => run.id === ref); + if (DATE_REF.test(ref)) return first((run) => run.createdAt.slice(0, 10) === ref); throw new Error( `Unknown run reference ${JSON.stringify(ref)}: use latest, a run id, a YYYY-MM-DD date, or a path.`, ); @@ -156,7 +165,8 @@ export function selectNightlyBaseline( * when the run has no such artifact (it failed before upload, or the artifact expired). */ export function fetchSummary(source: NightlySource, run: WorkflowRun): PerfSummary | undefined { - const dir = path.join(source.cacheDir, run.id); + // One run uploads a summary artifact per platform, so each artifact has its own directory. + const dir = path.join(source.cacheDir, run.id, source.artifact); const file = path.join(dir, SUMMARY_FILE); if (!fs.existsSync(file)) { fs.rmSync(dir, { recursive: true, force: true }); diff --git a/scripts/perf/platform-profiles.ts b/scripts/perf/platform-profiles.ts deleted file mode 100644 index 447d3b310f..0000000000 --- a/scripts/perf/platform-profiles.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { PerfConfig } from './config.ts'; - -// Local-convenience default for ad-hoc Android runs: an AVD name that `boot` can launch. CI -// passes --serial. The iOS suite drives the fixture app instead (see ios-suite.ts). -const DEFAULT_ANDROID_AVD = 'Pixel_9_Pro_XL_API_37'; - -export type ProfileSelectors = { - // A row on the Settings root that pushes a large sub-screen (big a11y tree). - deepScreen: string; - // The Settings search field (for press/focus; auto-picks a match). - searchField: string; - // A selector that uniquely targets the EDITABLE search field (for fill). - searchFieldEditable: string; - // A label reliably visible on the Settings root, for get/is (selector form). - anchorLabel: string; - // Plain text of the anchor, for wait text / find (not a selector). - anchorText: string; -}; - -export type ResolvedProfile = { - platform: 'android'; - deviceName: string; - serial?: string; - platformFlags: string[]; // --platform; applied to every call (only conflicts if it mismatches a locked session) - selectorFlags: string[]; // device selectors — ONLY on the session-establishing open / selectorless boot - appTarget: string; // `open` target for Settings - selectors: ProfileSelectors; -}; - -export function resolveProfile(cfg: PerfConfig): ResolvedProfile { - const avdName = cfg.device ?? DEFAULT_ANDROID_AVD; - const serialFlags = cfg.serial - ? ['--serial', cfg.serial, '--android-device-allowlist', cfg.serial] - : []; - return { - platform: 'android', - deviceName: cfg.serial ? `android (${cfg.serial})` : avdName, - serial: cfg.serial, - platformFlags: ['--platform', 'android'], - selectorFlags: ['--device', avdName, ...serialFlags], - appTarget: 'com.android.settings', - selectors: { - deepScreen: 'text="Network & internet"', - searchField: 'text="Search Settings"', - searchFieldEditable: 'editable', - anchorLabel: 'label="Network & internet"', - anchorText: 'Network & internet', - }, - }; -} diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts index 9e9b9e61c1..e3d3582739 100644 --- a/scripts/perf/profile.test.ts +++ b/scripts/perf/profile.test.ts @@ -2,12 +2,19 @@ import assert from 'node:assert/strict'; import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; import { RETRY_PHASE } from '@agent-device/host-kit/retry'; +import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + ANDROID_TOUCH_HELPER_GESTURE_PHASE, + type AndroidSnapshotHelperTransport, +} from '@agent-device/platform-android/mechanics'; import { test } from 'vitest'; import { aggregateProfiles, profileRequestLog, readClientDiagnostics, readSettleFacts, + readSnapshotFacts, } from './profile.ts'; import type { Sample } from './types.ts'; @@ -117,3 +124,133 @@ test('reads the settle wait of a --settle response and nothing from other respon ); assert.equal(readSettleFacts({ data: { message: 'Tapped' } }), undefined); }); + +// Trimmed from Android heavy-feed request logs. +const sessionTimer = (answered: boolean, event: Record = {}) => + line({ + phase: ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + ...event, + data: { + transport: 'persistent-session' satisfies AndroidSnapshotHelperTransport, + answered, + ...(event.data as Record | undefined), + }, + }); +const oneShotTimer = [ + line({ + level: 'debug', + phase: 'exec_command', + durationMs: 373, + data: { command: 'adb', event: 'settled' }, + }), + line({ + phase: ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + durationMs: 373, + data: { transport: 'instrumentation' satisfies AndroidSnapshotHelperTransport }, + }), +]; +const androidCapture = (...timers: string[]) => + [ + ...timers, + line({ phase: 'snapshot_capture', durationMs: 380, data: { backend: 'android' } }), + ].join('\n'); + +test('names the Android helper transport that answered each capture', () => { + const session = profileRequestLog(androidCapture(sessionTimer(true, { durationMs: 40 }))); + assert.deepEqual(session.snapshotSources, { + 'android-helper/persistent-session': { count: 1, totalMs: 40, failures: 0 }, + }); + assert.deepEqual(session.fallbacks, []); + + // With the session off, the session timer settles unanswered and a one-shot capture answers. + const sessionOff = profileRequestLog( + androidCapture(sessionTimer(false, { durationMs: 0 }), ...oneShotTimer), + ); + assert.deepEqual(sessionOff.snapshotSources, { + 'android-helper/persistent-session': { count: 1, totalMs: 0, failures: 1 }, + 'android-helper/instrumentation': { count: 1, totalMs: 373, failures: 0 }, + }); + assert.deepEqual(sessionOff.spawns, { adb: { count: 1, totalMs: 373 } }); + assert.deepEqual( + aggregateProfiles([{ round: 0, ok: true, wallClockMs: 1, profile: sessionOff }])!.producers, + { + 'android-helper/instrumentation': 1, + }, + ); + + const sessionFailed = profileRequestLog( + [ + androidCapture( + sessionTimer(false, { level: 'error', durationMs: 5000, data: { error: 'socket closed' } }), + ...oneShotTimer, + ), + line({ + level: 'warn', + phase: 'android_snapshot_helper_session_fallback', + data: { reason: 'socket closed' }, + }), + ].join('\n'), + ); + assert.deepEqual(sessionFailed.snapshotSources['android-helper/persistent-session'], { + count: 1, + totalMs: 5000, + failures: 1, + }); + assert.deepEqual(sessionFailed.fallbacks, [ + 'android-helper/persistent-session: socket closed', + 'android_snapshot_helper_session_fallback: socket closed', + ]); +}); + +test('reads whether the Android session answered from its own timer, not from event order', () => { + // A content re-capture can follow a session capture that answered. The answer still counts. + const recaptured = profileRequestLog( + androidCapture(sessionTimer(true, { durationMs: 40 }), ...oneShotTimer), + ); + assert.deepEqual(recaptured.snapshotSources['android-helper/persistent-session'], { + count: 1, + totalMs: 40, + failures: 0, + }); + + // A request that ends before any one-shot capture still records the unanswered session. + const unanswered = profileRequestLog(androidCapture(sessionTimer(false, { durationMs: 12 }))); + assert.deepEqual(unanswered.snapshotSources, { + 'android-helper/persistent-session': { count: 1, totalMs: 12, failures: 1 }, + }); +}); + +test('counts Android helper gestures as runner commands', () => { + const profile = profileRequestLog( + [ + line({ phase: ANDROID_TOUCH_HELPER_GESTURE_PHASE, durationMs: 210 }), + line({ + level: 'error', + phase: ANDROID_TOUCH_HELPER_GESTURE_PHASE, + durationMs: 30, + data: { error: 'busy' }, + }), + ].join('\n'), + ); + assert.deepEqual(profile.runnerCommands, { gesture: { count: 2, totalMs: 240 } }); + assert.deepEqual(profile.fallbacks, [`${ANDROID_TOUCH_HELPER_GESTURE_PHASE}: busy`]); +}); + +test('reads the Android helper transport from a snapshot response', () => { + const facts = readSnapshotFacts({ + data: { + nodes: [{ depth: 0 }, { depth: 60 }], + snapshotQuality: { state: 'healthy', backend: 'android-helper' }, + androidSnapshot: { backend: 'android-helper', helperTransport: 'instrumentation' }, + }, + }); + assert.deepEqual(facts, { + backend: 'android-helper', + transport: 'instrumentation', + state: 'healthy', + reasonCode: undefined, + nodeCount: 2, + maxDepth: 60, + truncated: undefined, + }); +}); diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts index 96a4a188f6..1124eb077c 100644 --- a/scripts/perf/profile.ts +++ b/scripts/perf/profile.ts @@ -1,6 +1,11 @@ import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; import { RETRY_PHASE } from '@agent-device/host-kit/retry'; +import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + ANDROID_TOUCH_HELPER_GESTURE_PHASE, +} from '@agent-device/platform-android/mechanics'; import { median } from './stats.ts'; import type { MeasurementProfile, @@ -49,8 +54,10 @@ function eventDuration(event: DiagnosticEvent): number | undefined { /** * Reads one daemon request log (`--debug` ndjson). Process spawns count once, at their `settled` - * event. Fallbacks name every degraded path the request took: snapshot-source errors, truncated - * recovery, and warn- or error-level phases. + * event. Runner commands are XCTest runner commands on iOS and helper gestures on Android. Snapshot + * sources are the iOS accessibility producers and the Android helper transports. Fallbacks name + * every degraded path the request took: snapshot-source errors, truncated recovery, and warn- or + * error-level phases. */ export function profileRequestLog(ndjson: string): SampleProfile { const profile: SampleProfile = { @@ -63,6 +70,15 @@ export function profileRequestLog(ndjson: string): SampleProfile { fallbacks: [], }; const fallbacks = new Set(); + const addSource = (producer: string, event: DiagnosticEvent, answered = true): void => { + const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); + entry.count += 1; + entry.totalMs += eventDuration(event) ?? 0; + const error = text(event.data?.error); + const failed = error !== undefined || event.level === 'error'; + if (failed || !answered) entry.failures += 1; + if (failed) fallbacks.add(`${producer}: ${error ?? 'failed'}`); + }; let firstTs: number | undefined; let lastTs: number | undefined; for (const line of ndjson.split('\n')) { @@ -88,15 +104,19 @@ export function profileRequestLog(ndjson: string): SampleProfile { } else if (phase === RUNNER_COMMAND_SEND_PHASE) { addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); } else if (phase === SNAPSHOT_SOURCE_ACQUIRE_PHASE) { - const producer = text(data.producer) ?? 'unknown'; - const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); - entry.count += 1; - entry.totalMs += duration ?? 0; - const error = text(data.error); - if (error || event.level === 'error') { - entry.failures += 1; - fallbacks.add(`${producer}: ${error ?? 'failed'}`); - } + addSource(text(data.producer) ?? 'unknown', event); + } else if ( + phase === ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE || + phase === ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE + ) { + addSource( + `android-helper/${text(data.transport) ?? 'unknown'}`, + event, + data.answered !== false, + ); + } else if (phase === ANDROID_TOUCH_HELPER_GESTURE_PHASE) { + addTiming(profile.runnerCommands, 'gesture', duration); + if (event.level === 'error') fallbacks.add(`${phase}: ${text(data.error) ?? 'failed'}`); } else if (phase === SNAPSHOT_SOURCE_RECOVERY_PHASE && data.truncated === true) { fallbacks.add( `${text(data.producer) ?? 'unknown'}: truncated at ${String(data.acceptedLevels)} levels`, @@ -157,8 +177,10 @@ export function readSnapshotFacts(json: unknown): SnapshotFacts | undefined { const nodes = data.nodes as { depth?: unknown }[]; const quality = (data.snapshotQuality ?? {}) as Record; const depths = nodes.map((node) => (typeof node.depth === 'number' ? node.depth : 0)); + const android = (data.androidSnapshot ?? {}) as Record; return { backend: text(quality.backend), + transport: text(android.helperTransport), state: text(quality.state), reasonCode: text(quality.reasonCode), nodeCount: nodes.length, diff --git a/scripts/perf/run.ts b/scripts/perf/run.ts index cd712056ae..81e51cad95 100644 --- a/scripts/perf/run.ts +++ b/scripts/perf/run.ts @@ -1,17 +1,17 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { parseConfig, REPO_ROOT, usesSourceCli, type PerfConfig } from './config.ts'; -import { runScenario, setupIsolation, teardownIsolation } from './harness.ts'; -import { iosRunFinalizer, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; +import { parseConfig, REPO_ROOT, type PerfConfig } from './config.ts'; +import { collectMetadata } from './metadata.ts'; +import { writeRunOutput } from './report.ts'; import { describeCoverage, FIXTURE_APP_EVENT_URL_TEMPLATE, - IOS_SUITE, - IOS_SUITE_EXCLUSIONS, -} from './ios-suite.ts'; -import { collectMetadata } from './metadata.ts'; -import { writeRunOutput } from './report.ts'; + SUITE_EXCLUSIONS, + suiteFor, + type SuiteScenario, +} from './suite.ts'; +import { runFinalizer, runSuite, type SuiteContext } from './suite-harness.ts'; import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary } from './types.ts'; function log(msg: string): void { @@ -29,73 +29,59 @@ function onInterrupt(cleanup: () => void): void { }); } -function selectScenarios(cfg: PerfConfig) { - if (!cfg.scenarios) return IOS_SUITE; - const unknown = cfg.scenarios.filter((id) => !IOS_SUITE.some((s) => s.id === id)); +function selectScenarios(cfg: PerfConfig): SuiteScenario[] { + const suite = suiteFor(cfg.platform); + if (!cfg.scenarios) return suite; + const unknown = cfg.scenarios.filter((id) => !suite.some((s) => s.id === id)); if (unknown.length > 0) { throw new Error( - `Unknown scenarios: ${unknown.join(', ')} (known: ${IOS_SUITE.map((s) => s.id).join(', ')})`, + `Unknown scenarios: ${unknown.join(', ')} (known: ${suite.map((s) => s.id).join(', ')})`, ); } - return IOS_SUITE.filter((s) => cfg.scenarios!.includes(s.id)); + return suite.filter((s) => cfg.scenarios!.includes(s.id)); } -async function runIos( +async function runPlatformSuite( cfg: PerfConfig, ): Promise<{ measurements: Measurement[]; coverage: PerfSummary['coverage'] }> { const scenarios = selectScenarios(cfg); - // The daemon inherits this from the first CLI call; it routes trigger-app-event to the fixture. + // The daemon inherits these from the first CLI call; they route trigger-app-event to the fixture. process.env.AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE ??= FIXTURE_APP_EVENT_URL_TEMPLATE; + process.env.AGENT_DEVICE_ANDROID_APP_EVENT_URL_TEMPLATE ??= FIXTURE_APP_EVENT_URL_TEMPLATE; const rawDir = path.join(cfg.outDir, 'raw'); - const ctx: IosSuiteContext = { + const ctx: SuiteContext = { + platform: cfg.platform, stateDir: fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-perf-')), - udid: cfg.udid!, + device: cfg.platform === 'ios' ? cfg.udid! : cfg.serial!, appId: cfg.appId, artifactsDir: path.join(rawDir, 'artifacts'), rawDir, }; fs.mkdirSync(ctx.artifactsDir, { recursive: true }); log(`state-dir: ${ctx.stateDir}`); - const finalize = iosRunFinalizer(ctx, cfg.keepArtifacts); + const finalize = runFinalizer(ctx, cfg.keepArtifacts); onInterrupt(finalize); try { - const measurements = await runIosSuite(ctx, cfg, scenarios); - return { measurements, coverage: describeCoverage(scenarios, IOS_SUITE_EXCLUSIONS) }; - } finally { - finalize(); - } -} - -function runAndroid(cfg: PerfConfig): { - measurements: Measurement[]; - coverage: PerfSummary['coverage']; -} { - const ctx = setupIsolation(cfg); - onInterrupt(() => teardownIsolation(ctx, cfg)); - try { - const measurements = runScenario(ctx, cfg); + const measurements = await runSuite(ctx, cfg, scenarios); return { measurements, - coverage: { measured: [...new Set(measurements.map((m) => m.command))].sort(), excluded: [] }, + coverage: describeCoverage(scenarios, SUITE_EXCLUSIONS[cfg.platform]), }; } finally { - teardownIsolation(ctx, cfg); + finalize(); } } async function main(): Promise { const cfg = parseConfig(process.argv.slice(2)); - // The iOS forced-backend probe loads the built SDK; the dist CLI needs the build as well. - if ( - (cfg.platform === 'ios' || !usesSourceCli()) && - !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src')) - ) { + // The CLI and the iOS forced-backend probe run from the build. + if (!fs.existsSync(path.join(REPO_ROOT, 'dist', 'src'))) { throw new Error('dist/ is missing — run `pnpm build` first.'); } const startedAt = new Date().toISOString(); // Read before the run, so a checkout change during a long run cannot relabel it. - const metadata = collectMetadata(cfg, cfg.serial ?? cfg.device ?? 'unknown'); - const { measurements, coverage } = cfg.platform === 'ios' ? await runIos(cfg) : runAndroid(cfg); + const metadata = collectMetadata(cfg); + const { measurements, coverage } = await runPlatformSuite(cfg); const summary: PerfSummary = { schema: PERF_SUMMARY_SCHEMA, platform: cfg.platform, diff --git a/scripts/perf/scenario.ts b/scripts/perf/scenario.ts deleted file mode 100644 index 1e34cad240..0000000000 --- a/scripts/perf/scenario.ts +++ /dev/null @@ -1,148 +0,0 @@ -import path from 'node:path'; -import type { ResolvedProfile } from './platform-profiles.ts'; - -// A structured batch step: maps through the same command input as MCP and the Node client. -export type BatchStepSpec = { - command: string; - input: Record; -}; - -type ScenarioStepBase = { - label: string; - command: string; - // When set, the harness runs an untimed `open --relaunch` (reset to root, top of list) - // before timing this step. Used for steps whose precondition is a clean root, since - // earlier commands (find/is, search) leave the list scrolled or in a different surface. - freshRoot?: boolean; -}; - -// Discriminated on execMode so the invoker gets the right payload without `!`/`?? []`: -// standalone carries full CLI args; batch carries one structured batch step. -export type ScenarioStep = - | (ScenarioStepBase & { execMode: 'standalone'; args: string[] }) - | (ScenarioStepBase & { execMode: 'batch'; step: BatchStepSpec; isSnapshot?: boolean }); - -export type StepContext = { artifactsDir: string }; - -function std(label: string, command: string, args: string[]): ScenarioStep { - return { label, command, execMode: 'standalone', args }; -} - -function bat( - label: string, - command: string, - step: BatchStepSpec, - opts: { isSnapshot?: boolean; freshRoot?: boolean } = {}, -): ScenarioStep { - return { label, command, execMode: 'batch' as const, step, ...opts }; -} - -// One ordered pass over Android Settings. The harness repeats this N (+warmup) times; -// the leading `open --relaunch` resets the app to its root each round, so every -// round starts from a known state while commands run in their natural order. -export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): ScenarioStep[] { - const s = p.selectors; - const shot = path.join(ctx.artifactsDir, 'shot.png'); - const rec = path.join(ctx.artifactsDir, 'rec.mp4'); - const trace = path.join(ctx.artifactsDir, 'trace.log'); - - // Android reveals the editable search field only after a tap on the search card. - const textEntry: ScenarioStep[] = [ - bat( - 'press search field', - 'press', - { - command: 'press', - input: { target: { kind: 'selector', selector: s.searchField } }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('fill search', 'fill', { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - ]; - - return [ - // --- reset to root via relaunch --- - std('open (relaunch → root)', 'open', ['open', p.appTarget, '--relaunch']), - - // --- reads on the root tree (snapshots first; anchor label is visible here) --- - bat( - 'snapshot -i (root)', - 'snapshot', - { command: 'snapshot', input: { interactiveOnly: true } }, - { isSnapshot: true }, - ), - bat('snapshot (root)', 'snapshot', { command: 'snapshot', input: {} }, { isSnapshot: true }), - - // --- navigate into a sub-screen from a fresh root (freshRoot resets scroll so the - // deep-screen row is in view), read it, then return --- - bat( - 'press → deep screen', - 'press', - { - command: 'press', - input: { target: { kind: 'selector', selector: s.deepScreen } }, - }, - { freshRoot: true }, - ), - bat('snapshot (deep)', 'snapshot', { command: 'snapshot', input: {} }, { isSnapshot: true }), - bat( - 'snapshot -i (deep)', - 'snapshot', - { command: 'snapshot', input: { interactiveOnly: true } }, - { isSnapshot: true }, - ), - bat('back', 'back', { command: 'back', input: {} }), - - // --- targeted reads against the visible anchor (freshRoot so the anchor is on screen) --- - bat( - 'wait text', - 'wait', - { command: 'wait', input: { text: s.anchorText, timeoutMs: 3000 } }, - { freshRoot: true }, - ), - bat('find', 'find', { command: 'find', input: { query: s.anchorText } }), - bat('get text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.anchorLabel }, - }, - }), - bat('is visible', 'is', { - command: 'is', - input: { predicate: 'visible', selector: s.anchorLabel }, - }), - - // --- text entry (platform-specific order; see textEntry above) then scroll results --- - ...textEntry, - bat('scroll down', 'scroll', { command: 'scroll', input: { direction: 'down' } }), - - // --- artifact-producing commands; record brackets the rest so the clip has >1s of - // footage (an instant start→stop makes simctl recordVideo fail to finalize) --- - std('record start', 'record', ['record', 'start', rec, '--hide-touches']), - bat('screenshot', 'screenshot', { command: 'screenshot', input: { path: shot } }), - bat('logs mark', 'logs', { - command: 'logs', - input: { action: 'mark', message: 'perf-mark' }, - }), - bat('logs clear', 'logs', { command: 'logs', input: { action: 'clear' } }), - std('trace start', 'trace', ['trace', 'start', trace]), - std('trace stop', 'trace', ['trace', 'stop']), - bat('perf frames', 'perf', { command: 'perf', input: { area: 'frames' } }), - std('record stop', 'record', ['record', 'stop']), - ]; -} diff --git a/scripts/perf/suite-harness.test.ts b/scripts/perf/suite-harness.test.ts new file mode 100644 index 0000000000..fb3b33c5d5 --- /dev/null +++ b/scripts/perf/suite-harness.test.ts @@ -0,0 +1,156 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { test, vi } from 'vitest'; +import type { PerfConfig } from './config.ts'; +import type { PlatformStep } from './suite.ts'; + +// Stands in for the CLI: records each teardown call, and whether the logs were already saved, +// then appends to the daemon log and fails like a teardown that went wrong. +const FAKE_CLI = ` +import fs from 'node:fs'; +const args = process.argv.slice(2); +const stateDir = args[args.indexOf('--state-dir') + 1]; +const command = args[0] === 'daemon' ? 'daemon stop' : args[0]; +const saved = fs.existsSync(process.env.PERF_FAKE_SAVED_LOG); +fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' saved=' + saved + '\\n'); +fs.appendFileSync(stateDir + '/daemon.log', command + '\\n'); +process.exit(1); +`; + +test('finalizing a run saves the logs before and after a failing teardown, once', async () => { + const root = mkdtempForTestSync('perf-finalize-'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + fs.writeFileSync(fakeCli, FAKE_CLI); + const stateDir = path.join(root, 'state'); + const rawDir = path.join(root, 'out', 'raw'); + fs.mkdirSync(path.join(stateDir, 'sessions', 'perf'), { recursive: true }); + fs.writeFileSync(path.join(stateDir, 'daemon.log'), 'suite\n'); + fs.writeFileSync(path.join(stateDir, 'sessions', 'perf', 'runner.log'), 'runner\n'); + fs.writeFileSync(path.join(stateDir, 'daemon.json'), '{"token":"secret"}'); + const calls = path.join(root, 'calls.log'); + vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); + vi.stubEnv('PERF_FAKE_CALLS', calls); + vi.stubEnv('PERF_FAKE_SAVED_LOG', path.join(rawDir, 'state', 'daemon.log')); + // The CLI command is read when the harness loads, so load it after the stub. + const { runFinalizer } = await import('./suite-harness.ts'); + + const finalize = runFinalizer( + { + platform: 'ios', + stateDir, + device: 'sim', + appId: 'app', + artifactsDir: path.join(rawDir, 'artifacts'), + rawDir, + }, + false, + ); + finalize(); + // The interrupt handler and the normal finish both call it; the second call does nothing. + finalize(); + + assert.equal(fs.readFileSync(calls, 'utf8'), 'close saved=true\ndaemon stop saved=true\n'); + const saved = path.join(rawDir, 'state'); + assert.equal( + fs.readFileSync(path.join(saved, 'daemon.log'), 'utf8'), + 'suite\nclose\ndaemon stop\n', + ); + assert.equal( + fs.readFileSync(path.join(saved, 'sessions', 'perf', 'runner.log'), 'utf8'), + 'runner\n', + ); + assert.equal(fs.existsSync(path.join(saved, 'daemon.json')), false); + assert.equal(fs.existsSync(stateDir), false); +}); + +// Stands in for the CLI and its daemon: records each call with the session variable it saw, and +// answers a snapshot with the transport that variable selects. +const FAKE_DAEMON_CLI = ` +import fs from 'node:fs'; +const args = process.argv.slice(2); +const command = args.slice(0, args.indexOf('--state-dir')).join(' '); +const session = process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION ?? 'default'; +fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' session=' + session + '\\n'); +if (process.env.PERF_FAKE_FAIL_RESTORE && command === 'open app --serial emulator-5554' && session === 'default') { + process.stdout.write(JSON.stringify({ success: false, error: { code: 'DEVICE_IN_USE', message: 'claimed' } })); + process.exit(1); +} +const transport = session === '0' ? 'instrumentation' : 'persistent-session'; +const data = args[0] === 'snapshot' + ? { nodes: [{ depth: 0 }], androidSnapshot: { helperTransport: transport } } + : {}; +process.stdout.write(JSON.stringify({ success: true, data })); +`; + +const ONE_SHOT: PlatformStep = { + id: 'one-shot', + command: 'snapshot', + args: ['snapshot', '--raw'], + daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' }, + expect: { transport: 'instrumentation' }, +}; + +async function runFakeDaemonSuite(steps: PlatformStep[]) { + const root = mkdtempForTestSync('perf-daemon-env-'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + fs.writeFileSync(fakeCli, FAKE_DAEMON_CLI); + const calls = path.join(root, 'calls.log'); + vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); + vi.stubEnv('PERF_FAKE_CALLS', calls); + vi.resetModules(); + const { runSuite } = await import('./suite-harness.ts'); + const measurements = await runSuite( + { + platform: 'android', + stateDir: path.join(root, 'state'), + device: 'emulator-5554', + appId: 'app', + artifactsDir: path.join(root, 'artifacts'), + rawDir: path.join(root, 'raw'), + }, + { warmup: 0, rounds: 1 } as PerfConfig, + [ + { + id: 'heavy-feed', + title: 'heavy feed', + setup: [], + steps, + }, + ], + ); + return { measurements, calls: fs.readFileSync(calls, 'utf8') }; +} + +test('a step with daemon variables runs against a restarted daemon and restores it after', async () => { + const { + measurements: [measurement], + calls, + } = await runFakeDaemonSuite([ONE_SHOT]); + + assert.equal( + calls, + [ + 'open app --relaunch --serial emulator-5554 session=default', + 'daemon stop session=default', + 'open app --serial emulator-5554 session=0', + 'snapshot --raw --debug session=0', + 'daemon stop session=default', + 'open app --serial emulator-5554 session=default', + '', + ].join('\n'), + ); + assert.equal(measurement!.failures, 0); + assert.equal(measurement!.samples[0]!.snapshot?.transport, 'instrumentation'); +}); + +test('a failed daemon restore skips the rest of the scenario as restore_failed', async () => { + vi.stubEnv('PERF_FAKE_FAIL_RESTORE', '1'); + const { measurements } = await runFakeDaemonSuite([ + ONE_SHOT, + { id: 'back', command: 'back', args: ['back'] }, + ]); + assert.equal(measurements[0]!.failures, 0); + assert.equal(measurements[1]!.samples[0]!.errorCode, 'restore_failed'); +}); diff --git a/scripts/perf/ios-harness.ts b/scripts/perf/suite-harness.ts similarity index 71% rename from scripts/perf/ios-harness.ts rename to scripts/perf/suite-harness.ts index 16579f7506..6cf0be7521 100644 --- a/scripts/perf/ios-harness.ts +++ b/scripts/perf/suite-harness.ts @@ -7,7 +7,7 @@ import { import { invokeCli } from './cli.ts'; import type { PerfConfig } from './config.ts'; import { captureWithForcedBackend } from './forced-backend.ts'; -import { expectationViolation, type SuiteScenario, type SuiteStep } from './ios-suite.ts'; +import { expectationViolation, type PlatformStep, type SuiteScenario } from './suite.ts'; import { buildMeasurement } from './measurement.ts'; import { profileRequestLog, @@ -15,39 +15,54 @@ import { readSettleFacts, readSnapshotFacts, } from './profile.ts'; -import type { CliResult, Measurement, Sample } from './types.ts'; +import type { CliResult, Measurement, Platform, Sample } from './types.ts'; const SESSION = 'perf'; -export type IosSuiteContext = { +export type SuiteContext = { + platform: Platform; stateDir: string; - udid: string; + // The iOS simulator UDID or the Android emulator serial. + device: string; appId: string; artifactsDir: string; rawDir: string; // Set when a request outlived the client timeout. That request still holds the session, so // the suite restarts the daemon before the next scenario. daemonStuck?: boolean; + // Variables that the running daemon was started with, set by a step with `daemonEnv`. + daemonEnv?: Readonly>; + // Set when the daemon restart after a `daemonEnv` step could not reopen the session. The rest + // of the scenario is skipped, and the next scenario reopens the session. + restoreFailed?: string; }; function log(msg: string): void { process.stderr.write(`[perf] ${msg}\n`); } -function baseFlags(ctx: IosSuiteContext): string[] { - return ['--state-dir', ctx.stateDir, '--session', SESSION, '--platform', 'ios']; +function baseFlags(ctx: SuiteContext): string[] { + return ['--state-dir', ctx.stateDir, '--session', SESSION, '--platform', ctx.platform]; } -function run(ctx: IosSuiteContext, args: readonly string[]): CliResult { - return watch(ctx, invokeCli(expandStatic(args, ctx), baseFlags(ctx))); +function deviceFlags(ctx: SuiteContext): string[] { + return ctx.platform === 'ios' ? ['--udid', ctx.device] : ['--serial', ctx.device]; } -function watch(ctx: IosSuiteContext, result: CliResult): CliResult { +function cli(ctx: SuiteContext, args: string[]): CliResult { + return watch(ctx, invokeCli(args, baseFlags(ctx), { env: ctx.daemonEnv })); +} + +function run(ctx: SuiteContext, args: readonly string[]): CliResult { + return cli(ctx, expandStatic(args, ctx)); +} + +function watch(ctx: SuiteContext, result: CliResult): CliResult { if (daemonTimedOut(result)) ctx.daemonStuck = true; return result; } -function expandStatic(args: readonly string[], ctx: IosSuiteContext): string[] { +function expandStatic(args: readonly string[], ctx: SuiteContext): string[] { return args.map((arg) => arg === '{app}' ? ctx.appId @@ -64,7 +79,7 @@ type SnapshotNode = { const TARGET_TOKEN = /^\{(ref|point):(.+)\}$/; /** Expands `{ref:id}` and `{point:id}` from one untimed interactive snapshot. */ -function resolveTargets(args: string[], ctx: IosSuiteContext): string[] | { error: string } { +function resolveTargets(args: string[], ctx: SuiteContext): string[] | { error: string } { if (!args.some((arg) => TARGET_TOKEN.test(arg))) return args; const snapshot = run(ctx, ['snapshot', '-i']); const nodes = ((snapshot.json as { data?: { nodes?: SnapshotNode[] } })?.data?.nodes ?? @@ -92,7 +107,7 @@ function resolveTargets(args: string[], ctx: IosSuiteContext): string[] | { erro /** Runs untimed calls in order and describes the first failure. */ function untimed( - ctx: IosSuiteContext, + ctx: SuiteContext, calls: readonly (readonly string[])[] = [], ): string | undefined { for (const args of calls) { @@ -117,14 +132,14 @@ function errorOf(result: CliResult): Pick }; } -function readProfile(ctx: IosSuiteContext, requestId: string | undefined) { +function readProfile(ctx: SuiteContext, requestId: string | undefined) { if (!requestId) return undefined; const sessionDir = resolveSessionDir(path.join(ctx.stateDir, 'sessions'), SESSION); const logPath = resolveSessionRequestLogPath(sessionDir, requestId); return fs.existsSync(logPath) ? profileRequestLog(fs.readFileSync(logPath, 'utf8')) : undefined; } -function recordFailure(ctx: IosSuiteContext, id: string, round: number, detail: unknown): void { +function recordFailure(ctx: SuiteContext, id: string, round: number, detail: unknown): void { const dir = path.join(ctx.rawDir, 'failures'); fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync( @@ -133,15 +148,37 @@ function recordFailure(ctx: IosSuiteContext, id: string, round: number, detail: ); } +/** + * Restarts the daemon with `env` and reopens the session on the current screen. A daemon reads + * its host environment once, at start. + */ +function restartDaemon( + ctx: SuiteContext, + env: Readonly> | undefined, +): string | undefined { + const stop = invokeCli(['daemon', 'stop'], ['--state-dir', ctx.stateDir]); + if (!stop.ok) return `daemon stop: ${errorOf(stop).errorMessage}`; + ctx.daemonEnv = env; + const failure = sessionError(openSession(ctx, { relaunch: false })); + // A fresh daemon holds no request from before the restart. + if (!failure) ctx.daemonStuck = false; + return failure; +} + async function runTimedStep( scenario: SuiteScenario, - step: SuiteStep, - ctx: IosSuiteContext, + step: PlatformStep, + ctx: SuiteContext, round: number, ): Promise { const id = `${scenario.id}/${step.id}`; - const before = untimed(ctx, step.before); + const restoreDaemon = (): void => { + if (step.daemonEnv) ctx.restoreFailed = restartDaemon(ctx, undefined); + }; + const before = + (step.daemonEnv ? restartDaemon(ctx, step.daemonEnv) : undefined) ?? untimed(ctx, step.before); if (before) { + restoreDaemon(); log(` ✗ ${id} (before_failed: ${before})`); return { round, ok: false, wallClockMs: 0, errorCode: 'before_failed', errorMessage: before }; } @@ -150,7 +187,7 @@ async function runTimedStep( const { result: forced, requestId } = await captureWithForcedBackend({ stateDir: ctx.stateDir, session: SESSION, - udid: ctx.udid, + udid: ctx.device, backend: 'tree', }); const result = watch(ctx, forced); @@ -176,7 +213,7 @@ async function runTimedStep( errorMessage: args.error, }; } else { - const result = watch(ctx, invokeCli([...args, '--debug'], baseFlags(ctx))); + const result = cli(ctx, [...args, '--debug']); const diagnostics = readClientDiagnostics(result.stderr); sample = { round, @@ -201,6 +238,7 @@ async function runTimedStep( const violation = sample.ok ? expectationViolation(step.expect, sample.snapshot) : undefined; if (violation) Object.assign(sample, { ok: false, errorCode: 'expectation_failed', errorMessage: violation }); + restoreDaemon(); const after = untimed(ctx, step.after); if (after) log(` ! ${id} after: ${after}`); if (!sample.ok) log(` ✗ ${id} (${sample.errorCode}: ${sample.errorMessage})`); @@ -214,23 +252,31 @@ function sessionError(result: CliResult): string | undefined { return message; } -function stopStuckDaemon(ctx: IosSuiteContext): void { - log(' ! a request outlived its timeout; restarting the daemon'); +function stopStuckDaemon(ctx: SuiteContext): void { + log( + ctx.restoreFailed + ? ` ! the daemon restore failed (${ctx.restoreFailed}); restarting the daemon` + : ' ! a request outlived its timeout; restarting the daemon', + ); invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); ctx.daemonStuck = false; + ctx.daemonEnv = undefined; + ctx.restoreFailed = undefined; } /** Opens the session with device selectors; later calls address it by session name alone. */ -function establishSession(ctx: IosSuiteContext): CliResult { - if (ctx.daemonStuck) stopStuckDaemon(ctx); - return watch( - ctx, - invokeCli(['open', ctx.appId, '--relaunch', '--udid', ctx.udid], baseFlags(ctx)), - ); +function openSession(ctx: SuiteContext, options: { relaunch: boolean }): CliResult { + const relaunch = options.relaunch ? ['--relaunch'] : []; + return cli(ctx, ['open', ctx.appId, ...relaunch, ...deviceFlags(ctx)]); +} + +function establishSession(ctx: SuiteContext): CliResult { + if (ctx.daemonStuck || ctx.restoreFailed) stopStuckDaemon(ctx); + return openSession(ctx, { relaunch: true }); } -export async function runIosSuite( - ctx: IosSuiteContext, +export async function runSuite( + ctx: SuiteContext, cfg: PerfConfig, scenarios: readonly SuiteScenario[], ): Promise { @@ -241,8 +287,10 @@ export async function runIosSuite( log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`); let sessionFailure = sessionError(establishSession(ctx)); for (const scenario of scenarios) { - // A request that outlived its timeout still holds the session; reopen before the next scenario. - if (ctx.daemonStuck) sessionFailure = sessionError(establishSession(ctx)); + // A request that outlived its timeout still holds the session, and a failed daemon restore + // lost it; reopen before the next scenario. + if (ctx.daemonStuck || ctx.restoreFailed) + sessionFailure = sessionError(establishSession(ctx)); log(` scenario ${scenario.id}`); const setupFailure = sessionFailure ?? untimed(ctx, scenario.setup); if (setupFailure) log(` ✗ setup failed: ${setupFailure}`); @@ -252,12 +300,14 @@ export async function runIosSuite( const id = `${scenario.id}/${step.id}`; const skipped = setupFailure ? { errorCode: 'setup_failed', errorMessage: setupFailure } - : ctx.daemonStuck - ? { - errorCode: 'daemon_stuck', - errorMessage: 'an earlier request still holds the session', - } - : undefined; + : ctx.restoreFailed + ? { errorCode: 'restore_failed', errorMessage: ctx.restoreFailed } + : ctx.daemonStuck + ? { + errorCode: 'daemon_stuck', + errorMessage: 'an earlier request still holds the session', + } + : undefined; const sample: Sample = skipped ? { round, ok: false, wallClockMs: 0, ...skipped } : await runTimedStep(scenario, step, ctx, round); @@ -273,7 +323,7 @@ export async function runIosSuite( label: step.args.join(' '), command: step.command, args: [...step.args], - platform: 'ios', + platform: ctx.platform, execMode: step.via === 'sdk-forced-tree' ? 'sdk' : 'standalone', samples: samples.get(`${scenario.id}/${step.id}`) ?? [], warmupDropped: cfg.warmup, @@ -290,7 +340,7 @@ const TEARDOWN_TIMEOUT_MS = 60_000; * the logs again with the teardown, and remove the state directory unless it is kept. It runs once, * so the interrupt handler and the normal finish can both call it. */ -export function iosRunFinalizer(ctx: IosSuiteContext, keepArtifacts: boolean): () => void { +export function runFinalizer(ctx: SuiteContext, keepArtifacts: boolean): () => void { let finalized = false; const saveLogs = (): void => { try { @@ -315,7 +365,7 @@ export function iosRunFinalizer(ctx: IosSuiteContext, keepArtifacts: boolean): ( } /** Copies daemon and session logs into the raw artifact; the daemon token file stays behind. */ -function copyRawState(ctx: IosSuiteContext): void { +function copyRawState(ctx: SuiteContext): void { const target = path.join(ctx.rawDir, 'state'); fs.mkdirSync(target, { recursive: true }); const daemonLog = path.join(ctx.stateDir, 'daemon.log'); diff --git a/scripts/perf/suite.test.ts b/scripts/perf/suite.test.ts new file mode 100644 index 0000000000..22ec9c85d4 --- /dev/null +++ b/scripts/perf/suite.test.ts @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import { test } from 'vitest'; +import { + describeCoverage, + expectationViolation, + SUITE, + SUITE_EXCLUSIONS, + suiteFor, + type SuiteScenario, + type SuiteStep, +} from './suite.ts'; +import type { Platform } from './types.ts'; + +const PLATFORMS: readonly Platform[] = ['ios', 'android']; + +test('on each platform, every public command is either measured or excluded with a reason', () => { + for (const platform of PLATFORMS) { + const { measured, excluded } = describeCoverage(suiteFor(platform), SUITE_EXCLUSIONS[platform]); + const classified = [...measured, ...excluded.map((e) => e.command)].sort(); + assert.deepEqual(classified, Object.values(PUBLIC_COMMANDS).sort(), platform); + for (const { reason } of excluded) assert.ok(reason.length > 0); + } +}); + +test('step ids are unique per platform, so comparisons key each step once', () => { + for (const platform of PLATFORMS) { + const ids = suiteFor(platform).flatMap((scenario) => + scenario.steps.map((s) => `${scenario.id}/${s.id}`), + ); + assert.equal(new Set(ids).size, ids.length, platform); + } +}); + +test('a platform suite drops the other platform steps and applies its own deltas', () => { + const shared = { id: 'swipe', command: PUBLIC_COMMANDS.swipe, args: ['swipe', '1'] }; + const suite: SuiteScenario[] = [ + { + id: 'scenario', + title: 'scenario', + setup: [], + steps: [ + { ...shared, on: { android: { args: ['swipe', '2'], before: [] } }, before: [['wait']] }, + { id: 'ios-only', command: PUBLIC_COMMANDS.actionButton, args: [], only: 'ios' }, + ], + }, + ]; + assert.deepEqual(suiteFor('ios', suite)[0]!.steps, [ + { ...shared, before: [['wait']] }, + { id: 'ios-only', command: PUBLIC_COMMANDS.actionButton, args: [] }, + ]); + assert.deepEqual(suiteFor('android', suite)[0]!.steps, [ + { ...shared, args: ['swipe', '2'], before: [] }, + ]); +}); + +test('every platform runs at least one step of every scenario', () => { + for (const scenario of SUITE) { + for (const platform of PLATFORMS) { + const resolved = suiteFor(platform).find((s) => s.id === scenario.id); + assert.ok(resolved && resolved.steps.length > 0, `${platform} ${scenario.id}`); + } + } +}); + +test('a heavy-feed capture breaks its expectation when the tree flattens', () => { + const depth = { maxDepthAtLeast: 64 }; + assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); + assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); + assert.match(expectationViolation(depth, undefined)!, /unknown/); +}); + +test('a pinned capture breaks its expectation unless the pinned backend served it', () => { + const pinned = { pinnedBackend: 'tree' } as const; + const served = { backend: 'tree', state: 'healthy', reasonCode: 'requested-backend' }; + assert.equal(expectationViolation(pinned, served), undefined); + // The raw plan ignores the pin and recovers through private AX on the heavy feed. + assert.match( + expectationViolation(pinned, { + backend: 'private-ax', + state: 'recovered', + reasonCode: 'ax-rejected', + })!, + /private-ax backend \(ax-rejected\)/, + ); + // An unpinned capture can still be tree-first on a light screen. + assert.match( + expectationViolation(pinned, { backend: 'tree', state: 'healthy' })!, + /no reason code/, + ); + assert.match(expectationViolation(pinned, undefined)!, /unknown backend/); +}); + +test('an Android capture breaks its expectation when another helper transport served it', () => { + const oneShot = { transport: 'instrumentation' }; + assert.equal(expectationViolation(oneShot, { transport: 'instrumentation' }), undefined); + assert.match( + expectationViolation(oneShot, { transport: 'persistent-session' })!, + /persistent-session transport served the capture, not instrumentation/, + ); + assert.match(expectationViolation(oneShot, {})!, /unknown transport/); +}); diff --git a/scripts/perf/ios-suite.ts b/scripts/perf/suite.ts similarity index 66% rename from scripts/perf/ios-suite.ts rename to scripts/perf/suite.ts index efc7795de1..27f09f47a5 100644 --- a/scripts/perf/ios-suite.ts +++ b/scripts/perf/suite.ts @@ -3,7 +3,7 @@ import type { SnapshotPreferredBackend, SnapshotQualityVerdict, } from '@agent-device/kernel/snapshot'; -import type { SnapshotFacts } from './types.ts'; +import type { Platform, SnapshotFacts } from './types.ts'; /** Routes `trigger-app-event` into the fixture's automation screen. */ export const FIXTURE_APP_EVENT_URL_TEMPLATE = @@ -21,7 +21,7 @@ type Cli = readonly string[]; * or center point of the element with that identifier in an untimed `snapshot -i` taken just * before the step. `before` and `after` run untimed around the step in every round. */ -export type SuiteStep = { +export type PlatformStep = { id: string; command: PublicCommand; args: Cli; @@ -30,17 +30,33 @@ export type SuiteStep = { // `sdk-forced-tree` captures a snapshot through the daemon with the XCTest tree backend pinned; // the CLI has no flag for it. via?: 'cli' | 'sdk-forced-tree'; + // The step runs against a daemon started with these variables. The harness restarts the daemon + // and reopens the session on the current screen, untimed, before and after the step. + daemonEnv?: Readonly>; // A sample that succeeds but breaks the expectation fails with `expectation_failed`, so a // fixture that stops reproducing its shape shows up as a new failure. expect?: StepExpectation; }; +type StepDelta = Partial>; + +/** + * A step of the shared suite. It runs on every platform unless `only` names one. `on` replaces + * fields on one platform, for example screen coordinates or a platform-only wait. + */ +export type SuiteStep = PlatformStep & { + only?: Platform; + on?: Partial>; +}; + export type StepExpectation = { // Deepest raw node depth, counted from the application at 0. maxDepthAtLeast?: number; // The pinned backend that must serve the capture. The runner marks a pinned capture with the // reason code `requested-backend`; any other backend or code means the pin was not honored. pinnedBackend?: SnapshotPreferredBackend; + // The Android helper transport that served the capture. + transport?: string; }; const PINNED_REASON_CODE: SnapshotQualityVerdict['reasonCode'] = 'requested-backend'; @@ -60,17 +76,33 @@ export function expectationViolation( ) { return `the ${snapshot?.backend ?? 'unknown'} backend (${snapshot?.reasonCode ?? 'no reason code'}) served the capture, not the pinned ${expect.pinnedBackend}`; } + if (expect.transport !== undefined && snapshot?.transport !== expect.transport) { + return `the ${snapshot?.transport ?? 'unknown'} transport served the capture, not ${expect.transport}`; + } return undefined; } /** A scenario starts from `setup` (untimed, every round) and runs its steps in order. */ -export type SuiteScenario = { +export type SuiteScenario = { id: string; title: string; setup: readonly Cli[]; - steps: SuiteStep[]; + steps: TStep[]; }; +/** Resolves the shared suite for one platform: drops other platforms' steps, applies deltas. */ +export function suiteFor( + platform: Platform, + suite: readonly SuiteScenario[] = SUITE, +): SuiteScenario[] { + return suite.map(({ steps, ...scenario }) => ({ + ...scenario, + steps: steps + .filter((s) => s.only === undefined || s.only === platform) + .map(({ only: _only, on, ...shared }) => ({ ...shared, ...on?.[platform] })), + })); +} + // A cold relaunch on a hosted runner can take longer than 15 s before the first landmark. const LANDMARK_WAIT_MS = '60000'; const relaunch: Cli = ['open', '{app}', '--relaunch']; @@ -86,6 +118,7 @@ const toSettingsLab = (button: string, landmark: string): Cli[] => [ ['click', `id="${button}"`], ['wait', landmark, LANDMARK_WAIT_MS], ]; +const SWIPE_COUNT_FLAGS = ['--count', '2', '--pause-ms', '50', '--pattern', 'ping-pong'] as const; const step = ( id: string, command: PublicCommand, @@ -93,7 +126,7 @@ const step = ( extra: Omit = {}, ): SuiteStep => ({ id, command, args, ...extra }); -export const IOS_SUITE: readonly SuiteScenario[] = [ +export const SUITE: readonly SuiteScenario[] = [ { id: 'lifecycle', title: 'App lifecycle, device state, and inventory', @@ -107,11 +140,30 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('orientation-landscape', C.orientation, ['orientation', 'landscape-left']), step('orientation-portrait', C.orientation, ['orientation', 'portrait']), step('settings-appearance', C.settings, ['settings', 'appearance', 'light']), - step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf']), - step('clipboard-read', C.clipboard, ['clipboard', 'read']), - step('action-button', C.actionButton, ['action-button']), - step('prepare-runner', C.prepare, ['prepare', 'ios-runner']), - step('devices', C.devices, ['devices', '--platform', 'ios']), + step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf'], { + only: 'ios', + }), + step('clipboard-read', C.clipboard, ['clipboard', 'read'], { only: 'ios' }), + step('action-button', C.actionButton, ['action-button'], { only: 'ios' }), + // `wake` is the one hardware key that leaves an awake device as it was. + step('hardware-button', C.hardwareButton, ['hardware-button', 'wake'], { only: 'android' }), + step( + 'push', + C.push, + [ + 'push', + '{app}', + '{"action":"com.callstack.agentdevicelab.PERF","extras":{"source":"perf"}}', + ], + { only: 'android' }, + ), + step('telephony-sms', C.telephony, ['telephony', 'sms', '5551234', 'agent-device perf'], { + only: 'android', + // The SMS notification can cover the app; reopen it for the next step. + after: [['open', '{app}']], + }), + step('prepare-runner', C.prepare, ['prepare', 'ios-runner'], { only: 'ios' }), + step('devices', C.devices, ['devices']), step('apps', C.apps, ['apps']), step('capabilities', C.capabilities, ['capabilities']), step('doctor', C.doctor, ['doctor']), @@ -128,6 +180,8 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ ['alert', 'accept'], ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], ], + // Android opens the event URL without a prompt. + on: { android: { after: [['wait', 'id="automation-title"', LANDMARK_WAIT_MS]] } }, }, ), ], @@ -143,6 +197,7 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ // Only the regular capture plan honors the pin; the raw plan runs its own backend ladder. The // XCTest tree rejects the heavy feed, so the pinned capture runs on this light screen. step('snapshot-forced-tree', C.snapshot, ['snapshot'], { + only: 'ios', via: 'sdk-forced-tree', expect: { pinnedBackend: 'tree' }, }), @@ -228,20 +283,13 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), step('scroll-top', C.scroll, ['scroll', 'top']), step('scroll-until', C.scroll, ['scroll', 'down', '--until', 'id="catalog-footer"']), - step('swipe', C.swipe, ['swipe', '200', '300', '200', '650']), - step('swipe-count', C.swipe, [ - 'swipe', - '200', - '650', - '200', - '450', - '--count', - '2', - '--pause-ms', - '50', - '--pattern', - 'ping-pong', - ]), + // Coordinates are iOS points and Android pixels; the Android values keep the same screen share. + step('swipe', C.swipe, ['swipe', '200', '300', '200', '650'], { + on: { android: { args: ['swipe', '540', '800', '540', '1750'] } }, + }), + step('swipe-count', C.swipe, ['swipe', '200', '650', '200', '450', ...SWIPE_COUNT_FLAGS], { + on: { android: { args: ['swipe', '540', '1750', '540', '1200', ...SWIPE_COUNT_FLAGS] } }, + }), ], }, { @@ -254,6 +302,8 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('type-suffix', C.type, ['type', '.test']), step('keyboard-dismiss', C.keyboard, ['keyboard', 'dismiss'], { before: [['wait', 'label="Done" visible=true', '10000']], + // Android emulators type through the headless test IME, which has no Done key. + on: { android: { before: [] } }, }), step('press-checkbox', C.press, ['press', 'id="checkbox-agree"'], { before: [['scroll', 'down', '--until', 'id="checkbox-agree"']], @@ -315,12 +365,16 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ }, { id: 'heavy-feed', - title: 'Heavy feed: Bluesky-depth tree (≥65 native levels) and virtualized list', + title: 'Heavy feed: Bluesky-depth tree and virtualized list', setup: toSettingsLab('open-heavy-feed', 'id="heavy-feed-title"'), steps: [ step('snapshot-i', C.snapshot, ['snapshot', '-i']), step('snapshot', C.snapshot, ['snapshot']), - step('snapshot-raw', C.snapshot, ['snapshot', '--raw'], { expect: { maxDepthAtLeast: 64 } }), + // The fixture renders more than 65 native levels on iOS and 60 on Android. + step('snapshot-raw', C.snapshot, ['snapshot', '--raw'], { + expect: { maxDepthAtLeast: 64 }, + on: { android: { expect: { maxDepthAtLeast: 56 } } }, + }), step('press-like-selector', C.press, ['press', 'id="heavy-post-0-like"']), step('press-like-settle', C.press, ['press', 'id="heavy-post-1-like"', '--settle']), step('press-like-ref', C.press, ['press', '{ref:heavy-post-0-like}']), @@ -333,7 +387,19 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('scroll-down', C.scroll, ['scroll', 'down']), step('snapshot-i-scrolled', C.snapshot, ['snapshot', '-i']), step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), - step('swipe', C.swipe, ['swipe', '200', '650', '200', '300']), + step('swipe', C.swipe, ['swipe', '200', '650', '200', '300'], { + on: { android: { args: ['swipe', '540', '1750', '540', '800'] } }, + }), + // Android has one snapshot engine. Its other path is the one-shot `am instrument` capture that + // serves a request when the helper session is off or fails; only the daemon environment + // turns the session off. + step('snapshot-raw-one-shot', C.snapshot, ['snapshot', '--raw'], { + only: 'android', + daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' }, + // The restarted daemon verifies the helper install on its first capture; keep that out. + before: [['snapshot', '-i']], + expect: { maxDepthAtLeast: 56, transport: 'instrumentation' }, + }), step('back', C.back, ['back']), ], }, @@ -345,32 +411,62 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ }, ]; -/** - * Catalog commands the iOS suite does not time, with the reason. Every public command is either - * measured by a step above or listed here; `ios-suite.test.ts` holds that partition. - */ -export const IOS_SUITE_EXCLUSIONS: Partial> = { - [C.boot]: - 'Simulator power cycle: CoreSimulator boot time dominates and it ends the session under test.', - [C.shutdown]: 'Simulator power cycle: it ends the session and the runner under test.', - [C.install]: - 'Bundle provisioning: simctl copy time dominates; the job installs the fixture once.', - [C.reinstall]: - 'Bundle provisioning: simctl copy time dominates and it resets the app under test.', - [C.uninstall]: 'Bundle provisioning: it removes the app under test.', - [C.installFromSource]: - 'Bundle provisioning from a source build; the job installs a cached Release fixture.', +const COMPOSITE_RUNNERS: Partial> = { [C.replay]: 'Composite runner: a replay is the sum of the measured commands it runs.', [C.test]: 'Composite runner: a test suite is the sum of the measured commands it runs.', - [C.push]: - 'The fixture holds no notification authorization, so simctl refuses the payload; iOS push coverage is contract-only.', - [C.audio]: 'Host audio capture needs a macOS audio permission that hosted runners do not grant.', - [C.hover]: 'The iOS runtime refuses hover; it is a web pointer contract.', - [C.viewport]: 'The iOS runtime refuses viewport; it resizes web viewports only.', - [C.tvRemote]: 'tvOS only.', - [C.fold]: 'iPhone Duo only; the suite device has one panel.', - [C.telephony]: 'The iOS simulator runtime refuses simulated calls and SMS.', - [C.hardwareButton]: 'The iOS simulator runtime refuses hardware-button presses.', +}; + +/** + * Catalog commands that the suite does not time on each platform, with the reason. On each + * platform, every public command is either measured by a step above or listed here; + * `suite.test.ts` holds that partition. + */ +export const SUITE_EXCLUSIONS: Record>> = { + ios: { + ...COMPOSITE_RUNNERS, + [C.boot]: + 'Simulator power cycle: CoreSimulator boot time dominates and it ends the session under test.', + [C.shutdown]: 'Simulator power cycle: it ends the session and the runner under test.', + [C.install]: + 'Bundle provisioning: simctl copy time dominates; the job installs the fixture once.', + [C.reinstall]: + 'Bundle provisioning: simctl copy time dominates and it resets the app under test.', + [C.uninstall]: 'Bundle provisioning: it removes the app under test.', + [C.installFromSource]: + 'Bundle provisioning from a source build; the job installs a cached Release fixture.', + [C.push]: + 'The fixture holds no notification authorization, so simctl refuses the payload; iOS push coverage is contract-only.', + [C.audio]: + 'Host audio capture needs a macOS audio permission that hosted runners do not grant.', + [C.hover]: 'The iOS runtime refuses hover; it is a web pointer contract.', + [C.viewport]: 'The iOS runtime refuses viewport; it resizes web viewports only.', + [C.tvRemote]: 'tvOS only.', + [C.fold]: 'iPhone Duo only; the suite device has one panel.', + [C.telephony]: 'The iOS simulator runtime refuses simulated calls and SMS.', + [C.hardwareButton]: 'The iOS simulator runtime refuses hardware-button presses.', + }, + android: { + ...COMPOSITE_RUNNERS, + [C.boot]: + 'Emulator power cycle: emulator boot time dominates and it ends the session under test.', + [C.shutdown]: 'Emulator power cycle: it ends the session and the helper under test.', + [C.install]: 'APK provisioning: adb install time dominates; the job installs the fixture once.', + [C.reinstall]: 'APK provisioning: adb install time dominates and it resets the app under test.', + [C.uninstall]: 'APK provisioning: it removes the app under test.', + [C.installFromSource]: + 'APK provisioning from a source build; the job installs a cached Release fixture.', + [C.prepare]: + 'prepare has only the ios-runner action; the snapshot helper installs on first use.', + [C.actionButton]: 'Android has no key event for the Action button.', + [C.clipboard]: + 'The API 35 and 36 emulator images ship no shell clipboard service, so adb cannot read or write the clipboard.', + [C.audio]: + 'Host audio capture uses macOS ScreenCaptureKit; the Linux emulator runner has none.', + [C.hover]: 'The Android runtime refuses hover; it is a web pointer contract.', + [C.viewport]: 'The Android runtime refuses viewport; it resizes web viewports only.', + [C.tvRemote]: 'Android TV only.', + [C.fold]: 'Foldable emulators only; the suite emulator is a pixel_7 with one display.', + }, }; /** Partitions the measured and excluded catalog commands for the report. */ diff --git a/scripts/perf/summary.fixtures.ts b/scripts/perf/summary.fixtures.ts index 5ad12534e6..b11085fddf 100644 --- a/scripts/perf/summary.fixtures.ts +++ b/scripts/perf/summary.fixtures.ts @@ -1,5 +1,11 @@ import { buildMeasurement } from './measurement.ts'; -import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary, type Sample } from './types.ts'; +import { + PERF_SUMMARY_SCHEMA, + type Measurement, + type PerfSummary, + type Platform, + type Sample, +} from './types.ts'; export type StepFixture = { id: string; @@ -37,11 +43,11 @@ function measurement(step: StepFixture): Measurement { /** A schema-valid summary whose steps carry the given timings. */ export function summaryFixture( steps: StepFixture[], - overrides: { startedAt?: string; xcode?: string; runId?: string } = {}, + overrides: { startedAt?: string; xcode?: string; runId?: string; platform?: Platform } = {}, ): PerfSummary { return { schema: PERF_SUMMARY_SCHEMA, - platform: 'ios', + platform: overrides.platform ?? 'ios', startedAt: overrides.startedAt ?? '2026-10-20T03:00:00.000Z', finishedAt: '2026-10-20T03:30:00.000Z', config: { diff --git a/scripts/perf/types.ts b/scripts/perf/types.ts index cfb14ffd2f..18b3d5c449 100644 --- a/scripts/perf/types.ts +++ b/scripts/perf/types.ts @@ -2,7 +2,7 @@ export type Platform = 'ios' | 'android'; -export type ExecMode = 'batch' | 'standalone' | 'sdk'; +export type ExecMode = 'standalone' | 'sdk'; export type CliResult = { exitCode: number; @@ -44,6 +44,8 @@ export type SettleFacts = { waitedMs: number; captures?: number; settled?: boole export type SnapshotFacts = { backend?: string; + // The Android helper transport: `persistent-session` or `instrumentation`. + transport?: string; state?: string; reasonCode?: string; nodeCount?: number; @@ -54,10 +56,9 @@ export type SnapshotFacts = { export type Sample = { round: number; wallClockMs: number; - // Daemon request round trip: the client's `daemon_request` timer for CLI calls, the batch step - // duration for batch calls, or the request-log span for SDK calls. + // Daemon request round trip: the client's `daemon_request` timer for CLI calls, or the + // request-log span for SDK calls. daemonMs?: number; - elementCount?: number; requestId?: string; snapshot?: SnapshotFacts; settle?: SettleFacts;