feat(perf): time every Android command against the fixture app

.github/workflows/perf-nightly.yml

@@ -1,16 +1,17 @@
 name: Perf Nightly
 
-# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3).
+# Nightly iOS and Android command performance against the fixture app, compared night over night
+# per platform (#1781 A3).
 # - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts
 #   failing. PR lanes check behavior, not time.
 # - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians
 #   swing up to +122% between single nights, so a step is judged against the median of up to seven
 #   earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. A night
 #   runs 3 rounds, the minimum for a timing verdict, so one failed sample withholds that step's
 #   timing verdict.
-# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30.
+# - Cost: one macos-26 job of about 45 minutes and one ubuntu emulator job a night; summaries are
+#   kept 90 days, raw logs 30.
 # - Delete or park when: no flagged change has been acted on for 60 nights.
-# Android stays report-only and runs only on request.
 
 on:
   schedule:
@@ -26,7 +27,7 @@ on:
         required: false
         default: '1'
       scenarios:
-        description: 'Comma-separated iOS scenario ids (empty runs all)'
+        description: 'Comma-separated scenario ids (empty runs all)'
         required: false
         default: ''
       baseline-nights:
@@ -37,8 +38,13 @@ on:
         description: 'Branch whose runs form the baseline'
         required: false
         default: 'main'
+      ios:
+        description: 'Run the iOS suite'
+        type: boolean
+        required: false
+        default: true
       android:
-        description: 'Also run the report-only Android Settings tour'
+        description: 'Run the Android suite'
         type: boolean
         required: false
         default: true
@@ -58,6 +64,7 @@ env:
 jobs:
   perf-ios:
     name: iOS Command Perf
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.ios == 'true' }}
     runs-on: macos-26
     timeout-minutes: 100
     env:
@@ -133,7 +140,7 @@ jobs:
           BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }}
           BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }}
         run: |
-          pnpm perf:compare --to perf-results \
+          pnpm perf:compare --platform ios --to perf-results \
             --branch "$BASELINE_BRANCH" \
             --nights "$BASELINE_NIGHTS" \
             --exclude-run "$GITHUB_RUN_ID" \
@@ -164,26 +171,44 @@ jobs:
 
   perf-android:
     name: Android Command Perf
-    if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.android == 'true' }}
     runs-on: ubuntu-latest
-    timeout-minutes: 80
-    env:
-      AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts'
+    timeout-minutes: 100
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Setup toolchain
         uses: ./.github/actions/setup-node-pnpm
+        with:
+          cache-dependency-path: |
+            pnpm-lock.yaml
+            examples/test-app/pnpm-lock.yaml
+
+      - name: Build the shared CLI
+        uses: ./.github/actions/run-gate
+        with:
+          gate: build
 
       - name: Setup Android replay host
-        id: android-replay-host
         uses: ./.github/actions/setup-android-replay-host
         with:
           package-helpers: 'true'
 
-      - name: Run Android command perf benchmark
+      - name: Restore the fixture APK
+        id: fixture-app
+        uses: ./.github/actions/setup-fixture-app
+        with:
+          platform: android
+          install: 'false'
+
+      # The emulator action runs each script line on its own, so the script is one folded line.
+      - name: Run Android command perf suite
         uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0
+        env:
+          APP_ID: ${{ steps.fixture-app.outputs.app-id }}
+          APP_PATH: ${{ steps.fixture-app.outputs.apk-path }}
+          SCENARIOS: ${{ github.event.inputs.scenarios }}
         with:
           api-level: 36
           arch: x86_64
@@ -192,15 +217,45 @@ jobs:
           emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics
           script: >-
             set -e;
-            adb -s emulator-5554 shell settings put global window_animation_scale 0 || true;
-            adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true;
-            adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true;
-            pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results"
+            adb -s emulator-5554 install -r "$APP_PATH";
+            pnpm perf --platform android --serial emulator-5554
+            --app-id "$APP_ID" --app-path "$APP_PATH"
+            --rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP"
+            ${SCENARIOS:+--scenarios "$SCENARIOS"}
+            --out-dir "$GITHUB_WORKSPACE/perf-results"
+
+      - name: Compare with earlier nights
+        if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }}
+          BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }}
+        run: |
+          pnpm perf:compare --platform android --to perf-results \
+            --branch "$BASELINE_BRANCH" \
+            --nights "$BASELINE_NIGHTS" \
+            --exclude-run "$GITHUB_RUN_ID" \
+            --out perf-results --github
+          echo "Full report: the perf-android-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload Android perf summary
+        if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: perf-android-summary
+          path: |
+            perf-results/summary.json
+            perf-results/report.md
+            perf-results/comparison.json
+            perf-results/comparison.md
+          retention-days: 90
+          if-no-files-found: error
 
-      - name: Upload Android perf report
-        if: always()
+      - name: Upload Android perf raw logs
+        if: ${{ always() }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: perf-android
-          path: perf-results/
+          name: perf-android-raw
+          path: perf-results/raw/
+          retention-days: 30
           if-no-files-found: warn

packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts

@@ -0,0 +1,74 @@
+import { afterEach, beforeEach, test } from 'vitest';
+import assert from 'node:assert/strict';
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { withDiagnosticsScope } from '@agent-device/host-kit/diagnostics';
+import type { DeviceInfo } from '@agent-device/kernel/device';
+import {
+  ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE as ONE_SHOT_PHASE,
+  ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE as SESSION_PHASE,
+  snapshotAndroid,
+} from '../snapshot.ts';
+import { resetAndroidSnapshotHelperInstallCache } from '../snapshot-helper-install.ts';
+import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts';
+import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts';
+import { createPersistentSnapshotHelperProvider } from './snapshot-helper-session.fixtures.ts';
+import { mkdtempForTest } from './test-utils/tmp-dir.ts';
+
+const device: DeviceInfo = {
+  platform: 'android',
+  id: 'emulator-5554',
+  name: 'Pixel',
+  kind: 'emulator',
+  booted: true,
+};
+
+beforeEach(async () => {
+  await resetAndroidSnapshotHelperSessions();
+  resetAndroidSnapshotHelperInstallCache();
+});
+
+afterEach(async () => {
+  await resetAndroidSnapshotHelperSessions();
+});
+
+async function captureTimers(sessionResponseMode?: 'malformed') {
+  const provider = createPersistentSnapshotHelperProvider({
+    calls: [],
+    spawnArgs: [],
+    processes: [],
+    sessionResponseMode,
+    oneShotXml: '<hierarchy><node text="one-shot fallback" bounds="[0,0][10,10]" /></hierarchy>',
+  });
+  const tmpDir = await mkdtempForTest('agent-device-android-capture-timers-');
+  const logPath = path.join(tmpDir, 'diag.ndjson');
+  try {
+    await withDiagnosticsScope(
+      { debug: true, logPath, session: 'android-test', requestId: 'req-1', command: 'snapshot' },
+      async () =>
+        await snapshotAndroid(device, {
+          helperAdb: provider,
+          helperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT,
+        }),
+    );
+    return (await fs.readFile(logPath, 'utf8'))
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => JSON.parse(line) as { phase: string; data?: Record<string, unknown> })
+      .filter(({ phase }) => phase === SESSION_PHASE || phase === ONE_SHOT_PHASE)
+      .map(({ phase, data }) => ({ phase, transport: data?.transport, answered: data?.answered }));
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+}
+
+test('each helper capture timer records its transport and whether the session answered', async () => {
+  assert.deepEqual(await captureTimers(), [
+    { phase: SESSION_PHASE, transport: 'persistent-session', answered: true },
+  ]);
+  await resetAndroidSnapshotHelperSessions();
+  assert.deepEqual(await captureTimers('malformed'), [
+    { phase: SESSION_PHASE, transport: 'persistent-session', answered: false },
+    { phase: ONE_SHOT_PHASE, transport: 'instrumentation', answered: undefined },
+  ]);
+});

packages/platform-android/src/mechanics.ts

@@ -308,7 +308,12 @@ export {
   resetAndroidSnapshotHelperRuntime,
   retireAndroidSnapshotHelperAfterContentFailure,
 } from './snapshot-helper-runtime.ts';
-export { snapshotAndroid, type AndroidSnapshotOptions } from './snapshot.ts';
+export {
+  ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE,
+  ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE,
+  snapshotAndroid,
+  type AndroidSnapshotOptions,
+} from './snapshot.ts';
 export { screenshotAndroid, type AndroidScreenshotOptions } from './screenshot.ts';
 export {
   ANDROID_SNAPSHOT_PRESENTATION_DEFAULT_DEADLINE_MS,
@@ -337,6 +342,7 @@ export { ensureAndroidSdkPathConfigured } from './sdk.ts';
 export { fillAndroid, typeAndroid } from './text-input.ts';
 export { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts';
 export {
+  ANDROID_TOUCH_HELPER_GESTURE_PHASE,
   ANDROID_TOUCH_PLAN_PROTOCOL,
   executeAndroidTouchHelperPlan,
   normalizeAndroidTouchHelperGestureRequest,

packages/platform-android/src/snapshot.ts

@@ -36,6 +36,7 @@ import {
   ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS,
   type AndroidHelperSessionScope,
   type AndroidSnapshotHelperCaptureOptions,
+  type AndroidSnapshotHelperTransport,
 } from './snapshot-helper-types.ts';
 import {
   captureAndroidSnapshotWithHelper,
@@ -102,6 +103,19 @@ const HELPER_CONTENT_RECAPTURE_DELAY_MS = 250;
  * a busy screen run a snapshot past the daemon request envelope.
  */
 const HELPER_CONTENT_RECAPTURE_WINDOW_MS = 10_000;
+/**
+ * The diagnostic phase of a capture through the snapshot helper's persistent session. Its
+ * `data.transport` names the session transport, and `data.answered` is false when the session
+ * returned no capture (off, backed off, failed to start, or retired) and a one-shot capture follows.
+ */
+export const ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE =
+  'android_snapshot_helper_session_capture';
+/**
+ * The diagnostic phase of a one-shot `am instrument` snapshot helper capture. Its `data.transport`
+ * names the instrumentation transport.
+ */
+export const ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE = 'android_snapshot_helper_capture';
+
 export type AndroidSnapshotOptions = SnapshotOptions & {
   idleWait?: AndroidSnapshotHelperCaptureOptions['idleWait'];
   appBundleId?: string;
@@ -463,15 +477,18 @@ async function captureAndroidUiHierarchyFromHelper(params: {
     signal,
   });
   captureOptions.idleWait = params.idleWait;
+  let sessionCapture: AndroidSnapshotHelperOutput | undefined;
   try {
-    const sessionCapture = await withDiagnosticTimer(
-      'android_snapshot_helper_session_capture',
-      async () => await captureAndroidSnapshotWithHelperSession(captureOptions),
-      {
+    sessionCapture = await withDiagnosticTimer(
+      ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE,
+      async () => (sessionCapture = await captureAndroidSnapshotWithHelperSession(captureOptions)),
+      () => ({
         packageName: artifact.manifest.packageName,
         version: artifact.manifest.version,
         timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS,
-      },
+        transport: 'persistent-session' satisfies AndroidSnapshotHelperTransport,
+        answered: sessionCapture !== undefined,
+      }),
     );
     if (sessionCapture) return sessionCapture;
   } catch (error) {
@@ -487,13 +504,14 @@ async function captureAndroidUiHierarchyFromHelper(params: {
     await resetAndroidSnapshotHelperRuntime(adb, artifact.manifest.packageName);
   }
   return await withDiagnosticTimer(
-    'android_snapshot_helper_capture',
+    ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE,
     async () => await captureAndroidSnapshotWithHelper(captureOptions),
     {
       packageName: artifact.manifest.packageName,
       version: artifact.manifest.version,
       timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS,
       commandTimeoutMs: ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS,
+      transport: 'instrumentation' satisfies AndroidSnapshotHelperTransport,
     },
   );
 }

packages/platform-android/src/touch-helper.ts

@@ -33,6 +33,8 @@ import {
 } from './snapshot-helper-types.ts';
 
 export const ANDROID_TOUCH_PLAN_PROTOCOL = 'android-touch-plan-v1';
+/** The diagnostic phase of one gesture that the snapshot helper injects. */
+export const ANDROID_TOUCH_HELPER_GESTURE_PHASE = 'android_touch_helper_gesture';
 const HELPER_GESTURE_TIMEOUT_MS = 45_000;
 const HELPER_GESTURE_TIMEOUT_OVERHEAD_MS = 15_000;
 const HELPER_VIEWPORT_TIMEOUT_MS = 45_000;
@@ -67,7 +69,7 @@ export async function executeAndroidTouchHelperPlan(
     JSON.stringify({ protocol: ANDROID_TOUCH_PLAN_PROTOCOL, ...request }),
   ).toString('base64');
   const output = await withDiagnosticTimer(
-    'android_touch_helper_gesture',
+    ANDROID_TOUCH_HELPER_GESTURE_PHASE,
     async () => {
       // A live snapshot session already owns UiAutomation; injecting through it avoids the
       // instrumentation stop/restart that previously taxed every gesture. Gesture session errors