diff --git a/.github/actions/smoke-ios-run/action.yml b/.github/actions/smoke-ios-run/action.yml new file mode 100644 index 0000000000..34830a6a14 --- /dev/null +++ b/.github/actions/smoke-ios-run/action.yml @@ -0,0 +1,144 @@ +name: Run iOS smoke tests +description: Run iOS smoke tests +runs: + using: composite + steps: + - name: Set fixture producer wait policy + id: fixture-producer + run: | + set -euo pipefail + WAIT_SECONDS=1800 + if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline." + WAIT_SECONDS=0 + fi + echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT" + shell: bash + - name: Select iOS runner XCTests + id: xctest-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + node --experimental-strip-types scripts/apple-ci-impact.ts xctest + shell: bash + - name: Establish host focus canary + id: host-focus-canary + run: | + set -euo pipefail + osascript -e 'tell application "Finder" to activate' + sleep 1 + FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" + test "$FRONTMOST" = "com.apple.finder" + mkdir -p .tmp + echo "$FRONTMOST" > .tmp/ios-host-focus-before + shell: bash + - name: Restore and build iOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build + with: + derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} + cache-key-prefix: ios-runner-prebuilt + cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64 + gate: swift-runner-ios + xcuitest-platform: ios + xcuitest-destination: generic/platform=iOS Simulator + - name: Boot iOS test simulator + id: ios-simulator + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + - name: Run targeted iOS runner XCTest regressions + id: ios-xctest + if: steps.xctest-impact.outputs.run != 'false' + run: | + set -euo pipefail + XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" + test -n "$XCTESTRUN_PATH" + IOS_PR_TEST_ARGS=() + IOS_PR_TEST_IDS="$(node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests)" + while IFS= read -r test_id; do + IOS_PR_TEST_ARGS+=("-only-testing:$test_id") + done <<< "$IOS_PR_TEST_IDS" + test "${#IOS_PR_TEST_ARGS[@]}" -gt 0 + mkdir -p "$GITHUB_WORKSPACE/test/artifacts" + xcodebuild test-without-building \ + -xctestrun "$XCTESTRUN_PATH" \ + -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ + -resultBundlePath "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \ + "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log + node --input-type=module -e ' + import { readFileSync } from "node:fs"; + const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8"); + if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) { + throw new Error("Optional observation ended the runner test before its wait completed"); + } + ' + shell: bash + - name: Preflight iOS runner through public CLI + run: | + pnpm clean:daemon + # The public-SDK record is the cold-startup evidence; the CLI timer only wraps attach. + node --input-type=module -e ' + const { performance } = await import("node:perf_hooks"); + const { ensureDaemon } = await import("./dist/src/index.js"); + const t0 = performance.now(); + const c = await ensureDaemon({ stateDir: process.env.AGENT_DEVICE_STATE_DIR }); + console.log(JSON.stringify({ phase: "daemon_cold_startup", started: c.started, pid: c.daemon.pid, durationMs: Math.round(performance.now() - t0) })); + ' + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + shell: bash + - name: Run iOS Settings replay smoke test + run: | + node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + shell: bash + - name: Fetch current fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + install: 'false' + wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }} + - name: Report fixture cache source + run: | + echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" + shell: bash + - name: Run gesture pan-duration smoke replay + run: | + xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" + node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml + shell: bash + - name: Run fixture-backed iOS simulator E2E smoke + env: + AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_IOS_E2E: '1' + AGENT_DEVICE_IOS_E2E_TIER: smoke + AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + run: | + xcrun simctl spawn "$AGENT_DEVICE_IOS_UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false + node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts + shell: bash + - name: Assert simulator automation preserved host focus + if: ${{ always() }} + run: | + set -euo pipefail + test "$(cat .tmp/ios-host-focus-before)" = "com.apple.finder" + FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" + test "$FRONTMOST" = "com.apple.finder" + shell: bash + - name: Run iOS physical device smoke replay + if: env.IOS_UDID != '' + run: | + pnpm clean:daemon + node bin/agent-device.mjs test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml + shell: bash + - name: Assert iOS XCTest selection was honored + if: always() + env: + SELECTED: ${{ steps.xctest-impact.outputs.run }} + OUTCOME: ${{ steps.ios-xctest.outcome }} + run: | + if [ "$SELECTED" = 'true' ] && [ "$OUTCOME" = 'skipped' ]; then + echo '::error::iOS runner XCTest was selected but its step was skipped.' + exit 1 + fi + shell: bash diff --git a/.github/actions/smoke-macos/action.yml b/.github/actions/smoke-macos/action.yml new file mode 100644 index 0000000000..361e5db50a --- /dev/null +++ b/.github/actions/smoke-macos/action.yml @@ -0,0 +1,101 @@ +name: Run macOS smoke tests +description: Run macOS smoke tests +runs: + using: composite + steps: + - name: Run macOS command coverage contract + uses: ./.github/actions/run-gate + with: + gate: macos-coverage + - name: Check the packaged runner Swift parses and keeps its line numbering + uses: ./.github/actions/run-gate + with: + gate: packaged-runner-swift + - name: Run iOS snapshot Swift/TypeScript differential + uses: ./.github/actions/run-gate + with: + gate: ios-snapshot-differential + - name: Restore and build macOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build + with: + derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} + cache-key-prefix: macos-runner-prebuilt + gate: swift-runner-macos + xcuitest-platform: macos + xcuitest-destination: platform=macOS,arch=arm64 + - name: Run the host-runnable runner XCTests + run: | + set -euo pipefail + XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" + test -n "$XCTESTRUN_PATH" + mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")" + xcodebuild test-without-building \ + -xctestrun "$XCTESTRUN_PATH" \ + -destination 'platform=macOS,arch=arm64' \ + -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \ + -resultBundlePath "$RESULT_BUNDLE_PATH" + shell: bash + - name: Extract the test-results summary + if: always() + run: | + set -uo pipefail + [ -d "$RESULT_BUNDLE_PATH" ] || exit 0 + xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" --compact \ + > "$RESULT_SUMMARY_PATH" 2>/dev/null || + xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" \ + > "$RESULT_SUMMARY_PATH" 2>/dev/null || + rm -f "$RESULT_SUMMARY_PATH" + exit 0 + shell: bash + - name: Report the run and assert it executed the source-derived set + if: always() + run: | + set -euo pipefail + if [ ! -s "$RESULT_SUMMARY_PATH" ]; then + echo 'No usable test-results summary was produced, so nothing here can show a test ran.' + echo 'The suite step above failed before or during the run; read its log first.' + echo '### iOS runner host XCTest lane' >> "$GITHUB_STEP_SUMMARY" + echo 'No result-bundle summary; see the run log and the uploaded bundle.' >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + node --experimental-strip-types scripts/xctest-run-summary.ts + shell: bash + - name: Upload result bundle + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: xctest-host-results-${{ github.run_id }}-${{ github.run_attempt }} + include-hidden-files: true + path: .tmp/xctest-host + if-no-files-found: warn + - name: macOS helper build and tests + uses: ./.github/actions/run-gate + with: + gate: macos-helper + - name: Run macOS integration test + uses: ./.github/actions/run-gate + with: + gate: replay-macos + args: | + --retries + 2 + --report-junit + test/artifacts/replays-macos.junit.xml + - name: Select Apple bridge proof + if: github.event_name == 'pull_request' + id: bridge-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + node --experimental-strip-types scripts/apple-ci-impact.ts bridge + shell: bash + - name: Verify the darwin-only native source proofs (snapshot bridge, fold-helper -Werror, recording scripts) + if: github.event_name == 'pull_request' && steps.bridge-impact.outputs.run != 'false' + run: | + pnpm exec vitest run \ + packages/platform-apple/src/snapshot-source/cache-native-runtime.test.ts \ + packages/platform-apple/src/foldable/fold-helper-cache.test.ts \ + packages/platform-apple/src/foldable/simulator-hid-native.test.ts \ + packages/capture-kit/src/recording/__tests__/swift-cache-recording-scripts.test.ts + pnpm check:package -- --verify-snapshot-bridge-preparation + shell: bash diff --git a/.github/actions/upload-agent-device-artifacts/action.yml b/.github/actions/upload-agent-device-artifacts/action.yml index 55c34e964f..9e75d20869 100644 --- a/.github/actions/upload-agent-device-artifacts/action.yml +++ b/.github/actions/upload-agent-device-artifacts/action.yml @@ -13,6 +13,11 @@ inputs: required: false default: '.agent-device-no-runner-derived' + additional-paths: + description: 'Optional additional diagnostic paths for a shared runner' + required: false + default: '' + runs: using: 'composite' steps: @@ -30,3 +35,4 @@ runs: ${{ inputs.runner-derived-path }}/Logs/** test/artifacts/** test/screenshots/** + ${{ inputs.additional-paths }} diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml new file mode 100644 index 0000000000..f56bb9dfbf --- /dev/null +++ b/.github/workflows/apple.yml @@ -0,0 +1,153 @@ +name: Apple +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + - converted_to_draft + paths-ignore: + - docs/** + - website/** + - README.md + - AGENTS.md + - CONTEXT.md + - CONTRIBUTING.md + - LICENSE + - SECURITY.md + - .github/actions/build-docs/action.yml + - .github/workflows/deploy.yml + - .github/workflows/pr-preview.yml + - .github/workflows/pr-preview-cleanup.yml + push: + branches: + - main +permissions: + contents: read + actions: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + smoke-apple: + name: Smoke Tests + if: ${{ !github.event.pull_request.draft }} + runs-on: macos-26 + # Keep each platform's former 80-minute ceiling, plus shared setup and cleanup. + timeout-minutes: 170 + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + IOS_STATE_DIR: ${{ github.workspace }}/.tmp/ios-agent-device-state + MACOS_STATE_DIR: ${{ github.workspace }}/.tmp/macos-agent-device-state + IOS_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived + MACOS_DERIVED_PATH: ${{ github.workspace }}/.tmp/macos-runner-derived + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + - name: Fetch the PR base for Apple selectors + if: github.event_name == 'pull_request' + shell: bash + run: | + git fetch origin "$BASE_SHA" --depth=1 || echo "Base fetch failed; Apple selection will fail open." + - name: Build the shared CLI + id: build + uses: ./.github/actions/run-gate + with: + gate: build + - name: Select iOS smoke tests + id: ios-impact + shell: bash + run: node --experimental-strip-types scripts/apple-ci-impact.ts smoke + # The macOS clean-install proof may raise permission UI. Run the iOS focus proof first. + - name: iOS smoke tests + id: ios + uses: ./.github/actions/smoke-ios-run + continue-on-error: true + timeout-minutes: 80 + env: + IOS_RUNTIME_VERSION: '26.2' + IOS_UDID: ${{ vars.IOS_UDID }} + AGENT_DEVICE_STATE_DIR: ${{ env.IOS_STATE_DIR }} + AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ env.IOS_DERIVED_PATH }} + AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' + AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' + AGENT_DEVICE_XCUITEST_ARCHS: arm64 + AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + - name: Restore the Apple host after iOS + id: ios-cleanup + if: ${{ always() && steps.build.outcome == 'success' }} + continue-on-error: true + env: + AGENT_DEVICE_STATE_DIR: ${{ env.IOS_STATE_DIR }} + shell: bash + run: bash scripts/teardown-ios-smoke.sh + - name: macOS smoke tests + id: macos + uses: ./.github/actions/smoke-macos + continue-on-error: true + timeout-minutes: 80 + env: + AGENT_DEVICE_STATE_DIR: ${{ env.MACOS_STATE_DIR }} + AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ env.MACOS_DERIVED_PATH }} + AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' + RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-host/RunnerTests.xcresult + RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-host/test-results-summary.json + XCTEST_LANE: host + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-cleanup.outcome == 'success' }} + - name: Diagnose the Apple host after a phase failure + if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure') }} + continue-on-error: true + shell: bash + run: | + mkdir -p test/screenshots + screencapture -x test/screenshots/apple-host-after-failure.png + osascript scripts/decline-hosted-compute-local-network.applescript diagnose + - name: Clean macOS daemon and retained runners + id: macos-cleanup + if: ${{ always() && steps.build.outcome == 'success' }} + continue-on-error: true + env: + AGENT_DEVICE_STATE_DIR: ${{ env.MACOS_STATE_DIR }} + shell: bash + run: | + pnpm clean:daemon + - name: Require both Apple phases to succeed + if: always() + env: + IOS_SELECTED: ${{ steps.ios-impact.outputs.run }} + IOS_OUTCOME: ${{ steps.ios.outcome }} + MACOS_OUTCOME: ${{ steps.macos.outcome }} + IOS_CLEANUP_OUTCOME: ${{ steps.ios-cleanup.outcome }} + MACOS_CLEANUP_OUTCOME: ${{ steps.macos-cleanup.outcome }} + shell: bash + run: | + set -euo pipefail + printf "### Apple smoke phases\n\n| Phase | Outcome |\n| --- | --- |\n| iOS | %s |\n| macOS | %s |\n" "$IOS_OUTCOME" "$MACOS_OUTCOME" >> "$GITHUB_STEP_SUMMARY" + if [ "$IOS_SELECTED" = false ]; then + test "$IOS_OUTCOME" = skipped + else + test "$IOS_OUTCOME" = success + fi + test "$MACOS_OUTCOME" = success + test "$IOS_CLEANUP_OUTCOME" = success + test "$MACOS_CLEANUP_OUTCOME" = success + - name: Upload Apple artifacts + if: always() + uses: ./.github/actions/upload-agent-device-artifacts + with: + artifact-name: apple-artifacts + agent-state-dir: ${{ env.IOS_STATE_DIR }} + runner-derived-path: ${{ env.IOS_DERIVED_PATH }} + additional-paths: | + ${{ env.MACOS_STATE_DIR }}/daemon.log + ${{ env.MACOS_STATE_DIR }}/sessions/** + ${{ env.MACOS_DERIVED_PATH }}/.agent-device-runner-cache.json + ${{ env.MACOS_DERIVED_PATH }}/Logs/** diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4faf6db405..47345f5942 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -210,7 +210,7 @@ jobs: with: { gate: gate-manifest } # Same family as the manifest above — a CI selection that has stopped selecting what - # it claims. ios.yml runs a hand-written subset of the runner XCTest methods through an + # it claims. The Apple iOS phase runs a source-derived subset of XCTest methods through an # `-only-testing:` list, and xcodebuild treats an identifier that matches nothing as # an empty selection rather than an error, so a rename drops a test with no signal — # in both directions, since a typo in xctest-nightly.yml's `-skip-testing:` entry diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml deleted file mode 100644 index c138a600d9..0000000000 --- a/.github/workflows/ios.yml +++ /dev/null @@ -1,255 +0,0 @@ -name: iOS - -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] - paths-ignore: - - 'docs/**' - - 'website/**' - - 'README.md' - - 'AGENTS.md' - - 'CONTEXT.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'SECURITY.md' - - '.github/actions/build-docs/action.yml' - - '.github/workflows/deploy.yml' - - '.github/workflows/pr-preview.yml' - - '.github/workflows/pr-preview-cleanup.yml' - # Routing (#1781 A9-2). Everything below is derived from the affected selector's - # device-lane ownership (scripts/check-affected/device-lanes.ts): unit tests no device - # lane runs, and trees another platform family owns. `pnpm check:gate-manifest` asserts - # this list against the selector over every tracked path, both ways — a path the selector - # routes to the iOS lane (or fails open on: tooling, workflows, package manifests) must not - # appear here, and a path it classifies as another family's or as a unit test must. The - # `push` trigger has no filter: main runs every lane unconditionally. - - '**/*.md' - - 'src/**/*.test.ts' - - 'src/**/__tests__/**/*.ts' - - 'packages/*/src/**/*.test.ts' - - 'packages/*/src/**/__tests__/**/*.ts' - - 'packages/platform-android/src/**' - - 'packages/capture-kit/src/snapshot/snapshot-presentation/android/**' - - 'packages/platform-harmonyos/src/**' - - 'packages/platform-linux/src/**' - - 'packages/platform-vega/src/**' - - 'packages/platform-web/src/**' - - 'android/**' - - 'linux/**' - - 'test/integration/replays/android/**' - - 'test/integration/replays/linux/**' - - 'test/integration/replays/macos/**' - - 'test/integration/android-**' - - 'test/integration/smoke-android-**' - - 'test/integration/smoke-linux-**' - - 'test/integration/linux-e2e/**' - - 'test/integration/smoke-macos-**' - - 'test/integration/smoke-web-**' - - 'test/integration/macos-e2e/**' - - 'test/integration/web-e2e/**' - push: - branches: - - main - -permissions: - contents: read - actions: read - -concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - smoke-ios: - name: Smoke Tests - if: ${{ !github.event.pull_request.draft }} - runs-on: macos-26 - timeout-minutes: 80 - env: - IOS_RUNTIME_VERSION: '26.2' - AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state - AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived - AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' - AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' - # A generic simulator destination leaves the active arch undefined and Xcode 26.6 - # defaults it to x86_64, which runs the whole runner under Rosetta on arm64 hosts. - # Pin the native slice: measured ~30% faster commits on identical CI hardware - # (#1874 diagnosis, PR #1941). - AGENT_DEVICE_XCUITEST_ARCHS: 'arm64' - AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Set fixture producer wait policy - id: fixture-producer - run: | - set -euo pipefail - WAIT_SECONDS=1800 - if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then - echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline." - WAIT_SECONDS=0 - fi - echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT" - - - name: Setup toolchain - uses: ./.github/actions/setup-node-pnpm - with: - cache-dependency-path: | - pnpm-lock.yaml - examples/test-app/pnpm-lock.yaml - - - name: Select iOS runner XCTests - id: xctest-impact - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - if [ "${{ github.event_name }}" = 'pull_request' ]; then - git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; XCTest selection will fail open.' - fi - node --experimental-strip-types scripts/apple-ci-impact.ts xctest - - - name: Establish host focus canary - id: host-focus-canary - run: | - set -euo pipefail - osascript -e 'tell application "Finder" to activate' - sleep 1 - FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" - test "$FRONTMOST" = "com.apple.finder" - mkdir -p .tmp - echo "$FRONTMOST" > .tmp/ios-host-focus-before - - - name: Restore and build iOS XCTest runner - uses: ./.github/actions/setup-apple-runner-build - with: - derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} - cache-key-prefix: ios-runner-prebuilt - cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64 - gate: swift-runner-ios - xcuitest-platform: ios - xcuitest-destination: generic/platform=iOS Simulator - - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro - - - name: Run targeted iOS runner XCTest regressions - id: ios-xctest - if: steps.xctest-impact.outputs.run != 'false' - run: | - set -euo pipefail - XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" - test -n "$XCTESTRUN_PATH" - IOS_PR_TEST_ARGS=() - IOS_PR_TEST_IDS="$(node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests)" - while IFS= read -r test_id; do - IOS_PR_TEST_ARGS+=("-only-testing:$test_id") - done <<< "$IOS_PR_TEST_IDS" - test "${#IOS_PR_TEST_ARGS[@]}" -gt 0 - xcodebuild test-without-building \ - -xctestrun "$XCTESTRUN_PATH" \ - -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ - "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log - node --input-type=module -e ' - import { readFileSync } from "node:fs"; - const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8"); - if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) { - throw new Error("Optional observation ended the runner test before its wait completed"); - } - ' - - - name: Build CLI used by every smoke step - run: pnpm gate build - - # Reuse the prepared daemon and runner through the shared AGENT_DEVICE_STATE_DIR. - - name: Preflight iOS runner through public CLI - run: | - pnpm clean:daemon - # The public-SDK record is the cold-startup evidence; the CLI timer only wraps attach. - node --input-type=module -e ' - const { performance } = await import("node:perf_hooks"); - const { ensureDaemon } = await import("./dist/src/index.js"); - const t0 = performance.now(); - const c = await ensureDaemon({ stateDir: process.env.AGENT_DEVICE_STATE_DIR }); - console.log(JSON.stringify({ phase: "daemon_cold_startup", started: c.started, pid: c.daemon.pid, durationMs: Math.round(performance.now() - t0) })); - ' - node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json - - # The replay reuses the prepared runner in the same daemon. --debug keeps every step's request - # log, not only the failing one, so a stale tap can be traced to its pre-tap capture (#3367). - - name: Run iOS Settings replay smoke test - run: | - node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml - - - name: Fetch current fixture app - id: fixture-app - uses: ./.github/actions/setup-fixture-app - with: - install: 'false' - wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }} - - - name: Report fixture cache source - run: | - echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" - - # #1584: isolated, cheap automatic guard for the #1562 class of regression (iOS silently - # ignoring a `gesture pan` duration). Split out of gesture-lab.ad, which stays full-tier - # (dispatch-only via replays-manual.yml) because its multi-touch commands are a separate, - # heavier concern. This is the only automatic lane this assertion runs in today. - # --timeout raises the per-attempt budget past the script's own 60s: it absorbs an - # in-replay runner rebuild (measured 48-61s build + ~40s launch) on a runner cache miss; - # the normal path finishes in 21-34s. - - name: Run gesture pan-duration smoke replay - run: | - xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" - node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml - - - name: Run fixture-backed iOS simulator E2E smoke - env: - AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} - AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} - AGENT_DEVICE_IOS_E2E: '1' - AGENT_DEVICE_IOS_E2E_TIER: smoke - AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} - run: | - xcrun simctl spawn "$AGENT_DEVICE_IOS_UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false - node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - - - name: Assert simulator automation preserved host focus - if: ${{ always() }} - run: | - set -euo pipefail - test "$(cat .tmp/ios-host-focus-before)" = "com.apple.finder" - FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" - test "$FRONTMOST" = "com.apple.finder" - - - name: Run iOS physical device smoke replay - if: env.IOS_UDID != '' - env: - IOS_UDID: ${{ vars.IOS_UDID }} - run: | - pnpm clean:daemon - node bin/agent-device.mjs test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml - - - name: Assert iOS XCTest selection was honored - if: always() - env: - SELECTED: ${{ steps.xctest-impact.outputs.run }} - OUTCOME: ${{ steps.ios-xctest.outcome }} - run: | - if [ "$SELECTED" = 'true' ] && [ "$OUTCOME" = 'skipped' ]; then - echo '::error::iOS runner XCTest was selected but its step was skipped.' - exit 1 - fi - - - name: Upload iOS artifacts - if: always() - uses: ./.github/actions/upload-agent-device-artifacts - with: - artifact-name: ios-artifacts - agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} - runner-derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index 306f6cfa1e..0000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: macOS - -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] - paths-ignore: - - 'docs/**' - - 'website/**' - - 'README.md' - - 'AGENTS.md' - - 'CONTEXT.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'SECURITY.md' - - '.github/actions/build-docs/action.yml' - - '.github/workflows/deploy.yml' - - '.github/workflows/pr-preview.yml' - - '.github/workflows/pr-preview-cleanup.yml' - push: - branches: - - main - -permissions: - contents: read - -concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # One macOS lane for everything the macOS host can prove: command coverage, - # the runner's pure-decision XCTests, the helper, and the replay smoke. All - # of it consumes one runner build, compiled with the unit-test flag so the - # host XCTest run below has a bundle to execute (#1781 A7). - # - # Which tests that build reaches is not a list: it is everything the macOS - # build compiles, and the simulator-only tests keep themselves out with an - # `os(iOS)` guard (see the classification note in RunnerTests.swift). The - # `-skip-testing:` entry is validated by `pnpm check:xctest-selection` for - # the same reason the nightly's is: `RunnerTests/testCommand` is the - # runner's 24-hour server entry point, and a typo re-arms the hang. The - # lane's reporter asserts the executed count equals the reach the check - # derives from source, so a build without the compile flag or a guard that - # compiles a file out reads as red rather than as a smaller green. - # - # Kill criterion: if the host run proves flaky where the simulator run of the - # same tests is not (a platform-branch difference this classification - # missed), gate the offender `os(iOS)` and it returns to the simulator lanes; - # if that happens to more than a handful, the host lane is the wrong tool and - # goes. - smoke-macos: - name: Smoke Tests - if: ${{ !github.event.pull_request.draft }} - runs-on: macos-26 - timeout-minutes: 80 - env: - AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state - AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/macos-runner-derived - AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' - RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-host/RunnerTests.xcresult - RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-host/test-results-summary.json - XCTEST_LANE: host - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup toolchain - uses: ./.github/actions/setup-node-pnpm - - - name: Run macOS command coverage contract - uses: ./.github/actions/run-gate - with: { gate: macos-coverage } - - # The npm package ships apple/runner/** as rewritten Swift source, and nothing in the repo - # compiles the result — the first consumer is a user's xcodebuild. This lane is the only one - # with a Swift toolchain, so it is where `swiftc -parse` over the packaged files runs; the - # line-parity half of the same gate needs no toolchain. Seconds, no simulator (#2461). - - name: Check the packaged runner Swift parses and keeps its line numbering - uses: ./.github/actions/run-gate - with: { gate: packaged-runner-swift } - - - name: Run iOS snapshot Swift/TypeScript differential - uses: ./.github/actions/run-gate - with: { gate: ios-snapshot-differential } - - - name: Restore and build macOS XCTest runner - uses: ./.github/actions/setup-apple-runner-build - with: - derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} - cache-key-prefix: macos-runner-prebuilt - gate: swift-runner-macos - xcuitest-platform: macos - xcuitest-destination: platform=macOS,arch=arm64 - - - name: Run the host-runnable runner XCTests - run: | - set -euo pipefail - XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" - test -n "$XCTESTRUN_PATH" - mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")" - xcodebuild test-without-building \ - -xctestrun "$XCTESTRUN_PATH" \ - -destination 'platform=macOS,arch=arm64' \ - -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \ - -resultBundlePath "$RESULT_BUNDLE_PATH" - - # Same shape as xctest-nightly.yml: extract is best-effort, the report step asserts. - - name: Extract the test-results summary - if: always() - run: | - set -uo pipefail - [ -d "$RESULT_BUNDLE_PATH" ] || exit 0 - xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" --compact \ - > "$RESULT_SUMMARY_PATH" 2>/dev/null || - xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" \ - > "$RESULT_SUMMARY_PATH" 2>/dev/null || - rm -f "$RESULT_SUMMARY_PATH" - exit 0 - - - name: Report the run and assert it executed the source-derived set - if: always() - run: | - set -euo pipefail - if [ ! -s "$RESULT_SUMMARY_PATH" ]; then - echo 'No usable test-results summary was produced, so nothing here can show a test ran.' - echo 'The suite step above failed before or during the run; read its log first.' - echo '### iOS runner host XCTest lane' >> "$GITHUB_STEP_SUMMARY" - echo 'No result-bundle summary; see the run log and the uploaded bundle.' >> "$GITHUB_STEP_SUMMARY" - exit 1 - fi - node --experimental-strip-types scripts/xctest-run-summary.ts - - - name: Upload result bundle - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: xctest-host-results-${{ github.run_id }}-${{ github.run_attempt }} - include-hidden-files: true - path: .tmp/xctest-host - if-no-files-found: warn - - - name: macOS helper build and tests - uses: ./.github/actions/run-gate - with: { gate: macos-helper } - - - name: Run macOS integration test - uses: ./.github/actions/run-gate - with: - gate: replay-macos - args: | - --retries - 2 - --report-junit - test/artifacts/replays-macos.junit.xml - - # Clean-install verification may raise macOS local-network permission UI; keep it after replay. - - name: Select Apple bridge proof - if: github.event_name == 'pull_request' - id: bridge-impact - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; bridge proof selection will fail open.' - node --experimental-strip-types scripts/apple-ci-impact.ts bridge - - - name: Verify the darwin-only native source proofs (snapshot bridge, fold-helper -Werror, recording scripts) - if: github.event_name == 'pull_request' && steps.bridge-impact.outputs.run != 'false' - run: | - pnpm build - pnpm exec vitest run \ - packages/platform-apple/src/snapshot-source/cache-native-runtime.test.ts \ - packages/platform-apple/src/foldable/fold-helper-cache.test.ts \ - packages/platform-apple/src/foldable/simulator-hid-native.test.ts \ - packages/capture-kit/src/recording/__tests__/swift-cache-recording-scripts.test.ts - pnpm check:package -- --verify-snapshot-bridge-preparation - - - name: Upload macOS artifacts - if: always() - uses: ./.github/actions/upload-agent-device-artifacts - with: - artifact-name: macos-artifacts - agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} - runner-derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} diff --git a/.github/workflows/xctest-nightly.yml b/.github/workflows/xctest-nightly.yml index 39ae0d1783..3c20c78b86 100644 --- a/.github/workflows/xctest-nightly.yml +++ b/.github/workflows/xctest-nightly.yml @@ -1,6 +1,6 @@ name: XCTest Nightly -# The full iOS runner XCTest suite (#1781 A7). The PR lane (ios.yml) names a hand-written +# The full iOS runner XCTest suite (#1781 A7). The iOS PR phase in apple.yml selects a source-derived # subset of the target's methods in an `-only-testing:` list; everything outside that list ran # nowhere at all. This lane drops the filter and runs the test plan whole, skipping only # `testCommand` — which is the runner's server entry point rather than a test (see the step @@ -16,28 +16,28 @@ name: XCTest Nightly # Under the lane rule in #1781: # # - Catches: regressions in the simulator-only tests (guard `… && os(iOS)`: they launch -# the host app, route through SpringBoard, or assert an iOS-only branch) that ios.yml's +# the host app, route through SpringBoard, or assert an iOS-only branch) that apple.yml's # PR list leaves out. Since the A7 classification the pure-decision tests also run on -# the macOS host lane on every PR (ci.yml "Swift Runner Host XCTests"), so this lane's +# the macOS phase on every PR (apple.yml), so this lane's # unique reach is the simulator-only remainder — `pnpm check:xctest-selection` prints # both numbers. `testCommand` is skipped, and the tests guarded to macOS/tvOS only # cannot execute on this lane's iOS Simulator destination (the host lane runs them). # - Evidence: the first execution found #1812 in the dark set (2 of 155 red, both real); # the other ~114 never-run tests all passed. # - Cost: this is a public repository, so nothing here is billed. The real cost is one macOS -# concurrency slot per night. The Apple runner build shares ios.yml's cache (identical +# concurrency slot per night. The Apple runner build shares apple.yml's cache (identical # inputs and build variant), but scheduled runs only see DEFAULT-BRANCH caches, so the hit -# depends on ios.yml's `push: main` run having populated the cache at the same source +# depends on apple.yml's `push: main` run having populated the cache at the same source # hash — on a miss this lane also pays a full runner build. Adds nothing to any PR. # - Kill criterion: what ONLY this lane reaches is 11 tests / ~42 s of simulator time # (measured from the first night's xcresult) — the app-launching, SpringBoard-probing -# class. Once ios.yml runs the whole iOS-compiled bundle instead of its hand-written +# class. Once apple.yml runs the whole iOS-compiled bundle instead of its selected # list — the follow-up stacked on the classification — every test reaches a PR lane and # this lane goes. That trade is not free: those 11 are the flakiest tests in the suite, # and they would land on a blocking 15-minute gate that is already cancelled ~19% of the # time, so measure their flake rate here before moving them. # -# Scheduled + manual only. It is deliberately NOT a PR gate: ios.yml is already over half of +# Scheduled + manual only. It is deliberately NOT a PR gate: apple.yml is already over half of # this repo's CI spend (#1781 A9), and moving the unrun set onto it would make that worse for # tests whose flakiness on a shared runner is unknown. # @@ -66,13 +66,13 @@ jobs: # Measured, not guessed. A full local run of this exact command on matching toolchain # (Xcode 26.2, iOS 26.2 Simulator) executed 154 tests in 120s wall — the suite itself is # ~2 minutes, not the bulk of the job. What dominates the budget is the runner build on a - # derived-data cache MISS, plus checkout and simulator boot. ios.yml carries 80 minutes + # derived-data cache MISS, plus checkout and simulator boot. the iOS phase in apple.yml has an 80-minute limit # while doing strictly more than this lane (same build and boot, plus replays, prepare, # and the fixture E2E), so half of that is generous headroom for a cache-miss night and # still caps a wedged run at 45 minutes instead of two hours of held macOS slot. timeout-minutes: 45 env: - # Must match ios.yml. The runtime version rides in the derived-data cache key, and + # Must match apple.yml. The runtime version rides in the derived-data cache key, and # AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS is BOTH an input to the cache's build-variant # hash and the `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` compile flag that makes the RunnerTests # methods exist at all. Dropping it would build a different variant (cache miss) whose @@ -98,7 +98,7 @@ jobs: pnpm-lock.yaml examples/test-app/pnpm-lock.yaml - # Byte-identical inputs to ios.yml's step, so both lanes restore the same cache entry + # Byte-identical inputs to apple.yml's step, so both lanes restore the same cache entry # and only the first one to run on a commit pays for the build. - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build @@ -117,7 +117,7 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - # Same command as ios.yml's targeted step minus every `-only-testing:` flag, so the + # Same command as apple.yml's targeted step minus every `-only-testing:` flag, so the # xctestrun's own test plan decides what runs — with one exception. # # `RunnerTests/testCommand` is not a test. It is the runner's server entry point: it diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 624666dcd2..50604c65f1 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -295,7 +295,7 @@ scale, and rotation values for one simultaneous two-finger gesture. `gesture pan`'s requested duration is actually observed by the app (a bucketed `pan duration` status on the Home screen), rather than just that the gesture activated. It's split out of `gesture-lab.ad` so it can run as an automatic PR-tier check -(`.github/workflows/ios.yml`) without depending on `gesture-lab.ad`'s multi-touch commands, +(`.github/workflows/apple.yml`) without depending on `gesture-lab.ad`'s multi-touch commands, which stay full-tier only. To target a specific iOS simulator or an installed Expo development build, run the diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index 3cc8e306fc..e8f1025c5f 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -1,14 +1,285 @@ import fs from 'node:fs'; -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import os from 'node:os'; import path from 'node:path'; import { expect, test } from 'vitest'; import { parse } from 'yaml'; -import { selectAppleBridgeProof, selectIosXctests } from '../apple-ci-impact.ts'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { selectAppleBridgeProof, selectIosSmoke, selectIosXctests } from '../apple-ci-impact.ts'; import { selectChecks } from '../check-affected/model.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); +type AppleStep = { + id?: string; + uses?: string; + if?: string; + run?: string; + env?: Record; + with?: Record; + 'continue-on-error'?: boolean; +}; + +function appleJob() { + const doc = parse( + fs.readFileSync(path.join(repoRoot, '.github/workflows/apple.yml'), 'utf8'), + ) as { + jobs: Record; + }; + expect(Object.keys(doc.jobs)).toHaveLength(1); + return Object.values(doc.jobs)[0]!; +} + +test('Apple smoke shares setup and build while keeping platform state and native products separate', () => { + const { steps } = appleJob(); + expect(steps.filter((step) => step.uses?.startsWith('actions/checkout@'))).toHaveLength(1); + expect(steps.filter((step) => step.uses === './.github/actions/setup-node-pnpm')).toHaveLength(1); + expect(steps.filter((step) => step.with?.gate === 'build')).toHaveLength(1); + const ios = steps.find((step) => step.id === 'ios')!; + const macos = steps.find((step) => step.id === 'macos')!; + expect(ios.uses).toBe('./.github/actions/smoke-ios-run'); + expect(macos.uses).toBe('./.github/actions/smoke-macos'); + expect(steps.indexOf(ios)).toBeLessThan(steps.indexOf(macos)); + expect(macos.if).toContain("steps.ios-cleanup.outcome == 'success'"); + for (const variable of ['AGENT_DEVICE_STATE_DIR', 'AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH']) { + expect(ios.env?.[variable]).toBeTruthy(); + expect(macos.env?.[variable]).toBeTruthy(); + expect(ios.env?.[variable]).not.toBe(macos.env?.[variable]); + } + for (const phase of [ios, macos]) { + expect(phase['continue-on-error']).toBe(true); + const cleanup = steps.find((step) => step.id === `${phase.id}-cleanup`)!; + expect(steps.indexOf(cleanup)).toBeGreaterThan(steps.indexOf(phase)); + if (phase === ios) expect(steps.indexOf(cleanup)).toBeLessThan(steps.indexOf(macos)); + expect(cleanup.if).toContain('always()'); + expect(cleanup['continue-on-error']).toBe(true); + expect(cleanup.env?.AGENT_DEVICE_STATE_DIR).toBe(phase.env?.AGENT_DEVICE_STATE_DIR); + expect(cleanup.run?.trim()).toBe( + phase === ios ? 'bash scripts/teardown-ios-smoke.sh' : 'pnpm clean:daemon', + ); + } +}); + +test.runIf(process.platform === 'darwin')( + 'hosted prompt cleanup refuses local execution before inspecting UI', + () => { + const result = spawnSync( + 'osascript', + [path.join(repoRoot, 'scripts/decline-hosted-compute-local-network.applescript')], + { + env: { ...process.env, GITHUB_ACTIONS: 'false' }, + encoding: 'utf8', + timeout: 10_000, + }, + ); + expect(result.error).toBeUndefined(); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain('This cleanup is only for GitHub Actions runners.'); + }, +); + +test.each([ + ['clean', {}, 0], + ['local invocation', { GITHUB_ACTIONS: 'false' }, 1], + ['daemon cleanup fails', { FIXTURE_DAEMON_FAIL: '1' }, 1], + ['simulator remains booted', { FIXTURE_BOOTED: '1' }, 1], + ['XCTest runner remains', { FIXTURE_PROCESSES: '/tmp/AgentDeviceRunnerUITests-Runner' }, 1], + [ + 'xcodebuild remains', + { FIXTURE_PROCESSES: '/Applications/Xcode.app/Contents/Developer/usr/bin/xcodebuild' }, + 1, + ], + ['focus not restored', { FIXTURE_FOCUS: 'com.apple.Simulator' }, 1], +] as const)( + 'between-phase teardown preserves host isolation: %s', + (_label, overrides, expected) => { + const directory = mkdtempForTestSync('apple-teardown-'); + const calls = path.join(directory, 'calls.ndjson'); + const stub = `#!${process.execPath} +const fs = require('node:fs'); +const path = require('node:path'); +const name = path.basename(process.argv[1]); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({ name, args }) + '\\n'); +if (name === 'pnpm' && process.env.FIXTURE_DAEMON_FAIL === '1') process.exit(1); +if (name === 'xcrun' && args.includes('list')) console.log(JSON.stringify({ devices: { ios: [{ udid: 'fixture-ios', state: process.env.FIXTURE_BOOTED === '1' ? 'Booted' : 'Shutdown' }] } })); +if (name === 'ps') console.log(process.env.FIXTURE_PROCESSES || ''); +if (name === 'osascript' && args.join(' ').includes('get bundle identifier')) console.log(process.env.FIXTURE_FOCUS || 'com.apple.finder'); +`; + for (const command of ['pnpm', 'xcrun', 'osascript', 'ps', 'screencapture']) + fs.writeFileSync(path.join(directory, command), stub, { mode: 0o755 }); + const result = spawnSync('/bin/bash', [path.join(repoRoot, 'scripts/teardown-ios-smoke.sh')], { + env: { + ...process.env, + PATH: `${directory}:${process.env.PATH}`, + GITHUB_ACTIONS: 'true', + FIXTURE_CALLS: calls, + ...overrides, + }, + encoding: 'utf8', + timeout: 10_000, + }); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(expected); + const invoked = fs.existsSync(calls) + ? fs + .readFileSync(calls, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { name: string; args: string[] }) + : []; + if ('GITHUB_ACTIONS' in overrides && overrides.GITHUB_ACTIONS === 'false') { + expect(invoked).toEqual([]); + } else { + expect(invoked.slice(0, 3)).toEqual([ + { name: 'pnpm', args: ['clean:daemon'] }, + { name: 'xcrun', args: ['simctl', 'shutdown', 'all'] }, + { + name: 'osascript', + args: [ + '-e', + 'if application "Simulator" is running then tell application "Simulator" to quit', + ], + }, + ]); + expect(invoked).toContainEqual({ + name: 'osascript', + args: ['scripts/decline-hosted-compute-local-network.applescript'], + }); + } + }, +); + +test('the workflow resolves the optional physical device variable before entering composites', () => { + const ios = appleJob().steps.find((step) => step.id === 'ios')!; + expect(ios.env?.IOS_UDID).toBe('${{ vars.IOS_UDID }}'); + for (const action of ['smoke-ios-run']) { + const source = fs.readFileSync( + path.join(repoRoot, `.github/actions/${action}/action.yml`), + 'utf8', + ); + expect(source).not.toMatch(/\$\{\{\s*vars\./); + } +}); + +test('Apple uploads shared diagnostics once and retains both platform logs after cleanup', () => { + const { steps } = appleJob(); + const uploads = steps.filter( + (step) => step.uses === './.github/actions/upload-agent-device-artifacts', + ); + expect(uploads).toHaveLength(1); + const upload = uploads[0]!; + expect(upload.if).toBe('always()'); + for (const platform of ['ios', 'macos']) { + const phase = steps.find((step) => step.id === platform)!; + const cleanup = steps.find((step) => step.id === `${platform}-cleanup`)!; + expect(steps.indexOf(upload)).toBeGreaterThan(steps.indexOf(cleanup)); + const state = phase.env!.AGENT_DEVICE_STATE_DIR!; + const derived = phase.env!.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH!; + if (platform === 'ios') { + expect(upload.with?.['agent-state-dir']).toBe(state); + expect(upload.with?.['runner-derived-path']).toBe(derived); + } else { + expect(upload.with?.['additional-paths']?.trim().split(/\r?\n/)).toEqual([ + `${state}/daemon.log`, + `${state}/sessions/**`, + `${derived}/.agent-device-runner-cache.json`, + `${derived}/Logs/**`, + ]); + } + const action = fs.readFileSync( + path.join( + repoRoot, + `.github/actions/smoke-${platform === 'ios' ? 'ios-run' : platform}/action.yml`, + ), + 'utf8', + ); + expect(action).not.toContain('./.github/actions/upload-agent-device-artifacts'); + } + const helper = parse( + fs.readFileSync( + path.join(repoRoot, '.github/actions/upload-agent-device-artifacts/action.yml'), + 'utf8', + ), + ) as { inputs: Record; runs: { steps: AppleStep[] } }; + expect(helper.inputs['additional-paths']?.default).toBe(''); + const paths = helper.runs.steps[0]!.with!.path!; + for (const path of [ + 'test/artifacts/**', + 'test/screenshots/**', + '~/.agent-device/logs/**', + '${{ inputs.additional-paths }}', + ]) { + expect(paths.split(/\r?\n/).filter((entry) => entry === path)).toHaveLength(1); + } +}); + +test.each([ + ['all successful', {}, 0], + ['iOS not selected', { IOS_SELECTED: 'false', IOS_OUTCOME: 'skipped' }, 0], + ['unselected iOS unexpectedly ran', { IOS_SELECTED: 'false' }, 1], + ['selected iOS fails', { IOS_SELECTED: 'true', IOS_OUTCOME: 'failure' }, 1], + ['iOS fails', { IOS_OUTCOME: 'failure' }, 1], + ['macOS fails', { MACOS_OUTCOME: 'failure' }, 1], + ['iOS silently skipped', { IOS_OUTCOME: 'skipped' }, 1], + ['macOS silently skipped', { MACOS_OUTCOME: 'skipped' }, 1], + ['iOS cleanup fails', { IOS_CLEANUP_OUTCOME: 'failure' }, 1], + ['macOS cleanup fails', { MACOS_CLEANUP_OUTCOME: 'failure' }, 1], +] as const)('the final Apple status guard preserves failure: %s', (_label, overrides, expected) => { + const guard = appleJob().steps.find((step) => step.env?.IOS_OUTCOME)!; + expect(guard.if).toBe('always()'); + expect(guard.env).toEqual({ + IOS_SELECTED: '${{ steps.ios-impact.outputs.run }}', + IOS_OUTCOME: '${{ steps.ios.outcome }}', + MACOS_OUTCOME: '${{ steps.macos.outcome }}', + IOS_CLEANUP_OUTCOME: '${{ steps.ios-cleanup.outcome }}', + MACOS_CLEANUP_OUTCOME: '${{ steps.macos-cleanup.outcome }}', + }); + const result = spawnSync('/bin/bash', ['-c', guard.run!], { + env: { + ...process.env, + IOS_SELECTED: 'true', + IOS_OUTCOME: 'success', + MACOS_OUTCOME: 'success', + IOS_CLEANUP_OUTCOME: 'success', + MACOS_CLEANUP_OUTCOME: 'success', + ...overrides, + GITHUB_STEP_SUMMARY: path.join(mkdtempForTestSync('apple-smoke-status-'), 'summary.md'), + }, + encoding: 'utf8', + }); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(expected); +}); + +test('iOS smoke uses affected-check ownership and fails open on uncertain diffs', () => { + const steps = appleJob().steps; + const select = steps.find((step) => step.id === 'ios-impact')!; + const ios = steps.find((step) => step.id === 'ios')!; + expect(select.run).toContain('scripts/apple-ci-impact.ts smoke'); + expect(ios.if).toContain("steps.ios-impact.outputs.run != 'false'"); + for (const file of [ + 'src/foo.test.ts', + 'packages/platform-android/src/index.ts', + 'test/integration/replays/macos/basic.ad', + ]) { + expect(selectIosSmoke('pull_request', [file]).run, file).toBe(false); + } + for (const file of [ + 'packages/platform-apple/src/index.ts', + 'src/index.ts', + '.github/actions/smoke-ios-run/action.yml', + 'examples/test-app/src/App.tsx', + ]) { + expect(selectIosSmoke('pull_request', [file]).run, file).toBe(true); + } + expect(selectIosSmoke('pull_request', ['src/foo.test.ts', 'src/index.ts']).run).toBe(true); + expect(selectIosSmoke('pull_request', null).run).toBe(true); + expect(selectIosSmoke('pull_request', []).run).toBe(true); + expect(selectIosSmoke('push', ['src/foo.test.ts']).run).toBe(true); +}); + function cacheInputs(action: string, stepId = 'source-hash'): string[] { const doc = parse(action) as { runs?: { steps?: Array<{ id?: string; run?: string }> } }; const hashStep = doc.runs?.steps?.find((step) => step.id === stepId)?.run ?? ''; @@ -28,10 +299,13 @@ type WorkflowStep = { run?: string; with?: { gate?: string } }; * scope reasons — or a `run:` block reindented — is not mistaken for a sequencing regression. */ function workflowStepIndex(workflow: string, matches: (step: WorkflowStep) => boolean): number { - const doc = parse(workflow) as { jobs?: Record }; - return Object.values(doc.jobs ?? {}) - .flatMap((job) => job.steps ?? []) - .findIndex(matches); + const doc = parse(workflow) as { + jobs?: Record; + runs?: { steps?: WorkflowStep[] }; + }; + return ( + doc.runs?.steps ?? Object.values(doc.jobs ?? {}).flatMap((job) => job.steps ?? []) + ).findIndex(matches); } test('native runner build-cache inputs trigger the PR XCTest lane', () => { @@ -153,7 +427,10 @@ test('restored native products are rebuilt before caching and icon patching', () }); test('the PR workflow applies the impact decision to the XCTest step', () => { - const workflow = fs.readFileSync(path.join(repoRoot, '.github/workflows/ios.yml'), 'utf8'); + const workflow = fs.readFileSync( + path.join(repoRoot, '.github/actions/smoke-ios-run/action.yml'), + 'utf8', + ); expect(workflow).toContain('node --experimental-strip-types scripts/apple-ci-impact.ts xctest'); expect(workflow).toMatch( /- name: Run targeted iOS runner XCTest regressions\n\s+id: ios-xctest\n\s+if: steps\.xctest-impact\.outputs\.run != 'false'/, @@ -165,7 +442,10 @@ test('macOS clean-install proof follows live UI replay', () => { // Ordered by what each step runs, not by its title: the clean-install proof must not fire // before the replay that can raise local-network permission UI, and a step renamed for scope // reasons is not a sequencing regression. - const workflow = fs.readFileSync(path.join(repoRoot, '.github/workflows/macos.yml'), 'utf8'); + const workflow = fs.readFileSync( + path.join(repoRoot, '.github/actions/smoke-macos/action.yml'), + 'utf8', + ); const replay = workflowStepIndex(workflow, (step) => step.with?.gate === 'replay-macos'); const proof = workflowStepIndex( workflow, @@ -183,7 +463,7 @@ test('native runner and golden-table changes run XCTest; TypeScript runtime chan 'apple/snapshot-presentation/Sources/Presenter.swift', 'packages/platform-apple/src/runner/runner-icon.ts', 'contracts/fixtures/scroll-gesture.json', - '.github/workflows/ios.yml', + '.github/workflows/apple.yml', 'scripts/apple-ci-impact.ts', ]) { expect(selectIosXctests('pull_request', [file]).run, file).toBe(true); @@ -214,7 +494,7 @@ test('bridge proof runs for its owning sources and uncertain tooling changes', ( 'packages/platform-apple/src/foldable/fold-helper-cache.ts', 'packages/platform-apple/src/new-module/runtime.ts', 'scripts/check-package.ts', - '.github/workflows/macos.yml', + '.github/workflows/apple.yml', ]) { expect(selectAppleBridgeProof([file]).run, file).toBe(true); } @@ -223,7 +503,7 @@ test('bridge proof runs for its owning sources and uncertain tooling changes', ( expect(selectAppleBridgeProof([]).run).toBe(true); }); -test('a shallow PR merge still yields a known change set for both selectors', () => { +test('a shallow PR merge still yields a known change set for all Apple selectors', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'apple-ci-impact-')); const source = path.join(root, 'source'); const shallow = path.join(root, 'shallow'); @@ -253,11 +533,16 @@ test('a shallow PR merge still yields a known change set for both selectors', () { cwd: shallow, encoding: 'utf8', - env: { ...process.env, BASE_SHA: base, GITHUB_EVENT_NAME: 'pull_request' }, + env: { + ...process.env, + BASE_SHA: base, + GITHUB_EVENT_NAME: 'pull_request', + }, }, ); expect(select('xctest')).toContain('skip;'); expect(select('bridge')).toContain('skip;'); + expect(select('smoke')).toContain('run; iOS device-lane input changed: src/feature.ts'); } finally { fs.rmSync(root, { recursive: true, force: true }); } diff --git a/scripts/__tests__/xctest-selection.test.ts b/scripts/__tests__/xctest-selection.test.ts index bc367120a9..40c56ec7b8 100644 --- a/scripts/__tests__/xctest-selection.test.ts +++ b/scripts/__tests__/xctest-selection.test.ts @@ -16,13 +16,13 @@ import { ENTRY_POINT_METHOD, formatSummary, GUARDED_WORKFLOWS, - HOST_WORKFLOW_FILE, + HOST_XCTEST_SOURCE_FILE, iosPrTestIdentifiers, LANES, loadReport, NIGHTLY_WORKFLOW_FILE, parseFlaggedTests, - PR_WORKFLOW_FILE, + PR_XCTEST_SOURCE_FILE, reportFailures, runnerPackageSourceFailures, type WorkflowSource, @@ -52,10 +52,10 @@ function realSources() { /** Workflow texts that skip the entry point and consume the guard-derived PR selection. */ function laneWorkflows(): WorkflowSource[] { return [ - { workflow: HOST_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, + { workflow: HOST_XCTEST_SOURCE_FILE, text: `-skip-testing:${ENTRY_POINT}` }, { workflow: NIGHTLY_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, { - workflow: PR_WORKFLOW_FILE, + workflow: PR_XCTEST_SOURCE_FILE, text: 'node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests\n' + 'IOS_PR_TEST_ARGS+=("-only-testing:$test_id")\n' + @@ -189,7 +189,7 @@ describe('the real tree', () => { describe('a planted typo', () => { test('a disconnected PR selector fails the workflow wiring check', () => { - const workflow = laneWorkflows().find((entry) => entry.workflow === PR_WORKFLOW_FILE); + const workflow = laneWorkflows().find((entry) => entry.workflow === PR_XCTEST_SOURCE_FILE); if (!workflow?.text) throw new Error('Missing synthetic PR workflow'); const report = buildReport( TARGET, @@ -197,7 +197,7 @@ describe('a planted typo', () => { `${ENTRY_SOURCE}#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n`, ), laneWorkflows().map((entry) => - entry.workflow === PR_WORKFLOW_FILE + entry.workflow === PR_XCTEST_SOURCE_FILE ? { ...entry, text: workflow.text?.replace('--ios-pr-tests', '--wrong-mode') ?? null } : entry, ), @@ -212,7 +212,7 @@ describe('a planted typo', () => { `${ENTRY_SOURCE}#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n`, ), laneWorkflows().map((entry) => - entry.workflow === PR_WORKFLOW_FILE + entry.workflow === PR_XCTEST_SOURCE_FILE ? { ...entry, text: `${entry.text}\n-only-testing:${TARGET}/RunnerTests/testIosOnly` } : entry, ), @@ -222,7 +222,7 @@ describe('a planted typo', () => { test.each([ ['nightly', NIGHTLY_WORKFLOW_FILE], - ['host', HOST_WORKFLOW_FILE], + ['host', HOST_XCTEST_SOURCE_FILE], ])( 'a renamed `-skip-testing:` entry on the %s lane is reported as unknown AND as the hang it re-arms', (laneId, workflow) => { @@ -308,7 +308,7 @@ describe('a planted typo', () => { `${ENTRY_SOURCE}#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n`, ), laneWorkflows().map((entry) => - entry.workflow === HOST_WORKFLOW_FILE + entry.workflow === HOST_XCTEST_SOURCE_FILE ? { ...entry, text: `${entry.text}\n-skip-testing:${TARGET}/RunnerTests/testIosOnly` } : entry, ), @@ -357,9 +357,9 @@ describe('a planted guard', () => { test('a whole-bundle lane that stops skipping the entry point is reported as reaching it', () => { const report = buildReport(TARGET, source(ENTRY_SOURCE), [ - { workflow: HOST_WORKFLOW_FILE, text: 'run: xcodebuild test-without-building' }, + { workflow: HOST_XCTEST_SOURCE_FILE, text: 'run: xcodebuild test-without-building' }, { workflow: NIGHTLY_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, - laneWorkflows().find((entry) => entry.workflow === PR_WORKFLOW_FILE)!, + laneWorkflows().find((entry) => entry.workflow === PR_XCTEST_SOURCE_FILE)!, ]); expect(report.entryPointReachedBy).toEqual(['host']); expect(reportFailures(report).join('\n')).toContain('reachable by lane(s): host'); @@ -379,7 +379,7 @@ describe('the workflow scan', () => { test('reads both flags on the multi-line xcodebuild invocation', () => { expect( parseFlaggedTests( - PR_WORKFLOW_FILE, + PR_XCTEST_SOURCE_FILE, [ ' xcodebuild test-without-building \\', ' -xctestrun "$XCTESTRUN_PATH" \\', @@ -389,13 +389,13 @@ describe('the workflow scan', () => { ), ).toEqual([ { - workflow: PR_WORKFLOW_FILE, + workflow: PR_XCTEST_SOURCE_FILE, flag: 'only-testing', identifier: `${TARGET}/RunnerTests/testOne`, line: 3, }, { - workflow: PR_WORKFLOW_FILE, + workflow: PR_XCTEST_SOURCE_FILE, flag: 'skip-testing', identifier: `${TARGET}/RunnerTests/testTwo`, line: 4, @@ -438,7 +438,9 @@ describe('the workflow scan', () => { }); test('a bare flag mention with no identifier after it is not a selection', () => { - expect(parseFlaggedTests(PR_WORKFLOW_FILE, 'run: echo "-only-testing: is a flag"')).toEqual([]); + expect( + parseFlaggedTests(PR_XCTEST_SOURCE_FILE, 'run: echo "-only-testing: is a flag"'), + ).toEqual([]); }); test('leaves another target alone rather than guessing about sources it cannot see', () => { @@ -447,10 +449,10 @@ describe('the workflow scan', () => { source(`${ENTRY_SOURCE}extension RunnerTests {\n func testOne() {}\n}\n`), [ { - workflow: PR_WORKFLOW_FILE, + workflow: PR_XCTEST_SOURCE_FILE, text: `-only-testing:SomeOtherTarget/OtherTests/testUnknown\n-only-testing:${TARGET}/RunnerTests/testOne`, }, - { workflow: HOST_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, + { workflow: HOST_XCTEST_SOURCE_FILE, text: `-skip-testing:${ENTRY_POINT}` }, { workflow: NIGHTLY_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, ], ); @@ -477,7 +479,7 @@ describe('the blind-parse guards', () => { }); test('an empty workflow scan fails instead of reporting a healthy list', () => { - const report = buildReport(TARGET, oneTest(), [{ workflow: PR_WORKFLOW_FILE, text: '' }]); + const report = buildReport(TARGET, oneTest(), [{ workflow: PR_XCTEST_SOURCE_FILE, text: '' }]); expect(reportFailures(report).join('\n')).toContain('stopped filtering'); }); }); diff --git a/scripts/apple-ci-impact.ts b/scripts/apple-ci-impact.ts index e696acdd83..3afb2c8515 100644 --- a/scripts/apple-ci-impact.ts +++ b/scripts/apple-ci-impact.ts @@ -5,6 +5,21 @@ import { selectChecks } from './check-affected/model.ts'; type Selection = { run: boolean; reason: string }; +export function selectIosSmoke( + eventName: string, + changedPaths: readonly string[] | null, +): Selection { + if (eventName !== 'pull_request') return { run: true, reason: 'main runs exercise iOS smoke' }; + const checked = checkOwnership(changedPaths); + if (checked.early) return checked.early; + const input = checked.plan.reasons.find( + (entry) => entry.check === 'swift-runner-ios' || entry.check === 'replay-ios', + ); + return input + ? { run: true, reason: `iOS device-lane input changed: ${input.path}` } + : { run: false, reason: 'the affected-check model selected no iOS device lane' }; +} + function checkOwnership( changedPaths: readonly string[] | null, ): { early: Selection; plan?: never } | { early?: never; plan: ReturnType } { @@ -62,18 +77,21 @@ function changedPathsFromGit(baseSha: string): string[] | null { if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { const target = process.argv[2]; - if (target !== 'xctest' && target !== 'bridge') { - process.stderr.write('Expected xctest or bridge target\n'); + if (target !== 'xctest' && target !== 'bridge' && target !== 'smoke') { + process.stderr.write('Expected xctest, bridge, or smoke target\n'); process.exitCode = 2; } else { const eventName = process.env.GITHUB_EVENT_NAME ?? ''; const changedPaths = eventName === 'pull_request' ? changedPathsFromGit(process.env.BASE_SHA ?? '') : []; const selection = - target === 'xctest' - ? selectIosXctests(eventName, changedPaths) - : selectAppleBridgeProof(changedPaths); - const label = target === 'xctest' ? 'iOS XCTest' : 'Apple bridge proof'; + target === 'smoke' + ? selectIosSmoke(eventName, changedPaths) + : target === 'xctest' + ? selectIosXctests(eventName, changedPaths) + : selectAppleBridgeProof(changedPaths); + const label = + target === 'smoke' ? 'iOS smoke' : target === 'xctest' ? 'iOS XCTest' : 'Apple bridge proof'; process.stdout.write(`${label}: ${selection.run ? 'run' : 'skip'}; ${selection.reason}\n`); if (process.env.GITHUB_OUTPUT) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `run=${selection.run}\n`); diff --git a/scripts/check-affected/device-lanes.test.ts b/scripts/check-affected/device-lanes.test.ts index f120c84c9c..db2852bd3a 100644 --- a/scripts/check-affected/device-lanes.test.ts +++ b/scripts/check-affected/device-lanes.test.ts @@ -1,5 +1,5 @@ // Device-lane ownership (#1781 A9-2): planted paths per family, and the two properties the -// ios.yml routing leans on — an Apple-owned change carries its iOS checks in a narrow plan, and +// Apple iOS phase routing leans on — an Apple-owned change carries its iOS checks in a narrow plan, and // a tooling change beside it still fails open to the full set. import assert from 'node:assert/strict'; @@ -183,7 +183,7 @@ test('a golden table owns the parity unit test and both runner builds', () => { test('a tooling change beside an Android-only change still fails open to the full set', () => { for (const tooling of [ - '.github/workflows/ios.yml', + '.github/workflows/apple.yml', '.github/actions/setup-apple-runner-build/action.yml', 'package.json', 'pnpm-lock.yaml', diff --git a/scripts/check-xctest-selection.ts b/scripts/check-xctest-selection.ts index f8c8a2b523..4723e0a56e 100644 --- a/scripts/check-xctest-selection.ts +++ b/scripts/check-xctest-selection.ts @@ -1,14 +1,14 @@ -// `pnpm check:xctest-selection` — derive, from the Swift sources and the workflow files, which +// `pnpm check:xctest-selection` — derive, from the Swift sources and CI definitions, which // runner XCTest methods each CI lane actually reaches, and fail when a method reaches none // (#1781 A7). Also ensures those unit-test methods are stripped from the Apple runner source // copied into the npm package. // // Three lanes run the `AgentDeviceRunnerUITests` bundle, and each reaches a different set: // -// - host macos.yml, macOS host, every PR: the whole bundle as compiled for macOS, minus +// - host smoke-macos/action.yml, macOS host, every PR: the whole bundle as compiled for macOS, minus // `-skip-testing:` — the pure runner-decision tests, whose guard is // `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone. -// - pr ios.yml, iOS Simulator: tests with iOS-specific Swift bodies, when selected. +// - pr smoke-ios-run/action.yml, iOS Simulator: tests with iOS-specific Swift bodies, when selected. // - nightly xctest-nightly.yml, iOS Simulator, scheduled: the whole bundle as compiled for // iOS, minus `-skip-testing:` — includes the simulator-only tests, whose guard is // `… && os(iOS)` (they launch the host app, route through SpringBoard, or assert an @@ -45,9 +45,9 @@ const repoRoot = path.resolve(import.meta.dirname, '..'); const packageAppleRunnerScript = path.join(repoRoot, 'scripts/package-apple-runner-source.mjs'); /** The macOS host lane, which runs the whole macOS-compiled bundle on every PR. */ -export const HOST_WORKFLOW_FILE = '.github/workflows/macos.yml'; +export const HOST_XCTEST_SOURCE_FILE = '.github/actions/smoke-macos/action.yml'; -export const PR_WORKFLOW_FILE = '.github/workflows/ios.yml'; +export const PR_XCTEST_SOURCE_FILE = '.github/actions/smoke-ios-run/action.yml'; /** The nightly lane, whose `-skip-testing:` list decides what the full simulator suite leaves out. */ export const NIGHTLY_WORKFLOW_FILE = '.github/workflows/xctest-nightly.yml'; @@ -67,14 +67,14 @@ export type Lane = { export const LANES: readonly Lane[] = [ { id: 'host', - workflow: HOST_WORKFLOW_FILE, + workflow: HOST_XCTEST_SOURCE_FILE, platform: 'macOS', selection: 'whole', title: 'iOS runner host XCTest lane (macOS, no simulator)', }, { id: 'pr', - workflow: PR_WORKFLOW_FILE, + workflow: PR_XCTEST_SOURCE_FILE, platform: 'iOS', selection: 'ios-specific', title: 'iOS-specific runner PR XCTest lane', @@ -312,7 +312,7 @@ function prWorkflowWiringFailures(text: string | null): string[] { if (!active.includes('-only-testing:$test_id') || !active.includes('"${IOS_PR_TEST_ARGS[@]}"')) { failures.push('PR workflow does not pass each generated XCTest identifier to xcodebuild.'); } - if (parseFlaggedTests(PR_WORKFLOW_FILE, active).length > 0) { + if (parseFlaggedTests(PR_XCTEST_SOURCE_FILE, active).length > 0) { failures.push('PR workflow still contains hand-maintained XCTest selection identifiers.'); } return failures; @@ -339,7 +339,7 @@ export function buildReport( ) as Record>; const entryPoint = `${target}/${ENTRY_POINT_METHOD}`; const reachedAnywhere = new Set(LANES.flatMap((entry) => [...reach[entry.id]])); - const prWorkflow = workflows.find((entry) => entry.workflow === PR_WORKFLOW_FILE); + const prWorkflow = workflows.find((entry) => entry.workflow === PR_XCTEST_SOURCE_FILE); return { target, declared, @@ -469,7 +469,7 @@ export function formatSummary(report: SelectionReport): string { const { declared, host, pr, nightly, dark } = counts(report); return ( `xctest selection: ${declared} declared ${report.target} methods — host lane ` + - `(${HOST_WORKFLOW_FILE}, macOS, every PR) reaches ${host}, PR iOS-specific lane (${PR_WORKFLOW_FILE}, ` + + `(${HOST_XCTEST_SOURCE_FILE}, macOS, every PR) reaches ${host}, PR iOS-specific lane (${PR_XCTEST_SOURCE_FILE}, ` + `iOS Simulator, selected PRs and main) selects ${pr}, nightly (${NIGHTLY_WORKFLOW_FILE}, iOS Simulator) ` + `reaches ${nightly}; ${dark} reachable by no lane; ${ENTRY_POINT_METHOD} skipped everywhere.\n` ); diff --git a/scripts/decline-hosted-compute-local-network.applescript b/scripts/decline-hosted-compute-local-network.applescript new file mode 100644 index 0000000000..c4ee5eef40 --- /dev/null +++ b/scripts/decline-hosted-compute-local-network.applescript @@ -0,0 +1,73 @@ +on matchingDeclineButtons() + set matches to {} + tell application "System Events" + repeat with candidateProcess in (application processes whose bundle identifier is "com.apple.UserNotificationCenter") + try + set candidateWindows to windows of candidateProcess + on error + set candidateWindows to {} + end try + repeat with candidateWindow in candidateWindows + try + set elements to entire contents of candidateWindow + on error + set elements to {} + end try + set headingMatches to false + set declineButtons to {} + repeat with candidateElement in elements + try + if role of candidateElement is "AXStaticText" and value of candidateElement is "Allow “hosted-compute-” to find devices on local networks?" then set headingMatches to true + if role of candidateElement is "AXButton" and name of candidateElement is in {"Don’t Allow", "Don't Allow"} then set end of declineButtons to contents of candidateElement + end try + end repeat + if headingMatches then + if (count of declineButtons) is not 1 then error "The hosted-compute local-network prompt has no unique decline button." + set end of matches to item 1 of declineButtons + end if + end repeat + end repeat + end tell + return matches +end matchingDeclineButtons + +on describePrompts() + tell application "System Events" + set notificationProcesses to application processes whose bundle identifier is "com.apple.UserNotificationCenter" + log "UserNotificationCenter processes: " & (count of notificationProcesses) + repeat with candidateProcess in notificationProcesses + set notificationWindows to windows of candidateProcess + log "UserNotificationCenter windows: " & (count of notificationWindows) + repeat with candidateWindow in notificationWindows + set elements to entire contents of candidateWindow + repeat with candidateElement in elements + set fields to {} + repeat with attributeName in {"AXRole", "AXTitle", "AXValue", "AXDescription"} + try + set attributeValue to value of attribute (contents of attributeName) of candidateElement + if attributeValue is not missing value then set end of fields to (contents of attributeName) & "=" & (attributeValue as text) + end try + end repeat + log fields + end repeat + end repeat + end repeat + end tell +end describePrompts + +on run arguments + if (system attribute "GITHUB_ACTIONS") is not "true" then error "This cleanup is only for GitHub Actions runners." + try + describePrompts() + on error messageText + log "Notification diagnostics failed: " & messageText + end try + if arguments is {"diagnose"} then return "Read-only hosted notification diagnostics." + set declineButtons to matchingDeclineButtons() + if (count of declineButtons) is 0 then return "No hosted-compute local-network prompt." + if (count of declineButtons) is not 1 then error "Multiple hosted-compute local-network prompts are present." + tell application "System Events" to click item 1 of declineButtons + delay 0.25 + if (count of matchingDeclineButtons()) is not 0 then error "The hosted-compute local-network prompt remained after declining." + return "Declined hosted-compute local-network access." +end run diff --git a/scripts/gate/audit.ts b/scripts/gate/audit.ts index 7c01f07153..eb3dc51c56 100644 --- a/scripts/gate/audit.ts +++ b/scripts/gate/audit.ts @@ -6,12 +6,9 @@ import { MANUAL_ONLY_OWNERS, type ManualOnlyOwner, REPORTING_SCRIPTS, - ROUTED_LANES, - type RoutedLane, UNPROVABLE_OWNERS, } from './declarations.ts'; import { categories, checkUnits, covered, scriptUnits, type Model } from './model.ts'; -import { routing } from './routing.ts'; export type Failure = { readonly assertion: string; readonly message: string }; @@ -19,13 +16,11 @@ export type Failure = { readonly assertion: string; readonly message: string }; export type GateDeclarations = { readonly manualOnly: Readonly>; readonly unprovable: Readonly>; - readonly routed?: readonly RoutedLane[]; }; const DECLARED: GateDeclarations = { manualOnly: MANUAL_ONLY_OWNERS, unprovable: UNPROVABLE_OWNERS, - routed: ROUTED_LANES, }; const HEADINGS: Readonly> = { @@ -34,7 +29,6 @@ const HEADINGS: Readonly> = { gate: 'Gate ids that name no registered check', surface: 'Execution surfaces the manifest does not model', 'path-coverage': 'Paths whose selected checks no triggered lane runs', - routing: 'Routed lanes whose paths-ignore disagrees with the selector', registered: 'Suites and projects no registered check covers', }; @@ -246,7 +240,6 @@ export function audit(model: Model, declarations: GateDeclarations = DECLARED): ...gateIds(model), ...laneSurfaces(model), ...pathCoverage(model, declarations), - ...routing(model, declarations.routed ?? []), ...unregisteredSuites(model), ...orphanProjects(model), ]; diff --git a/scripts/gate/declarations.ts b/scripts/gate/declarations.ts index ea6c562719..a1cc0b0660 100644 --- a/scripts/gate/declarations.ts +++ b/scripts/gate/declarations.ts @@ -72,25 +72,3 @@ export const MANUAL_ONLY_OWNERS: Readonly> = { ].join(' '), }, }; - -/** - * A `pull_request` lane whose `paths-ignore` list is routing, not hygiene (#1781 A9-2): the - * list is asserted against the affected selector over every tracked path, both ways. A path - * the selector fails open on, or routes to one of the lane's declared gates or to `sampled` - * (checks the lane runs a slice of through raw shell, invisible to the loader), must start the - * lane; a path the selector places on another family's device-lane surface, or classifies as - * a unit test, must not. GitHub evaluates the list before a runner is allocated, so the - * decision costs no macOS time and adds no job to the critical path. - */ -export type RoutedLane = { - /** Lane label, exactly as the loader builds it: ` / `. */ - readonly lane: string; - /** Checks the lane samples without declaring, in addition to its run-gate declarations. */ - readonly sampled: readonly string[]; -}; - -export const ROUTED_LANES: readonly RoutedLane[] = [ - // ios.yml declares swift-runner-ios; its raw-shell steps run one iOS simulator replay and, when - // IOS_UDID is set, one physical-device replay — a slice of the parked replay-ios lanes. - { lane: 'iOS / Smoke Tests', sampled: ['replay-ios', 'replay-ios-device'] }, -]; diff --git a/scripts/gate/routing.test.ts b/scripts/gate/routing.test.ts deleted file mode 100644 index c20de1fad4..0000000000 --- a/scripts/gate/routing.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -// Routed-lane witnesses (#1781 A9-2): the live ios.yml agrees with the selector, and each -// planted disagreement below is reported — an ignore that hides an Apple path or a tooling -// glob, and a missing ignore for a tree the selector says is another family's. - -import { execFileSync } from 'node:child_process'; -import assert from 'node:assert/strict'; -import path from 'node:path'; -import test from 'node:test'; -import { ROUTED_LANES } from './declarations.ts'; -import { loadModel, type Model } from './model.ts'; -import { routing } from './routing.ts'; -import type { Lane } from './workflows.ts'; - -const repoRoot = path.resolve(import.meta.dirname, '../..'); -const tracked = execFileSync('git', ['ls-files'], { cwd: repoRoot, encoding: 'utf8' }) - .split('\n') - .filter(Boolean); -const base = loadModel(repoRoot, tracked); -const IOS = ROUTED_LANES.find((routed) => routed.lane === 'iOS / Smoke Tests'); -assert.ok(IOS, 'the iOS lane is declared routed'); -const iosLane = base.lanes.find((lane) => lane.label === IOS.lane); -assert.ok(iosLane, 'ios.yml defines the routed lane'); - -function withIos(change: (lane: Lane) => Lane): Model { - return { - ...base, - lanes: base.lanes.map((lane) => (lane.label === IOS.lane ? change(lane) : lane)), - }; -} - -const messages = (model: Model) => routing(model, ROUTED_LANES).map((failure) => failure.message); - -test('the live ios.yml paths-ignore agrees with the selector over every tracked path', () => { - assert.deepEqual(messages(base), []); -}); - -test('the live iOS route carries no filters for the retired src/platforms tree', () => { - assert.deepEqual( - iosLane.pathsIgnore.filter((pattern) => pattern.startsWith('src/platforms/')), - [], - ); -}); - -test('the routed lane derives its needs from its declared gate plus the sampled checks', () => { - assert.ok(iosLane.gates.includes('swift-runner-ios'), 'ios.yml still declares the runner build'); - assert.deepEqual([...IOS.sampled], ['replay-ios', 'replay-ios-device']); -}); - -test('ignoring an Apple-owned tree is reported with the checks the selector routes it to', () => { - const model = withIos((lane) => ({ - ...lane, - pathsIgnore: [...lane.pathsIgnore, 'packages/platform-apple/src/**'], - })); - const found = messages(model); - assert.ok(found.length > 0); - assert.ok( - found.some( - (message) => - /ignores packages\/platform-apple\/src\//.test(message) && - /routes it to "replay-ios"/.test(message), - ), - found.slice(0, 3).join('\n'), - ); -}); - -test('ignoring a tooling glob is reported as fail-open, even under .github/', () => { - const model = withIos((lane) => ({ - ...lane, - pathsIgnore: [...lane.pathsIgnore, '.github/actions/**', 'package.json'], - })); - const found = messages(model); - assert.ok( - found.some( - (message) => - /ignores \.github\/actions\/setup-apple-runner-build\/action\.yml/.test(message) && - /fails open on it \(workflow-tooling\)/.test(message), - ), - found.slice(0, 3).join('\n'), - ); - assert.ok( - found.some((message) => /ignores package\.json, but the selector fails open/.test(message)), - ); -}); - -test('a .github path ignored by exact name stays the workflow’s own call', () => { - // The live list names deploy.yml and the docs preview workflows explicitly; the live tree - // is green above, so this only pins that the exemption is by exact path, not by prefix. - const model = withIos((lane) => ({ - ...lane, - pathsIgnore: [...lane.pathsIgnore, '.github/workflows/size.yml'], - })); - assert.deepEqual(messages(model), []); -}); - -// Review finding on #1857: the exact-name exemption was unbounded, so naming the lane's OWN -// build and boot actions skipped the lane that runs them and the manifest stayed green. The -// exemption now stops at `lane.uses` — the transitive composite-action closure plus the -// workflow file — and these are the reviewer's two planted cases. -test('the exemption cannot name an action the lane itself runs', () => { - for (const own of [ - '.github/actions/setup-apple-runner-build/action.yml', - '.github/actions/boot-ios-test-simulator/action.yml', - '.github/actions/run-gate/action.yml', // reached only through a composite action - '.github/workflows/ios.yml', // the lane's own definition - ]) { - assert.ok(iosLane.uses.includes(own), `${own} must be in the lane's uses closure`); - const model = withIos((lane) => ({ ...lane, pathsIgnore: [...lane.pathsIgnore, own] })); - assert.ok( - messages(model).some( - (message) => message.includes(`ignores ${own}`) && /fails open/.test(message), - ), - `naming ${own} exactly must still fail the routing assertion`, - ); - } -}); - -// Second review finding on #1857, one level past the first: `Lane.uses` recorded only the -// composite actions' `action.yml` descriptors, so a support file the descriptor *executes* -// (`bash "$GITHUB_ACTION_PATH/fetch-artifact.sh"`) was exemptible as if it were an unrelated -// sibling workflow. The closure is the action's directory now, which also covers the two files -// referenced no closer than inside that shell script. -test('the exemption cannot name a support file of an action the lane runs', () => { - const support = [ - '.github/actions/setup-fixture-app/fetch-artifact.sh', // named by the action.yml - '.github/actions/setup-fixture-app/resolve-artifact-name.sh', // named only inside that script - '.github/actions/setup-fixture-app/trusted-artifact.mjs', // likewise - ]; - for (const file of support) { - assert.ok(iosLane.uses.includes(file), `${file} must be in the lane's uses closure`); - const model = withIos((lane) => ({ ...lane, pathsIgnore: [...lane.pathsIgnore, file] })); - assert.ok( - messages(model).some( - (message) => message.includes(`ignores ${file}`) && /fails open/.test(message), - ), - `naming ${file} exactly must still fail the routing assertion`, - ); - } -}); - -// The trap fires correctly for a tracked non-TS file under an ignored family root, but the -// remedy is not "remove the ignore entry" — that would un-route every sibling `.ts` in the -// tree. The selector gap is the fix, and the message has to say so. -test('an unowned path under an ignored root asks for an owner, not for the entry’s removal', () => { - const planted = 'packages/platform-android/src/probe-fixture.json'; - const model = { - ...base, - trackedFiles: new Set([...base.trackedFiles, planted]), - }; - const found = routing(model, ROUTED_LANES).map((failure) => failure.message); - assert.equal(found.length, 1, found.join('\n')); - assert.match(found[0] ?? '', /fails open on it \(unknown-path\)/); - assert.match(found[0] ?? '', /Give it an owning check in scripts\/check-affected\//); - assert.ok(!/Remove the ignore entry/.test(found[0] ?? '')); -}); - -test('dropping a family root from the ignore list fails the routing claim for that tree', () => { - const model = withIos((lane) => ({ - ...lane, - pathsIgnore: lane.pathsIgnore.filter( - (pattern) => pattern !== 'packages/platform-android/src/**', - ), - })); - const found = messages(model); - assert.ok( - found.some( - (message) => - /starts on packages\/platform-android\/src\//.test(message) && - /android-owned \(lanes: replay-android\)/.test(message), - ), - found.slice(0, 3).join('\n'), - ); -}); - -test('dropping the unit-test ignore fails the routing claim for a unit test', () => { - const model = withIos((lane) => ({ - ...lane, - pathsIgnore: lane.pathsIgnore.filter((pattern) => pattern !== 'src/**/*.test.ts'), - })); - assert.ok( - messages(model).some((message) => - /\.test\.ts, which the selector classifies as a unit test/.test(message), - ), - ); -}); - -test('a routed lane that loses its pull_request trigger, or its job, is reported', () => { - const gone = { ...base, lanes: base.lanes.filter((lane) => lane.label !== IOS.lane) }; - assert.ok(messages(gone).some((message) => /not defined by any workflow/.test(message))); - const pushOnly = withIos((lane) => ({ ...lane, triggers: ['push'] })); - assert.ok(messages(pushOnly).some((message) => /no pull_request trigger/.test(message))); -}); diff --git a/scripts/gate/routing.ts b/scripts/gate/routing.ts deleted file mode 100644 index 3e13806532..0000000000 --- a/scripts/gate/routing.ts +++ /dev/null @@ -1,110 +0,0 @@ -// Routed lanes: a `paths-ignore` list held to the affected selector, both ways (#1781 A9-2). -// -// GitHub evaluates `paths-ignore` before it allocates a runner, so it is the one routing -// mechanism that costs no macOS time and adds no job to the critical path. Its weakness is -// that it is a hand-written glob list next to a derived selector — the two drift the first -// time someone adds a platform root or a unit-test convention. This assertion is what makes -// the YAML a derived artifact: over every tracked path, the lane must start whenever the -// selector says the change can reach it, and must not start on a path the selector places on -// another family's device-lane surface or classifies as a unit test. -// -// One limit is GitHub's, not this assertion's: path filters examine only the first 300 changed -// files, so a PR larger than that can skip a routed lane on the strength of its first 300 paths -// alone. The unfiltered `push` to main is the backstop. - -import { deviceLanesFor, isDeviceLaneSurface, isUnitTest } from '../check-affected/device-lanes.ts'; -import { isDocs, selectChecks, type CheckId } from '../check-affected/model.ts'; -import type { RoutedLane } from './declarations.ts'; -import type { Model } from './model.ts'; -import { type Lane, triggersOnPath } from './workflows.ts'; - -export type RoutingFailure = { readonly assertion: 'routing'; readonly message: string }; - -function failure(message: string): RoutingFailure { - return { assertion: 'routing', message }; -} - -// A `.github/**` path the lane ignores by its exact name is the workflow's own knowledge of a -// sibling workflow it does not use (deploy, docs preview); the selector's fail-open on -// `.github/**` is about *local* checks and cannot see that. -// -// The exemption stops at the lane's own machinery, and that is enforced rather than asserted in -// prose: `lane.uses` is the transitive closure of the composite actions the job's steps run, -// plus the workflow file itself, so naming `setup-apple-runner-build/action.yml` or -// `boot-ios-test-simulator/action.yml` exactly is refused the way a glob is. Reviewer planted -// both and the manifest stayed green before this check existed. -function exemptSiblingWorkflow(lane: Lane, file: string): boolean { - return ( - file.startsWith('.github/') && lane.pathsIgnore.includes(file) && !lane.uses.includes(file) - ); -} - -// Why the selector says the lane must start on `file`, and what to do about it, or null when -// the lane need not start. The remedy differs by cause: a path the selector *routes* to the -// lane, or fails open on because it is tooling, is one the ignore list must not name. A path it -// fails open on because it has no owner at all (`unknown-path`/`ambiguous-path` — a fixture -// under a family root, say) is a selector gap: deleting the ignore entry that happens to match -// it would un-route every sibling in that tree, which is the opposite of the fix. -const UNOWNED_RULES = new Set(['unknown-path', 'ambiguous-path']); - -function mustStart(model: Model, needs: ReadonlySet, file: string): string | null { - const plan = selectChecks({ changedFiles: [file], packageEntryFiles: model.packageEntryFiles }); - if (plan.failOpen) { - const rules = plan.failOpenReasons.map((reason) => reason.rule); - const remedy = rules.every((rule) => UNOWNED_RULES.has(rule)) - ? 'Give it an owning check in scripts/check-affected/ — the ignore entry that matches it ' + - 'is load-bearing for the rest of that tree' - : 'Remove the ignore entry'; - return `fails open on it (${rules.join(', ')}). ${remedy}.`; - } - const routedTo = plan.checks.filter((id) => needs.has(id)); - return routedTo.length > 0 - ? `routes it to ${routedTo.map((id) => `"${id}"`).join(', ')}. Remove the ignore entry.` - : null; -} - -// How the selector classifies a path the lane need not start, or null when it makes no claim -// (a path outside the device-lane surface is simply not the routing's business). -function mustNotStart(needs: ReadonlySet, file: string): string | null { - if (isUnitTest(file)) return 'a unit test no device lane runs'; - if (!isDeviceLaneSurface(file)) return null; - const { leaf, lanes } = deviceLanesFor(file); - if (lanes.some((id: CheckId) => needs.has(id))) return null; - return `${leaf}-owned (lanes: ${lanes.join(', ') || 'none'})`; -} - -function routingFor(model: Model, routed: RoutedLane): RoutingFailure[] { - const lane = model.lanes.find((candidate) => candidate.label === routed.lane); - if (!lane) { - return [failure(`routed lane "${routed.lane}" is not defined by any workflow.`)]; - } - if (!lane.triggers.includes('pull_request')) { - return [failure(`routed lane "${routed.lane}" has no pull_request trigger to route.`)]; - } - const needs = new Set([...lane.gates, ...routed.sampled]); - return [...model.trackedFiles] - .sort() - .filter((file) => !isDocs(file)) - .flatMap((file) => { - const starts = triggersOnPath(lane, file); - const why = mustStart(model, needs, file); - if (why !== null) { - return starts || exemptSiblingWorkflow(lane, file) - ? [] - : [failure(`${lane.workflow} ignores ${file}, but the selector ${why}`)]; - } - const claim = starts ? mustNotStart(needs, file) : null; - return claim === null - ? [] - : [ - failure( - `${lane.workflow} starts on ${file}, which the selector classifies as ${claim}. ` + - `Add it to paths-ignore, or the routing claim is false for that path.`, - ), - ]; - }); -} - -export function routing(model: Model, routedLanes: readonly RoutedLane[]): RoutingFailure[] { - return routedLanes.flatMap((routed) => routingFor(model, routed)); -} diff --git a/scripts/gate/workflows.test.ts b/scripts/gate/workflows.test.ts index 06a6b5b13d..978bb053b1 100644 --- a/scripts/gate/workflows.test.ts +++ b/scripts/gate/workflows.test.ts @@ -83,7 +83,7 @@ test('path filters use GitHub glob semantics', () => { test('lanes carry the workflow spelling the catalog used, and only real triggers qualify', () => { const labels = model.lanes.map((lane) => lane.label); assert.ok(labels.includes('Coverage'), 'CI jobs are named bare'); - assert.ok(labels.includes('iOS / Smoke Tests'), 'other workflows are prefixed'); + assert.ok(labels.includes('Apple / Smoke Tests'), 'other workflows are prefixed'); const deploy = model.lanes.find((lane) => lane.workflow === 'deploy.yml'); assert.equal(deploy?.qualifying, false, 'a push-only lane gates nothing on the way in'); }); @@ -336,9 +336,8 @@ test('device smoke lanes skip drafts and start when a draft is marked ready', () ); assert.deepEqual(smokeLanes.map(({ workflow }) => workflow.name).sort(), [ 'Android', + 'Apple', 'Linux', - 'iOS', - 'macOS', ]); for (const { workflow, job } of smokeLanes) { assert.deepEqual( diff --git a/scripts/gate/workflows.ts b/scripts/gate/workflows.ts index b711641ca4..5bafa455d2 100644 --- a/scripts/gate/workflows.ts +++ b/scripts/gate/workflows.ts @@ -22,8 +22,7 @@ export type Lane = { readonly gates: readonly CheckId[]; /** * Repo-relative files of the local composite actions this job's steps use, transitively, - * plus the workflow file itself: the lane's own machinery. Editing one of these changes what - * the lane does, so the lane cannot claim to be unaffected by it (`scripts/gate/routing.ts`). + * plus the workflow file itself. */ readonly uses: readonly string[]; readonly verbatim: readonly string[]; diff --git a/scripts/teardown-ios-smoke.sh b/scripts/teardown-ios-smoke.sh new file mode 100644 index 0000000000..cd28465966 --- /dev/null +++ b/scripts/teardown-ios-smoke.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "${GITHUB_ACTIONS:-}" != true ]; then + echo 'iOS smoke teardown is only for GitHub Actions runners.' >&2 + exit 1 +fi + +status=0 +pnpm clean:daemon || status=1 +xcrun simctl shutdown all || status=1 +osascript -e 'if application "Simulator" is running then tell application "Simulator" to quit' || status=1 +node --input-type=module <<'JS' || status=1 +import { execFileSync } from 'node:child_process'; +import path from 'node:path'; +const inventory = JSON.parse(execFileSync('xcrun', ['simctl', 'list', 'devices', '--json'], { encoding: 'utf8' })); +const booted = Object.values(inventory.devices ?? {}).flat().filter((device) => device.state === 'Booted'); +if (booted.length) throw new Error(`Simulators remain booted: ${booted.map((device) => device.udid).join(', ')}`); +const remaining = execFileSync('ps', ['-axo', 'comm='], { encoding: 'utf8' }).split('\n').map((entry) => path.basename(entry.trim())).filter((name) => /^(?:AgentDeviceRunner.*|xcodebuild|xctest|XCTRunner|Simulator)$/.test(name)); +if (remaining.length) throw new Error(`iOS smoke processes remain: ${remaining.join(', ')}`); +JS +mkdir -p test/screenshots +screencapture -x test/screenshots/apple-host-after-ios.png || echo "Could not capture the host teardown screenshot." >&2 +osascript scripts/decline-hosted-compute-local-network.applescript || status=1 +osascript -e 'tell application "Finder" to activate' || status=1 +frontmost="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" || status=1 +if [ "$frontmost" != com.apple.finder ]; then + echo "Host focus was not restored to Finder: $frontmost" >&2 + status=1 +fi +exit "$status" diff --git a/test/ci/root-docs-paths-ignore.test.ts b/test/ci/root-docs-paths-ignore.test.ts index e4e177c524..1cf48fb62f 100644 --- a/test/ci/root-docs-paths-ignore.test.ts +++ b/test/ci/root-docs-paths-ignore.test.ts @@ -1,7 +1,7 @@ // Regression pin for #1781 A9: the five root-level docs files // (AGENTS.md, CONTEXT.md, CONTRIBUTING.md, LICENSE, SECURITY.md) // must stay in the `pull_request` `paths-ignore` list of every workflow that -// also ignores `docs/**`/`website/**`/`README.md` — the four device lanes plus +// also ignores `docs/**`/`website/**`/`README.md` — the three device lanes plus // `ci.yml` and `size.yml`. Nothing else derives this: `check:gate-manifest` // only asks whether a *registered check* is reachable, generic Markdown // coverage does not look at workflow trigger config at all, and `actionlint` @@ -20,7 +20,7 @@ import { matchesGlob } from '../../scripts/gate/workflows.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); -const WORKFLOWS = ['ios.yml', 'android.yml', 'linux.yml', 'macos.yml', 'ci.yml', 'size.yml']; +const WORKFLOWS = ['apple.yml', 'android.yml', 'linux.yml', 'ci.yml', 'size.yml']; const ROOT_DOCS = ['AGENTS.md', 'CONTEXT.md', 'CONTRIBUTING.md', 'LICENSE', 'SECURITY.md']; diff --git a/test/ci/upload-agent-device-artifacts.test.ts b/test/ci/upload-agent-device-artifacts.test.ts index 81fe7bbf6a..dbac23a772 100644 --- a/test/ci/upload-agent-device-artifacts.test.ts +++ b/test/ci/upload-agent-device-artifacts.test.ts @@ -48,15 +48,19 @@ test('the shared diagnostics artifact includes hidden files only from its declar '${{ inputs.runner-derived-path }}/Logs/**', 'test/artifacts/**', 'test/screenshots/**', + '${{ inputs.additional-paths }}', ]); + expect(action.inputs['additional-paths']).toMatchObject({ required: false, default: '' }); expect( - paths(upload!).every((entry) => - [ - '${{ inputs.agent-state-dir }}/', - '${{ inputs.runner-derived-path }}/', - '~/.agent-device/logs/', - 'test/', - ].some((prefix) => entry.startsWith(prefix)), + paths(upload!).every( + (entry) => + entry === '${{ inputs.additional-paths }}' || + [ + '${{ inputs.agent-state-dir }}/', + '${{ inputs.runner-derived-path }}/', + '~/.agent-device/logs/', + 'test/', + ].some((prefix) => entry.startsWith(prefix)), ), ).toBe(true); }); diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index edd5ae2116..1a534b7e4c 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -644,7 +644,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'Android prepare fails closed through its unavailable prepareAppleRunner fact', ), iosSimulator: iosSimulator.workflowLive( - '.github/workflows/ios.yml', + '.github/actions/smoke-ios-run/action.yml', 'Preflight iOS runner through public CLI', 'cached XCTest runner is prepared once before Settings and fixture suites', ), diff --git a/test/integration/ios-simulator-e2e/behavior-coverage.ts b/test/integration/ios-simulator-e2e/behavior-coverage.ts index 7e8fdf6b13..05ff0d78c4 100644 --- a/test/integration/ios-simulator-e2e/behavior-coverage.ts +++ b/test/integration/ios-simulator-e2e/behavior-coverage.ts @@ -83,7 +83,7 @@ export const IOS_SIMULATOR_BEHAVIOR_COVERAGE = { 'when the hosted runner can establish a Finder canary, it remains frontmost across simulator automation', level: 'workflow-live', owner: { - path: '.github/workflows/ios.yml', + path: '.github/actions/smoke-ios-run/action.yml', test: 'Assert simulator automation preserved host focus', }, },