diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml deleted file mode 100644 index 12e120ca1c9ba..0000000000000 --- a/.github/workflows/infra.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: "infra" - -on: - push: - branches: - - main - - release-* - pull_request: - branches: - - main - - release-* - -jobs: - doc-and-lint: - name: "docs & lint" - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: lts/* - - run: npm ci - - run: npm run build - - run: npx playwright install --with-deps - - run: npm run lint - - name: Verify clean tree - run: | - if [[ -n $(git status -s) ]]; then - echo "ERROR: tree is dirty after npm run build:" - git diff - exit 1 - fi - - name: Audit prod NPM dependencies - run: node utils/check_audit.js - continue-on-error: true - lint-snippets: - name: "Lint snippets" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: lts/* - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.11' - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 - with: - dotnet-version: 8.0.x - - uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 - with: - distribution: 'zulu' - java-version: '21' - - run: npm ci - - run: pip install -r utils/doclint/linting-code-snippets/python/requirements.txt - - run: mvn package - working-directory: utils/doclint/linting-code-snippets/java - - run: node utils/doclint/linting-code-snippets/cli.js diff --git a/.github/workflows/tests_bidi.yml b/.github/workflows/tests_bidi.yml deleted file mode 100644 index e70fbd5a00384..0000000000000 --- a/.github/workflows/tests_bidi.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: tests BiDi - -on: - workflow_dispatch: - inputs: - ref: - description: Playwright SHA / ref to test. Use 'refs/pull/PULL_REQUEST_ID/head' to test a PR. Defaults to the current branch. - required: false - default: '' - pull_request: - branches: - - main - paths: - - .github/workflows/tests_bidi.yml - - packages/playwright-core/src/server/bidi/** - - tests/bidi/** - schedule: - # Run every day at midnight - - cron: '0 0 * * *' - -env: - FORCE_COLOR: 1 - -jobs: - test_bidi: - name: BiDi - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-24.04 - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - strategy: - fail-fast: false - matrix: - channel: [bidi-chromium, moz-firefox-nightly] - isPullRequest: - - ${{ github.event_name == 'pull_request' }} - exclude: - - isPullRequest: true - channel: bidi-chromium - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: github.event_name != 'workflow_dispatch' - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: github.event_name == 'workflow_dispatch' - with: - ref: ${{ github.event.inputs.ref }} - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 20 - - run: npm ci - - run: npm run build - # Needed for video tests - - run: npx playwright install ffmpeg - - run: npx playwright install --with-deps chromium - - if: matrix.channel == 'moz-firefox-nightly' - run: | - sudo apt-get update - sudo apt-get install -y libavcodec60 - echo "BIDI_FFPATH=$(npx -y @puppeteer/browsers install firefox@nightly | tail -n1 | sed 's/^[^ ]* *//')" >> "$GITHUB_ENV" - - name: Run tests - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run biditest -- --retries=${{ matrix.isPullRequest && 2 || 0 }} --project=${{ matrix.channel }}* - env: - PWTEST_USE_BIDI_EXPECTATIONS: ${{ matrix.isPullRequest && '1' || '' }} - - name: Upload csv report to GitHub - if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: csv-report-${{ matrix.channel }} - path: test-results/report.csv - retention-days: 7 - - - name: Upload json report to GitHub - if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: json-report-${{ matrix.channel }} - path: test-results/report.json - retention-days: 7 - - - name: Upload parquet report - if: ${{ !cancelled() }} - uses: ./.github/actions/upload-parquet-report - with: - job_name: ${{ matrix.channel }} - - - name: Azure Login - if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} - uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3.0.2 - with: - client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }} - - - name: Upload report.csv to Azure - if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} - run: | - REPORT_DIR='bidi-reports' - azcopy cp "./test-results/report.csv" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR/${{ matrix.channel }}.csv" - echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/${{ matrix.channel }}.csv" - env: - AZCOPY_AUTO_LOGIN_TYPE: AZCLI diff --git a/.github/workflows/tests_docker_changes.yml b/.github/workflows/tests_docker_changes.yml deleted file mode 100644 index 8807d32ae6788..0000000000000 --- a/.github/workflows/tests_docker_changes.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: tests docker (changes) - -on: - push: - branches: - - main - paths: - - 'utils/docker/**' - - '.github/workflows/tests_docker*.yml' - pull_request: - branches: - - main - - release-* - paths: - - 'utils/docker/**' - - '.github/workflows/tests_docker*.yml' - -jobs: - test_linux_docker: - uses: ./.github/workflows/tests_docker.yml - permissions: - id-token: write - contents: read - secrets: inherit diff --git a/.github/workflows/tests_extension.yml b/.github/workflows/tests_extension.yml deleted file mode 100644 index bcee6798551ff..0000000000000 --- a/.github/workflows/tests_extension.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: extension - -on: - push: - branches: - - main - - release-* - paths: - - 'packages/playwright-core/src/tools/**' - - '!packages/playwright-core/src/tools/dashboard/**' - - '!packages/playwright-core/src/tools/trace/**' - - 'packages/extension/**' - - 'tests/extension/**' - - '.github/workflows/tests_extension.yml' - pull_request: - branches: - - main - - release-* - paths: - - 'packages/playwright-core/src/tools/**' - - '!packages/playwright-core/src/tools/dashboard/**' - - '!packages/playwright-core/src/tools/trace/**' - - 'packages/extension/**' - - 'tests/extension/**' - - '.github/workflows/tests_extension.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - FORCE_COLOR: 1 - DEBUG: pw:mcp:error - -jobs: - test_extension: - name: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 - - run: npm ci - - run: npm run build - - run: npx playwright install --with-deps chromium - - run: npm run test-extension - - name: Upload parquet report - if: ${{ !cancelled() }} - uses: ./.github/actions/upload-parquet-report - with: - job_name: extension-${{ matrix.os }} diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.yml deleted file mode 100644 index 88dc1041c6229..0000000000000 --- a/.github/workflows/tests_mcp.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: MCP - -on: - push: - branches: - - main - - release-* - pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - - 'packages/extension/**' - - 'packages/playwright-core/src/server/bidi/**' - - 'tests/bidi/**' - - 'tests/extension/**' - branches: - - main - - release-* - -concurrency: - # For pull requests, cancel all currently-running jobs for this workflow - # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - # Force terminal colors. @see https://www.npmjs.com/package/colors - FORCE_COLOR: 1 - DEBUG: pw:mcp:error - -jobs: - test_mcp: - name: ${{ matrix.os }} - ${{ matrix.project }} - # Used for authentication of flakiness upload - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - project: [chrome, chromium, firefox, webkit] - include: - - os: windows-latest - project: chrome - - os: windows-latest - project: chromium - - os: windows-latest - project: firefox - - os: windows-latest - project: webkit - - os: windows-latest - project: msedge - runs-on: ${{ matrix.os }} - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - node-version: "22" - command: npm run test-mcp -- --project=${{ matrix.project }} - bot-name: "mcp-${{ matrix.os }}-${{ matrix.project }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml deleted file mode 100644 index c01376f55ab0e..0000000000000 --- a/.github/workflows/tests_primary.yml +++ /dev/null @@ -1,261 +0,0 @@ -name: "tests 1" - -on: - push: - branches: - - main - - release-* - pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - - 'packages/extension/**' - - 'packages/playwright-core/src/server/bidi/**' - - 'packages/playwright-core/src/tools/**' - - 'tests/bidi/**' - - 'tests/extension/**' - - 'tests/mcp/**' - branches: - - main - - release-* - -concurrency: - # For pull requests, cancel all currently-running jobs for this workflow - # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - # Force terminal colors. @see https://www.npmjs.com/package/colors - FORCE_COLOR: 1 - -jobs: - test_linux: - name: ${{ matrix.os }} (${{ matrix.browser }} - Node.js ${{ matrix.node-version }}) - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - browser: [chromium, firefox, webkit] - os: [ubuntu-22.04] - node-version: [20] - include: - - os: ubuntu-22.04 - node-version: 22 - browser: chromium - - os: ubuntu-22.04 - node-version: 24 - browser: chromium - - os: ubuntu-22.04-arm - node-version: 20 - browser: chromium - runs-on: ${{ matrix.os }} - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - node-version: ${{ matrix.node-version }} - browsers-to-install: ${{ matrix.browser }} chromium - command: npm run test -- --project=${{ matrix.browser }}-* - bot-name: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - test_test_runner: - name: Test Runner - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - node-version: [22] - shardIndex: [1, 2] - shardTotal: [2] - shardWeights: ['58:42'] - include: - - os: windows-latest - node-version: 22 - shardIndex: 1 - shardTotal: 3 - shardWeights: '44:33:23' - - os: windows-latest - node-version: 22 - shardIndex: 2 - shardTotal: 3 - shardWeights: '44:33:23' - - os: windows-latest - node-version: 22 - shardIndex: 3 - shardTotal: 3 - shardWeights: '44:33:23' - - os: ubuntu-latest - node-version: 20 - shardIndex: 1 - shardTotal: 2 - shardWeights: '58:42' - - os: ubuntu-latest - node-version: 20 - shardIndex: 2 - shardTotal: 2 - shardWeights: '58:42' - - os: ubuntu-latest - node-version: 24 - shardIndex: 1 - shardTotal: 2 - shardWeights: '58:42' - - os: ubuntu-latest - node-version: 24 - shardIndex: 2 - shardTotal: 2 - shardWeights: '58:42' - - os: ubuntu-latest - node-version: 26 - shardIndex: 1 - shardTotal: 2 - shardWeights: '58:42' - - os: ubuntu-latest - node-version: 26 - shardIndex: 2 - shardTotal: 2 - shardWeights: '58:42' - runs-on: ${{ matrix.os }} - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - node-version: ${{matrix.node-version}} - command: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} - bot-name: "${{ matrix.os }}-node${{ matrix.node-version }}" - shard-index: ${{ matrix.shardIndex }} - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_SHARD_WEIGHTS: ${{ matrix.shardWeights }} - - test_web_components: - name: Web Components - ${{ matrix.package }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - strategy: - fail-fast: false - matrix: - package: [html-reporter, web] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - node-version: 20 - browsers-to-install: chromium - command: npm run test-${{ matrix.package }} - bot-name: "web-components-${{ matrix.package }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - test_vscode_extension: - name: VSCode Extension - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 20 - - run: npm ci - env: - DEBUG: pw:install - - run: npm run build - - run: npx playwright install chromium - - name: Checkout extension - run: git clone https://github.com/microsoft/playwright-vscode.git - - name: Print extension revision - run: git rev-parse HEAD - working-directory: ./playwright-vscode - - name: Remove @playwright/test from extension dependencies - run: node -e "const p = require('./package.json'); delete p.devDependencies['@playwright/test']; fs.writeFileSync('./package.json', JSON.stringify(p, null, 2));" - working-directory: ./playwright-vscode - - name: Build extension - run: npm ci && npm run build - working-directory: ./playwright-vscode - - name: Run extension tests - run: npm run test - working-directory: ./playwright-vscode - env: - PW_TAG: "@vscode-extension" - - name: Upload blob report - if: ${{ !cancelled() && (github.event_name == 'pull_request' || failure()) }} - uses: ./.github/actions/upload-blob-report - with: - report_dir: playwright-vscode/blob-report - job_name: vscode-extension - - test_package_installations: - name: "Installation Test ${{ matrix.os }}" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - - macos-latest - - windows-latest - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - run: npm install -g yarn@1 - - run: npm install -g pnpm@8 - - name: Setup Ubuntu Binary Installation # TODO: Remove when https://github.com/electron/electron/issues/42510 is fixed - if: ${{ runner.os == 'Linux' }} - run: | - if grep -q "Ubuntu 24" /etc/os-release; then - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - fi - shell: bash - - uses: ./.github/actions/run-test - with: - command: npm run itest - node-version: 22 - bot-name: "package-installations-${{ matrix.os }}" - shell: ${{ matrix.os == 'windows-latest' && 'pwsh' || 'bash' }} - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - test_clock_frozen_time_linux: - name: time library - ${{ matrix.clock }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - strategy: - fail-fast: false - matrix: - clock: [frozen, realtime] - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - node-version: 22 - browsers-to-install: chromium - command: npm run test -- --project=chromium-* - bot-name: "${{ matrix.clock }}-time-library-chromium-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PW_CLOCK: ${{ matrix.clock }} diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml deleted file mode 100644 index 5622c1bb06b35..0000000000000 --- a/.github/workflows/tests_secondary.yml +++ /dev/null @@ -1,274 +0,0 @@ -name: "tests 2" - -on: - push: - branches: - - main - - release-* - pull_request: - types: [ labeled ] - branches: - - main - - release-* - -env: - # Force terminal colors. @see https://www.npmjs.com/package/colors - FORCE_COLOR: 1 - -permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - -jobs: - test_mac: - name: ${{ matrix.os }} (${{ matrix.browser }}) - if: ${{ github.event_name == 'push' || github.event.label.name == 'CQ1' }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - # Intel: *-large - # Arm64: *-xlarge - os: [macos-15-large, macos-15-xlarge] - browser: [chromium, firefox, webkit] - include: - - os: macos-14-xlarge - browser: chromium - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - browsers-to-install: ${{ matrix.browser }} chromium - command: npm run test -- --project=${{ matrix.browser }}-* - bot-name: "${{ matrix.browser }}-${{ matrix.os }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - test_win: - name: "Windows" - if: ${{ github.event_name == 'push' || github.event.label.name == 'CQ1' }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - browser: [chromium, firefox, webkit] - runs-on: windows-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - browsers-to-install: ${{ matrix.browser }} chromium - command: npm run test -- --project=${{ matrix.browser }}-* ${{ matrix.browser == 'firefox' && '--workers 1' || '' }} - bot-name: "${{ matrix.browser }}-windows-latest" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - test-package-installations-other-node-versions: - name: "Installation Test ${{ matrix.os }} (${{ matrix.node_version }})" - if: ${{ github.event_name == 'push' || github.event.label.name == 'CQ1' }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - node_version: 20 - - os: ubuntu-latest - node_version: 24 - - os: ubuntu-latest - node_version: 26 - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - run: npm install -g yarn@1 - - run: npm install -g pnpm@8 - - name: Setup Ubuntu Binary Installation # TODO: Remove when https://github.com/electron/electron/issues/42510 is fixed - if: ${{ runner.os == 'Linux' }} - run: | - if grep -q "Ubuntu 24" /etc/os-release; then - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - fi - shell: bash - - uses: ./.github/actions/run-test - with: - node-version: ${{ matrix.node_version }} - command: npm run itest - bot-name: "package-installations-${{ matrix.os }}-node${{ matrix.node_version }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - driver_linux: - name: "Driver" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chromium - command: npm run ctest - bot-name: "driver" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_MODE: driver - - tracing_linux: - name: Tracing ${{ matrix.browser }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - include: - - browser: chromium - runs-on: ubuntu-22.04 - - browser: firefox - runs-on: ubuntu-22.04 - # See https://github.com/microsoft/playwright/issues/35586 - - browser: webkit - runs-on: ubuntu-24.04 - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - browsers-to-install: ${{ matrix.browser }} chromium - command: npm run test -- --project=${{ matrix.browser }}-* - bot-name: "tracing-${{ matrix.browser }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_TRACE: 1 - - test_chromium_channels: - name: Test ${{ matrix.channel }} on ${{ matrix.runs-on }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ${{ matrix.runs-on }} - strategy: - fail-fast: false - matrix: - include: - - channel: chrome - runs-on: ubuntu-22.04 - - channel: chrome - runs-on: macos-latest - - channel: chrome - runs-on: windows-latest - - channel: chrome-beta - runs-on: ubuntu-22.04 - - channel: msedge - runs-on: windows-latest - - channel: msedge-dev - runs-on: windows-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - browsers-to-install: ${{ matrix.channel }} - command: npm run ctest - bot-name: ${{ matrix.channel }}-${{ matrix.runs-on }} - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: ${{ matrix.channel }} - - test_android: - name: Android - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: playwright-x64-ubuntu24-64-core - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 - with: - distribution: 'temurin' - java-version: '21' - - name: Enable KVM group perms - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 - - name: Create Android Emulator - run: utils/avd_recreate.sh - - name: Start Android Emulator - run: utils/avd_start.sh - - uses: ./.github/actions/run-test - with: - browsers-to-install: android - command: npm run atest - bot-name: "android" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - test_clock_frozen_time_test_runner: - name: time test runner - ${{ matrix.clock }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-22.04 - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - strategy: - fail-fast: false - matrix: - clock: [frozen, realtime] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/run-test - with: - node-version: 22 - command: npm run ttest - bot-name: "${{ matrix.clock }}-time-runner-chromium-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PW_CLOCK: ${{ matrix.clock }} - - test_electron: - name: Electron - ${{ matrix.os }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Setup Ubuntu Binary Installation # TODO: Remove when https://github.com/electron/electron/issues/42510 is fixed - if: ${{ runner.os == 'Linux' }} - run: | - if grep -q "Ubuntu 24" /etc/os-release; then - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - fi - shell: bash - # Prevent stalling when macOS decides to show the reopen windows prompt - # after killing enough Electron processes. See https://github.com/electron/electron/pull/51488. - - name: Turn off the reopen windows prompt on macOS - if: ${{ matrix.os == 'macos-latest' }} - run: defaults write com.github.Electron ApplePersistenceIgnoreState YES - shell: bash - - uses: ./.github/actions/run-test - with: - browsers-to-install: chromium - setup-command: npx install-electron --no - command: npm run etest - bot-name: "electron-${{ matrix.os }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} diff --git a/.github/workflows/tests_webview_simulator.yml b/.github/workflows/tests_webview_simulator.yml deleted file mode 100644 index 681594c4d2f8c..0000000000000 --- a/.github/workflows/tests_webview_simulator.yml +++ /dev/null @@ -1,166 +0,0 @@ -name: "tests WebView (iOS Simulator)" - -on: - workflow_dispatch: - pull_request: - paths: - - 'tests/webview/**' - - 'packages/playwright-core/src/server/webkit/webview/**' - - '.github/workflows/tests_webview_simulator.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - FORCE_COLOR: 1 - -jobs: - test_webview_simulator: - name: "WebView on iOS Simulator (${{ matrix.shard }}/4)" - runs-on: macos-15 - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 20 - - - name: Runner environment - run: | - echo "::group::OS / Xcode" - sw_vers - uname -a - xcode-select -p - xcodebuild -version - echo "::endgroup::" - echo "::group::Available iOS runtimes" - xcrun simctl list runtimes - echo "::endgroup::" - echo "::group::Available device types" - xcrun simctl list devicetypes | grep -i 'iPhone\|iPad' | head -40 - echo "::endgroup::" - echo "::group::Network config" - cat /etc/hosts - ifconfig lo0 - echo "::endgroup::" - - - name: Ensure ::1 localhost in /etc/hosts - run: | - if grep -qE '^::1[[:space:]]+localhost' /etc/hosts; then - echo "::1 localhost already present" - else - echo "::1 localhost" | sudo tee -a /etc/hosts - echo "Added ::1 localhost" - fi - echo "--- /etc/hosts after ---" - cat /etc/hosts - - - name: npm ci - run: | - echo "::group::npm ci" - npm ci - echo "::endgroup::" - - - name: npm run build - run: | - echo "::group::npm run build" - npm run build - echo "::endgroup::" - - - name: Install ios-webkit-debug-proxy - run: | - echo "::group::brew install ios-webkit-debug-proxy" - brew install ios-webkit-debug-proxy - which ios_webkit_debug_proxy - ios_webkit_debug_proxy --help 2>&1 | head -40 || true - echo "::endgroup::" - - - name: Boot iOS Simulator - uses: futureware-tech/simulator-action@e89aa8f93d3aec35083ff49d2854d07f7186f7f5 # v5 - with: - # Per wiki/Devices-macos-15.md only iPhone 16/17 series ship pre-installed; iPhone 15 isn't. - model: 'iPhone 16' - os_version: '18.6' - wait_for_boot: true - boot_timeout_seconds: 300 - - - name: Simulator state after boot - run: | - echo "::group::Booted devices" - xcrun simctl list devices booted - echo "::endgroup::" - echo "::group::Simulator processes" - pgrep -lf Simulator || true - pgrep -lf launchd_sim || true - echo "::endgroup::" - - - name: Locate simulator webinspectord socket - run: | - echo "::group::Locating com.apple.webinspectord_sim.socket" - # On modern macOS, ios_webkit_debug_proxy can no longer auto-discover the simulator; - # we have to point -s at the launchd-owned unix socket. - for i in $(seq 1 15); do - SOCK=$(lsof -aUc launchd_sim 2>/dev/null | awk '/com\.apple\.webinspectord_sim\.socket/{print $NF; exit}') - [[ -n "$SOCK" ]] && break - echo "attempt $i: socket not found yet" - sleep 1 - done - if [[ -z "$SOCK" ]]; then - echo "Failed to locate webinspectord_sim.socket" - echo "--- launchd_sim file table ---" - lsof -aUc launchd_sim 2>/dev/null || true - exit 1 - fi - echo "socket: $SOCK" - echo "SIM_WI_SOCKET=unix:$SOCK" >> $GITHUB_ENV - echo "::endgroup::" - - - name: Start ios-webkit-debug-proxy - run: | - echo "::group::Starting proxy (SIM_WI_SOCKET=$SIM_WI_SOCKET)" - ios_webkit_debug_proxy -F -d -s "$SIM_WI_SOCKET" -c "null:9221,:9222-9322" > "$RUNNER_TEMP/iwdp.log" 2>&1 & - PID=$! - echo "IWDP_PID=$PID" >> $GITHUB_ENV - echo "proxy pid=$PID" - sleep 3 - if ! kill -0 "$PID" 2>/dev/null; then - echo "Proxy died immediately. Log:" - cat "$RUNNER_TEMP/iwdp.log" - exit 1 - fi - echo "::endgroup::" - echo "::group::Listening ports" - lsof -nP -iTCP -sTCP:LISTEN | grep -E "9221|9222|ios_webkit" || true - echo "::endgroup::" - - - name: Run WebView tests - run: | - echo "::group::Test run (shard ${{ matrix.shard }}/4)" - npx playwright test --config tests/webview/playwright.config.ts --shard=${{ matrix.shard }}/4 - echo "::endgroup::" - - - name: Stop proxy - if: always() - run: | - [[ -n "$IWDP_PID" ]] && kill "$IWDP_PID" 2>/dev/null || true - # Simulator shutdown is owned by futureware-tech/simulator-action's post step. - - - name: Upload artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: webview-simulator-logs-${{ matrix.shard }} - path: | - ${{ github.workspace }}/test-results/** - if-no-files-found: ignore - - - name: Upload parquet report - if: ${{ !cancelled() }} - uses: ./.github/actions/upload-parquet-report - with: - job_name: webview-simulator-${{ matrix.shard }} diff --git a/.github/workflows/webkit_gc_repro.yml b/.github/workflows/webkit_gc_repro.yml new file mode 100644 index 0000000000000..00b1aeed67aee --- /dev/null +++ b/.github/workflows/webkit_gc_repro.yml @@ -0,0 +1,32 @@ +name: "webkit gc repro" + +# Diagnostic branch only. DO NOT MERGE. +# +# Every other workflow is removed on this branch so a draft PR runs nothing but the WebKit leak +# matrix below, on the one bot that actually reproduces the regression (macos-15-xlarge, arm64). +# pull_request only: a push to a personal fork cannot schedule a larger runner. + +on: + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + webkit_leaks: + name: webkit-page leaks (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: [macos-15-xlarge] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/run-test + with: + browsers-to-install: webkit chromium + command: npx playwright test --config=tests/library/playwright.config.ts tests/page/webkit-gc-repro.spec.ts tests/page/page-leaks.spec.ts --project=webkit-page --repeat-each=10 --retries=0 --reporter=line + bot-name: "webkit-gc-repro-${{ matrix.os }}" diff --git a/tests/page/webkit-gc-repro.spec.ts b/tests/page/webkit-gc-repro.spec.ts new file mode 100644 index 0000000000000..1cd954ea88d10 --- /dev/null +++ b/tests/page/webkit-gc-repro.spec.ts @@ -0,0 +1,158 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Diagnostic for the WebKit r2359 GC regression. DO NOT MERGE. +// +// `page-leaks.spec.ts` › `expect should not leak` fails ~70% of runs on webkit-macos-15-xlarge +// since the r2359 roll, while the click/fill/waitFor tests next to it are 100% green. The first +// run of this matrix showed that asymmetry is not about `expect` at all: the `waitFor` variant +// below failed 10/10 while page-leaks' own `waitFor` test passed 10/10, on the same bot, in the +// same job. +// +// The two tests differ in one thing. The `expect` test registers WeakRefs *before* the loop as +// well as after; click/fill/waitFor only register after. Every variant here copies the `expect` +// test's shape, so `registerBefore` reproduces it and the `-no-pre-refs` variants drop it. If +// those two stay green while their twins fail, the trigger is pre-loop registration plus a +// locator visibility check, not the assertion path. +// +// The raw-DOM variants run in the main world; the locator variants go through the injected +// script in the utility world. + +import { server as coreServer } from '../../packages/playwright-core/lib/coreBundle'; +const { nullProgress } = coreServer; +import { test, expect } from './pageTest'; + +const COUNT = 25; + +async function weakRefObjects(pageImpl: any, selector: string) { + for (const world of ['main', 'utility']) { + const context = await pageImpl.mainFrame().context(world); + await context.evaluate(selector => { + const elements = document.querySelectorAll(selector); + globalThis.weakRefs = globalThis.weakRefs || []; + for (const element of elements) + globalThis.weakRefs.push(new WeakRef(element)); + }, selector); + } +} + +async function weakRefCount(pageImpl: any): Promise<{ main: number, utility: number }> { + const result = { main: 0, utility: 0 }; + for (const world of ['main', 'utility']) { + await pageImpl.requestGC(nullProgress); + const context = await pageImpl.mainFrame().context(world); + result[world] = await context.evaluate(() => globalThis.weakRefs.filter(r => !!r.deref()).length); + } + return result; +} + +// Same bounds as page-leaks.spec.ts, and they hold either way: the static buttons leave 8 alive +// with pre-loop registration and 4 without. The failure is wholesale - every ref stays alive, 58 +// or 54 - and it is sticky, so a short toPass window is enough and keeps a failing run from +// burning the whole test timeout. +async function checkWeakRefs(pageImpl: any, from: number, to: number) { + await expect(async () => { + const counts = await weakRefCount(pageImpl); + expect(counts.main + counts.utility).toBeGreaterThanOrEqual(from); + expect(counts.main + counts.utility).toBeLessThan(to); + }).toPass({ timeout: 5000 }); +} + +type Visit = 'expect' | 'waitFor' | 'click' | 'dom-style' | 'dom-event' | 'dom-both' | 'none'; + +const kVariants: { name: string, registerBefore: boolean, visit: Visit }[] = [ + { name: 'expect', registerBefore: true, visit: 'expect' }, + { name: 'waitFor', registerBefore: true, visit: 'waitFor' }, + { name: 'click', registerBefore: true, visit: 'click' }, + { name: 'dom-style', registerBefore: true, visit: 'dom-style' }, + { name: 'dom-event', registerBefore: true, visit: 'dom-event' }, + { name: 'dom-both', registerBefore: true, visit: 'dom-both' }, + { name: 'none', registerBefore: true, visit: 'none' }, + // Identical, minus the pre-loop registration - i.e. exactly how page-leaks.spec.ts shapes the + // click/fill/waitFor tests that stay green. + { name: 'expect-no-pre-refs', registerBefore: false, visit: 'expect' }, + { name: 'waitFor-no-pre-refs', registerBefore: false, visit: 'waitFor' }, +]; + +for (const variant of kVariants) { + test(`${variant.name} should not leak`, async ({ page, mode, toImpl }) => { + test.skip(mode !== 'default'); + + await page.setContent(` + + +
+ `); + if (variant.registerBefore) + await weakRefObjects(toImpl(page), 'button'); + + const last = () => page.locator('#buttons > button').last(); + const visit = async () => { + switch (variant.visit) { + case 'expect': + await expect(last()).toBeVisible(); + break; + case 'waitFor': + await last().waitFor(); + break; + case 'click': + await last().click(); + break; + case 'dom-style': + await page.evaluate(() => { + const element = document.querySelector('#buttons > button:last-child')!; + const display = getComputedStyle(element).display; + element.getBoundingClientRect(); + return display; + }); + break; + case 'dom-event': + await page.evaluate(() => { + const element = document.querySelector('#buttons > button:last-child')!; + element.dispatchEvent(new CustomEvent('__mark__', { bubbles: true, cancelable: true, composed: true })); + }); + break; + case 'dom-both': + await page.evaluate(() => { + const element = document.querySelector('#buttons > button:last-child')!; + const display = getComputedStyle(element).display; + element.getBoundingClientRect(); + element.dispatchEvent(new CustomEvent('__mark__', { bubbles: true, cancelable: true, composed: true })); + return display; + }); + break; + case 'none': + break; + } + }; + + for (let i = 0; i < COUNT; ++i) { + await page.evaluate(i => { + const element = document.createElement('button'); + element.textContent = 'dynamic ' + i; + document.getElementById('buttons').appendChild(element); + }, i); + await visit(); + } + + await weakRefObjects(toImpl(page), 'button'); + await page.evaluate(() => { + document.getElementById('buttons').textContent = ''; + }); + + await checkWeakRefs(toImpl(page), 2, COUNT); + }); +}