diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a8373c..0d7dc7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: verify: name: Verify commercial application runs-on: ubuntu-24.04 - timeout-minutes: 25 + timeout-minutes: 45 env: REPO_DOCS_CHANNEL: ${{ github.event_name == 'pull_request' && 'preview' || 'stable' }} steps: diff --git a/.github/workflows/open-source.yml b/.github/workflows/open-source.yml index 6a65c8e..cb133fb 100644 --- a/.github/workflows/open-source.yml +++ b/.github/workflows/open-source.yml @@ -51,7 +51,7 @@ jobs: verify: name: Verify open-source application runs-on: ubuntu-24.04 - timeout-minutes: 25 + timeout-minutes: 45 env: REPO_DOCS_CHANNEL: ${{ github.event_name == 'pull_request' && 'preview' || 'stable' }} steps: diff --git a/.github/workflows/recover-pages-0.4.0.yml b/.github/workflows/recover-pages-0.4.0.yml new file mode 100644 index 0000000..0070f76 --- /dev/null +++ b/.github/workflows/recover-pages-0.4.0.yml @@ -0,0 +1,67 @@ +name: Recover the verified 0.4.0 Pages artifact + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-open-source-stable + cancel-in-progress: false + +jobs: + recover: + if: ${{ github.repository == 'weappjs/weapp.dev' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: docs-open-source-stable + permissions: + contents: write + actions: read + pages: write + id-token: write + env: + LEFTHOOK: '0' + REPO_DOCS_CONCURRENCY_GROUP: docs-open-source-stable + steps: + - name: Checkout recovery controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Checkout the frozen release adapter + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: 0b26eea7da1b67f66d31b639e5bd11087b5295d6 + path: .recovery/source + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + package_json_file: .recovery/source/package.json + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24.22.1 + - name: Install frozen adapter dependencies without build scripts + working-directory: .recovery/source + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Recover only the verified original tarball + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + github-token: ${{ github.token }} + script: | + const { spawnSync } = require('node:child_process') + const result = spawnSync('node', ['scripts/recover-pages-0.4.0.mjs'], { + stdio: 'inherit', env: process.env, + }) + if (result.error || result.status !== 0) throw new Error('Pages artifact recovery stopped; inspect sanitized diagnostics.') + - name: Preserve recovery evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: pages-recovery-${{ github.run_id }}-${{ github.run_attempt }} + path: .cache/pages-recovery + include-hidden-files: true + if-no-files-found: ignore + retention-days: 30 diff --git a/patches/@icebreakers__monorepo@6.0.1.patch b/patches/@icebreakers__monorepo@6.0.1.patch new file mode 100644 index 0000000..a04d1dc --- /dev/null +++ b/patches/@icebreakers__monorepo@6.0.1.patch @@ -0,0 +1,15 @@ +diff --git a/dist/commands-D1KtdxnV.mjs b/dist/commands-D1KtdxnV.mjs +index 79b651dba83031ed6f414bbc9d7afe77d49ab0a1..236e2472bfe1fe1f229904c889ee3942e5c49390 100644 +--- a/dist/commands-D1KtdxnV.mjs ++++ b/dist/commands-D1KtdxnV.mjs +@@ -5599,7 +5599,9 @@ function readAutomationState(request, key) { + } + function transient(error) { + if (!error || typeof error !== "object" || !("status" in error) || typeof error.status !== "number") return false; +- return error.status === 0 || error.status === 429 || error.status >= 500; ++ // Contents API can conflict when another file advances the shared branch. ++ // writeState only retries after rereading the unchanged target revision. ++ return error.status === 0 || error.status === 409 || error.status === 429 || error.status >= 500; + } + async function writeState(request, endpoint, state, revision, options = {}) { + if (!revision) try { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a5020c..6c7f3fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -166,6 +166,7 @@ overrides: tinypool@2.1.0: 2.1.2 patchedDependencies: + '@icebreakers/monorepo@6.0.1': 73737dde71d62987ce78a00dff0fc630cd8801c6cf2cebe5fabb20f42e1fbd80 '@weapp-tailwindcss/source-scan@0.1.2': bbed7bd62dea5a2138fd00bdc9a76b830ff777529efb3f10c0ebaa08e73e3a3f importers: @@ -9264,7 +9265,7 @@ snapshots: transitivePeerDependencies: - '@types/node' - '@icebreakers/monorepo@6.0.1(@commitlint/cli@21.2.4(conventional-commits-parser@7.1.3)(typescript@6.0.3))(@pnpm/logger@1001.0.1)(@vitest/coverage-v8@5.0.5)(conventional-changelog-conventionalcommits@10.4.1)(eslint-plugin-pnpm@1.9.1(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2)))(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2))(magicast@0.5.5)(postcss@8.5.31)(stylelint-config-html@2.0.0(postcss-html@2.0.0(postcss@8.5.31))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3)))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3))(supports-color@10.2.2)(tailwindcss@4.3.3)(typescript@6.0.3)(vite@8.3.5(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))': + '@icebreakers/monorepo@6.0.1(patch_hash=73737dde71d62987ce78a00dff0fc630cd8801c6cf2cebe5fabb20f42e1fbd80)(@commitlint/cli@21.2.4(conventional-commits-parser@7.1.3)(typescript@6.0.3))(@pnpm/logger@1001.0.1)(@vitest/coverage-v8@5.0.5)(conventional-changelog-conventionalcommits@10.4.1)(eslint-plugin-pnpm@1.9.1(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2)))(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2))(magicast@0.5.5)(postcss@8.5.31)(stylelint-config-html@2.0.0(postcss-html@2.0.0(postcss@8.5.31))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3)))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3))(supports-color@10.2.2)(tailwindcss@4.3.3)(typescript@6.0.3)(vite@8.3.5(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))': dependencies: '@arethetypeswrong/core': 0.18.5 '@babel/parser': 7.29.9 @@ -14287,7 +14288,7 @@ snapshots: repoctl@6.0.1(@commitlint/cli@21.2.4(conventional-commits-parser@7.1.3)(typescript@6.0.3))(@pnpm/logger@1001.0.1)(@vitest/coverage-v8@5.0.5)(conventional-changelog-conventionalcommits@10.4.1)(eslint-plugin-pnpm@1.9.1(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2)))(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2))(magicast@0.5.5)(postcss@8.5.31)(stylelint-config-html@2.0.0(postcss-html@2.0.0(postcss@8.5.31))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3)))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3))(supports-color@10.2.2)(tailwindcss@4.3.3)(typescript@6.0.3)(vite@8.3.5(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1))(vitest@5.0.5): dependencies: - '@icebreakers/monorepo': 6.0.1(@commitlint/cli@21.2.4(conventional-commits-parser@7.1.3)(typescript@6.0.3))(@pnpm/logger@1001.0.1)(@vitest/coverage-v8@5.0.5)(conventional-changelog-conventionalcommits@10.4.1)(eslint-plugin-pnpm@1.9.1(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2)))(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2))(magicast@0.5.5)(postcss@8.5.31)(stylelint-config-html@2.0.0(postcss-html@2.0.0(postcss@8.5.31))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3)))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3))(supports-color@10.2.2)(tailwindcss@4.3.3)(typescript@6.0.3)(vite@8.3.5(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)) + '@icebreakers/monorepo': 6.0.1(patch_hash=73737dde71d62987ce78a00dff0fc630cd8801c6cf2cebe5fabb20f42e1fbd80)(@commitlint/cli@21.2.4(conventional-commits-parser@7.1.3)(typescript@6.0.3))(@pnpm/logger@1001.0.1)(@vitest/coverage-v8@5.0.5)(conventional-changelog-conventionalcommits@10.4.1)(eslint-plugin-pnpm@1.9.1(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2)))(eslint@10.13.0(jiti@2.7.0)(supports-color@10.2.2))(magicast@0.5.5)(postcss@8.5.31)(stylelint-config-html@2.0.0(postcss-html@2.0.0(postcss@8.5.31))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3)))(stylelint@17.16.0(supports-color@10.2.2)(typescript@6.0.3))(supports-color@10.2.2)(tailwindcss@4.3.3)(typescript@6.0.3)(vite@8.3.5(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)) optionalDependencies: '@commitlint/cli': 21.2.4(conventional-commits-parser@7.1.3)(typescript@6.0.3) eslint: 10.13.0(jiti@2.7.0)(supports-color@10.2.2) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b608b1e..ee08e56 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -20,6 +20,8 @@ overrides: # Remove when oxfmt no longer pins the vulnerable tinypool 2.1.0. tinypool@2.1.0: 2.1.2 patchedDependencies: + # Remove when upstream retries branch conflicts after verifying the target state is unchanged. + '@icebreakers/monorepo@6.0.1': patches/@icebreakers__monorepo@6.0.1.patch # Remove after upstream includes Astro templates in source scanning and HMR. '@weapp-tailwindcss/source-scan@0.1.2': patches/@weapp-tailwindcss__source-scan@0.1.2.patch minimumReleaseAgeExclude: diff --git a/scripts/docs-deployment/cloudflare.mjs b/scripts/docs-deployment/cloudflare.mjs index c73ac83..e7ae341 100644 --- a/scripts/docs-deployment/cloudflare.mjs +++ b/scripts/docs-deployment/cloudflare.mjs @@ -1,7 +1,7 @@ import { mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' -import { run, verifyArtifact, writeResult } from './context.mjs' +import { pause, run, verifyArtifact, writeResult } from './context.mjs' export const deploymentMessage = identity => `repoctl-docs:${identity.deploymentId}` @@ -99,6 +99,25 @@ async function workerUrl(context, client) { return `https://${context.worker}.${result.subdomain}.workers.dev` } +async function confirmActiveVersion(client, worker, versionId, expected, options) { + const limit = options.pollLimit ?? 30 + for (let attempt = 0; attempt < limit; attempt++) { + const active = await activeVersion(client, worker) + if (active?.id === versionId) { + if (active.message !== expected) { + throw new Error('The uploaded Cloudflare version has an unexpected deployment identity.') + } + return active + } + // The deploy command can finish before the read API reflects activation. + // Only observe the known upload; a delayed read must never trigger a write. + if (attempt + 1 < limit) { + await (options.sleep ?? pause)(2000) + } + } + throw new Error('Cloudflare did not confirm this exact version at 100% traffic; retry the same deployment.') +} + export async function deployCloudflare(context, options = {}) { if (!context.worker) { throw new Error('This site channel does not use Cloudflare.') @@ -111,10 +130,7 @@ export async function deployCloudflare(context, options = {}) { if (active?.message !== expected) { const output = await invokeWrangler(context, ['deploy'], options.run ?? run) const uploaded = readWranglerOutput(output, context.worker, 'deploy') - active = await activeVersion(client, context.worker) - if (active?.id !== uploaded.version_id || active.message !== expected) { - throw new Error('Cloudflare did not confirm this exact version at 100% traffic; retry the same deployment.') - } + active = await confirmActiveVersion(client, context.worker, uploaded.version_id, expected, options) } return writeResult(context, { versionId: active.id, url: await workerUrl(context, client) }) } diff --git a/scripts/docs-deployment/deployment.test.mjs b/scripts/docs-deployment/deployment.test.mjs index d264aca..138fb1c 100644 --- a/scripts/docs-deployment/deployment.test.mjs +++ b/scripts/docs-deployment/deployment.test.mjs @@ -6,7 +6,7 @@ import { describe, expect, it, vi } from 'vitest' import { activeVersion, cloudflareClient, deployCloudflare, deploymentMessage, readWranglerOutput, uploadPrPreview, verifyWorkerTarget } from './cloudflare.mjs' import { assertIdentity, managedContext, prContext, writeBuildIdentity } from './context.mjs' import { sanitizeMessage, writeDiagnostic } from './diagnostics.mjs' -import { pagesReceiptStore } from './github.mjs' +import { githubClient, pagesReceiptStore } from './github.mjs' import { deployPages, waitForPages } from './pages.mjs' const sha = 'a'.repeat(40) @@ -140,20 +140,72 @@ describe('Cloudflare provider', () => { ? { deployments: [{ created_on: '2026-10-23', versions: [{ version_id: version, percentage: 100 }] }] } : { id: version, annotations: { 'workers/message': deploymentMessage(context.identity) } }) const execute = vi.fn() - expect(await deployCloudflare(context, { client, run: execute })).toEqual({ versionId: version, url: 'https://weapp.dev' }) + const sleep = vi.fn() + expect(await deployCloudflare(context, { client, run: execute, sleep })).toEqual({ versionId: version, url: 'https://weapp.dev' }) expect(execute).not.toHaveBeenCalled() + expect(sleep).not.toHaveBeenCalled() expect(JSON.parse(await readFile(context.env.REPO_DOCS_RESULT_FILE, 'utf8')).versionId).toBe(version) })) it('does not record an uploaded version until the API confirms it active', async () => fixture('web', async (context) => { - const client = async endpoint => endpoint.endsWith('/deployments') + const client = vi.fn(async endpoint => endpoint.endsWith('/deployments') ? { deployments: [{ created_on: '2026-10-23', versions: [{ version_id: oldVersion, percentage: 100 }] }] } - : { id: oldVersion, annotations: { 'workers/message': 'old' } } + : { id: oldVersion, annotations: { 'workers/message': 'old' } }) const execute = vi.fn(async (_command, args, options) => { expect(args).toContain('deploy') await writeFile(options.env.WRANGLER_OUTPUT_FILE_PATH, JSON.stringify({ type: 'deploy', worker_name: context.worker, version_id: version })) }) - await expect(deployCloudflare(context, { client, run: execute })).rejects.toThrow(/exact version/) + const sleep = vi.fn() + await expect(deployCloudflare(context, { client, run: execute, pollLimit: 3, sleep })).rejects.toThrow(/exact version/) + expect(execute).toHaveBeenCalledOnce() + expect(client.mock.calls.filter(([endpoint]) => endpoint.endsWith('/deployments'))).toHaveLength(4) + expect(sleep.mock.calls).toEqual([[2000], [2000]]) + await expect(readFile(context.env.REPO_DOCS_RESULT_FILE)).rejects.toMatchObject({ code: 'ENOENT' }) + })) + + it('waits for the uploaded version to become active without deploying twice', async () => fixture('web', async (context) => { + let reads = 0 + const client = vi.fn(async (endpoint) => { + if (endpoint.endsWith('/deployments')) { + const id = reads++ < 2 ? oldVersion : version + return { deployments: [{ created_on: '2026-10-23', versions: [{ version_id: id, percentage: 100 }] }] } + } + const id = endpoint.split('/').at(-1) + return { id, annotations: { 'workers/message': id === version ? deploymentMessage(context.identity) : 'old' } } + }) + const execute = vi.fn(async (_command, _args, options) => { + await writeFile(options.env.WRANGLER_OUTPUT_FILE_PATH, JSON.stringify({ type: 'deploy', worker_name: context.worker, version_id: version })) + }) + const sleep = vi.fn() + expect(await deployCloudflare(context, { client, run: execute, pollLimit: 3, sleep })).toEqual({ versionId: version, url: 'https://weapp.dev' }) + expect(execute).toHaveBeenCalledOnce() + expect(reads).toBe(3) + expect(sleep).toHaveBeenCalledExactlyOnceWith(2000) + expect(JSON.parse(await readFile(context.env.REPO_DOCS_RESULT_FILE, 'utf8')).versionId).toBe(version) + })) + + it.each(['split traffic', 'invalid response', 'wrong identity'])('stops confirmation immediately on %s', async failure => fixture('web', async (context) => { + let uploaded = false + const client = vi.fn(async (endpoint) => { + if (endpoint.endsWith('/deployments')) { + if (uploaded && failure === 'invalid response') { + return { deployments: null } + } + const id = uploaded ? version : oldVersion + return { deployments: [{ created_on: '2026-10-23', versions: [{ version_id: id, percentage: uploaded && failure === 'split traffic' ? 50 : 100 }] }] } + } + const id = endpoint.split('/').at(-1) + return { id, annotations: { 'workers/message': 'unexpected identity' } } + }) + const execute = vi.fn(async (_command, _args, options) => { + uploaded = true + await writeFile(options.env.WRANGLER_OUTPUT_FILE_PATH, JSON.stringify({ type: 'deploy', worker_name: context.worker, version_id: version })) + }) + const sleep = vi.fn() + const message = { 'split traffic': /100%/, 'invalid response': /Invalid Cloudflare/, 'wrong identity': /unexpected deployment identity/ }[failure] + await expect(deployCloudflare(context, { client, run: execute, pollLimit: 3, sleep })).rejects.toThrow(message) + expect(execute).toHaveBeenCalledOnce() + expect(sleep).not.toHaveBeenCalled() await expect(readFile(context.env.REPO_DOCS_RESULT_FILE)).rejects.toMatchObject({ code: 'ENOENT' }) })) @@ -225,7 +277,7 @@ describe('Pages provider', () => { const client = vi.fn(async (endpoint, options) => { if (options?.method === 'POST') { expect(store.current().phase).toBe('submitting') - expect(options.body).toEqual({ artifact_id: 42, environment: 'docs-open-source-stable', pages_build_version: identity, oidc_token: 'secret-token' }) + expect(options.body).toEqual({ artifact_id: 42, environment: 'docs-open-source-stable', pages_build_version: context.identity.sourceSha, oidc_token: 'secret-token' }) return { id: 'pages-123', page_url: 'https://weapp.js.org/' } } expect(endpoint).toBe('pages/deployments/pages-123') @@ -371,6 +423,23 @@ describe('Pages provider', () => { }) describe('deployment diagnostics', () => { + it('reports a safe provider explanation and request ID without leaking credentials', async () => { + const client = githubClient({ GITHUB_REPOSITORY: 'weappjs/weapp.dev', GITHUB_TOKEN: 'private-token' }, async () => Response.json({ + message: 'Missing build private-token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature', + debug: 'must-not-be-logged', + }, { status: 404, headers: { 'x-github-request-id': 'AB12:CD34' } })) + await expect(client('pages/deployments', { method: 'POST', body: { oidc_token: 'secret' } })).rejects.toMatchObject({ + status: 404, + method: 'POST', + message: 'GitHub POST pages/deployments failed (HTTP 404). Request ID: AB12:CD34. Missing build [REDACTED] [REDACTED JWT]', + }) + }) + + it('preserves the provider status when its error body is not JSON', async () => { + const client = githubClient({ GITHUB_REPOSITORY: 'weappjs/weapp.dev', GITHUB_TOKEN: 'private-token' }, async () => new Response('gateway', { status: 502 })) + await expect(client('pages/deployments')).rejects.toMatchObject({ status: 502, method: 'GET', message: 'GitHub GET pages/deployments failed (HTTP 502).' }) + }) + it('redacts environment credentials, bearer tokens and runtime JWTs', () => { expect(sanitizeMessage('error cf-secret ghp_abc123 Bearer unknown eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature', { CLOUDFLARE_API_TOKEN: 'cf-secret' })) .toBe('error [REDACTED] [REDACTED TOKEN] Bearer [REDACTED] [REDACTED JWT]') diff --git a/scripts/docs-deployment/github.mjs b/scripts/docs-deployment/github.mjs index 4425267..e31c48e 100644 --- a/scripts/docs-deployment/github.mjs +++ b/scripts/docs-deployment/github.mjs @@ -1,4 +1,5 @@ import { Buffer } from 'node:buffer' +import { sanitizeMessage } from './diagnostics.mjs' const stateBranch = 'repoctl-release-state' @@ -23,7 +24,12 @@ export function githubClient(env, request = fetch) { return undefined } if (!response.ok) { - const error = new Error(`GitHub ${options.method || 'GET'} ${endpoint.split('?')[0]} failed (HTTP ${response.status}).`) + // Keep the provider's selected explanation without persisting request + // payloads, authorization headers, OIDC tokens or arbitrary response data. + const details = await response.json().catch(() => undefined) + const message = typeof details?.message === 'string' ? sanitizeMessage(details.message, env).slice(0, 1000) : '' + const requestId = (response.headers.get('x-github-request-id') || '').replace(/[^a-z\d:-]/gi, '').slice(0, 100) + const error = new Error(`GitHub ${options.method || 'GET'} ${endpoint.split('?')[0]} failed (HTTP ${response.status}).${requestId ? ` Request ID: ${requestId}.` : ''}${message ? ` ${message}` : ''}`) error.status = response.status error.method = options.method || 'GET' throw error diff --git a/scripts/docs-deployment/pages.mjs b/scripts/docs-deployment/pages.mjs index 8bcfb37..2d2ab29 100644 --- a/scripts/docs-deployment/pages.mjs +++ b/scripts/docs-deployment/pages.mjs @@ -143,7 +143,10 @@ export async function deployPages(context, options = {}) { body: { artifact_id: artifactId, environment: 'docs-open-source-stable', - pages_build_version: context.identity.deploymentId, + // Match actions/deploy-pages: Pages resolves this value as a repository + // commit despite the REST schema describing it as a unique string. + // The full deployment identity remains in the receipt and site artifact. + pages_build_version: context.identity.sourceSha, oidc_token: idToken, }, }) diff --git a/scripts/recover-pages-0.4.0.mjs b/scripts/recover-pages-0.4.0.mjs new file mode 100644 index 0000000..916d373 --- /dev/null +++ b/scripts/recover-pages-0.4.0.mjs @@ -0,0 +1,227 @@ +import { strict as assert } from 'node:assert' +import { Buffer } from 'node:buffer' +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import process from 'node:process' +import { pathToFileURL } from 'node:url' + +const repository = 'weappjs/weapp.dev' +const sourceSha = '0b26eea7da1b67f66d31b639e5bd11087b5295d6' +const releaseId = '719206ee91ab8028f980ed8a0981fd72d4ce3c255e611094c1f774d9f3e490c9' +const deploymentId = 'addbffafaaf1fca47b893343905e0dd4f005aaad22e96b2dd65f44c3d01e9ed2' +const artifactId = 12505735843 +const originalRun = 39960105796 +const zipDigest = '35a102604b34007ec658b8decacf6118f02fb6634a989e9f1904bbe766c9eaf1' +const artifactName = 'pages-open-source-addbffafaaf1-39960105796-1-f1a54a1c-3272-497a-831d-d6d74812d095' +const source = path.resolve('.recovery/source') +const evidence = path.resolve('.cache/pages-recovery') +const resultFile = path.join(evidence, 'result.json') +const token = process.env.GITHUB_TOKEN +assert.equal(process.env.GITHUB_ACTIONS, 'true') +assert.equal(process.env.GITHUB_REPOSITORY, repository) +assert.equal(process.env.GITHUB_REF, 'refs/heads/main') +assert.equal(process.env.GITHUB_EVENT_NAME, 'workflow_dispatch') +assert.equal(process.env.REPO_DOCS_CONCURRENCY_GROUP, 'docs-open-source-stable') +assert.ok(token) +assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: source, encoding: 'utf8' }).trim(), sourceSha) +await mkdir(evidence, { recursive: true }) +const moduleAt = filename => import(pathToFileURL(path.join(source, 'scripts/docs-deployment', filename)).href) +const { deployPages } = await moduleAt('pages.mjs') +const { githubClient, pagesReceiptStore } = await moduleAt('github.mjs') +const { managedContext, assertIdentity } = await moduleAt('context.mjs') +const { sanitizeMessage } = await moduleAt('diagnostics.mjs') +const apiEvidence = [] +async function safeFetch(input, options) { + const response = await fetch(input, options) + const url = new URL(input) + if (url.origin === 'https://api.github.com') { + const item = { method: options?.method || 'GET', path: url.pathname, status: response.status, requestId: response.headers.get('x-github-request-id') } + if (!response.ok) { + try { + const body = await response.clone().json() + if (typeof body.message === 'string') { + item.message = sanitizeMessage(body.message).slice(0, 2000) + } + } + catch {} + } + apiEvidence.push(item) + await writeFile(path.join(evidence, 'api.json'), `${JSON.stringify(apiEvidence, null, 2)}\n`) + console.log(JSON.stringify(item)) + } + return response +} +const env = { + ...process.env, + REPO_DOCS_SITE: 'open-source', + REPO_DOCS_CHANNEL: 'stable', + REPO_DOCS_SOURCE_SHA: sourceSha, + REPO_DOCS_RELEASE_ID: releaseId, + REPO_DOCS_DEPLOYMENT_ID: deploymentId, + REPO_DOCS_RESULT_FILE: resultFile, +} +const client = githubClient(env, safeFetch) +async function providerClient(endpoint, options = {}) { + if (options.method === 'POST') { + assert.equal(endpoint, 'pages/deployments') + assert.equal(options.body.pages_build_version, deploymentId) + // Provider compatibility translation only: Pages requires a real source + // commit for build-version lookup. The frozen artifact and full repoctl + // identity remain unchanged in the public file and provider receipt. + return client(endpoint, { ...options, body: { ...options.body, pages_build_version: sourceSha } }) + } + return client(endpoint, options) +} +const context = managedContext(env, source) +const store = pagesReceiptStore(client, deploymentId) +async function readState(filename) { + const file = await client(`contents/${filename}?ref=repoctl-release-state`) + assert.equal(file.encoding, 'base64') + return { revision: file.sha, value: JSON.parse(Buffer.from(file.content, 'base64').toString()) } +} +async function guard() { + const record = (await readState(`.repoctl-automation/releases/${releaseId}.json`)).value + assert.equal(record.id, releaseId) + assert.equal(record.repository, repository) + assert.equal(record.sourceSha, sourceSha) + assert.equal(record.branch, 'main') + assert.equal(record.distTag, 'latest') + assert.deepEqual(record.packages, [{ name: 'weapp', version: '0.4.0', sourceSha }]) + assert.deepEqual(record.releasedPackages, ['weapp']) + const pending = (await readState('.repoctl-automation/docs/open-source/stable.json')).value + assert.equal(pending.repository, repository) + assert.equal(pending.site, 'open-source') + assert.equal(pending.channel, 'stable') + assert.equal(pending.pending?.phase, 'deploying') + assert.equal(pending.pending.deploymentId, deploymentId) + assert.equal(pending.pending.snapshot.sourceSha, sourceSha) + assert.equal(pending.pending.snapshot.releaseId, releaseId) + assert.deepEqual(pending.pending.snapshot.commits, []) + assert.deepEqual(pending.pending.snapshot.requestedCommits, []) + const receipt = await store.read() + assert.ok(receipt.value, 'Existing Pages provider receipt required') + assert.equal(receipt.value.deploymentId, deploymentId) + if (receipt.value.phase === 'failed') { + assert.equal(receipt.value.rejectedStatus, 404, 'Only the explicit initial 404 rejection may create a replacement') + assert.equal(receipt.value.artifactId, artifactId) + assert.equal(receipt.value.runId, String(originalRun)) + assert.equal(receipt.value.pagesId, undefined) + assert.deepEqual(receipt.value.previousAttempts, []) + } + else { + assert.ok(['submitting', 'created', 'complete'].includes(receipt.value.phase)) + // A prior invocation may already have submitted. The frozen adapter will + // reuse its Pages ID or check public identity; it will not POST again. + } + return receipt +} + +const metadata = await client(`actions/artifacts/${artifactId}`) +assert.equal(metadata.id, artifactId) +assert.equal(metadata.name, artifactName) +assert.equal(metadata.expired, false) +assert.equal(metadata.digest, `sha256:${zipDigest}`) +assert.equal(metadata.size_in_bytes, 1774700) +assert.equal(metadata.workflow_run.id, originalRun) +assert.equal(metadata.workflow_run.repository_id, 1346469709) +assert.equal(metadata.workflow_run.head_repository_id, 1346469709) +assert.equal(metadata.workflow_run.head_branch, 'main') +assert.equal(metadata.workflow_run.head_sha, sourceSha) +await guard() +const temporary = await mkdtemp(path.join(tmpdir(), 'weapp-pages-recovery-')) +try { + // Read the signed redirect separately; never send GITHUB_TOKEN to storage. + const archive = await safeFetch(`https://api.github.com/repos/${repository}/actions/artifacts/${artifactId}/zip`, { + headers: { 'Authorization': `Bearer ${token}`, 'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2026-03-10' }, + redirect: 'manual', + signal: AbortSignal.timeout(30000), + }) + assert.equal(archive.status, 302) + const location = new URL(archive.headers.get('location')) + assert.equal(location.protocol, 'https:') + assert.equal(location.username, '') + assert.equal(location.password, '') + const downloaded = await fetch(location, { signal: AbortSignal.timeout(60000) }) + assert.equal(downloaded.status, 200) + const bytes = Buffer.from(await downloaded.arrayBuffer()) + assert.equal(bytes.byteLength, 1774700) + assert.equal(createHash('sha256').update(bytes).digest('hex'), zipDigest) + const archiveFile = path.join(temporary, 'archive.zip') + const tarFile = path.join(temporary, 'artifact.tar') + await writeFile(archiveFile, bytes) + // Validate every member before writing. No links, devices, absolute paths, + // traversal, duplicate paths or archives exceeding the bounded size. + execFileSync('python3', ['-c', ` +import json,pathlib,sys,tarfile,zipfile +archive,tar_path,dist=map(pathlib.Path,sys.argv[1:]) +limit=100*1024*1024 +with zipfile.ZipFile(archive) as z: + assert z.namelist()==['artifact.tar'], 'Unexpected ZIP members' + item=z.getinfo('artifact.tar') + assert item.file_size <= limit and not item.flag_bits & 1, 'Invalid tar entry' + tar_path.write_bytes(z.read(item)) +with tarfile.open(tar_path,mode='r:') as t: + members=t.getmembers() + assert len(members)<=10000 and sum(m.size for m in members)<=limit + paths=set() + for member in members: + p=pathlib.PurePosixPath(member.name) + assert not p.is_absolute() and '..' not in p.parts and '\\\\' not in member.name, 'Unsafe tar path' + assert member.isfile() or member.isdir(), 'Unsupported tar member' + normalized=str(p) + assert normalized not in paths, 'Duplicate tar path' + paths.add(normalized) + assert 'repoctl-deployment.json' in paths and 'CNAME' in paths + dist.mkdir(parents=True,exist_ok=False) + for member in members: + destination=dist/pathlib.PurePosixPath(member.name) + if member.isdir(): destination.mkdir(parents=True,exist_ok=True) + else: + destination.parent.mkdir(parents=True,exist_ok=True) + with t.extractfile(member) as stream: destination.write_bytes(stream.read()) +`, archiveFile, tarFile, context.dist], { stdio: ['ignore', 'pipe', 'pipe'] }) + const identity = JSON.parse(await readFile(path.join(context.dist, 'repoctl-deployment.json'), 'utf8')) + assertIdentity(identity, context.identity) + assert.equal((await readFile(path.join(context.dist, 'CNAME'), 'utf8')).trim(), 'weapp.js.org') + const tarDigest = createHash('sha256').update(await readFile(tarFile)).digest('hex') + await writeFile(path.join(evidence, 'verified-artifact.json'), `${JSON.stringify({ artifactId, originalRun, zipDigest, tarDigest, identity }, null, 2)}\n`) + const receipt = await guard() + const sdkRoot = path.join(source, 'node_modules/@actions/artifact') + const sdkManifest = JSON.parse(await readFile(path.join(sdkRoot, 'package.json'), 'utf8')) + assert.equal(sdkManifest.version, '6.3.1') + const { DefaultArtifactClient } = await import(pathToFileURL(path.join(sdkRoot, 'lib/artifact.js')).href) + const sdk = new DefaultArtifactClient() + let uploadCount = 0 + const artifact = { + async uploadArtifact(name, files, root, options) { + assert.equal(++uploadCount, 1) + assert.equal(receipt.value.phase, 'failed', 'Recovery must not upload for an uncertain existing submission') + assert.equal(files.length, 1) + assert.equal(createHash('sha256').update(await readFile(files[0])).digest('hex'), tarDigest) + const uploaded = await sdk.uploadArtifact(name, files, root, options) + assert.ok(uploaded.id && uploaded.id !== artifactId) + await writeFile(path.join(evidence, 'new-artifact.json'), `${JSON.stringify({ id: uploaded.id, originalArtifactId: artifactId, tarDigest, runId: process.env.GITHUB_RUN_ID })}\n`) + return uploaded + }, + } + const execute = async (command, args) => { + assert.equal(command, 'tar') + const output = args[args.indexOf('-cf') + 1] + assert.ok(output && path.basename(output) === 'artifact.tar') + await copyFile(tarFile, output) + } + const result = await deployPages(context, { client: providerClient, store, artifact, run: execute, fetch: safeFetch }) + await writeFile(path.join(evidence, 'completed.json'), `${JSON.stringify({ result, uploadCount, sourceSha, releaseId, deploymentId })}\n`) + console.log(JSON.stringify({ result, uploadCount, next: 'Resume the same release with the managed repo docs workflow to finalize its state.' })) +} +catch (error) { + await writeFile(path.join(evidence, 'failure.json'), `${JSON.stringify({ message: sanitizeMessage(error.message).slice(0, 4000) })}\n`) + throw new Error(sanitizeMessage(error.message)) +} +finally { + await rm(temporary, { recursive: true, force: true }) + await rm(context.dist, { recursive: true, force: true }) +} diff --git a/scripts/repoctl-state-concurrency.test.mjs b/scripts/repoctl-state-concurrency.test.mjs new file mode 100644 index 0000000..ba94686 --- /dev/null +++ b/scripts/repoctl-state-concurrency.test.mjs @@ -0,0 +1,92 @@ +import { strict as assert } from 'node:assert' +import { Buffer } from 'node:buffer' +import { GitHubClient } from 'repoctl' +import { it } from 'vitest' + +const key = 'docs/open-source/stable' +const originalRevision = 'a'.repeat(40) +const updatedRevision = 'b'.repeat(40) +const original = { pending: { phase: 'building' } } +const requested = { pending: { phase: 'deploying' } } + +// Exercise the installed client's actual HTTP reconciliation. GitHub Contents +// can reject a file update when another file advances the shared state branch. +function fixture(onPut) { + const calls = [] + const delays = [] + const remote = { revision: originalRevision, value: original } + const client = new GitHubClient({ + token: 'fixture-token', + repository: 'weappjs/weapp.dev', + apiUrl: 'https://github.example.invalid', + retryAttempts: 3, + retryDelay: 5, + sleep: async (delay) => { delays.push(delay) }, + fetch: async (input, options) => { + const url = new URL(input) + assert.equal(url.origin, 'https://github.example.invalid') + assert.equal(url.pathname, `/repos/weappjs/weapp.dev/contents/.repoctl-automation/${key}.json`) + calls.push(options.method) + if (options.method === 'GET') { + assert.equal(url.searchParams.get('ref'), 'repoctl-release-state') + return Response.json({ sha: remote.revision, content: Buffer.from(JSON.stringify(remote.value)).toString('base64') }) + } + assert.equal(options.method, 'PUT') + const body = JSON.parse(options.body) + assert.equal(body.branch, 'repoctl-release-state') + assert.equal(body.sha, originalRevision, 'A retry must not overwrite a changed target revision') + assert.deepEqual(JSON.parse(Buffer.from(body.content, 'base64').toString()), requested) + return onPut({ remote, putCount: calls.filter(method => method === 'PUT').length }) + }, + }) + return { client, calls, delays, remote } +} + +const conflict = () => Response.json({ message: 'State branch advanced while updating another file' }, { status: 409 }) + +it('retries a shared-branch conflict after confirming the target file is unchanged', async () => { + const test = fixture(({ remote, putCount }) => { + if (putCount === 1) { + return conflict() + } + remote.revision = updatedRevision + remote.value = requested + return Response.json({ content: { sha: updatedRevision } }) + }) + assert.equal(await test.client.writeAutomationState(key, requested, originalRevision), updatedRevision) + assert.deepEqual(test.calls, ['PUT', 'GET', 'PUT']) + assert.deepEqual(test.delays, [5]) +}) + +it('refuses to retry when a competing writer changed the target file', async () => { + const competing = { current: { deploymentId: 'other-deployment' } } + const test = fixture(({ remote }) => { + remote.revision = updatedRevision + remote.value = competing + return conflict() + }) + await assert.rejects(test.client.writeAutomationState(key, requested, originalRevision), /another publisher advanced it/) + assert.deepEqual(test.calls, ['PUT', 'GET']) + assert.deepEqual(test.delays, []) + assert.equal(test.remote.value, competing) +}) + +it('stops after bounded retries when the shared branch stays busy', async () => { + const test = fixture(conflict) + await assert.rejects(test.client.writeAutomationState(key, requested, originalRevision), /checkpoint write failed/) + assert.deepEqual(test.calls, ['PUT', 'GET', 'PUT', 'GET', 'PUT', 'GET']) + assert.deepEqual(test.delays, [5, 10]) + assert.equal(test.remote.revision, originalRevision) + assert.equal(test.remote.value, original) +}) + +it('recovers an accepted write with a lost response without submitting it twice', async () => { + const test = fixture(({ remote }) => { + remote.revision = updatedRevision + remote.value = requested + throw new TypeError('Response was lost after GitHub accepted the write') + }) + assert.equal(await test.client.writeAutomationState(key, requested, originalRevision), updatedRevision) + assert.deepEqual(test.calls, ['PUT', 'GET']) + assert.deepEqual(test.delays, []) +})