diff --git a/.github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml b/.github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml new file mode 100644 index 000000000..d83753f81 --- /dev/null +++ b/.github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml @@ -0,0 +1,79 @@ +name: OpenCode Coverage Artifact Rerun Quality CI + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/opencode-review-dispatch.yml" + - ".github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml" + - "tests/test_opencode_coverage_artifact_rerun_contract.py" + - "docs/doctoring/opencode-coverage-artifact-reruns.md" + - "CHANGELOG.md" + - "requirements-opencode-review-ci-hashes.txt" + - "pyproject.toml" + push: + branches: [main] + paths: + - ".github/workflows/opencode-review-dispatch.yml" + - ".github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml" + - "tests/test_opencode_coverage_artifact_rerun_contract.py" + - "docs/doctoring/opencode-coverage-artifact-reruns.md" + - "CHANGELOG.md" + - "requirements-opencode-review-ci-hashes.txt" + - "pyproject.toml" + +concurrency: + group: opencode-coverage-artifact-rerun-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + exact-head-contract: + name: Python 3.14 attempt-scoped artifact contract + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Checkout exact source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up current stable Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + cache: pip + cache-dependency-path: requirements-opencode-review-ci-hashes.txt + + - name: Install hash-locked quality tooling + run: python -m pip install --disable-pip-version-check --require-hashes -r requirements-opencode-review-ci-hashes.txt + + - name: Run attempt-scoped artifact regression + run: python -m pytest tests/test_opencode_coverage_artifact_rerun_contract.py -q + + - name: Enforce complete central test and branch coverage + run: | + set -euo pipefail + python -m coverage erase + python -m coverage run -m pytest tests -q + python -m coverage report --show-missing + + - name: Enforce production docstring coverage + run: python -m interrogate scripts/ci + + - name: Compile permanent contracts + run: python -m compileall -q scripts tests + + - name: Reject uncommitted generated state + run: git diff --exit-code --check && test -z "$(git status --porcelain)" diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 83f6830d5..c1e54d8eb 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -222,6 +222,9 @@ jobs: permissions: contents: read id-token: write + outputs: + coverage_source_artifact_id: ${{ steps.coverage_source_upload.outputs.artifact-id }} + coverage_source_run_attempt: ${{ steps.coverage_source_attempt.outputs.run_attempt }} env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true steps: @@ -349,10 +352,23 @@ jobs: git -C "$COVERAGE_SOURCE_WORKDIR" status --short tar -cf "$COVERAGE_SOURCE_ARCHIVE" -C "$COVERAGE_SOURCE_WORKDIR" . + - name: Record coverage source workflow attempt + id: coverage_source_attempt + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + if ! [[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Coverage producer workflow attempt is not a positive integer." + exit 1 + fi + printf 'run_attempt=%s\n' "$GITHUB_RUN_ATTEMPT" >>"$GITHUB_OUTPUT" + - name: Upload materialized pull request merge tree + id: coverage_source_upload uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: opencode-coverage-source + name: opencode-coverage-source-${{ github.run_attempt }} path: ${{ runner.temp }}/opencode-coverage-source.tar if-no-files-found: error retention-days: 1 @@ -433,12 +449,48 @@ jobs: echo "::error::Coverage source tree could not be materialized; see the coverage-source-tree job log for the exact target repository, base SHA, head SHA, and fetch or merge failure." exit 1 - - name: Download materialized pull request merge tree + - name: Verify coverage source identity for current workflow attempt + id: coverage_source_identity + continue-on-error: true + env: + COVERAGE_SOURCE_ARTIFACT_ID: ${{ needs.coverage-source-tree.outputs.coverage_source_artifact_id }} + COVERAGE_SOURCE_RUN_ATTEMPT: ${{ needs.coverage-source-tree.outputs.coverage_source_run_attempt }} + CURRENT_RUN_ATTEMPT: ${{ github.run_attempt }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + if ! [[ "$CURRENT_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] || \ + [ "$COVERAGE_SOURCE_RUN_ATTEMPT" != "$CURRENT_RUN_ATTEMPT" ]; then + echo "::error::Coverage source was not produced in current workflow attempt ${CURRENT_RUN_ATTEMPT:-missing}; producer attempt=${COVERAGE_SOURCE_RUN_ATTEMPT:-missing}." + echo "::error::Use a full rerun or a fresh repository dispatch; failed-jobs-only reruns cannot reuse prior-attempt source evidence." + exit 1 + fi + if ! [[ "$COVERAGE_SOURCE_ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Coverage source artifact ID is missing or malformed for current workflow attempt." + echo "::error::Use a full rerun or a fresh repository dispatch so the producer publishes current-attempt evidence." + exit 1 + fi + artifact_id=$COVERAGE_SOURCE_ARTIFACT_ID + printf 'artifact_id=%s\n' "$artifact_id" >>"$GITHUB_OUTPUT" + + - name: Download current-attempt materialized pull request merge tree + if: steps.coverage_source_identity.outcome == 'success' + id: coverage_source_download + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: opencode-coverage-source + artifact-ids: ${{ steps.coverage_source_identity.outputs.artifact_id }} path: ${{ runner.temp }}/opencode-coverage-artifact + - name: Report missing current-attempt coverage source + if: steps.coverage_source_identity.outcome != 'success' || steps.coverage_source_download.outcome != 'success' + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + set -euo pipefail + echo "::error::Coverage source evidence is unavailable for workflow run attempt ${GITHUB_RUN_ATTEMPT}; a failed-jobs-only rerun cannot safely reconstruct or reuse source evidence from another attempt." + echo "::error::Use a full rerun or a fresh repository dispatch so coverage-source-tree uploads exact current-attempt evidence." + exit 1 + - name: Prepare pull request merge tree for coverage measurement env: COVERAGE_SOURCE_ARCHIVE: ${{ runner.temp }}/opencode-coverage-artifact/opencode-coverage-source.tar diff --git a/CHANGELOG.md b/CHANGELOG.md index c993bf7cb..a3ebe4df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Bound OpenCode coverage source evidence to a validated immutable artifact ID and producer-attested workflow attempt, retained one-day source evidence, and made selective reruns fail closed before download on missing, malformed, or prior-attempt identity with full-rerun or fresh-dispatch guidance. - Bound each review-agent invocation key to the wrapper's complete canonical payload, including the base branch and requesting actor; altered fields with a valid-format key now fail before durable-leader election or forwarding, and wrapper write permission is job-scoped. - Bound both trusted-uv quality jobs to `github.event.pull_request.head.sha` and added a permanent two-checkout regression contract so exact-head compatibility, coverage, docstring, and compilation claims cannot silently measure GitHub's generated pull-request merge revision. - Made Strix treat only a single LiteLLM provider-error line containing NVIDIA NIM context and model-catalog 404 evidence as cross-model fallback evidence, rejecting cross-line signal assembly and provider-like target source literals; moved the public default to Nemotron 3 Super 120B and added a second NVIDIA hosted candidate before GitHub Models without neutralizing reported vulnerabilities. diff --git a/docs/doctoring/opencode-coverage-artifact-reruns.md b/docs/doctoring/opencode-coverage-artifact-reruns.md new file mode 100644 index 000000000..5c57a37ae --- /dev/null +++ b/docs/doctoring/opencode-coverage-artifact-reruns.md @@ -0,0 +1,95 @@ +# OpenCode coverage artifact rerun contract + +## Decision + +The central OpenCode review workflow binds every materialized pull-request merge tree to one workflow-run attempt and one immutable GitHub Actions artifact identifier. The credential-free `coverage-evidence` job may consume only that exact artifact identifier. It never searches by a mutable artifact name and never falls back to an artifact produced by another run or attempt. + +The producer also exports a step-recorded literal workflow attempt. Before download, the consumer verifies that this attempt equals its current `github.run_attempt` and that the immutable artifact ID is a positive decimal identifier. Artifact immutability selects one upload; attempt attestation proves that the producer executed in the current attempt. + +The source artifact retains the existing one-day retention period. A failed-jobs-only rerun that does not rerun the successful producer is therefore expected to fail closed once that producer artifact expires. The operator response is a **full rerun or a fresh repository dispatch**, both of which rerun `coverage-source-tree` and create current-attempt evidence. Increasing retention or reusing prior-attempt source evidence is not an accepted repair. + +## Incident + +On August 7, 2026, failed-jobs-only rerun attempt 2 of OpenCode workflow run `31022108085` retried `coverage-evidence` for `ContextualWisdomLab/pg-llm-batch#53` without retrying the successful `coverage-source-tree` producer. The attempt-1 artifact `opencode-coverage-source` had a one-day retention period and was already expired. `actions/download-artifact` therefore returned `Artifact not found` before any current-head tests or docstring checks could run. + +The product pull request was not the source of this failure. The failing boundary was the central producer/consumer lifecycle: a static name did not prove that the consumer received evidence uploaded by the current attempt. + +## Contract + +```mermaid +sequenceDiagram + participant D as Repository dispatch + participant V as validate-pr-metadata + participant P as coverage-source-tree + participant A as Immutable Actions artifact + participant C as coverage-evidence + + D->>V: Exact repository, PR, base SHA, head SHA + V->>P: Validated current-head metadata + P->>P: Materialize exact merge tree + P->>A: Upload attempt-scoped name + A-->>P: artifact-id + P-->>C: Immutable artifact-id job output + C->>A: Download exact artifact-id + alt Artifact belongs to current producer attempt + A-->>C: Merge-tree archive + C->>C: Validate archive, sandbox tests, coverage, docstrings + else Producer was omitted or evidence expired + A-->>C: Download failure + C-->>D: Fail closed; require full rerun or fresh dispatch + end +``` + +The implementation must preserve all of the following properties: + +- `coverage-source-tree` remains the only job with repository-read and OIDC credentials for target-repository materialization. +- `coverage-evidence` remains limited to `actions: read`; it receives no repository-content token, OIDC credential, model secret, or review-write credential. +- The upload name includes `github.run_attempt` for operator diagnostics and collision resistance. +- The upload step exports the immutable `artifact-id`; the consumer validates that it is a positive decimal identifier and passes only the validated step output to `download-artifact`. +- The producer exports its step-recorded run attempt; the consumer rejects empty or prior-attempt provenance before download. +- Retention remains one day to minimize retention of private source evidence. +- Missing current-attempt evidence produces a bounded diagnostic containing the run attempt and the required recovery action. +- Exact-head metadata validation, same-repository validation, merge-tree construction, archive-member validation, isolated execution, coverage, docstring, security, and approval gates remain unchanged. + +## Rerun operations + +| Operator action | Producer behavior | Consumer behavior | Accepted outcome | +|---|---|---|---| +| Fresh repository dispatch | Producer runs and uploads a new attempt-scoped artifact | Downloads the producer's immutable artifact ID | Accepted | +| Full workflow rerun | Producer reruns and uploads a new attempt-scoped artifact | Downloads the new immutable artifact ID | Accepted | +| Failed-jobs-only rerun while producer is omitted | Producer attempt marker or artifact ID is missing or belongs to an earlier attempt | Rejects identity before download | Expected failure | +| Attempt to reuse an earlier artifact by name | Current-attempt identity is not proven | Rejected by contract | Rejected | +| Increase retention to hide missing producer execution | Stale source remains available longer | Does not repair attempt identity | Rejected | + +## Security and privacy rationale + +Artifact immutability prevents later jobs from mutating a successfully uploaded archive, but immutability alone does not identify which workflow attempt produced the archive. The producer's exact `artifact-id` closes upload-selection ambiguity, while its step-recorded attempt closes execution-attempt ambiguity. The consumer validates both before download; attempt-qualified names remain diagnostic only. + +The one-day retention period is intentionally short because the archive can contain proprietary or otherwise sensitive source code. Recovery must create fresh, exact-head evidence rather than preserve source archives for a longer period. No product test executes in the credentialed producer. No trusted follow-up consumes command files after untrusted coverage execution begins. + +## Rollback + +Rollback consists of reverting the attempt-scoped producer output and exact-ID consumer selection together. Reverting only one side leaves the workflow unable to exchange evidence. A rollback must preserve one-day retention, credential separation, and fail-closed behavior; it must not restore mutable-name fallback across attempts. + +## Verification + +The permanent regression suite must verify: + +1. attempt-scoped artifact naming and immutable `artifact-id` producer output; +2. producer-attested attempt output and pre-download current-attempt equality; +3. positive-decimal artifact-ID validation and exact-ID download; +4. actionable failure for missing, malformed, or prior-attempt evidence; +5. one-day retention; and +6. absence of repository, OIDC, secret, and review-write credentials from `coverage-evidence`. + +The complete repository test suite, Python compilation, production statement and branch coverage, public docstring gate, security and supply-chain checks, current-head review, independent approval, and protected merge remain required. + +## References + +GitHub. (2026a). *Downloading workflow artifacts*. GitHub Actions documentation. https://docs.github.com/en/actions/how-tos/manage-workflow-runs/download-workflow-artifacts + +GitHub. (2026b). *Re-running workflows and jobs*. GitHub Actions documentation. https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs + +GitHub. (2026c). *actions/download-artifact* [Computer software]. GitHub. https://github.com/actions/download-artifact + +GitHub. (2026d). *actions/upload-artifact* [Computer software]. GitHub. https://github.com/actions/upload-artifact diff --git a/tests/test_opencode_coverage_artifact_rerun_contract.py b/tests/test_opencode_coverage_artifact_rerun_contract.py new file mode 100644 index 000000000..7a891ac36 --- /dev/null +++ b/tests/test_opencode_coverage_artifact_rerun_contract.py @@ -0,0 +1,187 @@ +"""Contracts for rerun-safe OpenCode coverage artifact handoff.""" + +from pathlib import Path + + +WORKFLOW_PATH = Path(".github/workflows/opencode-review-dispatch.yml") +TEMPORARY_REPAIR_GLOBS = ( + ".github/opencode-attempt-scoped-coverage-artifact*.trigger", + ".github/pr812*.trigger", + ".github/workflows/*opencode*artifact*materializ*.yml", + ".github/workflows/*opencode*artifact*repair*.yml", + ".github/workflows/pr812-finalize*.yml", + "scripts/ci/*opencode*artifact*patch*.py", +) + + +def _workflow_text() -> str: + """Return the protected OpenCode repository-dispatch workflow source.""" + return WORKFLOW_PATH.read_text(encoding="utf-8") + + +def _job_block(workflow: str, job_name: str, next_job_name: str) -> str: + """Return one top-level workflow job block bounded by the next job.""" + start = workflow.index(f" {job_name}:\n") + end = workflow.index(f"\n {next_job_name}:\n", start) + return workflow[start:end] + + +def _step_block(job: str, step_name: str, next_step_name: str) -> str: + """Return one workflow step bounded by the following named step.""" + start = job.index(f" - name: {step_name}\n") + end = job.index(f"\n - name: {next_step_name}\n", start) + return job[start:end] + + +def test_coverage_source_artifact_is_attempt_scoped_and_downloaded_by_id() -> None: + """Bind every producer attempt to its immutable uploaded artifact ID.""" + workflow = _workflow_text() + source_job = _job_block(workflow, "coverage-source-tree", "coverage-evidence") + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + + assert ( + "coverage_source_artifact_id: " + "${{ steps.coverage_source_upload.outputs.artifact-id }}" + in source_job + ) + assert "id: coverage_source_upload" in source_job + assert "name: opencode-coverage-source-${{ github.run_attempt }}" in source_job + assert "retention-days: 1" in source_job + + identity = _step_block( + evidence_job, + "Verify coverage source identity for current workflow attempt", + "Download current-attempt materialized pull request merge tree", + ) + download = _step_block( + evidence_job, + "Download current-attempt materialized pull request merge tree", + "Report missing current-attempt coverage source", + ) + assert "id: coverage_source_identity" in identity + assert ( + "COVERAGE_SOURCE_ARTIFACT_ID: " + "${{ needs.coverage-source-tree.outputs.coverage_source_artifact_id }}" + in identity + ) + assert '[[ "$COVERAGE_SOURCE_ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]]' in identity + assert "artifact_id=$COVERAGE_SOURCE_ARTIFACT_ID" in identity + assert ( + "artifact-ids: ${{ steps.coverage_source_identity.outputs.artifact_id }}" + in download + ) + assert ( + "artifact-ids: " + "${{ needs.coverage-source-tree.outputs.coverage_source_artifact_id }}" + not in download + ) + assert "name: opencode-coverage-source\n" not in download + + +def test_coverage_source_requires_current_producer_attempt() -> None: + """Reject reused producer output when a selective rerun advances the attempt.""" + workflow = _workflow_text() + source_job = _job_block(workflow, "coverage-source-tree", "coverage-evidence") + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + identity = _step_block( + evidence_job, + "Verify coverage source identity for current workflow attempt", + "Download current-attempt materialized pull request merge tree", + ) + + assert ( + "coverage_source_run_attempt: " + "${{ steps.coverage_source_attempt.outputs.run_attempt }}" + in source_job + ) + assert "id: coverage_source_attempt" in source_job + assert "GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}" in source_job + assert "run_attempt=%s" in source_job + + assert ( + "COVERAGE_SOURCE_RUN_ATTEMPT: " + "${{ needs.coverage-source-tree.outputs.coverage_source_run_attempt }}" + in identity + ) + assert "CURRENT_RUN_ATTEMPT: ${{ github.run_attempt }}" in identity + assert '[ "$COVERAGE_SOURCE_RUN_ATTEMPT" != "$CURRENT_RUN_ATTEMPT" ]' in identity + assert "failed-jobs-only reruns cannot reuse prior-attempt source evidence" in identity + assert "full rerun or a fresh repository dispatch" in identity + + guard_index = evidence_job.index( + "- name: Verify coverage source identity for current workflow attempt" + ) + download_index = evidence_job.index( + "- name: Download current-attempt materialized pull request merge tree" + ) + assert guard_index < download_index + + +def test_missing_or_expired_artifact_fails_with_bounded_recovery_guidance() -> None: + """Keep fail-closed recovery reachable after producer or download failures.""" + workflow = _workflow_text() + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + producer_failure = _step_block( + evidence_job, + "Report coverage source materialization failure", + "Verify coverage source identity for current workflow attempt", + ) + identity = _step_block( + evidence_job, + "Verify coverage source identity for current workflow attempt", + "Download current-attempt materialized pull request merge tree", + ) + download = _step_block( + evidence_job, + "Download current-attempt materialized pull request merge tree", + "Report missing current-attempt coverage source", + ) + recovery = _step_block( + evidence_job, + "Report missing current-attempt coverage source", + "Prepare pull request merge tree for coverage measurement", + ) + + assert "if: needs.coverage-source-tree.result != 'success'" in producer_failure + assert "exit 1" not in producer_failure + assert "id: coverage_source_identity" in identity + assert "if: always()" in identity + assert "continue-on-error: true" in identity + assert "id: coverage_source_download" in download + assert "continue-on-error: true" in download + assert "needs.coverage-source-tree.result == 'success'" in download + assert "steps.coverage_source_identity.outcome == 'success'" in download + assert "if: always() && (" in recovery + assert "needs.coverage-source-tree.result != 'success'" in recovery + assert "steps.coverage_source_identity.outcome != 'success'" in recovery + assert "steps.coverage_source_download.outcome != 'success'" in recovery + assert "failed-jobs-only rerun" in recovery + assert "full rerun or a fresh repository dispatch" in recovery + assert "GITHUB_RUN_ATTEMPT" in recovery + assert "exit 1" in recovery + assert "list-artifacts" not in identity + download + recovery + + +def test_coverage_consumer_remains_credential_free() -> None: + """Keep repository and OIDC credentials outside the untrusted-test job.""" + workflow = _workflow_text() + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + permissions = evidence_job.split(" outputs:\n", 1)[0] + + assert "actions: read" in permissions + assert "contents:" not in permissions + assert "id-token:" not in permissions + assert "secrets." not in evidence_job + assert "GH_TOKEN:" not in evidence_job + + +def test_temporary_branch_writers_are_absent_from_final_tree() -> None: + """Reject versioned or renamed materializers and branch finalizers.""" + unexpected = sorted( + { + str(path) + for pattern in TEMPORARY_REPAIR_GLOBS + for path in Path(".").glob(pattern) + } + ) + assert unexpected == []