diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 6cad4cf5..55db5d8b 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -37,6 +37,7 @@ hypatia (scanner) → findings JSONL → fleet-coordinator.sh → dispatch | `fix-*.sh` | `scripts/` | ~50 fix scripts for eliminate-tier patterns | | `shared-context/` | `shared-context/` | Rust crate for inter-bot communication | | `robot-repo-automaton/` | `robot-repo-automaton/` | Rust CLI: scan, fix, PR creation | +| `fleet-report-back.sh`, `fleet-report-back-build.sh` | `scripts/` | Build the enrolled adapters, run them on a PR head, and write the report-back envelopes the Hypatia verdict grid reads (`docs/REPORT-BACK.adoc`) | ## Build & Run diff --git a/.gitbot-fleet/enrolment.json b/.gitbot-fleet/enrolment.json new file mode 100644 index 00000000..15b2b205 --- /dev/null +++ b/.gitbot-fleet/enrolment.json @@ -0,0 +1 @@ +{"reporters":{"accessibilitybot":{"enrolled":false,"reason":"no served HTML surface; the HTML in the tree is accessibilitybot's own test fixtures and gsbot's templates"},"cipherbot":{"enrolled":true,"title":"cipherbot (cryptographic hygiene)"},"echidnabot":{"enrolled":false,"reason":"the only proof files here are echidnabot's own dogfood fixtures (bots/echidnabot/proofs); no proofs about the fleet"},"finishingbot":{"enrolled":true,"title":"finishingbot (release readiness)"},"glambot":{"enrolled":true,"title":"glambot (presentation quality)"},"panicbot":{"enrolled":true,"title":"panicbot (panic recovery, error handling)"},"rhodibot":{"enrolled":true,"title":"rhodibot (RSR compliance)"},"seambot":{"enrolled":false,"reason":"no seam register is declared (spec/seams/seam-register.json)"},"sustainabot":{"enrolled":false,"reason":"reserved fleet slot; no adapter exists (bots/sustainabot/README.adoc)"}},"schema":1} diff --git a/.gitbot-fleet/report-backs/README.md b/.gitbot-fleet/report-backs/README.md new file mode 100644 index 00000000..1507cc03 --- /dev/null +++ b/.gitbot-fleet/report-backs/README.md @@ -0,0 +1,16 @@ +# gitbot-fleet report-backs + +One JSON envelope per enrolled fleet member, `.json`, in the shape +`docs/VERDICT-GRID-SPEC.adoc` of metadatastician/standards defines and +`docs/REPORT-BACK.adoc` in this repository explains. + +**This directory is empty in git on purpose, and stays so.** Envelopes are +produced per run by `scripts/fleet-report-back.sh` (through +`.github/workflows/fleet-report-back-reusable.yml`), uploaded as the +`fleet-report-backs` artifact, and placed here by the Hypatia lane only at +render time. A committed envelope would be stale evidence about an older +commit, so none is ever committed. An enrolled member with no envelope +renders **orange** ("enrolled, but no report-back was received"); only a real +run may change that. + +Only `*.json` files are read, so this README is inert. diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 5f1aa7d3..b175c6d4 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -22,6 +22,13 @@ workflows: - 'hyperpolymath/k9-ecosystem@main' '.github/workflows/e2e.yml': - 'actions/checkout@v7.0.1' + '.github/workflows/fleet-report-back-reusable.yml': + - 'actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9' + - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' + - 'actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9' + '.github/workflows/fleet-report-back.yml': + - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' + - 'actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333' '.github/workflows/governance.yml': - 'hyperpolymath/standards@75bd5994470661a099df562d6d5ad2fe744dbd66' '.github/workflows/hypatia-dispatch-intake.yml': @@ -94,6 +101,11 @@ dependencies: commit: 'sha1-368f82528645a54fb793d4d04e342629a3f51346' owner_id: 44036562 repo_id: 438112499 + 'actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333': + ref: 'v8.0.2' + commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333' + owner_id: 44036562 + repo_id: 192626254 'actions/download-artifact@v8.0.2': ref: 'v8.0.2' commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333' diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3dff4035..1090100b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -30,3 +30,7 @@ jobs: run: bash tests/retired-descriptile-policy-test.sh - name: Verify star-list enrollment discovery run: bash tests/enroll-discovery-test.sh + - name: Verify the fleet report-back driver (envelope contract) + run: bash tests/fleet-report-back-test.sh + - name: Fleet enrolment fixer (report-back rollout) + run: bash tests/fix-fleet-enrolment-test.sh diff --git a/.github/workflows/fleet-report-back-reusable.yml b/.github/workflows/fleet-report-back-reusable.yml new file mode 100644 index 00000000..cd6ca42e --- /dev/null +++ b/.github/workflows/fleet-report-back-reusable.yml @@ -0,0 +1,280 @@ +# This workflow is managed by gh actions-lock. +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# fleet-report-back-reusable.yml — produce the gitbot-fleet report-backs for +# one pull request, as an artifact the Hypatia verdict grid reads. +# +# WHY. The verdict grid (standards: docs/VERDICT-GRID-SPEC.adoc) gives every +# fleet member a repository enrols in `.gitbot-fleet/enrolment.json` one row, +# and that row is orange ("enrolled, but no report-back was received") until a +# `.gitbot-fleet/report-backs/.json` envelope exists when the grid +# renders. Until this workflow existed nothing in the estate wrote one, so the +# fleet reported nothing on any PR. This is the producer. +# +# WHAT IT DOES. Checks out the caller's tree, checks out gitbot-fleet at the +# commit the caller pinned, builds only the adapters the manifest enrols +# (cached with actions/cache), runs `scripts/fleet-report-back.sh`, and +# uploads the envelopes as the `fleet-report-backs` artifact. The Hypatia lane +# (standards: hypatia-scan-reusable.yml, input `report-backs-artifact`) +# downloads that artifact into `.gitbot-fleet/report-backs/` before rendering. +# +# WHAT IT DOES NOT DO. It posts nothing and gates nothing: an adapter that is +# missing, crashes or times out becomes an `error` envelope with its reason, +# and the grid renders that orange. The job fails only when the driver itself +# cannot run. It writes nothing into the caller's repository; the envelopes +# live in the artifact for the run, never in git. +# +# Permissions: `contents: read` only. A called workflow may not request more +# than its caller grants, and this one needs nothing more. +# +# Caller sketch (see docs/REPORT-BACK.adoc for the full adoption note): +# +# jobs: +# fleet: +# uses: hyperpolymath/gitbot-fleet/.github/workflows/fleet-report-back-reusable.yml@ +# permissions: { contents: read } +# hypatia: +# needs: [fleet] +# if: ${{ always() }} +# uses: /.github/workflows/hypatia-scan-reusable.yml@ +# ... +# +# A repository whose Actions policy refuses reusable workflows from another +# owner (metadatastician/* calling hyperpolymath/*) runs the same steps inline +# instead: check out gitbot-fleet at a pinned SHA and call the driver. The +# standards canon's own fork-governance.yml is the worked example of that. + +name: Fleet report-backs (reusable) + +on: + workflow_call: + inputs: + enrolment: + description: Path of the enrolment manifest inside the caller's tree. + type: string + required: false + default: ".gitbot-fleet/enrolment.json" + members: + description: >- + Comma-separated member names to run instead of every enrolled + member. A member the manifest marks `enrolled: false` is still + skipped: the repository's statement wins. + type: string + required: false + default: "" + artifact-name: + description: Name of the uploaded artifact holding the envelopes. + type: string + required: false + default: "fleet-report-backs" + adapter-timeout-seconds: + description: >- + Seconds each adapter may run before its row reads error. At most 300: + the job limit is sized for seven adapters at 300 s after the build + budget, so a larger value is capped (with a warning) and a value that + is not a positive integer falls back to 300. + type: string + required: false + default: "300" + outputs: + members: + description: >- + JSON object mapping each member that ran to its envelope status + (ran | not_run | error). Empty object when nothing was enrolled. + value: ${{ jobs.report-back.outputs.members }} + artifact-name: + description: The artifact name, echoed for the consumer's download step. + value: ${{ jobs.report-back.outputs.artifact-name }} + +permissions: + contents: read + +jobs: + report-back: + name: Fleet report-backs + runs-on: ubuntu-latest + # Warm (cache hit): about two minutes. Cold: up to six Rust adapters plus + # panic-attack from source (~15 min). The limit is the worst case, so a + # stalled phase never kills the job before the upload: the build budget + # (fleet-report-back-build.sh --build-budget, 2400 s) + up to seven adapters + # at adapter-timeout-seconds each (7 × 300 s = 2100 s) + checkout, cache and + # upload (~5 min) = 80 min. GNU timeout bounds each command on its own; only + # this limit bounds their sum. + timeout-minutes: 85 + permissions: + contents: read + outputs: + members: ${{ steps.run.outputs.members }} + artifact-name: ${{ inputs.artifact-name }} + env: + # panic-attack, built from source at this commit for panicbot. Kept in + # step with the standards Hypatia lane's PANIC_ATTACK_SHA so the two + # panic-attack-derived rows on one grid describe the same tool. + PANIC_ATTACK_SHA: b689a74d2762a1c19839ae3fbd59b7fe5ef920ea + steps: + - name: Checkout the repository to report on + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The PR head, not the merge ref: the grid's change scope is computed + # from the same head, so paths and line numbers agree with it. + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Checkout gitbot-fleet at the pinned commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/gitbot-fleet + # `job.workflow_sha` is the commit of THIS file, which is what the + # caller pinned. It is not `github.workflow_sha` (the caller's file) + # and `github.job_workflow_sha` does not exist. + # It is deliberately NOT an input: a ref a caller can choose is untrusted + # code run with a cache the default branch shares (CodeQL + # actions/cache-poisoning, three alerts on gitbot-fleet PR 626). The pin + # in `uses:` is the only way to choose the fleet commit, and it is immutable. + ref: ${{ job.workflow_sha }} + path: .fleet-src + persist-credentials: false + + # The fleet checkout must not sit inside the tree being reported on: + # walk-based adapters (glambot, finishing-bot, cipherbot) would scan the + # fleet's own source and emit findings no PR diff can match. actions/checkout + # only writes under the workspace, so the checkout is moved out to + # RUNNER_TEMP before anything reads the caller's tree. The lockfile hash + # for the cache key is taken first, while hashFiles() can still see it. + - name: Move the fleet checkout out of the scanned tree + id: fleet + run: | + set -euo pipefail + lockhash=$(cat .fleet-src/bots/*/Cargo.lock .fleet-src/shared-context/Cargo.lock 2>/dev/null | sha256sum | cut -c1-16) + mv .fleet-src "$RUNNER_TEMP/fleet-src" + echo "dir=$RUNNER_TEMP/fleet-src" >> "$GITHUB_OUTPUT" + echo "lockhash=$lockhash" >> "$GITHUB_OUTPUT" + + - name: Plan which adapters to build + id: plan + env: + FLEET_DIR: ${{ steps.fleet.outputs.dir }} + ENROLMENT: ${{ inputs.enrolment }} + MEMBERS: ${{ inputs.members }} + run: | + set -euo pipefail + args=(--plan --enrolment "$ENROLMENT") + IFS=',' read -r -a _m <<< "${MEMBERS:-}" + for m in "${_m[@]}"; do m="${m// /}"; [ -n "$m" ] && args+=(--member "$m"); done + plan=$(bash "$FLEET_DIR/scripts/fleet-report-back.sh" "${args[@]}") + echo "plan:"; printf '%s\n' "$plan" | sed 's/^/ /' + # Crates to build: the non-empty second column, deduplicated. + crates=$(printf '%s\n' "$plan" | awk -F'\t' '$2 != "" {print $2}' | sort -u | tr '\n' ' ') + echo "crates=${crates% }" >> "$GITHUB_OUTPUT" + if printf '%s\n' "$plan" | awk -F'\t' '$1 == "panicbot" {found=1} END {exit !found}'; then + echo "panicbot=true" >> "$GITHUB_OUTPUT" + else + echo "panicbot=false" >> "$GITHUB_OUTPUT" + fi + { + echo "workspaces< target"; done + echo "$FLEET_DIR/shared-context -> target" + echo "WS" + } >> "$GITHUB_OUTPUT" + [ -n "${crates% }" ] && echo "build=true" >> "$GITHUB_OUTPUT" || echo "build=false" >> "$GITHUB_OUTPUT" + + # Rust comes with the GitHub-hosted runner image (rustup, stable), so no + # third-party toolchain action is needed; the cache is actions/cache, keyed + # on the fleet commit's lockfiles. Only GitHub-owned actions, pinned by + # SHA, appear in this file: a caller whose Actions policy requires SHA pins + # or forbids unverified creators can still call it (metamanifold-webui + # PR 296 died at plan time with zero jobs on the tag-pinned version). + - name: Cache adapter builds + if: steps.plan.outputs.build == 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + ${{ runner.temp }}/fleet-src/bots/*/target + key: fleet-report-back-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.fleet.outputs.lockhash }}-${{ steps.plan.outputs.crates }} + restore-keys: | + fleet-report-back-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.fleet.outputs.lockhash }}- + fleet-report-back-v3-${{ runner.os }}-${{ runner.arch }}- + + - name: Report the Rust toolchain in use + if: steps.plan.outputs.build == 'true' || steps.plan.outputs.panicbot == 'true' + run: | + set -euo pipefail + command -v cargo >/dev/null || { echo "::error::cargo is not on this runner; the adapters cannot be built"; exit 1; } + cargo --version; rustc --version + + - name: Restore panic-attack build cache + if: steps.plan.outputs.panicbot == 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.panic-attack + key: panic-attack-v1-${{ runner.os }}-${{ runner.arch }}-${{ env.PANIC_ATTACK_SHA }} + + - name: Build the enrolled adapters + if: steps.plan.outputs.build == 'true' || steps.plan.outputs.panicbot == 'true' + env: + FLEET_DIR: ${{ steps.fleet.outputs.dir }} + CRATES: ${{ steps.plan.outputs.crates }} + PANICBOT: ${{ steps.plan.outputs.panicbot }} + run: | + set -euo pipefail + # One build recipe for the reusable and for inline callers; a crate + # that fails to build is a warning here and an `error` row later. + args=(--fleet "$FLEET_DIR" --bin-dir "$RUNNER_TEMP/fleet-bin" --crates "$CRATES") + [ "$PANICBOT" = true ] && args+=(--panic-attack "$PANIC_ATTACK_SHA") + bash "$FLEET_DIR/scripts/fleet-report-back-build.sh" "${args[@]}" + + - name: Run the enrolled adapters and write the envelopes + id: run + # The envelopes are written under RUNNER_TEMP, never into the scanned tree: + # each member's JSON would otherwise be scanned by the walk-based adapters + # that run after it, and a tracked report-backs/ directory in the caller + # would leak into the artifact. + env: + FLEET_DIR: ${{ steps.fleet.outputs.dir }} + ENROLMENT: ${{ inputs.enrolment }} + MEMBERS: ${{ inputs.members }} + ADAPTER_TIMEOUT: ${{ inputs.adapter-timeout-seconds }} + FLEET_REF: ${{ job.workflow_sha }} + run: | + set -euo pipefail + # The job limit (timeout-minutes) assumes at most 300 s per adapter; a + # larger input would let the scans outrun it and lose every envelope. + case "$ADAPTER_TIMEOUT" in + ''|*[!0-9]*|0) echo "::warning title=adapter-timeout-seconds::'$ADAPTER_TIMEOUT' is not a positive integer; using 300"; ADAPTER_TIMEOUT=300 ;; + esac + if [ "$ADAPTER_TIMEOUT" -gt 300 ]; then + echo "::warning title=adapter-timeout-seconds::$ADAPTER_TIMEOUT s exceeds the 300 s the job limit is sized for; capped at 300" + ADAPTER_TIMEOUT=300 + fi + mkdir -p "$RUNNER_TEMP/fleet-bin" "$RUNNER_TEMP/report-backs" + args=(--repo . --out "$RUNNER_TEMP/report-backs" --bin-dir "$RUNNER_TEMP/fleet-bin" + --enrolment "$ENROLMENT" --fleet-ref "$FLEET_REF" + --timeout "$ADAPTER_TIMEOUT" --summary "$GITHUB_STEP_SUMMARY") + IFS=',' read -r -a _m <<< "${MEMBERS:-}" + for m in "${_m[@]}"; do m="${m// /}"; [ -n "$m" ] && args+=(--member "$m"); done + bash "$FLEET_DIR/scripts/fleet-report-back.sh" "${args[@]}" + # Status map for the caller, and a non-empty directory for the + # upload: a run that enrolled nobody still uploads the note below so + # the consumer can tell "nothing enrolled" from "upload failed". + if ls "$RUNNER_TEMP"/report-backs/*.json >/dev/null 2>&1; then + members=$(jq -s -c 'map({(.reporter): .status}) | add' "$RUNNER_TEMP"/report-backs/*.json) + else + members='{}' + printf 'No member is enrolled in %s; no report-back was produced.\n' "$ENROLMENT" > "$RUNNER_TEMP/report-backs/NONE-ENROLLED.txt" + fi + echo "members=$members" >> "$GITHUB_OUTPUT" + echo "members: $members" + + - name: Upload the report-backs + if: ${{ always() && steps.run.conclusion != 'skipped' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ inputs.artifact-name }} + path: ${{ runner.temp }}/report-backs + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/fleet-report-back.yml b/.github/workflows/fleet-report-back.yml new file mode 100644 index 00000000..7dfe15fa --- /dev/null +++ b/.github/workflows/fleet-report-back.yml @@ -0,0 +1,112 @@ +# This workflow is managed by gh actions-lock. +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# fleet-report-back.yml — the fleet dogfoods its own report-back reusable. +# +# Every pull request here runs the enrolled adapters against this repository +# through `fleet-report-back-reusable.yml`, then proves the artifact is the +# evidence the Hypatia verdict grid accepts: the real renderer from +# metadatastician/standards is run over the envelopes and the enrolment +# manifest, and the job FAILS if any enrolled member is still reported as +# "enrolled, but no report-back was received". That is the one claim this +# workflow exists to test: the fleet's PR feedback is no longer silent. +# +# The grid rendered here carries only fleet rows (no Hypatia, panic-attack or +# pons evidence is passed), so its overall state is orange by construction and +# is not a verdict on this repository. The repository's own Hypatia lane is +# hypatia-scan.yml; once its standards pin carries the `report-backs-artifact` +# input, that lane consumes the same artifact and the rows appear in the PR +# comment. Until then this summary is where the rows are visible. + +name: Fleet report-backs + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # metadatastician/standards commit whose scripts/verdict-grid.sh renders the + # grid. Bump deliberately; the renderer's envelope contract is what the + # driver's test suite (tests/fleet-report-back-test.sh) is written against. + STANDARDS_GRID_SHA: 9ae23875d29a7c6de71a17eb026d54b914a3d2bf + +jobs: + report-back: + name: Produce the report-backs + uses: ./.github/workflows/fleet-report-back-reusable.yml + permissions: + contents: read + + render-check: + name: The grid accepts the envelopes + needs: [report-back] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout this repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Checkout the verdict grid renderer + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: metadatastician/standards + ref: ${{ env.STANDARDS_GRID_SHA }} + path: .standards-grid + persist-credentials: false + sparse-checkout: | + scripts/verdict-grid.sh + scripts/lib/verdict-grid.jq + sparse-checkout-cone-mode: false + + - name: Download the report-backs + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: fleet-report-backs + path: report-backs + + - name: Run the driver's test suite against the real renderer + run: STANDARDS_DIR="$PWD/.standards-grid" bash tests/fleet-report-back-test.sh + + - name: Render the grid from the envelopes and fail on any silent member + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + MEMBERS: ${{ needs.report-back.outputs.members }} + run: | + set -euo pipefail + echo "members reported by the reusable: $MEMBERS" + ls -l report-backs + bash .standards-grid/scripts/verdict-grid.sh render --out grid \ + --hypatia /dev/null --panic /dev/null --pons /dev/null \ + --report-backs report-backs --enrolment .gitbot-fleet/enrolment.json \ + --sha "$HEAD_SHA" --run "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > /dev/null + { + echo "## Fleet rows as the Hypatia verdict grid renders them" + echo + echo "_Fleet rows only: no Hypatia, panic-attack or pons evidence is passed here, so the lane state is orange by construction._" + echo + cat grid/report.md + } >> "$GITHUB_STEP_SUMMARY" + silent=$(jq -r '[.reporters[] | select(.reason == "enrolled, but no report-back was received") | .id] | join(", ")' grid/report.json) + if [ -n "$silent" ]; then + echo "::error::enrolled member(s) produced no report-back: $silent" + exit 1 + fi + enrolled=$(jq -r '[.reporters | to_entries[] | select(.value.enrolled == true) | .key] | length' .gitbot-fleet/enrolment.json) + reported=$(jq -r '[.reporters[] | select(.status != "not_applicable")] | length' grid/report.json) + echo "enrolled=$enrolled fleet rows with evidence or a stated reason=$((reported - 3))" + echo "every enrolled member reported back" diff --git a/README.adoc b/README.adoc index 636c9fc5..7ad1f137 100644 --- a/README.adoc +++ b/README.adoc @@ -144,6 +144,7 @@ Authoritative documents: * link:EXPLAINME.adoc[EXPLAINME.adoc] — long-form architecture and rationale * link:ROADMAP.adoc[ROADMAP.adoc] — forward plan, all 11 bots * link:READINESS.md[READINESS.md] — Component Readiness Grades (CRG) assessment +* link:docs/REPORT-BACK.adoc[docs/REPORT-BACK.adoc] — how each bot's verdict reaches the Hypatia verdict grid on a pull request (the `fleet-report-backs` artifact) * link:docs/[docs/] — operational guides, performance notes, branch protection * link:docs/archive/[docs/archive/] — historical session reports kept for audit * link:docs/wiki-source/[docs/wiki-source/] — repo source for the GitHub wiki diff --git a/ROADMAP.adoc b/ROADMAP.adoc index e320f4f0..f4610c9d 100644 --- a/ROADMAP.adoc +++ b/ROADMAP.adoc @@ -80,7 +80,7 @@ Robot-repo-automaton is currently at ~5% completion. This milestone brings it to * [ ] GitHub Actions integration for automated fleet runs * [ ] GitLab CI integration via mirror workflows * [ ] Scheduled fleet scans (cron-based) -* [ ] PR-triggered scans with bot comments +* [~] PR-triggered scans with bot comments — report-backs land on every PR grid via `fleet-report-back-reusable.yml` (2026-10-27, link:docs/REPORT-BACK.adoc[]); comments are posted by the fork-governance engine in standards, not by the fleet. Estate rollout: new repos inherit from rsr-template-repo, existing repos via `scripts/fix-fleet-enrolment.sh` on their standards pin-bump branch (REPORT-BACK.adoc §6); gated on the standards lane reading the artifact * [ ] Status checks: block merge on Eliminate-tier findings == v1.3.0 -- Observability and Analytics diff --git a/bots/README.adoc b/bots/README.adoc index e3c9f27f..3ce8a8b1 100644 --- a/bots/README.adoc +++ b/bots/README.adoc @@ -28,3 +28,15 @@ reset to a placeholder. See `sustainabot/README.adoc`. If a bot needs a heavyweight capability that deserves its own project, build that project in its **own repository** and have the adapter here depend on it as an external crate or service. Keep the adapter thin. + +== How an adapter reports back to a pull request + +Each adapter keeps a local-directory JSON mode (`rhodibot canon --path`, +`panicbot scan`, `seambot check`, `finishing-bot audit`, `glambot audit`, +`accessibilitybot fleet`, `cipherbot scan`). `scripts/fleet-report-back.sh` +runs the members a repository enrols in `.gitbot-fleet/enrolment.json` and +translates each output into the report-back envelope the Hypatia verdict grid +reads, so the member's row on the pull request stops being orange. A new +adapter joins that path by adding one `case` arm to the driver and one stub to +`tests/fleet-report-back-test.sh`; it must print JSON to stdout and write +nothing into the scanned tree. See link:../docs/REPORT-BACK.adoc[docs/REPORT-BACK.adoc]. diff --git a/docs/ARCHITECTURE.adoc b/docs/ARCHITECTURE.adoc index 5365e639..f364f910 100644 --- a/docs/ARCHITECTURE.adoc +++ b/docs/ARCHITECTURE.adoc @@ -54,6 +54,11 @@ fix, PR creation |`+bots/*+` |`+bots/+` |Rust (thin adapters — see `+bots/README.adoc+`) |Per-bot specialised logic + +|`+scripts/fleet-report-back.sh+` |`+scripts/+` |bash |Runs the adapters a +repository enrols and writes one report-back envelope per member for the +Hypatia verdict grid (`+docs/REPORT-BACK.adoc+`); called by +`+.github/workflows/fleet-report-back-reusable.yml+` |=== === Safety triangle diff --git a/docs/REPORT-BACK.adoc b/docs/REPORT-BACK.adoc new file mode 100644 index 00000000..41d55f8a --- /dev/null +++ b/docs/REPORT-BACK.adoc @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) += Fleet report-backs — how a bot's verdict reaches a pull request +:toc: left +:toclevels: 3 +:revdate: 2026-10-27 + +== 1. The problem this solves + +Every pull request in an estate repository carries one generated +*Fork governance* comment whose third lane is the *Hypatia verdict grid* +(metadatastician/standards: `3-practice/fork-governance/`, +`docs/VERDICT-GRID-SPEC.adoc`). The grid has three permanent rows (Hypatia, +panic-attack, pons asinorum) and one row per gitbot-fleet member the repository +enrols in `.gitbot-fleet/enrolment.json`. + +A fleet row leaves 🟠 *indeterminate* only when a file +`.gitbot-fleet/report-backs/.json` is present in the scanned tree when +the grid renders. Until 2026-10-27 nothing in the estate produced one. Measured +on hyperpolymath/metamanifold-webui#295: seven enrolled members, seven rows +reading "enrolled, but no report-back was received". The fleet was offering +nothing, and the grid said so honestly. + +This document describes the producer that closes the gap and how a repository +adopts it. + +== 2. Components + +[cols="1,3"] +|=== +| `scripts/fleet-report-back.sh` +| The driver. Reads the enrolment manifest, runs each enrolled adapter's + existing local-directory JSON mode, translates the output into the grid's + envelope, writes `/.json`. Exit 0 whenever envelopes were + written, whatever they say. `--plan` prints which crates a workflow must + build. `--summary` appends a Markdown table. + +| `scripts/fleet-report-back-build.sh` +| Builds the enrolled adapter crates (`cargo build --release --locked`) into one + directory and installs panic-attack at the pinned commit for panicbot. One + recipe for the reusable workflow and for inline callers; a crate that fails to + build is a warning here and an `error` row later, never a dead run. + +| `tests/fleet-report-back-test.sh` +| 48 checks over stubbed adapters whose JSON shapes were taken from live runs: + enrolment rules, every adapter's translation, path relativisation, the + failure modes, envelope shape, `--plan`, the summary, and (with + `STANDARDS_DIR` set) the real renderer turning the rows non-orange. + +| `.github/workflows/fleet-report-back-reusable.yml` +| `workflow_call`. Checks out the caller and gitbot-fleet at the pinned commit, + builds only the enrolled adapters (actions/cache; only GitHub-owned, SHA-pinned actions), builds panic-attack + at the SHA the standards Hypatia lane pins, runs the driver, uploads the + `fleet-report-backs` artifact. `contents: read` only. Posts nothing. + +| `.github/workflows/fleet-report-back.yml` +| This repository's dogfood: runs the reusable on its own PRs, then renders the + envelopes with the real standards renderer and fails if any enrolled member + is still reported silent. + +| `.gitbot-fleet/enrolment.json`, `.gitbot-fleet/report-backs/` +| This repository's own manifest and the (deliberately empty) directory the + lane fills at render time. +|=== + +== 3. The envelope + +One JSON object per member, key-sorted and compact with one final LF (the +I-JSON/JCS shape the estate asks of every `.json` it writes, as far as `jq` +produces it): + +[source,json] +---- +{ "findings": [ { "file": "src/a.jl", "line": 0, "message": "eval in src/a.jl", + "rule_id": "PA018", "severity": "warning" } ], + "reason": null, + "ref": "", + "reporter": "panicbot", + "status": "ran", + "title": "panicbot (panic recovery, error handling)" } +---- + +`status` is `ran`, `not_run` or `error`; `reason` is required unless the member +ran. The driver never writes `not_applicable`: that is a statement about the +repository and belongs in the enrolment manifest (`"enrolled": false` with a +reason), where the grid renders it grey. `severity` is the fleet's own word +(`error`, `warning`, `info`, `suggestion`) or a scanner word (`critical`, +`high`, `medium`, `low`); the renderer normalises both +(`error` → high, `warning` → medium, `info`/`suggestion` → minor). File paths +are repository-relative with no `./`, so the grid's change-scope (delta) +grading can match them against the diff. + +== 4. What each member reports, and how + +[cols="1,2,3",options="header"] +|=== +| Member | Invocation | Translation and honesty notes + +| rhodibot +| `rhodibot canon --path --format json` +| File-presence criteria against the vendored RSR canon. `satisfied` rows are + not findings; `missing` → error, `deprecated` → warning, `relocated` → info. + Advisory by rhodibot's own design: hypatia's `rsr-conformance` is the + normative checker. + +| panicbot +| `panicbot --format json scan ` +| Wraps `panic-attack assail`; panicbot's own translator maps critical → error, + high → warning. The grid already carries a direct panic-attack row from the + Hypatia lane, so a repository that enrols panicbot sees the same tool twice, + once through the bot's categorisation. `fleet` mode is *not* used: it writes a + `.panicbot/*.a2ml` debt register into the scanned tree, and A2ML is retired. + +| seambot +| `seambot --path --format json check` +| Seam register completeness, drift and hidden channels. A repository with no + `spec/seams/seam-register.json` gets one `error` finding saying so; whether + that is wanted is the enrolment manifest's call. + +| finishingbot +| `finishing-bot --path --format json audit` +| Eight analyzers (license, placeholders, claims, release, SCM files, testing, + tooling, v1 readiness) flattened to one list. Exits 1 when findings block a + release; that is still a run. + +| glambot +| `glambot --path --format json audit` +| Visual, accessibility, SEO, machine-readability and git-SEO analyzers, + flattened. + +| accessibilitybot +| `accessibilitybot fleet ` +| WCAG 2.3 static analysis of HTML/CSS; absolute paths are relativised. Exits 1 + when findings block a release; still a run. + +| cipherbot +| `cipherbot --format json scan ` +| Cryptographic hygiene and post-quantum readiness findings. + +| echidnabot +| none +| echidnabot verifies proofs by dispatching to an ECHIDNA engine and has no + local mode. Without `ECHIDNA_URL` the envelope is `not_run` with that reason: + still orange, but the comment now says *why*. The row turns green or red once + the engine leg lands; that is tracked in this repository, not papered over + with a placeholder. + +| sustainabot +| none +| Reserved slot (`bots/sustainabot/README.adoc`); `not_run` with that reason. + A repository should mark it `enrolled: false` instead. +|=== + +Adapter failures are never silent and never fatal to the run: a missing +binary, a crash without JSON, or a timeout each become `error` with the first +diagnostic line as the reason, and the grid renders that orange. Failing the +job would have turned every other member's evidence into a missing artifact. + +== 5. Adoption + +A repository needs three things. + +. *An enrolment manifest* at `.gitbot-fleet/enrolment.json`, stating for every + member whether it applies, with a reason when it does not. Enrol what is + true of the repository, not what would look complete: a grey row with a + reason is honest; an orange row forever is noise. +. *The producer job*, before the Hypatia lane in the fork-governance workflow. + Where the repository's Actions policy allows a reusable workflow from + `hyperpolymath`: ++ +[source,yaml] +---- +jobs: + fleet: + name: Fleet report-backs + uses: hyperpolymath/gitbot-fleet/.github/workflows/fleet-report-back-reusable.yml@ + permissions: + contents: read + hypatia: + needs: [fleet] + if: ${{ always() }} + uses: metadatastician/standards/.github/workflows/hypatia-scan-reusable.yml@ + # the reusable downloads the `fleet-report-backs` artifact by default +---- ++ +Where it does not (a repository under another owner calling a reusable from +`hyperpolymath` fails at plan time with zero jobs), run the same steps inline: +check out `hyperpolymath/gitbot-fleet` at a pinned SHA with `actions/checkout`, +run `scripts/fleet-report-back.sh`, upload the artifact under the same name. +The standards canon's own `fork-governance.yml` is the worked example. +. *A Hypatia lane that reads the artifact.* The `report-backs-artifact` input + (default `fleet-report-backs`; `""` opts out) and the download step before + "Render the verdict grid" are a change to metadatastician/standards + `hypatia-scan-reusable.yml` that is prepared but NOT yet on its `main`: no + pinned standards revision at the time of writing carries them, so a caller + on today's standards pin produces the artifact and the lane does not read it + yet. Until that change lands and callers bump their pin, a repository gets + rows only through a fork-local copy of the reusable that carries the step + (hyperpolymath/metamanifold-webui does, as its fork delta 4). This document + will name the first standards commit that carries the input once it exists. + +== 6. Rolling it out across the estate + +The rows are not metamanifold-webui's: every repository that runs the fork +governance comment can have them. Two routes, chosen by the owner 2026-10-27: + +* *New repositories* inherit the manifest and the inline `fleet` job from + `hyperpolymath/rsr-template-repo` (its `.gitbot-fleet/` and + `.github/workflows/fork-governance.yml`). +* *Existing repositories* get them from `scripts/fix-fleet-enrolment.sh`, + which rides the standards pin-propagation PR each repository already + receives (standards `scripts/propagate-workflow-pins.sh`): run both in the + same branch, so the lane that reads the artifact and the job that produces it + land together. Registered as recipe `recipe-add-fleet-enrolment` and category + `MissingFleetEnrolment` in `scripts/fix-script-registry.json`; no Hypatia + rule emits that finding yet, so today the fixer is run by hand or from a + campaign manifest. + +[source,bash] +---- +# in a consumer checkout, on the branch that bumps its standards pins +STANDARDS_DIR=~/standards FLEET_REF= \ + bash ~/gitbot-fleet/scripts/fix-fleet-enrolment.sh . +# exit 0: wired end to end. exit 3: written, with the MANUAL lines naming what +# is still missing (a lock to regenerate, a standards pin that does not read the +# artifact, a hypatia job to wire by hand). exit 2: refused, nothing changed. +---- + +What the fixer writes, and what it refuses to: + +[cols="1,3",options="header"] +|=== +| Item | Rule +| `.gitbot-fleet/enrolment.json` +| Written only when absent. rhodibot, finishingbot, glambot and cipherbot are enrolled everywhere; panicbot, seambot, accessibilitybot and echidnabot are enrolled when the tree shows what they read (a `Cargo.toml`, `spec/seams/seam-register.json`, HTML files, proof files) and otherwise carry that observation as the reason. An existing manifest is the owner's and is never rewritten. +| `.gitbot-fleet/report-backs/README.md` +| Written only when absent. +| `fleet` job +| Inserted before `hypatia:` in `fork-governance.yml`. Under the hyperpolymath owner it calls `fleet-report-back-reusable.yml` (`scripts/lib/fleet-job.reusable.yml`); under any other owner, where a cross-owner reusable plans zero jobs, the steps go inline (`scripts/lib/fleet-job.inline.yml`, kept step for step with the reusable by `tests/fix-fleet-enrolment-test.sh`). The pin is `FLEET_REF` or this checkout's `origin/main`, never a feature-branch HEAD. +| `hypatia` wiring +| Gains `needs: [fleet]` (appended to a flow list it already has) and, only when it has no `if:`, `if: ${{ !cancelled() }}`. A block-list `needs:` is left to a human. An existing `always()` is pointed out, not changed. +| `actions.lock` +| Never edited. When one exists and a job was inserted the fixer exits 3 naming `gh actions-lock`. +| Standards pin +| Never bumped. The fixer reads the pinned `hypatia-scan-reusable.yml` from `STANDARDS_DIR` and exits 3 unless it takes `report-backs-artifact`: an artifact no lane downloads is unwired. +|=== + +Until the standards change in §5 step 3 is on `main`, every run against a +remote standards pin ends in that exit 3. That is the intended order: the +campaign starts when the standards pin can move, not before. diff --git a/scripts/fix-fleet-enrolment.sh b/scripts/fix-fleet-enrolment.sh new file mode 100755 index 00000000..418506c0 --- /dev/null +++ b/scripts/fix-fleet-enrolment.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# fix-fleet-enrolment.sh — give a repository what its Hypatia verdict grid needs +# before a gitbot-fleet row can say anything (docs/REPORT-BACK.adoc §5): +# +# 1. `.gitbot-fleet/enrolment.json`, written only when absent. Members that +# apply to every estate repository (rhodibot, finishingbot, glambot, +# cipherbot) are enrolled; panicbot, seambot, accessibilitybot and +# echidnabot are enrolled when the tree shows what they read (a Cargo.toml, +# spec/seams/seam-register.json, HTML files, proof files) and otherwise +# carry the observed reason; sustainabot stays the reserved slot. An +# existing manifest is never rewritten: the owner's enrolment decisions +# stand, and changing a manifest resets the grid's baseline. +# 2. `.gitbot-fleet/report-backs/README.md`, saying the directory is empty on +# purpose (envelopes are produced per run, never committed). +# 3. A `fleet` job in `.github/workflows/fork-governance.yml`, inserted before +# the `hypatia` job, which gains `needs: [fleet]` (and `if: !cancelled()` +# when it had no `if:`). Under the hyperpolymath owner the job calls this +# repository's reusable workflow (lib/fleet-job.reusable.yml); under any +# other owner, where a cross-owner reusable fails at plan time with zero +# jobs, the same steps go inline (lib/fleet-job.inline.yml). The pin is +# FLEET_REF, or this checkout's origin/main — never a feature-branch HEAD +# (the 2026-09-04 incident: a PR-intermediate commit propagated to 70 +# repositories before it had landed). +# 4. A check that the Hypatia lane the workflow pins actually READS the +# artifact (its `report-backs-artifact` input). Producing an artifact no +# lane downloads is unwired, and unwired is not done, so that state exits +# 3 with the bump named — the fixer never bumps a standards pin itself; +# standards' scripts/propagate-workflow-pins.sh does, and this fixer is +# meant to run in the same branch as that bump. +# +# Idempotent: a second run changes nothing and reports `ok` per item. Touches +# only the three paths above. Never edits actions.lock: when the repository +# carries one and a job was inserted, the lock needs `gh actions-lock` (or the +# repository's check-lock-sync.sh) and the fixer says so. +# +# Exit codes: 0 everything present and wired · 2 usage, refused, or nothing +# changed because an input was invalid · 3 files were written or checked but +# the repository is not yet wired end to end (each reason is printed as a +# MANUAL/UNVERIFIED line) — a dispatcher must not record 3 as a clean success. +# +# Usage: +# fix-fleet-enrolment.sh [finding-json] +# +# Environment: +# FLEET_REF gitbot-fleet commit to pin (40 hex). Default: this +# checkout's refs/remotes/origin/main. +# PANIC_ATTACK_SHA panic-attack commit for the inline job. Default: the +# value in this repository's fleet-report-back-reusable.yml. +# FLEET_JOB_FORM auto (default) | inline | reusable. +# STANDARDS_DIR a standards checkout used to read the pinned Hypatia +# reusable; without it a remote pin is UNVERIFIED. +# The finding JSON (dispatch-runner's second argument) is accepted and ignored: +# the fixer is driven by the tree, not by the finding. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FLEET_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_PATH="${1:?Usage: fix-fleet-enrolment.sh [finding-json]}" + +[ -d "$REPO_PATH" ] || { echo "fix-fleet-enrolment: not a directory: $REPO_PATH" >&2; exit 2; } +REPO_PATH="$(cd "$REPO_PATH" && pwd)" + +MANIFEST="$REPO_PATH/.gitbot-fleet/enrolment.json" +RB_README="$REPO_PATH/.gitbot-fleet/report-backs/README.md" +WORKFLOW="$REPO_PATH/.github/workflows/fork-governance.yml" +LOCK="$REPO_PATH/.github/workflows/actions.lock" + +rc=0 +manual() { echo " MANUAL: $*"; rc=3; } +unverified() { echo " UNVERIFIED: $*"; rc=3; } + +# ── inputs ─────────────────────────────────────────────────────────────────── +fleet_ref="${FLEET_REF:-}" +if [ -z "$fleet_ref" ]; then + fleet_ref=$(git -C "$FLEET_DIR" rev-parse --verify -q refs/remotes/origin/main 2>/dev/null || true) + [ -n "$fleet_ref" ] || { echo "fix-fleet-enrolment: FLEET_REF is unset and this checkout has no origin/main to pin; nothing was changed" >&2; exit 2; } +fi +[[ "$fleet_ref" =~ ^[0-9a-f]{40}$ ]] || { echo "fix-fleet-enrolment: FLEET_REF must be a 40-hex commit, got '$fleet_ref'; nothing was changed" >&2; exit 2; } +# The pin must carry the producer, or every consumer would get a fleet job that +# plans zero jobs (reusable form) or finds no driver (inline form). +for f in .github/workflows/fleet-report-back-reusable.yml scripts/fleet-report-back.sh; do + git -C "$FLEET_DIR" cat-file -e "$fleet_ref:$f" 2>/dev/null \ + || { echo "fix-fleet-enrolment: commit ${fleet_ref:0:12} does not carry $f in this checkout (fetch it, or pin a commit that does); nothing was changed" >&2; exit 2; } +done + +panic_sha="${PANIC_ATTACK_SHA:-}" +if [ -z "$panic_sha" ]; then + panic_sha=$(grep -m1 -oE 'PANIC_ATTACK_SHA: [0-9a-f]{40}' "$FLEET_DIR/.github/workflows/fleet-report-back-reusable.yml" 2>/dev/null | cut -d' ' -f2 || true) + [ -n "$panic_sha" ] || { echo "fix-fleet-enrolment: PANIC_ATTACK_SHA is unset and the reusable workflow carries none; nothing was changed" >&2; exit 2; } +fi +[[ "$panic_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "fix-fleet-enrolment: PANIC_ATTACK_SHA must be a 40-hex commit, got '$panic_sha'; nothing was changed" >&2; exit 2; } + +form="${FLEET_JOB_FORM:-auto}" +case "$form" in + auto|inline|reusable) ;; + *) echo "fix-fleet-enrolment: FLEET_JOB_FORM must be auto, inline or reusable, got '$form'" >&2; exit 2 ;; +esac +if [ "$form" = auto ]; then + origin=$(git -C "$REPO_PATH" remote get-url origin 2>/dev/null || true) + # github.com/hyperpolymath/ over https or ssh; anything else is another owner. + if [[ "$origin" =~ github\.com[:/]hyperpolymath/ ]]; then form=reusable; else form=inline; fi +fi +JOB_TEMPLATE="$SCRIPT_DIR/lib/fleet-job.$form.yml" +[ -r "$JOB_TEMPLATE" ] || { echo "fix-fleet-enrolment: job template not readable: $JOB_TEMPLATE" >&2; exit 2; } + +# One workflow-pin fixer per checkout at a time (see lib/pin-fixer-lock.sh). +# shellcheck source-path=SCRIPTDIR source=lib/pin-fixer-lock.sh +. "$SCRIPT_DIR/lib/pin-fixer-lock.sh" +pin_fixer_lock "$REPO_PATH" + +echo "=== Fleet enrolment (report-backs) ===" +echo " Repo: $REPO_PATH" +echo " Fleet pin: $fleet_ref job form: $form" + +# ── tree facts the manifest is written from ────────────────────────────────── +# Vendored and generated trees are not this repository's surface. +tree_has() { # find-expression... + [ -n "$(find "$REPO_PATH" \( -name .git -o -name node_modules -o -name target -o -name vendor -o -name .fleet-src \) -prune -o -type f \( "$@" \) -print -quit 2>/dev/null)" ] +} +has_rust=false; has_seams=false; has_html=false; has_proofs=false +tree_has -name Cargo.toml && has_rust=true +[ -f "$REPO_PATH/spec/seams/seam-register.json" ] && has_seams=true +tree_has -name '*.html' -o -name '*.htm' && has_html=true +if [ -d "$REPO_PATH/proofs" ] || tree_has -name '*.agda' -o -name '*.lean' -o -name '*.idr' -o -name '*.v'; then has_proofs=true; fi + +# ── 1. manifest ────────────────────────────────────────────────────────────── +if [ -f "$MANIFEST" ]; then + if jq -e '.reporters | type == "object"' "$MANIFEST" >/dev/null 2>&1; then + echo " ok: .gitbot-fleet/enrolment.json exists (left as the owner wrote it)" + else + manual ".gitbot-fleet/enrolment.json exists but does not parse as a manifest with a .reporters object; not overwritten" + fi +else + mkdir -p "$(dirname "$MANIFEST")" + # Key-sorted, compact, one final LF: the I-JSON/JCS shape the estate's + # JSON policy asks of every .json (standards 3-practice/JSON-POLICY.adoc). + jq -nSc --argjson rust "$has_rust" --argjson seams "$has_seams" --argjson html "$has_html" --argjson proofs "$has_proofs" ' + {schema: 1, + reporters: { + rhodibot: {title: "rhodibot (RSR compliance)", enrolled: true}, + finishingbot: {title: "finishingbot (release readiness)", enrolled: true}, + glambot: {title: "glambot (presentation quality)", enrolled: true}, + cipherbot: {title: "cipherbot (cryptographic hygiene)", enrolled: true}, + panicbot: (if $rust then {title: "panicbot (panic recovery, error handling)", enrolled: true} + else {enrolled: false, reason: "no Cargo.toml in the tree; panic-attack scans Rust"} end), + seambot: (if $seams then {title: "seambot (ABI/FFI seam checks)", enrolled: true} + else {enrolled: false, reason: "no seam register is declared (spec/seams/seam-register.json)"} end), + accessibilitybot: (if $html then {title: "accessibilitybot (WCAG conformance)", enrolled: true} + else {enrolled: false, reason: "no HTML files in the tree"} end), + echidnabot: (if $proofs then {title: "echidnabot (formal verification)", enrolled: true} + else {enrolled: false, reason: "no proof files in the tree"} end), + sustainabot: {enrolled: false, reason: "reserved fleet slot; no adapter exists"}}}' > "$MANIFEST" + echo " CREATED .gitbot-fleet/enrolment.json (rust=$has_rust seams=$has_seams html=$has_html proofs=$has_proofs; edit it — enrol what is true of this repository)" +fi + +# ── 2. report-backs directory ──────────────────────────────────────────────── +if [ -f "$RB_README" ]; then + echo " ok: .gitbot-fleet/report-backs/README.md exists" +else + mkdir -p "$(dirname "$RB_README")" + cat > "$RB_README" <<'EOF' + +# Fleet report-backs + +Empty on purpose. The `fleet` job of `.github/workflows/fork-governance.yml` +writes one `.json` envelope per enrolled gitbot-fleet member into an +artifact at run time, and the Hypatia lane downloads it here before rendering +the verdict grid. Nothing in this directory is committed: a committed envelope +would pose as evidence about a head it was not produced from, and the lane +clears the directory before the download for that reason. + +Envelope contract: `docs/VERDICT-GRID-SPEC.adoc` in standards. Producer: +`docs/REPORT-BACK.adoc` in hyperpolymath/gitbot-fleet. +EOF + echo " CREATED .gitbot-fleet/report-backs/README.md" +fi + +# ── 3. the fleet job ───────────────────────────────────────────────────────── +job_inserted=false +if [ ! -f "$WORKFLOW" ]; then + manual "no .github/workflows/fork-governance.yml; adopt fork governance first (standards 3-practice/fork-governance/ADOPTION.adoc), then rerun" +elif grep -qE '^ fleet:[[:space:]]*$' "$WORKFLOW"; then + # A job merely named fleet is not the producer: it must call the reusable or + # upload the fleet-report-backs artifact, else hypatia would wait on the + # wrong job and the artifact would never exist. + if awk '/^ fleet:[[:space:]]*$/ {p=1; next} p && /^ [A-Za-z_][A-Za-z0-9_-]*:/ {exit} p' "$WORKFLOW" | grep -qE 'fleet-report-back-reusable\.yml@|name: fleet-report-backs'; then + echo " ok: fork-governance.yml already has the fleet report-back job" + else + manual "fork-governance.yml has a job named fleet that is not the report-back producer (no fleet-report-back-reusable.yml call and no fleet-report-backs upload); rename it or add the producer by hand (docs/REPORT-BACK.adoc §5)" + fi +elif ! grep -qE '^ hypatia:[[:space:]]*$' "$WORKFLOW"; then + manual "fork-governance.yml has no 'hypatia:' job at two-space indent to insert the fleet job before; add the job by hand (docs/REPORT-BACK.adoc §5)" +else + jobtmp=$(mktemp) + # The template's SPDX/explanatory header lines are for the template file. + sed -e '/^# /d' -e "s/__FLEET_REF__/$fleet_ref/g" -e "s/__PANIC_ATTACK_SHA__/$panic_sha/g" "$JOB_TEMPLATE" > "$jobtmp" + if grep -q '__[A-Z_]*__' "$jobtmp"; then + rm -f "$jobtmp"; echo "fix-fleet-enrolment: template $JOB_TEMPLATE left a placeholder unfilled; workflow not changed" >&2; exit 2 + fi + out=$(mktemp) + awk -v tmpl="$jobtmp" ' + BEGIN { while ((getline l < tmpl) > 0) job = job l "\n"; close(tmpl) } + /^ hypatia:[[:space:]]*$/ && !done { printf "%s", job; done = 1 } + { print }' "$WORKFLOW" > "$out" + pin_fixer_replace "$out" "$WORKFLOW"; rm -f "$jobtmp" + job_inserted=true + echo " INSERTED fleet job ($form form) before the hypatia job in fork-governance.yml" +fi + +# ── 3b. hypatia runs after the fleet job ───────────────────────────────────── +# Does a flow-list `needs: [a, b]` line name $2 as an exact item? `fleet-scan` is +# not `fleet`: a word-boundary match would call it present and never append. +needs_has_item() { # needs-line item + printf "%s\n" "$1" | sed -E "s/^[^[]*\[//; s/\].*$//" | tr "," "\n" | sed -E "s/^[[:space:]]+//; s/[[:space:]]+$//; s/^[\x27\"]//; s/[\x27\"]$//" | grep -qx -- "$2" +} +# The hypatia block: from its header to the next two-space job header. +hypatia_block() { + awk '/^ hypatia:[[:space:]]*$/ {p=1; print; next} p && /^ [A-Za-z_][A-Za-z0-9_-]*:/ {exit} p {print}' "$WORKFLOW" +} +if [ -f "$WORKFLOW" ] && grep -qE '^ hypatia:[[:space:]]*$' "$WORKFLOW"; then + block=$(hypatia_block) + needs_line=$(printf '%s\n' "$block" | grep -m1 -E '^ needs:' || true) + if [ -z "$needs_line" ]; then + # No needs: add needs and, when the job has no if: of its own, the + # !cancelled() guard (always() would let a superseded run post a stale grid). + add=' needs: [fleet]' + # shellcheck disable=SC2016 # the ${{ }} is GitHub's expression syntax, written literally + printf '%s\n' "$block" | grep -qE '^ if:' || add="$add"$'\n'' if: ${{ !cancelled() }}' + out=$(mktemp) + awk -v add="$add" '{ print } /^ hypatia:[[:space:]]*$/ && !done { print add; done = 1 }' "$WORKFLOW" > "$out" + pin_fixer_replace "$out" "$WORKFLOW" + echo " WIRED hypatia: needs: [fleet]" + elif needs_has_item "$needs_line" fleet || [[ "$needs_line" =~ ^\ \ \ \ needs:[[:space:]]*fleet[[:space:]]*$ ]]; then + echo " ok: hypatia already needs fleet" + elif [[ "$needs_line" =~ ^\ \ \ \ needs:[[:space:]]*([A-Za-z0-9_-]+)[[:space:]]*$ ]]; then + # The scalar form `needs: other` becomes a flow list with fleet appended. + other="${BASH_REMATCH[1]}" + out=$(mktemp) + awk -v other="$other" '!done && /^ hypatia:[[:space:]]*$/ {p=1} p && /^ needs:/ { $0 = " needs: [" other ", fleet]"; done = 1; p = 0 } { print }' "$WORKFLOW" > "$out" + pin_fixer_replace "$out" "$WORKFLOW" + echo " WIRED hypatia: needs: [$other, fleet]" + elif printf '%s' "$needs_line" | grep -qE '^ needs:[[:space:]]*\[.*\][[:space:]]*$'; then + out=$(mktemp) + awk '!done && /^ hypatia:[[:space:]]*$/ {p=1} p && /^ needs:[[:space:]]*\[/ { if ($0 ~ /\[[[:space:]]*\]/) sub(/\[[[:space:]]*\]/, "[fleet]"); else sub(/\]/, ", fleet]"); done = 1; p = 0 } { print }' "$WORKFLOW" > "$out" + pin_fixer_replace "$out" "$WORKFLOW" + echo " WIRED hypatia: fleet added to its needs list" + else + manual "hypatia's needs: is a block list; add '- fleet' to it by hand" + fi + if printf '%s\n' "$block" | grep -qE '^ if:.*always\(\)'; then + echo " NOTE: hypatia runs under always(); with cancel-in-progress a superseded run can post a stale grid — prefer \${{ !cancelled() }}" + fi +fi + +# A job inserted into a workflow with a lock needs the lock regenerated; the +# fixer never edits actions.lock (tool-owned). +if [ "$job_inserted" = true ] && [ -f "$LOCK" ]; then + manual "actions.lock exists; run 'gh actions-lock' (or the repository's check-lock-sync.sh) so the new uses: entries are locked" +fi +if [ "$job_inserted" = true ] && command -v yq >/dev/null 2>&1; then + if ! yq -e '.jobs.fleet' "$WORKFLOW" >/dev/null 2>&1; then + manual "fork-governance.yml does not parse with a fleet job after the edit; inspect it" + fi +fi + +# ── 4. does the pinned Hypatia lane read the artifact? ─────────────────────── +# A reusable reads the artifact when it both takes the report-backs-artifact +# input and has a download-artifact step: the input name alone (or a comment +# mentioning it) places no envelope where the renderer looks. +reads_artifact() { # stdin: a workflow + local c; c=$(cat) + printf "%s\n" "$c" | grep -q "report-backs-artifact" && printf "%s\n" "$c" | grep -qE "uses:[[:space:]]*actions/download-artifact@" +} +if [ -f "$WORKFLOW" ]; then + uses=$(hypatia_block | grep -m1 -E '^ uses:' | sed -E 's/^ uses:[[:space:]]*//; s/[[:space:]]+#.*$//' || true) + case "$uses" in + '') + manual "hypatia job has no uses: line; cannot tell whether the lane reads the fleet-report-backs artifact" ;; + ./*) + local_reusable="$REPO_PATH/.github/workflows/${uses#./.github/workflows/}" + if [ -f "$local_reusable" ] && reads_artifact < "$local_reusable"; then + echo " ok: local Hypatia reusable reads the fleet-report-backs artifact" + else + manual "local Hypatia reusable ($uses) has no report-backs-artifact input with a download-artifact step; the artifact is produced but never read" + fi ;; + */standards/.github/workflows/hypatia-scan-reusable.yml@*) + sha="${uses##*@}" + if [ -n "${STANDARDS_DIR:-}" ] && git -C "$STANDARDS_DIR" cat-file -e "$sha:.github/workflows/hypatia-scan-reusable.yml" 2>/dev/null; then + if git -C "$STANDARDS_DIR" cat-file -p "$sha:.github/workflows/hypatia-scan-reusable.yml" | reads_artifact; then + echo " ok: standards pin ${sha:0:12} reads the fleet-report-backs artifact" + else + manual "standards pin ${sha:0:12} does not read the artifact; bump the standards pins (standards scripts/propagate-workflow-pins.sh --fix --to ) in this same branch" + fi + else + unverified "standards pin ${sha:0:12}: set STANDARDS_DIR to a standards checkout holding that commit to confirm its Hypatia reusable reads the artifact" + fi ;; + *) + manual "hypatia job uses '$uses', which this fixer cannot read; confirm by hand that the lane takes report-backs-artifact" ;; + esac +fi + +echo +if [ "$rc" -eq 0 ]; then echo "Fleet enrolment: wired end to end"; else echo "Fleet enrolment: not yet wired end to end (see MANUAL/UNVERIFIED lines)"; fi +exit "$rc" diff --git a/scripts/fix-script-registry.json b/scripts/fix-script-registry.json index 473baa6d..8f35a496 100644 --- a/scripts/fix-script-registry.json +++ b/scripts/fix-script-registry.json @@ -47,6 +47,7 @@ "MissingProvenRef": "fix-missing-proven-ref.sh", "MissingVerisimdbFeed": "fix-missing-verisimdb-feed.sh", "MissingFeedbackIntegration": "fix-missing-feedback-integration.sh", + "MissingFleetEnrolment": "fix-fleet-enrolment.sh", "MissingVexometerHooks": "fix-missing-vexometer-hooks.sh", "TrackedNpmLockfile": "fix-tracked-package-lock.sh", "LicensePMPLDrift": "fix-pmpl-drift.sh", @@ -107,6 +108,7 @@ "recipe-add-proven-ref": "fix-missing-proven-ref.sh", "recipe-add-verisimdb-feed": "fix-missing-verisimdb-feed.sh", "recipe-add-feedback-integration": "fix-missing-feedback-integration.sh", + "recipe-add-fleet-enrolment": "fix-fleet-enrolment.sh", "recipe-add-vexometer-hooks": "fix-missing-vexometer-hooks.sh", "recipe-remove-tracked-package-lock": "fix-tracked-package-lock.sh", "cicd_rules/tracked_npm_lockfile": "fix-tracked-package-lock.sh", diff --git a/scripts/fleet-report-back-build.sh b/scripts/fleet-report-back-build.sh new file mode 100755 index 00000000..790c6ad1 --- /dev/null +++ b/scripts/fleet-report-back-build.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# fleet-report-back-build.sh — build the adapter binaries a report-back run +# needs, into one directory the driver can be pointed at with --bin-dir. +# +# One script, so the reusable workflow and the inline callers (repositories +# whose Actions policy refuses a reusable from another owner) build the same +# way. Per crate: `cargo build --release --locked`. A crate that fails to build +# is reported and skipped, never fatal: the driver then writes that member's +# envelope as `error: adapter binary ... not found`, and the build log holds +# the reason. Failing here would turn every OTHER member's evidence into a +# missing artifact. +# +# panic-attack (for panicbot) is a `cargo install --git` at a pinned commit, +# cached by the caller under $HOME/.panic-attack and reused only when the +# recorded source commit matches (the same recipe as the standards Hypatia +# lane, so both panic-attack-derived rows describe one tool). +# +# Usage: +# fleet-report-back-build.sh --fleet DIR --bin-dir DIR [--crates "a b c"] +# [--panic-attack SHA] [--build-timeout SECONDS] +# [--build-budget SECONDS] +# +# --fleet a gitbot-fleet checkout (bots//Cargo.toml) +# --bin-dir where the built binaries are copied (created) +# --crates crate directory names under bots/, space-separated; +# default: read `fleet-report-back.sh --plan` output on stdin +# --panic-attack also install panic-attack at this commit into --bin-dir +# --build-timeout seconds per cargo invocation (default 900) +# --build-budget seconds for every build together (default 2400). A crate +# whose turn comes after the budget is spent is not built +# (warned; its row reads error) so one job timeout cannot +# swallow the envelopes of the members that did build. +# +# Exit 0 when the script ran to completion (whatever built); 2 on usage error. + +set -euo pipefail + +fleet="" bindir="" crates="" pa_sha="" build_timeout=900 build_budget=2400 +while [ $# -gt 0 ]; do + case "$1" in + --fleet) fleet="$2"; shift 2 ;; + --bin-dir) bindir="$2"; shift 2 ;; + --crates) crates="$2"; shift 2 ;; + --panic-attack) pa_sha="$2"; shift 2 ;; + --build-timeout) build_timeout="$2"; shift 2 ;; + --build-budget) build_budget="$2"; shift 2 ;; + *) echo "unknown option: $1" >&2; exit 2 ;; + esac +done +if [ -z "$fleet" ] || [ -z "$bindir" ]; then + echo "usage: $0 --fleet DIR --bin-dir DIR [--crates \"a b\"] [--panic-attack SHA]" >&2; exit 2 +fi +[ -d "$fleet/bots" ] || { echo "not a gitbot-fleet checkout: $fleet" >&2; exit 2; } +mkdir -p "$bindir"; bindir="$(cd "$bindir" && pwd)" +command -v cargo >/dev/null 2>&1 || { echo "::warning title=no cargo::cargo is not on PATH; no adapter can be built" >&2; exit 0; } + +if [ -z "$crates" ] && [ ! -t 0 ]; then + # From `fleet-report-back.sh --plan`: membercratebinary + crates=$(awk -F'\t' '$2 != "" {print $2}' | sort -u | tr '\n' ' ') +fi + +# Print a group heading for $1, using Actions grouping when GITHUB_ACTIONS is nonempty. +group() { if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "::group::$1"; else echo "== $1"; fi; } +# Close an Actions log group; emit nothing when GITHUB_ACTIONS is empty or unset. +endgroup() { if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "::endgroup::"; fi; } +# Report title $1 and message $2 as an Actions warning, or plain text to stderr. +warn() { if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "::warning title=$1::$2"; else echo "warning: $1: $2" >&2; fi; } +# Seconds of the build budget still unspent (never negative). SECONDS is bash's +# monotonic counter since the script started. +remaining() { local r=$((build_budget - SECONDS)); [ "$r" -gt 0 ] && echo "$r" || echo 0; } +# The timeout for the next build: the per-build limit $1, capped by the budget. +slice() { local r; r=$(remaining); [ "$1" -lt "$r" ] && echo "$1" || echo "$r"; } + +built=0 failed=0 +for c in $crates; do + if ! [[ "$c" =~ ^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$ ]]; then + warn "unknown crate" "$(printf '%q' "$c") is not a crate directory name; skipped"; failed=$((failed + 1)); continue + fi + if [ ! -f "$fleet/bots/$c/Cargo.toml" ]; then + warn "unknown crate" "bots/$c has no Cargo.toml; skipped"; failed=$((failed + 1)); continue + fi + if [ "$(remaining)" -eq 0 ]; then + warn "build budget exhausted" "bots/$c not built: the ${build_budget}s build budget was spent on earlier crates; its row will read error"; failed=$((failed + 1)); continue + fi + group "cargo build --release --locked (bots/$c)" + if (cd "$fleet/bots/$c" && timeout "$(slice "$build_timeout")" cargo build --release --locked 2>&1); then + # Every executable the crate produced; the driver picks the one it needs. + find "$fleet/bots/$c/target/release" -maxdepth 1 -type f -perm -u+x ! -name '*.d' \ + -exec cp -t "$bindir" {} + + built=$((built + 1)) + else + warn "adapter build failed" "bots/$c did not build; its row will read error" + failed=$((failed + 1)) + fi + endgroup +done + +if [ -n "$pa_sha" ]; then + root="$HOME/.panic-attack"; bin="$root/bin/panic-attack" + group "panic-attack at $pa_sha" + if [ -x "$bin" ] && grep -qF "#$pa_sha" "$root/.crates.toml" 2>/dev/null; then + echo "panic-attack restored from cache at $pa_sha" + elif [ "$(remaining)" -eq 0 ]; then + warn "panic-attack not installed" "the ${build_budget}s build budget was spent before its turn; panicbot will read error" + elif ! timeout "$(slice 600)" cargo install --git https://github.com/hyperpolymath/panic-attack \ + --rev "$pa_sha" --locked --force --root "$root" panic-attack; then + rm -rf "$root" # leave nothing for the cache to save from a failed build + warn "panic-attack not installed" "source build at $pa_sha failed or exceeded 10 minutes; panicbot will read error" + fi + if [ -x "$bin" ]; then cp "$bin" "$bindir/"; fi + endgroup +fi + +echo "fleet-report-back-build: $built crate(s) built, $failed failed; binaries in $bindir:" +find "$bindir" -maxdepth 1 -type f -printf ' %f\n' | sort +exit 0 diff --git a/scripts/fleet-report-back.sh b/scripts/fleet-report-back.sh new file mode 100755 index 00000000..2ff1374c --- /dev/null +++ b/scripts/fleet-report-back.sh @@ -0,0 +1,432 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# fleet-report-back.sh — run every fleet member a repository has enrolled and +# write one report-back envelope per member. +# +# The Hypatia verdict grid (standards: docs/VERDICT-GRID-SPEC.adoc, rendered by +# scripts/verdict-grid.sh) gives each gitbot-fleet member named in the scanned +# repository's `.gitbot-fleet/enrolment.json` one row. The row stays orange +# ("enrolled, but no report-back was received") until a file +# `.gitbot-fleet/report-backs/.json` is present when the grid renders. +# Nothing in the estate produced those files before this script existed, which +# is why every fleet row on every PR read orange. +# +# This script is the producer. It is deliberately a thin shell driver over the +# adapters' EXISTING local-directory JSON modes: it adds no analysis of its own, +# it only translates each adapter's output into the envelope the grid reads: +# +# { "reporter": "", "title": "", "status": "ran", +# "reason": null, "ref": "<gitbot-fleet commit>", +# "findings": [ { "rule_id", "severity", "file", "line", "message" } ] } +# +# status is one of ran | not_run | error | not_applicable; reason is required +# unless the member ran. Severity words are the fleet's own (error, warning, +# info, suggestion) or the scanner words (critical, high, medium, low); the +# grid normalises both. File paths are made relative to the repository root so +# the grid's change-scope (delta) grading can match them against the diff. +# +# Honesty rules, in order of precedence: +# * A member whose adapter is missing or crashed is `error` with the reason, +# never silently skipped: an enrolled member that stays silent is exactly +# what the orange row exists to report, so the reason must be stated. +# * A member that cannot run in this environment (echidnabot without an +# ECHIDNA engine, the reserved sustainabot slot) is `not_run` with the +# reason. It renders orange, like silence, but the comment now says WHY. +# * `not_applicable` is a statement about the REPOSITORY and belongs in the +# enrolment manifest (`enrolled: false`, with a reason), not here. This +# script never writes it. +# * Members marked `enrolled: false` are not run and get no file: the grid +# renders them grey from the manifest alone. +# * Nothing is ever written into the scanned repository. The envelopes go to +# --out, which the workflow uploads as an artifact; the lane places them +# under .gitbot-fleet/report-backs/ at render time. A committed envelope +# would be stale evidence, so none is ever committed. +# +# Usage: +# fleet-report-back.sh --repo DIR --out DIR [--enrolment FILE] [--bin-dir DIR] +# [--member NAME]... [--fleet-ref SHA] [--summary FILE] +# [--timeout SECONDS] +# fleet-report-back.sh --plan [--enrolment FILE] [--member NAME]... +# +# --repo the checkout to report on (its root; paths are made relative) +# --out directory receiving <member>.json (created) +# --enrolment manifest; default <repo>/.gitbot-fleet/enrolment.json +# --bin-dir directory searched for adapter binaries before PATH +# --member run only these members (repeatable); they must be enrolled +# unless --enrolment is explicitly "" (no manifest) +# --fleet-ref gitbot-fleet commit recorded as `ref`; default: this +# checkout's HEAD, else "unknown" +# --summary append a Markdown table of the members and their status +# --timeout seconds allowed per adapter (default 300) +# --plan print "member<TAB>crate-dir<TAB>binary" for every member +# that would run, so a workflow can build only those crates +# +# Exit 0 whenever envelopes were written (whatever they say); 2 on usage +# error. An adapter failing never fails this script: the failure is the +# envelope's content. + +set -euo pipefail + +FLEET_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# Print the usage text to stderr and terminate the script with status 2. +usage() { + sed -n '/^# Usage:/,/^# error\./p' "$0" | sed 's/^# \{0,1\}//' >&2 + exit 2 +} + +# ── member table ───────────────────────────────────────────────────────────── +# member -> crate directory under bots/ (empty: no adapter) and binary name. +crate_dir_of() { + case "$1" in + rhodibot) echo rhodibot ;; + panicbot) echo panicbot ;; + seambot) echo seambot ;; + finishingbot|finishbot|finishing-bot) echo finishingbot ;; + glambot) echo glambot ;; + accessibilitybot) echo accessibilitybot ;; + cipherbot) echo cipherbot ;; + echidnabot) echo "" ;; # no local mode: never built for a report-back + sustainabot) echo "" ;; + *) echo "" ;; + esac +} +# Print the adapter executable name for a member, or an empty line if unknown +# or unsupported locally. The finishingbot aliases all use finishing-bot. +binary_of() { + case "$1" in + finishingbot|finishbot|finishing-bot) echo finishing-bot ;; + rhodibot|panicbot|seambot|glambot|accessibilitybot|cipherbot) echo "$1" ;; + *) echo "" ;; + esac +} +# Print the built-in display title, falling back to the member name. +default_title_of() { + case "$1" in + rhodibot) echo "rhodibot (RSR compliance)" ;; + panicbot) echo "panicbot (panic recovery, error handling)" ;; + echidnabot) echo "echidnabot (formal verification)" ;; + seambot) echo "seambot (integration health)" ;; + finishingbot) echo "finishingbot (release readiness)" ;; + accessibilitybot) echo "accessibilitybot (WCAG)" ;; + glambot) echo "glambot (presentation quality)" ;; + cipherbot) echo "cipherbot (cryptographic hygiene)" ;; + sustainabot) echo "sustainabot (eco/econ)" ;; + *) echo "$1" ;; + esac +} + +# ── argument parsing ───────────────────────────────────────────────────────── +repo="" out="" enrol="__default__" bindir="" fleet_ref="" summary="" plan=false +timeout_s=300 +members=() +while [ $# -gt 0 ]; do + case "$1" in + --repo) repo="$2"; shift 2 ;; + --out) out="$2"; shift 2 ;; + --enrolment) enrol="$2"; shift 2 ;; + --bin-dir) bindir="$2"; shift 2 ;; + --member) members+=("$2"); shift 2 ;; + --fleet-ref) fleet_ref="$2"; shift 2 ;; + --summary) summary="$2"; shift 2 ;; + --timeout) timeout_s="$2"; shift 2 ;; + --plan) plan=true; shift ;; + -h|--help) usage ;; + *) echo "unknown option: $1" >&2; usage ;; + esac +done + +if [ "$plan" = false ]; then + if [ -z "$repo" ] || [ -z "$out" ]; then usage; fi + [ -d "$repo" ] || { echo "not a directory: $repo" >&2; exit 2; } + repo="$(cd "$repo" && pwd)" +fi +if [ "$enrol" = "__default__" ]; then + if [ -n "$repo" ]; then enrol="$repo/.gitbot-fleet/enrolment.json"; else enrol=""; fi +fi +command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 2; } +# GNU timeout reads 0 as "no limit", which would let one hung adapter spend the +# whole job; the limit must be a positive integer. +[[ "$timeout_s" =~ ^[1-9][0-9]*$ ]] || { echo "--timeout must be a positive integer (0 would disable the limit)" >&2; exit 2; } + +# ── enrolment ──────────────────────────────────────────────────────────────── +# Which members run. An explicit --member list wins; otherwise every reporter +# the manifest marks `enrolled: true`. A manifest that cannot be read is NOT +# treated as "nobody enrolled": the grid already renders that as one orange +# row, so this script runs nothing and says so. +enrolment_json='null' +if [ -n "$enrol" ]; then + if [ -s "$enrol" ] && jq -e 'type == "object" and (.reporters | type) == "object"' "$enrol" >/dev/null 2>&1; then + enrolment_json=$(jq -c . "$enrol") + elif [ -e "$enrol" ]; then + echo "warning: enrolment manifest unreadable: $enrol" >&2 + enrolment_json='"unreadable"' + fi +fi + +if [ "${#members[@]}" -eq 0 ]; then + if [ "$enrolment_json" = 'null' ] || [ "$enrolment_json" = '"unreadable"' ]; then + members=() + else + mapfile -t members < <(jq -r '.reporters | to_entries[] | select(.value.enrolled == true) | .key' <<<"$enrolment_json") + fi +else + # Honour the manifest even when members are named explicitly: a member the + # repository marked not enrolled is grey by its own statement, and running + # it anyway would contradict that statement in the same comment. + if [ "$enrolment_json" != 'null' ] && [ "$enrolment_json" != '"unreadable"' ]; then + kept=() + for m in "${members[@]}"; do + if jq -e --arg m "$m" '.reporters[$m].enrolled == false' <<<"$enrolment_json" >/dev/null 2>&1; then + echo "note: $m is marked not enrolled in $enrol; skipped" >&2 + else + kept+=("$m") + fi + done + members=("${kept[@]+"${kept[@]}"}") + fi +fi + +# A member name comes from the scanned repository's manifest, so it is untrusted: +# it becomes a tab-delimited plan field, an envelope file name and an adapter +# argument. Only a slug may pass; anything else is dropped with a warning (a +# traversal or a tab in a key must never reach the build or the output tree). +slugs=() +for m in "${members[@]+"${members[@]}"}"; do + if [[ "$m" =~ ^[a-z][a-z0-9-]{0,63}$ ]]; then slugs+=("$m") + else echo "warning: ignoring member $(printf '%q' "$m"): a member name is lowercase letters, digits and hyphens" >&2; fi +done +members=("${slugs[@]+"${slugs[@]}"}") + +if [ "$plan" = true ]; then + for m in "${members[@]+"${members[@]}"}"; do + printf '%s\t%s\t%s\n' "$m" "$(crate_dir_of "$m")" "$(binary_of "$m")" + done + exit 0 +fi + +mkdir -p "$out" +out="$(cd "$out" && pwd)" + +if [ -z "$fleet_ref" ]; then + fleet_ref=$(git -C "$FLEET_DIR" rev-parse HEAD 2>/dev/null || echo unknown) +fi + +# Print the member's nonempty manifest title, or its default display title. +title_of() { + local m="$1" t="" + if [ "$enrolment_json" != 'null' ] && [ "$enrolment_json" != '"unreadable"' ]; then + t=$(jq -r --arg m "$m" '.reporters[$m].title // empty' <<<"$enrolment_json") + fi + # The title rides into jq as an argument; a manifest is PR-controlled, so it is + # bounded here (120 characters) rather than trusted to fit the command line. + if [ -n "$t" ]; then echo "${t:0:120}"; else default_title_of "$m"; fi +} + +# Print the executable found in bindir first, then PATH; return 1 if absent. +find_binary() { + local name="$1" + if [ -n "$bindir" ] && [ -x "$bindir/$name" ]; then echo "$bindir/$name"; return 0; fi + command -v "$name" 2>/dev/null || return 1 +} + +# ── envelope writers ───────────────────────────────────────────────────────── +# Every envelope goes through one writer so the shape cannot drift between +# members. Output is key-sorted, compact, one final LF: the I-JSON/JCS shape +# the estate's JSON policy asks of every .json it writes (standards +# 3-practice/JSON-POLICY.adoc), as far as jq can produce it. +# Write or replace out/<member>.json using the supplied status, reason, and +# findings JSON array. An empty reason becomes null; title and ref come from +# the current run. Temporary-file, JSON, and output I/O failures are not caught. +write_envelope() { # member status reason findings_json + local m="$1" status="$2" reason="$3" findings="$4" ftmp + # The findings go in through a FILE, never argv: a few hundred findings + # exceed ARG_MAX and jq dies with "Argument list too long" (seen live on + # gitbot-fleet's own PR 626 with panicbot's 146 findings). + ftmp=$(mktemp); printf '%s' "$findings" > "$ftmp" + jq -S -c -n --arg reporter "$m" --arg title "$(title_of "$m")" --arg status "$status" \ + --arg reason "$reason" --arg ref "$fleet_ref" --slurpfile findings "$ftmp" \ + '{reporter: $reporter, title: $title, status: $status, + reason: (if $reason == "" then null else $reason end), + ref: $ref, findings: $findings[0]}' > "$out/$m.json" + rm -f "$ftmp" +} + +# Emit a jq rel filter that strips the repo/ prefix and then one leading ./; +# nonstrings become empty strings and other paths are unchanged. Adapters differ: +# accessibilitybot prints absolute paths, glambot and finishing-bot "./x", +# seambot and rhodibot repository-relative. The grid matches paths against +# `git diff --name-only`, which is repository-relative with no "./". +relativize_filter() { # jq filter text; expects $repo bound + cat <<'JQ' +def rel: if type != "string" then "" else + (if startswith($repo + "/") then .[($repo | length) + 1:] else . end) + | (if startswith("./") then .[2:] else . end) end; +JQ +} + +# Run an adapter, capturing stdout/stderr separately. Non-zero exit with valid +# JSON on stdout is still a run: several adapters exit 1 when their findings +# would block a release, and that is a finding, not a failure. +# Run in repo with timeout_s seconds allowed. Set ADAPTER_OUT and ADAPTER_ERR +# to temporary capture files and ADAPTER_RC to the exit status (124 on timeout) +# without propagating it. The caller must remove the files; creation failures +# are not caught. +run_adapter() { # binary args... ; sets ADAPTER_OUT, ADAPTER_ERR, ADAPTER_RC + local bin="$1"; shift + ADAPTER_OUT=$(mktemp); ADAPTER_ERR=$(mktemp); ADAPTER_RC=0 + # --bin-dir is also prepended to PATH: panicbot finds panic-attack with `which`. + ( cd "$repo" && PATH="${bindir:+$bindir:}$PATH" RUST_LOG=off timeout "$timeout_s" "$bin" "$@" ) > "$ADAPTER_OUT" 2> "$ADAPTER_ERR" || ADAPTER_RC=$? +} +# Print up to 240 characters of the first nonblank ADAPTER_ERR line excluding +# Stack backtrace headers, or an ADAPTER_RC diagnostic if no line is selected. +first_err_line() { + local l + l=$(grep -v '^\s*$' "$ADAPTER_ERR" 2>/dev/null | grep -v '^Stack backtrace' | head -n1 | cut -c1-240) + [ -n "$l" ] && echo "$l" || echo "exit status $ADAPTER_RC with no diagnostic" +} +# Remove the most recent adapter's stdout and stderr capture files. +cleanup_adapter() { rm -f "$ADAPTER_OUT" "$ADAPTER_ERR"; } + +# Translate when stdout parses with the given jq program (which must yield the +# findings array), accepting nonzero adapter exits with usable output. Exit +# status 124 always produces a timeout error; invalid output produces an error +# with a stderr diagnostic. Bind $repo for relativize_filter, write the member's +# envelope, and remove the capture files. Envelope I/O failures are not caught. +translate_or_error() { # member jq-program + local m="$1" prog="$2" findings + if [ "$ADAPTER_RC" -eq 124 ]; then + write_envelope "$m" error "adapter exceeded the ${timeout_s}s timeout" '[]' + elif findings=$(jq -c --arg repo "$repo" "$(relativize_filter) $prog" "$ADAPTER_OUT" 2> "$ADAPTER_ERR.jq") \ + && [ -n "$findings" ] && jq -e 'type == "array"' <<<"$findings" >/dev/null 2>&1; then + write_envelope "$m" ran "" "$findings" + elif grep -q 'unexpected output shape' "$ADAPTER_ERR.jq" 2>/dev/null; then + # The shape guard fired: valid JSON, but not what this adapter documents. + write_envelope "$m" error "adapter output had an $(grep -o 'unexpected output shape: [^(]*' "$ADAPTER_ERR.jq" | head -n1 | sed 's/ *$//')" '[]' + else + write_envelope "$m" error "adapter produced no readable JSON: $(first_err_line)" '[]' + fi + rm -f "$ADAPTER_ERR.jq" + cleanup_adapter +} + +# Every adapter program begins with its shape guard: a crash or a regression that +# prints `{}` or `42` must become an error envelope, never a clean scan with no +# findings (the per-member filters would otherwise default a missing array to []). +SHAPE_ARRAY='if type != "array" then error("unexpected output shape: expected an array") else . end | ' +SHAPE_OBJ_FINDINGS='if type != "object" or (.findings | type) != "array" then error("unexpected output shape: expected an object with a findings array") else . end | ' +# finishing-bot and glambot print their analyzers keyed by name (an object, or +# an array), each with a findings array; an empty set means the run died. +SHAPE_ANALYZERS='if (type == "object" or type == "array") and length > 0 and all(.[]; type == "object" and (.findings | type) == "array") then . else error("unexpected output shape: expected analyzer objects each with a findings array") end | ' + +# One finding shape for every adapter: { rule_id, severity, file, line, message } +FINDING='{rule_id: (.rule_id // .id // "?" | tostring), severity: (.severity // "unknown" | tostring), + file: ((.file // .location // "") | rel), line: ((.line // 0) | if type == "number" then . else 0 end), + message: (.message // "" | tostring)}' + +# Run one member against repo and write its translated report-back envelope. +# echidnabot and sustainabot yield not_run, even with ECHIDNA_URL configured; +# unknown members, unavailable binaries, timeouts, and unusable output yield +# error envelopes. Envelope I/O failures are not converted to adapter errors. +run_member() { + local m="$1" bin crate + crate=$(crate_dir_of "$m") + case "$m" in + echidnabot) + # echidnabot dispatches proofs to an ECHIDNA engine; it has no local + # mode. Without an engine the honest row is not_run WITH the reason. + if [ -z "${ECHIDNA_URL:-}" ]; then + write_envelope "$m" not_run "no ECHIDNA engine endpoint is configured for this repository (ECHIDNA_URL unset); echidnabot verifies proofs by dispatching to an engine and has no local mode" '[]' + else + write_envelope "$m" not_run "ECHIDNA_URL is set, but this driver does not yet dispatch to an engine; the row turns green or red once that leg lands in gitbot-fleet" '[]' + fi + return ;; + sustainabot) + write_envelope "$m" not_run "reserved fleet slot: no sustainabot adapter exists (bots/sustainabot/README.adoc); OikosBot is a separate project" '[]' + return ;; + esac + if [ -z "$crate" ]; then + write_envelope "$m" error "unknown fleet member: no adapter is registered under this name" '[]' + return + fi + if ! bin=$(find_binary "$(binary_of "$m")"); then + write_envelope "$m" error "adapter binary '$(binary_of "$m")' not found (searched ${bindir:-<no --bin-dir>} and PATH); build bots/$crate" '[]' + return + fi + case "$m" in + rhodibot) + # canon --path: file-presence criteria against the vendored RSR canon. + # "satisfied" rows are not findings; missing > deprecated > relocated. + run_adapter "$bin" canon --path "$repo" --format json + translate_or_error "$m" "$SHAPE_OBJ_FINDINGS"' + [ .findings[] + | select(.severity != "satisfied") + # A relocated or deprecated file is reported at the path where it WAS + # found, so the grid can match it against the PR diff; a missing one + # can only be named by where the canon keeps it. + | { rule_id: .id, + file: ((if .severity == "missing" then (.canon_keeps_it_at // (.expected // [])[0] // "") + else ((.found // [])[0] // .canon_keeps_it_at // (.expected // [])[0] // "") end) | tostring), + line: 0, + severity: ({missing: "error", deprecated: "warning", relocated: "info"}[.severity] // .severity), + message: ("\(.name): \(.severity)" + (if (.details // []) | length > 0 then " — " + ((.details // []) | join("; ")) else "" end)) } + | .file |= rel ]' ;; + panicbot) + # `scan` translates panic-attack output into fleet Findings and writes + # nothing. (`fleet` mode would write a .panicbot/*.a2ml debt register + # into the scanned tree; A2ML is retired, so it is not used.) + run_adapter "$bin" --format json scan "$repo" + translate_or_error "$m" "$SHAPE_ARRAY [ .[] | $FINDING ]" ;; + cipherbot) + run_adapter "$bin" --format json scan "$repo" + translate_or_error "$m" "$SHAPE_ARRAY [ .[] | $FINDING ]" ;; + seambot) + run_adapter "$bin" --path "$repo" --format json check + translate_or_error "$m" "$SHAPE_OBJ_FINDINGS"' + [ .findings[] + | { rule_id: ("SEAM-" + (.check // "?" | tostring)), severity: (.severity // "unknown" | tostring), + file: ((.location // "") | rel), line: 0, message: (.message // "" | tostring) } ]' ;; + finishingbot|finishbot|finishing-bot) + run_adapter "$bin" --path "$repo" --format json audit + translate_or_error "$m" "$SHAPE_ANALYZERS [ .[] | .findings[] | $FINDING ]" ;; + glambot) + run_adapter "$bin" --path "$repo" --format json audit + translate_or_error "$m" "$SHAPE_ANALYZERS [ .[] | .findings[] | $FINDING ]" ;; + accessibilitybot) + run_adapter "$bin" fleet "$repo" + translate_or_error "$m" "$SHAPE_OBJ_FINDINGS [ .findings[] | $FINDING ]" ;; + *) + write_envelope "$m" error "no invocation is defined for $m" '[]' ;; + esac +} + +if [ "${#members[@]}" -eq 0 ]; then + echo "fleet-report-back: no enrolled members (manifest: ${enrol:-none}); nothing written" >&2 +fi +for m in "${members[@]+"${members[@]}"}"; do + run_member "$m" + echo "fleet-report-back: $m -> $(jq -r '.status + (if .status == "ran" then " (\(.findings | length) finding(s))" else ": " + (.reason // "") end)' "$out/$m.json")" +done + +# ── summary ────────────────────────────────────────────────────────────────── +# Print a Markdown summary of this run from the members' envelope files, or a +# note if no members were selected. Envelope read/parse errors are not caught. +render_summary() { + echo "### gitbot-fleet report-backs" + echo + echo "ref \`$fleet_ref\` · repository \`$repo\` · manifest \`${enrol:-none}\`" + echo + if [ "${#members[@]}" -eq 0 ]; then + echo "_No member is enrolled, so no report-back was produced._" + return + fi + echo "| Member | Status | Findings | Reason |" + echo "|---|---|---:|---|" + for m in "${members[@]}"; do + jq -r '"| \(.reporter) | \(.status) | \(.findings | length) | \(.reason // "") |"' "$out/$m.json" + done +} +if [ -n "$summary" ]; then render_summary >> "$summary"; fi +exit 0 diff --git a/scripts/lib/fleet-job.inline.yml b/scripts/lib/fleet-job.inline.yml new file mode 100644 index 00000000..fb4f601a --- /dev/null +++ b/scripts/lib/fleet-job.inline.yml @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: MPL-2.0 +# fleet-job.inline.yml — the fork-governance `fleet` job fix-fleet-enrolment.sh +# inserts before the Hypatia lane in a repository whose Actions policy cannot +# call a reusable workflow from another owner. __FLEET_REF__ and +# __PANIC_ATTACK_SHA__ are filled in by the fixer. Mirrors rsr-template-repo's +# .github/workflows/fork-governance.yml and this repository's reusable step for +# step (tests/fix-fleet-enrolment-test.sh asserts the step names agree). + fleet: + name: Fleet report-backs + runs-on: ubuntu-24.04 + # Worst case, so a stalled phase never kills the job before the upload: the + # 2400 s build budget + seven adapters at 300 s + checkout, cache and upload. + timeout-minutes: 85 + permissions: + contents: read + env: + # hyperpolymath/gitbot-fleet commit whose driver and adapters run. + FLEET_REF: __FLEET_REF__ + # panic-attack commit for panicbot; equal to the standards Hypatia lane's pin. + PANIC_ATTACK_SHA: __PANIC_ATTACK_SHA__ + steps: + - name: Checkout the repository to report on + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Checkout gitbot-fleet at the pinned commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/gitbot-fleet + ref: ${{ env.FLEET_REF }} + path: .fleet-src + persist-credentials: false + + # The fleet checkout must not sit inside the tree being reported on: walk- + # based adapters would scan the fleet's own source and attribute its findings + # to this repository. actions/checkout only writes under the workspace, so + # the checkout is moved out to RUNNER_TEMP first; the lockfile hash for the + # cache key is taken before the move. + - name: Move the fleet checkout out of the scanned tree + id: fleet + run: | + set -euo pipefail + lockhash=$(cat .fleet-src/bots/*/Cargo.lock .fleet-src/shared-context/Cargo.lock 2>/dev/null | sha256sum | cut -c1-16) + mv .fleet-src "$RUNNER_TEMP/fleet-src" + echo "dir=$RUNNER_TEMP/fleet-src" >> "$GITHUB_OUTPUT" + echo "lockhash=$lockhash" >> "$GITHUB_OUTPUT" + + - name: Plan which adapters to build + id: plan + run: | + set -euo pipefail + plan=$(bash "$RUNNER_TEMP/fleet-src/scripts/fleet-report-back.sh" --plan --enrolment .gitbot-fleet/enrolment.json) + echo "plan:"; printf '%s\n' "$plan" | sed 's/^/ /' + crates=$(printf '%s\n' "$plan" | awk -F'\t' '$2 != "" {print $2}' | sort -u | tr '\n' ' ') + echo "crates=${crates% }" >> "$GITHUB_OUTPUT" + if printf '%s\n' "$plan" | awk -F'\t' '$1 == "panicbot" {f=1} END {exit !f}'; then echo panicbot=true; else echo panicbot=false; fi >> "$GITHUB_OUTPUT" + { echo "workspaces<<WS"; for c in $crates; do echo "$RUNNER_TEMP/fleet-src/bots/$c -> target"; done; echo "$RUNNER_TEMP/fleet-src/shared-context -> target"; echo WS; } >> "$GITHUB_OUTPUT" + if [ -n "${crates% }" ]; then echo build=true; else echo build=false; fi >> "$GITHUB_OUTPUT" + + # Rust comes with the GitHub-hosted runner image; the cache is actions/cache + # keyed on the fleet commit's lockfiles. Only GitHub-owned, SHA-pinned + # actions appear in this job, as in gitbot-fleet's reusable. + - name: Cache adapter builds + if: steps.plan.outputs.build == 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + ${{ runner.temp }}/fleet-src/bots/*/target + key: fleet-report-back-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.fleet.outputs.lockhash }}-${{ steps.plan.outputs.crates }} + restore-keys: | + fleet-report-back-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.fleet.outputs.lockhash }}- + fleet-report-back-v3-${{ runner.os }}-${{ runner.arch }}- + + - name: Report the Rust toolchain in use + if: steps.plan.outputs.build == 'true' || steps.plan.outputs.panicbot == 'true' + run: | + set -euo pipefail + command -v cargo >/dev/null || { echo "::error::cargo is not on this runner; the adapters cannot be built"; exit 1; } + cargo --version; rustc --version + + - name: Restore panic-attack build cache + if: steps.plan.outputs.panicbot == 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.panic-attack + key: panic-attack-v1-${{ runner.os }}-${{ runner.arch }}-${{ env.PANIC_ATTACK_SHA }} + + - name: Build the enrolled adapters + if: steps.plan.outputs.build == 'true' || steps.plan.outputs.panicbot == 'true' + env: + CRATES: ${{ steps.plan.outputs.crates }} + PANICBOT: ${{ steps.plan.outputs.panicbot }} + run: | + set -euo pipefail + args=(--fleet "$RUNNER_TEMP/fleet-src" --bin-dir "$RUNNER_TEMP/fleet-bin" --crates "$CRATES") + [ "$PANICBOT" = true ] && args+=(--panic-attack "$PANIC_ATTACK_SHA") + bash "$RUNNER_TEMP/fleet-src/scripts/fleet-report-back-build.sh" "${args[@]}" + + - name: Run the enrolled adapters and write the envelopes + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/fleet-bin" "$RUNNER_TEMP/report-backs" + bash "$RUNNER_TEMP/fleet-src/scripts/fleet-report-back.sh" --repo . --out "$RUNNER_TEMP/report-backs" \ + --bin-dir "$RUNNER_TEMP/fleet-bin" --enrolment .gitbot-fleet/enrolment.json \ + --fleet-ref "$FLEET_REF" --summary "$GITHUB_STEP_SUMMARY" + ls "$RUNNER_TEMP/report-backs"/*.json >/dev/null 2>&1 \ + || echo "No member is enrolled in .gitbot-fleet/enrolment.json." > "$RUNNER_TEMP/report-backs/NONE-ENROLLED.txt" + + - name: Upload the report-backs + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: fleet-report-backs + path: ${{ runner.temp }}/report-backs + if-no-files-found: error + retention-days: 90 + diff --git a/scripts/lib/fleet-job.reusable.yml b/scripts/lib/fleet-job.reusable.yml new file mode 100644 index 00000000..cd5cf891 --- /dev/null +++ b/scripts/lib/fleet-job.reusable.yml @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: MPL-2.0 +# fleet-job.reusable.yml — the fork-governance `fleet` job fix-fleet-enrolment.sh +# inserts before the Hypatia lane in a repository under the hyperpolymath owner, +# where the gitbot-fleet reusable workflow can be called. __FLEET_REF__ is +# filled in by the fixer. + # Produces .gitbot-fleet/report-backs/*.json as the `fleet-report-backs` + # artifact; the Hypatia lane downloads it before rendering (docs/REPORT-BACK.adoc + # in hyperpolymath/gitbot-fleet). Bump the pin deliberately. + fleet: + name: Fleet report-backs + uses: hyperpolymath/gitbot-fleet/.github/workflows/fleet-report-back-reusable.yml@__FLEET_REF__ + permissions: + contents: read + diff --git a/tests/fix-fleet-enrolment-test.sh b/tests/fix-fleet-enrolment-test.sh new file mode 100755 index 00000000..3142422f --- /dev/null +++ b/tests/fix-fleet-enrolment-test.sh @@ -0,0 +1,407 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Offline fixture test for scripts/fix-fleet-enrolment.sh: the rollout fixer +# that gives a repository the enrolment manifest, the report-backs directory, +# the fork-governance fleet job and the hypatia wiring, and refuses to call +# the result wired until the pinned Hypatia lane reads the artifact. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIXER="$ROOT/scripts/fix-fleet-enrolment.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +export PIN_FIXER_LOCK_DIR="$TMP/locks"; mkdir -m 700 "$PIN_FIXER_LOCK_DIR" + +pass=0; fail=0 +check() { # name expected actual + if [ "$2" = "$3" ]; then pass=$((pass + 1)); echo "ok $1" + else fail=$((fail + 1)); echo "FAIL $1"; echo " want: $2"; echo " got: $3"; fi +} +check_grep() { # name pattern file + if grep -qE -- "$2" "$3"; then pass=$((pass + 1)); echo "ok $1" + else fail=$((fail + 1)); echo "FAIL $1 (no match for '$2' in $3)"; sed 's/^/ | /' "$3" | head -40; fi +} +check_no_grep() { # name pattern file + if grep -qE -- "$2" "$3"; then fail=$((fail + 1)); echo "FAIL $1 (unexpected match for '$2' in $3)" + else pass=$((pass + 1)); echo "ok $1"; fi +} + +REF=$(git -C "$ROOT" rev-parse HEAD) # a commit that carries the producer +PANIC=2222222222222222222222222222222222222222 +export FLEET_REF="$REF" PANIC_ATTACK_SHA="$PANIC" + +# The canonical caller from standards' ADOPTION.adoc §2 (hypatia with no needs). +canonical_workflow() { # standards-sha + cat <<EOF +# SPDX-License-Identifier: MPL-2.0 +name: Fork governance + +on: + pull_request: + +permissions: + contents: read + +jobs: + deed: + name: DEED syntax + uses: hyperpolymath/standards/.github/workflows/deed-conformance.yml@$1 + + hypatia: + name: Hypatia verdict grid + uses: hyperpolymath/standards/.github/workflows/hypatia-scan-reusable.yml@$1 + permissions: + actions: read + contents: read + security-events: write + + verdict: + name: Post the fork governance summary + needs: [deed, hypatia] + if: \${{ always() && github.event_name == 'pull_request' }} + uses: hyperpolymath/standards/.github/workflows/fork-governance-reusable.yml@$1 +EOF +} + +mk_repo() { # dir origin-url + mkdir -p "$1/.github/workflows" + git -C "$1" init -q + git -C "$1" remote add origin "$2" +} + +# A fake standards checkout: one commit whose Hypatia reusable reads the +# artifact, one whose reusable does not. +STD="$TMP/standards"; mkdir -p "$STD/.github/workflows"; git -C "$STD" init -q +git -C "$STD" config user.email t@example.invalid; git -C "$STD" config user.name t +printf 'on:\n workflow_call:\n inputs:\n scan-ref:\n type: string\n' > "$STD/.github/workflows/hypatia-scan-reusable.yml" +git -C "$STD" add -A; git -C "$STD" -c commit.gpgsign=false commit -q -m old +OLD_SHA=$(git -C "$STD" rev-parse HEAD) +printf 'on:\n workflow_call:\n inputs:\n report-backs-artifact:\n type: string\n default: fleet-report-backs\njobs:\n scan:\n steps:\n - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333\n' > "$STD/.github/workflows/hypatia-scan-reusable.yml" +git -C "$STD" add -A; git -C "$STD" -c commit.gpgsign=false commit -q -m new +NEW_SHA=$(git -C "$STD" rev-parse HEAD) +export STANDARDS_DIR="$STD" + +# ── 0. the inline template mirrors the reusable step for step ─────────────── +step_names() { grep -E '^ *- name: ' "$1" | sed -E 's/^ *- name: //'; } +check "inline job template has the reusable's steps, in order" \ + "$(step_names "$ROOT/.github/workflows/fleet-report-back-reusable.yml")" \ + "$(step_names "$ROOT/scripts/lib/fleet-job.inline.yml")" +check_grep "inline template pins the fleet by placeholder" '__FLEET_REF__' "$ROOT/scripts/lib/fleet-job.inline.yml" +check_grep "reusable template pins the fleet by placeholder" '@__FLEET_REF__$' "$ROOT/scripts/lib/fleet-job.reusable.yml" + +# ── 1. hyperpolymath repo, canonical workflow, new standards pin ──────────── +A="$TMP/a"; mk_repo "$A" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" > "$A/.github/workflows/fork-governance.yml"; chmod 644 "$A/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$A" > "$TMP/a.out" 2>&1; rc=$?; set -e +check "A: exit 0 — wired end to end" 0 "$rc" +check_grep "A: manifest created" 'CREATED \.gitbot-fleet/enrolment\.json' "$TMP/a.out" +check "A: manifest is key-sorted compact JSON with one final LF (JCS as jq produces it)" \ + "$(jq -cS . "$A/.gitbot-fleet/enrolment.json")" "$(cat "$A/.gitbot-fleet/enrolment.json")" +check "A: manifest ends with exactly one LF" "1" "$(tail -c1 "$A/.gitbot-fleet/enrolment.json" | wc -l | tr -d ' ')" +check "A: every-repo members enrolled" "true true true true" \ + "$(jq -r '[.reporters.rhodibot, .reporters.finishingbot, .reporters.glambot, .reporters.cipherbot][] | .enrolled' "$A/.gitbot-fleet/enrolment.json" | paste -sd' ' -)" +check "A: tree-dependent members not enrolled in an empty tree, with reasons" "4" \ + "$(jq -r '[.reporters.panicbot, .reporters.seambot, .reporters.accessibilitybot, .reporters.echidnabot][] | select(.enrolled == false and (.reason | length) > 0)' "$A/.gitbot-fleet/enrolment.json" | jq -s length)" +check "A: sustainabot stays the reserved slot" "false" "$(jq -r '.reporters.sustainabot.enrolled' "$A/.gitbot-fleet/enrolment.json")" +check "A: report-backs README created" "yes" "$([ -f "$A/.gitbot-fleet/report-backs/README.md" ] && echo yes || echo no)" +WF="$A/.github/workflows/fork-governance.yml" +check_grep "A: reusable form chosen for the hyperpolymath owner" "^ uses: hyperpolymath/gitbot-fleet/.github/workflows/fleet-report-back-reusable.yml@$REF$" "$WF" +check "A: fleet job sits before hypatia" "fleet hypatia" \ + "$(grep -nE '^ (fleet|hypatia):' "$WF" | sort -n | sed -E 's/^[0-9]+: //; s/://' | paste -sd' ' -)" +# shellcheck disable=SC2016 # literal GitHub expression +check "A: hypatia gained needs and the cancel guard, right under its header" \ + ' needs: [fleet]| if: ${{ !cancelled() }}' \ + "$(awk '/^ hypatia:/{p=1;next} p&&n<2{print;n++}' "$WF" | paste -sd'|' -)" +check_no_grep "A: no placeholder left in the workflow" '__[A-Z_]+__' "$WF" +check_grep "A: pinned standards lane confirmed to read the artifact" 'ok: standards pin .* reads the fleet-report-backs artifact' "$TMP/a.out" +cp -r "$A" "$TMP/a-before" +set +e; bash "$FIXER" "$A" > "$TMP/a2.out" 2>&1; rc=$?; set -e +check "A: second run exits 0" 0 "$rc" +check "A: second run changes nothing (idempotent)" "" "$(diff -r "$TMP/a-before" "$A" || true)" +check_grep "A: second run reports the manifest as the owner's" 'ok: .gitbot-fleet/enrolment.json exists' "$TMP/a2.out" +if command -v yq >/dev/null 2>&1; then + check "A: edited workflow parses and has four jobs" "4" "$(yq -r '.jobs | length' "$WF" 2>/dev/null || echo parse-error)" +fi + +# ── 2. other owner, old standards pin, a lock file, Cargo/HTML/seams/proofs ─ +B="$TMP/b"; mk_repo "$B" git@github.com:metadatastician/example.git +canonical_workflow "$OLD_SHA" > "$B/.github/workflows/fork-governance.yml" +printf 'workflows: {}\n' > "$B/.github/workflows/actions.lock" +mkdir -p "$B/spec/seams" "$B/www" "$B/proofs" "$B/node_modules/x" +: > "$B/Cargo.toml"; : > "$B/spec/seams/seam-register.json"; : > "$B/www/index.html"; : > "$B/proofs/Thm.agda" +set +e; bash "$FIXER" "$B" > "$TMP/b.out" 2>&1; rc=$?; set -e +check "B: exit 3 — written but not wired" 3 "$rc" +WFB="$B/.github/workflows/fork-governance.yml" +check_grep "B: inline form chosen for another owner" '^ FLEET_REF: '"$REF"'$' "$WFB" +check_grep "B: panic-attack pin filled in" '^ PANIC_ATTACK_SHA: '"$PANIC"'$' "$WFB" +check_no_grep "B: no reusable call under another owner" 'fleet-report-back-reusable.yml@' "$WFB" +check_no_grep "B: template header comments are not copied into the workflow" '^# fleet-job' "$WFB" +check "B: inline job has the reusable's steps" \ + "$(step_names "$ROOT/.github/workflows/fleet-report-back-reusable.yml")" \ + "$(awk '/^ fleet:/{p=1;next} /^ hypatia:/{p=0} p' "$WFB" | grep -E '^ *- name: ' | sed -E 's/^ *- name: //')" +check "B: tree-dependent members enrolled from what the tree shows" "true true true true" \ + "$(jq -r '[.reporters.panicbot, .reporters.seambot, .reporters.accessibilitybot, .reporters.echidnabot][] | .enrolled' "$B/.gitbot-fleet/enrolment.json" | paste -sd' ' -)" +check_grep "B: old standards pin named as not reading the artifact" 'MANUAL: standards pin .* does not read the artifact; bump' "$TMP/b.out" +check_grep "B: lock regeneration named" 'MANUAL: actions.lock exists' "$TMP/b.out" +check "B: actions.lock itself untouched" 'workflows: {}' "$(cat "$B/.github/workflows/actions.lock")" +if command -v yq >/dev/null 2>&1; then + check "B: inline workflow parses with the fleet job's steps" \ + "$(step_names "$ROOT/.github/workflows/fleet-report-back-reusable.yml" | wc -l | tr -d ' ')" \ + "$(yq -r '.jobs.fleet.steps | length' "$WFB" 2>/dev/null || echo parse-error)" +fi + +# ── 3. hypatia already has a flow needs list and an always() guard ────────── +C="$TMP/c"; mk_repo "$C" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" | awk '{print} /^ name: Hypatia verdict grid/ {print " needs: [workflow-contract]"; print " if: ${{ always() }}"}' > "$C/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$C" > "$TMP/c.out" 2>&1; rc=$?; set -e +check "C: exit 0 (a NOTE is not a blocker)" 0 "$rc" +check_grep "C: fleet appended to the existing needs list" '^ needs: \[workflow-contract, fleet\]$' "$C/.github/workflows/fork-governance.yml" +check "C: the existing if: is left alone, and not duplicated" "1" "$(awk '/^ hypatia:/{p=1;next} /^ [a-z]+:/{p=0} p && /^ if: /{n++} END{print n+0}' "$C/.github/workflows/fork-governance.yml")" +check_grep "C: always() is pointed out" 'NOTE: hypatia runs under always\(\)' "$TMP/c.out" +set +e; bash "$FIXER" "$C" > /dev/null 2>&1; set -e +check "C: fleet not appended twice" "1" "$(grep -c 'fleet' <<< "$(grep -E '^ needs:' "$C/.github/workflows/fork-governance.yml")")" + +# ── 4. no fork-governance workflow at all ─────────────────────────────────── +D="$TMP/d"; mk_repo "$D" https://github.com/hyperpolymath/example.git +set +e; bash "$FIXER" "$D" > "$TMP/d.out" 2>&1; rc=$?; set -e +check "D: exit 3" 3 "$rc" +check "D: manifest still written" "yes" "$([ -f "$D/.gitbot-fleet/enrolment.json" ] && echo yes || echo no)" +check_grep "D: adoption named" 'MANUAL: no .github/workflows/fork-governance.yml; adopt fork governance first' "$TMP/d.out" + +# ── 5. an existing manifest is the owner's ────────────────────────────────── +E="$TMP/e"; mk_repo "$E" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" > "$E/.github/workflows/fork-governance.yml" +mkdir -p "$E/.gitbot-fleet"; printf '{"reporters":{"rhodibot":{"enrolled":false,"reason":"owner says so"}},"schema":1}\n' > "$E/.gitbot-fleet/enrolment.json" +set +e; bash "$FIXER" "$E" > "$TMP/e.out" 2>&1; rc=$?; set -e +check "E: exit 0" 0 "$rc" +check "E: existing manifest byte-identical" '{"reporters":{"rhodibot":{"enrolled":false,"reason":"owner says so"}},"schema":1}' "$(cat "$E/.gitbot-fleet/enrolment.json")" + +# ── 6. block-list needs is left to a human ────────────────────────────────── +F="$TMP/f"; mk_repo "$F" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" | awk '{print} /^ name: Hypatia verdict grid/ {print " needs:"; print " - workflow-contract"}' > "$F/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$F" > "$TMP/f.out" 2>&1; rc=$?; set -e +check "F: exit 3" 3 "$rc" +check_grep "F: fleet job still inserted" '^ fleet:$' "$F/.github/workflows/fork-governance.yml" +check_grep "F: block list named for a human" "MANUAL: hypatia's needs: is a block list" "$TMP/f.out" + +# ── 7. refusals change nothing ────────────────────────────────────────────── +G="$TMP/g"; mk_repo "$G" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" > "$G/.github/workflows/fork-governance.yml" +cp -r "$G" "$TMP/g-before" +set +e; FLEET_REF=main bash "$FIXER" "$G" > "$TMP/g.out" 2>&1; rc=$?; set -e +check "G: a non-SHA FLEET_REF is refused with exit 2" 2 "$rc" +check "G: refusal changed nothing" "" "$(diff -r "$TMP/g-before" "$G" || true)" +set +e; FLEET_JOB_FORM=yaml bash "$FIXER" "$G" > /dev/null 2>&1; rc=$?; set -e +check "G: an unknown job form is refused with exit 2" 2 "$rc" +set +e; bash "$FIXER" "$TMP/does-not-exist" > /dev/null 2>&1; rc=$?; set -e +check "G: a missing repo path is refused with exit 2" 2 "$rc" + +# ── 8. a remote pin with no standards checkout is UNVERIFIED, not ok ──────── +H="$TMP/h"; mk_repo "$H" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" > "$H/.github/workflows/fork-governance.yml" +set +e; STANDARDS_DIR='' bash "$FIXER" "$H" > "$TMP/h.out" 2>&1; rc=$?; set -e +check "H: exit 3 without a standards checkout to read the pin" 3 "$rc" +check_grep "H: pin reported UNVERIFIED" 'UNVERIFIED: standards pin' "$TMP/h.out" + +# ── 9. a local Hypatia reusable (a fork) is read in place ─────────────────── +I="$TMP/i"; mk_repo "$I" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" | sed 's|uses: hyperpolymath/standards/.github/workflows/hypatia-scan-reusable.yml@.*|uses: ./.github/workflows/hypatia-scan-reusable.yml|' > "$I/.github/workflows/fork-governance.yml" +printf 'on:\n workflow_call:\n inputs:\n scan-ref:\n type: string\n' > "$I/.github/workflows/hypatia-scan-reusable.yml" +set +e; bash "$FIXER" "$I" > "$TMP/i.out" 2>&1; rc=$?; set -e +check "I: local reusable without the input → exit 3" 3 "$rc" +check_grep "I: local reusable named" 'MANUAL: local Hypatia reusable .* has no report-backs-artifact input with a download-artifact step' "$TMP/i.out" +printf ' report-backs-artifact: # the input alone\n' >> "$I/.github/workflows/hypatia-scan-reusable.yml" +set +e; bash "$FIXER" "$I" > "$TMP/i1.out" 2>&1; rc=$?; set -e +check "I: the input name alone is not wiring → exit 3" 3 "$rc" +printf 'jobs:\n scan:\n steps:\n - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333\n' >> "$I/.github/workflows/hypatia-scan-reusable.yml" +set +e; bash "$FIXER" "$I" > "$TMP/i2.out" 2>&1; rc=$?; set -e +check "I: local reusable with the input → exit 0" 0 "$rc" + +# ── 10. a needs item that merely starts with "fleet" is not fleet ─────────── +J="$TMP/j"; mk_repo "$J" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" | awk '{print} /^ name: Hypatia verdict grid/ {print " needs: [fleet-scan]"}' > "$J/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$J" > "$TMP/j.out" 2>&1; rc=$?; set -e +check "J: exit 0" 0 "$rc" +check_grep "J: fleet appended beside fleet-scan, not mistaken for it" '^ needs: \[fleet-scan, fleet\]$' "$J/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$J" > "$TMP/j2.out" 2>&1; set -e +check_grep "J: second run sees fleet as present" 'ok: hypatia already needs fleet' "$TMP/j2.out" + +# ── 11. a job merely named fleet is not the producer ──────────────────────── +K="$TMP/k"; mk_repo "$K" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" | awk '{print} /^jobs:$/ {print " fleet:"; print " name: Something else entirely"; print " runs-on: ubuntu-24.04"; print " steps:"; print " - run: true"; print ""}' > "$K/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$K" > "$TMP/k.out" 2>&1; rc=$?; set -e +check "K: exit 3" 3 "$rc" +check_grep "K: the impostor is named for a human" 'MANUAL: fork-governance.yml has a job named fleet that is not the report-back producer' "$TMP/k.out" +check "K: no second fleet job inserted" "1" "$(grep -cE '^ fleet:$' "$K/.github/workflows/fork-governance.yml" | tr -d ' ')" +check_no_grep "K: the impostor is not called the producer" 'already has the fleet report-back job' "$TMP/k.out" +set +e; bash "$FIXER" "$A" > "$TMP/a3.out" 2>&1; set -e +check_grep "A: the real producer job is recognised" 'ok: fork-governance.yml already has the fleet report-back job' "$TMP/a3.out" + +# ── 12. the scalar needs form ─────────────────────────────────────────────── +L="$TMP/l"; mk_repo "$L" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" | awk '{print} /^ name: Hypatia verdict grid/ {print " needs: fleet"}' > "$L/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$L" > "$TMP/l.out" 2>&1; rc=$?; set -e +check "L: scalar needs: fleet is already wired → exit 0" 0 "$rc" +check_grep "L: reported as present" 'ok: hypatia already needs fleet' "$TMP/l.out" +check "L: the scalar line is left as it was" "1" "$(grep -cE '^ needs: fleet$' "$L/.github/workflows/fork-governance.yml" | tr -d ' ')" +N="$TMP/n"; mk_repo "$N" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" | awk '{print} /^ name: Hypatia verdict grid/ {print " needs: workflow-contract"}' > "$N/.github/workflows/fork-governance.yml" +chmod 644 "$N/.github/workflows/fork-governance.yml" +set +e; bash "$FIXER" "$N" > "$TMP/n.out" 2>&1; rc=$?; set -e +check "N: scalar needs: other → exit 0" 0 "$rc" +check_grep "N: scalar other becomes a flow list with fleet" '^ needs: \[workflow-contract, fleet\]$' "$N/.github/workflows/fork-governance.yml" +check "N: the rewritten workflow keeps its mode (not mktemp's 0600)" "644" "$(stat -c %a "$N/.github/workflows/fork-governance.yml")" +check "A: the inserted-job workflow keeps its mode too" "644" "$(stat -c %a "$A/.github/workflows/fork-governance.yml")" + +# ── 13. a pin that does not carry the producer is refused ─────────────────── +# The parent of the commit that added the driver lacks it; a shallow clone may +# not hold that commit, in which case the case is reported as skipped. +ADDED=$(git -C "$ROOT" log --format=%H --diff-filter=A -1 -- scripts/fleet-report-back.sh || true) +if [ -n "$ADDED" ] && OLD=$(git -C "$ROOT" rev-parse -q --verify "${ADDED}^{commit}" 2>/dev/null) && OLD=$(git -C "$ROOT" rev-parse -q --verify "${OLD}^" 2>/dev/null) \ + && ! git -C "$ROOT" cat-file -e "$OLD:scripts/fleet-report-back.sh" 2>/dev/null; then + O="$TMP/o"; mk_repo "$O" https://github.com/hyperpolymath/example.git + canonical_workflow "$NEW_SHA" > "$O/.github/workflows/fork-governance.yml" + cp -r "$O" "$TMP/o-before" + set +e; FLEET_REF="$OLD" bash "$FIXER" "$O" > "$TMP/o.out" 2>&1; rc=$?; set -e + check "O: a pin without the producer is refused with exit 2" 2 "$rc" + check_grep "O: the missing producer file is named" 'does not carry .github/workflows/fleet-report-back-reusable.yml' "$TMP/o.out" + check "O: refusal changed nothing" "" "$(diff -r "$TMP/o-before" "$O" || true)" +else + echo "skip O: no commit without the producer is reachable from this checkout" +fi + +# ── 14. dependencies and generated trees are not enrolment evidence ──────── +run_fixer() { # label repo [finding-json]; records rc and output + local label="$1"; shift + rc=0 + bash "$FIXER" "$@" > "$TMP/$label.out" 2>&1 || rc=$? +} +dependent_enrolments() { + jq -c '[.reporters.panicbot.enrolled, .reporters.seambot.enrolled, + .reporters.accessibilitybot.enrolled, .reporters.echidnabot.enrolled]' "$1/.gitbot-fleet/enrolment.json" +} +for excluded in .git node_modules target vendor .fleet-src; do + P="$TMP/pruned-$excluded"; mk_repo "$P" https://github.com/hyperpolymath/example.git + canonical_workflow "$NEW_SHA" > "$P/.github/workflows/fork-governance.yml" + mkdir -p "$P/$excluded/dependency/spec/seams" + : > "$P/$excluded/dependency/Cargo.toml" + : > "$P/$excluded/dependency/index.htm" + : > "$P/$excluded/dependency/Theorem.lean" + : > "$P/$excluded/dependency/spec/seams/seam-register.json" + run_fixer "pruned-$excluded" "$P" + check "$excluded: a tree containing only excluded evidence is still wired" 0 "$rc" + check "$excluded: generated or vendored files enrol no conditional member" \ + '[false,false,false,false]' "$(dependent_enrolments "$P")" + check "$excluded: every conditional exclusion carries an observed reason" 4 \ + "$(jq '[.reporters.panicbot,.reporters.seambot,.reporters.accessibilitybot,.reporters.echidnabot] | map(select(.enrolled == false and (.reason | length) > 0)) | length' "$P/.gitbot-fleet/enrolment.json")" +done + +# Proofs can live outside proofs/, in any of the supported source languages. +# Conversely an explicitly declared, currently empty proofs/ is sufficient. +for proof in theorem.agda theorem.lean theorem.idr theorem.v proofs-directory; do + P="$TMP/proof-$proof"; mk_repo "$P" https://github.com/hyperpolymath/example.git + canonical_workflow "$NEW_SHA" > "$P/.github/workflows/fork-governance.yml" + if [ "$proof" = proofs-directory ]; then + mkdir -p "$P/proofs" + else + mkdir -p "$P/nested"; : > "$P/nested/$proof" + fi + run_fixer "proof-$proof" "$P" + check "$proof: proof evidence wires successfully" 0 "$rc" + check "$proof: only the proof-dependent member is enabled" \ + '[false,false,false,true]' "$(dependent_enrolments "$P")" +done +P="$TMP/nested source with spaces"; mk_repo "$P" https://github.com/hyperpolymath/example.git +canonical_workflow "$NEW_SHA" > "$P/.github/workflows/fork-governance.yml" +mkdir -p "$P/crates/app" "$P/site" "$P/docs" +: > "$P/crates/app/Cargo.toml"; : > "$P/site/index.htm"; : > "$P/docs/proof.txt" +run_fixer nested-source "$P" '{"ignored":"finding input is not tree evidence"}' +check "nested Cargo and .htm files enable their members, while prose is not a proof" \ + '[true,false,true,false]' "$(dependent_enrolments "$P")" +check "nested source in a path with spaces wires successfully" 0 "$rc" + +# ── 15. owner detection and explicit job-form overrides ──────────────────── +for choice in ssh-auto force-inline force-reusable; do + P="$TMP/$choice" + case "$choice" in + ssh-auto) origin=git@github.com:hyperpolymath/example.git; form=auto; expected=reusable ;; + force-inline) origin=https://github.com/hyperpolymath/example.git; form=inline; expected=inline ;; + force-reusable) origin=https://github.com/another-owner/example.git; form=reusable; expected=reusable ;; + esac + mk_repo "$P" "$origin" + canonical_workflow "$NEW_SHA" > "$P/.github/workflows/fork-governance.yml" + FLEET_JOB_FORM="$form" run_fixer "$choice" "$P" + check "$choice: explicit form or detected SSH owner wires successfully" 0 "$rc" + check_grep "$choice: chosen job form is reported" "job form: $expected$" "$TMP/$choice.out" + if [ "$expected" = reusable ]; then + check_grep "$choice: reusable job pins the selected producer revision" \ + "uses: hyperpolymath/gitbot-fleet/.github/workflows/fleet-report-back-reusable.yml@$REF$" "$P/.github/workflows/fork-governance.yml" + else + check_grep "$choice: inline job pins the selected producer revision" \ + "^ FLEET_REF: $REF$" "$P/.github/workflows/fork-governance.yml" + fi +done + +# ── 16. existing flow dependencies and guards keep their meaning ─────────── +for needs in '[]' '["deed", "fleet"]' "[ 'fleet', 'deed' ]"; do + P="$TMP/flow-$pass"; mk_repo "$P" https://github.com/hyperpolymath/example.git + canonical_workflow "$NEW_SHA" | awk -v needs="$needs" \ + '{print} /^ name: Hypatia verdict grid/ {print " needs: " needs; print " if: ${{ success() }}"}' \ + > "$P/.github/workflows/fork-governance.yml" + run_fixer flow "$P" + check "flow $needs: wiring succeeds" 0 "$rc" + expected="$needs"; [ "$needs" != '[]' ] || expected='[fleet]' + check_grep "flow $needs: dependency list preserves exact quoted fleet items" \ + '^ needs:' "$P/.github/workflows/fork-governance.yml" + check "flow $needs: expected dependency list is retained" " needs: $expected" \ + "$(grep '^ needs:' "$P/.github/workflows/fork-governance.yml" | head -n1)" + check_grep "flow $needs: existing guard is preserved" \ + '^ if: \$\{\{ success\(\) \}\}$' "$P/.github/workflows/fork-governance.yml" + cp "$P/.github/workflows/fork-governance.yml" "$TMP/flow-before.yml" + run_fixer flow-again "$P" + check "flow $needs: repeated wiring succeeds" 0 "$rc" + check "flow $needs: repeated wiring is byte-identical" '' \ + "$(diff "$TMP/flow-before.yml" "$P/.github/workflows/fork-governance.yml" || true)" +done + +# ── 17. invalid owner manifests and existing notes are never overwritten ──── +for invalid in malformed wrong-shape; do + P="$TMP/manifest-$invalid"; mk_repo "$P" https://github.com/hyperpolymath/example.git + canonical_workflow "$NEW_SHA" > "$P/.github/workflows/fork-governance.yml" + mkdir -p "$P/.gitbot-fleet/report-backs" + case "$invalid" in + malformed) printf 'not JSON\n' > "$P/.gitbot-fleet/enrolment.json" ;; + wrong-shape) printf '{"reporters":[]}\n' > "$P/.gitbot-fleet/enrolment.json" ;; + esac + printf 'Owner-maintained report-back instructions\n' > "$P/.gitbot-fleet/report-backs/README.md" + before=$(sha256sum "$P/.gitbot-fleet/enrolment.json" "$P/.gitbot-fleet/report-backs/README.md") + run_fixer "manifest-$invalid" "$P" + check "$invalid: invalid owner manifest requires manual action" 3 "$rc" + check_grep "$invalid: manifest validation failure is explicit" \ + 'MANUAL: .gitbot-fleet/enrolment.json exists but does not parse' "$TMP/manifest-$invalid.out" + check "$invalid: both owner files remain byte-identical" "$before" \ + "$(sha256sum "$P/.gitbot-fleet/enrolment.json" "$P/.gitbot-fleet/report-backs/README.md")" + check_grep "$invalid: independent workflow wiring still completes" \ + '^ fleet:$' "$P/.github/workflows/fork-governance.yml" +done + +# A malformed panic-attack pin is rejected before creating any rollout files. +cp -r "$G" "$TMP/panic-ref-before" +PANIC_ATTACK_SHA=main run_fixer invalid-panic-ref "$G" +check "an invalid panic-attack pin is refused" 2 "$rc" +check_grep "invalid panic-attack pin explains the required format" \ + 'PANIC_ATTACK_SHA must be a 40-hex commit' "$TMP/invalid-panic-ref.out" +check "invalid panic-attack pin leaves the entire repository unchanged" '' \ + "$(diff -r "$TMP/panic-ref-before" "$G" || true)" + +# ── 18. a standards checkout without the pinned commit is unverified ──────── +P="$TMP/unknown-standards-pin"; mk_repo "$P" https://github.com/hyperpolymath/example.git +canonical_workflow 1111111111111111111111111111111111111111 > "$P/.github/workflows/fork-governance.yml" +run_fixer unknown-standards-pin "$P" +check "an absent standards commit is not treated as successful wiring" 3 "$rc" +check_grep "the absent standards commit is identified as unverified" \ + 'UNVERIFIED: standards pin 111111111111' "$TMP/unknown-standards-pin.out" + +echo +echo "fix-fleet-enrolment: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/tests/fleet-report-back-build-test.sh b/tests/fleet-report-back-build-test.sh new file mode 100755 index 00000000..fe895d84 --- /dev/null +++ b/tests/fleet-report-back-build-test.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk> +# +# Unit tests for fleet-report-back-build.sh. Cargo and timeout are stubbed: +# exercise planning, copying and failure isolation without a toolchain, +# downloads, or touching the user's panic-attack cache. +# Run: bash tests/fleet-report-back-build-test.sh +# Also run by tests/fleet-report-back-test.sh, the existing CI entry point. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILDER="$ROOT/scripts/fleet-report-back-build.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +pass=0; fail=0 +check() { + local label="$1" expected="$2" actual="$3" + if [[ "$expected" == "$actual" ]]; then + pass=$((pass + 1)); echo "PASS: $label" + else + fail=$((fail + 1)); echo "FAIL: $label (expected '$expected', got '$actual')" + fi +} + +FLEET="$TMP/fleet with spaces"; TOOLS="$TMP/tools" +mkdir -p "$FLEET/bots" "$TOOLS" +for crate in rhodibot panicbot finishingbot glambot broken slow library-only Adapter_2-name; do + mkdir -p "$FLEET/bots/$crate" + printf '[package]\nname = "%s"\nversion = "0.0.0"\n' "$crate" > "$FLEET/bots/$crate/Cargo.toml" +done +export FLEET_BUILD_CALLS="$TMP/cargo-calls" FLEET_BUILD_TIMEOUTS="$TMP/timeout-calls" + +cat > "$TOOLS/cargo" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +crate="${PWD##*/}" +printf '%s\t%s\n' "$crate" "$*" >> "$FLEET_BUILD_CALLS" +# Refuse installs and any flags other than the locked release build recipe. +# An accidental real dependency install must never escape the fixture. +[[ "$*" == 'build --release --locked' ]] || exit 99 +mkdir -p target/release/deps +if [[ "$crate" == library-only ]]; then + printf 'library\n' > target/release/libfixture.rlib + exit 0 +fi +binary="$crate"; [[ "$crate" != finishingbot ]] || binary=finishing-bot +printf '#!/usr/bin/env bash\necho %s\n' "$crate" > "target/release/$binary" +chmod +x "target/release/$binary" +[[ "$crate" != broken ]] || exit 1 +# Only top-level executables belong in the destination. +printf 'library\n' > target/release/libfixture.rlib +printf 'dependency metadata\n' > target/release/fixture.d +chmod +x target/release/fixture.d +printf '#!/usr/bin/env bash\nexit 0\n' > target/release/deps/internal-helper +chmod +x target/release/deps/internal-helper +if [[ "$crate" == rhodibot ]]; then + cp "target/release/$binary" target/release/rhodi-helper +fi +STUB +cat > "$TOOLS/timeout" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\t%s\n' "${PWD##*/}" "$*" >> "$FLEET_BUILD_TIMEOUTS" +# Model a timed-out crate deterministically, without waiting or spawning it. +[[ "${PWD##*/}" != slow ]] || exit 124 +shift +exec "$@" +STUB +chmod +x "$TOOLS/cargo" "$TOOLS/timeout" + +run_build() { # label arguments... ; sets rc, clears invocation logs + local label="$1"; shift + : > "$FLEET_BUILD_CALLS"; : > "$FLEET_BUILD_TIMEOUTS" + rc=0 + PATH="$TOOLS:$PATH" GITHUB_ACTIONS='' bash "$BUILDER" \ + --fleet "$FLEET" --bin-dir "$TMP/$label" "$@" \ + < "$TMP/plan" > "$TMP/$label.log" 2>&1 || rc=$? +} +files() { find "$1" -maxdepth 1 -type f -printf '%f\n' | sort | paste -sd ' ' -; } +: > "$TMP/plan" + +# Explicit crate list: continue after both an unknown crate and a failed build. +run_build mixed --crates 'rhodibot unknown broken finishingbot glambot' +check "mixed successful and failing builds do not fail the helper" 0 "$rc" +check "builds run inside each selected crate with release and locked flags" \ + $'rhodibot\tbuild --release --locked\nbroken\tbuild --release --locked\nfinishingbot\tbuild --release --locked\nglambot\tbuild --release --locked' \ + "$(cat "$FLEET_BUILD_CALLS")" +check "only top-level executables from successful builds are copied" \ + 'finishing-bot glambot rhodi-helper rhodibot' "$(files "$TMP/mixed")" +check "copied adapters remain executable" yes "$([ -x "$TMP/mixed/finishing-bot" ] && echo yes || echo no)" +check "copied adapter bytes match the crate output" \ + "$(sha256sum < "$FLEET/bots/rhodibot/target/release/rhodibot")" "$(sha256sum < "$TMP/mixed/rhodibot")" +check "failed crates' partial binaries are not copied" no "$([ -e "$TMP/mixed/broken" ] && echo yes || echo no)" +check "unknown crate warning names the absent manifest" yes \ + "$(grep -q 'bots/unknown has no Cargo.toml' "$TMP/mixed.log" && echo yes || echo no)" +check "failed crate warning explains its report-back consequence" yes \ + "$(grep -q 'bots/broken did not build; its row will read error' "$TMP/mixed.log" && echo yes || echo no)" +check "build summary counts successes and failures" yes \ + "$(grep -q '3 crate(s) built, 2 failed' "$TMP/mixed.log" && echo yes || echo no)" +check "default timeout is applied to every attempted build" 4 \ + "$(grep -c $'\t900 cargo build --release --locked$' "$FLEET_BUILD_TIMEOUTS")" + +# Consume the real driver's plan, including aliases for the same crate and +# rows with deliberately empty crate columns. Each crate should build once. +bash "$ROOT/scripts/fleet-report-back.sh" --plan --enrolment "" \ + --member finishingbot --member finishbot --member rhodibot \ + --member echidnabot --member sustainabot > "$TMP/plan" +run_build planned +check "stdin plan completes successfully" 0 "$rc" +check "stdin plan deduplicates aliases and skips members without crates" \ + $'finishingbot\tbuild --release --locked\nrhodibot\tbuild --release --locked' "$(cat "$FLEET_BUILD_CALLS")" +check "stdin plan publishes just the selected crates' executables" \ + 'finishing-bot rhodi-helper rhodibot' "$(files "$TMP/planned")" + +# --crates takes precedence over a nonempty stdin plan. +run_build explicit --crates glambot +check "an explicit crate list overrides stdin" $'glambot\tbuild --release --locked' "$(cat "$FLEET_BUILD_CALLS")" +check "unselected adapters are not copied" glambot "$(files "$TMP/explicit")" + +: > "$TMP/plan" +run_build empty +check "an empty plan succeeds" 0 "$rc" +check "an empty plan invokes no Cargo commands" 0 "$(wc -l < "$FLEET_BUILD_CALLS")" +check "an empty plan publishes no binaries" '' "$(files "$TMP/empty")" +check "an empty plan summary reports zero work" yes \ + "$(grep -q '0 crate(s) built, 0 failed' "$TMP/empty.log" && echo yes || echo no)" + +run_build timed --crates 'slow panicbot' --build-timeout 23 +check "a timed-out build does not fail the helper" 0 "$rc" +check "a timed-out build still allows the next crate to build" panicbot "$(files "$TMP/timed")" +check "the configured timeout reaches each build" \ + $'slow\t23 cargo build --release --locked\npanicbot\t23 cargo build --release --locked' "$(cat "$FLEET_BUILD_TIMEOUTS")" +check "a timed-out crate counts as a failure" yes \ + "$(grep -q '1 crate(s) built, 1 failed' "$TMP/timed.log" && echo yes || echo no)" + +run_build failed --crates 'unknown broken' +check "even when every crate fails, the helper completes" 0 "$rc" +check "all-failed builds publish nothing" '' "$(files "$TMP/failed")" +check "all-failed summary counts both failures" yes \ + "$(grep -q '0 crate(s) built, 2 failed' "$TMP/failed.log" && echo yes || echo no)" + +# Exercise the Actions log format as well as local stderr warnings. +rc=0 +PATH="$TOOLS:$PATH" GITHUB_ACTIONS=true bash "$BUILDER" \ + --fleet "$FLEET" --bin-dir "$TMP/actions" --crates broken \ + > "$TMP/actions.log" 2>&1 || rc=$? +check "Actions builds preserve advisory failure handling" 0 "$rc" +check "Actions build groups open and close" $'::group::cargo build --release --locked (bots/broken)\n::endgroup::' \ + "$(grep -E '^::(group|endgroup)::' "$TMP/actions.log")" +check "Actions failures use warning annotations" yes \ + "$(grep -q '^::warning title=adapter build failed::bots/broken' "$TMP/actions.log" && echo yes || echo no)" + +# A minimal PATH makes the missing-Cargo branch independent of the host's +# installed Rust toolchains. Invoke bash by its existing absolute path. +BASH_BIN="$(command -v bash)" +mkdir -p "$TMP/no-cargo" +ln -s "$(command -v mkdir)" "$TMP/no-cargo/mkdir" +rc=0 +PATH="$TMP/no-cargo" "$BASH_BIN" "$BUILDER" --fleet "$FLEET" \ + --bin-dir "$TMP/no-toolchain" --crates rhodibot > "$TMP/no-toolchain.log" 2>&1 || rc=$? +check "missing Cargo does not fail the producer" 0 "$rc" +check "missing Cargo gets an explicit warning" yes \ + "$(grep -q 'cargo is not on PATH; no adapter can be built' "$TMP/no-toolchain.log" && echo yes || echo no)" +check "missing Cargo publishes no binaries" '' "$(files "$TMP/no-toolchain")" + +expect_usage() { + local label="$1" rc=0; shift + bash "$BUILDER" "$@" < /dev/null > "$TMP/usage.log" 2>&1 || rc=$? + check "$label exits with usage status" 2 "$rc" +} +expect_usage "builder missing required options" +expect_usage "builder missing bin-dir" --fleet "$FLEET" +expect_usage "builder invalid fleet directory" --fleet "$TMP/absent" --bin-dir "$TMP/invalid" +expect_usage "builder unknown option" --unknown +check "invalid builder arguments create no bin directory" no "$([ -e "$TMP/invalid" ] && echo yes || echo no)" + +# --- the build budget: one job timeout must not swallow every envelope -------- +run_build budget-zero --crates "rhodibot panicbot" --panic-attack 0123456789abcdef0123456789abcdef01234567 --build-budget 0 +check "budget spent: no cargo invocation is attempted" "0" "$(grep -c . "$FLEET_BUILD_CALLS" || true)" +check "budget spent: every crate is warned as unbuilt" "2" "$(grep -c 'build budget exhausted' "$TMP/budget-zero.log" || true)" +check "budget spent: panic-attack is warned, not attempted" "1" "$(grep -c 'spent before its turn' "$TMP/budget-zero.log" || true)" +check "budget spent: the run still completes (exit 0)" 0 "$rc" +run_build budget-wide --crates rhodibot --build-budget 100000 --build-timeout 900 +check "budget above the per-build limit: the per-build limit applies" "1" "$(grep -c $'\t900 cargo build --release --locked$' "$FLEET_BUILD_TIMEOUTS" || true)" +run_build budget-narrow --crates rhodibot --build-budget 100000 --build-timeout 60 +check "per-build limit below the budget: the limit applies" "1" "$(grep -c $'\t60 cargo build --release --locked$' "$FLEET_BUILD_TIMEOUTS" || true)" + +# --- a crate name that is a path is refused before any cargo invocation -------- +run_build traversal --crates "../../evil rhodibot" +check "a path in the crate list is skipped" "1" "$(grep -c 'is not a crate directory name' "$TMP/traversal.log" || true)" +check "...and only the real crate is built" $'rhodibot\tbuild --release --locked' "$(cat "$FLEET_BUILD_CALLS")" + +# A positive total budget below the per-build limit must cap timeout's +# argument. Allow for time spent starting the helper instead of assuming +# Bash's SECONDS counter is still exactly zero on a busy machine. +run_build budget-capped --crates rhodibot --build-budget 120 --build-timeout 900 +check "a build under a smaller total budget still succeeds" 0 "$rc" +limit=$(awk -F'\t' '{split($2, args, " "); print args[1]}' "$FLEET_BUILD_TIMEOUTS") +check "remaining budget caps the per-build timeout at a positive value" yes \ + "$([[ "$limit" =~ ^[0-9]+$ ]] && [ "$limit" -gt 0 ] && [ "$limit" -le 120 ] && echo yes || echo no)" +check "the budget-capped build publishes its executables" 'rhodi-helper rhodibot' \ + "$(files "$TMP/budget-capped")" + +# A successful library-only crate produces no executable to publish; that +# must not fail this helper or prevent a subsequent adapter build. +run_build library-only --crates 'library-only panicbot' +check "a crate with no executable does not fail the helper" 0 "$rc" +check "library-only output is not copied and the later adapter is available" panicbot \ + "$(files "$TMP/library-only")" +check "successful library-only crates remain successful in the build summary" yes \ + "$(grep -q '2 crate(s) built, 0 failed' "$TMP/library-only.log" && echo yes || echo no)" + +run_build crate-names --crates '-option bad/name dotted.name _leading Adapter_2-name' +check "invalid crate tokens never reach cargo" $'Adapter_2-name\tbuild --release --locked' \ + "$(cat "$FLEET_BUILD_CALLS")" +check "each invalid crate token is warned about" 4 \ + "$(grep -c 'is not a crate directory name' "$TMP/crate-names.log")" +check "valid crate names may contain uppercase letters, underscores, digits and hyphens" \ + Adapter_2-name "$(files "$TMP/crate-names")" +check "invalid crate tokens count as failures while the valid build completes" yes \ + "$(grep -q '1 crate(s) built, 4 failed' "$TMP/crate-names.log" && echo yes || echo no)" + +echo +echo "fleet-report-back-build-test: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/tests/fleet-report-back-test.sh b/tests/fleet-report-back-test.sh new file mode 100755 index 00000000..fe0d0c05 --- /dev/null +++ b/tests/fleet-report-back-test.sh @@ -0,0 +1,495 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk> +# +# fleet-report-back-test.sh — the report-back driver writes exactly the +# envelope the Hypatia verdict grid reads (standards docs/VERDICT-GRID-SPEC.adoc), +# and never turns a silent, missing or crashed adapter into a clean row. +# +# Adapters are stubbed: each stub prints the real adapter's JSON SHAPE (taken +# from a live run against hyperpolymath/metamanifold-webui on 2026-10-27), so +# the translation is tested without building seven Rust crates. The stubs +# write nothing, so a stub that is invoked with the wrong arguments fails the +# test rather than quietly passing. +# +# Covers: +# 1. enrolment: only `enrolled: true` members run; `false` ones get no file; +# an unreadable manifest runs nothing; --member honours `enrolled: false`. +# 2. every adapter's output translates to {rule_id, severity, file, line, +# message} with repository-relative paths (absolute and "./" stripped). +# 3. missing binary, crash without JSON, and timeout are `error` WITH a +# reason; non-zero exit WITH valid JSON is still `ran`. +# 4. echidnabot and sustainabot are `not_run` with a reason; an unknown +# member is `error`; the driver never writes `not_applicable`. +# 5. the envelope is key-sorted compact JSON with one final LF, carrying +# the fleet ref and the manifest's title. +# 6. --plan lists member, crate and binary for the members that would run. +# 7. the summary table has one row per member. +# 8. (when STANDARDS_DIR points at a hyperpolymath/standards checkout) the +# real renderer turns the rows non-orange from these envelopes alone. +# 9. empty scans, malformed output, normalization boundaries, aliases, +# failure isolation, invocation context, and an unchanged scanned tree. +# 10. the build-helper unit suite (stubbed Cargo, no Rust builds or network). +# +# Run: bash tests/fleet-report-back-test.sh + +# shellcheck disable=SC2016 # stub bodies are single-quoted on purpose: they expand when the stub runs +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DRIVER="$ROOT/scripts/fleet-report-back.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +pass=0; fail=0 +check() { + local label="$1" expected="$2" actual="$3" + if [[ "$expected" == "$actual" ]]; then + pass=$((pass + 1)); echo "PASS: $label" + else + fail=$((fail + 1)); echo "FAIL: $label (expected '$expected', got '$actual')" + fi +} +field() { jq -r "$2" "$1"; } + +# ── fixture repository and stub adapters ───────────────────────────────────── +REPO="$TMP/repo with spaces"; mkdir -p "$REPO/.gitbot-fleet" "$REPO/src" +echo 'x' > "$REPO/src/a.jl" +cat > "$REPO/.gitbot-fleet/enrolment.json" <<'JSON' +{ "schema": 1, + "reporters": { + "rhodibot": { "title": "rhodibot (RSR compliance)", "enrolled": true }, + "panicbot": { "enrolled": true }, + "echidnabot": { "title": "echidnabot (formal verification: proofs/)", "enrolled": true }, + "seambot": { "enrolled": true }, + "finishingbot": { "enrolled": true }, + "accessibilitybot": { "enrolled": true }, + "glambot": { "enrolled": true }, + "cipherbot": { "enrolled": false, "reason": "no cryptography is implemented" }, + "sustainabot": { "enrolled": false, "reason": "reserved slot" } } } +JSON + +BIN="$TMP/bin with spaces"; mkdir -p "$BIN" +stub() { # name body + printf '#!/usr/bin/env bash\nset -u\n%s\n' "$2" > "$BIN/$1"; chmod +x "$BIN/$1" +} +# Each stub asserts the invocation the driver documents, then prints the shape. +stub rhodibot '[ "$1 $2 $4 $5" = "canon --path --format json" ] || { echo "bad args: $*" >&2; exit 9; } +cat <<J +{"subject":"$3","criteria":30,"scored":20,"not_applicable":10,"counts":{"satisfied":18,"relocated":1,"deprecated":0,"missing":1}, + "findings":[ + {"id":"1.1.2","name":"justfile","tier":"T1","severity":"missing","canon_keeps_it_at":"Justfile","expected":["Justfile"],"found":[],"deprecated_copies":[],"details":["absent"]}, + {"id":"2.3","name":"readme","tier":"T1","severity":"satisfied","canon_keeps_it_at":"README.adoc","expected":["README.adoc"],"found":["README.adoc"],"deprecated_copies":[],"details":[]}, + {"id":"4.1","name":"security.txt","tier":"T2","severity":"relocated","canon_keeps_it_at":"www/.well-known/security.txt","expected":["www/.well-known/security.txt"],"found":[".well-known/security.txt"],"deprecated_copies":[],"details":["found elsewhere"]}]} +J' +stub panicbot '[ "$1 $2 $3" = "--format json scan" ] || { echo "bad args: $*" >&2; exit 9; } +command -v panic-attack >/dev/null || { echo "Error: Failed to run panic-attack assail" >&2; exit 1; } +cat <<J +[{"id":"u1","source":"panicbot","rule_id":"PA018","rule_name":"DynamicCodeExecution","severity":"warning","message":"eval in src/a.jl","category":"security","file":"src/a.jl","line":null,"column":null,"fixable":false,"fixed":false}] +J' +stub panic-attack 'echo "stub"' +stub seambot '[ "$1 $3 $4 $5" = "--path --format json check" ] || { echo "bad args: $*" >&2; exit 9; } +cat <<J +{"status":"fail","findings":[{"check":"register-exists","severity":"error","message":"Seam register not found at spec/seams/seam-register.json","location":"spec/seams/seam-register.json","suggestion":"Run seambot init","seam_id":null}],"summary":{"errors":1}} +J' +stub finishing-bot '[ "$1 $3 $4 $5" = "--path --format json audit" ] || { echo "bad args: $*" >&2; exit 9; } +cat <<J +{"license":{"findings":[{"id":"LIC-002","name":"Missing SPDX Header","severity":"warning","message":"Source file missing SPDX license identifier","file":"./config/tools.yml","line":null,"column":null,"suggestion":"s","fixable":true}]}, + "placeholder":{"findings":[]}, + "release":{"findings":[{"id":"REL-001","name":"No release","severity":"error","message":"No release artefacts","file":null,"line":null,"column":null,"suggestion":null,"fixable":false}]}} +J +exit 1' +stub glambot '[ "$1 $3 $4 $5" = "--path --format json audit" ] || { echo "bad args: $*" >&2; exit 9; } +cat <<J +{"visual":{"findings":[{"id":"VIS-005","name":"Line too long","severity":"info","message":"Line 88 characters exceeds max length of 80.","file":"./SECURITY.md","line":7,"column":null,"suggestion":"s","fixable":false}]},"seo":{"findings":[]}} +J' +stub accessibilitybot '[ "$1" = "fleet" ] || { echo "bad args: $*" >&2; exit 9; } +cat <<J +{"findings":[{"id":"u2","source":"accessibilitybot","rule_id":"WCAG-1.3.1-first-heading","rule_name":"Heading Hierarchy","severity":"error","message":"First heading is <h2>","category":"accessibility/wcag-a","file":"$2/src/index.html","line":12,"column":null,"fixable":false,"fixed":false}]} +J +exit 1' +stub cipherbot '[ "$1 $2 $3" = "--format json scan" ] || { echo "bad args: $*" >&2; exit 9; } +echo "[]"' + +run() { # out-dir extra-args... + bash "$DRIVER" --repo "$REPO" --out "$1" --bin-dir "$BIN" --fleet-ref deadbeef "${@:2}" > "$1.log" 2>&1 +} + +# 1. enrolment ──────────────────────────────────────────────────────────────── +repo_state() { + find "$REPO" -printf '%P %y %l\n' | sort + find "$REPO" -type f -exec sha256sum {} + | sort +} +repo_before=$(repo_state) +ECHIDNA_URL='' run "$TMP/o1" --summary "$TMP/o1.md" +check "enrolled members each get a file" "accessibilitybot echidnabot finishingbot glambot panicbot rhodibot seambot" \ + "$(find "$TMP/o1" -name '*.json' -printf '%f\n' | sed 's/\.json$//' | sort | tr '\n' ' ' | sed 's/ $//')" +check "a member marked enrolled:false gets no file" no "$([ -e "$TMP/o1/cipherbot.json" ] && echo yes || echo no)" +check "the driver never writes not_applicable" 0 "$(cat "$TMP/o1"/*.json | jq -r .status | grep -c not_applicable || true)" + +run "$TMP/o1b" --member cipherbot --member rhodibot +check "--member honours enrolled:false" "rhodibot" "$(find "$TMP/o1b" -name '*.json' -printf '%f\n' | sed 's/\.json$//' | tr '\n' ' ' | sed 's/ $//')" + +echo 'nonsense' > "$TMP/bad.json" +run "$TMP/o1c" --enrolment "$TMP/bad.json" +check "an unreadable manifest runs nothing" 0 "$(find "$TMP/o1c" -name '*.json' | wc -l)" +check "...and says so" yes "$(grep -q 'enrolment manifest unreadable' "$TMP/o1c.log" && echo yes || echo no)" + +# 2. translation ─────────────────────────────────────────────────────────────── +r="$TMP/o1/rhodibot.json" +check "rhodibot: satisfied rows are not findings" 2 "$(field "$r" '.findings | length')" +check "rhodibot: missing is error" error "$(field "$r" '.findings[] | select(.rule_id == "1.1.2") | .severity')" +check "rhodibot: relocated is info" info "$(field "$r" '.findings[] | select(.rule_id == "4.1") | .severity')" +check "rhodibot: a missing file is named where the canon keeps it" Justfile "$(field "$r" '.findings[] | select(.rule_id == "1.1.2") | .file')" +check "rhodibot: a relocated file is named where it was found, so the diff can match it" .well-known/security.txt "$(field "$r" '.findings[] | select(.rule_id == "4.1") | .file')" +check "rhodibot: title comes from the manifest" "rhodibot (RSR compliance)" "$(field "$r" .title)" +check "panicbot: fleet Finding passes through" "PA018|warning|src/a.jl|0" "$(field "$TMP/o1/panicbot.json" '.findings[0] | "\(.rule_id)|\(.severity)|\(.file)|\(.line)"')" +check "seambot: check name becomes the rule id" "SEAM-register-exists|error|spec/seams/seam-register.json" \ + "$(field "$TMP/o1/seambot.json" '.findings[0] | "\(.rule_id)|\(.severity)|\(.file)"')" +check "finishingbot: analyzers are flattened" 2 "$(field "$TMP/o1/finishingbot.json" '.findings | length')" +check "finishingbot: ./ prefix is stripped" config/tools.yml "$(field "$TMP/o1/finishingbot.json" '.findings[] | select(.rule_id == "LIC-002") | .file')" +check "finishingbot: a repository-level finding has an empty path" "" "$(field "$TMP/o1/finishingbot.json" '.findings[] | select(.rule_id == "REL-001") | .file')" +check "glambot: line numbers survive" 7 "$(field "$TMP/o1/glambot.json" '.findings[0].line')" +check "accessibilitybot: absolute paths become repository-relative" src/index.html "$(field "$TMP/o1/accessibilitybot.json" '.findings[0].file')" +check "a default title is used when the manifest has none" "glambot (presentation quality)" "$(field "$TMP/o1/glambot.json" .title)" + +# 3. failure modes ───────────────────────────────────────────────────────────── +check "non-zero exit with valid JSON is still a run (finishingbot)" ran "$(field "$TMP/o1/finishingbot.json" .status)" +check "non-zero exit with valid JSON is still a run (accessibilitybot)" ran "$(field "$TMP/o1/accessibilitybot.json" .status)" + +rm "$BIN/panic-attack" +run "$TMP/o3" --member panicbot +check "a crash without JSON is error" error "$(field "$TMP/o3/panicbot.json" .status)" +check "...with the adapter's first diagnostic as the reason" yes \ + "$(field "$TMP/o3/panicbot.json" .reason | grep -q 'Failed to run panic-attack' && echo yes || echo no)" +stub panic-attack 'echo stub' + +run "$TMP/o4" --member glambot --bin-dir "$TMP/nowhere" +check "a missing binary is error" error "$(field "$TMP/o4/glambot.json" .status)" +check "...naming the binary and the crate" yes \ + "$(field "$TMP/o4/glambot.json" .reason | grep -q "glambot.*bots/glambot" && echo yes || echo no)" + +stub seambot 'sleep 5; echo "{}"' +run "$TMP/o5" --member seambot --timeout 1 +check "a timeout is error" error "$(field "$TMP/o5/seambot.json" .status)" +check "...and says so" yes "$(field "$TMP/o5/seambot.json" .reason | grep -q timeout && echo yes || echo no)" + +# 4. members without a local mode ────────────────────────────────────────────── +check "echidnabot without an engine is not_run" not_run "$(field "$TMP/o1/echidnabot.json" .status)" +check "...naming ECHIDNA_URL" yes "$(field "$TMP/o1/echidnabot.json" .reason | grep -q ECHIDNA_URL && echo yes || echo no)" +run "$TMP/o6" --enrolment "" --member sustainabot --member nosuchbot +check "sustainabot is not_run (reserved slot)" not_run "$(field "$TMP/o6/sustainabot.json" .status)" +check "an unknown member is error" error "$(field "$TMP/o6/nosuchbot.json" .status)" + +# 4b. a large findings list must not die on ARG_MAX (seen live: 146 panicbot +# findings killed jq with "Argument list too long" when passed via argv). +stub cipherbot '[ "$1 $2 $3" = "--format json scan" ] || { echo "bad args: $*" >&2; exit 9; } +jq -n -c '"'"'[range(6000) | {rule_id: "CIPHER-\(.)", severity: "warning", file: "src/f\(.).jl", line: ., message: ("m" * 60)}]'"'"'' +run "$TMP/o4b" --enrolment "" --member cipherbot +check "a 6000-finding envelope is written (no ARG_MAX failure)" ran "$(field "$TMP/o4b/cipherbot.json" .status)" +check "...with every finding kept" 6000 "$(field "$TMP/o4b/cipherbot.json" '.findings | length')" +stub cipherbot '[ "$1 $2 $3" = "--format json scan" ] || { echo "bad args: $*" >&2; exit 9; } +echo "[]"' + +# 5. envelope shape ──────────────────────────────────────────────────────────── +check "envelope keys are exactly the spec's, sorted" "findings reason ref reporter status title" \ + "$(field "$r" 'keys | join(" ")')" +check "the envelope is compact with one final LF" 1 "$(wc -l < "$r")" +check "the fleet ref is recorded" deadbeef "$(field "$r" .ref)" +check "reason is null when the member ran" null "$(field "$r" .reason)" +check "finding keys are exactly the spec's" "file line message rule_id severity" "$(field "$r" '.findings[0] | keys | join(" ")')" + +# 6. --plan ──────────────────────────────────────────────────────────────────── +plan=$(bash "$DRIVER" --plan --enrolment "$REPO/.gitbot-fleet/enrolment.json") +check "--plan lists only enrolled members" 7 "$(printf '%s\n' "$plan" | wc -l)" +check "--plan names the finishingbot crate and binary" "finishingbot finishingbot finishing-bot" "$(printf '%s\n' "$plan" | grep '^finishingbot')" +check "--plan gives echidnabot no crate: nothing is built for a member without a local mode" "echidnabot " "$(printf '%s\n' "$plan" | grep '^echidnabot')" + +# 7. summary ─────────────────────────────────────────────────────────────────── +check "summary has one row per member" 7 "$(grep -c '^| [a-z]' "$TMP/o1.md")" +check "summary states the error reason" yes "$(grep -q '| echidnabot | not_run | 0 | no ECHIDNA' "$TMP/o1.md" && echo yes || echo no)" + +# 9. boundary and regression cases ──────────────────────────────────────────── +# Only boolean true enrols a member; JSON truthiness must not enrol a string. +cat > "$TMP/strict.json" <<'JSON' +{"reporters":{"cipherbot":{"enrolled":true,"title":""}, + "panicbot":{"enrolled":"true"},"glambot":{"enrolled":1}, + "seambot":{"enrolled":null},"rhodibot":{},"sustainabot":{"enrolled":false}}} +JSON +run "$TMP/strict" --enrolment "$TMP/strict.json" +check "only boolean true enrols a member" cipherbot \ + "$(find "$TMP/strict" -name '*.json' -printf '%f\n' | sed 's/\.json$//')" +check "an empty title falls back to the built-in title" 'cipherbot (cryptographic hygiene)' \ + "$(field "$TMP/strict/cipherbot.json" .title)" + +for manifest in missing empty wrong-shape nobody; do + case "$manifest" in + missing) ;; # Deliberately absent. + empty) : > "$TMP/$manifest.json" ;; + wrong-shape) echo '{"reporters":[]}' > "$TMP/$manifest.json" ;; + nobody) echo '{"reporters":{}}' > "$TMP/$manifest.json" ;; + esac + run "$TMP/manifest-$manifest" --enrolment "$TMP/$manifest.json" + check "$manifest manifest produces no envelopes" 0 \ + "$(find "$TMP/manifest-$manifest" -name '*.json' | wc -l)" +done +run "$TMP/disabled" --member cipherbot --member sustainabot --summary "$TMP/disabled.md" +check "an explicitly selected list of disabled members stays empty" 0 \ + "$(find "$TMP/disabled" -name '*.json' | wc -l)" +check "an empty selection gets an explanatory summary" yes \ + "$(grep -q 'No member is enrolled' "$TMP/disabled.md" && echo yes || echo no)" + +# A clean scan has an envelope, for each distinct adapter JSON shape. +for member in rhodibot panicbot seambot finishingbot glambot accessibilitybot cipherbot; do + case "$member" in + panicbot|cipherbot) shape='[]' ;; + finishingbot|glambot) shape='{"first":{"findings":[]},"second":{"findings":[]}}' ;; + *) shape='{"findings":[]}' ;; + esac + binary="$member"; [ "$member" != finishingbot ] || binary=finishing-bot + stub "$binary" "printf '%s\n' '$shape'" + run "$TMP/clean-$member" --enrolment "" --member "$member" + check "$member clean scan is ran with no findings or reason" 'ran|0|null' \ + "$(field "$TMP/clean-$member/$member.json" '"\(.status)|\(.findings | length)|\(.reason)"')" +done + +for alias in finishbot finishing-bot; do + run "$TMP/alias-$alias" --enrolment "" --member "$alias" + check "$alias uses finishing-bot and retains the requested reporter name" "$alias|ran" \ + "$(field "$TMP/alias-$alias/$alias.json" '"\(.reporter)|\(.status)"')" +done +plan=$(bash "$DRIVER" --plan --enrolment "" --member finishbot --member finishing-bot --member cipherbot) +check "--plan maps aliases and cipherbot to their crates and binaries" \ + $'finishbot\tfinishingbot\tfinishing-bot\nfinishing-bot\tfinishingbot\tfinishing-bot\ncipherbot\tcipherbot\tcipherbot' "$plan" + +stub rhodibot 'cat <<JSON +{"findings":[ + {"id":"D","name":"Old canon","severity":"deprecated","found":["./legacy.txt"],"canon_keeps_it_at":"new.txt","details":["replace","retire"]}, + {"id":"M","name":"Missing","severity":"missing","expected":["expected.txt"],"found":["wrong.txt"]}, + {"id":"R","name":"Relocated","severity":"relocated","found":[],"expected":["fallback.txt"]}, + {"id":"S","name":"Satisfied","severity":"satisfied"}]} +JSON' +run "$TMP/canon" --member rhodibot +check "canon deprecated severity, actual path, details and fallback paths survive" \ + '[{"file":"legacy.txt","line":0,"message":"Old canon: deprecated — replace; retire","rule_id":"D","severity":"warning"},{"file":"expected.txt","line":0,"message":"Missing: missing","rule_id":"M","severity":"error"},{"file":"fallback.txt","line":0,"message":"Relocated: relocated","rule_id":"R","severity":"info"}]' \ + "$(jq -S -c .findings "$TMP/canon/rhodibot.json")" + +stub cipherbot 'cat <<JSON +[{"id":"fallback","severity":null,"file":42,"line":"12","message":17}, + {}, + {"rule_id":"primary","id":"ignored","severity":"high","file":"$4/src/a.jl","line":12,"message":"message"}, + {"rule_id":7,"severity":3,"location":"./src/b.jl","line":null}, + {"rule_id":"sibling","file":"$4-other/src/a.jl","line":0}] +JSON' +run "$TMP/normalize" --enrolment "" --member cipherbot +check "fleet fields have stable defaults, types and precedence" \ + '[{"file":"","line":0,"message":"17","rule_id":"fallback","severity":"unknown"},{"file":"","line":0,"message":"","rule_id":"?","severity":"unknown"},{"file":"src/a.jl","line":12,"message":"message","rule_id":"primary","severity":"high"},{"file":"src/b.jl","line":0,"message":"","rule_id":"7","severity":"3"}]' \ + "$(jq -S -c '.findings[:4]' "$TMP/normalize/cipherbot.json")" +check "relativization does not strip a sibling directory with the same prefix" "$REPO-other/src/a.jl" \ + "$(field "$TMP/normalize/cipherbot.json" '.findings[4].file')" + +stub cipherbot 'jq -n '\''["error","warning","info","suggestion","critical","high","medium","low"] | map({severity: .})'\''' +run "$TMP/severities" --enrolment "" --member cipherbot +check "fleet and scanner severities reach the renderer unchanged" \ + 'error warning info suggestion critical high medium low' \ + "$(field "$TMP/severities/cipherbot.json" '[.findings[].severity] | join(" ")')" + +# Success exit codes alone do not prove the adapter produced usable findings. +for output in empty malformed wrong-shape; do + case "$output" in + empty) stub cipherbot 'exit 0' ;; + malformed) stub cipherbot 'echo "not JSON"' ;; + wrong-shape) stub cipherbot 'echo 42' ;; + esac + run "$TMP/output-$output" --enrolment "" --member cipherbot --member glambot + check "$output adapter output is an error with no findings" 'error|0' \ + "$(field "$TMP/output-$output/cipherbot.json" '"\(.status)|\(.findings | length)"')" + case "$output" in wrong-shape) phrase="unexpected output shape" ;; *) phrase="no readable JSON" ;; esac + check "$output adapter output explains the error" yes \ + "$(field "$TMP/output-$output/cipherbot.json" .reason | grep -q "$phrase" && echo yes || echo no)" + check "$output adapter output does not prevent a later member running" ran \ + "$(field "$TMP/output-$output/glambot.json" .status)" +done + +stub cipherbot 'printf "\nStack backtrace:\nfirst diagnostic\nsecond diagnostic\n" >&2; exit 17' +run "$TMP/diagnostic" --enrolment "" --member cipherbot +check "errors use the first nonblank diagnostic, skipping backtrace headers" \ + 'adapter produced no readable JSON: first diagnostic' "$(field "$TMP/diagnostic/cipherbot.json" .reason)" +stub cipherbot 'exit 17' +run "$TMP/no-diagnostic" --enrolment "" --member cipherbot +check "a silent crash records its exit status" \ + 'adapter produced no readable JSON: exit status 17 with no diagnostic' "$(field "$TMP/no-diagnostic/cipherbot.json" .reason)" +stub cipherbot 'printf "%0300d\n" 0 >&2; exit 1' +run "$TMP/long-diagnostic" --enrolment "" --member cipherbot +check "diagnostics are bounded to 240 characters" 240 \ + "$(field "$TMP/long-diagnostic/cipherbot.json" '.reason | ltrimstr("adapter produced no readable JSON: ") | length')" +stub cipherbot 'echo "[]"; exit 124' +run "$TMP/timeout-json" --enrolment "" --member cipherbot +check "a timeout cannot become a clean scan even with valid JSON" \ + 'error|adapter exceeded the 300s timeout|0' \ + "$(field "$TMP/timeout-json/cipherbot.json" '"\(.status)|\(.reason)|\(.findings | length)"')" +stub cipherbot 'echo "[]"; exit 2' +run "$TMP/nonzero-json" --enrolment "" --member cipherbot +check "valid empty JSON with a nonzero exit is still a completed scan" 'ran|0|null' \ + "$(field "$TMP/nonzero-json/cipherbot.json" '"\(.status)|\(.findings | length)|\(.reason)"')" + +ECHIDNA_URL=https://engine.invalid run "$TMP/engine" --enrolment "" --member echidnabot +check "a configured engine still reports the unsupported dispatch honestly" 'not_run|0' \ + "$(field "$TMP/engine/echidnabot.json" '"\(.status)|\(.findings | length)"')" +check "a configured engine reason names the missing dispatch implementation" yes \ + "$(field "$TMP/engine/echidnabot.json" .reason | grep -q 'does not yet dispatch' && echo yes || echo no)" + +# Verify process context and argument boundaries using paths containing spaces. +stub cipherbot '[ "$#" -eq 4 ] && [ "$1 $2 $3" = "--format json scan" ] && [ "$PWD" = "$4" ] && [ "$RUST_LOG" = off ] || exit 9 +jq -n --arg path "$(command -v cipherbot)" '\''[{rule_id: "CONTEXT", severity: "info", message: $path}]'\''' +run "$TMP/context with spaces" --enrolment "" --member cipherbot +check "adapters run in the scanned tree with RUST_LOG off and bin-dir on PATH" "$BIN/cipherbot" \ + "$(field "$TMP/context with spaces/cipherbot.json" '.findings[0].message')" +PATH="$BIN:$PATH" run "$TMP/path-fallback" --enrolment "" --member cipherbot --bin-dir "$TMP/no-bin" +check "an absent bin-dir falls back to PATH" ran "$(field "$TMP/path-fallback/cipherbot.json" .status)" + +printf 'Existing summary\n' > "$TMP/append.md" +run "$TMP/append" --enrolment "" --member cipherbot --summary "$TMP/append.md" +check "--summary appends and preserves existing content" 'Existing summary' "$(head -n1 "$TMP/append.md")" +check "the default ref is the fleet checkout revision" "$(git -C "$ROOT" rev-parse HEAD)" \ + "$(bash "$DRIVER" --repo "$REPO" --out "$TMP/default-ref" --enrolment "" --member sustainabot >/dev/null; field "$TMP/default-ref/sustainabot.json" .ref)" + +expect_usage() { + local label="$1" rc=0; shift + bash "$DRIVER" "$@" > "$TMP/usage.log" 2>&1 || rc=$? + check "$label exits with usage status" 2 "$rc" +} +expect_usage "missing required paths" +expect_usage "nonexistent repository" --repo "$TMP/absent" --out "$TMP/unused" +expect_usage "unknown option" --unrecognized +for seconds in 0 -1 1.5 invalid; do + expect_usage "invalid timeout $seconds" --repo "$REPO" --out "$TMP/unused" --timeout "$seconds" +done +# Valid JSON of the wrong shape is a shape error, never a clean scan. +stub cipherbot 'echo "{}"' +run "$TMP/shape-obj" --enrolment "" --member cipherbot +check "an array adapter printing {} is an error" "error" "$(field "$TMP/shape-obj/cipherbot.json" .status)" +check "...naming the expected shape" yes "$(field "$TMP/shape-obj/cipherbot.json" .reason | grep -q "unexpected output shape: expected an array" && echo yes || echo no)" +stub seambot 'echo "[]"' +run "$TMP/shape-arr" --enrolment "" --member seambot +check "an object adapter printing [] is an error" "error" "$(field "$TMP/shape-arr/seambot.json" .status)" +stub accessibilitybot 'echo "{\"findings\": null}"' +run "$TMP/shape-null" --enrolment "" --member accessibilitybot +check "a null findings array is an error, not a clean scan" "error" "$(field "$TMP/shape-null/accessibilitybot.json" .status)" +# A manifest title is PR-controlled and rides into jq as an argument: bounded. +stub cipherbot 'echo "[]"' +jq -n --arg t "$(printf '%0500d' 0)" '{schema: 1, reporters: {cipherbot: {enrolled: true, title: $t}}}' > "$TMP/longtitle.json" +run "$TMP/longtitle" --enrolment "$TMP/longtitle.json" --member cipherbot +check "a 500-character manifest title is cut to 120" 120 "$(field "$TMP/longtitle/cipherbot.json" '.title | length')" + +# Analyzer collections may be arrays as well as name-keyed objects. Keep +# findings from every analyzer, including when the adapter exits nonzero. +for member in finishingbot glambot; do + binary="$member"; [ "$member" != finishingbot ] || binary=finishing-bot + stub "$binary" 'printf '\''[{"findings":[{"id":"FIRST","file":"./first.txt"}]},{"findings":[]},{"findings":[{"id":"LAST","file":"./last.txt"}]}]\n'\''; exit 1' + run "$TMP/analyzer-array-$member" --enrolment "" --member "$member" + check "$member accepts analyzer arrays after a nonzero exit" 'ran|null' \ + "$(field "$TMP/analyzer-array-$member/$member.json" '"\(.status)|\(.reason)"')" + check "$member keeps findings from every analyzer in order" 'FIRST:first.txt LAST:last.txt' \ + "$(field "$TMP/analyzer-array-$member/$member.json" '[.findings[] | "\(.rule_id):\(.file)"] | join(" ")')" + + # A partial analyzer report must never masquerade as a clean scan. In + # particular, validate every analyzer, even when the first one is valid. + shape_index=0 + for shape in '{}' '[]' '{"first":{}}' '{"first":{"findings":null}}' \ + '{"first":{"findings":{}}}' \ + '{"first":{"findings":[{"id":"PARTIAL"}]},"second":7}'; do + shape_index=$((shape_index + 1)) + stub "$binary" "printf '%s\n' '$shape'" + run "$TMP/analyzer-shape-$member-$shape_index" --enrolment "" --member "$member" + check "$member rejects incomplete analyzer shape $shape_index without findings" 'error|0' \ + "$(field "$TMP/analyzer-shape-$member-$shape_index/$member.json" '"\(.status)|\(.findings | length)"')" + check "$member explains the required analyzer shape $shape_index" \ + 'adapter output had an unexpected output shape: expected analyzer objects each with a findings array' \ + "$(field "$TMP/analyzer-shape-$member-$shape_index/$member.json" .reason)" + done +done + +stub seambot 'echo '\''{"findings":[{}, {"check":17,"severity":null,"location":42,"message":23,"line":99}]}'\''' +run "$TMP/seam-defaults" --member seambot +check "seambot normalizes missing and nonstring fields and uses repository-level lines" \ + '[{"file":"","line":0,"message":"","rule_id":"SEAM-?","severity":"unknown"},{"file":"","line":0,"message":"23","rule_id":"SEAM-17","severity":"unknown"}]' \ + "$(jq -S -c .findings "$TMP/seam-defaults/seambot.json")" + +# An explicitly supplied manifest replaces the default manifest's decisions. +# Quotes, backslashes and newlines in its title remain data in the envelope. +custom_title=$'Audit "quoted" \\ path\nsecond line' +jq -n --arg title "$custom_title" \ + '{reporters:{cipherbot:{enrolled:true,title:$title},rhodibot:{enrolled:false}}}' > "$TMP/override.json" +run "$TMP/override" --enrolment "$TMP/override.json" --member cipherbot --member rhodibot +check "explicit manifest overrides the default enrolment and filters selected members" cipherbot.json \ + "$(find "$TMP/override" -type f -printf '%f\n')" +check "manifest title metacharacters survive JSON construction" "$custom_title" \ + "$(field "$TMP/override/cipherbot.json" .title)" +plan=$(bash "$DRIVER" --plan --enrolment "$TMP/override.json" --member cipherbot --member rhodibot 2> "$TMP/override-plan.err") +check "planning honours explicit manifest exclusions just like execution" $'cipherbot\tcipherbot\tcipherbot' "$plan" + +# The slug limit is inclusive at 64 characters, and validation also applies +# to explicit members before the plan can reach the build helper. +slug64="a$(printf '%063d' 0)" +plan=$(bash "$DRIVER" --plan --enrolment "" --member "$slug64" --member bot-2 \ + --member "${slug64}a" --member 2bot --member under_score --member Uppercase --member dotted.name \ + --repo "$REPO" --out "$TMP/plan-only" --summary "$TMP/plan-only.md" 2> "$TMP/slug-limits.err") +check "planning accepts a 64-character slug and lowercase digits and hyphens" \ + "$(printf '%s\t\t\nbot-2\t\t' "$slug64")" "$plan" +check "planning warns for overlong slugs and invalid initial or interior characters" 5 \ + "$(grep -c 'warning: ignoring member' "$TMP/slug-limits.err")" +check "planning creates no output directory or summary" no \ + "$([ -e "$TMP/plan-only" ] || [ -e "$TMP/plan-only.md" ] && echo yes || echo no)" +run "$TMP/slug-limit" --enrolment "" --member "$slug64" --member "${slug64}a" +check "execution uses the same inclusive slug limit as planning" "$slug64.json" \ + "$(find "$TMP/slug-limit" -type f -printf '%f\n')" +check "a valid unregistered slug receives an explicit error envelope" 'error|unknown fleet member: no adapter is registered under this name' \ + "$(field "$TMP/slug-limit/$slug64.json" '"\(.status)|\(.reason)"')" + +# PATH fallback is already covered above; conflicting executables verify +# that an explicitly selected adapter directory takes precedence. +mkdir -p "$TMP/shadow-bin" +printf '#!/usr/bin/env bash\nexit 9\n' > "$TMP/shadow-bin/cipherbot" +chmod +x "$TMP/shadow-bin/cipherbot" +PATH="$TMP/shadow-bin:$PATH" run "$TMP/bin-precedence" --enrolment "" --member cipherbot +check "bin-dir takes precedence over a failing adapter on PATH" ran \ + "$(field "$TMP/bin-precedence/cipherbot.json" .status)" + +# A member name from the manifest is untrusted: a tab or a path in a key must +# never become a plan field, an envelope file name or an adapter argument. +printf '{"reporters":{"rhodibot":{"enrolled":true},"x\\t../../../evil\\tz":{"enrolled":true},"Bad Name":{"enrolled":true}},"schema":1}\n' > "$TMP/hostile.json" +plan=$(bash "$DRIVER" --plan --enrolment "$TMP/hostile.json" 2> "$TMP/hostile.err") +check "a member key with a tab or a path never reaches the plan" "rhodibot rhodibot rhodibot" "$plan" +check "...and each dropped name is warned about" 2 "$(grep -c "warning: ignoring member" "$TMP/hostile.err")" +mkdir -p "$TMP/hostile-out" +bash "$DRIVER" --repo "$REPO" --out "$TMP/hostile-out" --bin-dir "$BIN" --enrolment "" --member "../escape" > "$TMP/hostile-run.log" 2>&1 || true +check "an explicit --member that is not a slug writes nothing" "" "$(find "$TMP/hostile-out" -type f | sort | tr "\n" " ")" +check "...and leaves no file beside the output directory" no "$([ -e "$TMP/escape.json" ] && echo yes || echo no)" +check "usage errors create no output directory" no "$([ -e "$TMP/unused" ] && echo yes || echo no)" +check "reporting never changes the scanned tree's files or directories" "$repo_before" "$(repo_state)" + +# 8. the real renderer ───────────────────────────────────────────────────────── +GRID="${STANDARDS_DIR:-}/scripts/verdict-grid.sh" +if [ -n "${STANDARDS_DIR:-}" ] && [ -f "$GRID" ]; then + ( cd "$REPO" && bash "$GRID" render --out "$TMP/grid" --hypatia /dev/null --panic /dev/null --pons /dev/null \ + --report-backs "$TMP/o1" --enrolment .gitbot-fleet/enrolment.json --sha 0000000 >/dev/null ) + rowstate() { jq -r --arg id "$1" '.reporters[] | select(.id == $id) | .state' "$TMP/grid/report.json"; } + check "renderer: rhodibot's error finding is a red row" red "$(rowstate rhodibot)" + check "renderer: glambot's info finding is a blue row" blue "$(rowstate glambot)" + check "renderer: panicbot's warning is a yellow row" yellow "$(rowstate panicbot)" + check "renderer: echidnabot not_run is orange" orange "$(rowstate echidnabot)" + check "renderer: cipherbot not enrolled is grey" grey "$(rowstate cipherbot)" + check "renderer: no fleet row is left as 'enrolled, but no report-back'" 0 \ + "$(jq '[.reporters[] | select(.reason == "enrolled, but no report-back was received")] | length' "$TMP/grid/report.json")" + check "renderer: fleet rows are advisory, so the lane stays orange on missing Hypatia evidence" orange "$(jq -r .state "$TMP/grid/report.json")" +else + echo "SKIP: STANDARDS_DIR not set or has no scripts/verdict-grid.sh; renderer round-trip not run" +fi + +echo +echo "fleet-report-back-test: $pass passed, $fail failed" +[ "$fail" -eq 0 ] + +# Keep the build helper covered by the same entry point used by e2e.yml. +bash "$ROOT/tests/fleet-report-back-build-test.sh"