From f1649506ba66837847eb8f30a3ae8778790b56e5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:29:44 -0400 Subject: [PATCH] ci: check the whole-tree test plan in lint-repo, not in the planner suite The live cases of scripts/plan-test-lanes.test.sh planned the whole live tree, so the suite read every code file and .gitignore without declaring any of them: 2,378 of the 3,653 selection gaps the trace audit saw in shards 2 and 3. They move to a test-plan-corpus gate step in lint-repo, which runs on every non-draft pull request and merge group, with the same four checks: the bash plan is the runner's corpus with each suite once, the Python plan is every test module but the eval fixtures, no leg is empty, and every Node suite has a wrapper or a Node package. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/pr-require-checks.yml | 26 +++++++++++++++++++++++++ scripts/plan-test-lanes.test.sh | 22 ++++----------------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pr-require-checks.yml b/.github/workflows/pr-require-checks.yml index a982347dfb..439e542213 100644 --- a/.github/workflows/pr-require-checks.yml +++ b/.github/workflows/pr-require-checks.yml @@ -699,6 +699,31 @@ jobs: continue-on-error: true run: scripts/check-lane-coverage.sh --check + # The whole-tree test plan of the live tree: the bash plan is the bash + # runner's corpus, each suite once; the Python plan is every test module + # but the eval fixtures; no leg is empty; and every Node suite runs + # through a wrapper or a Node package. Each reads the whole tree, so no + # test selection can say when it must run, and it runs here on every pull + # request rather than as a case of scripts/plan-test-lanes.test.sh. + - name: Check the whole-tree test plan covers the live corpus + id: test_plan_corpus + continue-on-error: true + run: | + rc=0 + plan="$(bash scripts/plan-test-lanes.sh 2>/dev/null)" || { echo "::error::the whole-tree plan failed"; exit 1; } + legs() { sed -n "s/^$1_plan=//p" <<<"$plan"; } + diff <(legs bash | jq -r '.[][]' | LC_ALL=C sort) <(bash scripts/run-plugin-tests.sh --list | LC_ALL=C sort) || + { echo "::error::the whole-tree bash plan is not the bash runner's corpus, each suite once"; rc=1; } + diff <(legs python | jq -r '.[][]' | LC_ALL=C sort) <(git ls-files --cached --others --exclude-standard | + awk '{ b = $0; sub(/.*\//, "", b) } b ~ /^test_.*\.py$/ && !/\/evals\/fixtures\//' | LC_ALL=C sort) || + { echo "::error::the whole-tree Python plan is not every test module but the eval fixtures"; rc=1; } + [[ "$(legs bash | jq '[.[] | length > 0] | all')" == true && "$(legs python | jq '[.[] | length > 0] | all')" == true ]] || + { echo "::error::a leg of the whole-tree plan is empty"; rc=1; } + mapfile -t node_suites < <(git ls-files | grep -E '\.test\.(js|mjs|cjs)$') + bash scripts/plan-test-lanes.sh -- "${node_suites[@]}" >/dev/null || + { echo "::error::a Node suite runs through neither a wrapper nor a Node package"; rc=1; } + exit "$rc" + # A DIFF LINTS THE MARKDOWN IT CHANGED. markdownlint's rules read one file # at a time, so an unchanged file's verdict is the one its last green run # gave. The whole tree when there is no diff base (a schedule or dispatch @@ -986,6 +1011,7 @@ jobs: stale-base-overlap=${{ steps.stale_base.outcome }} docs-only-contract=${{ steps.docs_only_contract.outcome }} lane-coverage=${{ steps.lane_coverage.outcome }} + test-plan-corpus=${{ steps.test_plan_corpus.outcome }} markdown=${{ steps.markdown.outcome }} typos=${{ steps.typos.outcome }} gitleaks=${{ steps.gitleaks.outcome }} diff --git a/scripts/plan-test-lanes.test.sh b/scripts/plan-test-lanes.test.sh index aa7bf45763..b2594c1532 100755 --- a/scripts/plan-test-lanes.test.sh +++ b/scripts/plan-test-lanes.test.sh @@ -4,8 +4,7 @@ # goes wider than the selection only where the planner's header says, and a # selected suite no lane runs fails the plan. The fixture cases run a copy of # the planner and the selector in a throwaway repo; the LIVE cases hold the -# real tree: the whole-tree plan covers the whole corpus, every Node suite has -# a home, and scripts/test-windows-plan.txt names exactly the steps +# real tree: scripts/test-windows-plan.txt names exactly the steps # pr-test-windows.yml gates. # test-scope: .github/workflows/pr-test-windows.yml set -uo pipefail @@ -229,22 +228,9 @@ is "$RC" 2 check "--base with no ref is a usage error" $? # --- LIVE: the real tree ------------------------------------------------------- - -live="$(cd "$REPO_ROOT" && bash scripts/plan-test-lanes.sh 2>/dev/null)" || fail "the whole-tree plan of the live repo failed" -OUT="$live" -is "$(suites bash)" "$(cd "$REPO_ROOT" && bash scripts/run-plugin-tests.sh --list | LC_ALL=C sort)" && - is "$(key bash_plan | jq -r '[.[][]] | length' | tr -d '\r')" "$(suites bash | wc -l | tr -d ' ')" -check "live: the whole-tree bash plan is the runner's corpus, each suite once" $? -is "$(suites python)" "$(cd "$REPO_ROOT" && git ls-files --cached --others --exclude-standard | - awk '{ b = $0; sub(/.*\//, "", b) } b ~ /^test_.*\.py$/ && !/\/evals\/fixtures\//' | LC_ALL=C sort)" -check "live: the whole-tree Python plan is every test module but the eval fixtures" $? -is "$(key bash_plan | jq '[.[] | length > 0] | all' | tr -d '\r')" true && - is "$(key python_plan | jq '[.[] | length > 0] | all' | tr -d '\r')" true -check "live: no leg of the whole-tree plan is empty" $? - -mapfile -t node_suites < <(cd "$REPO_ROOT" && git ls-files | grep -E '\.test\.(js|mjs|cjs)$') -(cd "$REPO_ROOT" && bash scripts/plan-test-lanes.sh -- "${node_suites[@]}" >/dev/null 2>"$ERR") -check "live: every Node suite in the tree runs through a wrapper or a Node package" $? +# The whole-tree plan of the live tree reads every file, so it is the +# `test-plan-corpus` gate step of pr-require-checks.yml's lint-repo job, which +# runs on every pull request, not a case here. # The (job, key) pairs pr-test-windows.yml gates on, against the plan's lines. yml_pairs="$(awk '