diff --git a/core/scripts/check-repo-brain.sh b/core/scripts/check-repo-brain.sh index 02793f7..f79f2af 100755 --- a/core/scripts/check-repo-brain.sh +++ b/core/scripts/check-repo-brain.sh @@ -8,10 +8,24 @@ # in-repo copy). Installed into target repos by the documentation skill's # BOOTSTRAP pass (/wire-repo-brain). # -# Usage: bash scripts/check-repo-brain.sh [--fix] [repo-root] (default: cwd) +# Usage: bash scripts/check-repo-brain.sh [--fix] [--scope [--words ]] [repo-root] +# (repo-root default: cwd) # CI: one line — bash scripts/check-repo-brain.sh # --fix: rewrite drifted index lines from each target doc's `description` # (the one mechanical repair; everything else stays report-only) +# --scope a review's scope: one path per line, relative to repo-root +# (code files and docs, deleted ones included). The gate then checks +# only the docs in scope. A listed doc gets every check. A doc whose +# text names a listed file's basename, or that cites a token with a +# dot-separated part in --words, reports only its links into a listed +# file, those tokens, and (an index) its lines to the listed docs. +# Code edges are checked for the listed code files only, Q3 is skipped +# (root wiring is the repository's state, not a diff's), and with no +# doc in scope the doc checks are skipped. One stdout line says which: +# "check-repo-brain: scope — doc(s) to check" or +# "check-repo-brain: scope — no doc in scope; doc checks skipped". +# --words identifiers the scoped change removes, one per line; a doc +# that cites one of them in backticks is in scope # # Doc roots are discovered at the repo root AND at every sub-project (a # directory holding the language's project marker — see the adapter below), @@ -58,8 +72,11 @@ # against its declaring package or receiver — never against a same-named # member elsewhere. A token missing from the set still resolves when it # appears as a whole word in any non-markdown repo file (config keys, -# alert names, test helpers). A `pkg.Sym` whose package is not declared in -# this repo is external (stdlib, dependencies) and exempt. +# alert names, test helpers): in a git work tree the tracked and +# untracked-but-not-ignored files; elsewhere every file outside the +# dependency and build-output directories (bazel-*, node_modules, …). +# A `pkg.Sym` whose package is not declared in this repo is external +# (stdlib, dependencies) and exempt. # - lines carrying the ⚠️ stale flag or a *(planned)* marker are exempt from # symbol resolution and the description copy check (R9 Q2/Q7 exemptions); # the file:line ban has no exemption beyond URL spans, fenced code blocks, @@ -72,22 +89,37 @@ # 2 usage error, or an internal scanner failure (a failed scan is # inconclusive, never silently clean) # -# Uses only POSIX-portable tools: find, grep, sed, awk, head, sort, wc. No jq/python. +# Uses only POSIX-portable tools: find, grep, sed, awk, head, sort, wc, tr, xargs, +# plus git when the repository is a git work tree. No jq/python. # awk programs avoid interval expressions ({m,n}) — mawk, Debian's default, rejects them. set -u FIX=0 -if [[ "${1:-}" == "--fix" ]]; then - FIX=1 - shift -fi +SCOPE_LIST="" +WORDS_LIST="" +abs_list() { # -> absolute path of an existing file, or exit 2 + [[ -f "$1" ]] || { echo "check-repo-brain: no such list: $1" >&2; exit 2; } + printf '%s/%s\n' "$(cd "$(dirname "$1")" && pwd)" "$(basename "$1")" +} +while (( $# > 0 )); do + case "$1" in + --fix) FIX=1; shift ;; + --scope) SCOPE_LIST=$(abs_list "${2:-}") || exit 2; shift 2 ;; + --words) WORDS_LIST=$(abs_list "${2:-}") || exit 2; shift 2 ;; + --*) echo "check-repo-brain: unknown option: $1" >&2; exit 2 ;; + *) break ;; + esac +done +[[ -n "$WORDS_LIST" && -z "$SCOPE_LIST" ]] && { echo "check-repo-brain: --words needs --scope" >&2; exit 2; } REPO_ROOT="${1:-$(pwd)}" if [[ ! -d "$REPO_ROOT" ]]; then echo "check-repo-brain: not a directory: $REPO_ROOT" >&2 exit 2 fi cd "$REPO_ROOT" || exit 2 +SCOPED=0 +[[ -n "$SCOPE_LIST" ]] && SCOPED=1 {{include "scripts/repo-brain-adapter.sh"}} @@ -204,27 +236,88 @@ lifecycle_stale() { # -> reason when frontmatter marks the doc stale ('' have_code=0 lang_has_code && have_code=1 -# ---------- declaration set: built once, queried per token ---------- -DECLS="" PKGS="" -if (( have_code )); then - DECLS=$(mktemp) PKGS=$(mktemp) DECL_ALL=$(mktemp) - trap 'rm -f "$DECLS" "$PKGS"' EXIT - if ! ( set -o pipefail; lang_declarations | sort -u > "$DECL_ALL" ); then +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# ---------- declaration set: built once, on the first token to resolve ---------- +DECLS="$WORK/decls" PKGS="$WORK/pkgs" +decls_built=0 +build_declarations() { + (( decls_built )) && return 0 + decls_built=1 + if ! ( set -o pipefail; lang_declarations | sort -u > "$WORK/decl-all" ); then echo "check-repo-brain: internal scanner error — the declaration scan failed; treating the run as inconclusive" >&2 exit 2 fi - grep '^pkg:' "$DECL_ALL" | sed 's/^pkg://' > "$PKGS" - grep -v '^pkg:' "$DECL_ALL" > "$DECLS" - rm -f "$DECL_ALL" -fi + grep '^pkg:' "$WORK/decl-all" | sed 's/^pkg://' > "$PKGS" + grep -v '^pkg:' "$WORK/decl-all" > "$DECLS" + rm -f "$WORK/decl-all" +} is_repo_pkg() { grep -qxF "$1" "$PKGS" 2>/dev/null; } +# repo_files — the repository's non-markdown files, NUL-separated. A git work +# tree lists them through git — the tracked files, those of its submodules +# included, then the untracked ones git does not ignore — which never walks +# build outputs or symlinked caches; elsewhere find skips the directories that +# hold dependencies and build outputs. +repo_files() { + local -a spec=(-- . ':(exclude,glob)**/*.md' ':(exclude,glob)**/vendor/**') + if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + { git ls-files -z --cached --recurse-submodules "${spec[@]}" 2>/dev/null \ + || git ls-files -z --cached "${spec[@]}" 2>/dev/null; } \ + && git ls-files -z --others --exclude-standard "${spec[@]}" 2>/dev/null && return 0 + fi + find . \( -name .git -o -name vendor -o -name node_modules -o -name 'bazel-*' -o -name .venv \ + -o -name venv -o -name __pycache__ -o -name .tox -o -name dist -o -name target \) -prune \ + -o -type f ! -name '*.md' -print0 +} + +# repo_words — the repository's distinct words (runs of [A-Za-z0-9_.$]) in +# its non-markdown text files, one per line, read once per gate run and kept. +# grep ends every file's last line, so two files never fuse into one word, and +# skips binary files. +REPO_WORDS="$WORK/repo-words" +repo_words() { + if [[ ! -f "$REPO_WORDS" ]]; then + repo_files | xargs -0 grep -Ih -e '' -- 2>/dev/null | LC_ALL=C tr -cs 'A-Za-z0-9_.$' '\n' \ + | LC_ALL=C awk '!seen[$0]++' > "$REPO_WORDS" + fi + cat "$REPO_WORDS" +} + +# resolve_words — the tokens that occur as a whole word in any +# non-markdown repository file, one per line. A dotted token such as +# `Type.Method` matches any run of whole dot-separated parts of a word, as +# grep -w would match it, and a word that carries $ (a JavaScript identifier) +# also offers its $-free parts. A lookup in the word set is linear in the +# repository's size, where a fixed-string grep for hundreds of tokens is not. +resolve_words() { + repo_words | LC_ALL=C awk -v list="$1" ' + BEGIN { while ((getline t < list) > 0) if (t != "") want[t] = 1 } + { + n = split($0, p, ".") + for (i = 1; i <= n; i++) { + s = p[i] + if (s in want) { print s; delete want[s] } + for (j = i + 1; j <= n; j++) { + s = s "." p[j] + if (s in want) { print s; delete want[s] } + } + } + if (index($0, "$") > 0) { + n = split($0, p, /[$.]/) + for (i = 1; i <= n; i++) if (p[i] in want) { print p[i]; delete want[p[i]] } + } + }' + return 0 +} + # One fence-aware awk pass per bundle extracts everything Q2's doc scan needs: # P — a file:line citation outside fences/URLs/globs # S — a backticked symbol-shaped token to resolve # Tokens are then resolved as SETS (one grep against the declaration file, one -# repo-wide word grep for the whole unresolved batch) — never per token. +# pass over the repository for the whole unresolved batch) — never per token. # Language-specific shapes arrive as -v variables: file_ext, file_re, sym_re, qual_re. DOCSCAN_AWK=' FNR == 1 { fence = 0 } @@ -258,8 +351,89 @@ FNR == 1 { fence = 0 } } ' -# ---------- Q2: code→docs edges (repo-wide; resolved from repo root, then the -# citing file's own sub-project) ---------- +# ---------- the scope: which docs a scoped run checks, and how far ---------- +# A doc the scope lists (a doc the diff touches) gets every check. A doc that +# only cites the scope — its text names the basename of a listed file as a +# whole word, or it cites in backticks a word the change removes (a symbol the +# diff renames or deletes) — reports only what the scope can break in it: a +# doc link into a listed file, the backticked tokens that carry a removed +# word, and an index line to a listed doc (whose description the diff can +# change). Its other findings are the repository's state, for a +# whole-repository run. +# Every list below holds relative paths as find(1) prints them under a root. +SCOPE_DOCS="$WORK/scope-docs" # every doc a scoped run reads +FULL_DOCS="$WORK/full-docs" # the listed docs: every check +SCOPE_PATHS="$WORK/scope-paths" +BASENAMES="$WORK/basenames" +if (( SCOPED )); then + sed -e 's|^\./||' -e '/^$/d' "$SCOPE_LIST" | LC_ALL=C sort -u > "$SCOPE_PATHS" + sed 's|.*/||' "$SCOPE_PATHS" | LC_ALL=C sort -u > "$BASENAMES" + for r in "${ROOTS[@]}"; do find "$r" -type f -name '*.md'; done | LC_ALL=C sort -u > "$WORK/all-docs" + grep -xF -f "$SCOPE_PATHS" "$WORK/all-docs" > "$FULL_DOCS" + { + cat "$FULL_DOCS" + if [[ -s "$BASENAMES" && -s "$WORK/all-docs" ]]; then + tr '\n' '\0' < "$WORK/all-docs" | xargs -0 grep -lwF -f "$BASENAMES" 2>/dev/null + fi + if [[ -n "$WORDS_LIST" && -s "$WORDS_LIST" && -s "$WORK/all-docs" ]]; then + tr '\n' '\0' < "$WORK/all-docs" \ + | xargs -0 awk -v file_ext="$LANG_FILE_EXT" -v file_re="$LANG_FILE_RE" \ + -v sym_re="$LANG_SYMBOL_RE" -v qual_re="$LANG_QUALIFIED_RE" "$DOCSCAN_AWK" \ + | awk -F'\t' -v list="$WORDS_LIST" ' + BEGIN { while ((getline w < list) > 0) want[w] = 1 } + $1 == "S" { if ($4 in want) { print $2; next } + n = split($4, part, "."); for (i = 1; i <= n; i++) if (part[i] in want) { print $2; next } }' + fi + } | LC_ALL=C sort -u > "$SCOPE_DOCS" + if [[ -s "$SCOPE_DOCS" ]]; then + echo "check-repo-brain: scope — $(wc -l < "$SCOPE_DOCS" | tr -d ' ') doc(s) to check" \ + "($(wc -l < "$FULL_DOCS" | tr -d ' ') touched, every check; the rest cite the scope, scope lines only)" + else + echo "check-repo-brain: scope — no doc in scope; doc checks skipped" + fi +fi + +# bundle_docs [name] [full] — the docs a run checks under one root: +# every *.md (or every file of that name); on a scoped run only the docs in +# scope, or with "full" only the docs that get every check +bundle_docs() { + local name="${2:-*.md}" list="$SCOPE_DOCS" + [[ "${3:-}" == full ]] && list="$FULL_DOCS" + if (( SCOPED )); then + find "$1" -type f -name "$name" | grep -xF -f "$list" + else + find "$1" -type f -name "$name" + fi + return 0 +} + +# full_doc — exit 0 iff the doc gets every check +full_doc() { (( ! SCOPED )) || grep -qxF -- "$1" "$FULL_DOCS"; } + +# link_path — the link's target as a +# repository-relative path, the title and anchor stripped and . and .. folded +# by text, so a target whose directory does not exist still has a path +link_path() { + local t="${2%%#*}" + t="${t%% *}" + if [[ "$t" == /* ]]; then t="$3/${t#/}"; else t="$(dirname "$1")/$t"; fi + printf '%s\n' "$t" | awk -F/ '{ + n = 0 + for (i = 1; i <= NF; i++) { + if ($i == "" || $i == ".") continue + if ($i == ".." && n > 0 && st[n] != "..") { n--; continue } + st[++n] = $i + } + o = ""; for (i = 1; i <= n; i++) o = o (i > 1 ? "/" : "") st[i]; print o + }' +} + +# in_list — exit 0 iff the path is a line of the list +in_list() { grep -qxF -- "$1" "$2"; } + +# ---------- Q2: code→docs edges (repo-wide, or the scope's code files on a +# scoped run; resolved from repo root, then the citing file's own +# sub-project) ---------- docroot_for_file() { # -> docroot of the longest matching project dir local f="${1#./}" best="" i for i in "${!PROJS[@]}"; do @@ -283,7 +457,13 @@ if (( have_code )); then CUR_DOCROOT=$(docroot_for_file "$file") fail "[Q2] $file:$line — code edge points at missing $target" fi - done < <(lang_code_edges) + done < <(if (( SCOPED )); then + lang_code_edges | awk -v list="$SCOPE_PATHS" ' + BEGIN { while ((getline l < list) > 0) s[l] = 1 } + { f = substr($0, 1, index($0, ":") - 1); sub(/^\.\//, "", f); if (f in s) print }' + else + lang_code_edges + fi) fi # ---------- per-bundle checks ---------- @@ -293,6 +473,8 @@ check_bundle() { # local root_index="$docroot/index.md" local root_index_c root_index_c=$(canon "$root_index") + # a scoped run skips a bundle that holds no doc in scope + (( SCOPED )) && [[ -z "$(bundle_docs "$docroot")" ]] && return 0 # --- Q1: transitive reachability from the bundle's root index --- local reachable="" visited="" queue=("$root_index") @@ -311,7 +493,7 @@ check_bundle() { # [[ "$(basename "$resolved")" == "index.md" ]] && queue+=("$resolved") done < <(grep -oE '\]\([^)]+\)' "$idx" 2>/dev/null) done - if [[ ! -f "$root_index" ]]; then + if [[ ! -f "$root_index" ]] && [[ -n "$(bundle_docs "$docroot" '*.md' full)" ]]; then fail "[Q1] $docroot — no index.md: the bundle has no map" fi while IFS= read -r doc; do @@ -321,14 +503,17 @@ check_bundle() { # case "$reachable" in *"$c"$'\n'*) ;; *) fail "[Q1] $doc — orphan: not reachable from $root_index" ;; esac - done < <(find "$docroot" -type f -name '*.md') + done < <(bundle_docs "$docroot" '*.md' full) # --- Q2: every doc link resolves (a missing parent directory is just as - # broken as a missing file — only URLs/anchors are exempt) --- + # broken as a missing file — only URLs/anchors are exempt); a doc that only + # cites the scope reports only its links into a listed file --- while IFS= read -r md; do + local full=0; full_doc "$md" && full=1 while IFS= read -r raw; do local t="${raw#](}"; t="${t%)}" [[ "$t" == *.md* ]] || continue + (( full )) || in_list "$(link_path "$md" "$t" "$docroot")" "$SCOPE_PATHS" || continue local resolved rc=0 resolved=$(resolve_link "$md" "$t" "$docroot") || rc=$? (( rc == 1 )) && continue @@ -336,13 +521,14 @@ check_bundle() { # fail "[Q2] $md — link target does not exist: $t" fi done < <(grep -oE '\]\([^)]+\)' "$md" 2>/dev/null) - done < <(find "$docroot" -type f -name '*.md') + done < <(bundle_docs "$docroot") # --- Q2: doc scan — file:line ban + docs→code symbol resolution. - # One awk pass extracts; resolution is set-based (see DOCSCAN_AWK above). --- + # One awk pass extracts; resolution is set-based (see DOCSCAN_AWK above). A + # doc that only cites the scope keeps only its backticked removed words. --- local scan; scan=$(mktemp) if ! ( set -o pipefail - find "$docroot" -type f -name '*.md' -print0 \ + bundle_docs "$docroot" | tr '\n' '\0' \ | xargs -0 awk -v file_ext="$LANG_FILE_EXT" -v file_re="$LANG_FILE_RE" \ -v sym_re="$LANG_SYMBOL_RE" -v qual_re="$LANG_QUALIFIED_RE" \ "$DOCSCAN_AWK" > "$scan" ); then @@ -350,6 +536,14 @@ check_bundle() { # rm -f "$scan" exit 2 fi + if (( SCOPED )); then + awk -F'\t' -v full="$FULL_DOCS" -v words="${WORDS_LIST:-/dev/null}" ' + BEGIN { while ((getline l < full) > 0) f[l] = 1; while ((getline l < words) > 0) w[l] = 1 } + ($2 in f) { print; next } + $1 == "S" { if ($4 in w) { print; next } + n = split($4, part, "."); for (i = 1; i <= n; i++) if (part[i] in w) { print; next } }' "$scan" > "$scan.kept" + mv -f "$scan.kept" "$scan" + fi local f ln while IFS=$'\t' read -r _ f ln; do fail "[Q2] $f:$ln — cites a file path or line number (churn-prone coordinate)" @@ -357,6 +551,7 @@ check_bundle() { # if (( have_code )) && grep -q $'^S\t' "$scan"; then local toks check members unres bad toks=$(mktemp) check=$(mktemp) members=$(mktemp) unres=$(mktemp) bad=$(mktemp) + build_declarations grep $'^S\t' "$scan" | cut -f4 | sort -u > "$toks" # tokens to resolve (external pkg.Sym exempt). A qualified token resolves # as the PAIR itself — the declaration set carries pkg.Ident / Type.Method @@ -374,12 +569,11 @@ check_bundle() { # esac done < "$toks" > "$check" cut -f2 "$check" | sort -u > "$members" - grep -vxF -f "$DECLS" "$members" > "$unres" || true + grep -vxF -f "$DECLS" "$members" | grep -v '^$' > "$unres" || true if [[ -s "$unres" ]]; then - # ONE repo-wide word grep for the whole unresolved batch + # ONE pass over the repository for the whole unresolved batch local found; found=$(mktemp) - grep -rIhoFw --exclude-dir=vendor --exclude-dir=.git --exclude='*.md' \ - -f "$unres" . 2>/dev/null | sort -u > "$found" + resolve_words "$unres" | sort -u > "$found" grep -vxF -f "$found" "$unres" > "$bad" || true rm -f "$found" fi @@ -397,24 +591,27 @@ check_bundle() { # fi rm -f "$scan" - # --- Q3: root wiring (exact path; sub-roots may ride the repo-root index) --- - local rel="$docroot" - [[ "$proj" != "." ]] && rel="${docroot#$proj/}" - local claude="CLAUDE.md" agents="AGENTS.md" - [[ "$proj" != "." ]] && { claude="$proj/CLAUDE.md"; agents="$proj/AGENTS.md"; } - local wired_claude=0 wired_agents=0 - [[ -f "$claude" ]] && grep -qF -- "$rel/index.md" "$claude" && wired_claude=1 - [[ -f "$agents" ]] && grep -qF -- "$rel/index.md" "$agents" && wired_agents=1 - if (( ! wired_claude && ! wired_agents )); then - local via_root=0 - if [[ "$proj" != "." && -n "$ROOT_BUNDLE" ]]; then - grep -rqF -- "$docroot/index.md" "$ROOT_BUNDLE" --include='*.md' 2>/dev/null && via_root=1 - fi - if (( ! via_root )); then - fail "[Q3] $proj — neither $claude nor $agents references $rel/index.md" + # --- Q3: root wiring (exact path; sub-roots may ride the repo-root index); + # a scoped run leaves it to a whole-repository run --- + if (( ! SCOPED )); then + local rel="$docroot" + [[ "$proj" != "." ]] && rel="${docroot#$proj/}" + local claude="CLAUDE.md" agents="AGENTS.md" + [[ "$proj" != "." ]] && { claude="$proj/CLAUDE.md"; agents="$proj/AGENTS.md"; } + local wired_claude=0 wired_agents=0 + [[ -f "$claude" ]] && grep -qF -- "$rel/index.md" "$claude" && wired_claude=1 + [[ -f "$agents" ]] && grep -qF -- "$rel/index.md" "$agents" && wired_agents=1 + if (( ! wired_claude && ! wired_agents )); then + local via_root=0 + if [[ "$proj" != "." && -n "$ROOT_BUNDLE" ]]; then + grep -rqF -- "$docroot/index.md" "$ROOT_BUNDLE" --include='*.md' 2>/dev/null && via_root=1 + fi + if (( ! via_root )); then + fail "[Q3] $proj — neither $claude nor $agents references $rel/index.md" + fi + elif (( ! wired_agents )); then + note "[Q3] $agents lacks the $rel/index.md routing reference — AGENTS.md-reading tools start blind" fi - elif (( ! wired_agents )); then - note "[Q3] $agents lacks the $rel/index.md routing reference — AGENTS.md-reading tools start blind" fi # --- Q7: bundle contract --- @@ -479,12 +676,15 @@ check_bundle() { # else fail "[Q7] $md — frontmatter missing 'description:'" fi - done < <(find "$docroot" -type f -name '*.md') + done < <(bundle_docs "$docroot" '*.md' full) # --- Q7: drift check — index line text == target's description (⚠️ exempt; - # --fix rewrites drifted lines after the read loop, never during it) --- + # --fix rewrites drifted lines after the read loop, never during it). An + # index that only cites the scope checks only its lines to the touched docs, + # whose description the diff can change. --- while IFS= read -r idx; do - local lineno=0 line fixes="" + local lineno=0 line fixes="" ifull=0 + full_doc "$idx" && ifull=1 while IFS= read -r line; do lineno=$((lineno + 1)) case "$line" in *'⚠️'*) continue ;; esac @@ -492,6 +692,7 @@ check_bundle() { # local t; t=$(printf '%s' "$line" | sed -E 's/^- \[[^]]*\]\(([^)]*)\).*/\1/') local tail="${line#* — }" tail=$(printf '%s' "$tail" | sed -e 's/[[:space:]]*$//') + (( ifull )) || in_list "$(link_path "$idx" "$t" "$docroot")" "$FULL_DOCS" || continue local resolved resolved=$(resolve_link "$idx" "$t" "$docroot") || continue # broken links are Q2's report [[ -f "$resolved" ]] || continue @@ -517,12 +718,12 @@ check_bundle() { # echo " fixed: $idx:$n — index line rewritten from its target's description" done <<< "$fixes" fi - done < <(find "$docroot" -type f -name 'index.md') + done < <(bundle_docs "$docroot" index.md) # --- Q7: no log.md --- while IFS= read -r lg; do fail "[Q7] $lg — log.md is reserved for change history; docs describe current behavior" - done < <(find "$docroot" -type f -name 'log.md') + done < <(bundle_docs "$docroot" log.md full) } for i in "${!PROJS[@]}"; do diff --git a/core/scripts/check-repo-brain_test.sh b/core/scripts/check-repo-brain_test.sh index 30fadd5..bd4b269 100755 --- a/core/scripts/check-repo-brain_test.sh +++ b/core/scripts/check-repo-brain_test.sh @@ -545,6 +545,113 @@ run_gate expect_exit 0 && expect_not "does not resolve" && expect_has "check-repo-brain: clean" && ok finish +# --- scoped runs (--scope, --words) and the symbol resolver --- +LISTS=$(mktemp -d) # the scope and word lists live outside the repository they describe +begin "--scope: a code-only scope that no doc cites skips the doc checks" +mk_conformant; mk_doc orphan "an unlinked doc" "Nothing links here." +printf '%s\n' "$FX_CODE_FILE" > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 0 && expect_has "scope — no doc in scope; doc checks skipped" && expect_not "orphan" && ok +finish + +begin "--scope: a listed doc is checked, an unlisted orphan and the root wiring are not" +mk_conformant; mk_doc orphan "an unlinked doc" "Nothing links here."; mk_doc other "another unlinked doc" "Nothing." +rm "$REPO/CLAUDE.md" "$REPO/AGENTS.md" +printf '%s\n' "$FX_CODE_FILE" docs/orphan.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "scope — 1 doc(s) to check" && expect_has "[Q1] docs/orphan.md — orphan" \ + && expect_not "docs/other.md" && expect_not "[Q3]" && ok +finish + +begin "--scope: a doc that only names a scope file reports only its links into the scope" +mk_conformant +mk_doc linker "links the scope" "See [the old doc](gone.md) and [a missing one](missing.md); code: \`$FX_CODE_FILE\` at $FX_CODE_FILE:12." +printf '%s\n' "$FX_CODE_FILE" docs/gone.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "0 touched" && expect_has "[Q2] docs/linker.md — link target does not exist: gone.md" \ + && expect_not "missing.md" && expect_not "orphan" && expect_not "file path or line number" && ok +finish + +begin "--scope --words: a doc that cites a removed symbol is checked" +mk_conformant +mk_doc ghost "a ghost symbol" "Uses \`SnapshotRunner\` and \`OtherGhost\`." +append_index "- [ghost.md](ghost.md) — a ghost symbol" +printf '%s\n' "$FX_CODE_FILE" > "$LISTS/scope.txt"; printf 'SnapshotRunner\n' > "$LISTS/words.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 0 && expect_has "no doc in scope" \ + && { run_gate --scope "$LISTS/scope.txt" --words "$LISTS/words.txt"; expect_exit 1; } \ + && expect_has "backticked \`SnapshotRunner\` does not resolve" && expect_not "OtherGhost" && ok +finish + +begin "Q2 in a git work tree a token resolves from tracked and untracked files, never from ignored ones" +mk_conformant +printf 'build/\n' > "$REPO/.gitignore" +mkdir -p "$REPO/build"; printf 'IgnoredOnlyName\n' > "$REPO/build/gen.yaml" +printf 'TrackedName\n' > "$REPO/tracked.yaml" +mk_doc names "alert names" "Fires \`TrackedName\`, \`UntrackedName\` and \`IgnoredOnlyName\`." +append_index "- [names.md](names.md) — alert names" +(cd "$REPO" && git init -q && git add -A && git -c user.name=t -c user.email=t@t commit -qm init) +printf 'UntrackedName\n' > "$REPO/untracked.yaml" +run_gate +expect_exit 1 && expect_has "\`IgnoredOnlyName\` does not resolve" \ + && expect_not "\`TrackedName\` does not resolve" && expect_not "\`UntrackedName\` does not resolve" && ok +finish + +begin "Q2 outside git a token does not resolve from a bazel-* output directory" +mk_conformant +mkdir -p "$REPO/bazel-out"; printf 'BazelOnlyName\n' > "$REPO/bazel-out/gen.yaml" +mk_doc bazel "bazel names" "Fires \`BazelOnlyName\`." +append_index "- [bazel.md](bazel.md) — bazel names" +run_gate +expect_exit 1 && expect_has "\`BazelOnlyName\` does not resolve" && ok +finish + +begin "Q2 a dotted token resolves on whole dot-separated words only" +mk_conformant +printf 'limits:\n key: Cfg.MaxRetries.value\n other: Cfg.MaxRetriesX\n' > "$REPO/limits.yaml" +mk_doc dotted "dotted names" "Reads \`Cfg.MaxRetries\`; \`Cfg.MaxRetri\` is a typo." +append_index "- [dotted.md](dotted.md) — dotted names" +run_gate +expect_exit 1 && expect_has "\`Cfg.MaxRetri\` does not resolve" && expect_not "\`Cfg.MaxRetries\` does not resolve" && ok +finish +begin "Q2 two files with no final newline do not fuse their last and first words" +mk_conformant +printf 'FirstToken' > "$REPO/a.yaml"; printf 'SecondToken' > "$REPO/b.yaml" +mk_doc fused "fused names" "Fires \`FirstToken\` and \`SecondToken\`." +append_index "- [fused.md](fused.md) — fused names" +run_gate +expect_exit 0 && expect_not "does not resolve" && ok +finish + +begin "--scope --words: a doc citing Type.Method is checked when the diff removes the type" +mk_conformant +mk_doc renamed "a renamed type" "Calls \`OldType.Run\`." +append_index "- [renamed.md](renamed.md) — a renamed type" +printf '%s\n' "$FX_CODE_FILE" > "$LISTS/scope.txt"; printf 'OldType\n' > "$LISTS/words.txt" +run_gate --scope "$LISTS/scope.txt" --words "$LISTS/words.txt" +expect_exit 1 && expect_has "backticked \`OldType.Run\` does not resolve" && ok +finish + +begin "--scope: a citing doc reports a link to the listed doc, never a link to a same-named doc elsewhere" +mk_conformant +mk_doc linker2 "links two guides" "See [old](a/guide.md \"Title\") and [other](b/guide.md)." +append_index "- [linker2.md](linker2.md) — links two guides" +printf '%s\n' "$FX_CODE_FILE" docs/a/guide.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "[Q2] docs/linker2.md — link target does not exist: a/guide.md" \ + && expect_not "b/guide.md" && ok +finish + +begin "--scope: a touched doc's new description is checked against the index that lists it" +mk_conformant +sed 's/^description: .*/description: a new description/' "$REPO/docs/retry-policy.md" > "$REPO/docs/rp.tmp" && mv "$REPO/docs/rp.tmp" "$REPO/docs/retry-policy.md" +printf '%s\n' docs/retry-policy.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "[Q7] docs/index.md:" && expect_has "drifted from retry-policy.md" && ok +finish + +rm -rf "$LISTS" + } # run_row_cases for ROW in $FX_ROWS; do diff --git a/core/scripts/ldd-detect.sh b/core/scripts/ldd-detect.sh index 25647f5..b351fd8 100755 --- a/core/scripts/ldd-detect.sh +++ b/core/scripts/ldd-detect.sh @@ -25,7 +25,14 @@ # --plan print the parsed detect lines and exit; nothing runs # --glob the scope is one group of a language the table # does not know, matched by this glob -# --test-re test-file pattern over relative paths, same case +# --test-re test-file pattern over relative paths, same case. +# A bundle carries the pattern ldd-scope.sh used +# (test-re.txt), and a custom group its glob +# (language.txt): the bundle's values win, and a +# --glob or --test-re that differs is a usage error +# --gate-budget wall-clock seconds the R9 gate may run (default: no +# limit); on expiry the gate is stopped and its +# questions are counted "inconclusive", never 0 # # The scope's language is the bundle's language.txt; without it, the files in # files.txt tell it by extension, and a list that spans several languages is @@ -39,10 +46,14 @@ # Detect-path: `` over the scope's relative paths (layer directories, # role-named packages); a hit's line is 0 # Detect-gate: Q the R9 gate's [Q] report lines, from one run of -# check-repo-brain.sh beside this script over --root; on a -# scoped bundle (no dirs.txt) only the lines whose file is in -# the scope — the gate's repository-wide findings belong to -# a whole-repository review +# check-repo-brain.sh beside this script over --root. On a +# scoped bundle (no dirs.txt) the gate runs with --scope: it +# checks the bundle's docs (docs.txt) in full, and in the +# docs that name a scope file or cite a symbol the diff +# removes only those links and symbols; it checks the code +# edges of the scope's code files and no root wiring; with +# no such doc its doc checks are skipped. The repository-wide findings belong to a +# whole-repository review # Detect: judgment no mechanical lead; the hunter runs the prose # A question with no detect line, two of them, or a malformed one is an error: # the generator's lint-core keeps the rule files honest, and this script refuses @@ -53,15 +64,18 @@ # the overflow row of a capped question: file "-", line 0, # excerpt "+ more hit(s) not listed" # hits-all.tsv the same columns, every hit, no cap and no overflow row -# counts.tsv rule question kind hits (judgment rows carry "-") +# counts.tsv rule question kind hits (judgment rows carry "-", gate +# rows "inconclusive" when the gate ran out of --gate-budget) # the last row is SUPPRESS - grep : suppression directives # on the diff's added lines when the bundle has a diff.patch, # else over every scope file # Two runs over one tree write byte-identical counts.tsv. # # Exit codes: 0 tables written · 2 usage error, a detect line that does not -# parse, a pattern grep rejects, or a gate run that failed -# (inconclusive, never silently clean) +# parse, a pattern grep rejects, a gate run that failed +# (inconclusive, never silently clean), or a non-empty scope in +# which no file matches the source glob (a glob mismatch, never +# an empty table) # # Uses only POSIX-portable tools: grep, sed, awk, sort, wc, head. GNU grep's # \b and \s are assumed, as the rules already assume them. @@ -77,6 +91,7 @@ CAP=40 PLAN=0 OPT_GLOB="" OPT_TEST_RE="" +GATE_BUDGET=0 BUNDLE="" usage() { sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } @@ -94,12 +109,14 @@ while (( $# > 0 )); do --plan) PLAN=1; shift ;; --glob) value "$1" $#; OPT_GLOB="$2"; shift 2 ;; --test-re) value "$1" $#; OPT_TEST_RE="$2"; shift 2 ;; + --gate-budget) value "$1" $#; GATE_BUDGET="$2"; shift 2 ;; -h|--help) usage ;; --*) die "unknown option: $1" ;; *) [[ -n "$BUNDLE" ]] && die "one bundle directory, got also: $1"; BUNDLE="$1"; shift ;; esac done [[ "$CAP" =~ ^[0-9]+$ ]] || die "--cap takes a number, got: $CAP" +[[ "$GATE_BUDGET" =~ ^[0-9]+$ ]] || die "--gate-budget takes a number of seconds, got: $GATE_BUDGET" [[ -d "$RULES_DIR" ]] || die "no rules directory: $RULES_DIR" RULES_DIR=$(cd "$RULES_DIR" && pwd) if (( ! PLAN )); then @@ -136,6 +153,7 @@ if (( ! PLAN )) && [[ -f "$BUNDLE/groups.txt" ]]; then fi args=(--root "$ROOT" --cap "$CAP") [[ -n "$OPT_TEST_RE" ]] && args+=(--test-re "$OPT_TEST_RE") + (( GATE_BUDGET > 0 )) && args+=(--gate-budget "$GATE_BUDGET") bash "$script" "${args[@]}" "$bundle" || { failed=1; echo "$SCRIPT_NAME: the $id group's detection pass failed; it is inconclusive" >&2; } echo done < "$BUNDLE/groups.txt" @@ -150,7 +168,9 @@ LANG_ID="" if (( ! PLAN )); then if [[ -f "$BUNDLE/language.txt" ]]; then IFS=$'\t' read -r LANG_ID glob < "$BUNDLE/language.txt" - [[ "$LANG_ID" == "custom" && -z "$OPT_GLOB" ]] && OPT_GLOB="$glob" + [[ "$LANG_ID" == "custom" && -n "$OPT_GLOB" && "$OPT_GLOB" != "$glob" ]] \ + && die "--glob '$OPT_GLOB' differs from the bundle's glob '$glob' (ldd-scope.sh wrote it; drop --glob)" + [[ "$LANG_ID" == "custom" ]] && OPT_GLOB="$glob" else ids="" while IFS= read -r line; do @@ -165,6 +185,14 @@ if (( ! PLAN )); then [[ "$ids" == *" "* ]] && die "the scope list spans several languages ($ids); ldd-scope.sh writes one bundle per language" LANG_ID="$ids" fi + # the test-file pattern ldd-scope.sh used, so the two scripts cannot disagree + # on what a test file is + if [[ -f "$BUNDLE/test-re.txt" ]]; then + b_test_re=$(cat "$BUNDLE/test-re.txt") + [[ -n "$OPT_TEST_RE" && "$OPT_TEST_RE" != "$b_test_re" ]] \ + && die "--test-re '$OPT_TEST_RE' differs from the bundle's '$b_test_re' (ldd-scope.sh wrote it; drop --test-re)" + OPT_TEST_RE="$b_test_re" + fi lang_configure "$LANG_ID" || die "this plugin has no language block for $LANG_ID" fi @@ -210,7 +238,7 @@ END { if (insec) flush() } ' plan_file=$(mktemp) -trap 'rm -f "$plan_file" "${tmp_hits:-}" "${tmp_hits_all:-}" "${tmp_counts:-}" "${gate_out:-}"' EXIT +trap 'rm -f "$plan_file" "${tmp_hits:-}" "${tmp_hits_all:-}" "${tmp_counts:-}" "${gate_out:-}" "${gate_scope:-}" "${gate_words:-}"' EXIT rule_files=$(find "$RULES_DIR" -maxdepth 1 -name 'R*.md' | awk -F/ '{ n = $NF; sub(/^R/, "", n); sub(/-.*$/, "", n); print n "\t" $0 }' | sort -n | cut -f2) [[ -n "$rule_files" ]] || die "no rule files (R*.md) under $RULES_DIR" while IFS= read -r f; do @@ -236,11 +264,13 @@ fi SRC_FILES=() TEST_FILES=() ALL_FILES=() +n_present=0 while IFS= read -r line; do f="${line%% (not bundled:*}" f="${f#./}" [[ -z "$f" ]] && continue [[ -f "$f" ]] || continue + n_present=$((n_present + 1)) [[ "$(lang_of_path "$f")" == "$LANG_ID" ]] || continue printf '%s\n' "$f" | grep -qE -- "$LANG_EXCLUDE_RE" && continue ALL_FILES+=("$f") @@ -248,6 +278,12 @@ done < <(LC_ALL=C sort -u "$FILES_LIST") for f in ${ALL_FILES[@]+"${ALL_FILES[@]}"}; do if lang_is_test "$f"; then TEST_FILES+=("$f"); else SRC_FILES+=("$f"); fi done +# A scope whose files exist but none of which is a source file is a glob +# mismatch between the scope and this run: an all-zero table would read as a +# clean review of nothing. +if (( ${#ALL_FILES[@]} == 0 && n_present > 0 )); then + die "0 source + 0 test files: none of the $n_present file(s) in $FILES_LIST is a $LANG_ID source file (pass the --glob ldd-scope.sh used)" +fi tmp_hits=$(mktemp) tmp_hits_all=$(mktemp) @@ -361,24 +397,98 @@ run_path() { # rm -f "$raw" "$err" } -# The R9 gate runs once, lazily; its [Q] lines are split by question. +# gate_scope_lists — what a scoped gate run checks: +# every path of the scope (deleted ones too: a doc can still link to them) and +# of docs.txt; and the identifiers the diff removes — on its removed lines and +# on none of its added ones — the symbols a renamed or deleted declaration +# takes away from the docs that cite them. Comment lines (a line that opens +# with //, #, /*, *, -- or """) carry prose, not symbols, and are skipped, +# and a word the scope's files still carry after the change is not removed. +gate_scope_lists() { + local scope="$1" words="$2" line f + { + while IFS= read -r line; do + f="${line%% (not bundled:*}"; f="${f#./}" + [[ -n "$f" ]] && printf '%s\n' "$f" + done < "$FILES_LIST" + [[ -f "$BUNDLE/docs.txt" ]] && cat "$BUNDLE/docs.txt" + } | LC_ALL=C sort -u > "$scope" + : > "$words" + [[ -s "$BUNDLE/diff.patch" ]] || return 0 + grep -vE '^(\+\+\+|---) ' "$BUNDLE/diff.patch" | grep -E '^[-+]' \ + | LC_ALL=C awk ' + { + s = substr($0, 1, 1); t = substr($0, 2) + if (t ~ /^[ \t]*(\/\/|#|\/\*|\*|--|""")/) next + while (match(t, /[A-Za-z_][A-Za-z0-9_]*/)) { + w = substr(t, RSTART, RLENGTH); t = substr(t, RSTART + RLENGTH) + if (s == "+") add[w] = 1; else del[w] = 1 + } + } + END { for (w in del) if (!(w in add)) print w }' \ + | LC_ALL=C sort > "$words.removed" + if (( ${#ALL_FILES[@]} > 0 )); then + cat -- "${ALL_FILES[@]}" | LC_ALL=C tr -cs 'A-Za-z0-9_' '\n' \ + | LC_ALL=C awk -v list="$words.removed" ' + BEGIN { while ((getline w < list) > 0) gone[w] = 1 } + ($0 in gone) { delete gone[$0] } + END { for (w in gone) print w }' | LC_ALL=C sort > "$words" + else + cp "$words.removed" "$words" + fi + rm -f "$words.removed" +} + +# The R9 gate runs once, lazily; its [Q] lines are split by question. With +# --gate-budget the gate runs as its own process group, so expiry stops every +# process it started, not only its shell. +GATE_STATE="whole repository" run_gate_once() { [[ -n "$gate_out" ]] && return 0 gate_out=$(mktemp) - local gate="$SCRIPT_DIR/check-repo-brain.sh" code + local gate="$SCRIPT_DIR/check-repo-brain.sh" code=0 start pid scope_line + local -a args=() [[ -f "$gate" ]] || die "Detect-gate needs $gate beside this script" - bash "$gate" "$ROOT" > "$gate_out" 2>&1 - code=$? + if [[ ! -f "$BUNDLE/dirs.txt" ]]; then + gate_scope=$(mktemp); gate_words=$(mktemp) + gate_scope_lists "$gate_scope" "$gate_words" + args=(--scope "$gate_scope" --words "$gate_words") + fi + if (( GATE_BUDGET == 0 )); then + bash "$gate" ${args[@]+"${args[@]}"} "$ROOT" > "$gate_out" 2>&1 || code=$? + else + start=$SECONDS + set -m + bash "$gate" ${args[@]+"${args[@]}"} "$ROOT" > "$gate_out" 2>&1 & + pid=$! + set +m + while kill -0 "$pid" 2>/dev/null; do + if (( SECONDS - start >= GATE_BUDGET )); then + kill -TERM -- "-$pid" 2>/dev/null + wait "$pid" 2>/dev/null + GATE_STATE="inconclusive — stopped after the ${GATE_BUDGET}s --gate-budget" + return 0 + fi + sleep 0.2 + done + wait "$pid" || code=$? + fi if (( code == 2 )); then head -5 "$gate_out" >&2 die "the R9 gate failed to run (exit 2); its questions are inconclusive" fi + scope_line=$(grep -m1 '^check-repo-brain: scope — ' "$gate_out") + [[ -n "$scope_line" ]] && GATE_STATE="scoped, ${scope_line#check-repo-brain: scope — }" return 0 } run_gate() { # local rule="$1" q="$2" qn="$3" raw tab run_gate_once + if [[ "$GATE_STATE" == inconclusive* ]]; then + printf '%s\t%s\tgate\tinconclusive\n' "$rule" "$q" >> "$tmp_counts" + return 0 + fi raw=$(mktemp) tab=$(printf '\t') # "[Qn] [:line] — — see " becomes @@ -396,15 +506,9 @@ run_gate() { # if (match(loc, /:[0-9]+$/)) { file = substr(loc, 1, RSTART - 1); line = substr(loc, RSTART + 1) + 0 } print file "\t" line "\t" rest }' > "$raw" - # A scoped review owns only the gate lines about its own files: an orphan - # doc, an unwired root or another file's broken edge is the repository's - # state, not the diff's, and is not a lead for this review's hunter. - if [[ ! -f "$BUNDLE/dirs.txt" ]] && [[ -s "$raw" ]]; then - scoped=$(mktemp); scope_list=$(mktemp) - printf '%s\n' ${ALL_FILES[@]+"${ALL_FILES[@]}"} > "$scope_list" - awk -F'\t' 'NR == FNR { in_scope[$0] = 1; next } ($1 in in_scope)' "$scope_list" "$raw" > "$scoped" - mv "$scoped" "$raw"; rm -f "$scope_list" - fi + # A scoped review owns only the gate lines about its own files and the docs + # that cite them; the gate's --scope run reports nothing else (an orphan doc + # elsewhere or an unwired root is the repository's state, not the diff's). record "$rule" "$q" gate "$raw" 0 rm -f "$raw" } @@ -454,6 +558,7 @@ n_gate=$(awk -F'\t' '$3 == "gate"' "$plan_file" | wc -l | tr -d ' ') n_judg=$(awk -F'\t' '$3 == "judgment"' "$plan_file" | wc -l | tr -d ' ') n_q=$(wc -l < "$plan_file" | tr -d ' ') echo "$SCRIPT_NAME: $n_q questions ($n_grep grep, $n_path path, $n_gate gate, $n_judg judgment) over ${#SRC_FILES[@]} source + ${#TEST_FILES[@]} test files → $BUNDLE/hits.tsv, counts.tsv" +(( n_gate > 0 )) && [[ -n "$gate_out" ]] && echo "R9 gate: $GATE_STATE" echo awk -F'\t' 'BEGIN { printf "%-9s %-4s %-9s %s\n", "rule", "q", "kind", "hits" } { q = ($2 == "-") ? "-" : "Q" $2; printf "%-9s %-4s %-9s %s\n", $1, q, $3, $4 }' "$BUNDLE/counts.tsv" @@ -473,12 +578,14 @@ awk -F'\t' ' f = fam[$1]; if (f == "") f = "other" nq[f]++ if ($3 == "judgment") { nj[f]++; next } + if ($4 == "inconclusive") { ni[f]++; next } if ($4 + 0 > 0) { hits[f] += $4; nh[f]++; rh[f, $1] += $4 } } END { for (i = 1; i <= 4; i++) { f = order[i] printf "%-15s %5d hit(s) in %d of %d questions, %d judgment", f, hits[f] + 0, nh[f] + 0, nq[f] + 0, nj[f] + 0 + if (ni[f] + 0 > 0) printf ", %d inconclusive", ni[f] per = ""; n = split(rules[f], rs, " ") for (j = 1; j <= n; j++) if (rh[f, rs[j]] + 0 > 0) per = per (per == "" ? "" : " · ") rs[j] " " rh[f, rs[j]] if (per != "") printf " — %s", per diff --git a/core/scripts/ldd-detect_test.sh b/core/scripts/ldd-detect_test.sh index cd70ffb..b6d7277 100644 --- a/core/scripts/ldd-detect_test.sh +++ b/core/scripts/ldd-detect_test.sh @@ -49,6 +49,11 @@ run_detect() { # [options...] — runs the script over $REPO with $BUNDLE OUT=$(cd "$REPO" && bash "$DETECT" "$@" --root . "$BUNDLE" 2>"$BUNDLE/.stderr"); CODE=$? ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" } +run_detect_bundle() { # [options...] — runs the script over $REPO with that bundle + local b="$1"; shift + OUT=$(cd "$REPO" && bash "$DETECT" "$@" --root . "$b" 2>"$BUNDLE/.stderr"); CODE=$? + ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" +} run_scope() { # [args...] — runs ldd-scope.sh over $REPO into $BUNDLE/scope-out rm -rf "$BUNDLE/scope-out" OUT=$(cd "$REPO" && bash "$SCOPE_SH" --out "$BUNDLE/scope-out" "$@" 2>"$BUNDLE/.stderr"); CODE=$? @@ -75,8 +80,9 @@ expect_not() { return 0 } expect_quiet() { [[ -z "$ERR" ]] || { bad "stderr not empty"; return 1; }; } -count_of() { # — the hits cell of counts.tsv, "-" for judgment - awk -F'\t' -v r="$1" -v q="$2" '$1 == r && $2 == q { print $4 }' "$BUNDLE/counts.tsv" +count_of() { # — the hits cell of counts.tsv (COUNTS_FILE, default the + # case bundle's), "-" for judgment + awk -F'\t' -v r="$1" -v q="$2" '$1 == r && $2 == q { print $4 }' "${COUNTS_FILE:-$BUNDLE/counts.tsv}" } expect_count() { # =n> local got; got=$(count_of "$1" "$2") @@ -216,6 +222,97 @@ expect_exit 0 && expect_count SUPPRESS - 1 \ && grep -q "^SUPPRESS - grep $FX_SRC_FILE 2 " "$BUNDLE/hits.tsv" && ok finish +# --- the bundle ldd-scope.sh writes, read by detection --- +SO="" # the scope bundle of the case: $BUNDLE/scope-out + +begin "a code-only scoped diff skips the R9 gate's doc checks" +mk_repo; git_commit_repo +printf '\n' >> "$REPO/$FX_SRC_FILE" # a code-only change +(cd "$REPO" && git -c user.name=t -c user.email=t@t commit -qam change) +run_scope --base HEAD~1; SO="$BUNDLE/scope-out" +run_detect_bundle "$SO" +expect_exit 0 && expect_has "R9 gate: scoped, no doc in scope; doc checks skipped" \ + && COUNTS_FILE="$SO/counts.tsv" expect_count R9 1 0 && ok +finish + +begin "a scoped diff that touches a doc gets that doc checked by the R9 gate" +mk_repo; git_commit_repo +printf '\n' >> "$REPO/$FX_SRC_FILE" # a code-only change +printf '\nMore text.\n' >> "$REPO/docs/orphan.md" +(cd "$REPO" && git -c user.name=t -c user.email=t@t commit -qam change) +run_scope --base HEAD~1; SO="$BUNDLE/scope-out" +expect_exit 0 && { grep -qx 'docs/orphan.md' "$SO/docs.txt" || bad "docs.txt lacks the touched doc"; } \ + && grep -qx 'docs/orphan.md' "$SO/docs.txt" \ + && { run_detect_bundle "$SO"; expect_exit 0; } \ + && expect_has "R9 gate: scoped, 1 doc(s) to check" \ + && COUNTS_FILE="$SO/counts.tsv" expect_count R9 1 1 \ + && { grep -q "^R9 1 gate docs/orphan.md " "$SO/hits.tsv" || bad "no R9 Q1 hit row for docs/orphan.md"; } \ + && grep -q "^R9 1 gate docs/orphan.md " "$SO/hits.tsv" && ok +finish + +begin "detection reads the source glob from the bundle ldd-scope.sh wrote" +mk_repo; mkdir -p "$REPO/lib"; printf 'thing\n' > "$REPO/lib/thing.zz"; git_commit_repo +run_scope --glob '*.zz' lib/thing.zz; SO="$BUNDLE/scope-out" +run_detect_bundle "$SO" +expect_exit 0 && expect_has "over 1 source + 0 test files" && ok +finish + +begin "a --glob that differs from the bundle's is a usage error (exit 2)" +mk_repo; mkdir -p "$REPO/lib"; printf 'thing\n' > "$REPO/lib/thing.zz"; git_commit_repo +run_scope --glob '*.zz' lib/thing.zz; SO="$BUNDLE/scope-out" +run_detect_bundle "$SO" --glob '*.yy' +expect_exit 2 && expect_has "differs from the bundle's glob" && ok +finish + +begin "a scope whose files match no source glob is an error, not an all-zero table (exit 2)" +mk_repo; printf 'notes\n' > "$REPO/notes.txt"; printf 'notes.txt\n' > "$BUNDLE/files.txt" +printf 'custom\t*.zz\n' > "$BUNDLE/language.txt" +run_detect +expect_exit 2 && expect_has "0 source + 0 test files" && ok +finish + +begin "--gate-budget stops a slow gate and its children; the gate questions are inconclusive" +mk_repo; : > "$BUNDLE/dirs.txt" +FAKE=$(mktemp -d) +cp "$DETECT" "$FAKE/ldd-detect.sh" +printf '#!/usr/bin/env bash\nsleep 37\necho never\n' > "$FAKE/check-repo-brain.sh" +t0=$SECONDS +OUT=$(cd "$REPO" && bash "$FAKE/ldd-detect.sh" --rules "$RULES" --gate-budget 1 --root . "$BUNDLE" 2>"$BUNDLE/.stderr"); CODE=$? +ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" +took=$((SECONDS - t0)) +expect_exit 0 && { (( took < 15 )) || bad "the run took ${took}s with a 1s gate budget"; } && (( took < 15 )) \ + && expect_has "R9 gate: inconclusive" && expect_count R9 1 inconclusive && expect_has "inconclusive" \ + && { ! pgrep -f 'sleep 37' >/dev/null || bad "the gate's child process outlived the budget"; } \ + && ! pgrep -f 'sleep 37' >/dev/null && ok +pkill -f 'sleep 37' 2>/dev/null +rm -rf "$FAKE" +finish + +begin "a --gate-budget the gate finishes within writes the same counts as no budget" +mk_repo; : > "$BUNDLE/dirs.txt"; run_detect +cp "$BUNDLE/counts.tsv" "$BUNDLE/first.tsv" +run_detect --gate-budget 60 +expect_exit 0 && { cmp -s "$BUNDLE/first.tsv" "$BUNDLE/counts.tsv" || bad "counts.tsv differs under a budget the gate meets"; } \ + && cmp -s "$BUNDLE/first.tsv" "$BUNDLE/counts.tsv" && ok +finish + +begin "a gate that fails under --gate-budget fails the run (exit 2)" +mk_repo; : > "$BUNDLE/dirs.txt" +FAKE=$(mktemp -d) +cp "$DETECT" "$FAKE/ldd-detect.sh" +printf '#!/usr/bin/env bash\necho "check-repo-brain: internal scanner error"\nexit 2\n' > "$FAKE/check-repo-brain.sh" +OUT=$(cd "$REPO" && bash "$FAKE/ldd-detect.sh" --rules "$RULES" --gate-budget 30 --root . "$BUNDLE" 2>"$BUNDLE/.stderr"); CODE=$? +ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" +expect_exit 2 && expect_has "the R9 gate failed to run" && ok +rm -rf "$FAKE" +finish + +begin "a --gate-budget with no value is a usage error (exit 2)" +mk_repo +OUT=$(cd "$REPO" && bash "$DETECT" --root . "$BUNDLE" --gate-budget 2>&1); CODE=$?; ERR="" +expect_exit 2 && expect_has "--gate-budget needs a value" && ok +finish + } # run_row_cases write_installed_plugins "$SIBLINGS_FILE" go-linter-driven-development python-linter-driven-development linter-driven-development ts-react-linter-driven-development diff --git a/core/scripts/ldd-scope.sh b/core/scripts/ldd-scope.sh index 17acc26..b0150e5 100755 --- a/core/scripts/ldd-scope.sh +++ b/core/scripts/ldd-scope.sh @@ -46,6 +46,11 @@ # # The bundle: # language.txt the group's language id; "custom TAB " under --glob +# test-re.txt the --test-re pattern, when one was passed, so ldd-detect.sh +# tells test files the same way +# docs.txt on a scoped rung, the markdown files the rung touches, one +# relative path per line, deleted ones included: the docs the +# R9 gate checks in full; --all writes none # files.txt the scope, one relative path per line. A file left out of # scope/ keeps its line with the reason: (not bundled: deleted), # (not bundled: binary), (not bundled: lines) over the @@ -226,6 +231,10 @@ write_bundle() { mkdir -p "$OUT/scope" if [[ "$GROUP_ID" == "custom" ]]; then printf 'custom\t%s\n' "$OPT_GLOB" > "$OUT/language.txt" else printf '%s\n' "$GROUP_ID" > "$OUT/language.txt"; fi + [[ -n "$OPT_TEST_RE" ]] && printf '%s\n' "$OPT_TEST_RE" > "$OUT/test-re.txt" + if [[ "$RUNG" != "all" ]]; then + sed 's|^\./||' "$CANDIDATES" | grep -E '\.md$' | LC_ALL=C sort -u > "$OUT/docs.txt" + fi # not_bundled_reason — prints the reason, or nothing when it is bundled not_bundled_reason() { @@ -273,10 +282,23 @@ write_bundle() { # comment lines plus every comment line of a bundled file the diff does not # touch (untracked, or committed and named on the command line); every comment # line of the bundle on --all; directives never count. + # comment_filter — keeps the "file:line:text" lines whose text carries a + # comment marker at the line start or after whitespace, with every + # double-quoted string literal blanked first, so a # or // inside a trace + # format string is not a comment; a """ docstring marker is kept + comment_filter() { + LC_ALL=C awk -v re="(^|[[:space:]])($LANG_COMMENT_RE)" '{ + i = index($0, ":"); rest = substr($0, i + 1); t = substr(rest, index(rest, ":") + 1) + gsub(/"""/, "\001", t) + gsub(/"([^"\\]|\\.)*"/, "\"\"", t) + gsub(/\001/, "\"\"\"", t) + if (t ~ re) print + }' + } # comment_lines_of ... — every comment line of the files, grep -nH shape comment_lines_of() { (( $# > 0 )) || return 0 - LC_ALL=C grep -nHE -e "$LANG_COMMENT_RE" -- "$@" 2>/dev/null | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" + LC_ALL=C grep -nH -e '' -- "$@" 2>/dev/null | comment_filter | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" } : > "$OUT/comments.txt" if [[ "$RUNG" == "all" ]]; then @@ -290,7 +312,7 @@ write_bundle() { /^\+/ { n++; print f ":" n ":" substr($0, 2); next } /^-/ { next } { n++ } - ' "$OUT/diff.patch" | LC_ALL=C grep -E -- "^[^:]*:[0-9]+:.*($LANG_COMMENT_RE)" | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" >> "$OUT/comments.txt" + ' "$OUT/diff.patch" | comment_filter | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" >> "$OUT/comments.txt" fi # a bundled file the diff does not touch has no added lines: every comment # line of it is the critic's (an untracked file, or a committed file named diff --git a/core/scripts/ldd-scope_test.sh b/core/scripts/ldd-scope_test.sh index d138e18..9e0feb4 100755 --- a/core/scripts/ldd-scope_test.sh +++ b/core/scripts/ldd-scope_test.sh @@ -215,6 +215,17 @@ else fi finish +begin "a comment marker inside a string literal is not a comment line" +mk_repo; mkdir -p "$REPO/lib" +{ fx_comment_line; printf 'x = "#tag #tag" + "//tag // tag"\n'; } > "$REPO/lib/thing.zz" +git_commit_repo +run_scope --glob '*.zz' lib/thing.zz +expect_exit 0 && { grep -q "added by the test" "$BUNDLE/scope-out/comments.txt" || bad "the real comment line is missing"; } \ + && grep -q "added by the test" "$BUNDLE/scope-out/comments.txt" \ + && { ! grep -q '^lib/thing.zz:2:' "$BUNDLE/scope-out/comments.txt" || bad "the string literal line is listed as a comment"; } \ + && ! grep -q '^lib/thing.zz:2:' "$BUNDLE/scope-out/comments.txt" && ok +finish + } # run_row_cases write_installed_plugins "$SIBLINGS_FILE" go-linter-driven-development python-linter-driven-development linter-driven-development ts-react-linter-driven-development diff --git a/core/skills/pre-commit-review/SKILL.md b/core/skills/pre-commit-review/SKILL.md index 3579d0a..abd49c2 100644 --- a/core/skills/pre-commit-review/SKILL.md +++ b/core/skills/pre-commit-review/SKILL.md @@ -80,11 +80,15 @@ writes `hits.tsv`, `hits-all.tsv` and `counts.tsv` into the bundle, and prints t question only a hunter's reading answers — then one line per rule family. That table, about 300 tokens, is the pre-filter: the parent prints no rule file, composes no detection command, and never reads a Falsifying-questions section. On a scoped review the R9 -gate's rows are the scope's own files' — an orphan doc or an unwired root is the -repository's state, for a `--all` review or BROADER CONTEXT, never this diff's -finding. When the +gate checks the docs the diff touches in full; in a doc that links to a scope file +or cites a symbol the diff removes it checks only those links and symbols; and it +checks the code edges of the scope's own files — an orphan doc elsewhere, a citing +doc's older drift or an unwired root is the repository's state, for a `--all` review or BROADER CONTEXT, never this diff's finding. The line +`R9 gate: …` under the summary says which docs the gate checked. A gate row whose +hits cell is `inconclusive` means the gate ran out of its `--gate-budget`: it is not +a `0`, and the documentation hunter answers that question from the prose. When the detection script exits 2 instead — a pattern its grep rejects, the R9 gate failing to -run — the review did not run: report its message, spawn nothing, and never render a +run, a scope in which no file matches the source glob — the review did not run: report its message, spawn nothing, and never render a rule as `skipped` or compose the pass by hand. A rule family none of whose rules has a hit — `0` on every row of theirs that is not `judgment` — is skipped: no hunter for it. @@ -108,7 +112,8 @@ zero prior use; the fix is a discussion or a separate PR, never silent inclusion The bundle exists (step 1 wrote it: `files.txt`, `diff.patch` on a scoped review, `scope/.txt` per bundled file with the file's own line numbers, `dirs.txt` on -`--all`, `comments.txt`, `hits.tsv`, `hits-all.tsv`, `counts.tsv`); nothing is +`--all`, `docs.txt` with the touched docs on a scoped review, `test-re.txt` with the +test-file pattern when one was passed, `comments.txt`, `hits.tsv`, `hits-all.tsv`, `counts.tsv`); nothing is written here. One Bash command, only when the scope is not empty, lists every rule path the prompts will carry and prints `sed -n '/^## Waiting for agents/,/^## The merged report/p' diff --git a/docs/token-budget.md b/docs/token-budget.md index 0d70a4b..2af6add 100644 --- a/docs/token-budget.md +++ b/docs/token-budget.md @@ -375,7 +375,7 @@ defaults carries one **detect line** beside its prose, of one of four kinds. |---|---| | `Detect-grep: PATTERN [files=src\|test\|all] [exclude-path=ERE,…] [context=n]` — the pattern an ERE in backticks | `grep -nHE` over the scope's source files — the non-test ones unless `files=` says otherwise, minus paths matching an `exclude-path` entry; `context` appends the next n lines to each hit's excerpt | | `Detect-path: PATTERN` | the pattern over the scope's relative paths: layer directories, role-named packages | -| `Detect-gate: Q` | the R9 gate's `[Q]` report lines, from one run of `check-repo-brain.sh` over the repository | +| `Detect-gate: Q` | the R9 gate's `[Q]` report lines, from one run of `check-repo-brain.sh` over the repository, or over the scope's docs on a scoped review; `inconclusive` when the run outlives `--gate-budget` | | `Detect: judgment` | no mechanical lead; the hunter runs the prose, as it does for a question whose only procedure depends on an earlier hit (R1's second question) or on a read | A pattern is a literal: the placeholder forms the prose used to carry — ``** stops a gate that runs too long and + counts its questions `inconclusive`, never 0. +- **The two review scripts cannot disagree.** `ldd-detect.sh` reads the + `--test-re` that `ldd-scope.sh` used. A `--glob` or `--test-re` that differs + from the bundle is a usage error. A scope whose files are not source files of + its language is an error, not a table of zeros. +- **`comments.txt` skips string literals.** A `#` or `//` inside a string literal + is not a comment line. ### Removed diff --git a/go-linter-driven-development/scripts/check-repo-brain.sh b/go-linter-driven-development/scripts/check-repo-brain.sh index d2b885c..85da053 100755 --- a/go-linter-driven-development/scripts/check-repo-brain.sh +++ b/go-linter-driven-development/scripts/check-repo-brain.sh @@ -8,10 +8,24 @@ # in-repo copy). Installed into target repos by the documentation skill's # BOOTSTRAP pass (/wire-repo-brain). # -# Usage: bash scripts/check-repo-brain.sh [--fix] [repo-root] (default: cwd) +# Usage: bash scripts/check-repo-brain.sh [--fix] [--scope [--words ]] [repo-root] +# (repo-root default: cwd) # CI: one line — bash scripts/check-repo-brain.sh # --fix: rewrite drifted index lines from each target doc's `description` # (the one mechanical repair; everything else stays report-only) +# --scope a review's scope: one path per line, relative to repo-root +# (code files and docs, deleted ones included). The gate then checks +# only the docs in scope. A listed doc gets every check. A doc whose +# text names a listed file's basename, or that cites a token with a +# dot-separated part in --words, reports only its links into a listed +# file, those tokens, and (an index) its lines to the listed docs. +# Code edges are checked for the listed code files only, Q3 is skipped +# (root wiring is the repository's state, not a diff's), and with no +# doc in scope the doc checks are skipped. One stdout line says which: +# "check-repo-brain: scope — doc(s) to check" or +# "check-repo-brain: scope — no doc in scope; doc checks skipped". +# --words identifiers the scoped change removes, one per line; a doc +# that cites one of them in backticks is in scope # # Doc roots are discovered at the repo root AND at every sub-project (a # directory holding the language's project marker — see the adapter below), @@ -58,8 +72,11 @@ # against its declaring package or receiver — never against a same-named # member elsewhere. A token missing from the set still resolves when it # appears as a whole word in any non-markdown repo file (config keys, -# alert names, test helpers). A `pkg.Sym` whose package is not declared in -# this repo is external (stdlib, dependencies) and exempt. +# alert names, test helpers): in a git work tree the tracked and +# untracked-but-not-ignored files; elsewhere every file outside the +# dependency and build-output directories (bazel-*, node_modules, …). +# A `pkg.Sym` whose package is not declared in this repo is external +# (stdlib, dependencies) and exempt. # - lines carrying the ⚠️ stale flag or a *(planned)* marker are exempt from # symbol resolution and the description copy check (R9 Q2/Q7 exemptions); # the file:line ban has no exemption beyond URL spans, fenced code blocks, @@ -72,22 +89,37 @@ # 2 usage error, or an internal scanner failure (a failed scan is # inconclusive, never silently clean) # -# Uses only POSIX-portable tools: find, grep, sed, awk, head, sort, wc. No jq/python. +# Uses only POSIX-portable tools: find, grep, sed, awk, head, sort, wc, tr, xargs, +# plus git when the repository is a git work tree. No jq/python. # awk programs avoid interval expressions ({m,n}) — mawk, Debian's default, rejects them. set -u FIX=0 -if [[ "${1:-}" == "--fix" ]]; then - FIX=1 - shift -fi +SCOPE_LIST="" +WORDS_LIST="" +abs_list() { # -> absolute path of an existing file, or exit 2 + [[ -f "$1" ]] || { echo "check-repo-brain: no such list: $1" >&2; exit 2; } + printf '%s/%s\n' "$(cd "$(dirname "$1")" && pwd)" "$(basename "$1")" +} +while (( $# > 0 )); do + case "$1" in + --fix) FIX=1; shift ;; + --scope) SCOPE_LIST=$(abs_list "${2:-}") || exit 2; shift 2 ;; + --words) WORDS_LIST=$(abs_list "${2:-}") || exit 2; shift 2 ;; + --*) echo "check-repo-brain: unknown option: $1" >&2; exit 2 ;; + *) break ;; + esac +done +[[ -n "$WORDS_LIST" && -z "$SCOPE_LIST" ]] && { echo "check-repo-brain: --words needs --scope" >&2; exit 2; } REPO_ROOT="${1:-$(pwd)}" if [[ ! -d "$REPO_ROOT" ]]; then echo "check-repo-brain: not a directory: $REPO_ROOT" >&2 exit 2 fi cd "$REPO_ROOT" || exit 2 +SCOPED=0 +[[ -n "$SCOPE_LIST" ]] && SCOPED=1 # ======================= language adapter: go ======================= # Everything language-specific lives between these two marker comments. The @@ -315,27 +347,88 @@ lifecycle_stale() { # -> reason when frontmatter marks the doc stale ('' have_code=0 lang_has_code && have_code=1 -# ---------- declaration set: built once, queried per token ---------- -DECLS="" PKGS="" -if (( have_code )); then - DECLS=$(mktemp) PKGS=$(mktemp) DECL_ALL=$(mktemp) - trap 'rm -f "$DECLS" "$PKGS"' EXIT - if ! ( set -o pipefail; lang_declarations | sort -u > "$DECL_ALL" ); then +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# ---------- declaration set: built once, on the first token to resolve ---------- +DECLS="$WORK/decls" PKGS="$WORK/pkgs" +decls_built=0 +build_declarations() { + (( decls_built )) && return 0 + decls_built=1 + if ! ( set -o pipefail; lang_declarations | sort -u > "$WORK/decl-all" ); then echo "check-repo-brain: internal scanner error — the declaration scan failed; treating the run as inconclusive" >&2 exit 2 fi - grep '^pkg:' "$DECL_ALL" | sed 's/^pkg://' > "$PKGS" - grep -v '^pkg:' "$DECL_ALL" > "$DECLS" - rm -f "$DECL_ALL" -fi + grep '^pkg:' "$WORK/decl-all" | sed 's/^pkg://' > "$PKGS" + grep -v '^pkg:' "$WORK/decl-all" > "$DECLS" + rm -f "$WORK/decl-all" +} is_repo_pkg() { grep -qxF "$1" "$PKGS" 2>/dev/null; } +# repo_files — the repository's non-markdown files, NUL-separated. A git work +# tree lists them through git — the tracked files, those of its submodules +# included, then the untracked ones git does not ignore — which never walks +# build outputs or symlinked caches; elsewhere find skips the directories that +# hold dependencies and build outputs. +repo_files() { + local -a spec=(-- . ':(exclude,glob)**/*.md' ':(exclude,glob)**/vendor/**') + if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + { git ls-files -z --cached --recurse-submodules "${spec[@]}" 2>/dev/null \ + || git ls-files -z --cached "${spec[@]}" 2>/dev/null; } \ + && git ls-files -z --others --exclude-standard "${spec[@]}" 2>/dev/null && return 0 + fi + find . \( -name .git -o -name vendor -o -name node_modules -o -name 'bazel-*' -o -name .venv \ + -o -name venv -o -name __pycache__ -o -name .tox -o -name dist -o -name target \) -prune \ + -o -type f ! -name '*.md' -print0 +} + +# repo_words — the repository's distinct words (runs of [A-Za-z0-9_.$]) in +# its non-markdown text files, one per line, read once per gate run and kept. +# grep ends every file's last line, so two files never fuse into one word, and +# skips binary files. +REPO_WORDS="$WORK/repo-words" +repo_words() { + if [[ ! -f "$REPO_WORDS" ]]; then + repo_files | xargs -0 grep -Ih -e '' -- 2>/dev/null | LC_ALL=C tr -cs 'A-Za-z0-9_.$' '\n' \ + | LC_ALL=C awk '!seen[$0]++' > "$REPO_WORDS" + fi + cat "$REPO_WORDS" +} + +# resolve_words — the tokens that occur as a whole word in any +# non-markdown repository file, one per line. A dotted token such as +# `Type.Method` matches any run of whole dot-separated parts of a word, as +# grep -w would match it, and a word that carries $ (a JavaScript identifier) +# also offers its $-free parts. A lookup in the word set is linear in the +# repository's size, where a fixed-string grep for hundreds of tokens is not. +resolve_words() { + repo_words | LC_ALL=C awk -v list="$1" ' + BEGIN { while ((getline t < list) > 0) if (t != "") want[t] = 1 } + { + n = split($0, p, ".") + for (i = 1; i <= n; i++) { + s = p[i] + if (s in want) { print s; delete want[s] } + for (j = i + 1; j <= n; j++) { + s = s "." p[j] + if (s in want) { print s; delete want[s] } + } + } + if (index($0, "$") > 0) { + n = split($0, p, /[$.]/) + for (i = 1; i <= n; i++) if (p[i] in want) { print p[i]; delete want[p[i]] } + } + }' + return 0 +} + # One fence-aware awk pass per bundle extracts everything Q2's doc scan needs: # P — a file:line citation outside fences/URLs/globs # S — a backticked symbol-shaped token to resolve # Tokens are then resolved as SETS (one grep against the declaration file, one -# repo-wide word grep for the whole unresolved batch) — never per token. +# pass over the repository for the whole unresolved batch) — never per token. # Language-specific shapes arrive as -v variables: file_ext, file_re, sym_re, qual_re. DOCSCAN_AWK=' FNR == 1 { fence = 0 } @@ -369,8 +462,89 @@ FNR == 1 { fence = 0 } } ' -# ---------- Q2: code→docs edges (repo-wide; resolved from repo root, then the -# citing file's own sub-project) ---------- +# ---------- the scope: which docs a scoped run checks, and how far ---------- +# A doc the scope lists (a doc the diff touches) gets every check. A doc that +# only cites the scope — its text names the basename of a listed file as a +# whole word, or it cites in backticks a word the change removes (a symbol the +# diff renames or deletes) — reports only what the scope can break in it: a +# doc link into a listed file, the backticked tokens that carry a removed +# word, and an index line to a listed doc (whose description the diff can +# change). Its other findings are the repository's state, for a +# whole-repository run. +# Every list below holds relative paths as find(1) prints them under a root. +SCOPE_DOCS="$WORK/scope-docs" # every doc a scoped run reads +FULL_DOCS="$WORK/full-docs" # the listed docs: every check +SCOPE_PATHS="$WORK/scope-paths" +BASENAMES="$WORK/basenames" +if (( SCOPED )); then + sed -e 's|^\./||' -e '/^$/d' "$SCOPE_LIST" | LC_ALL=C sort -u > "$SCOPE_PATHS" + sed 's|.*/||' "$SCOPE_PATHS" | LC_ALL=C sort -u > "$BASENAMES" + for r in "${ROOTS[@]}"; do find "$r" -type f -name '*.md'; done | LC_ALL=C sort -u > "$WORK/all-docs" + grep -xF -f "$SCOPE_PATHS" "$WORK/all-docs" > "$FULL_DOCS" + { + cat "$FULL_DOCS" + if [[ -s "$BASENAMES" && -s "$WORK/all-docs" ]]; then + tr '\n' '\0' < "$WORK/all-docs" | xargs -0 grep -lwF -f "$BASENAMES" 2>/dev/null + fi + if [[ -n "$WORDS_LIST" && -s "$WORDS_LIST" && -s "$WORK/all-docs" ]]; then + tr '\n' '\0' < "$WORK/all-docs" \ + | xargs -0 awk -v file_ext="$LANG_FILE_EXT" -v file_re="$LANG_FILE_RE" \ + -v sym_re="$LANG_SYMBOL_RE" -v qual_re="$LANG_QUALIFIED_RE" "$DOCSCAN_AWK" \ + | awk -F'\t' -v list="$WORDS_LIST" ' + BEGIN { while ((getline w < list) > 0) want[w] = 1 } + $1 == "S" { if ($4 in want) { print $2; next } + n = split($4, part, "."); for (i = 1; i <= n; i++) if (part[i] in want) { print $2; next } }' + fi + } | LC_ALL=C sort -u > "$SCOPE_DOCS" + if [[ -s "$SCOPE_DOCS" ]]; then + echo "check-repo-brain: scope — $(wc -l < "$SCOPE_DOCS" | tr -d ' ') doc(s) to check" \ + "($(wc -l < "$FULL_DOCS" | tr -d ' ') touched, every check; the rest cite the scope, scope lines only)" + else + echo "check-repo-brain: scope — no doc in scope; doc checks skipped" + fi +fi + +# bundle_docs [name] [full] — the docs a run checks under one root: +# every *.md (or every file of that name); on a scoped run only the docs in +# scope, or with "full" only the docs that get every check +bundle_docs() { + local name="${2:-*.md}" list="$SCOPE_DOCS" + [[ "${3:-}" == full ]] && list="$FULL_DOCS" + if (( SCOPED )); then + find "$1" -type f -name "$name" | grep -xF -f "$list" + else + find "$1" -type f -name "$name" + fi + return 0 +} + +# full_doc — exit 0 iff the doc gets every check +full_doc() { (( ! SCOPED )) || grep -qxF -- "$1" "$FULL_DOCS"; } + +# link_path — the link's target as a +# repository-relative path, the title and anchor stripped and . and .. folded +# by text, so a target whose directory does not exist still has a path +link_path() { + local t="${2%%#*}" + t="${t%% *}" + if [[ "$t" == /* ]]; then t="$3/${t#/}"; else t="$(dirname "$1")/$t"; fi + printf '%s\n' "$t" | awk -F/ '{ + n = 0 + for (i = 1; i <= NF; i++) { + if ($i == "" || $i == ".") continue + if ($i == ".." && n > 0 && st[n] != "..") { n--; continue } + st[++n] = $i + } + o = ""; for (i = 1; i <= n; i++) o = o (i > 1 ? "/" : "") st[i]; print o + }' +} + +# in_list — exit 0 iff the path is a line of the list +in_list() { grep -qxF -- "$1" "$2"; } + +# ---------- Q2: code→docs edges (repo-wide, or the scope's code files on a +# scoped run; resolved from repo root, then the citing file's own +# sub-project) ---------- docroot_for_file() { # -> docroot of the longest matching project dir local f="${1#./}" best="" i for i in "${!PROJS[@]}"; do @@ -394,7 +568,13 @@ if (( have_code )); then CUR_DOCROOT=$(docroot_for_file "$file") fail "[Q2] $file:$line — code edge points at missing $target" fi - done < <(lang_code_edges) + done < <(if (( SCOPED )); then + lang_code_edges | awk -v list="$SCOPE_PATHS" ' + BEGIN { while ((getline l < list) > 0) s[l] = 1 } + { f = substr($0, 1, index($0, ":") - 1); sub(/^\.\//, "", f); if (f in s) print }' + else + lang_code_edges + fi) fi # ---------- per-bundle checks ---------- @@ -404,6 +584,8 @@ check_bundle() { # local root_index="$docroot/index.md" local root_index_c root_index_c=$(canon "$root_index") + # a scoped run skips a bundle that holds no doc in scope + (( SCOPED )) && [[ -z "$(bundle_docs "$docroot")" ]] && return 0 # --- Q1: transitive reachability from the bundle's root index --- local reachable="" visited="" queue=("$root_index") @@ -422,7 +604,7 @@ check_bundle() { # [[ "$(basename "$resolved")" == "index.md" ]] && queue+=("$resolved") done < <(grep -oE '\]\([^)]+\)' "$idx" 2>/dev/null) done - if [[ ! -f "$root_index" ]]; then + if [[ ! -f "$root_index" ]] && [[ -n "$(bundle_docs "$docroot" '*.md' full)" ]]; then fail "[Q1] $docroot — no index.md: the bundle has no map" fi while IFS= read -r doc; do @@ -432,14 +614,17 @@ check_bundle() { # case "$reachable" in *"$c"$'\n'*) ;; *) fail "[Q1] $doc — orphan: not reachable from $root_index" ;; esac - done < <(find "$docroot" -type f -name '*.md') + done < <(bundle_docs "$docroot" '*.md' full) # --- Q2: every doc link resolves (a missing parent directory is just as - # broken as a missing file — only URLs/anchors are exempt) --- + # broken as a missing file — only URLs/anchors are exempt); a doc that only + # cites the scope reports only its links into a listed file --- while IFS= read -r md; do + local full=0; full_doc "$md" && full=1 while IFS= read -r raw; do local t="${raw#](}"; t="${t%)}" [[ "$t" == *.md* ]] || continue + (( full )) || in_list "$(link_path "$md" "$t" "$docroot")" "$SCOPE_PATHS" || continue local resolved rc=0 resolved=$(resolve_link "$md" "$t" "$docroot") || rc=$? (( rc == 1 )) && continue @@ -447,13 +632,14 @@ check_bundle() { # fail "[Q2] $md — link target does not exist: $t" fi done < <(grep -oE '\]\([^)]+\)' "$md" 2>/dev/null) - done < <(find "$docroot" -type f -name '*.md') + done < <(bundle_docs "$docroot") # --- Q2: doc scan — file:line ban + docs→code symbol resolution. - # One awk pass extracts; resolution is set-based (see DOCSCAN_AWK above). --- + # One awk pass extracts; resolution is set-based (see DOCSCAN_AWK above). A + # doc that only cites the scope keeps only its backticked removed words. --- local scan; scan=$(mktemp) if ! ( set -o pipefail - find "$docroot" -type f -name '*.md' -print0 \ + bundle_docs "$docroot" | tr '\n' '\0' \ | xargs -0 awk -v file_ext="$LANG_FILE_EXT" -v file_re="$LANG_FILE_RE" \ -v sym_re="$LANG_SYMBOL_RE" -v qual_re="$LANG_QUALIFIED_RE" \ "$DOCSCAN_AWK" > "$scan" ); then @@ -461,6 +647,14 @@ check_bundle() { # rm -f "$scan" exit 2 fi + if (( SCOPED )); then + awk -F'\t' -v full="$FULL_DOCS" -v words="${WORDS_LIST:-/dev/null}" ' + BEGIN { while ((getline l < full) > 0) f[l] = 1; while ((getline l < words) > 0) w[l] = 1 } + ($2 in f) { print; next } + $1 == "S" { if ($4 in w) { print; next } + n = split($4, part, "."); for (i = 1; i <= n; i++) if (part[i] in w) { print; next } }' "$scan" > "$scan.kept" + mv -f "$scan.kept" "$scan" + fi local f ln while IFS=$'\t' read -r _ f ln; do fail "[Q2] $f:$ln — cites a file path or line number (churn-prone coordinate)" @@ -468,6 +662,7 @@ check_bundle() { # if (( have_code )) && grep -q $'^S\t' "$scan"; then local toks check members unres bad toks=$(mktemp) check=$(mktemp) members=$(mktemp) unres=$(mktemp) bad=$(mktemp) + build_declarations grep $'^S\t' "$scan" | cut -f4 | sort -u > "$toks" # tokens to resolve (external pkg.Sym exempt). A qualified token resolves # as the PAIR itself — the declaration set carries pkg.Ident / Type.Method @@ -485,12 +680,11 @@ check_bundle() { # esac done < "$toks" > "$check" cut -f2 "$check" | sort -u > "$members" - grep -vxF -f "$DECLS" "$members" > "$unres" || true + grep -vxF -f "$DECLS" "$members" | grep -v '^$' > "$unres" || true if [[ -s "$unres" ]]; then - # ONE repo-wide word grep for the whole unresolved batch + # ONE pass over the repository for the whole unresolved batch local found; found=$(mktemp) - grep -rIhoFw --exclude-dir=vendor --exclude-dir=.git --exclude='*.md' \ - -f "$unres" . 2>/dev/null | sort -u > "$found" + resolve_words "$unres" | sort -u > "$found" grep -vxF -f "$found" "$unres" > "$bad" || true rm -f "$found" fi @@ -508,24 +702,27 @@ check_bundle() { # fi rm -f "$scan" - # --- Q3: root wiring (exact path; sub-roots may ride the repo-root index) --- - local rel="$docroot" - [[ "$proj" != "." ]] && rel="${docroot#$proj/}" - local claude="CLAUDE.md" agents="AGENTS.md" - [[ "$proj" != "." ]] && { claude="$proj/CLAUDE.md"; agents="$proj/AGENTS.md"; } - local wired_claude=0 wired_agents=0 - [[ -f "$claude" ]] && grep -qF -- "$rel/index.md" "$claude" && wired_claude=1 - [[ -f "$agents" ]] && grep -qF -- "$rel/index.md" "$agents" && wired_agents=1 - if (( ! wired_claude && ! wired_agents )); then - local via_root=0 - if [[ "$proj" != "." && -n "$ROOT_BUNDLE" ]]; then - grep -rqF -- "$docroot/index.md" "$ROOT_BUNDLE" --include='*.md' 2>/dev/null && via_root=1 - fi - if (( ! via_root )); then - fail "[Q3] $proj — neither $claude nor $agents references $rel/index.md" + # --- Q3: root wiring (exact path; sub-roots may ride the repo-root index); + # a scoped run leaves it to a whole-repository run --- + if (( ! SCOPED )); then + local rel="$docroot" + [[ "$proj" != "." ]] && rel="${docroot#$proj/}" + local claude="CLAUDE.md" agents="AGENTS.md" + [[ "$proj" != "." ]] && { claude="$proj/CLAUDE.md"; agents="$proj/AGENTS.md"; } + local wired_claude=0 wired_agents=0 + [[ -f "$claude" ]] && grep -qF -- "$rel/index.md" "$claude" && wired_claude=1 + [[ -f "$agents" ]] && grep -qF -- "$rel/index.md" "$agents" && wired_agents=1 + if (( ! wired_claude && ! wired_agents )); then + local via_root=0 + if [[ "$proj" != "." && -n "$ROOT_BUNDLE" ]]; then + grep -rqF -- "$docroot/index.md" "$ROOT_BUNDLE" --include='*.md' 2>/dev/null && via_root=1 + fi + if (( ! via_root )); then + fail "[Q3] $proj — neither $claude nor $agents references $rel/index.md" + fi + elif (( ! wired_agents )); then + note "[Q3] $agents lacks the $rel/index.md routing reference — AGENTS.md-reading tools start blind" fi - elif (( ! wired_agents )); then - note "[Q3] $agents lacks the $rel/index.md routing reference — AGENTS.md-reading tools start blind" fi # --- Q7: bundle contract --- @@ -590,12 +787,15 @@ check_bundle() { # else fail "[Q7] $md — frontmatter missing 'description:'" fi - done < <(find "$docroot" -type f -name '*.md') + done < <(bundle_docs "$docroot" '*.md' full) # --- Q7: drift check — index line text == target's description (⚠️ exempt; - # --fix rewrites drifted lines after the read loop, never during it) --- + # --fix rewrites drifted lines after the read loop, never during it). An + # index that only cites the scope checks only its lines to the touched docs, + # whose description the diff can change. --- while IFS= read -r idx; do - local lineno=0 line fixes="" + local lineno=0 line fixes="" ifull=0 + full_doc "$idx" && ifull=1 while IFS= read -r line; do lineno=$((lineno + 1)) case "$line" in *'⚠️'*) continue ;; esac @@ -603,6 +803,7 @@ check_bundle() { # local t; t=$(printf '%s' "$line" | sed -E 's/^- \[[^]]*\]\(([^)]*)\).*/\1/') local tail="${line#* — }" tail=$(printf '%s' "$tail" | sed -e 's/[[:space:]]*$//') + (( ifull )) || in_list "$(link_path "$idx" "$t" "$docroot")" "$FULL_DOCS" || continue local resolved resolved=$(resolve_link "$idx" "$t" "$docroot") || continue # broken links are Q2's report [[ -f "$resolved" ]] || continue @@ -628,12 +829,12 @@ check_bundle() { # echo " fixed: $idx:$n — index line rewritten from its target's description" done <<< "$fixes" fi - done < <(find "$docroot" -type f -name 'index.md') + done < <(bundle_docs "$docroot" index.md) # --- Q7: no log.md --- while IFS= read -r lg; do fail "[Q7] $lg — log.md is reserved for change history; docs describe current behavior" - done < <(find "$docroot" -type f -name 'log.md') + done < <(bundle_docs "$docroot" log.md full) } for i in "${!PROJS[@]}"; do diff --git a/go-linter-driven-development/scripts/check-repo-brain_test.sh b/go-linter-driven-development/scripts/check-repo-brain_test.sh index 59aeac7..00e99c9 100644 --- a/go-linter-driven-development/scripts/check-repo-brain_test.sh +++ b/go-linter-driven-development/scripts/check-repo-brain_test.sh @@ -591,6 +591,113 @@ run_gate expect_exit 0 && expect_not "does not resolve" && expect_has "check-repo-brain: clean" && ok finish +# --- scoped runs (--scope, --words) and the symbol resolver --- +LISTS=$(mktemp -d) # the scope and word lists live outside the repository they describe +begin "--scope: a code-only scope that no doc cites skips the doc checks" +mk_conformant; mk_doc orphan "an unlinked doc" "Nothing links here." +printf '%s\n' "retry/policy.go" > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 0 && expect_has "scope — no doc in scope; doc checks skipped" && expect_not "orphan" && ok +finish + +begin "--scope: a listed doc is checked, an unlisted orphan and the root wiring are not" +mk_conformant; mk_doc orphan "an unlinked doc" "Nothing links here."; mk_doc other "another unlinked doc" "Nothing." +rm "$REPO/CLAUDE.md" "$REPO/AGENTS.md" +printf '%s\n' "retry/policy.go" docs/orphan.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "scope — 1 doc(s) to check" && expect_has "[Q1] docs/orphan.md — orphan" \ + && expect_not "docs/other.md" && expect_not "[Q3]" && ok +finish + +begin "--scope: a doc that only names a scope file reports only its links into the scope" +mk_conformant +mk_doc linker "links the scope" "See [the old doc](gone.md) and [a missing one](missing.md); code: \`retry/policy.go\` at retry/policy.go:12." +printf '%s\n' "retry/policy.go" docs/gone.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "0 touched" && expect_has "[Q2] docs/linker.md — link target does not exist: gone.md" \ + && expect_not "missing.md" && expect_not "orphan" && expect_not "file path or line number" && ok +finish + +begin "--scope --words: a doc that cites a removed symbol is checked" +mk_conformant +mk_doc ghost "a ghost symbol" "Uses \`SnapshotRunner\` and \`OtherGhost\`." +append_index "- [ghost.md](ghost.md) — a ghost symbol" +printf '%s\n' "retry/policy.go" > "$LISTS/scope.txt"; printf 'SnapshotRunner\n' > "$LISTS/words.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 0 && expect_has "no doc in scope" \ + && { run_gate --scope "$LISTS/scope.txt" --words "$LISTS/words.txt"; expect_exit 1; } \ + && expect_has "backticked \`SnapshotRunner\` does not resolve" && expect_not "OtherGhost" && ok +finish + +begin "Q2 in a git work tree a token resolves from tracked and untracked files, never from ignored ones" +mk_conformant +printf 'build/\n' > "$REPO/.gitignore" +mkdir -p "$REPO/build"; printf 'IgnoredOnlyName\n' > "$REPO/build/gen.yaml" +printf 'TrackedName\n' > "$REPO/tracked.yaml" +mk_doc names "alert names" "Fires \`TrackedName\`, \`UntrackedName\` and \`IgnoredOnlyName\`." +append_index "- [names.md](names.md) — alert names" +(cd "$REPO" && git init -q && git add -A && git -c user.name=t -c user.email=t@t commit -qm init) +printf 'UntrackedName\n' > "$REPO/untracked.yaml" +run_gate +expect_exit 1 && expect_has "\`IgnoredOnlyName\` does not resolve" \ + && expect_not "\`TrackedName\` does not resolve" && expect_not "\`UntrackedName\` does not resolve" && ok +finish + +begin "Q2 outside git a token does not resolve from a bazel-* output directory" +mk_conformant +mkdir -p "$REPO/bazel-out"; printf 'BazelOnlyName\n' > "$REPO/bazel-out/gen.yaml" +mk_doc bazel "bazel names" "Fires \`BazelOnlyName\`." +append_index "- [bazel.md](bazel.md) — bazel names" +run_gate +expect_exit 1 && expect_has "\`BazelOnlyName\` does not resolve" && ok +finish + +begin "Q2 a dotted token resolves on whole dot-separated words only" +mk_conformant +printf 'limits:\n key: Cfg.MaxRetries.value\n other: Cfg.MaxRetriesX\n' > "$REPO/limits.yaml" +mk_doc dotted "dotted names" "Reads \`Cfg.MaxRetries\`; \`Cfg.MaxRetri\` is a typo." +append_index "- [dotted.md](dotted.md) — dotted names" +run_gate +expect_exit 1 && expect_has "\`Cfg.MaxRetri\` does not resolve" && expect_not "\`Cfg.MaxRetries\` does not resolve" && ok +finish +begin "Q2 two files with no final newline do not fuse their last and first words" +mk_conformant +printf 'FirstToken' > "$REPO/a.yaml"; printf 'SecondToken' > "$REPO/b.yaml" +mk_doc fused "fused names" "Fires \`FirstToken\` and \`SecondToken\`." +append_index "- [fused.md](fused.md) — fused names" +run_gate +expect_exit 0 && expect_not "does not resolve" && ok +finish + +begin "--scope --words: a doc citing Type.Method is checked when the diff removes the type" +mk_conformant +mk_doc renamed "a renamed type" "Calls \`OldType.Run\`." +append_index "- [renamed.md](renamed.md) — a renamed type" +printf '%s\n' "retry/policy.go" > "$LISTS/scope.txt"; printf 'OldType\n' > "$LISTS/words.txt" +run_gate --scope "$LISTS/scope.txt" --words "$LISTS/words.txt" +expect_exit 1 && expect_has "backticked \`OldType.Run\` does not resolve" && ok +finish + +begin "--scope: a citing doc reports a link to the listed doc, never a link to a same-named doc elsewhere" +mk_conformant +mk_doc linker2 "links two guides" "See [old](a/guide.md \"Title\") and [other](b/guide.md)." +append_index "- [linker2.md](linker2.md) — links two guides" +printf '%s\n' "retry/policy.go" docs/a/guide.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "[Q2] docs/linker2.md — link target does not exist: a/guide.md" \ + && expect_not "b/guide.md" && ok +finish + +begin "--scope: a touched doc's new description is checked against the index that lists it" +mk_conformant +sed 's/^description: .*/description: a new description/' "$REPO/docs/retry-policy.md" > "$REPO/docs/rp.tmp" && mv "$REPO/docs/rp.tmp" "$REPO/docs/retry-policy.md" +printf '%s\n' docs/retry-policy.md > "$LISTS/scope.txt" +run_gate --scope "$LISTS/scope.txt" +expect_exit 1 && expect_has "[Q7] docs/index.md:" && expect_has "drifted from retry-policy.md" && ok +finish + +rm -rf "$LISTS" + # ========================= summary ========================= echo if [[ -n "$GATE_REF" ]]; then diff --git a/go-linter-driven-development/scripts/ldd-detect.sh b/go-linter-driven-development/scripts/ldd-detect.sh index b4ecc2b..23f2112 100755 --- a/go-linter-driven-development/scripts/ldd-detect.sh +++ b/go-linter-driven-development/scripts/ldd-detect.sh @@ -25,7 +25,14 @@ # --plan print the parsed detect lines and exit; nothing runs # --glob the scope is one group of a language the table # does not know, matched by this glob -# --test-re test-file pattern over relative paths, same case +# --test-re test-file pattern over relative paths, same case. +# A bundle carries the pattern ldd-scope.sh used +# (test-re.txt), and a custom group its glob +# (language.txt): the bundle's values win, and a +# --glob or --test-re that differs is a usage error +# --gate-budget wall-clock seconds the R9 gate may run (default: no +# limit); on expiry the gate is stopped and its +# questions are counted "inconclusive", never 0 # # The scope's language is the bundle's language.txt; without it, the files in # files.txt tell it by extension, and a list that spans several languages is @@ -39,10 +46,14 @@ # Detect-path: `` over the scope's relative paths (layer directories, # role-named packages); a hit's line is 0 # Detect-gate: Q the R9 gate's [Q] report lines, from one run of -# check-repo-brain.sh beside this script over --root; on a -# scoped bundle (no dirs.txt) only the lines whose file is in -# the scope — the gate's repository-wide findings belong to -# a whole-repository review +# check-repo-brain.sh beside this script over --root. On a +# scoped bundle (no dirs.txt) the gate runs with --scope: it +# checks the bundle's docs (docs.txt) in full, and in the +# docs that name a scope file or cite a symbol the diff +# removes only those links and symbols; it checks the code +# edges of the scope's code files and no root wiring; with +# no such doc its doc checks are skipped. The repository-wide findings belong to a +# whole-repository review # Detect: judgment no mechanical lead; the hunter runs the prose # A question with no detect line, two of them, or a malformed one is an error: # the generator's lint-core keeps the rule files honest, and this script refuses @@ -53,15 +64,18 @@ # the overflow row of a capped question: file "-", line 0, # excerpt "+ more hit(s) not listed" # hits-all.tsv the same columns, every hit, no cap and no overflow row -# counts.tsv rule question kind hits (judgment rows carry "-") +# counts.tsv rule question kind hits (judgment rows carry "-", gate +# rows "inconclusive" when the gate ran out of --gate-budget) # the last row is SUPPRESS - grep : suppression directives # on the diff's added lines when the bundle has a diff.patch, # else over every scope file # Two runs over one tree write byte-identical counts.tsv. # # Exit codes: 0 tables written · 2 usage error, a detect line that does not -# parse, a pattern grep rejects, or a gate run that failed -# (inconclusive, never silently clean) +# parse, a pattern grep rejects, a gate run that failed +# (inconclusive, never silently clean), or a non-empty scope in +# which no file matches the source glob (a glob mismatch, never +# an empty table) # # Uses only POSIX-portable tools: grep, sed, awk, sort, wc, head. GNU grep's # \b and \s are assumed, as the rules already assume them. @@ -77,6 +91,7 @@ CAP=40 PLAN=0 OPT_GLOB="" OPT_TEST_RE="" +GATE_BUDGET=0 BUNDLE="" usage() { sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } @@ -94,12 +109,14 @@ while (( $# > 0 )); do --plan) PLAN=1; shift ;; --glob) value "$1" $#; OPT_GLOB="$2"; shift 2 ;; --test-re) value "$1" $#; OPT_TEST_RE="$2"; shift 2 ;; + --gate-budget) value "$1" $#; GATE_BUDGET="$2"; shift 2 ;; -h|--help) usage ;; --*) die "unknown option: $1" ;; *) [[ -n "$BUNDLE" ]] && die "one bundle directory, got also: $1"; BUNDLE="$1"; shift ;; esac done [[ "$CAP" =~ ^[0-9]+$ ]] || die "--cap takes a number, got: $CAP" +[[ "$GATE_BUDGET" =~ ^[0-9]+$ ]] || die "--gate-budget takes a number of seconds, got: $GATE_BUDGET" [[ -d "$RULES_DIR" ]] || die "no rules directory: $RULES_DIR" RULES_DIR=$(cd "$RULES_DIR" && pwd) if (( ! PLAN )); then @@ -242,6 +259,7 @@ if (( ! PLAN )) && [[ -f "$BUNDLE/groups.txt" ]]; then fi args=(--root "$ROOT" --cap "$CAP") [[ -n "$OPT_TEST_RE" ]] && args+=(--test-re "$OPT_TEST_RE") + (( GATE_BUDGET > 0 )) && args+=(--gate-budget "$GATE_BUDGET") bash "$script" "${args[@]}" "$bundle" || { failed=1; echo "$SCRIPT_NAME: the $id group's detection pass failed; it is inconclusive" >&2; } echo done < "$BUNDLE/groups.txt" @@ -256,7 +274,9 @@ LANG_ID="" if (( ! PLAN )); then if [[ -f "$BUNDLE/language.txt" ]]; then IFS=$'\t' read -r LANG_ID glob < "$BUNDLE/language.txt" - [[ "$LANG_ID" == "custom" && -z "$OPT_GLOB" ]] && OPT_GLOB="$glob" + [[ "$LANG_ID" == "custom" && -n "$OPT_GLOB" && "$OPT_GLOB" != "$glob" ]] \ + && die "--glob '$OPT_GLOB' differs from the bundle's glob '$glob' (ldd-scope.sh wrote it; drop --glob)" + [[ "$LANG_ID" == "custom" ]] && OPT_GLOB="$glob" else ids="" while IFS= read -r line; do @@ -271,6 +291,14 @@ if (( ! PLAN )); then [[ "$ids" == *" "* ]] && die "the scope list spans several languages ($ids); ldd-scope.sh writes one bundle per language" LANG_ID="$ids" fi + # the test-file pattern ldd-scope.sh used, so the two scripts cannot disagree + # on what a test file is + if [[ -f "$BUNDLE/test-re.txt" ]]; then + b_test_re=$(cat "$BUNDLE/test-re.txt") + [[ -n "$OPT_TEST_RE" && "$OPT_TEST_RE" != "$b_test_re" ]] \ + && die "--test-re '$OPT_TEST_RE' differs from the bundle's '$b_test_re' (ldd-scope.sh wrote it; drop --test-re)" + OPT_TEST_RE="$b_test_re" + fi lang_configure "$LANG_ID" || die "this plugin has no language block for $LANG_ID" fi @@ -316,7 +344,7 @@ END { if (insec) flush() } ' plan_file=$(mktemp) -trap 'rm -f "$plan_file" "${tmp_hits:-}" "${tmp_hits_all:-}" "${tmp_counts:-}" "${gate_out:-}"' EXIT +trap 'rm -f "$plan_file" "${tmp_hits:-}" "${tmp_hits_all:-}" "${tmp_counts:-}" "${gate_out:-}" "${gate_scope:-}" "${gate_words:-}"' EXIT rule_files=$(find "$RULES_DIR" -maxdepth 1 -name 'R*.md' | awk -F/ '{ n = $NF; sub(/^R/, "", n); sub(/-.*$/, "", n); print n "\t" $0 }' | sort -n | cut -f2) [[ -n "$rule_files" ]] || die "no rule files (R*.md) under $RULES_DIR" while IFS= read -r f; do @@ -342,11 +370,13 @@ fi SRC_FILES=() TEST_FILES=() ALL_FILES=() +n_present=0 while IFS= read -r line; do f="${line%% (not bundled:*}" f="${f#./}" [[ -z "$f" ]] && continue [[ -f "$f" ]] || continue + n_present=$((n_present + 1)) [[ "$(lang_of_path "$f")" == "$LANG_ID" ]] || continue printf '%s\n' "$f" | grep -qE -- "$LANG_EXCLUDE_RE" && continue ALL_FILES+=("$f") @@ -354,6 +384,12 @@ done < <(LC_ALL=C sort -u "$FILES_LIST") for f in ${ALL_FILES[@]+"${ALL_FILES[@]}"}; do if lang_is_test "$f"; then TEST_FILES+=("$f"); else SRC_FILES+=("$f"); fi done +# A scope whose files exist but none of which is a source file is a glob +# mismatch between the scope and this run: an all-zero table would read as a +# clean review of nothing. +if (( ${#ALL_FILES[@]} == 0 && n_present > 0 )); then + die "0 source + 0 test files: none of the $n_present file(s) in $FILES_LIST is a $LANG_ID source file (pass the --glob ldd-scope.sh used)" +fi tmp_hits=$(mktemp) tmp_hits_all=$(mktemp) @@ -467,24 +503,98 @@ run_path() { # rm -f "$raw" "$err" } -# The R9 gate runs once, lazily; its [Q] lines are split by question. +# gate_scope_lists — what a scoped gate run checks: +# every path of the scope (deleted ones too: a doc can still link to them) and +# of docs.txt; and the identifiers the diff removes — on its removed lines and +# on none of its added ones — the symbols a renamed or deleted declaration +# takes away from the docs that cite them. Comment lines (a line that opens +# with //, #, /*, *, -- or """) carry prose, not symbols, and are skipped, +# and a word the scope's files still carry after the change is not removed. +gate_scope_lists() { + local scope="$1" words="$2" line f + { + while IFS= read -r line; do + f="${line%% (not bundled:*}"; f="${f#./}" + [[ -n "$f" ]] && printf '%s\n' "$f" + done < "$FILES_LIST" + [[ -f "$BUNDLE/docs.txt" ]] && cat "$BUNDLE/docs.txt" + } | LC_ALL=C sort -u > "$scope" + : > "$words" + [[ -s "$BUNDLE/diff.patch" ]] || return 0 + grep -vE '^(\+\+\+|---) ' "$BUNDLE/diff.patch" | grep -E '^[-+]' \ + | LC_ALL=C awk ' + { + s = substr($0, 1, 1); t = substr($0, 2) + if (t ~ /^[ \t]*(\/\/|#|\/\*|\*|--|""")/) next + while (match(t, /[A-Za-z_][A-Za-z0-9_]*/)) { + w = substr(t, RSTART, RLENGTH); t = substr(t, RSTART + RLENGTH) + if (s == "+") add[w] = 1; else del[w] = 1 + } + } + END { for (w in del) if (!(w in add)) print w }' \ + | LC_ALL=C sort > "$words.removed" + if (( ${#ALL_FILES[@]} > 0 )); then + cat -- "${ALL_FILES[@]}" | LC_ALL=C tr -cs 'A-Za-z0-9_' '\n' \ + | LC_ALL=C awk -v list="$words.removed" ' + BEGIN { while ((getline w < list) > 0) gone[w] = 1 } + ($0 in gone) { delete gone[$0] } + END { for (w in gone) print w }' | LC_ALL=C sort > "$words" + else + cp "$words.removed" "$words" + fi + rm -f "$words.removed" +} + +# The R9 gate runs once, lazily; its [Q] lines are split by question. With +# --gate-budget the gate runs as its own process group, so expiry stops every +# process it started, not only its shell. +GATE_STATE="whole repository" run_gate_once() { [[ -n "$gate_out" ]] && return 0 gate_out=$(mktemp) - local gate="$SCRIPT_DIR/check-repo-brain.sh" code + local gate="$SCRIPT_DIR/check-repo-brain.sh" code=0 start pid scope_line + local -a args=() [[ -f "$gate" ]] || die "Detect-gate needs $gate beside this script" - bash "$gate" "$ROOT" > "$gate_out" 2>&1 - code=$? + if [[ ! -f "$BUNDLE/dirs.txt" ]]; then + gate_scope=$(mktemp); gate_words=$(mktemp) + gate_scope_lists "$gate_scope" "$gate_words" + args=(--scope "$gate_scope" --words "$gate_words") + fi + if (( GATE_BUDGET == 0 )); then + bash "$gate" ${args[@]+"${args[@]}"} "$ROOT" > "$gate_out" 2>&1 || code=$? + else + start=$SECONDS + set -m + bash "$gate" ${args[@]+"${args[@]}"} "$ROOT" > "$gate_out" 2>&1 & + pid=$! + set +m + while kill -0 "$pid" 2>/dev/null; do + if (( SECONDS - start >= GATE_BUDGET )); then + kill -TERM -- "-$pid" 2>/dev/null + wait "$pid" 2>/dev/null + GATE_STATE="inconclusive — stopped after the ${GATE_BUDGET}s --gate-budget" + return 0 + fi + sleep 0.2 + done + wait "$pid" || code=$? + fi if (( code == 2 )); then head -5 "$gate_out" >&2 die "the R9 gate failed to run (exit 2); its questions are inconclusive" fi + scope_line=$(grep -m1 '^check-repo-brain: scope — ' "$gate_out") + [[ -n "$scope_line" ]] && GATE_STATE="scoped, ${scope_line#check-repo-brain: scope — }" return 0 } run_gate() { # local rule="$1" q="$2" qn="$3" raw tab run_gate_once + if [[ "$GATE_STATE" == inconclusive* ]]; then + printf '%s\t%s\tgate\tinconclusive\n' "$rule" "$q" >> "$tmp_counts" + return 0 + fi raw=$(mktemp) tab=$(printf '\t') # "[Qn] [:line] — — see " becomes @@ -502,15 +612,9 @@ run_gate() { # if (match(loc, /:[0-9]+$/)) { file = substr(loc, 1, RSTART - 1); line = substr(loc, RSTART + 1) + 0 } print file "\t" line "\t" rest }' > "$raw" - # A scoped review owns only the gate lines about its own files: an orphan - # doc, an unwired root or another file's broken edge is the repository's - # state, not the diff's, and is not a lead for this review's hunter. - if [[ ! -f "$BUNDLE/dirs.txt" ]] && [[ -s "$raw" ]]; then - scoped=$(mktemp); scope_list=$(mktemp) - printf '%s\n' ${ALL_FILES[@]+"${ALL_FILES[@]}"} > "$scope_list" - awk -F'\t' 'NR == FNR { in_scope[$0] = 1; next } ($1 in in_scope)' "$scope_list" "$raw" > "$scoped" - mv "$scoped" "$raw"; rm -f "$scope_list" - fi + # A scoped review owns only the gate lines about its own files and the docs + # that cite them; the gate's --scope run reports nothing else (an orphan doc + # elsewhere or an unwired root is the repository's state, not the diff's). record "$rule" "$q" gate "$raw" 0 rm -f "$raw" } @@ -560,6 +664,7 @@ n_gate=$(awk -F'\t' '$3 == "gate"' "$plan_file" | wc -l | tr -d ' ') n_judg=$(awk -F'\t' '$3 == "judgment"' "$plan_file" | wc -l | tr -d ' ') n_q=$(wc -l < "$plan_file" | tr -d ' ') echo "$SCRIPT_NAME: $n_q questions ($n_grep grep, $n_path path, $n_gate gate, $n_judg judgment) over ${#SRC_FILES[@]} source + ${#TEST_FILES[@]} test files → $BUNDLE/hits.tsv, counts.tsv" +(( n_gate > 0 )) && [[ -n "$gate_out" ]] && echo "R9 gate: $GATE_STATE" echo awk -F'\t' 'BEGIN { printf "%-9s %-4s %-9s %s\n", "rule", "q", "kind", "hits" } { q = ($2 == "-") ? "-" : "Q" $2; printf "%-9s %-4s %-9s %s\n", $1, q, $3, $4 }' "$BUNDLE/counts.tsv" @@ -579,12 +684,14 @@ awk -F'\t' ' f = fam[$1]; if (f == "") f = "other" nq[f]++ if ($3 == "judgment") { nj[f]++; next } + if ($4 == "inconclusive") { ni[f]++; next } if ($4 + 0 > 0) { hits[f] += $4; nh[f]++; rh[f, $1] += $4 } } END { for (i = 1; i <= 4; i++) { f = order[i] printf "%-15s %5d hit(s) in %d of %d questions, %d judgment", f, hits[f] + 0, nh[f] + 0, nq[f] + 0, nj[f] + 0 + if (ni[f] + 0 > 0) printf ", %d inconclusive", ni[f] per = ""; n = split(rules[f], rs, " ") for (j = 1; j <= n; j++) if (rh[f, rs[j]] + 0 > 0) per = per (per == "" ? "" : " · ") rs[j] " " rh[f, rs[j]] if (per != "") printf " — %s", per diff --git a/go-linter-driven-development/scripts/ldd-detect_test.sh b/go-linter-driven-development/scripts/ldd-detect_test.sh index 7d7a85e..58e5647 100644 --- a/go-linter-driven-development/scripts/ldd-detect_test.sh +++ b/go-linter-driven-development/scripts/ldd-detect_test.sh @@ -49,6 +49,11 @@ run_detect() { # [options...] — runs the script over $REPO with $BUNDLE OUT=$(cd "$REPO" && bash "$DETECT" "$@" --root . "$BUNDLE" 2>"$BUNDLE/.stderr"); CODE=$? ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" } +run_detect_bundle() { # [options...] — runs the script over $REPO with that bundle + local b="$1"; shift + OUT=$(cd "$REPO" && bash "$DETECT" "$@" --root . "$b" 2>"$BUNDLE/.stderr"); CODE=$? + ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" +} run_scope() { # [args...] — runs ldd-scope.sh over $REPO into $BUNDLE/scope-out rm -rf "$BUNDLE/scope-out" OUT=$(cd "$REPO" && bash "$SCOPE_SH" --out "$BUNDLE/scope-out" "$@" 2>"$BUNDLE/.stderr"); CODE=$? @@ -75,8 +80,9 @@ expect_not() { return 0 } expect_quiet() { [[ -z "$ERR" ]] || { bad "stderr not empty"; return 1; }; } -count_of() { # — the hits cell of counts.tsv, "-" for judgment - awk -F'\t' -v r="$1" -v q="$2" '$1 == r && $2 == q { print $4 }' "$BUNDLE/counts.tsv" +count_of() { # — the hits cell of counts.tsv (COUNTS_FILE, default the + # case bundle's), "-" for judgment + awk -F'\t' -v r="$1" -v q="$2" '$1 == r && $2 == q { print $4 }' "${COUNTS_FILE:-$BUNDLE/counts.tsv}" } expect_count() { # =n> local got; got=$(count_of "$1" "$2") @@ -479,6 +485,97 @@ expect_exit 0 && expect_count SUPPRESS - 1 \ && grep -q "^SUPPRESS - grep $FX_SRC_FILE 2 " "$BUNDLE/hits.tsv" && ok finish +# --- the bundle ldd-scope.sh writes, read by detection --- +SO="" # the scope bundle of the case: $BUNDLE/scope-out + +begin "a code-only scoped diff skips the R9 gate's doc checks" +mk_repo; git_commit_repo +printf '\n' >> "$REPO/$FX_SRC_FILE" # a code-only change +(cd "$REPO" && git -c user.name=t -c user.email=t@t commit -qam change) +run_scope --base HEAD~1; SO="$BUNDLE/scope-out" +run_detect_bundle "$SO" +expect_exit 0 && expect_has "R9 gate: scoped, no doc in scope; doc checks skipped" \ + && COUNTS_FILE="$SO/counts.tsv" expect_count R9 1 0 && ok +finish + +begin "a scoped diff that touches a doc gets that doc checked by the R9 gate" +mk_repo; git_commit_repo +printf '\n' >> "$REPO/$FX_SRC_FILE" # a code-only change +printf '\nMore text.\n' >> "$REPO/docs/orphan.md" +(cd "$REPO" && git -c user.name=t -c user.email=t@t commit -qam change) +run_scope --base HEAD~1; SO="$BUNDLE/scope-out" +expect_exit 0 && { grep -qx 'docs/orphan.md' "$SO/docs.txt" || bad "docs.txt lacks the touched doc"; } \ + && grep -qx 'docs/orphan.md' "$SO/docs.txt" \ + && { run_detect_bundle "$SO"; expect_exit 0; } \ + && expect_has "R9 gate: scoped, 1 doc(s) to check" \ + && COUNTS_FILE="$SO/counts.tsv" expect_count R9 1 1 \ + && { grep -q "^R9 1 gate docs/orphan.md " "$SO/hits.tsv" || bad "no R9 Q1 hit row for docs/orphan.md"; } \ + && grep -q "^R9 1 gate docs/orphan.md " "$SO/hits.tsv" && ok +finish + +begin "detection reads the source glob from the bundle ldd-scope.sh wrote" +mk_repo; mkdir -p "$REPO/lib"; printf 'thing\n' > "$REPO/lib/thing.zz"; git_commit_repo +run_scope --glob '*.zz' lib/thing.zz; SO="$BUNDLE/scope-out" +run_detect_bundle "$SO" +expect_exit 0 && expect_has "over 1 source + 0 test files" && ok +finish + +begin "a --glob that differs from the bundle's is a usage error (exit 2)" +mk_repo; mkdir -p "$REPO/lib"; printf 'thing\n' > "$REPO/lib/thing.zz"; git_commit_repo +run_scope --glob '*.zz' lib/thing.zz; SO="$BUNDLE/scope-out" +run_detect_bundle "$SO" --glob '*.yy' +expect_exit 2 && expect_has "differs from the bundle's glob" && ok +finish + +begin "a scope whose files match no source glob is an error, not an all-zero table (exit 2)" +mk_repo; printf 'notes\n' > "$REPO/notes.txt"; printf 'notes.txt\n' > "$BUNDLE/files.txt" +printf 'custom\t*.zz\n' > "$BUNDLE/language.txt" +run_detect +expect_exit 2 && expect_has "0 source + 0 test files" && ok +finish + +begin "--gate-budget stops a slow gate and its children; the gate questions are inconclusive" +mk_repo; : > "$BUNDLE/dirs.txt" +FAKE=$(mktemp -d) +cp "$DETECT" "$FAKE/ldd-detect.sh" +printf '#!/usr/bin/env bash\nsleep 37\necho never\n' > "$FAKE/check-repo-brain.sh" +t0=$SECONDS +OUT=$(cd "$REPO" && bash "$FAKE/ldd-detect.sh" --rules "$RULES" --gate-budget 1 --root . "$BUNDLE" 2>"$BUNDLE/.stderr"); CODE=$? +ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" +took=$((SECONDS - t0)) +expect_exit 0 && { (( took < 15 )) || bad "the run took ${took}s with a 1s gate budget"; } && (( took < 15 )) \ + && expect_has "R9 gate: inconclusive" && expect_count R9 1 inconclusive && expect_has "inconclusive" \ + && { ! pgrep -f 'sleep 37' >/dev/null || bad "the gate's child process outlived the budget"; } \ + && ! pgrep -f 'sleep 37' >/dev/null && ok +pkill -f 'sleep 37' 2>/dev/null +rm -rf "$FAKE" +finish + +begin "a --gate-budget the gate finishes within writes the same counts as no budget" +mk_repo; : > "$BUNDLE/dirs.txt"; run_detect +cp "$BUNDLE/counts.tsv" "$BUNDLE/first.tsv" +run_detect --gate-budget 60 +expect_exit 0 && { cmp -s "$BUNDLE/first.tsv" "$BUNDLE/counts.tsv" || bad "counts.tsv differs under a budget the gate meets"; } \ + && cmp -s "$BUNDLE/first.tsv" "$BUNDLE/counts.tsv" && ok +finish + +begin "a gate that fails under --gate-budget fails the run (exit 2)" +mk_repo; : > "$BUNDLE/dirs.txt" +FAKE=$(mktemp -d) +cp "$DETECT" "$FAKE/ldd-detect.sh" +printf '#!/usr/bin/env bash\necho "check-repo-brain: internal scanner error"\nexit 2\n' > "$FAKE/check-repo-brain.sh" +OUT=$(cd "$REPO" && bash "$FAKE/ldd-detect.sh" --rules "$RULES" --gate-budget 30 --root . "$BUNDLE" 2>"$BUNDLE/.stderr"); CODE=$? +ERR=$(cat "$BUNDLE/.stderr"); rm -f "$BUNDLE/.stderr" +expect_exit 2 && expect_has "the R9 gate failed to run" && ok +rm -rf "$FAKE" +finish + +begin "a --gate-budget with no value is a usage error (exit 2)" +mk_repo +OUT=$(cd "$REPO" && bash "$DETECT" --root . "$BUNDLE" --gate-budget 2>&1); CODE=$?; ERR="" +expect_exit 2 && expect_has "--gate-budget needs a value" && ok +finish + } # run_row_cases write_installed_plugins "$SIBLINGS_FILE" go-linter-driven-development python-linter-driven-development linter-driven-development ts-react-linter-driven-development diff --git a/go-linter-driven-development/scripts/ldd-scope.sh b/go-linter-driven-development/scripts/ldd-scope.sh index 8b0e0e3..0e9e505 100755 --- a/go-linter-driven-development/scripts/ldd-scope.sh +++ b/go-linter-driven-development/scripts/ldd-scope.sh @@ -46,6 +46,11 @@ # # The bundle: # language.txt the group's language id; "custom TAB " under --glob +# test-re.txt the --test-re pattern, when one was passed, so ldd-detect.sh +# tells test files the same way +# docs.txt on a scoped rung, the markdown files the rung touches, one +# relative path per line, deleted ones included: the docs the +# R9 gate checks in full; --all writes none # files.txt the scope, one relative path per line. A file left out of # scope/ keeps its line with the reason: (not bundled: deleted), # (not bundled: binary), (not bundled: lines) over the @@ -332,6 +337,10 @@ write_bundle() { mkdir -p "$OUT/scope" if [[ "$GROUP_ID" == "custom" ]]; then printf 'custom\t%s\n' "$OPT_GLOB" > "$OUT/language.txt" else printf '%s\n' "$GROUP_ID" > "$OUT/language.txt"; fi + [[ -n "$OPT_TEST_RE" ]] && printf '%s\n' "$OPT_TEST_RE" > "$OUT/test-re.txt" + if [[ "$RUNG" != "all" ]]; then + sed 's|^\./||' "$CANDIDATES" | grep -E '\.md$' | LC_ALL=C sort -u > "$OUT/docs.txt" + fi # not_bundled_reason — prints the reason, or nothing when it is bundled not_bundled_reason() { @@ -379,10 +388,23 @@ write_bundle() { # comment lines plus every comment line of a bundled file the diff does not # touch (untracked, or committed and named on the command line); every comment # line of the bundle on --all; directives never count. + # comment_filter — keeps the "file:line:text" lines whose text carries a + # comment marker at the line start or after whitespace, with every + # double-quoted string literal blanked first, so a # or // inside a trace + # format string is not a comment; a """ docstring marker is kept + comment_filter() { + LC_ALL=C awk -v re="(^|[[:space:]])($LANG_COMMENT_RE)" '{ + i = index($0, ":"); rest = substr($0, i + 1); t = substr(rest, index(rest, ":") + 1) + gsub(/"""/, "\001", t) + gsub(/"([^"\\]|\\.)*"/, "\"\"", t) + gsub(/\001/, "\"\"\"", t) + if (t ~ re) print + }' + } # comment_lines_of ... — every comment line of the files, grep -nH shape comment_lines_of() { (( $# > 0 )) || return 0 - LC_ALL=C grep -nHE -e "$LANG_COMMENT_RE" -- "$@" 2>/dev/null | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" + LC_ALL=C grep -nH -e '' -- "$@" 2>/dev/null | comment_filter | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" } : > "$OUT/comments.txt" if [[ "$RUNG" == "all" ]]; then @@ -396,7 +418,7 @@ write_bundle() { /^\+/ { n++; print f ":" n ":" substr($0, 2); next } /^-/ { next } { n++ } - ' "$OUT/diff.patch" | LC_ALL=C grep -E -- "^[^:]*:[0-9]+:.*($LANG_COMMENT_RE)" | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" >> "$OUT/comments.txt" + ' "$OUT/diff.patch" | comment_filter | LC_ALL=C grep -vE -- "$LANG_DIRECTIVE_RE" >> "$OUT/comments.txt" fi # a bundled file the diff does not touch has no added lines: every comment # line of it is the critic's (an untracked file, or a committed file named diff --git a/go-linter-driven-development/scripts/ldd-scope_test.sh b/go-linter-driven-development/scripts/ldd-scope_test.sh index fe5d9fc..a658855 100755 --- a/go-linter-driven-development/scripts/ldd-scope_test.sh +++ b/go-linter-driven-development/scripts/ldd-scope_test.sh @@ -478,6 +478,17 @@ else fi finish +begin "a comment marker inside a string literal is not a comment line" +mk_repo; mkdir -p "$REPO/lib" +{ fx_comment_line; printf 'x = "#tag #tag" + "//tag // tag"\n'; } > "$REPO/lib/thing.zz" +git_commit_repo +run_scope --glob '*.zz' lib/thing.zz +expect_exit 0 && { grep -q "added by the test" "$BUNDLE/scope-out/comments.txt" || bad "the real comment line is missing"; } \ + && grep -q "added by the test" "$BUNDLE/scope-out/comments.txt" \ + && { ! grep -q '^lib/thing.zz:2:' "$BUNDLE/scope-out/comments.txt" || bad "the string literal line is listed as a comment"; } \ + && ! grep -q '^lib/thing.zz:2:' "$BUNDLE/scope-out/comments.txt" && ok +finish + } # run_row_cases write_installed_plugins "$SIBLINGS_FILE" go-linter-driven-development python-linter-driven-development linter-driven-development ts-react-linter-driven-development diff --git a/go-linter-driven-development/skills/pre-commit-review/SKILL.md b/go-linter-driven-development/skills/pre-commit-review/SKILL.md index 5da72c9..61faad3 100644 --- a/go-linter-driven-development/skills/pre-commit-review/SKILL.md +++ b/go-linter-driven-development/skills/pre-commit-review/SKILL.md @@ -80,11 +80,15 @@ writes `hits.tsv`, `hits-all.tsv` and `counts.tsv` into the bundle, and prints t question only a hunter's reading answers — then one line per rule family. That table, about 300 tokens, is the pre-filter: the parent prints no rule file, composes no detection command, and never reads a Falsifying-questions section. On a scoped review the R9 -gate's rows are the scope's own files' — an orphan doc or an unwired root is the -repository's state, for a `--all` review or BROADER CONTEXT, never this diff's -finding. When the +gate checks the docs the diff touches in full; in a doc that links to a scope file +or cites a symbol the diff removes it checks only those links and symbols; and it +checks the code edges of the scope's own files — an orphan doc elsewhere, a citing +doc's older drift or an unwired root is the repository's state, for a `--all` review or BROADER CONTEXT, never this diff's finding. The line +`R9 gate: …` under the summary says which docs the gate checked. A gate row whose +hits cell is `inconclusive` means the gate ran out of its `--gate-budget`: it is not +a `0`, and the documentation hunter answers that question from the prose. When the detection script exits 2 instead — a pattern its grep rejects, the R9 gate failing to -run — the review did not run: report its message, spawn nothing, and never render a +run, a scope in which no file matches the source glob — the review did not run: report its message, spawn nothing, and never render a rule as `skipped` or compose the pass by hand. A rule family none of whose rules has a hit — `0` on every row of theirs that is not `judgment` — is skipped: no hunter for it. @@ -110,7 +114,8 @@ zero prior use; the fix is a discussion or a separate PR, never silent inclusion The bundle exists (step 1 wrote it: `files.txt`, `diff.patch` on a scoped review, `scope/.txt` per bundled file with the file's own line numbers, `dirs.txt` on -`--all`, `comments.txt`, `hits.tsv`, `hits-all.tsv`, `counts.tsv`); nothing is +`--all`, `docs.txt` with the touched docs on a scoped review, `test-re.txt` with the +test-file pattern when one was passed, `comments.txt`, `hits.tsv`, `hits-all.tsv`, `counts.tsv`); nothing is written here. One Bash command, only when the scope is not empty, lists every rule path the prompts will carry and prints `sed -n '/^## Waiting for agents/,/^## The merged report/p' diff --git a/lang/generic/passthrough/CHANGELOG.md b/lang/generic/passthrough/CHANGELOG.md index 80a0749..a42f862 100644 --- a/lang/generic/passthrough/CHANGELOG.md +++ b/lang/generic/passthrough/CHANGELOG.md @@ -33,6 +33,23 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versio gap linter applies: the repository linter's rule where it has one, else 450 code lines per file and 8 and 13 files per directory. The package-size procedure runs the gap linter in place of a hand-written `find` command. +- **A scoped review runs the documentation gate on its scope only.** The gate + checks the docs the diff touches in full. In a doc that names a scope file or + cites a symbol the diff removes, it reports only those links and symbols. A + code-only diff that no doc cites skips the doc checks. On a 6-file diff in a + large monorepo, the detection pass took 46 seconds, down from about 90 minutes. +- **Doc symbols resolve in one pass over the repository.** In a git work tree the + gate reads the tracked files, submodule files and untracked files that git does + not ignore. Outside git it skips `bazel-*`, `node_modules` and other build + output directories. +- **`ldd-detect.sh --gate-budget `** stops a gate that runs too long and + counts its questions `inconclusive`, never 0. +- **The two review scripts cannot disagree.** `ldd-detect.sh` reads the + `--test-re` that `ldd-scope.sh` used. A `--glob` or `--test-re` that differs + from the bundle is a usage error. A scope whose files are not source files of + its language is an error, not a table of zeros. +- **`comments.txt` skips string literals.** A `#` or `//` inside a string literal + is not a comment line. ### Fixed @@ -44,6 +61,9 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versio `ldd-detect.sh` or `ldd-scope.sh` got an option such as `--root` or `--base` as its last argument, the script looped forever. It now stops with "