diff --git a/CHANGELOG.md b/CHANGELOG.md index d1c655ae..33fffd7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--snapshot-report-unused` lists snapshot files no test resolved — the ones a rename or deletion leaves behind. Reports only, never deletes; refused on partial runs (#902) - `--no-snapshot-create` / `BASHUNIT_SNAPSHOT_CREATE=false` fails on a missing snapshot instead of recording it — the recommended CI setting, since a never-committed snapshot used to make CI green while asserting nothing (#901) - `--snapshot-update` / `BASHUNIT_SNAPSHOT_UPDATE=true` re-records existing snapshots (combine with `--filter`); snapshots holding a placeholder are left alone (#900) - `bashunit::mock `: a lone all-digits argument is an exit code, matching `bashunit::spy`; no throwaway `return 1` helper needed (#898) diff --git a/completions/_bashunit b/completions/_bashunit index d2f52cf2..8b9afce5 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -85,6 +85,7 @@ _bashunit() { '--rerun-failed[Replay only the tests that failed on the last run]' \ '--snapshot-update[Rewrite existing snapshots from the actual value]' \ '--no-snapshot-create[Fail instead of recording a missing snapshot]' \ + '--snapshot-report-unused[List snapshot files no test resolved]' \ '(-vvv --verbose)'{-vvv,--verbose}'[Show execution details]' \ '--debug[Enable shell debug mode]::file:_files' \ '--no-output[Suppress all output]' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 544cd910..88525c84 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -19,7 +19,8 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --coverage --coverage-exclude \ --random-order --report-html \ --report-json --report-junit --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ ---skip-env-file --snapshot-update --stop-on-failure --strict --tag \ +--skip-env-file --snapshot-report-unused --snapshot-update \ +--stop-on-failure --strict --tag \ --test-timeout --verbose \ --watch -R -S -a -e -f -h -j -l -p -r -s -vvv -w" diff --git a/docs/command-line.md b/docs/command-line.md index 339e29ac..b3c6b527 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -85,6 +85,7 @@ bashunit test tests/ --parallel --simple | `--rerun-failed` | Replay only the tests that failed on the last run | | `--snapshot-update` | Rewrite existing snapshots from the actual value | | `--no-snapshot-create` | Fail on a missing snapshot instead of recording it | +| `--snapshot-report-unused` | List snapshot files no test resolved (deletes nothing) | | `--show-skipped` | Show skipped tests summary at end | | `--show-incomplete` | Show incomplete tests summary at end | | `-vvv, --verbose` | Show execution details | @@ -599,6 +600,31 @@ Notes: - Review the diff afterwards: this rewrites files, so `git diff` is what tells you the new output is the output you meant. +### Snapshot report unused + +> `bashunit test --snapshot-report-unused` + +Lists the snapshot files that no test resolved during the run — the ones left +behind when a test was renamed or deleted, since the filename is derived from +the test file and function name: + +``` +Unused snapshots (1), no test resolved them: + tests/snapshots/header_test_sh.test_old_name.snapshot +Nothing was deleted. Delete them yourself once you have checked the tests are gone. +``` + +Nothing is removed, on purpose: a snapshot deleted by mistake is silently +re-recorded on the next run and never fails again, so an automatic cleanup could +turn a real assertion into one that asserts nothing. + +The report only considers snapshots belonging to the test files the run +discovered, so running a single file or directory reports only that scope +instead of everything else in the same `snapshots/` directory. A run that +executes a *subset of the tests* in those files would still be misleading, so +the flag is refused alongside `--filter`, `--tag`, `--exclude-tag`, `--shard` +and `--rerun-failed`. + ### No snapshot create > `bashunit test --no-snapshot-create` diff --git a/docs/snapshots.md b/docs/snapshots.md index abeb51f4..7a75d15e 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -91,6 +91,28 @@ echo 'Run at ::ignore::' > snapshots/example.snapshot assert_match_snapshot "Run at $(date)" ``` +## Finding unused snapshots + +A snapshot file is named after its test file and test function, so renaming or deleting a +test orphans its snapshot: nothing reads it, nothing reports it, and it stays on disk. + +`--snapshot-report-unused` lists the snapshot files no test resolved during the run: + +```bash +./bashunit --snapshot-report-unused tests/ +``` +``` +Unused snapshots (1), no test resolved them: + tests/snapshots/header_test_sh.test_old_name.snapshot +Nothing was deleted. Delete them yourself once you have checked the tests are gone. +``` + +It never deletes anything — a snapshot removed by mistake is re-recorded on the next run +and never fails again, so an automatic cleanup could quietly turn a real assertion into a +rubber stamp. Only snapshots belonging to the test files of the run are considered, and +the flag is refused with `--filter`, `--tag`, `--exclude-tag`, `--shard` and +`--rerun-failed`, whose partial runs would report live files as unused. + ## Snapshots in CI Recommended CI setting: `--no-snapshot-create` (or `BASHUNIT_SNAPSHOT_CREATE=false`). diff --git a/src/assert_snapshot.sh b/src/assert_snapshot.sh index e1c59698..aff3a23e 100644 --- a/src/assert_snapshot.sh +++ b/src/assert_snapshot.sh @@ -44,6 +44,97 @@ function assert_match_snapshot_ignore_colors() { bashunit::snapshot::assert "$actual" "$snapshot_file" "$test_fn" } +# Strips the `./` resolve_file prepends and squeezes doubled slashes, so a path +# recorded by an assertion and the same path as `find` prints it compare equal. +function bashunit::snapshot::normalize_path() { + local path="$1" + while [ "${path#./}" != "$path" ]; do + path="${path#./}" + done + while [ "${path%%//*}" != "$path" ]; do + path="${path%%//*}/${path#*//}" + done + builtin echo "$path" +} + +# Lists the snapshot files under $@ that no test resolved this run, and says so. +# Deliberately reports only: a snapshot deleted by mistake is re-recorded on the +# next run and never fails again, so an automatic cleanup could quietly turn a +# real assertion into one that asserts nothing. +function bashunit::snapshot::report_unused() { + local -a search_paths=() + local path + for path in "$@"; do + if [ -d "$path" ]; then + search_paths[${#search_paths[@]}]="$path" + elif [ -f "$path" ]; then + case "$path" in + */*) search_paths[${#search_paths[@]}]="${path%/*}" ;; + *) search_paths[${#search_paths[@]}]="." ;; + esac + fi + done + if [ "${#search_paths[@]}" -eq 0 ]; then + search_paths[0]="." + fi + + # A snapshot is named "..snapshot", + # so it can be attributed to the test file that owns it. Only the files this + # run discovered are considered: running one file or one directory must not + # report every snapshot belonging to the files it did not run. + local owners="" + for path in "$@"; do + [ -f "$path" ] || continue + bashunit::helper::normalize_variable_name_to_slot "${path##*/}" + owners="$owners$_BASHUNIT_HELPER_VARNAME_OUT +" + done + + local used="" + if [ -f "${SNAPSHOT_USED_OUTPUT_PATH:-}" ]; then + local used_path + while IFS= read -r used_path; do + [ -z "$used_path" ] && continue + used="$used$(bashunit::snapshot::normalize_path "$used_path") +" + done <"$SNAPSHOT_USED_OUTPUT_PATH" + fi + + local unused="" + local total=0 + local dir file normalized + while IFS= read -r dir; do + [ -z "$dir" ] && continue + for file in "$dir"/*.snapshot; do + [ -f "$file" ] || continue + normalized="$(bashunit::snapshot::normalize_path "$file")" + case "$used" in + *"$normalized"$'\n'*) continue ;; + esac + local owner="${file##*/}" + owner="${owner%%.*}" + case "$owners" in + *"$owner"$'\n'*) ;; + *) continue ;; + esac + total=$((total + 1)) + unused="$unused $normalized +" + done + done < <(find "${search_paths[@]}" -type d -name snapshots 2>/dev/null | sort -u) + + if [ "$total" -eq 0 ]; then + printf "\n%sNo unused snapshots.%s\n" \ + "${_BASHUNIT_COLOR_FAINT}" "${_BASHUNIT_COLOR_DEFAULT}" + return + fi + + printf "\n%sUnused snapshots (%s), no test resolved them:%s\n%s" \ + "${_BASHUNIT_COLOR_SKIPPED}" "$total" "${_BASHUNIT_COLOR_DEFAULT}" "$unused" + printf "%sNothing was deleted. Delete them yourself once you have checked the tests are gone.%s\n" \ + "${_BASHUNIT_COLOR_FAINT}" "${_BASHUNIT_COLOR_DEFAULT}" +} + # The shared tail of both snapshot assertions: record a first-time snapshot, # rewrite it under --snapshot-update, or compare against it. # Arguments: $1 - actual value, $2 - snapshot path, $3 - test function name @@ -52,6 +143,10 @@ function bashunit::snapshot::assert() { local snapshot_file="$2" local test_fn="$3" + if bashunit::env::is_snapshot_report_unused_enabled; then + printf '%s\n' "$snapshot_file" >>"${SNAPSHOT_USED_OUTPUT_PATH:-/dev/null}" 2>/dev/null || true + fi + if [ ! -f "$snapshot_file" ]; then if ! bashunit::env::is_snapshot_create_enabled; then bashunit::snapshot::fail_missing "$snapshot_file" "$test_fn" diff --git a/src/console_header.sh b/src/console_header.sh index 8d4ae9fc..e8c31382 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -142,6 +142,7 @@ Options: --rerun-failed Replay only the tests that failed on the last run (.bashunit/last-failed) --snapshot-update Rewrite existing snapshots from the actual value (combine with --filter) --no-snapshot-create Fail on a missing snapshot instead of recording it (for CI) + --snapshot-report-unused List snapshot files no test resolved (full runs only, deletes nothing) -vvv, --verbose Show execution details --debug [file] Enable shell debug mode --no-output Suppress all output diff --git a/src/env.sh b/src/env.sh index 79df78ef..64e5326e 100644 --- a/src/env.sh +++ b/src/env.sh @@ -259,6 +259,8 @@ _BASHUNIT_DEFAULT_RERUN_FAILED="false" _BASHUNIT_DEFAULT_SNAPSHOT_UPDATE="false" # Record a snapshot the first time it is asserted (false = a missing one fails) _BASHUNIT_DEFAULT_SNAPSHOT_CREATE="true" +# Report snapshot files that no test resolved during the run +_BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_PARALLEL_JOBS:=$_BASHUNIT_DEFAULT_PARALLEL_JOBS}" @@ -303,6 +305,7 @@ _BASHUNIT_DEFAULT_SNAPSHOT_CREATE="true" # setting that should be reachable by a generic name from the environment. : "${BASHUNIT_SNAPSHOT_UPDATE:=$_BASHUNIT_DEFAULT_SNAPSHOT_UPDATE}" : "${BASHUNIT_SNAPSHOT_CREATE:=$_BASHUNIT_DEFAULT_SNAPSHOT_CREATE}" +: "${BASHUNIT_SNAPSHOT_REPORT_UNUSED:=$_BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED}" # Support NO_COLOR standard (https://no-color.org) if [ -n "${NO_COLOR:-}" ]; then BASHUNIT_NO_COLOR="true" @@ -550,6 +553,10 @@ function bashunit::env::is_tap_output_enabled() { [ "$BASHUNIT_OUTPUT_FORMAT" = "tap" ] } +function bashunit::env::is_snapshot_report_unused_enabled() { + [ "$BASHUNIT_SNAPSHOT_REPORT_UNUSED" = "true" ] +} + function bashunit::env::is_snapshot_create_enabled() { [ "$BASHUNIT_SNAPSHOT_CREATE" = "true" ] } @@ -678,6 +685,10 @@ WORKER_STDERR_OUTPUT_PREFIX="$_BASHUNIT_RUN_OUTPUT_DIR/worker-stderr" # Collects ":" for every failing test in a run so the # next --rerun-failed can replay just those. Shared across parallel subshells. RERUN_FAILED_OUTPUT_PATH="$_BASHUNIT_RUN_OUTPUT_DIR/rerun-failed" +# Collects every snapshot path a test resolved, so --snapshot-report-unused can +# diff it against what is on disk. Appended from the test subshells, only when +# the flag is on, so a normal run pays nothing. +SNAPSHOT_USED_OUTPUT_PATH="$_BASHUNIT_RUN_OUTPUT_DIR/snapshots-used" # Shared temp directory, initialized once at startup for performance. BASHUNIT_TEMP_DIR="${TMPDIR:-/tmp}/bashunit/tmp" diff --git a/src/main.sh b/src/main.sh index e42548dd..a6cc91b7 100644 --- a/src/main.sh +++ b/src/main.sh @@ -285,6 +285,10 @@ function bashunit::main::cmd_test() { BASHUNIT_SNAPSHOT_CREATE=false export -n BASHUNIT_SNAPSHOT_CREATE ;; + --snapshot-report-unused) + BASHUNIT_SNAPSHOT_REPORT_UNUSED=true + export -n BASHUNIT_SNAPSHOT_REPORT_UNUSED + ;; -w | --watch) BASHUNIT_WATCH_MODE=true export -n BASHUNIT_WATCH_MODE @@ -538,6 +542,23 @@ function bashunit::main::cmd_test() { fi fi + # A run that executes a subset resolves a subset of the snapshots, so every + # snapshot the subset skipped would be reported as unused. Refusing beats + # printing a list that invites deleting live files. + if bashunit::env::is_snapshot_report_unused_enabled; then + local _partial_flag="" + [ -n "$filter" ] && _partial_flag="--filter" + [ -n "$tag_filter" ] && _partial_flag="--tag" + [ -n "$exclude_tag_filter" ] && _partial_flag="--exclude-tag" + [ -n "${BASHUNIT_SHARD_INDEX:-}" ] && _partial_flag="--shard" + bashunit::rerun::is_enabled && _partial_flag="--rerun-failed" + if [ -n "$_partial_flag" ]; then + printf "%sError: --snapshot-report-unused needs a full run; %s only runs a subset.%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "$_partial_flag" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + fi + fi + # --rerun-failed: restrict discovery to the files recorded as failing last # run. Function-level filtering happens in the runner; --filter/--tag still # apply on top. With no recorded failures, fall back to the full suite. @@ -1062,6 +1083,12 @@ function bashunit::main::exec_tests() { bashunit::console_results::print_profile_and_reset fi + # Report only: it never touches exit_code, captured above. With no discovered + # file the helper falls back to the working directory. + if bashunit::env::is_snapshot_report_unused_enabled; then + bashunit::snapshot::report_unused ${test_files[@]+"${test_files[@]}"} + fi + if [ -n "$BASHUNIT_LOG_JUNIT" ]; then bashunit::reports::generate_junit_xml "$BASHUNIT_LOG_JUNIT" fi diff --git a/tests/acceptance/bashunit_snapshot_unused_test.sh b/tests/acceptance/bashunit_snapshot_unused_test.sh new file mode 100644 index 00000000..559282ef --- /dev/null +++ b/tests/acceptance/bashunit_snapshot_unused_test.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +function set_up_before_script() { + BASHUNIT_BIN="$(pwd)/bashunit" + FIXTURES_DIR="$(pwd)/tests/acceptance/fixtures/snapshot_update" +} + +function set_up() { + WORKDIR="$(mktemp -d)" + cp "$FIXTURES_DIR/snap.sh" "$WORKDIR/snap.sh" + mkdir -p "$WORKDIR/snapshots" + echo "alpha value" >"$WORKDIR/snapshots/snap_sh.test_snapshot_alpha.snapshot" + echo "beta value" >"$WORKDIR/snapshots/snap_sh.test_snapshot_beta.snapshot" + ORPHAN="$WORKDIR/snapshots/snap_sh.test_snapshot_deleted.snapshot" + echo "left behind" >"$ORPHAN" +} + +function tear_down() { + rm -rf "$WORKDIR" +} + +function test_reports_the_snapshot_no_test_resolved() { + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --snapshot-report-unused snap.sh 2>&1) || true + + assert_contains "snap_sh.test_snapshot_deleted.snapshot" "$output" + assert_not_contains "snap_sh.test_snapshot_alpha.snapshot" "$output" +} + +function test_reports_nothing_when_every_snapshot_is_used() { + rm -f "$ORPHAN" + + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --snapshot-report-unused snap.sh 2>&1) || true + + assert_contains "No unused snapshots" "$output" +} + +function test_a_snapshot_of_a_test_file_outside_the_run_is_not_reported() { + echo "not mine" >"$WORKDIR/snapshots/other_test_sh.test_something.snapshot" + + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --snapshot-report-unused snap.sh 2>&1) || true + + assert_not_contains "other_test_sh" "$output" + assert_contains "snap_sh.test_snapshot_deleted.snapshot" "$output" +} + +function test_the_report_does_not_delete_anything() { + (cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --snapshot-report-unused snap.sh) >/dev/null 2>&1 || true + + assert_file_exists "$ORPHAN" +} + +function test_no_report_without_the_flag() { + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file snap.sh 2>&1) || true + + assert_not_contains "nused snapshot" "$output" +} + +function test_the_flag_is_refused_on_a_partial_run() { + local code=0 + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --snapshot-report-unused --filter alpha snap.sh 2>&1) || code=$? + + assert_same "1" "$code" + assert_contains "--filter" "$output" + assert_not_contains "test_snapshot_deleted" "$output" +} + +function test_the_flag_is_refused_alongside_a_shard() { + local code=0 + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --snapshot-report-unused --shard 1/2 snap.sh 2>&1) || code=$? + + assert_same "1" "$code" + assert_contains "--shard" "$output" +}