diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bd197ef..d1c655ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--no-snapshot-create` / `BASHUNIT_SNAPSHOT_CREATE=false` fails on a missing snapshot instead of recording it — the recommended CI setting, since a never-committed snapshot used to make CI green while asserting nothing (#901) - `--snapshot-update` / `BASHUNIT_SNAPSHOT_UPDATE=true` re-records existing snapshots (combine with `--filter`); snapshots holding a placeholder are left alone (#900) - `bashunit::mock `: a lone all-digits argument is an exit code, matching `bashunit::spy`; no throwaway `return 1` helper needed (#898) - `assert_have_been_called_with_any `: passes when *any* recorded call matches, instead of only the last one (#897) diff --git a/completions/_bashunit b/completions/_bashunit index 1bb15fbf..d2f52cf2 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -84,6 +84,7 @@ _bashunit() { '--shard[Run shard i of n]:shard:' \ '--rerun-failed[Replay only the tests that failed on the last run]' \ '--snapshot-update[Rewrite existing snapshots from the actual value]' \ + '--no-snapshot-create[Fail instead of recording a missing snapshot]' \ '(-vvv --verbose)'{-vvv,--verbose}'[Show execution details]' \ '--debug[Enable shell debug mode]::file:_files' \ '--no-output[Suppress all output]' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 81a06661..544cd910 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -15,7 +15,8 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --coverage --coverage-exclude \ --debug --detailed --env --exclude-tag --fail-on-risky --failures-only \ --filter --help --jobs --log-gha --log-junit --login --no-color \ --no-coverage-report --no-output --no-output-on-failure --no-parallel \ ---no-progress --output --parallel --profile --random-order --report-html \ +--no-progress --no-snapshot-create --output --parallel --profile \ +--random-order --report-html \ --report-json --report-junit --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ --skip-env-file --snapshot-update --stop-on-failure --strict --tag \ diff --git a/docs/ai-agents.md b/docs/ai-agents.md index a2e4015b..1ce9a341 100644 --- a/docs/ai-agents.md +++ b/docs/ai-agents.md @@ -66,7 +66,8 @@ not ok 2 - Fails The exit code is non-zero when a test **failed**. It is `0` for a run that was entirely skipped, incomplete, risky, or that only recorded new snapshots, so `$?` alone does not mean "everything passed" — read `summary.total` against `summary.passed`, or add -`--fail-on-risky`. +`--fail-on-risky` and `--no-snapshot-create` (the latter turns a first-time snapshot +recording into a failure, so a snapshot that was never committed cannot pass silently). ## Keep the loop tight diff --git a/docs/command-line.md b/docs/command-line.md index 956512b7..339e29ac 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -84,6 +84,7 @@ bashunit test tests/ --parallel --simple | `--shard /` | Run shard i of n (split suite across runners) | | `--rerun-failed` | Replay only the tests that failed on the last run | | `--snapshot-update` | Rewrite existing snapshots from the actual value | +| `--no-snapshot-create` | Fail on a missing snapshot instead of recording it | | `--show-skipped` | Show skipped tests summary at end | | `--show-incomplete` | Show incomplete tests summary at end | | `-vvv, --verbose` | Show execution details | @@ -598,6 +599,28 @@ Notes: - Review the diff afterwards: this rewrites files, so `git diff` is what tells you the new output is the output you meant. +### No snapshot create + +> `bashunit test --no-snapshot-create` + +Require every snapshot to exist already: a missing one fails the test instead of +being recorded, and the failure names the resolved path so you know what to +commit. + +``` +✗ Failed: Renders the header + Expected './tests/snapshots/header_test_sh.test_renders_the_header.snapshot' + does not exist; record it with a run without '--no-snapshot-create' +``` + +This is the CI setting. By default a first run records the snapshot and passes, +so a snapshot that was never committed — or is gitignored, or was lost — is +re-created on the fly and CI stays green while asserting nothing. Record +locally, commit the file, and let CI run with this flag. + +The two snapshot flags are opposites and pair up: `--snapshot-update` records +deliberately, `--no-snapshot-create` forbids recording by accident. + ### Rerun failed > `bashunit test --rerun-failed` diff --git a/docs/configuration.md b/docs/configuration.md index 10a10e24..e224c1b0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -222,6 +222,27 @@ BASHUNIT_SNAPSHOT_UPDATE=false ``` ::: +## Snapshot create + +> `BASHUNIT_SNAPSHOT_CREATE=true|false` + +Whether a missing snapshot is recorded on the fly. Enabled by default, which is +what makes the first run of a snapshot test pass. Set it to `false` in CI so a +snapshot that was never committed fails the run instead of being re-created +silently. + +Similar as using `--no-snapshot-create` option on the +[command line](/command-line#no-snapshot-create). + +::: code-group +```bash [CI: a missing snapshot fails] +BASHUNIT_SNAPSHOT_CREATE=false +``` +```bash [Default] +BASHUNIT_SNAPSHOT_CREATE=true +``` +::: + ## Rerun failed > `BASHUNIT_RERUN_FAILED=true|false` diff --git a/docs/snapshots.md b/docs/snapshots.md index b18c3aeb..abeb51f4 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -91,6 +91,24 @@ echo 'Run at ::ignore::' > snapshots/example.snapshot assert_match_snapshot "Run at $(date)" ``` +## Snapshots in CI + +Recommended CI setting: `--no-snapshot-create` (or `BASHUNIT_SNAPSHOT_CREATE=false`). + +By default the first run of a snapshot test writes the snapshot and passes. That is what +you want locally, and exactly what you do not want in CI: a snapshot that was never +committed, is gitignored, or was lost gets re-created on the fly, and the run is green +while asserting nothing. With the flag, a missing snapshot fails and the message names +the file to commit: + +``` +✗ Failed: Renders the header + Expected './tests/snapshots/header_test_sh.test_renders_the_header.snapshot' + does not exist; record it with a run without '--no-snapshot-create' +``` + +So: record locally, commit the snapshot, run CI with `--no-snapshot-create`. + ## Re-recording snapshots When an output change is intentional, run with `--snapshot-update` (or diff --git a/src/assert_snapshot.sh b/src/assert_snapshot.sh index 58f00782..e1c59698 100644 --- a/src/assert_snapshot.sh +++ b/src/assert_snapshot.sh @@ -21,16 +21,7 @@ function assert_match_snapshot() { bashunit::snapshot::resolve_file "${2:-}" "$test_fn" local snapshot_file=$_BASHUNIT_SNAPSHOT_FILE_OUT - if [ ! -f "$snapshot_file" ]; then - bashunit::snapshot::initialize "$snapshot_file" "$actual" - return - fi - - if bashunit::snapshot::update "$snapshot_file" "$actual"; then - return - fi - - bashunit::snapshot::compare "$actual" "$snapshot_file" "$test_fn" + bashunit::snapshot::assert "$actual" "$snapshot_file" "$test_fn" } function assert_match_snapshot_ignore_colors() { @@ -50,7 +41,22 @@ function assert_match_snapshot_ignore_colors() { bashunit::snapshot::resolve_file "${2:-}" "$test_fn" local snapshot_file=$_BASHUNIT_SNAPSHOT_FILE_OUT + bashunit::snapshot::assert "$actual" "$snapshot_file" "$test_fn" +} + +# The shared tail of both snapshot assertions: record a first-time snapshot, +# rewrite it under --snapshot-update, or compare against it. +# Arguments: $1 - actual value, $2 - snapshot path, $3 - test function name +function bashunit::snapshot::assert() { + local actual="$1" + local snapshot_file="$2" + local test_fn="$3" + if [ ! -f "$snapshot_file" ]; then + if ! bashunit::env::is_snapshot_create_enabled; then + bashunit::snapshot::fail_missing "$snapshot_file" "$test_fn" + return + fi bashunit::snapshot::initialize "$snapshot_file" "$actual" return fi @@ -62,6 +68,20 @@ function assert_match_snapshot_ignore_colors() { bashunit::snapshot::compare "$actual" "$snapshot_file" "$test_fn" } +# Fails because the snapshot has to exist already (--no-snapshot-create). The +# resolved path goes in the message: it is derived from the test file and +# function name, so a reader cannot otherwise tell which file to commit. +function bashunit::snapshot::fail_missing() { + local path="$1" + local func_name="$2" + local label + label=$(bashunit::helper::normalize_test_function_name "$func_name") + + bashunit::state::add_assertions_failed + bashunit::console_results::print_failed_test "$label" "$path" \ + "does not exist; record it with a run without" "--no-snapshot-create" +} + function bashunit::snapshot::match_with_placeholder() { local actual="$1" local snapshot="$2" diff --git a/src/console_header.sh b/src/console_header.sh index 7e7ca311..8d4ae9fc 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -141,6 +141,7 @@ Options: --shard / Run shard i of n (split the suite across runners) --rerun-failed Replay only the tests that failed on the last run (.bashunit/last-failed) --snapshot-update Rewrite existing snapshots from the actual value (combine with --filter) + --no-snapshot-create Fail on a missing snapshot instead of recording it (for CI) -vvv, --verbose Show execution details --debug [file] Enable shell debug mode --no-output Suppress all output diff --git a/src/env.sh b/src/env.sh index 1ce7a00a..79df78ef 100644 --- a/src/env.sh +++ b/src/env.sh @@ -257,6 +257,8 @@ _BASHUNIT_DEFAULT_SHARD_TOTAL="" _BASHUNIT_DEFAULT_RERUN_FAILED="false" # Rewrite existing snapshots from the actual value instead of comparing _BASHUNIT_DEFAULT_SNAPSHOT_UPDATE="false" +# Record a snapshot the first time it is asserted (false = a missing one fails) +_BASHUNIT_DEFAULT_SNAPSHOT_CREATE="true" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_PARALLEL_JOBS:=$_BASHUNIT_DEFAULT_PARALLEL_JOBS}" @@ -300,6 +302,7 @@ _BASHUNIT_DEFAULT_SNAPSHOT_UPDATE="false" # No bare SNAPSHOT_UPDATE alias either: rewriting files on disk is the last # setting that should be reachable by a generic name from the environment. : "${BASHUNIT_SNAPSHOT_UPDATE:=$_BASHUNIT_DEFAULT_SNAPSHOT_UPDATE}" +: "${BASHUNIT_SNAPSHOT_CREATE:=$_BASHUNIT_DEFAULT_SNAPSHOT_CREATE}" # Support NO_COLOR standard (https://no-color.org) if [ -n "${NO_COLOR:-}" ]; then BASHUNIT_NO_COLOR="true" @@ -547,6 +550,10 @@ function bashunit::env::is_tap_output_enabled() { [ "$BASHUNIT_OUTPUT_FORMAT" = "tap" ] } +function bashunit::env::is_snapshot_create_enabled() { + [ "$BASHUNIT_SNAPSHOT_CREATE" = "true" ] +} + function bashunit::env::is_snapshot_update_enabled() { [ "$BASHUNIT_SNAPSHOT_UPDATE" = "true" ] } diff --git a/src/main.sh b/src/main.sh index 0f7b8a19..e42548dd 100644 --- a/src/main.sh +++ b/src/main.sh @@ -281,6 +281,10 @@ function bashunit::main::cmd_test() { BASHUNIT_SNAPSHOT_UPDATE=true export -n BASHUNIT_SNAPSHOT_UPDATE ;; + --no-snapshot-create) + BASHUNIT_SNAPSHOT_CREATE=false + export -n BASHUNIT_SNAPSHOT_CREATE + ;; -w | --watch) BASHUNIT_WATCH_MODE=true export -n BASHUNIT_WATCH_MODE diff --git a/tests/acceptance/bashunit_snapshot_create_test.sh b/tests/acceptance/bashunit_snapshot_create_test.sh new file mode 100644 index 00000000..9d91e730 --- /dev/null +++ b/tests/acceptance/bashunit_snapshot_create_test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +function set_up_before_script() { + BASHUNIT_BIN="$(pwd)/bashunit" + FIXTURES_DIR="$(pwd)/tests/acceptance/fixtures/snapshot_update" +} + +function set_up() { + WORKDIR="$(mktemp -d)" + cp "$FIXTURES_DIR/snap.sh" "$WORKDIR/snap.sh" + cp "$FIXTURES_DIR/ignore_colors.sh" "$WORKDIR/ignore_colors.sh" + SNAPSHOT_ALPHA="$WORKDIR/snapshots/snap_sh.test_snapshot_alpha.snapshot" +} + +function tear_down() { + rm -rf "$WORKDIR" +} + +function test_a_missing_snapshot_is_recorded_by_default() { + local code=0 + (cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file snap.sh) >/dev/null 2>&1 || code=$? + + assert_same "0" "$code" + assert_file_exists "$SNAPSHOT_ALPHA" +} + +function test_a_missing_snapshot_fails_with_no_snapshot_create() { + local code=0 + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --no-snapshot-create snap.sh 2>&1) || code=$? + + assert_not_same "0" "$code" + assert_contains "snapshots/snap_sh.test_snapshot_alpha.snapshot" "$output" + assert_file_not_exists "$SNAPSHOT_ALPHA" +} + +function test_no_snapshot_create_also_covers_the_ignore_colors_assertion() { + local output + output=$(cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --no-snapshot-create ignore_colors.sh 2>&1) || true + + assert_contains "ignore_colors_sh.test_snapshot_without_colors.snapshot" "$output" + assert_contains "failed" "$output" +} + +function test_an_existing_snapshot_still_passes_with_no_snapshot_create() { + (cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file snap.sh) >/dev/null 2>&1 || true + + local code=0 + (cd "$WORKDIR" && "$BASHUNIT_BIN" --no-parallel --skip-env-file \ + --no-snapshot-create snap.sh) >/dev/null 2>&1 || code=$? + + assert_same "0" "$code" +} + +function test_env_variable_forbids_recording_too() { + local code=0 + (cd "$WORKDIR" && BASHUNIT_SNAPSHOT_CREATE=false "$BASHUNIT_BIN" \ + --no-parallel --skip-env-file snap.sh) >/dev/null 2>&1 || code=$? + + assert_not_same "0" "$code" + assert_file_not_exists "$SNAPSHOT_ALPHA" +} diff --git a/tests/acceptance/fixtures/snapshot_update/ignore_colors.sh b/tests/acceptance/fixtures/snapshot_update/ignore_colors.sh new file mode 100644 index 00000000..3c70e960 --- /dev/null +++ b/tests/acceptance/fixtures/snapshot_update/ignore_colors.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +function test_snapshot_without_colors() { + assert_match_snapshot_ignore_colors "plain value" +}