GNU tests: compare against a checked-in baseline

.github/workflows/run-gnu-test.yml

@@ -7,6 +7,17 @@ on:
     branches:
       - 'main'
 
+# The job only needs to check out the repository.
+permissions:
+  contents: read
+
+# Cancel superseded runs of a pull request or branch when it is updated.
+# On main, let a run in progress finish (GitHub still replaces a pending
+# run in the group with a newer one).
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
 jobs:
   native:
     name: Run GNU tests (native)
@@ -39,30 +50,9 @@ jobs:
           ./configure
           make libsmartcols/src/filter-parser.c libsmartcols/src/filter-scanner.c
           make -j$(nproc) check-programs
-      - name: Download test baseline if exists
-        id: download-test-baseline
-        continue-on-error: true
-        uses: actions/download-artifact@v8
-        with:
-          name: gnu-test-failures
-          path: uutils-util-linux/.reference/
-
       - name: Run GNU tests
         env:
           GNU_PROJECT_DIR: ${{ github.workspace }}/gnu-util-linux
         run: |
           cd uutils-util-linux
           ./scripts/run-tests.sh
-
-      - name: Update test baseline
-        if: github.ref == 'refs/heads/main'
-        run: |
-          cd uutils-util-linux
-          ./scripts/check-new-gnu-failures.sh --update-baseline
-
-      - name: Upload test baseline artifact
-        if: github.ref == 'refs/heads/main' || steps.download-test-baseline.outcome == 'failure'
-        uses: actions/upload-artifact@v7
-        with:
-          name: gnu-test-failures
-          path: uutils-util-linux/.reference/gnu-test-failures.txt

.gitignore

@@ -3,4 +3,3 @@ syntax: glob
 /target/
 
 .test-helpers
-.reference

scripts/check-new-gnu-failures.sh

@@ -4,9 +4,8 @@ set -euo pipefail
 cd "$(dirname "$0")/../"
 PROJECT_DIR="$(pwd)"
 DIFF_DIR="$PROJECT_DIR/.test-helpers/tests/diff"
-BASELINE="$PROJECT_DIR/.reference/gnu-test-failures.txt"
+BASELINE="$PROJECT_DIR/tests/gnu-test-failures.txt"
 UPDATE_BASELINE=false
-EMPTY_BASELINE=false
 
 for arg in "$@"; do
     case "$arg" in
@@ -15,21 +14,41 @@ for arg in "$@"; do
     esac
 done
 
-# Collect current failures (exclude .err companion files)
+# run-tests.sh deletes DIFF_DIR before running the GNU tests, and the GNU
+# harness (ts_gen_diff in tests/functions.sh) creates DIFF_DIR/<component>
+# whenever it compares a test's output, deleting only the diff files of
+# tests that pass. So no component directory means that no test produced
+# results: run.sh failed to start, died early, or skipped every test.
+# Without this check, such a run would have no failures, so every known
+# failure would be reported as fixed and the check would pass (or
+# --update-baseline would record an empty baseline).
+if [[ ! -d "$DIFF_DIR" ]] \
+    || [[ -z "$(find "$DIFF_DIR" -mindepth 1 -maxdepth 1 -type d)" ]]; then
+    echo "ERROR: No GNU test results in $DIFF_DIR" >&2
+    echo "  The GNU test run failed or did not run any tests." >&2
+    exit 2
+fi
+
+# Collect current failures. The GNU harness (ts_gen_diff in
+# tests/functions.sh) fails a test if its stdout or its stderr differs from
+# the expected output, writing <test> or <test>.err respectively, so a test
+# whose stdout matches but whose stderr differs leaves only <test>.err.
+# Map both to <test> and de-duplicate so such failures are counted too.
 current_tmp=$(mktemp)
 trap 'rm -f "$current_tmp"' EXIT
-if [[ -d "$DIFF_DIR" ]]; then
-    find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f ! -name "*.err" -print0 \
-        | sort -z \
-        | while IFS= read -r -d '' path; do
-            echo "${path#"$DIFF_DIR/"}"
-          done > "$current_tmp"
-fi
+find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f -print0 \
+    | while IFS= read -r -d '' path; do
+        path="${path#"$DIFF_DIR/"}"
+        echo "${path%.err}"
+      done \
+    | sort -u > "$current_tmp"
 
 # --update-baseline mode
 if [[ "$UPDATE_BASELINE" == true ]]; then
     {
         echo "# Known GNU test failures - format: util/testname"
+        echo "# Must match CI: run against the util-linux version checked out in"
+        echo "# .github/workflows/run-gnu-test.yml."
         echo "# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline"
         cat "$current_tmp"
     } > "$BASELINE"
@@ -38,16 +57,39 @@ if [[ "$UPDATE_BASELINE" == true ]]; then
     exit 0
 fi
 
+# The baseline is checked in, so a missing file is an error rather than a
+# reason to create one: creating it from the current results would accept
+# every current failure, including regressions, and pass.
+if [[ ! -f "$BASELINE" ]]; then
+    echo "ERROR: Baseline not found: $BASELINE" >&2
+    echo "  Restore it from version control, or create it with:" >&2
+    echo "    ./scripts/check-new-gnu-failures.sh --update-baseline" >&2
+    exit 2
+fi
+
 # Load baseline (strip comments and blank lines)
 baseline_tmp=$(mktemp)
 trap 'rm -f "$current_tmp" "$baseline_tmp"' EXIT
-if [[ -f "$BASELINE" ]]; then
-    grep -v '^\s*#' "$BASELINE" | grep -v '^\s*$' | sort > "$baseline_tmp"
-else
-    echo "WARNING: Baseline not found: $BASELINE" >&2
-    echo "  Run with --update-baseline to create it." >&2
-    touch "$baseline_tmp"
-    EMPTY_BASELINE=true
+# Use sed rather than grep -v: grep exits 1 when it selects no lines,
+# which pipefail and set -e would turn into an abort for a baseline with
+# no entries (e.g. after every known failure has been fixed).
+sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$/d' "$BASELINE" \
+    | sort > "$baseline_tmp"
+
+# As in the check on DIFF_DIR above: if a component with known failures
+# has no DIFF_DIR/<component>, none of its tests produced results (e.g.
+# they were all skipped, or the utility is no longer tested), so its known
+# failures must not be reported as fixed.
+missing_components=$(cut -d/ -f1 "$baseline_tmp" | sort -u \
+    | while IFS= read -r component; do
+        [[ -d "$DIFF_DIR/$component" ]] || echo "$component"
+      done)
+if [[ -n "$missing_components" ]]; then
+    echo "ERROR: No GNU test results for components with known failures:" >&2
+    awk '{print "  " $0}' <<< "$missing_components" >&2
+    echo "  Check the GNU test output for skipped tests, or remove the" >&2
+    echo "  components' entries from the baseline if they are no longer tested." >&2
+    exit 2
 fi
 
 # comm: -13 = lines only in current (new failures), -23 = lines only in baseline (fixed)
@@ -60,18 +102,6 @@ echo "Baseline failures: $(wc -l < "$baseline_tmp")"
 echo "Current failures:  $(wc -l < "$current_tmp")"
 echo ""
 
-if [[ "$EMPTY_BASELINE" == true ]]; then
-    mkdir -p "$(dirname "$BASELINE")"
-    {
-        echo "# Known GNU test failures - format: util/testname"
-        echo "# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline"
-        cat "$current_tmp"
-    } > "$BASELINE"
-    count=$(wc -l < "$current_tmp")
-    echo "Initial baseline created: $BASELINE ($count failures recorded)"
-    exit 0
-fi
-
 if [[ -n "$fixed_tests" ]]; then
     echo "Tests newly FIXED:"
     awk '{print " [FIXED] " $0}' <<< "$fixed_tests"

tests/gnu-test-failures.txt

@@ -0,0 +1,49 @@
+# Known GNU test failures - format: util/testname
+# Must match CI: run against the util-linux version checked out in
+# .github/workflows/run-gnu-test.yml.
+# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline
+cal/column-5
+cal/column-6
+cal/column-auto
+cal/jan1753-3w
+cal/jan1753-m3w
+cal/weeknum-3mjw
+cal/weeknum-3mw
+cal/weeknum-3sjw
+cal/weeknum-3sw
+cal/weeknum-ymjw
+cal/weeknum-ymw
+cal/weeknum-ysjw
+cal/weeknum-ysw
+cal/year-ym
+cal/year-ymj
+cal/year-ymjw
+cal/year-ymw
+cal/year-ys
+cal/year-ysj
+cal/year-ysjw
+cal/year-ysw
+lscpu/lscpu-arm-A510-A710-A715-X3
+lscpu/lscpu-armv7
+lscpu/lscpu-loongarch-kvm_on_loongson_3c6000
+lscpu/lscpu-loongarch-loongson_3a5000_hv
+lscpu/lscpu-ppc-qemu
+lscpu/lscpu-ppc64-POWER7
+lscpu/lscpu-ppc64-POWER7-64cpu
+lscpu/lscpu-rv64-linux
+lscpu/lscpu-rv64-milkvpioneer
+lscpu/lscpu-rv64-visionfive2
+lscpu/lscpu-s390-kvm
+lscpu/lscpu-s390-lpar
+lscpu/lscpu-s390-lpar-drawer
+lscpu/lscpu-s390-nested-virt
+lscpu/lscpu-s390-zvm
+lscpu/lscpu-sparc64
+lscpu/lscpu-vbox-win
+lscpu/lscpu-vmware_fpe
+lscpu/lscpu-x86_64-64cpu
+lscpu/lscpu-x86_64-64cpu-linux6.2
+lscpu/lscpu-x86_64-dell_e4310
+lscpu/lscpu-x86_64-epyc_7451
+lsmem/lsmem-s390-zvm-6g
+lsmem/lsmem-x86_64-16g