From b712c744f9c506c9b80ca6bee8e4ed64112a7f91 Mon Sep 17 00:00:00 2001 From: Jesse Rosenstock Date: Wed, 21 Oct 2026 16:20:10 +0200 Subject: [PATCH 1/5] check-new-gnu-failures.sh: accept a baseline with no entries The baseline is loaded with grep -v '^\s*#' "$BASELINE" | grep -v '^\s*$' | sort grep exits 1 when it selects no lines, so for a baseline that contains only the header comments, pipefail makes the pipeline fail and set -e aborts the script with exit status 1 and no message. Such a baseline is what --update-baseline (or the initial-baseline path) writes after a run with no failures, so once every known failure is fixed, every following check fails without saying why. Strip comments and blank lines with sed instead, which exits 0 whether or not it deletes anything. `{ grep ... || true; }` would also work but would hide real grep errors such as an unreadable file. --- scripts/check-new-gnu-failures.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/check-new-gnu-failures.sh b/scripts/check-new-gnu-failures.sh index b19b8cbb..8c834675 100755 --- a/scripts/check-new-gnu-failures.sh +++ b/scripts/check-new-gnu-failures.sh @@ -42,7 +42,11 @@ fi baseline_tmp=$(mktemp) trap 'rm -f "$current_tmp" "$baseline_tmp"' EXIT if [[ -f "$BASELINE" ]]; then - grep -v '^\s*#' "$BASELINE" | grep -v '^\s*$' | sort > "$baseline_tmp" + # Use sed rather than grep -v: grep exits 1 when it selects no lines, + # which pipefail and set -e would turn into an abort for a baseline + # with no entries (e.g. after every known failure has been fixed). + sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$/d' "$BASELINE" \ + | sort > "$baseline_tmp" else echo "WARNING: Baseline not found: $BASELINE" >&2 echo " Run with --update-baseline to create it." >&2 From d8673409a254a64090c17ce353f0c4a3e7b94754 Mon Sep 17 00:00:00 2001 From: Jesse Rosenstock Date: Wed, 21 Oct 2026 16:21:04 +0200 Subject: [PATCH 2/5] check-new-gnu-failures.sh: fail if the GNU tests produced no results If the GNU test run produced no results, the script found no failures and reported every baseline entry as fixed, so the check passed. This happens if run.sh is missing or exits early (e.g. on an unknown test component), or if every test of a component is skipped. In the same situation, --update-baseline recorded an empty baseline. Use the diff directory to tell whether tests ran. run-tests.sh deletes .test-helpers/tests/diff before the run, and the GNU harness (ts_gen_diff in tests/functions.sh) creates diff/ whenever it compares a test's output, deleting only the diff files of tests that pass. So: * If the diff directory has no component directories, no test produced results: exit 2 with an error, also in --update-baseline mode. * If a component with baseline entries has no diff directory, none of its tests produced results: exit 2 rather than report its known failures as fixed. Alternatives considered: * run.sh's exit status: it exits 1 both when tests fail and when it cannot run them (unknown component, xargs error), so it cannot tell the two apart. * Parsing run.sh's summary ("N tests of M FAILED", "All M tests PASSED"): run-tests.sh would have to capture the output, and the check would depend on its wording. * tests/output//: the harness creates an output file for every test that starts, including skipped ones, and run-tests.sh does not delete tests/output, so results from an earlier run would satisfy the check. * Requiring results for every utility in .test-helpers/utils.list: some utilities have every test skipped in some environments (e.g. dmesg and rev when test helpers are not built), so this would fail runs that are fine. --- scripts/check-new-gnu-failures.sh | 43 ++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/scripts/check-new-gnu-failures.sh b/scripts/check-new-gnu-failures.sh index 8c834675..acc77244 100755 --- a/scripts/check-new-gnu-failures.sh +++ b/scripts/check-new-gnu-failures.sh @@ -15,16 +15,29 @@ for arg in "$@"; do esac done +# run-tests.sh deletes DIFF_DIR before running the GNU tests, and the GNU +# harness (ts_gen_diff in tests/functions.sh) creates DIFF_DIR/ +# whenever it compares a test's output, deleting only the diff files of +# tests that pass. So no component directory means that no test produced +# results: run.sh failed to start, died early, or skipped every test. +# Without this check, such a run would have no failures, so every known +# failure would be reported as fixed and the check would pass (or +# --update-baseline would record an empty baseline). +if [[ ! -d "$DIFF_DIR" ]] \ + || [[ -z "$(find "$DIFF_DIR" -mindepth 1 -maxdepth 1 -type d)" ]]; then + echo "ERROR: No GNU test results in $DIFF_DIR" >&2 + echo " The GNU test run failed or did not run any tests." >&2 + exit 2 +fi + # Collect current failures (exclude .err companion files) current_tmp=$(mktemp) trap 'rm -f "$current_tmp"' EXIT -if [[ -d "$DIFF_DIR" ]]; then - find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f ! -name "*.err" -print0 \ - | sort -z \ - | while IFS= read -r -d '' path; do - echo "${path#"$DIFF_DIR/"}" - done > "$current_tmp" -fi +find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f ! -name "*.err" -print0 \ + | sort -z \ + | while IFS= read -r -d '' path; do + echo "${path#"$DIFF_DIR/"}" + done > "$current_tmp" # --update-baseline mode if [[ "$UPDATE_BASELINE" == true ]]; then @@ -54,6 +67,22 @@ else EMPTY_BASELINE=true fi +# As in the check on DIFF_DIR above: if a component with known failures +# has no DIFF_DIR/, none of its tests produced results (e.g. +# they were all skipped, or the utility is no longer tested), so its known +# failures must not be reported as fixed. +missing_components=$(cut -d/ -f1 "$baseline_tmp" | sort -u \ + | while IFS= read -r component; do + [[ -d "$DIFF_DIR/$component" ]] || echo "$component" + done) +if [[ -n "$missing_components" ]]; then + echo "ERROR: No GNU test results for components with known failures:" >&2 + awk '{print " " $0}' <<< "$missing_components" >&2 + echo " Check the GNU test output for skipped tests, or remove the" >&2 + echo " components' entries from the baseline if they are no longer tested." >&2 + exit 2 +fi + # comm: -13 = lines only in current (new failures), -23 = lines only in baseline (fixed) new_failures=$(comm -13 "$baseline_tmp" "$current_tmp") fixed_tests=$(comm -23 "$baseline_tmp" "$current_tmp") From 9cb99f50e4e30767370092f4942ebf21be6cd153 Mon Sep 17 00:00:00 2001 From: Jesse Rosenstock Date: Wed, 21 Oct 2026 15:56:40 +0200 Subject: [PATCH 3/5] check-new-gnu-failures.sh: count stderr-only failures The GNU harness (ts_gen_diff in tests/functions.sh) fails a test if either its stdout or its stderr differs from the expected output. It writes the stdout diff to tests/diff// and the stderr diff to tests/diff//.err, and removes each when there is no difference. The script collected failures by listing diff files and skipping *.err, so a test whose stdout matches but whose stderr differs was never counted: a change that only breaks an error message passed the check even though run.sh reports the test as FAILED. For example, invoking the multicall binary through a link by full path makes rename print the full path instead of `rename:` in its error messages. run.sh then reports rename/exit_codes and rename/symlink as FAILED, but both only have .err diffs, so the script did not see them. Map both kinds of diff file to the test name and de-duplicate. The baseline format is unchanged (`util/testname`, one per line), but it can now contain tests that fail only on stderr. A baseline written by the old script lacks those, so they are reported as new failures until the baseline is updated with --update-baseline. --- scripts/check-new-gnu-failures.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/check-new-gnu-failures.sh b/scripts/check-new-gnu-failures.sh index acc77244..2c82f8b7 100755 --- a/scripts/check-new-gnu-failures.sh +++ b/scripts/check-new-gnu-failures.sh @@ -30,14 +30,19 @@ if [[ ! -d "$DIFF_DIR" ]] \ exit 2 fi -# Collect current failures (exclude .err companion files) +# Collect current failures. The GNU harness (ts_gen_diff in +# tests/functions.sh) fails a test if its stdout or its stderr differs from +# the expected output, writing or .err respectively, so a test +# whose stdout matches but whose stderr differs leaves only .err. +# Map both to and de-duplicate so such failures are counted too. current_tmp=$(mktemp) trap 'rm -f "$current_tmp"' EXIT -find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f ! -name "*.err" -print0 \ - | sort -z \ +find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f -print0 \ | while IFS= read -r -d '' path; do - echo "${path#"$DIFF_DIR/"}" - done > "$current_tmp" + path="${path#"$DIFF_DIR/"}" + echo "${path%.err}" + done \ + | sort -u > "$current_tmp" # --update-baseline mode if [[ "$UPDATE_BASELINE" == true ]]; then From 41ade69fdee11ff33c5890d7e81e88a90a168b82 Mon Sep 17 00:00:00 2001 From: Jesse Rosenstock Date: Wed, 21 Oct 2026 15:59:03 +0200 Subject: [PATCH 4/5] ci: check the GNU test baseline into the repo The GNU test workflow never detected a regression. It tried to fetch the baseline with actions/download-artifact@v8 using only `name` and `path`. The action README documents the `github-token` input as "Optional. If unspecified, the action will download artifacts from the current repo and the current workflow run.", and `run-id` as applying only "If github-token is specified". Without a token it can only see artifacts uploaded earlier in the same run, but the baseline was uploaded by earlier runs, so the step always failed with "Artifact not found for name: gnu-test-failures" (hidden by `continue-on-error`). The check script then found no baseline, wrote one from the current results and exited 0. This happened on pull requests and on main alike: main also never compared, it only re-uploaded the new file, which no later run could download. Both are visible in the logs of main run 39571991580 and pull request run 39609494316. Fixing the download would need a token with actions:read and a lookup of the latest successful main run ID, since artifacts are addressable only by run; uutils/coreutils keeps its reference results on a separate branch instead. A file in the repo is simpler, versioned with the code it describes, and lets a pull request that fixes or knowingly breaks tests update the baseline in the same change: * Add tests/gnu-test-failures.txt. A non-hidden path next to the other test data is easier to find and review than .reference/, which was named for the downloaded artifact; stop ignoring .reference. * Remove the download, main-only update and upload steps. * Make a missing baseline an error in check-new-gnu-failures.sh instead of silently creating one and passing. This is not limited to CI: with the file checked in, a missing baseline means a broken checkout, and recreating it from local results would hide regressions and record results from a different util-linux version than CI uses. --update-baseline still creates it. * Note in the baseline header that it must be generated against the util-linux version the workflow checks out. The initial file has the entries of the gnu-test-failures artifact from main run 39571991580 (commit c05b0e93, util-linux v2.41.4, ubuntu-24.04), the latest successful main run, written with --update-baseline from a diff directory containing those entries. That artifact was produced before stderr-only failures were counted, so it was checked against the run log: the 45 tests run.sh reported as FAILED are exactly the 45 entries, so none were missing. --- .github/workflows/run-gnu-test.yml | 21 ------------- .gitignore | 1 - scripts/check-new-gnu-failures.sh | 44 +++++++++++---------------- tests/gnu-test-failures.txt | 49 ++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 48 deletions(-) create mode 100644 tests/gnu-test-failures.txt diff --git a/.github/workflows/run-gnu-test.yml b/.github/workflows/run-gnu-test.yml index a58f48fe..e457a750 100644 --- a/.github/workflows/run-gnu-test.yml +++ b/.github/workflows/run-gnu-test.yml @@ -39,30 +39,9 @@ jobs: ./configure make libsmartcols/src/filter-parser.c libsmartcols/src/filter-scanner.c make -j$(nproc) check-programs - - name: Download test baseline if exists - id: download-test-baseline - continue-on-error: true - uses: actions/download-artifact@v8 - with: - name: gnu-test-failures - path: uutils-util-linux/.reference/ - - name: Run GNU tests env: GNU_PROJECT_DIR: ${{ github.workspace }}/gnu-util-linux run: | cd uutils-util-linux ./scripts/run-tests.sh - - - name: Update test baseline - if: github.ref == 'refs/heads/main' - run: | - cd uutils-util-linux - ./scripts/check-new-gnu-failures.sh --update-baseline - - - name: Upload test baseline artifact - if: github.ref == 'refs/heads/main' || steps.download-test-baseline.outcome == 'failure' - uses: actions/upload-artifact@v7 - with: - name: gnu-test-failures - path: uutils-util-linux/.reference/gnu-test-failures.txt diff --git a/.gitignore b/.gitignore index 89305739..300fc31a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,3 @@ syntax: glob /target/ .test-helpers -.reference diff --git a/scripts/check-new-gnu-failures.sh b/scripts/check-new-gnu-failures.sh index 2c82f8b7..9b6f93fd 100755 --- a/scripts/check-new-gnu-failures.sh +++ b/scripts/check-new-gnu-failures.sh @@ -4,9 +4,8 @@ set -euo pipefail cd "$(dirname "$0")/../" PROJECT_DIR="$(pwd)" DIFF_DIR="$PROJECT_DIR/.test-helpers/tests/diff" -BASELINE="$PROJECT_DIR/.reference/gnu-test-failures.txt" +BASELINE="$PROJECT_DIR/tests/gnu-test-failures.txt" UPDATE_BASELINE=false -EMPTY_BASELINE=false for arg in "$@"; do case "$arg" in @@ -48,6 +47,8 @@ find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f -print0 \ if [[ "$UPDATE_BASELINE" == true ]]; then { echo "# Known GNU test failures - format: util/testname" + echo "# Must match CI: run against the util-linux version checked out in" + echo "# .github/workflows/run-gnu-test.yml." echo "# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline" cat "$current_tmp" } > "$BASELINE" @@ -56,21 +57,24 @@ if [[ "$UPDATE_BASELINE" == true ]]; then exit 0 fi +# The baseline is checked in, so a missing file is an error rather than a +# reason to create one: creating it from the current results would accept +# every current failure, including regressions, and pass. +if [[ ! -f "$BASELINE" ]]; then + echo "ERROR: Baseline not found: $BASELINE" >&2 + echo " Restore it from version control, or create it with:" >&2 + echo " ./scripts/check-new-gnu-failures.sh --update-baseline" >&2 + exit 2 +fi + # Load baseline (strip comments and blank lines) baseline_tmp=$(mktemp) trap 'rm -f "$current_tmp" "$baseline_tmp"' EXIT -if [[ -f "$BASELINE" ]]; then - # Use sed rather than grep -v: grep exits 1 when it selects no lines, - # which pipefail and set -e would turn into an abort for a baseline - # with no entries (e.g. after every known failure has been fixed). - sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$/d' "$BASELINE" \ - | sort > "$baseline_tmp" -else - echo "WARNING: Baseline not found: $BASELINE" >&2 - echo " Run with --update-baseline to create it." >&2 - touch "$baseline_tmp" - EMPTY_BASELINE=true -fi +# Use sed rather than grep -v: grep exits 1 when it selects no lines, +# which pipefail and set -e would turn into an abort for a baseline with +# no entries (e.g. after every known failure has been fixed). +sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$/d' "$BASELINE" \ + | sort > "$baseline_tmp" # As in the check on DIFF_DIR above: if a component with known failures # has no DIFF_DIR/, none of its tests produced results (e.g. @@ -98,18 +102,6 @@ echo "Baseline failures: $(wc -l < "$baseline_tmp")" echo "Current failures: $(wc -l < "$current_tmp")" echo "" -if [[ "$EMPTY_BASELINE" == true ]]; then - mkdir -p "$(dirname "$BASELINE")" - { - echo "# Known GNU test failures - format: util/testname" - echo "# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline" - cat "$current_tmp" - } > "$BASELINE" - count=$(wc -l < "$current_tmp") - echo "Initial baseline created: $BASELINE ($count failures recorded)" - exit 0 -fi - if [[ -n "$fixed_tests" ]]; then echo "Tests newly FIXED:" awk '{print " [FIXED] " $0}' <<< "$fixed_tests" diff --git a/tests/gnu-test-failures.txt b/tests/gnu-test-failures.txt new file mode 100644 index 00000000..7edf7c18 --- /dev/null +++ b/tests/gnu-test-failures.txt @@ -0,0 +1,49 @@ +# Known GNU test failures - format: util/testname +# Must match CI: run against the util-linux version checked out in +# .github/workflows/run-gnu-test.yml. +# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline +cal/column-5 +cal/column-6 +cal/column-auto +cal/jan1753-3w +cal/jan1753-m3w +cal/weeknum-3mjw +cal/weeknum-3mw +cal/weeknum-3sjw +cal/weeknum-3sw +cal/weeknum-ymjw +cal/weeknum-ymw +cal/weeknum-ysjw +cal/weeknum-ysw +cal/year-ym +cal/year-ymj +cal/year-ymjw +cal/year-ymw +cal/year-ys +cal/year-ysj +cal/year-ysjw +cal/year-ysw +lscpu/lscpu-arm-A510-A710-A715-X3 +lscpu/lscpu-armv7 +lscpu/lscpu-loongarch-kvm_on_loongson_3c6000 +lscpu/lscpu-loongarch-loongson_3a5000_hv +lscpu/lscpu-ppc-qemu +lscpu/lscpu-ppc64-POWER7 +lscpu/lscpu-ppc64-POWER7-64cpu +lscpu/lscpu-rv64-linux +lscpu/lscpu-rv64-milkvpioneer +lscpu/lscpu-rv64-visionfive2 +lscpu/lscpu-s390-kvm +lscpu/lscpu-s390-lpar +lscpu/lscpu-s390-lpar-drawer +lscpu/lscpu-s390-nested-virt +lscpu/lscpu-s390-zvm +lscpu/lscpu-sparc64 +lscpu/lscpu-vbox-win +lscpu/lscpu-vmware_fpe +lscpu/lscpu-x86_64-64cpu +lscpu/lscpu-x86_64-64cpu-linux6.2 +lscpu/lscpu-x86_64-dell_e4310 +lscpu/lscpu-x86_64-epyc_7451 +lsmem/lsmem-s390-zvm-6g +lsmem/lsmem-x86_64-16g From 9c993ef5a707bd7af2849a17e64a18b3cdcf610a Mon Sep 17 00:00:00 2001 From: Jesse Rosenstock Date: Wed, 21 Oct 2026 16:00:15 +0200 Subject: [PATCH 5/5] run-gnu-test.yml: add concurrency and read-only permissions Each push to a pull request starts a new GNU test run while the previous one keeps building util-linux and our binary, although only the latest result matters. Add the concurrency settings used by uutils/coreutils GnuTests.yml: group runs by workflow and ref, cancel an in-progress run when its pull request or branch is updated, but not on main. Per the GitHub Actions docs, a newer queued run on main still replaces an older pending one in the same group. Also set the default GITHUB_TOKEN permissions to `contents: read`, as coreutils does. Since the previous commit removed the artifact steps, the job only checks out code (this repository and util-linux), restores the Rust cache (Swatinem/rust-cache, which coreutils also uses under `contents: read`), installs packages and runs scripts, so it needs no other permissions. --- .github/workflows/run-gnu-test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/run-gnu-test.yml b/.github/workflows/run-gnu-test.yml index e457a750..13f920b9 100644 --- a/.github/workflows/run-gnu-test.yml +++ b/.github/workflows/run-gnu-test.yml @@ -7,6 +7,17 @@ on: branches: - 'main' +# The job only needs to check out the repository. +permissions: + contents: read + +# Cancel superseded runs of a pull request or branch when it is updated. +# On main, let a run in progress finish (GitHub still replaces a pending +# run in the group with a newer one). +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + jobs: native: name: Run GNU tests (native)