diff --git a/.github/workflows/run-gnu-test.yml b/.github/workflows/run-gnu-test.yml index a58f48fe..13f920b9 100644 --- a/.github/workflows/run-gnu-test.yml +++ b/.github/workflows/run-gnu-test.yml @@ -7,6 +7,17 @@ on: branches: - 'main' +# The job only needs to check out the repository. +permissions: + contents: read + +# Cancel superseded runs of a pull request or branch when it is updated. +# On main, let a run in progress finish (GitHub still replaces a pending +# run in the group with a newer one). +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + jobs: native: name: Run GNU tests (native) @@ -39,30 +50,9 @@ jobs: ./configure make libsmartcols/src/filter-parser.c libsmartcols/src/filter-scanner.c make -j$(nproc) check-programs - - name: Download test baseline if exists - id: download-test-baseline - continue-on-error: true - uses: actions/download-artifact@v8 - with: - name: gnu-test-failures - path: uutils-util-linux/.reference/ - - name: Run GNU tests env: GNU_PROJECT_DIR: ${{ github.workspace }}/gnu-util-linux run: | cd uutils-util-linux ./scripts/run-tests.sh - - - name: Update test baseline - if: github.ref == 'refs/heads/main' - run: | - cd uutils-util-linux - ./scripts/check-new-gnu-failures.sh --update-baseline - - - name: Upload test baseline artifact - if: github.ref == 'refs/heads/main' || steps.download-test-baseline.outcome == 'failure' - uses: actions/upload-artifact@v7 - with: - name: gnu-test-failures - path: uutils-util-linux/.reference/gnu-test-failures.txt diff --git a/.gitignore b/.gitignore index 89305739..300fc31a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,3 @@ syntax: glob /target/ .test-helpers -.reference diff --git a/scripts/check-new-gnu-failures.sh b/scripts/check-new-gnu-failures.sh index b19b8cbb..9b6f93fd 100755 --- a/scripts/check-new-gnu-failures.sh +++ b/scripts/check-new-gnu-failures.sh @@ -4,9 +4,8 @@ set -euo pipefail cd "$(dirname "$0")/../" PROJECT_DIR="$(pwd)" DIFF_DIR="$PROJECT_DIR/.test-helpers/tests/diff" -BASELINE="$PROJECT_DIR/.reference/gnu-test-failures.txt" +BASELINE="$PROJECT_DIR/tests/gnu-test-failures.txt" UPDATE_BASELINE=false -EMPTY_BASELINE=false for arg in "$@"; do case "$arg" in @@ -15,21 +14,41 @@ for arg in "$@"; do esac done -# Collect current failures (exclude .err companion files) +# run-tests.sh deletes DIFF_DIR before running the GNU tests, and the GNU +# harness (ts_gen_diff in tests/functions.sh) creates DIFF_DIR/ +# whenever it compares a test's output, deleting only the diff files of +# tests that pass. So no component directory means that no test produced +# results: run.sh failed to start, died early, or skipped every test. +# Without this check, such a run would have no failures, so every known +# failure would be reported as fixed and the check would pass (or +# --update-baseline would record an empty baseline). +if [[ ! -d "$DIFF_DIR" ]] \ + || [[ -z "$(find "$DIFF_DIR" -mindepth 1 -maxdepth 1 -type d)" ]]; then + echo "ERROR: No GNU test results in $DIFF_DIR" >&2 + echo " The GNU test run failed or did not run any tests." >&2 + exit 2 +fi + +# Collect current failures. The GNU harness (ts_gen_diff in +# tests/functions.sh) fails a test if its stdout or its stderr differs from +# the expected output, writing or .err respectively, so a test +# whose stdout matches but whose stderr differs leaves only .err. +# Map both to and de-duplicate so such failures are counted too. current_tmp=$(mktemp) trap 'rm -f "$current_tmp"' EXIT -if [[ -d "$DIFF_DIR" ]]; then - find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f ! -name "*.err" -print0 \ - | sort -z \ - | while IFS= read -r -d '' path; do - echo "${path#"$DIFF_DIR/"}" - done > "$current_tmp" -fi +find "$DIFF_DIR" -mindepth 2 -maxdepth 2 -type f -print0 \ + | while IFS= read -r -d '' path; do + path="${path#"$DIFF_DIR/"}" + echo "${path%.err}" + done \ + | sort -u > "$current_tmp" # --update-baseline mode if [[ "$UPDATE_BASELINE" == true ]]; then { echo "# Known GNU test failures - format: util/testname" + echo "# Must match CI: run against the util-linux version checked out in" + echo "# .github/workflows/run-gnu-test.yml." echo "# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline" cat "$current_tmp" } > "$BASELINE" @@ -38,16 +57,39 @@ if [[ "$UPDATE_BASELINE" == true ]]; then exit 0 fi +# The baseline is checked in, so a missing file is an error rather than a +# reason to create one: creating it from the current results would accept +# every current failure, including regressions, and pass. +if [[ ! -f "$BASELINE" ]]; then + echo "ERROR: Baseline not found: $BASELINE" >&2 + echo " Restore it from version control, or create it with:" >&2 + echo " ./scripts/check-new-gnu-failures.sh --update-baseline" >&2 + exit 2 +fi + # Load baseline (strip comments and blank lines) baseline_tmp=$(mktemp) trap 'rm -f "$current_tmp" "$baseline_tmp"' EXIT -if [[ -f "$BASELINE" ]]; then - grep -v '^\s*#' "$BASELINE" | grep -v '^\s*$' | sort > "$baseline_tmp" -else - echo "WARNING: Baseline not found: $BASELINE" >&2 - echo " Run with --update-baseline to create it." >&2 - touch "$baseline_tmp" - EMPTY_BASELINE=true +# Use sed rather than grep -v: grep exits 1 when it selects no lines, +# which pipefail and set -e would turn into an abort for a baseline with +# no entries (e.g. after every known failure has been fixed). +sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$/d' "$BASELINE" \ + | sort > "$baseline_tmp" + +# As in the check on DIFF_DIR above: if a component with known failures +# has no DIFF_DIR/, none of its tests produced results (e.g. +# they were all skipped, or the utility is no longer tested), so its known +# failures must not be reported as fixed. +missing_components=$(cut -d/ -f1 "$baseline_tmp" | sort -u \ + | while IFS= read -r component; do + [[ -d "$DIFF_DIR/$component" ]] || echo "$component" + done) +if [[ -n "$missing_components" ]]; then + echo "ERROR: No GNU test results for components with known failures:" >&2 + awk '{print " " $0}' <<< "$missing_components" >&2 + echo " Check the GNU test output for skipped tests, or remove the" >&2 + echo " components' entries from the baseline if they are no longer tested." >&2 + exit 2 fi # comm: -13 = lines only in current (new failures), -23 = lines only in baseline (fixed) @@ -60,18 +102,6 @@ echo "Baseline failures: $(wc -l < "$baseline_tmp")" echo "Current failures: $(wc -l < "$current_tmp")" echo "" -if [[ "$EMPTY_BASELINE" == true ]]; then - mkdir -p "$(dirname "$BASELINE")" - { - echo "# Known GNU test failures - format: util/testname" - echo "# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline" - cat "$current_tmp" - } > "$BASELINE" - count=$(wc -l < "$current_tmp") - echo "Initial baseline created: $BASELINE ($count failures recorded)" - exit 0 -fi - if [[ -n "$fixed_tests" ]]; then echo "Tests newly FIXED:" awk '{print " [FIXED] " $0}' <<< "$fixed_tests" diff --git a/tests/gnu-test-failures.txt b/tests/gnu-test-failures.txt new file mode 100644 index 00000000..7edf7c18 --- /dev/null +++ b/tests/gnu-test-failures.txt @@ -0,0 +1,49 @@ +# Known GNU test failures - format: util/testname +# Must match CI: run against the util-linux version checked out in +# .github/workflows/run-gnu-test.yml. +# Update with: ./scripts/check-new-gnu-failures.sh --update-baseline +cal/column-5 +cal/column-6 +cal/column-auto +cal/jan1753-3w +cal/jan1753-m3w +cal/weeknum-3mjw +cal/weeknum-3mw +cal/weeknum-3sjw +cal/weeknum-3sw +cal/weeknum-ymjw +cal/weeknum-ymw +cal/weeknum-ysjw +cal/weeknum-ysw +cal/year-ym +cal/year-ymj +cal/year-ymjw +cal/year-ymw +cal/year-ys +cal/year-ysj +cal/year-ysjw +cal/year-ysw +lscpu/lscpu-arm-A510-A710-A715-X3 +lscpu/lscpu-armv7 +lscpu/lscpu-loongarch-kvm_on_loongson_3c6000 +lscpu/lscpu-loongarch-loongson_3a5000_hv +lscpu/lscpu-ppc-qemu +lscpu/lscpu-ppc64-POWER7 +lscpu/lscpu-ppc64-POWER7-64cpu +lscpu/lscpu-rv64-linux +lscpu/lscpu-rv64-milkvpioneer +lscpu/lscpu-rv64-visionfive2 +lscpu/lscpu-s390-kvm +lscpu/lscpu-s390-lpar +lscpu/lscpu-s390-lpar-drawer +lscpu/lscpu-s390-nested-virt +lscpu/lscpu-s390-zvm +lscpu/lscpu-sparc64 +lscpu/lscpu-vbox-win +lscpu/lscpu-vmware_fpe +lscpu/lscpu-x86_64-64cpu +lscpu/lscpu-x86_64-64cpu-linux6.2 +lscpu/lscpu-x86_64-dell_e4310 +lscpu/lscpu-x86_64-epyc_7451 +lsmem/lsmem-s390-zvm-6g +lsmem/lsmem-x86_64-16g