Merge upstream mg5amcnlo 3.8.0 into main

.github/actions/check_heptools/action.yml

@@ -0,0 +1,42 @@
+name: Check HEPTools cache content
+description: >
+  Verify that the requested tools are really present in the HEPTools install
+  prefix.  mg5_aMC exits 0 even when "install <tool>" fails (truncated
+  download, broken build, ...), so without this check a cache-building job
+  finishes green while saving a cache with a tool missing.
+
+inputs:
+  tools:
+    description: >
+      Space separated list of components to check, among:
+      lhapdf boost pythia8 emela contur looptools
+    required: true
+  prefix:
+    description: 'HEPTools install prefix to inspect (default ~/.cache/HEPtools)'
+    required: false
+    default: ''
+  mode:
+    description: >
+      "fatal" (default) makes the step fail on the first incomplete component,
+      "report" only emits warnings and sets the "valid" output.
+    required: false
+    default: 'fatal'
+
+outputs:
+  valid:
+    description: 'true when every requested component is complete'
+    value: ${{ steps.check.outputs.valid }}
+
+runs:
+  using: composite
+  steps:
+    - name: Check HEPTools content
+      id: check
+      shell: bash
+      env:
+        HEPTOOLS_PREFIX: ${{ inputs.prefix }}
+        CHECK_MODE: ${{ inputs.mode }}
+        CHECK_TOOLS: ${{ inputs.tools }}
+      # invoked through "bash" so the check does not depend on the executable
+      # bit surviving the checkout
+      run: bash "${{ github.action_path }}/check_heptools.sh" $CHECK_TOOLS

.github/actions/check_heptools/check_heptools.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+#
+# Content check for the HEPTools cache tree.
+#
+# "./bin/madgraph cmd" exits 0 even when the "install <tool>" it contains
+# failed (e.g. the boost tarball came down truncated, so bootstrap.sh/b2
+# never ran and eMELA aborted).  A warm_cache job therefore ends green and
+# actions/cache happily saves a cache with the tool missing, which only shows
+# up much later as a broken acceptance test.
+#
+# Every path asserted below mirrors what HEPToolInstaller.py installs
+# ('install_path' = <prefix>/<tool>, libraries looked up in lib/ or lib64/)
+# and what the .github/actions/restore_heptools_* actions then hand to MG5
+# through input/mg7_configuration.txt.
+#
+# Usage: check_heptools.sh <component> [<component> ...]
+#   HEPTOOLS_PREFIX  install prefix to inspect  (default ~/.cache/HEPtools)
+#   CHECK_MODE       fatal (default) | report
+
+set -o pipefail
+
+HEP="${HEPTOOLS_PREFIX:-}"
+[ -n "$HEP" ] || HEP="$HOME/.cache/HEPtools"
+MODE="${CHECK_MODE:-fatal}"
+
+missing=()
+
+need_file() { [ -f "$1" ] || missing+=("missing file: $1"); }
+need_exec() { [ -x "$1" ] || missing+=("missing executable: $1"); }
+need_dir()  { [ -d "$1" ] || missing+=("missing directory: $1"); }
+
+# need_glob <description> <pattern> [<pattern> ...]
+# Satisfied as soon as one pattern matches something (used for the lib/lib64
+# ambiguity and for library extensions that differ between builds).
+need_glob() {
+    local desc="$1"; shift
+    local pattern
+    for pattern in "$@"; do
+        if compgen -G "$pattern" > /dev/null 2>&1; then
+            return 0
+        fi
+    done
+    missing+=("missing $desc (no match for: $*)")
+}
+
+check_lhapdf() {
+    need_exec "$HEP/lhapdf6_py3/bin/lhapdf-config"
+    need_exec "$HEP/bin/lhapdf-config"
+    need_glob "libLHAPDF" "$HEP/lhapdf6_py3/lib/libLHAPDF.*" "$HEP/lhapdf6_py3/lib64/libLHAPDF.*"
+    # the two PDF sets the NLO acceptance tests need are unpacked by the
+    # warm_cache lhapdf job itself, not by the installer
+    need_dir "$HEP/lhapdf6_py3/share/LHAPDF/cteq6l1"
+    need_dir "$HEP/lhapdf6_py3/share/LHAPDF/NNPDF23_nlo_as_0118_qed"
+}
+
+check_boost() {
+    # exactly what HEPToolInstaller.find_dependency('boost') looks for: without
+    # it eMELA re-downloads and rebuilds boost from scratch
+    need_glob "libboost_system" \
+        "$HEP/boost/lib/libboost_system.*" "$HEP/boost/lib/libboost_system-mt.*" \
+        "$HEP/boost/lib64/libboost_system.*" "$HEP/boost/lib64/libboost_system-mt.*"
+    need_file "$HEP/boost/include/boost/version.hpp"
+}
+
+check_pythia8() {
+    # a network-failed pythia8 install leaves an incomplete pythia8/ dir; MG5
+    # then nullifies pythia8_path at startup, leaving the parton shower
+    # unavailable and breaking the rivet/contur acceptance tests
+    need_file "$HEP/pythia8/include/Pythia8/Pythia.h"
+    need_exec "$HEP/pythia8/bin/pythia8-config"
+    need_glob "libpythia8" "$HEP/pythia8/lib/libpythia8.*" "$HEP/pythia8/lib64/libpythia8.*"
+    need_dir  "$HEP/hepmc"
+    need_file "$HEP/MG5aMC_PY8_interface/MG5aMC_PY8_interface"
+}
+
+check_emela() {
+    # bin/eMELA-config is the symlink finalize_installation() creates and the
+    # path restore_heptools_emela writes into mg7_configuration.txt
+    need_exec "$HEP/bin/eMELA-config"
+    need_glob "eMELA grids" "$HEP/EMELA/share/eMELA/*"
+}
+
+check_contur() {
+    # a missing fastjet-config means NLO subprocesses cannot compile
+    # (fastjet/ClusterSequence.hh)
+    need_exec "$HEP/fastjet/bin/fastjet-config"
+    need_glob "libfastjet" "$HEP/fastjet/lib/libfastjet.*" "$HEP/fastjet/lib64/libfastjet.*"
+    need_glob "libRivet"   "$HEP/rivet/lib/libRivet.*"     "$HEP/rivet/lib64/libRivet.*"
+    need_glob "libYODA"    "$HEP/yoda/lib/libYODA.*"       "$HEP/yoda/lib64/libYODA.*"
+    need_dir  "$HEP/contur"
+    need_dir  "$HEP/hepmc3"
+}
+
+check_looptools() {
+    need_file "$HEP/CutTools/lib/libcts.a"
+    need_file "$HEP/CutTools/lib/mpmodule.mod"
+    need_file "$HEP/IREGI/src/libiregi.a"
+    need_glob "libcollier" "$HEP/collier/libcollier.*" \
+        "$HEP/collier/lib/libcollier.*" "$HEP/collier/lib64/libcollier.*"
+    need_glob "libninja"   "$HEP/ninja/lib/libninja.*" "$HEP/ninja/lib64/libninja.*"
+}
+
+if [ "$#" -eq 0 ]; then
+    echo "::error::check_heptools: no component requested"
+    exit 1
+fi
+
+echo "Checking HEPTools content under $HEP for: $*"
+for component in "$@"; do
+    case "$component" in
+        lhapdf|boost|pythia8|emela|contur|looptools) "check_$component" ;;
+        *) echo "::error::check_heptools: unknown component '$component'"; exit 1 ;;
+    esac
+done
+
+if [ "${#missing[@]}" -eq 0 ]; then
+    echo "All requested components are complete."
+    [ -n "${GITHUB_OUTPUT:-}" ] && echo "valid=true" >> "$GITHUB_OUTPUT"
+    exit 0
+fi
+
+level=error
+[ "$MODE" = "fatal" ] || level=warning
+for entry in "${missing[@]}"; do
+    echo "::${level}::HEPTools cache incomplete ($*): ${entry}"
+done
+[ -n "${GITHUB_OUTPUT:-}" ] && echo "valid=false" >> "$GITHUB_OUTPUT"
+
+if [ "$MODE" = "fatal" ]; then
+    echo "Refusing to treat this build as successful: the cache would be saved incomplete."
+    exit 1
+fi
+exit 0

.github/actions/restore-pip-cache/action.yml

@@ -8,10 +8,14 @@ runs:
       run: echo "CACHE_KEY=pip-numpy-$ImageOS" >> $GITHUB_ENV
       shell: bash
 
+    # Generation-suffixed keys: see the comment in restore_heptools.
     - uses: actions/cache/restore@v5
       with:
         path: ~/.cache/pip-packages
-        key: ${{ env.CACHE_KEY }}
+        key: ${{ env.CACHE_KEY }}-${{ github.run_id }}
+        restore-keys: |
+          ${{ env.CACHE_KEY }}-
+          ${{ env.CACHE_KEY }}
 
     - run: |
         sudo apt install meson ninja-build

.github/actions/restore_delphes/action.yml

@@ -24,11 +24,15 @@ runs:
       run: echo "CACHE_KEY=delphes-$ImageOS" >> $GITHUB_ENV
       shell: bash
 
+    # Generation-suffixed keys: see the comment in restore_heptools.
     - uses: actions/cache/restore@v5
       if: ${{ inputs.setup_only != 'true' }}
       with:
         path: ~/.cache/Delphes
-        key: ${{ env.CACHE_KEY }}
+        key: ${{ env.CACHE_KEY }}-${{ github.run_id }}
+        restore-keys: |
+          ${{ env.CACHE_KEY }}-
+          ${{ env.CACHE_KEY }}
 
     - run: |
         cd $GITHUB_WORKSPACE

.github/actions/restore_heptools/action.yml

@@ -4,71 +4,64 @@ description: Restores heptools from cache and sets MG5 accordingly
 runs:
   using: composite
   steps:
-    - name: get os 
+    - name: get os
       run: echo "CACHE_KEY=heptools-$ImageOS" >> $GITHUB_ENV
       shell: bash
 
+    # warm_cache.yml saves the combined cache under a generation-suffixed key,
+    # "heptools-<ImageOS>-<run_id>", so that a refresh adds a new entry instead
+    # of having to delete the current one first (GitHub caches are immutable,
+    # and the delete-then-save it used to need left a ~30 min window in which
+    # every consumer missed). Nothing outside that run can know the generation,
+    # so the exact key below is deliberately unmatchable here and restore-keys
+    # does the work: GitHub serves the newest entry with the given prefix, i.e.
+    # the previous generation while a new one is being built.
+    # The bare prefix on the last line is transitional, for the unversioned
+    # entries that predate this change; it can be dropped once warm_cache has
+    # run one full cycle.
     - name: restore full cache
       id: heptools-cache
       uses: actions/cache/restore@v5
       with:
         path: ~/.cache/HEPtools
-        key: ${{ env.CACHE_KEY }}
-        # Crash loudly (naming the missing key) instead of silently continuing
-        # with an empty HEPtools dir: a missing combined cache otherwise leaves
-        # the tool paths configured but the binaries absent, so downstream tests
-        # self-skip and (because test_manager.py treats a skip as a failure)
-        # fail for an opaque reason. This makes it obvious that the
-        # heptools-<os> cache itself needs to be rebuilt (run 'warm cache').
-        # Safe: warm_cache.yml builds the caches without this action, and jobs
-        # that currently pass already have the cache present.
-        fail-on-cache-miss: true
+        key: ${{ env.CACHE_KEY }}-${{ github.run_id }}
+        restore-keys: |
+          ${{ env.CACHE_KEY }}-
+          ${{ env.CACHE_KEY }}
 
+    # NB: cache-hit is true only on an *exact* key match, which now essentially
+    # never happens here; cache-matched-key is non-empty whenever something was
+    # actually restored (exact match or restore-keys). That is the condition
+    # that means "we have a combined cache to look at".
+
+    # Same content check the warm_cache jobs run before saving, in "report"
+    # mode: here an incomplete combined cache is not fatal, we just fall back
+    # to the (still present) sub-caches below.
     - name: validate cache content
       id: validate-cache
-      if: steps.heptools-cache.outputs.cache-hit == 'true'
-      run: |
-        # Check that all expected HEPtool binaries/directories are present
-        VALID=true
-        test -d $HOME/.cache/HEPtools/lhapdf6_py3                    || VALID=false
-        # check the header/binary, not just the dir: a network-failed pythia8
-        # install can leave an incomplete pythia8/ dir that MG5 then rejects
-        # (pythia8_path nullified -> no shower -> rivet/contur tests fail).
-        test -f $HOME/.cache/HEPtools/pythia8/include/Pythia8/Pythia.h || VALID=false
-        test -x $HOME/.cache/HEPtools/pythia8/bin/pythia8-config      || VALID=false
-        test -x $HOME/.cache/HEPtools/bin/eMELA-config                || VALID=false
-        test -x $HOME/.cache/HEPtools/fastjet/bin/fastjet-config      || VALID=false
-        test -d $HOME/.cache/HEPtools/rivet                           || VALID=false
-        # looptools: check the libraries themselves, not just the ninja/ dir.
-        # A "install collier"/"install ninja" that failed still leaves the dirs
-        # behind, and a combined cache built from such a run has MadLoop paths
-        # configured but nothing to link against (MG5 looks for exactly these
-        # two files, see activate_dependence in madgraph/various/misc.py).
-        test -f $HOME/.cache/HEPtools/ninja/lib/libninja.a            || VALID=false
-        test -f $HOME/.cache/HEPtools/collier/libcollier.a            || VALID=false
-        test -f $HOME/.cache/HEPtools/CutTools/lib/libcts.a           || VALID=false
-        test -f $HOME/.cache/HEPtools/IREGI/src/libiregi.a            || VALID=false
-        echo "cache_valid=$VALID" >> $GITHUB_OUTPUT
-        if [ "$VALID" = "false" ]; then
-          echo "::warning::heptools cache is incomplete, will restore from sub-caches"
-        fi
-      shell: bash
+      if: steps.heptools-cache.outputs.cache-matched-key != ''
+      uses: ./.github/actions/check_heptools
+      with:
+        tools: lhapdf boost pythia8 emela contur looptools
+        mode: report
 
-    # setup other environment of previous level of cache 
+    # setup other environment of previous level of cache
     # try to use their cache if previous cache failed or content is incomplete
     - uses: ./.github/actions/restore_heptools_lhapdf
-      with: 
-        setup_only: ${{ steps.heptools-cache.outputs.cache-hit == 'true' && steps.validate-cache.outputs.cache_valid == 'true' }}
+      with:
+        setup_only: ${{ steps.heptools-cache.outputs.cache-matched-key != '' && steps.validate-cache.outputs.valid == 'true' }}
+    - uses: ./.github/actions/restore_heptools_boost
+      with:
+        setup_only: ${{ steps.heptools-cache.outputs.cache-matched-key != '' && steps.validate-cache.outputs.valid == 'true' }}
     - uses: ./.github/actions/restore_heptools_pythia8
-      with: 
-        setup_only: ${{ steps.heptools-cache.outputs.cache-hit == 'true' && steps.validate-cache.outputs.cache_valid == 'true' }}
+      with:
+        setup_only: ${{ steps.heptools-cache.outputs.cache-matched-key != '' && steps.validate-cache.outputs.valid == 'true' }}
     - uses: ./.github/actions/restore_heptools_emela
-      with: 
-        setup_only: ${{ steps.heptools-cache.outputs.cache-hit == 'true' && steps.validate-cache.outputs.cache_valid == 'true' }}
+      with:
+        setup_only: ${{ steps.heptools-cache.outputs.cache-matched-key != '' && steps.validate-cache.outputs.valid == 'true' }}
     - uses: ./.github/actions/restore_heptools_contur
-      with: 
-        setup_only: ${{ steps.heptools-cache.outputs.cache-hit == 'true' && steps.validate-cache.outputs.cache_valid == 'true' }}
+      with:
+        setup_only: ${{ steps.heptools-cache.outputs.cache-matched-key != '' && steps.validate-cache.outputs.valid == 'true' }}
     - uses: ./.github/actions/restore_heptools_looptools
-      with: 
-        setup_only: ${{ steps.heptools-cache.outputs.cache-hit == 'true' && steps.validate-cache.outputs.cache_valid == 'true' }}
-
+      with:
+        setup_only: ${{ steps.heptools-cache.outputs.cache-matched-key != '' && steps.validate-cache.outputs.valid == 'true' }}

.github/actions/restore_heptools_boost/action.yml

@@ -0,0 +1,37 @@
+name: Restore boost cache (if it exists and in any case configure even if it does not)
+description: Restores boost (stacked on lhapdf) from cache and sets MG5 accordingly
+
+inputs:
+  setup_only:
+    required: false
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+
+    # Call previous cache to ensure that environment is setup correctly
+    - name: restore lhapdf
+      uses: ./.github/actions/restore_heptools_lhapdf
+      with:
+        setup_only: ${{ inputs.setup_only }}
+
+    - name: Set cache key
+      run: echo "CACHE_KEY=boost-$ImageOS" >> $GITHUB_ENV
+      shell: bash
+
+    # check if the cache exists (if not an higher level will be called)
+    # Generation-suffixed keys: see the comment in restore_heptools_lhapdf.
+    - uses: actions/cache/restore@v5
+      if: ${{ inputs.setup_only != 'true' }}
+      with:
+        path: ~/.cache/HEPtools
+        key: ${{ env.CACHE_KEY }}-${{ github.run_id }}
+        restore-keys: |
+          ${{ env.CACHE_KEY }}-
+          ${{ env.CACHE_KEY }}
+
+    # Nothing to write into mg7_configuration.txt: boost has no MG5 option.
+    # HEPToolInstaller finds it on its own as soon as
+    # <heptools_install_dir>/boost/lib/libboost_system.so is present, and then
+    # skips the (~6 min, download-flaky) boost build when installing eMELA.