Improve interactive debugging

EESSI-install-software.sh

@@ -84,6 +84,41 @@ TOPDIR=$(dirname $(realpath $0))
 
 source $TOPDIR/scripts/utils.sh
 
+# Debugging aid, only active if EESSI_INTERACTIVE_BUILD is set (see bot/build.sh): start an interactive shell
+# in the current environment. We are running inside 'startprefix' with stdin consumed by a here-string, so we
+# talk to /dev/tty directly. The new shell inherits all exported variables (Prefix, Lmod state, ...); we
+# additionally dump the non-exported variables, functions, aliases and shell options of this script into an
+# rcfile, so that e.g. ${EB} and our helper functions are available too.
+debug_shell() {
+    local reason="$1"
+    local rcfile
+    rcfile=$(mktemp "${TMPDIR:-/tmp}/debug_shell_rc.XXXXXX")
+    {
+        # 'grep -v' filters out read-only variables (e.g. BASHOPTS, EUID), as trying to load those in a debug shell
+        # would cause errors. If grep -v returns a non-zero output (i.e. the only variables are read-only variables)
+        # the '|| true' ensures this still passes if 'set -e' is ever set
+        declare -p 2>/dev/null | grep -v '^declare -[a-zA-Z-]*r[a-zA-Z-]* ' || true
+        declare -f
+        alias -p
+        shopt -p
+        # the debugging shell should not abort on errors, nor re-trigger our hooks
+        echo "set +e +u; trap - EXIT"
+        echo "PS1='[EESSI debug] \\w \\$ '"
+        echo "cd '${PWD}'"
+    } > "${rcfile}" 2>/dev/null
+    echo_yellow ">> Dropping into an interactive debug shell: ${reason}"
+    echo_yellow ">> Working directory: ${PWD}; exit the shell (Ctrl-D) to continue/finish"
+    bash --rcfile "${rcfile}" -i < /dev/tty > /dev/tty 2>&1
+    rm -f "${rcfile}"
+}
+
+# If EESSI_INTERACTIVE_BUILD=onfail, then define a trap so that if an error is encounted that would cause the shell to exit
+# (e.g. fatal_error) we get dropped into the debug_shell as well. That will allow us to not only debug EasyBuild build 
+# failures interactively, but anything resulting in fatal_error as well.
+if [[ "${EESSI_INTERACTIVE_BUILD}" == "onfail" ]]; then
+    trap 'ec_exit=$?; if [[ ${ec_exit} -ne 0 ]]; then debug_shell "script exiting with exit code ${ec_exit}"; fi' EXIT
+fi
+
 # honor $TMPDIR if it is already defined, use /tmp otherwise
 if [ -z $TMPDIR ]; then
     export WORKDIR=/tmp/$USER
@@ -425,18 +460,34 @@ else
 
             echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
+            # full EasyBuild command for this easystack file (also shown in the debug shell messages)
+            if [[ ${easystack_file} == *"/rebuilds/"* ]]; then
+                eb_cmd="${EB} --easystack ${easystack_file} --rebuild"
+            else
+                eb_cmd="${EB} --easystack ${easystack_file} --robot"
+            fi
+
+            # If EESSI_INTERACTIVE_BUILD=before, you'll get a debug shell right before easybuild starts
+            # You can execute ${eb_cmd} manually, or just exit the shell again
+            # to let the script continue (e.g. because the item you want to debug is in the 2nd iteration of this for-loop over
+            # easystack files)
+            if [[ "${EESSI_INTERACTIVE_BUILD}" == "before" ]]; then
+                debug_shell "environment is ready to run: ${eb_cmd}"
+            fi
+
             if [ -f ${easystack_file} ]; then
                 echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
 
-                if [[ ${easystack_file} == *"/rebuilds/"* ]]; then
-                    ${EB} --easystack ${easystack_file} --rebuild
-                else
-                    ${EB} --easystack ${easystack_file} --robot
-                fi
+                ${eb_cmd}
                 ec=$?
 
                 # copy EasyBuild log file if EasyBuild exited with an error
                 if [ ${ec} -ne 0 ]; then
+                    # If easybuild had a non-zero exit code, and EESSI_INTERACTIVE_BUILD=onfail, you'll get a debug
+                    # shell so you can debug the failure
+                    if [[ "${EESSI_INTERACTIVE_BUILD}" == "onfail" ]]; then
+                        debug_shell "EasyBuild failed with exit code ${ec} for command: ${eb_cmd}"
+                    fi
                     eb_last_log=$(eb --last-log | grep ^/.*\.log)
                     # copy to current working directory if file exhists
                     if [ -f ${eb_last_log} ]; then

bot/build.sh

@@ -18,6 +18,28 @@
 #    file 'job.cfg' has been deposited
 #  - the directory may contain any additional files referenced in job.cfg
 
+# Debugging interactive builds
+# This script respects the environment variable EESSI_INTERACTIVE_BUILD=<mode> which may be set if one wants to debug interactively
+# To debug interactively, run:
+# EESSI_INTERACTIVE_BUILD=<mode> <path-to-bot-build-jobscript>
+# where the path-to-bot-build-jobscript is the job script referred to as build_job_script by the app.cfg of the build bot.
+# That is typically eessi-bot-software-layer/scripts/bot-build.slurm, which then calls bot/build.sh (i.e. this script) with
+# EESSI_INTERACTIVE_BUILD=<mode> set. This way, environment setup that is part of the bot-build.slurm script (including e.g.
+# sourcing of the site_config_script as pointed to in the bot's app.cfg) is included. Then, this script sets up more of the
+# build environment, before eventually running the build, dropping you in an interactive debugging shell. Supported modes:
+#   shell  - start a plain shell in the build container instead of running the build
+#   before - run the build, but start a shell (inside the compat layer, EasyBuild loaded) right before
+#            EasyBuild is called
+#   onfail - run the build, and start a shell when EasyBuild fails (or the build script exits non-zero)
+# The state of the bot's build step (a tarball in previous_tmp/build_step) is resumed, and one accelerator target is
+# handled. Select them with EESSI_INTERACTIVE_BUILD_RESUME_FROM (a tmp directory or tarball, or 'none' to not resume from
+# anything and start from scratch) and EESSI_INTERACTIVE_BUILD_ACCELERATOR (e.g. accel/nvidia/cc80). If one of them is set, it
+# is used as given. If it is not set, you are asked to choose from the available accelerator targets / tarballs (the n-th
+# tarball is normally the one written after the n-th accelerator build), which needs stdin to be a terminal. Nothing is
+# assumed: without a terminal the script stops with an error asking you to set the variable(s). The only choices that
+# are not asked are those with just one option: a single accelerator target, or no tarballs to resume from.
+# Nothing is saved, no tarball is created.
+
 # stop as soon as something fails
 set -e
 
@@ -260,9 +282,20 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
 
 # prepare arguments to eessi_container.sh specific to build step
 declare -a BUILD_STEP_ARGS=()
-BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
+# We don't want interactive debugging sessions to save tarballs, so only save if EESSI_INTERACTIVE_BUILD isn't set
+if [[ -z "${EESSI_INTERACTIVE_BUILD}" ]]; then
+    BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
+fi
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 
+# interactive debugging: check the requested mode. The state of the bot's build step to resume from (tarball) and the
+# accelerator target are selected further below, once the list of accelerator targets is known
+if [[ -n "${EESSI_INTERACTIVE_BUILD}" ]]; then
+    if [[ ! "${EESSI_INTERACTIVE_BUILD}" =~ ^(shell|before|onfail)$ ]]; then
+        fatal_error "EESSI_INTERACTIVE_BUILD must be one of: shell, before, onfail (got '${EESSI_INTERACTIVE_BUILD}')"
+    fi
+fi
+
 # Retain location for host injections so we don't reinstall CUDA
 # (Always need to run the driver installation as available driver may change)
 if [[ ! -z ${SHARED_FS_PATH} ]]; then
@@ -296,6 +329,20 @@ else
 fi
 RESUME_DIR=""
 
+# interactive debugging only handles a single accelerator target, and resumes from a single tarball of the bot's build step
+# (or from none). Select them as described at the top of this script, see scripts/interactive_build.sh
+if [[ -n "${EESSI_INTERACTIVE_BUILD}" ]]; then
+    source $software_layer_dir/scripts/interactive_build.sh
+    select_interactive_state
+    if [[ "${selected_tarball}" == "none" ]]; then
+        echo "bot/build.sh: EESSI_INTERACTIVE_BUILD='${EESSI_INTERACTIVE_BUILD}', accelerator '${selected_accel_target}', not resuming from a tarball"
+    else
+        echo "bot/build.sh: EESSI_INTERACTIVE_BUILD='${EESSI_INTERACTIVE_BUILD}', accelerator '${selected_accel_target}', resuming from '${selected_tarball}'"
+        BUILD_STEP_ARGS+=("--resume" "${selected_tarball}")
+    fi
+    EESSI_ACCELERATOR_TARGET_OVERRIDES=("${selected_accel_target}")
+fi
+
 for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do
     # copy the common build step arguments to a a
     BUILD_STEP_ARGS_ACCEL=("${BUILD_STEP_ARGS[@]}")
@@ -322,6 +369,19 @@ for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do
     echo "Executing command to build software:"
     echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS_ACCEL[@]}"
     echo "                     -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}"
+    if [[ -n "${EESSI_INTERACTIVE_BUILD}" ]]; then
+        # Interactive debugging: same setup as above. No 'tee' as it would break the interactive terminal.
+        if [[ "${EESSI_INTERACTIVE_BUILD}" == "shell" ]]; then
+            echo "bot/build.sh: starting a shell in the build container instead of the build; to run the build step manually use:"
+            echo "                     $software_layer_dir/install_software_layer.sh ${INSTALL_SCRIPT_ARGS[@]} $@"
+            # a later --mode overrides the --mode exec in COMMON_ARGS
+            $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS_ACCEL[@]}" --mode shell
+        else
+            $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS_ACCEL[@]}" \
+                                 -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@"
+        fi
+        exit 0
+    fi
     $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS_ACCEL[@]}" \
                          -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr}
 

run_in_compat_layer_env.sh

@@ -59,6 +59,9 @@ fi
 if [ ! -z ${EESSI_INIT_PREFIX} ]; then
     INPUT="export EESSI_INIT_PREFIX=${EESSI_INIT_PREFIX}; ${INPUT}"
 fi
+if [ ! -z ${EESSI_INTERACTIVE_BUILD} ]; then
+    INPUT="export EESSI_INTERACTIVE_BUILD=${EESSI_INTERACTIVE_BUILD}; ${INPUT}"
+fi
 if [ ! -z ${EESSI_SITE_INSTALL_FORCE} ]; then
     INPUT="export EESSI_SITE_INSTALL_FORCE=${EESSI_SITE_INSTALL_FORCE}; ${INPUT}"
     if [ ! -z ${EESSI_SITE_SOFTWARE_PREFIX} ]; then

scripts/interactive_build.sh

@@ -0,0 +1,102 @@
+# Functions for debugging a bot build interactively, sourced by bot/build.sh if EESSI_INTERACTIVE_BUILD is set.
+# See the comments at the top of bot/build.sh for the user-facing description.
+#
+# Requires fatal_error (scripts/utils.sh).
+
+# Ask the user to choose a number between $2 (lowest number) and $3 (highest number) for $1 (description of what is
+# chosen). The result is returned in CHOSEN_NUMBER. There is no default: an empty answer is invalid. Returns 1 if no
+# answer can be read (e.g. at EOF because user pressed Ctrl+D / terminal disappears, which also avoids an infinite loop).
+# Note that all output of the dialog goes to stderr, like the prompt of 'read -p' does (bash's own convention for prompts)
+interactive_choice() {
+    local description="$1" lowest_number="$2" highest_number="$3" answer
+    # loop until we get a valid answer
+    while true; do
+        if ! read -r -p "bot/build.sh: select ${description} [${lowest_number}-${highest_number}]: " answer; then
+            echo >&2
+            return 1
+        fi
+        # Check answer is valid and return
+        if [[ "${answer}" =~ ^[0-9]+$ ]] && (( answer >= lowest_number && answer <= highest_number )); then
+            CHOSEN_NUMBER=${answer}
+            return
+        fi
+        echo "bot/build.sh: invalid choice '${answer}'" >&2
+    done
+}
+
+# Choose one entry of the array named $4, and return its number in CHOSEN_NUMBER: N for entry N of the array (counting from
+# 1), and 0 for 'none' (only allowed if $3, the lowest number, is 0).
+#   $1 description of what is chosen, $2 name of the environment variable with which the user can set it
+# If there is only one option, we don't need to choose. Otherwise we ask the user, if stdin is a terminal, or else stop
+# with an error: we don't assume anything.
+choose_option() {
+    local description="$1" env_var_name="$2" lowest_number="$3"
+    local -n option_array="$4"
+    local highest_number=${#option_array[@]}
+    if [[ ${highest_number} -eq ${lowest_number} ]]; then
+        CHOSEN_NUMBER=${highest_number}
+    elif [[ -t 0 ]]; then
+        interactive_choice "${description}" ${lowest_number} ${highest_number} ||
+            fatal_error "no answer given for ${description}, set ${env_var_name} to choose it without being asked"
+    else
+        fatal_error "stdin is not a terminal, so cannot ask which ${description}, set ${env_var_name} to choose it"
+    fi
+}
+
+# Select the accelerator target and the tarball of the bot's build step to resume from, based on
+# EESSI_INTERACTIVE_BUILD_ACCELERATOR and EESSI_INTERACTIVE_BUILD_RESUME_FROM, asking for what isn't set (see bot/build.sh).
+# Needs EESSI_ACCELERATOR_TARGET_OVERRIDES (the accelerator targets of the build) and TARBALL_TMP_BUILD_STEP_DIR.
+# Returns the accelerator target in selected_accel_target, and the tarball (or directory) in selected_tarball, which is
+# the word 'none' if we shouldn't resume from anything.
+select_interactive_state() {
+    local -a accel_targets=("${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}")
+    local -a tarball_paths
+    mapfile -t tarball_paths < <(ls -tr "${TARBALL_TMP_BUILD_STEP_DIR}"/tmp_storage-* 2>/dev/null)
+    local i
+
+    # select the accelerator target
+    if [[ -n "${EESSI_INTERACTIVE_BUILD_ACCELERATOR}" ]]; then
+        # Format for EESSI_INTERACTIVE_BUILD_ACCELERATOR is similar to EESSI_ACCELERATOR_TARGET_OVERRIDE,
+        # e.g. accel/nvidia/cc80
+        selected_accel_target="${EESSI_INTERACTIVE_BUILD_ACCELERATOR}"
+        local found=0 accel_target
+        for accel_target in "${accel_targets[@]}"; do
+            [[ "${accel_target}" == "${selected_accel_target}" ]] && found=1
+        done
+        if [[ ${found} -eq 0 ]]; then
+            fatal_error "EESSI_INTERACTIVE_BUILD_ACCELERATOR='${EESSI_INTERACTIVE_BUILD_ACCELERATOR}' not found in accelerator targets of ${JOB_CFG_FILE}: ${accel_targets[*]}"
+        fi
+    else
+        # List accelerators from the job.cfg file
+        # (skip listing if there is only one - choose_option will select it by default)
+        if [[ ${#accel_targets[@]} -gt 1 ]]; then
+            echo "bot/build.sh: accelerator targets of this build (in the order the bot built them):"
+            for i in "${!accel_targets[@]}"; do
+                echo "  [$((i+1))] '${accel_targets[$i]}'"
+            done
+        fi
+        choose_option "accelerator target" EESSI_INTERACTIVE_BUILD_ACCELERATOR 1 accel_targets
+        selected_accel_target="${accel_targets[$((CHOSEN_NUMBER-1))]}"
+    fi
+
+    # select the tarball to resume from, or none
+    if [[ -n "${EESSI_INTERACTIVE_BUILD_RESUME_FROM}" ]]; then
+        selected_tarball="${EESSI_INTERACTIVE_BUILD_RESUME_FROM}"
+    else
+        echo "bot/build.sh: tarballs of the bot's build step in ${TARBALL_TMP_BUILD_STEP_DIR} (oldest first):"
+        echo "  [0] none (do not resume from a tarball, start from scratch)"
+        for i in "${!tarball_paths[@]}"; do
+            echo "  [$((i+1))] $(basename "${tarball_paths[$i]}")   ($(date -r "${tarball_paths[$i]}" '+%Y-%m-%d %H:%M'), $(du -h "${tarball_paths[$i]}" | cut -f1))"
+        done
+        echo "  note: normally, tarball [n] is written after the n-th accelerator build, since each build should produce one tarball"
+        [[ ${#tarball_paths[@]} -gt 0 ]] || echo "  no tarballs found, so not resuming from a tarball"
+        choose_option "tarball to resume from (0 = none)" EESSI_INTERACTIVE_BUILD_RESUME_FROM 0 tarball_paths
+        selected_tarball="none"
+        [[ ${CHOSEN_NUMBER} -gt 0 ]] && selected_tarball="${tarball_paths[$((CHOSEN_NUMBER-1))]}"
+    fi
+    # accept 'none' in any capitalisation (and normalize to lower case for easier subsequent checks)
+    [[ "${selected_tarball,,}" == "none" ]] && selected_tarball="none"
+    if [[ "${selected_tarball}" != "none" && ! -e "${selected_tarball}" ]]; then
+        fatal_error "state to resume from does not exist: ${selected_tarball}"
+    fi
+}