diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 6ed3985a..52b8968b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -84,6 +84,41 @@ TOPDIR=$(dirname $(realpath $0)) source $TOPDIR/scripts/utils.sh +# Debugging aid, only active if EESSI_INTERACTIVE_BUILD is set (see bot/build.sh): start an interactive shell +# in the current environment. We are running inside 'startprefix' with stdin consumed by a here-string, so we +# talk to /dev/tty directly. The new shell inherits all exported variables (Prefix, Lmod state, ...); we +# additionally dump the non-exported variables, functions, aliases and shell options of this script into an +# rcfile, so that e.g. ${EB} and our helper functions are available too. +debug_shell() { + local reason="$1" + local rcfile + rcfile=$(mktemp "${TMPDIR:-/tmp}/debug_shell_rc.XXXXXX") + { + # 'grep -v' filters out read-only variables (e.g. BASHOPTS, EUID), as trying to load those in a debug shell + # would cause errors. If grep -v returns a non-zero output (i.e. the only variables are read-only variables) + # the '|| true' ensures this still passes if 'set -e' is ever set + declare -p 2>/dev/null | grep -v '^declare -[a-zA-Z-]*r[a-zA-Z-]* ' || true + declare -f + alias -p + shopt -p + # the debugging shell should not abort on errors, nor re-trigger our hooks + echo "set +e +u; trap - EXIT" + echo "PS1='[EESSI debug] \\w \\$ '" + echo "cd '${PWD}'" + } > "${rcfile}" 2>/dev/null + echo_yellow ">> Dropping into an interactive debug shell: ${reason}" + echo_yellow ">> Working directory: ${PWD}; exit the shell (Ctrl-D) to continue/finish" + bash --rcfile "${rcfile}" -i < /dev/tty > /dev/tty 2>&1 + rm -f "${rcfile}" +} + +# If EESSI_INTERACTIVE_BUILD=onfail, then define a trap so that if an error is encounted that would cause the shell to exit +# (e.g. fatal_error) we get dropped into the debug_shell as well. That will allow us to not only debug EasyBuild build +# failures interactively, but anything resulting in fatal_error as well. +if [[ "${EESSI_INTERACTIVE_BUILD}" == "onfail" ]]; then + trap 'ec_exit=$?; if [[ ${ec_exit} -ne 0 ]]; then debug_shell "script exiting with exit code ${ec_exit}"; fi' EXIT +fi + # honor $TMPDIR if it is already defined, use /tmp otherwise if [ -z $TMPDIR ]; then export WORKDIR=/tmp/$USER @@ -425,18 +460,34 @@ else echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." + # full EasyBuild command for this easystack file (also shown in the debug shell messages) + if [[ ${easystack_file} == *"/rebuilds/"* ]]; then + eb_cmd="${EB} --easystack ${easystack_file} --rebuild" + else + eb_cmd="${EB} --easystack ${easystack_file} --robot" + fi + + # If EESSI_INTERACTIVE_BUILD=before, you'll get a debug shell right before easybuild starts + # You can execute ${eb_cmd} manually, or just exit the shell again + # to let the script continue (e.g. because the item you want to debug is in the 2nd iteration of this for-loop over + # easystack files) + if [[ "${EESSI_INTERACTIVE_BUILD}" == "before" ]]; then + debug_shell "environment is ready to run: ${eb_cmd}" + fi + if [ -f ${easystack_file} ]; then echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - if [[ ${easystack_file} == *"/rebuilds/"* ]]; then - ${EB} --easystack ${easystack_file} --rebuild - else - ${EB} --easystack ${easystack_file} --robot - fi + ${eb_cmd} ec=$? # copy EasyBuild log file if EasyBuild exited with an error if [ ${ec} -ne 0 ]; then + # If easybuild had a non-zero exit code, and EESSI_INTERACTIVE_BUILD=onfail, you'll get a debug + # shell so you can debug the failure + if [[ "${EESSI_INTERACTIVE_BUILD}" == "onfail" ]]; then + debug_shell "EasyBuild failed with exit code ${ec} for command: ${eb_cmd}" + fi eb_last_log=$(eb --last-log | grep ^/.*\.log) # copy to current working directory if file exhists if [ -f ${eb_last_log} ]; then diff --git a/bot/build.sh b/bot/build.sh index a60054fe..45cf4e4b 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -18,6 +18,28 @@ # file 'job.cfg' has been deposited # - the directory may contain any additional files referenced in job.cfg +# Debugging interactive builds +# This script respects the environment variable EESSI_INTERACTIVE_BUILD= which may be set if one wants to debug interactively +# To debug interactively, run: +# EESSI_INTERACTIVE_BUILD= +# where the path-to-bot-build-jobscript is the job script referred to as build_job_script by the app.cfg of the build bot. +# That is typically eessi-bot-software-layer/scripts/bot-build.slurm, which then calls bot/build.sh (i.e. this script) with +# EESSI_INTERACTIVE_BUILD= set. This way, environment setup that is part of the bot-build.slurm script (including e.g. +# sourcing of the site_config_script as pointed to in the bot's app.cfg) is included. Then, this script sets up more of the +# build environment, before eventually running the build, dropping you in an interactive debugging shell. Supported modes: +# shell - start a plain shell in the build container instead of running the build +# before - run the build, but start a shell (inside the compat layer, EasyBuild loaded) right before +# EasyBuild is called +# onfail - run the build, and start a shell when EasyBuild fails (or the build script exits non-zero) +# The state of the bot's build step (a tarball in previous_tmp/build_step) is resumed, and one accelerator target is +# handled. Select them with EESSI_INTERACTIVE_BUILD_RESUME_FROM (a tmp directory or tarball, or 'none' to not resume from +# anything and start from scratch) and EESSI_INTERACTIVE_BUILD_ACCELERATOR (e.g. accel/nvidia/cc80). If one of them is set, it +# is used as given. If it is not set, you are asked to choose from the available accelerator targets / tarballs (the n-th +# tarball is normally the one written after the n-th accelerator build), which needs stdin to be a terminal. Nothing is +# assumed: without a terminal the script stops with an error asking you to set the variable(s). The only choices that +# are not asked are those with just one option: a single accelerator target, or no tarballs to resume from. +# Nothing is saved, no tarball is created. + # stop as soon as something fails set -e @@ -260,9 +282,20 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} # prepare arguments to eessi_container.sh specific to build step declare -a BUILD_STEP_ARGS=() -BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +# We don't want interactive debugging sessions to save tarballs, so only save if EESSI_INTERACTIVE_BUILD isn't set +if [[ -z "${EESSI_INTERACTIVE_BUILD}" ]]; then + BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +fi BUILD_STEP_ARGS+=("--storage" "${STORAGE}") +# interactive debugging: check the requested mode. The state of the bot's build step to resume from (tarball) and the +# accelerator target are selected further below, once the list of accelerator targets is known +if [[ -n "${EESSI_INTERACTIVE_BUILD}" ]]; then + if [[ ! "${EESSI_INTERACTIVE_BUILD}" =~ ^(shell|before|onfail)$ ]]; then + fatal_error "EESSI_INTERACTIVE_BUILD must be one of: shell, before, onfail (got '${EESSI_INTERACTIVE_BUILD}')" + fi +fi + # Retain location for host injections so we don't reinstall CUDA # (Always need to run the driver installation as available driver may change) if [[ ! -z ${SHARED_FS_PATH} ]]; then @@ -296,6 +329,20 @@ else fi RESUME_DIR="" +# interactive debugging only handles a single accelerator target, and resumes from a single tarball of the bot's build step +# (or from none). Select them as described at the top of this script, see scripts/interactive_build.sh +if [[ -n "${EESSI_INTERACTIVE_BUILD}" ]]; then + source $software_layer_dir/scripts/interactive_build.sh + select_interactive_state + if [[ "${selected_tarball}" == "none" ]]; then + echo "bot/build.sh: EESSI_INTERACTIVE_BUILD='${EESSI_INTERACTIVE_BUILD}', accelerator '${selected_accel_target}', not resuming from a tarball" + else + echo "bot/build.sh: EESSI_INTERACTIVE_BUILD='${EESSI_INTERACTIVE_BUILD}', accelerator '${selected_accel_target}', resuming from '${selected_tarball}'" + BUILD_STEP_ARGS+=("--resume" "${selected_tarball}") + fi + EESSI_ACCELERATOR_TARGET_OVERRIDES=("${selected_accel_target}") +fi + for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do # copy the common build step arguments to a a BUILD_STEP_ARGS_ACCEL=("${BUILD_STEP_ARGS[@]}") @@ -322,6 +369,19 @@ for ACCEL_OVERRIDE in "${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}"; do echo "Executing command to build software:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS_ACCEL[@]}" echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" + if [[ -n "${EESSI_INTERACTIVE_BUILD}" ]]; then + # Interactive debugging: same setup as above. No 'tee' as it would break the interactive terminal. + if [[ "${EESSI_INTERACTIVE_BUILD}" == "shell" ]]; then + echo "bot/build.sh: starting a shell in the build container instead of the build; to run the build step manually use:" + echo " $software_layer_dir/install_software_layer.sh ${INSTALL_SCRIPT_ARGS[@]} $@" + # a later --mode overrides the --mode exec in COMMON_ARGS + $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS_ACCEL[@]}" --mode shell + else + $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS_ACCEL[@]}" \ + -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" + fi + exit 0 + fi $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS_ACCEL[@]}" \ -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 07501a7c..989f1d6b 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -59,6 +59,9 @@ fi if [ ! -z ${EESSI_INIT_PREFIX} ]; then INPUT="export EESSI_INIT_PREFIX=${EESSI_INIT_PREFIX}; ${INPUT}" fi +if [ ! -z ${EESSI_INTERACTIVE_BUILD} ]; then + INPUT="export EESSI_INTERACTIVE_BUILD=${EESSI_INTERACTIVE_BUILD}; ${INPUT}" +fi if [ ! -z ${EESSI_SITE_INSTALL_FORCE} ]; then INPUT="export EESSI_SITE_INSTALL_FORCE=${EESSI_SITE_INSTALL_FORCE}; ${INPUT}" if [ ! -z ${EESSI_SITE_SOFTWARE_PREFIX} ]; then diff --git a/scripts/interactive_build.sh b/scripts/interactive_build.sh new file mode 100644 index 00000000..fa19e820 --- /dev/null +++ b/scripts/interactive_build.sh @@ -0,0 +1,102 @@ +# Functions for debugging a bot build interactively, sourced by bot/build.sh if EESSI_INTERACTIVE_BUILD is set. +# See the comments at the top of bot/build.sh for the user-facing description. +# +# Requires fatal_error (scripts/utils.sh). + +# Ask the user to choose a number between $2 (lowest number) and $3 (highest number) for $1 (description of what is +# chosen). The result is returned in CHOSEN_NUMBER. There is no default: an empty answer is invalid. Returns 1 if no +# answer can be read (e.g. at EOF because user pressed Ctrl+D / terminal disappears, which also avoids an infinite loop). +# Note that all output of the dialog goes to stderr, like the prompt of 'read -p' does (bash's own convention for prompts) +interactive_choice() { + local description="$1" lowest_number="$2" highest_number="$3" answer + # loop until we get a valid answer + while true; do + if ! read -r -p "bot/build.sh: select ${description} [${lowest_number}-${highest_number}]: " answer; then + echo >&2 + return 1 + fi + # Check answer is valid and return + if [[ "${answer}" =~ ^[0-9]+$ ]] && (( answer >= lowest_number && answer <= highest_number )); then + CHOSEN_NUMBER=${answer} + return + fi + echo "bot/build.sh: invalid choice '${answer}'" >&2 + done +} + +# Choose one entry of the array named $4, and return its number in CHOSEN_NUMBER: N for entry N of the array (counting from +# 1), and 0 for 'none' (only allowed if $3, the lowest number, is 0). +# $1 description of what is chosen, $2 name of the environment variable with which the user can set it +# If there is only one option, we don't need to choose. Otherwise we ask the user, if stdin is a terminal, or else stop +# with an error: we don't assume anything. +choose_option() { + local description="$1" env_var_name="$2" lowest_number="$3" + local -n option_array="$4" + local highest_number=${#option_array[@]} + if [[ ${highest_number} -eq ${lowest_number} ]]; then + CHOSEN_NUMBER=${highest_number} + elif [[ -t 0 ]]; then + interactive_choice "${description}" ${lowest_number} ${highest_number} || + fatal_error "no answer given for ${description}, set ${env_var_name} to choose it without being asked" + else + fatal_error "stdin is not a terminal, so cannot ask which ${description}, set ${env_var_name} to choose it" + fi +} + +# Select the accelerator target and the tarball of the bot's build step to resume from, based on +# EESSI_INTERACTIVE_BUILD_ACCELERATOR and EESSI_INTERACTIVE_BUILD_RESUME_FROM, asking for what isn't set (see bot/build.sh). +# Needs EESSI_ACCELERATOR_TARGET_OVERRIDES (the accelerator targets of the build) and TARBALL_TMP_BUILD_STEP_DIR. +# Returns the accelerator target in selected_accel_target, and the tarball (or directory) in selected_tarball, which is +# the word 'none' if we shouldn't resume from anything. +select_interactive_state() { + local -a accel_targets=("${EESSI_ACCELERATOR_TARGET_OVERRIDES[@]}") + local -a tarball_paths + mapfile -t tarball_paths < <(ls -tr "${TARBALL_TMP_BUILD_STEP_DIR}"/tmp_storage-* 2>/dev/null) + local i + + # select the accelerator target + if [[ -n "${EESSI_INTERACTIVE_BUILD_ACCELERATOR}" ]]; then + # Format for EESSI_INTERACTIVE_BUILD_ACCELERATOR is similar to EESSI_ACCELERATOR_TARGET_OVERRIDE, + # e.g. accel/nvidia/cc80 + selected_accel_target="${EESSI_INTERACTIVE_BUILD_ACCELERATOR}" + local found=0 accel_target + for accel_target in "${accel_targets[@]}"; do + [[ "${accel_target}" == "${selected_accel_target}" ]] && found=1 + done + if [[ ${found} -eq 0 ]]; then + fatal_error "EESSI_INTERACTIVE_BUILD_ACCELERATOR='${EESSI_INTERACTIVE_BUILD_ACCELERATOR}' not found in accelerator targets of ${JOB_CFG_FILE}: ${accel_targets[*]}" + fi + else + # List accelerators from the job.cfg file + # (skip listing if there is only one - choose_option will select it by default) + if [[ ${#accel_targets[@]} -gt 1 ]]; then + echo "bot/build.sh: accelerator targets of this build (in the order the bot built them):" + for i in "${!accel_targets[@]}"; do + echo " [$((i+1))] '${accel_targets[$i]}'" + done + fi + choose_option "accelerator target" EESSI_INTERACTIVE_BUILD_ACCELERATOR 1 accel_targets + selected_accel_target="${accel_targets[$((CHOSEN_NUMBER-1))]}" + fi + + # select the tarball to resume from, or none + if [[ -n "${EESSI_INTERACTIVE_BUILD_RESUME_FROM}" ]]; then + selected_tarball="${EESSI_INTERACTIVE_BUILD_RESUME_FROM}" + else + echo "bot/build.sh: tarballs of the bot's build step in ${TARBALL_TMP_BUILD_STEP_DIR} (oldest first):" + echo " [0] none (do not resume from a tarball, start from scratch)" + for i in "${!tarball_paths[@]}"; do + echo " [$((i+1))] $(basename "${tarball_paths[$i]}") ($(date -r "${tarball_paths[$i]}" '+%Y-%m-%d %H:%M'), $(du -h "${tarball_paths[$i]}" | cut -f1))" + done + echo " note: normally, tarball [n] is written after the n-th accelerator build, since each build should produce one tarball" + [[ ${#tarball_paths[@]} -gt 0 ]] || echo " no tarballs found, so not resuming from a tarball" + choose_option "tarball to resume from (0 = none)" EESSI_INTERACTIVE_BUILD_RESUME_FROM 0 tarball_paths + selected_tarball="none" + [[ ${CHOSEN_NUMBER} -gt 0 ]] && selected_tarball="${tarball_paths[$((CHOSEN_NUMBER-1))]}" + fi + # accept 'none' in any capitalisation (and normalize to lower case for easier subsequent checks) + [[ "${selected_tarball,,}" == "none" ]] && selected_tarball="none" + if [[ "${selected_tarball}" != "none" && ! -e "${selected_tarball}" ]]; then + fatal_error "state to resume from does not exist: ${selected_tarball}" + fi +}