From e0c110ae890c164695943219862896f6c49145e5 Mon Sep 17 00:00:00 2001 From: Jea-Eok-Kim Date: Thu, 3 Sep 2026 23:11:47 +0900 Subject: [PATCH 1/2] slurm: batch the queries and parallelize the writes in the GPU power/clock helpers set_gpu_power_levels.sh and set_gpu_clocks.sh called nvidia-smi once per GPU to read the target value and once more to apply it, all serially. Both "nvidia-smi -pl" and "nvidia-smi -ac" take roughly a second per GPU, so on an 8-GPU node the two helpers together add about 8 s to the prolog of every job that 50-exclusive-gpu runs for. srun reports this as: srun: Prolog hung on node Read the values for all GPUs in a single --query-gpu call, then apply them in parallel and collect each child's exit status so a failure on any GPU still fails the script. Behaviour is otherwise unchanged: the same values are written to the same GPUs. The "default" branch of set_gpu_clocks.sh already operated on all GPUs at once and is untouched. Observed on DGX OS 7.5.0 (8x B300), Slurm 26.05.1: prolog took 6-8 s per job while 50-exclusive-gpu was running. Signed-off-by: Jea-Eok-Kim --- .../etc/slurm/shared/bin/set_gpu_clocks.sh | 19 ++++--- .../slurm/shared/bin/set_gpu_power_levels.sh | 50 ++++++++++++------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh index 13918792e..84a8ae4ee 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -e -gpu_count="$(nvidia-smi -L | wc -l)" - case "$1" in default) nvidia-smi -rac # Reset application clocks @@ -10,11 +8,20 @@ case "$1" in nvidia-smi -c DEFAULT # Reset compute mode to default ;; max) - for i in $(seq 0 "$(( gpu_count - 1 ))" ) ; do - nextSM="$(nvidia-smi -i "$i" --query-gpu=clocks.max.sm --format=csv,noheader,nounits)" - nextMEM="$(nvidia-smi -i "$i" --query-gpu=clocks.max.mem --format=csv,noheader,nounits)" - nvidia-smi -i "${i}" -ac "${nextMEM}","${nextSM}" + # Query every GPU in a single call, then apply in parallel: + # "nvidia-smi -ac" also costs roughly a second per GPU. + readarray -t maxSM < <(nvidia-smi --query-gpu=clocks.max.sm --format=csv,noheader,nounits) + readarray -t maxMEM < <(nvidia-smi --query-gpu=clocks.max.mem --format=csv,noheader,nounits) + pids=() + for i in "${!maxSM[@]}" ; do + nvidia-smi -i "${i}" -ac "${maxMEM[$i]}","${maxSM[$i]}" >/dev/null & + pids+=("$!") + done + rc=0 + for pid in "${pids[@]}" ; do + wait "$pid" || rc=1 done + exit "$rc" ;; *) echo "Usage: $0 [default|max]" diff --git a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh index 4b3cb54e5..e609d458a 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh @@ -1,24 +1,38 @@ #!/usr/bin/env bash set -e -gpu_count="$(nvidia-smi -L | wc -l)" +case "$1" in + max) + query=power.max_limit + ;; + default) + query=power.default_limit + ;; + min) + query=power.min_limit + ;; + *) + echo "Usage: $0 [max,default,min]" + exit 1 + ;; +esac -for i in $(seq 0 "$(( gpu_count - 1 ))" ) +# Query every GPU in a single call instead of one call per GPU. +readarray -t limits < <(nvidia-smi --query-gpu="$query" --format=csv,noheader,nounits) + +# "nvidia-smi -pl" takes roughly a second per GPU, so applying the limits +# serially adds ~8 s to the prolog of every full-node job on an 8-GPU node. +# Apply them in parallel and collect the exit status of each child. +pids=() +for i in "${!limits[@]}" +do + nvidia-smi -i "$i" -pl "${limits[$i]}" >/dev/null & + pids+=("$!") +done + +rc=0 +for pid in "${pids[@]}" do - case "$1" in - max) - next="$(nvidia-smi -i "$i" --query-gpu=power.max_limit --format=csv,noheader,nounits)" - ;; - default) - next="$(nvidia-smi -i "$i" --query-gpu=power.default_limit --format=csv,noheader,nounits)" - ;; - min) - next="$(nvidia-smi -i "$i" --query-gpu=power.min_limit --format=csv,noheader,nounits)" - ;; - *) - echo "Usage: $0 [max,default,min]" - exit 1 - ;; - esac - nvidia-smi -i "$i" -pl "$next" + wait "$pid" || rc=1 done +exit "$rc" From c95d3a294fcbbdeb802200bfe3cf1f4640e60041 Mon Sep 17 00:00:00 2001 From: 100milliongold Date: Mon, 7 Sep 2026 09:58:50 +0900 Subject: [PATCH 2/2] fix: validate the batched GPU queries before writing anything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `readarray -t limits < <(nvidia-smi ...)` hides the query's exit status from both readarray and `set -e`. The helpers therefore reported success in every failure mode: an empty result made the write loop run zero times, a truncated result configured only some of the GPUs, and a nonzero exit was not seen at all. The query result now goes through a file so its status can be checked, the index is selected alongside the values so a write targets the GPU nvidia-smi reported instead of an array subscript, and every row is parsed and range-checked before the first write is launched. The row count is compared against `nvidia-smi -L`, so a partial result is rejected rather than silently applied. set_gpu_clocks.sh additionally selected clocks.max.mem and clocks.max.sm in two separate queries. If the two returned different row counts, `${maxMEM[$i]}` was empty for the trailing GPUs and produced an `-ac ,1980` argument. Both values now come from the same query, so they cannot drift out of alignment. Verified on a DGX B300 (Ubuntu 24.04, bash 5.2) with an nvidia-smi stub that honours --query-gpu and records writes instead of performing them. Identical results for both helpers: case before after ---------- ---------------------------- --------------------- ok rc=0 8 writes rc=0 8 writes empty rc=0 0 writes rc=1 0 writes partial rc=0 3 writes (of 8 GPUs) rc=1 0 writes fail rc=0 0 writes rc=1 0 writes nonnumeric rc=0 8 writes ("N/A" passed) rc=1 0 writes One limitation of the stub is worth stating: it records writes rather than performing them, so the `nonnumeric` row shows 8 writes for the old code. On real hardware `nvidia-smi -pl N/A` fails and `wait` would surface rc=1 — but only after eight bad invocations. The new code rejects the row before the first one. The `ok` case is unchanged, so the parallel-write speedup this branch adds is preserved. --- .../etc/slurm/shared/bin/set_gpu_clocks.sh | 51 +++++++++++++++++-- .../slurm/shared/bin/set_gpu_power_levels.sh | 47 +++++++++++++++-- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh index 84a8ae4ee..d171abf2f 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_clocks.sh @@ -10,11 +10,54 @@ case "$1" in max) # Query every GPU in a single call, then apply in parallel: # "nvidia-smi -ac" also costs roughly a second per GPU. - readarray -t maxSM < <(nvidia-smi --query-gpu=clocks.max.sm --format=csv,noheader,nounits) - readarray -t maxMEM < <(nvidia-smi --query-gpu=clocks.max.mem --format=csv,noheader,nounits) + # + # Index, memory clock and SM clock come from one query so the three + # values of a GPU cannot drift out of alignment. Reading them through + # "readarray -t maxSM < <(nvidia-smi ...)" hid the query's exit status + # from both readarray and "set -e": a failed query left the arrays + # empty and the helper exited 0 having set nothing, and two separate + # queries returning different row counts made "${maxMEM[$i]}" empty for + # the trailing GPUs, producing an "-ac ,1980" argument. + tmp=$(mktemp) + trap 'rm -f "$tmp"' EXIT + + if ! nvidia-smi --query-gpu=index,clocks.max.mem,clocks.max.sm \ + --format=csv,noheader,nounits > "$tmp"; then + echo "$0: querying the maximum clocks failed" >&2 + exit 1 + fi + + # Every GPU that nvidia-smi lists has to be present in the query result + # before anything is written. + if ! expected=$(nvidia-smi -L | grep -c '^GPU '); then + echo "$0: could not determine the number of GPUs" >&2 + exit 1 + fi + + indexes=() + maxMEM=() + maxSM=() + while IFS=', ' read -r index mem sm _; do + [ -z "$index" ] && continue + if ! [[ "$index" =~ ^[0-9]+$ ]] \ + || ! [[ "$mem" =~ ^[0-9]+$ ]] \ + || ! [[ "$sm" =~ ^[0-9]+$ ]]; then + echo "$0: unexpected row from nvidia-smi: '$index, $mem, $sm'" >&2 + exit 1 + fi + indexes+=("$index") + maxMEM+=("$mem") + maxSM+=("$sm") + done < "$tmp" + + if [ "${#indexes[@]}" -ne "$expected" ]; then + echo "$0: got ${#indexes[@]} usable clock rows, expected $expected" >&2 + exit 1 + fi + pids=() - for i in "${!maxSM[@]}" ; do - nvidia-smi -i "${i}" -ac "${maxMEM[$i]}","${maxSM[$i]}" >/dev/null & + for i in "${!indexes[@]}" ; do + nvidia-smi -i "${indexes[$i]}" -ac "${maxMEM[$i]}","${maxSM[$i]}" >/dev/null & pids+=("$!") done rc=0 diff --git a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh index e609d458a..da140c160 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/set_gpu_power_levels.sh @@ -18,15 +18,56 @@ case "$1" in esac # Query every GPU in a single call instead of one call per GPU. -readarray -t limits < <(nvidia-smi --query-gpu="$query" --format=csv,noheader,nounits) +# +# The result is written to a file rather than read through +# "readarray -t limits < <(nvidia-smi ...)": inside a process substitution the +# query's exit status is invisible to both readarray and "set -e", so a failed +# query left the array empty, the write loop ran zero times, and the helper +# still exited 0 having configured nothing. A truncated result configured only +# some of the GPUs and also returned success. +# +# The index is queried alongside the value so each write targets the GPU that +# nvidia-smi actually reported, rather than assuming the array subscript equals +# the GPU index. +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT + +if ! nvidia-smi --query-gpu=index,"$query" --format=csv,noheader,nounits > "$tmp"; then + echo "$0: querying $query failed" >&2 + exit 1 +fi + +# Every GPU that nvidia-smi lists has to be present in the query result before +# anything is written. +if ! expected=$(nvidia-smi -L | grep -c '^GPU '); then + echo "$0: could not determine the number of GPUs" >&2 + exit 1 +fi + +indexes=() +limits=() +while IFS=', ' read -r index limit _; do + [ -z "$index" ] && continue + if ! [[ "$index" =~ ^[0-9]+$ ]] || ! [[ "$limit" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then + echo "$0: unexpected row from nvidia-smi: '$index, $limit'" >&2 + exit 1 + fi + indexes+=("$index") + limits+=("$limit") +done < "$tmp" + +if [ "${#indexes[@]}" -ne "$expected" ]; then + echo "$0: got ${#indexes[@]} usable rows for $query, expected $expected" >&2 + exit 1 +fi # "nvidia-smi -pl" takes roughly a second per GPU, so applying the limits # serially adds ~8 s to the prolog of every full-node job on an 8-GPU node. # Apply them in parallel and collect the exit status of each child. pids=() -for i in "${!limits[@]}" +for i in "${!indexes[@]}" do - nvidia-smi -i "$i" -pl "${limits[$i]}" >/dev/null & + nvidia-smi -i "${indexes[$i]}" -pl "${limits[$i]}" >/dev/null & pids+=("$!") done