diff --git a/docs/slurm-cluster/README.md b/docs/slurm-cluster/README.md index 4d5bda512..0f5952d31 100644 --- a/docs/slurm-cluster/README.md +++ b/docs/slurm-cluster/README.md @@ -240,6 +240,12 @@ DeepOps no longer bundles an Open OnDemand installer. Sites that want the [Open [Pyxis](https://github.com/NVIDIA/pyxis) and [Enroot](https://github.com/NVIDIA/enroot) are installed by default and can be disabled by setting `slurm_install_enroot` and `slurm_install_pyxis` to no. They are the supported, release-validated container runtime for Slurm in DeepOps. +On Ubuntu systems that restrict unprivileged user namespaces, DeepOps installs +a command-scoped AppArmor profile for `/usr/bin/enroot-nsenter`. This preserves +the host-wide security default while allowing Pyxis jobs to create their user +namespace. Set `pyxis_configure_enroot_apparmor_userns: false` only when site +policy provides an equivalent profile. + The DeepOps Singularity wrapper role has been retired. Singularity lives on upstream as [Apptainer](https://apptainer.org/); sites that still want it can install Apptainer/Singularity separately. ## Large deployments diff --git a/playbooks/nvidia-software/nvidia-driver.yml b/playbooks/nvidia-software/nvidia-driver.yml index c445e5574..48316f784 100644 --- a/playbooks/nvidia-software/nvidia-driver.yml +++ b/playbooks/nvidia-software/nvidia-driver.yml @@ -29,4 +29,5 @@ when: - ansible_local['gpus']['count'] - is_dgx.stat.exists == False + - nvidia_driver_test_enabled | default(true) | bool environment: "{{proxy_env if proxy_env is defined else{}}}" diff --git a/playbooks/slurm-cluster.yml b/playbooks/slurm-cluster.yml index 3f0788308..7bc6da0da 100644 --- a/playbooks/slurm-cluster.yml +++ b/playbooks/slurm-cluster.yml @@ -123,6 +123,10 @@ - name: Install monitoring exporters | Install nvidia dcgm exporter import_playbook: slurm-cluster/nvidia-dcgm-exporter.yml + vars: + # The driver was already validated before Slurm enabled the direct-login + # GPU guard. The final allocated srun validator is the authoritative proof. + nvidia_dcgm_exporter_driver_test_enabled: false when: slurm_enable_monitoring - name: Set up rsyslog forwarding from compute nodes to head node | Install rsyslog server diff --git a/playbooks/slurm-cluster/nvidia-dcgm-exporter.yml b/playbooks/slurm-cluster/nvidia-dcgm-exporter.yml index 88f8d849a..36414250a 100644 --- a/playbooks/slurm-cluster/nvidia-dcgm-exporter.yml +++ b/playbooks/slurm-cluster/nvidia-dcgm-exporter.yml @@ -4,6 +4,8 @@ - name: Install NVIDIA driver import_playbook: ../nvidia-software/nvidia-driver.yml + vars: + nvidia_driver_test_enabled: "{{ nvidia_dcgm_exporter_driver_test_enabled | default(true) }}" - name: Install NVIDIA container runtime import_playbook: ../container/nvidia-docker.yml diff --git a/roles/nvidia-peer-memory/tasks/main.yml b/roles/nvidia-peer-memory/tasks/main.yml index 04025e7f0..d0f3c8120 100644 --- a/roles/nvidia-peer-memory/tasks/main.yml +++ b/roles/nvidia-peer-memory/tasks/main.yml @@ -4,19 +4,76 @@ path: /etc/dgx-release register: is_dgx -- name: Autoinstall DKMS modules +- name: Check for in-tree nvidia_peermem module + command: modinfo nvidia_peermem + register: nvidia_peermem_info + failed_when: false + changed_when: false + when: + - ansible_local['gpus']['count'] + - is_dgx.stat.exists + +- name: Collect service state before selecting peer-memory implementation + service_facts: + when: + - ansible_local['gpus']['count'] + - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) == 0 + +- name: Stop and disable legacy nv_peer_mem service + service: + name: nv_peer_mem + state: stopped + enabled: false + when: + - ansible_local['gpus']['count'] + - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) == 0 + - "'nv_peer_mem.service' in ansible_facts.services or 'nv_peer_mem' in ansible_facts.services" + +- name: Check whether legacy nv_peer_mem module is loaded + stat: + path: /sys/module/nv_peer_mem + register: nv_peer_mem_loaded + when: + - ansible_local['gpus']['count'] + - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) == 0 + +- name: Unload legacy nv_peer_mem module before using nvidia_peermem + modprobe: + name: nv_peer_mem + state: absent + when: + - ansible_local['gpus']['count'] + - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) == 0 + - nv_peer_mem_loaded.stat.exists | default(false) + +- name: Load in-tree nvidia_peermem module + modprobe: + name: nvidia_peermem + state: present + when: + - ansible_local['gpus']['count'] + - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) == 0 + +- name: Autoinstall legacy nv_peer_mem DKMS module command: dkms autoinstall when: - ansible_local['gpus']['count'] - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) != 0 -- name: Modprobe nv_peer_mem +- name: Load legacy nv_peer_mem module modprobe: name: nv_peer_mem state: present when: - ansible_local['gpus']['count'] - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) != 0 - name: Start nv_peer_mem service service: @@ -25,3 +82,4 @@ when: - ansible_local['gpus']['count'] - is_dgx.stat.exists + - nvidia_peermem_info.rc | default(1) != 0 diff --git a/roles/nvidia_dcgm/defaults/main.yml b/roles/nvidia_dcgm/defaults/main.yml index 087cf9c0c..20a61d59d 100644 --- a/roles/nvidia_dcgm/defaults/main.yml +++ b/roles/nvidia_dcgm/defaults/main.yml @@ -1,5 +1,6 @@ --- dcgm_pkg_name: "datacenter-gpu-manager" +dcgm_dgx_cuda13_pkg_name: "datacenter-gpu-manager-4-cuda13" # RedHat family epel_package: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" diff --git a/roles/nvidia_dcgm/tasks/install-dgx.yml b/roles/nvidia_dcgm/tasks/install-dgx.yml index 0d512b3e7..141be0e50 100644 --- a/roles/nvidia_dcgm/tasks/install-dgx.yml +++ b/roles/nvidia_dcgm/tasks/install-dgx.yml @@ -1,5 +1,24 @@ --- -- name: install DCGM from repos +- name: Select DGX-compatible DCGM package + set_fact: + dcgm_dgx_selected_pkg_name: >- + {{ dcgm_dgx_cuda13_pkg_name + if ansible_distribution == 'Ubuntu' and + ansible_distribution_version is version('24.04', '>=') + else dcgm_pkg_name }} + +- name: Install the selected DGX-compatible DCGM package package: - name: "datacenter-gpu-manager" + name: "{{ dcgm_dgx_selected_pkg_name }}" state: present + +- name: Refresh package facts after DCGM convergence + package_facts: + manager: auto + +- name: Verify the selected DGX-compatible DCGM package is installed + assert: + that: + - dcgm_dgx_selected_pkg_name in ansible_facts.packages + fail_msg: >- + Required DGX DCGM package {{ dcgm_dgx_selected_pkg_name }} is not installed diff --git a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 index 502c7bf3c..f3b2c5b3e 100644 --- a/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 +++ b/roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2 @@ -12,7 +12,7 @@ ExecStartPre=-/usr/bin/docker rm %n {% if not slurm_exporter_build_image %} ExecStartPre=/usr/bin/docker pull {{ slurm_exporter_container }} {% endif %} -ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} +ExecStart=/usr/bin/docker run --rm --network host --name %n --env PATH=/usr/local/bin:/usr/bin:/bin -v {{ slurm_install_prefix }}/bin/sdiag:/usr/local/bin/sdiag:ro -v {{ slurm_install_prefix }}/bin/sinfo:/usr/local/bin/sinfo:ro -v {{ slurm_install_prefix }}/bin/squeue:/usr/local/bin/squeue:ro -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }} [Install] WantedBy=multi-user.target diff --git a/roles/pyxis/defaults/main.yml b/roles/pyxis/defaults/main.yml index 7db494eca..21c520608 100644 --- a/roles/pyxis/defaults/main.yml +++ b/roles/pyxis/defaults/main.yml @@ -9,10 +9,16 @@ is_compute: no pyxis_ubuntu_deps: - "bsdmainutils" +- "apparmor" pyxis_el_deps: - "util-linux" +# Ubuntu restricts unprivileged user namespaces when this kernel knob exists. +# Install a command-scoped AppArmor profile instead of weakening the host-wide +# setting. Disable only when site policy supplies an equivalent profile. +pyxis_configure_enroot_apparmor_userns: true + # /run is default partition of pyxis runtime_path resize_run_partition: false diff --git a/roles/pyxis/tasks/main.yml b/roles/pyxis/tasks/main.yml index 433a8b45f..1f0ba9da3 100644 --- a/roles/pyxis/tasks/main.yml +++ b/roles/pyxis/tasks/main.yml @@ -13,6 +13,38 @@ with_items: "{{ pyxis_el_deps }}" when: ansible_os_family == "RedHat" +- name: Check for AppArmor unprivileged user namespace restriction + stat: + path: /proc/sys/kernel/apparmor_restrict_unprivileged_userns + register: apparmor_userns_knob + when: + - ansible_distribution == "Ubuntu" + - is_compute + - pyxis_configure_enroot_apparmor_userns | bool + +- name: Install command-scoped AppArmor profile for enroot-nsenter + template: + src: enroot-nsenter.apparmor.j2 + dest: /etc/apparmor.d/enroot-nsenter + owner: root + group: root + mode: "0644" + register: enroot_apparmor_profile + when: + - ansible_distribution == "Ubuntu" + - is_compute + - pyxis_configure_enroot_apparmor_userns | bool + - apparmor_userns_knob.stat.exists | default(false) + +- name: Load command-scoped AppArmor profile for enroot-nsenter + command: apparmor_parser -r /etc/apparmor.d/enroot-nsenter + changed_when: enroot_apparmor_profile.changed + when: + - ansible_distribution == "Ubuntu" + - is_compute + - pyxis_configure_enroot_apparmor_userns | bool + - apparmor_userns_knob.stat.exists | default(false) + - name: install slurm-pmi hook file: path: /etc/enroot/hooks.d/50-slurm-pmi.sh diff --git a/roles/pyxis/templates/enroot-nsenter.apparmor.j2 b/roles/pyxis/templates/enroot-nsenter.apparmor.j2 new file mode 100644 index 000000000..e89373d83 --- /dev/null +++ b/roles/pyxis/templates/enroot-nsenter.apparmor.j2 @@ -0,0 +1,10 @@ +# This profile grants only enroot-nsenter the user namespace permission that +# Ubuntu otherwise restricts for unconfined applications. + +include + +profile enroot-nsenter /usr/bin/enroot-nsenter flags=(unconfined) { + userns, + + include if exists +} diff --git a/roles/slurm/tasks/login-compute-setup.yml b/roles/slurm/tasks/login-compute-setup.yml index 2644c9bc3..be888956b 100644 --- a/roles/slurm/tasks/login-compute-setup.yml +++ b/roles/slurm/tasks/login-compute-setup.yml @@ -1,18 +1,18 @@ --- -- name: Hide GPUs for regular user logins via sshd.service. +- name: Hide GPUs for regular user logins via the platform SSH service + vars: + ssh_unit: "{{ 'sshd.service' if ansible_os_family == 'RedHat' else 'ssh.service' }}" shell: | set -o pipefail - deviceprop=$(systemctl show sshd.service -p DeviceAllow | grep -i nvidiactl) + deviceprop=$(systemctl show {{ ssh_unit }} -p DeviceAllow | grep -i nvidiactl) if [ -z "$deviceprop" ] ; then - systemctl set-property sshd.service DeviceAllow="/dev/nvidiactl" + systemctl set-property {{ ssh_unit }} DeviceAllow="/dev/nvidiactl" fi args: executable: /bin/bash - creates: "{{ '/etc/systemd/system.control/sshd.service.d/50-DeviceAllow.conf' \ - if ansible_os_family == 'RedHat' else \ - '/etc/systemd/system.control/ssh.service.d/50-DeviceAllow.conf' }}" + creates: "/etc/systemd/system.control/{{ ssh_unit }}.d/50-DeviceAllow.conf" when: is_controller tags: - config diff --git a/roles/slurm/tasks/service-files.yml b/roles/slurm/tasks/service-files.yml index 7ecbff88d..4cbacc591 100644 --- a/roles/slurm/tasks/service-files.yml +++ b/roles/slurm/tasks/service-files.yml @@ -21,3 +21,24 @@ - slurmd.service when: is_compute failed_when: false + +- name: Create slurmd systemd override directory + file: + path: /etc/systemd/system/slurmd.service.d + state: directory + owner: root + group: root + mode: "0755" + when: is_compute + +- name: Put custom-prefix Slurm clients on the slurmd PATH + copy: + dest: /etc/systemd/system/slurmd.service.d/10-slurm-path.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + Environment="PATH={{ slurm_install_prefix }}/bin:{{ slurm_install_prefix }}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + when: is_compute + notify: restart slurmd diff --git a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh index f748196ca..f894f17c5 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh @@ -2,6 +2,7 @@ # This could all be done with run-parts using regexes on ubuntu. # However, centos' version of run-parts is just a simple bash script with no useful flags. set -e +set -o pipefail if [ "$#" -ne 1 ]; then echo "Usage: $0 parts_dir" @@ -13,17 +14,37 @@ log () { logger -s -t slurm "$@" } -# Find out if we are running in exclusive mode +# Use the configured prefix because slurmd does not inherit it in PATH. +squeue_bin="{{ slurm_install_prefix }}/bin/squeue" + +# Find out if we are running in exclusive mode. Failed or incomplete scheduler +# queries fail closed to non-exclusive while ordinary scripts still run. exclusive=0 -numcpus_sys=$(( $(grep -c ^processor /proc/cpuinfo) * $(scontrol show job "$SLURM_JOBID" | grep -Eio "TRES=.*node=[0-9]+" | cut -d= -f5) )) -numcpus_job=$(scontrol show job "$SLURM_JOBID" | grep -Eio "TRES=cpu=[0-9]+" | cut -d= -f3) -if [ "$numcpus_sys" == "$numcpus_job" ] ; then +numcpus_job="" +numnodes_job="" +if ! numcpus_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %C 2>/dev/null); then + log "Unable to query allocated CPUs for job $SLURM_JOBID; treating it as non-exclusive." +fi +if ! numnodes_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %D 2>/dev/null); then + log "Unable to query allocated nodes for job $SLURM_JOBID; treating it as non-exclusive." +fi +if [[ "$numcpus_job" =~ ^[0-9]+$ ]] && + [[ "$numnodes_job" =~ ^[1-9][0-9]*$ ]] && + [ $(( $(grep -c ^processor /proc/cpuinfo) * numnodes_job )) -eq "$numcpus_job" ]; then exclusive=1 fi # Find out if there are any more jobs on this node for this user last_user_job=0 -num_jobs=$(squeue -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running | wc -l) +num_jobs=1 +if ! num_jobs=$("$squeue_bin" -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running | wc -l); then + log "Unable to query remaining jobs for $SLURM_JOB_USER; preserving last-user cleanup state." + num_jobs=1 +fi +if [[ ! "$num_jobs" =~ ^[0-9]+$ ]]; then + log "Invalid remaining-job count for $SLURM_JOB_USER; preserving last-user cleanup state." + num_jobs=1 +fi if [ "$num_jobs" -eq 0 ]; then last_user_job=1 fi diff --git a/scripts/validation/tests/test_release_blocker_contracts.py b/scripts/validation/tests/test_release_blocker_contracts.py new file mode 100644 index 000000000..9f3cc9602 --- /dev/null +++ b/scripts/validation/tests/test_release_blocker_contracts.py @@ -0,0 +1,177 @@ +"""Regression tests for the 26.09 release-blocker fixes. + +Run with: python3 -m unittest discover scripts/validation/tests +""" + +import os +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] + + +class SlurmRunPartsTests(unittest.TestCase): + def run_fixture(self, mode): + source = ( + ROOT / "roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh" + ).read_text(encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + prefix = root / "slurm" + bin_dir = prefix / "bin" + parts = root / "parts" + mocks = root / "mocks" + output = root / "output" + bin_dir.mkdir(parents=True) + parts.mkdir() + mocks.mkdir() + + rendered = root / "run-parts.sh" + rendered.write_text( + source.replace("{{ slurm_install_prefix }}", str(prefix)).replace( + "/var/log/slurm/prolog-epilog", str(root / "prolog-epilog.log") + ), + encoding="utf-8", + ) + rendered.chmod(0o755) + + (bin_dir / "squeue").write_text( + textwrap.dedent( + """\ + #!/usr/bin/env bash + case "$SQUEUE_MODE:$*" in + fail:*) exit 1 ;; + empty:*) exit 0 ;; + invalid:*"-o %C"*) echo invalid ;; + invalid:*"-o %D"*) echo 1 ;; + invalid:*) echo invalid ;; + valid:*"-o %C"*) grep -c '^processor' /proc/cpuinfo ;; + valid:*"-o %D"*) echo 1 ;; + valid:*) exit 0 ;; + esac + """ + ), + encoding="utf-8", + ) + (bin_dir / "squeue").chmod(0o755) + (mocks / "logger").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + (mocks / "logger").chmod(0o755) + + for name in ("10-normal", "20-exclusive-test", "30-lastuserjob-test"): + path = parts / name + path.write_text( + f"#!/bin/sh\necho {name} >> \"$OUTPUT\"\n", + encoding="utf-8", + ) + path.chmod(0o755) + + env = { + **os.environ, + "HOSTNAME": "node1", + "OUTPUT": str(output), + "PATH": f"{mocks}:{os.environ['PATH']}", + "SLURM_JOBID": "42", + "SLURM_JOB_USER": "test-user", + "SQUEUE_MODE": mode, + } + result = subprocess.run( + [str(rendered), str(parts)], + env=env, + text=True, + capture_output=True, + check=False, + ) + lines = ( + output.read_text(encoding="utf-8").splitlines() + if output.exists() + else [] + ) + return result, lines + + def test_failed_scheduler_queries_run_only_unconditional_parts(self): + result, lines = self.run_fixture("fail") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(lines, ["10-normal"]) + + def test_empty_scheduler_queries_run_only_unconditional_parts(self): + result, lines = self.run_fixture("empty") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(lines, ["10-normal", "30-lastuserjob-test"]) + + def test_invalid_scheduler_output_fails_closed(self): + result, lines = self.run_fixture("invalid") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(lines, ["10-normal"]) + + def test_complete_scheduler_queries_select_exclusive_and_last_user_parts(self): + result, lines = self.run_fixture("valid") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + lines, + ["10-normal", "20-exclusive-test", "30-lastuserjob-test"], + ) + + +class ReleaseBlockerSourceContracts(unittest.TestCase): + def read(self, path): + return (ROOT / path).read_text(encoding="utf-8") + + def test_login_guard_uses_platform_service_consistently(self): + source = self.read("roles/slurm/tasks/login-compute-setup.yml") + self.assertIn("ansible_os_family == 'RedHat'", source) + self.assertIn("systemctl show {{ ssh_unit }}", source) + self.assertIn("systemctl set-property {{ ssh_unit }}", source) + self.assertIn("system.control/{{ ssh_unit }}.d", source) + + def test_peer_memory_transitions_legacy_before_loading_in_tree_module(self): + source = self.read("roles/nvidia-peer-memory/tasks/main.yml") + stop = source.index("Stop and disable legacy nv_peer_mem service") + unload = source.index("Unload legacy nv_peer_mem module") + load = source.index("Load in-tree nvidia_peermem module") + self.assertLess(stop, unload) + self.assertLess(unload, load) + + def test_pyxis_uses_command_scoped_apparmor_not_global_sysctl(self): + tasks = self.read("roles/pyxis/tasks/main.yml") + profile = self.read("roles/pyxis/templates/enroot-nsenter.apparmor.j2") + self.assertNotIn("ansible.posix.sysctl", tasks) + self.assertIn("/etc/apparmor.d/enroot-nsenter", tasks) + self.assertIn("profile enroot-nsenter /usr/bin/enroot-nsenter", profile) + self.assertIn("userns,", profile) + + def test_dgx_dcgm_selects_and_verifies_cuda13_package(self): + defaults = self.read("roles/nvidia_dcgm/defaults/main.yml") + tasks = self.read("roles/nvidia_dcgm/tasks/install-dgx.yml") + self.assertIn("datacenter-gpu-manager-4-cuda13", defaults) + self.assertIn("ansible_distribution_version is version('24.04', '>=')", tasks) + self.assertIn("dcgm_dgx_selected_pkg_name in ansible_facts.packages", tasks) + + def test_slurm_dcgm_exporter_avoids_guarded_direct_driver_test(self): + cluster = self.read("playbooks/slurm-cluster.yml") + exporter = self.read("playbooks/slurm-cluster/nvidia-dcgm-exporter.yml") + driver = self.read("playbooks/nvidia-software/nvidia-driver.yml") + self.assertIn("nvidia_dcgm_exporter_driver_test_enabled: false", cluster) + self.assertIn("nvidia_dcgm_exporter_driver_test_enabled | default(true)", exporter) + self.assertIn("nvidia_driver_test_enabled | default(true) | bool", driver) + + def test_slurmd_exports_custom_prefix_for_enroot_hooks(self): + source = self.read("roles/slurm/tasks/service-files.yml") + self.assertIn("/etc/systemd/system/slurmd.service.d/10-slurm-path.conf", source) + self.assertIn("PATH={{ slurm_install_prefix }}/bin", source) + self.assertIn("{{ slurm_install_prefix }}/sbin", source) + + def test_exporter_mounts_custom_prefix_clients_on_path(self): + source = self.read( + "roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2" + ) + self.assertIn("--env PATH=/usr/local/bin:/usr/bin:/bin", source) + for command in ("sdiag", "sinfo", "squeue"): + self.assertIn(f"/bin/{command}:/usr/local/bin/{command}", source) + + +if __name__ == "__main__": + unittest.main()