fix: harden GPU runtime convergence

docs/slurm-cluster/README.md

@@ -240,6 +240,12 @@ DeepOps no longer bundles an Open OnDemand installer. Sites that want the [Open
 
 [Pyxis](https://github.com/NVIDIA/pyxis) and [Enroot](https://github.com/NVIDIA/enroot) are installed by default and can be disabled by setting `slurm_install_enroot` and `slurm_install_pyxis` to no. They are the supported, release-validated container runtime for Slurm in DeepOps.
 
+On Ubuntu systems that restrict unprivileged user namespaces, DeepOps installs
+a command-scoped AppArmor profile for `/usr/bin/enroot-nsenter`. This preserves
+the host-wide security default while allowing Pyxis jobs to create their user
+namespace. Set `pyxis_configure_enroot_apparmor_userns: false` only when site
+policy provides an equivalent profile.
+
 The DeepOps Singularity wrapper role has been retired. Singularity lives on upstream as [Apptainer](https://apptainer.org/); sites that still want it can install Apptainer/Singularity separately.
 
 ## Large deployments

playbooks/nvidia-software/nvidia-driver.yml

@@ -29,4 +29,5 @@
       when:
         - ansible_local['gpus']['count']
         - is_dgx.stat.exists == False
+        - nvidia_driver_test_enabled | default(true) | bool
   environment: "{{proxy_env if proxy_env is defined else{}}}"

playbooks/slurm-cluster.yml

@@ -123,6 +123,10 @@
 
 - name: Install monitoring exporters | Install nvidia dcgm exporter
   import_playbook: slurm-cluster/nvidia-dcgm-exporter.yml
+  vars:
+    # The driver was already validated before Slurm enabled the direct-login
+    # GPU guard. The final allocated srun validator is the authoritative proof.
+    nvidia_dcgm_exporter_driver_test_enabled: false
   when: slurm_enable_monitoring
 
 - name: Set up rsyslog forwarding from compute nodes to head node | Install rsyslog server

playbooks/slurm-cluster/nvidia-dcgm-exporter.yml

@@ -4,6 +4,8 @@
 
 - name: Install NVIDIA driver
   import_playbook: ../nvidia-software/nvidia-driver.yml
+  vars:
+    nvidia_driver_test_enabled: "{{ nvidia_dcgm_exporter_driver_test_enabled | default(true) }}"
 
 - name: Install NVIDIA container runtime
   import_playbook: ../container/nvidia-docker.yml

roles/nvidia-peer-memory/tasks/main.yml

@@ -4,19 +4,76 @@
     path: /etc/dgx-release
   register: is_dgx
 
-- name: Autoinstall DKMS modules
+- name: Check for in-tree nvidia_peermem module
+  command: modinfo nvidia_peermem
+  register: nvidia_peermem_info
+  failed_when: false
+  changed_when: false
+  when:
+    - ansible_local['gpus']['count']
+    - is_dgx.stat.exists
+
+- name: Collect service state before selecting peer-memory implementation
+  service_facts:
+  when:
+    - ansible_local['gpus']['count']
+    - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) == 0
+
+- name: Stop and disable legacy nv_peer_mem service
+  service:
+    name: nv_peer_mem
+    state: stopped
+    enabled: false
+  when:
+    - ansible_local['gpus']['count']
+    - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) == 0
+    - "'nv_peer_mem.service' in ansible_facts.services or 'nv_peer_mem' in ansible_facts.services"
+
+- name: Check whether legacy nv_peer_mem module is loaded
+  stat:
+    path: /sys/module/nv_peer_mem
+  register: nv_peer_mem_loaded
+  when:
+    - ansible_local['gpus']['count']
+    - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) == 0
+
+- name: Unload legacy nv_peer_mem module before using nvidia_peermem
+  modprobe:
+    name: nv_peer_mem
+    state: absent
+  when:
+    - ansible_local['gpus']['count']
+    - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) == 0
+    - nv_peer_mem_loaded.stat.exists | default(false)
+
+- name: Load in-tree nvidia_peermem module
+  modprobe:
+    name: nvidia_peermem
+    state: present
+  when:
+    - ansible_local['gpus']['count']
+    - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) == 0
+
+- name: Autoinstall legacy nv_peer_mem DKMS module
   command: dkms autoinstall
   when:
     - ansible_local['gpus']['count']
     - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) != 0
 
-- name: Modprobe nv_peer_mem
+- name: Load legacy nv_peer_mem module
   modprobe:
     name: nv_peer_mem
     state: present
   when:
     - ansible_local['gpus']['count']
     - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) != 0
 
 - name: Start nv_peer_mem service
   service:
@@ -25,3 +82,4 @@
   when:
     - ansible_local['gpus']['count']
     - is_dgx.stat.exists
+    - nvidia_peermem_info.rc | default(1) != 0

roles/nvidia_dcgm/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
 dcgm_pkg_name: "datacenter-gpu-manager"
+dcgm_dgx_cuda13_pkg_name: "datacenter-gpu-manager-4-cuda13"
 
 # RedHat family
 epel_package: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm"

roles/nvidia_dcgm/tasks/install-dgx.yml

@@ -1,5 +1,24 @@
 ---
-- name: install DCGM from repos
+- name: Select DGX-compatible DCGM package
+  set_fact:
+    dcgm_dgx_selected_pkg_name: >-
+      {{ dcgm_dgx_cuda13_pkg_name
+         if ansible_distribution == 'Ubuntu' and
+            ansible_distribution_version is version('24.04', '>=')
+         else dcgm_pkg_name }}
+
+- name: Install the selected DGX-compatible DCGM package
   package:
-    name: "datacenter-gpu-manager"
+    name: "{{ dcgm_dgx_selected_pkg_name }}"
     state: present
+
+- name: Refresh package facts after DCGM convergence
+  package_facts:
+    manager: auto
+
+- name: Verify the selected DGX-compatible DCGM package is installed
+  assert:
+    that:
+      - dcgm_dgx_selected_pkg_name in ansible_facts.packages
+    fail_msg: >-
+      Required DGX DCGM package {{ dcgm_dgx_selected_pkg_name }} is not installed

roles/prometheus-slurm-exporter/templates/docker.slurm-exporter.service.j2

@@ -12,7 +12,7 @@ ExecStartPre=-/usr/bin/docker rm %n
 {% if not slurm_exporter_build_image %}
 ExecStartPre=/usr/bin/docker pull {{ slurm_exporter_container }}
 {% endif %}
-ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ slurm_install_prefix }}/bin/sdiag:{{ slurm_install_prefix }}/bin/sdiag -v {{ slurm_install_prefix }}/bin/sinfo:{{ slurm_install_prefix }}/bin/sinfo -v {{ slurm_install_prefix }}/bin/squeue:{{ slurm_install_prefix }}/bin/squeue -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }}
+ExecStart=/usr/bin/docker run --rm --network host --name %n --env PATH=/usr/local/bin:/usr/bin:/bin -v {{ slurm_install_prefix }}/bin/sdiag:/usr/local/bin/sdiag:ro -v {{ slurm_install_prefix }}/bin/sinfo:/usr/local/bin/sinfo:ro -v {{ slurm_install_prefix }}/bin/squeue:/usr/local/bin/squeue:ro -v /etc/slurm:/etc/slurm:ro -v {{ slurm_install_prefix }}/lib:{{ slurm_install_prefix }}/lib:ro -v /etc/hosts:/etc/hosts:ro -v /var/run/munge:/var/run/munge:ro {{ slurm_exporter_container }}
 
 [Install]
 WantedBy=multi-user.target

roles/pyxis/defaults/main.yml

@@ -9,10 +9,16 @@ is_compute: no
 
 pyxis_ubuntu_deps:
 - "bsdmainutils"
+- "apparmor"
 
 pyxis_el_deps:
 - "util-linux"
 
+# Ubuntu restricts unprivileged user namespaces when this kernel knob exists.
+# Install a command-scoped AppArmor profile instead of weakening the host-wide
+# setting. Disable only when site policy supplies an equivalent profile.
+pyxis_configure_enroot_apparmor_userns: true
+
 # /run is default partition of pyxis runtime_path
 resize_run_partition: false
 

roles/pyxis/tasks/main.yml

@@ -13,6 +13,38 @@
   with_items: "{{ pyxis_el_deps }}"
   when: ansible_os_family == "RedHat"
 
+- name: Check for AppArmor unprivileged user namespace restriction
+  stat:
+    path: /proc/sys/kernel/apparmor_restrict_unprivileged_userns
+  register: apparmor_userns_knob
+  when:
+    - ansible_distribution == "Ubuntu"
+    - is_compute
+    - pyxis_configure_enroot_apparmor_userns | bool
+
+- name: Install command-scoped AppArmor profile for enroot-nsenter
+  template:
+    src: enroot-nsenter.apparmor.j2
+    dest: /etc/apparmor.d/enroot-nsenter
+    owner: root
+    group: root
+    mode: "0644"
+  register: enroot_apparmor_profile
+  when:
+    - ansible_distribution == "Ubuntu"
+    - is_compute
+    - pyxis_configure_enroot_apparmor_userns | bool
+    - apparmor_userns_knob.stat.exists | default(false)
+
+- name: Load command-scoped AppArmor profile for enroot-nsenter
+  command: apparmor_parser -r /etc/apparmor.d/enroot-nsenter
+  changed_when: enroot_apparmor_profile.changed
+  when:
+    - ansible_distribution == "Ubuntu"
+    - is_compute
+    - pyxis_configure_enroot_apparmor_userns | bool
+    - apparmor_userns_knob.stat.exists | default(false)
+
 - name: install slurm-pmi hook
   file:
     path: /etc/enroot/hooks.d/50-slurm-pmi.sh

roles/pyxis/templates/enroot-nsenter.apparmor.j2

@@ -0,0 +1,10 @@
+# This profile grants only enroot-nsenter the user namespace permission that
+# Ubuntu otherwise restricts for unconfined applications.
+
+include <tunables/global>
+
+profile enroot-nsenter /usr/bin/enroot-nsenter flags=(unconfined) {
+  userns,
+
+  include if exists <local/enroot-nsenter>
+}

roles/slurm/tasks/login-compute-setup.yml

@@ -1,18 +1,18 @@
 ---
 
-- name: Hide GPUs for regular user logins via sshd.service.
+- name: Hide GPUs for regular user logins via the platform SSH service
+  vars:
+    ssh_unit: "{{ 'sshd.service' if ansible_os_family == 'RedHat' else 'ssh.service' }}"
   shell: |
     set -o pipefail
-    deviceprop=$(systemctl show sshd.service -p DeviceAllow | grep -i nvidiactl)
+    deviceprop=$(systemctl show {{ ssh_unit }} -p DeviceAllow | grep -i nvidiactl)
 
     if [ -z "$deviceprop" ] ; then
-      systemctl set-property sshd.service DeviceAllow="/dev/nvidiactl"
+      systemctl set-property {{ ssh_unit }} DeviceAllow="/dev/nvidiactl"
     fi
   args:
     executable: /bin/bash
-    creates: "{{ '/etc/systemd/system.control/sshd.service.d/50-DeviceAllow.conf' \
-                 if ansible_os_family == 'RedHat' else \
-                 '/etc/systemd/system.control/ssh.service.d/50-DeviceAllow.conf' }}"
+    creates: "/etc/systemd/system.control/{{ ssh_unit }}.d/50-DeviceAllow.conf"
   when: is_controller
   tags:
     - config

roles/slurm/tasks/service-files.yml

@@ -21,3 +21,24 @@
     - slurmd.service
   when: is_compute
   failed_when: false
+
+- name: Create slurmd systemd override directory
+  file:
+    path: /etc/systemd/system/slurmd.service.d
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when: is_compute
+
+- name: Put custom-prefix Slurm clients on the slurmd PATH
+  copy:
+    dest: /etc/systemd/system/slurmd.service.d/10-slurm-path.conf
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      [Service]
+      Environment="PATH={{ slurm_install_prefix }}/bin:{{ slurm_install_prefix }}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  when: is_compute
+  notify: restart slurmd

roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh

@@ -2,6 +2,7 @@
 # This could all be done with run-parts using regexes on ubuntu.
 # However, centos' version of run-parts is just a simple bash script with no useful flags.
 set -e
+set -o pipefail
 
 if [ "$#" -ne 1 ]; then
     echo "Usage: $0 parts_dir"
@@ -13,17 +14,37 @@ log () {
     logger -s -t slurm "$@"
 }
 
-# Find out if we are running in exclusive mode
+# Use the configured prefix because slurmd does not inherit it in PATH.
+squeue_bin="{{ slurm_install_prefix }}/bin/squeue"
+
+# Find out if we are running in exclusive mode. Failed or incomplete scheduler
+# queries fail closed to non-exclusive while ordinary scripts still run.
 exclusive=0
-numcpus_sys=$(( $(grep -c ^processor /proc/cpuinfo) * $(scontrol show job "$SLURM_JOBID" | grep -Eio "TRES=.*node=[0-9]+" | cut -d= -f5) ))
-numcpus_job=$(scontrol show job "$SLURM_JOBID" | grep -Eio "TRES=cpu=[0-9]+" | cut -d= -f3)
-if [ "$numcpus_sys" == "$numcpus_job" ] ; then
+numcpus_job=""
+numnodes_job=""
+if ! numcpus_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %C 2>/dev/null); then
+    log "Unable to query allocated CPUs for job $SLURM_JOBID; treating it as non-exclusive."
+fi
+if ! numnodes_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %D 2>/dev/null); then
+    log "Unable to query allocated nodes for job $SLURM_JOBID; treating it as non-exclusive."
+fi
+if [[ "$numcpus_job" =~ ^[0-9]+$ ]] &&
+   [[ "$numnodes_job" =~ ^[1-9][0-9]*$ ]] &&
+   [ $(( $(grep -c ^processor /proc/cpuinfo) * numnodes_job )) -eq "$numcpus_job" ]; then
     exclusive=1
 fi
 
 # Find out if there are any more jobs on this node for this user
 last_user_job=0
-num_jobs=$(squeue -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running | wc -l)
+num_jobs=1
+if ! num_jobs=$("$squeue_bin" -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running | wc -l); then
+    log "Unable to query remaining jobs for $SLURM_JOB_USER; preserving last-user cleanup state."
+    num_jobs=1
+fi
+if [[ ! "$num_jobs" =~ ^[0-9]+$ ]]; then
+    log "Invalid remaining-job count for $SLURM_JOB_USER; preserving last-user cleanup state."
+    num_jobs=1
+fi
 if [ "$num_jobs" -eq 0 ]; then
     last_user_job=1
 fi