From 0ebfebcf36a7cc4beac6a20a12fdcfdf768aa096 Mon Sep 17 00:00:00 2001 From: Abrar Shivani Date: Fri, 24 Jul 2026 18:02:19 -0700 Subject: [PATCH] feat(driver): mount vGPU licensing config as a directory for live updates The licensing Secret/ConfigMap is mounted into the driver container only via subPath mounts at /drivers/gridd.conf and /drivers/ClientConfigToken/client_configuration_token.tok. subPath mounts are resolved once when the container starts and kubelet explicitly excludes them from its atomic-symlink refresh, so an in-place update of the backing object (for example an NLS token rotated by ExternalSecrets/Vault) is never visible to a running nvidia-driver-daemonset pod. Recovering the new license today requires a `kubectl rollout restart`, which unloads the kernel module and evicts every GPU workload on the node. Add a third mount of the same licensing volume as a plain directory at /drivers/licensing-config with no subPath, so kubelet keeps its contents in sync while the pod runs. This gives the driver container entrypoint a live view of gridd.conf (and client_configuration_token.tok when NLS is enabled) that it can watch in order to restart nvidia-gridd in place instead of rolling the daemonset. The volume is reused rather than duplicated: the Items projection is identical, so a second volume would only add a redundant copy of the same projected keys and a second source of truth to keep in sync. Keys are still projected through Items so that extra keys carried by a user's Secret do not land in the mount. The two existing subPath mounts are retained unchanged. Every published driver image entrypoint copies from those exact paths, and removing them would break all of them. Applied to both the NVIDIADriver CR path (internal/state) and the ClusterPolicy path (controllers), which is still live whenever driver.useNvidiaDriverCRD is false (the default). Refs #2279 --- controllers/object_controls.go | 7 ++ controllers/transforms_test.go | 18 ++- internal/consts/consts.go | 4 + internal/state/driver_test.go | 105 ++++++++++++++++++ internal/state/driver_volumes.go | 8 ++ .../golden/driver-vgpu-licensing-secret.yaml | 6 +- .../golden/driver-vgpu-licensing.yaml | 6 +- 7 files changed, 146 insertions(+), 8 deletions(-) diff --git a/controllers/object_controls.go b/controllers/object_controls.go index c20387b9ae..1786c3cebd 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -3471,6 +3471,13 @@ func applyLicensingConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, nlsTokenVolMount) } + // Additionally mount the whole licensing volume as a directory. The subPath mounts above are + // resolved once at container start, so an in-place update of the Secret/ConfigMap is never + // visible to a running driver pod. A directory mount is kept in sync by kubelet, allowing the + // driver container to pick up a rotated license without a daemonset rollout. + licensingConfigDirVolMount := corev1.VolumeMount{Name: "licensing-config", ReadOnly: true, MountPath: consts.VGPULicensingConfigDirMountPath} + driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, licensingConfigDirVolMount) + var licensingConfigVolumeSource corev1.VolumeSource if config.Driver.LicensingConfig.SecretName != "" { licensingConfigVolumeSource = corev1.VolumeSource{ diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 9a0881072f..ffd563986c 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -3443,11 +3443,16 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { MountPath: consts.VGPULicensingConfigMountPath, SubPath: consts.VGPULicensingFileName, }, + { + Name: "licensing-config", + ReadOnly: true, + MountPath: consts.VGPULicensingConfigDirMountPath, + }, }, Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "1164839178", + Value: "838227286", }, }, }).WithInitContainer(corev1.Container{ @@ -3457,7 +3462,7 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "1164839178", + Value: "838227286", }, }, }).WithVolume(corev1.Volume{ @@ -3509,11 +3514,16 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { MountPath: consts.VGPULicensingConfigMountPath, SubPath: consts.VGPULicensingFileName, }, + { + Name: "licensing-config", + ReadOnly: true, + MountPath: consts.VGPULicensingConfigDirMountPath, + }, }, Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "3123249180", + Value: "14574464", }, }, }).WithInitContainer(corev1.Container{ @@ -3523,7 +3533,7 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "3123249180", + Value: "14574464", }, }, }).WithVolume(corev1.Volume{ diff --git a/internal/consts/consts.go b/internal/consts/consts.go index f28c507942..c2ad46487e 100644 --- a/internal/consts/consts.go +++ b/internal/consts/consts.go @@ -57,6 +57,10 @@ const ( NLSClientTokenMountPath = "/drivers/ClientConfigToken/client_configuration_token.tok" // NLSClientTokenFileName is the NLS client config token filename NLSClientTokenFileName = "client_configuration_token.tok" + // VGPULicensingConfigDirMountPath indicates the target mount path for the directory holding the + // vGPU licensing configuration files. This mount does not use a subPath, so kubelet keeps its + // contents in sync with the backing Secret/ConfigMap while the driver pod is running. + VGPULicensingConfigDirMountPath = "/drivers/licensing-config" // VGPUTopologyConfigMountPath indicates target mount path for vGPU topology daemon configuration file VGPUTopologyConfigMountPath = "/etc/nvidia/nvidia-topologyd.conf" // VGPUTopologyConfigFileName is the vGPU topology daemon configuration filename diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index f682de2ebb..5f53aa1dc3 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -606,6 +606,103 @@ func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) { } } +func TestDriverAdditionalConfigsVGPULicensing(t *testing.T) { + nlsDisabled := false + + testCases := []struct { + description string + licensingConfig *nvidiav1alpha1.DriverLicensingConfigSpec + expectTokenSubcfg bool + expectedItemKeys []string + }{ + { + description: "secret with NLS enabled mounts gridd.conf and the client token", + licensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{ + SecretName: "licensing-config-secret", + }, + expectTokenSubcfg: true, + expectedItemKeys: []string{consts.VGPULicensingFileName, consts.NLSClientTokenFileName}, + }, + { + description: "configmap with NLS disabled mounts only gridd.conf", + licensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{ + Name: "licensing-config-configmap", + NLSEnabled: &nlsDisabled, + }, + expectTokenSubcfg: false, + expectedItemKeys: []string{consts.VGPULicensingFileName}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + stateDriver := &stateDriver{ + stateSkel: stateSkel{ + client: fake.NewClientBuilder().WithScheme(scheme.Scheme).Build(), + namespace: "test-ns", + }, + } + driver := &nvidiav1alpha1.NVIDIADriver{} + driver.Spec.LicensingConfig = tc.licensingConfig + + configs, err := stateDriver.getDriverAdditionalConfigs( + context.Background(), + driver, + testClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}, + ) + require.NoError(t, err) + + mountsByPath := map[string]corev1.VolumeMount{} + for _, m := range configs.VolumeMounts { + if m.Name == "licensing-config" { + mountsByPath[m.MountPath] = m + } + } + + // The directory mount is what makes rotated licensing files visible to a running + // pod: kubelet only refreshes mounts that do not use a subPath. + dirMount, ok := mountsByPath[consts.VGPULicensingConfigDirMountPath] + require.True(t, ok, "expected a directory mount at %s", consts.VGPULicensingConfigDirMountPath) + assert.Empty(t, dirMount.SubPath, "directory mount must not set a subPath") + assert.True(t, dirMount.ReadOnly) + + // The pre-existing subPath mounts are retained so that driver images which copy from + // the legacy paths keep working. + griddMount, ok := mountsByPath[consts.VGPULicensingConfigMountPath] + require.True(t, ok, "expected the legacy gridd.conf subPath mount to be retained") + assert.Equal(t, consts.VGPULicensingFileName, griddMount.SubPath) + + tokenMount, ok := mountsByPath[consts.NLSClientTokenMountPath] + assert.Equal(t, tc.expectTokenSubcfg, ok, "NLS client token mount presence") + if tc.expectTokenSubcfg { + assert.Equal(t, consts.NLSClientTokenFileName, tokenMount.SubPath) + } + + var items []corev1.KeyToPath + for _, v := range configs.Volumes { + if v.Name != "licensing-config" { + continue + } + switch { + case v.Secret != nil: + items = v.Secret.Items + case v.ConfigMap != nil: + items = v.ConfigMap.Items + } + } + // Keys are projected explicitly so that extra keys carried by a user's Secret + // (for example one synced from Vault) never land in the mounted directory. + require.Len(t, items, len(tc.expectedItemKeys)) + var gotKeys []string + for _, item := range items { + gotKeys = append(gotKeys, item.Key) + } + assert.ElementsMatch(t, tc.expectedItemKeys, gotKeys) + }) + } +} + func TestDriverConfigPathHelpers(t *testing.T) { repoConfigPath, err := getRepoConfigPath("rhel") require.NoError(t, err) @@ -1014,6 +1111,10 @@ func TestDriverVGPULicensing(t *testing.T) { MountPath: "/drivers/ClientConfigToken/client_configuration_token.tok", SubPath: "client_configuration_token.tok", }, + { + Name: "licensing-config", + MountPath: "/drivers/licensing-config", + }, }, Volumes: []corev1.Volume{ { @@ -1084,6 +1185,10 @@ func TestDriverVGPULicensingSecret(t *testing.T) { MountPath: "/drivers/ClientConfigToken/client_configuration_token.tok", SubPath: "client_configuration_token.tok", }, + { + Name: "licensing-config", + MountPath: "/drivers/licensing-config", + }, }, Volumes: []corev1.Volume{ { diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index 327a3c5efd..6c9a749834 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -279,6 +279,14 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, nlsTokenVolMount) } + // Additionally mount the whole licensing volume as a directory. The subPath mounts above are + // resolved once at container start, so an in-place update of the Secret/ConfigMap is never + // visible to a running driver pod. A directory mount is kept in sync by kubelet, allowing the + // driver container to pick up a rotated license without a daemonset rollout. + licensingConfigDirVolMount := corev1.VolumeMount{Name: "licensing-config", ReadOnly: true, + MountPath: consts.VGPULicensingConfigDirMountPath} + additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, licensingConfigDirVolMount) + var licensingConfigVolumeSource corev1.VolumeSource if cr.Spec.LicensingConfig.SecretName != "" { licensingConfigVolumeSource = corev1.VolumeSource{ diff --git a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml index c7e5f1f51f..422770c5c4 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml @@ -185,7 +185,7 @@ spec: fieldRef: fieldPath: status.hostIP - name: DRIVER_CONFIG_DIGEST - value: "1341669320" + value: "2473959235" image: nvcr.io/nvidia/driver:525.85.03-ubuntu22.04 imagePullPolicy: IfNotPresent lifecycle: @@ -253,6 +253,8 @@ spec: - mountPath: /drivers/ClientConfigToken/client_configuration_token.tok name: licensing-config subPath: client_configuration_token.tok + - mountPath: /drivers/licensing-config + name: licensing-config hostPID: true initContainers: - args: @@ -283,7 +285,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: DRIVER_CONFIG_DIGEST - value: "1341669320" + value: "2473959235" image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager diff --git a/internal/state/testdata/golden/driver-vgpu-licensing.yaml b/internal/state/testdata/golden/driver-vgpu-licensing.yaml index ba82a165fe..b6cf82a79a 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing.yaml @@ -185,7 +185,7 @@ spec: fieldRef: fieldPath: status.hostIP - name: DRIVER_CONFIG_DIGEST - value: "1619279977" + value: "748513754" image: nvcr.io/nvidia/driver:525.85.03-ubuntu22.04 imagePullPolicy: IfNotPresent lifecycle: @@ -253,6 +253,8 @@ spec: - mountPath: /drivers/ClientConfigToken/client_configuration_token.tok name: licensing-config subPath: client_configuration_token.tok + - mountPath: /drivers/licensing-config + name: licensing-config hostPID: true initContainers: - args: @@ -283,7 +285,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: DRIVER_CONFIG_DIGEST - value: "1619279977" + value: "748513754" image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager