Mount vGPU licensing config as a directory for live updates

controllers/object_controls.go

@@ -3471,6 +3471,13 @@ func applyLicensingConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec
 		driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, nlsTokenVolMount)
 	}
 
+	// Additionally mount the whole licensing volume as a directory. The subPath mounts above are
+	// resolved once at container start, so an in-place update of the Secret/ConfigMap is never
+	// visible to a running driver pod. A directory mount is kept in sync by kubelet, allowing the
+	// driver container to pick up a rotated license without a daemonset rollout.
+	licensingConfigDirVolMount := corev1.VolumeMount{Name: "licensing-config", ReadOnly: true, MountPath: consts.VGPULicensingConfigDirMountPath}
+	driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, licensingConfigDirVolMount)
+
 	var licensingConfigVolumeSource corev1.VolumeSource
 	if config.Driver.LicensingConfig.SecretName != "" {
 		licensingConfigVolumeSource = corev1.VolumeSource{

controllers/transforms_test.go

@@ -3443,11 +3443,16 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) {
 						MountPath: consts.VGPULicensingConfigMountPath,
 						SubPath:   consts.VGPULicensingFileName,
 					},
+					{
+						Name:      "licensing-config",
+						ReadOnly:  true,
+						MountPath: consts.VGPULicensingConfigDirMountPath,
+					},
 				},
 				Env: []corev1.EnvVar{
 					{
 						Name:  "DRIVER_CONFIG_DIGEST",
-						Value: "1164839178",
+						Value: "838227286",
 					},
 				},
 			}).WithInitContainer(corev1.Container{
@@ -3457,7 +3462,7 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) {
 				Env: []corev1.EnvVar{
 					{
 						Name:  "DRIVER_CONFIG_DIGEST",
-						Value: "1164839178",
+						Value: "838227286",
 					},
 				},
 			}).WithVolume(corev1.Volume{
@@ -3509,11 +3514,16 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) {
 						MountPath: consts.VGPULicensingConfigMountPath,
 						SubPath:   consts.VGPULicensingFileName,
 					},
+					{
+						Name:      "licensing-config",
+						ReadOnly:  true,
+						MountPath: consts.VGPULicensingConfigDirMountPath,
+					},
 				},
 				Env: []corev1.EnvVar{
 					{
 						Name:  "DRIVER_CONFIG_DIGEST",
-						Value: "3123249180",
+						Value: "14574464",
 					},
 				},
 			}).WithInitContainer(corev1.Container{
@@ -3523,7 +3533,7 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) {
 				Env: []corev1.EnvVar{
 					{
 						Name:  "DRIVER_CONFIG_DIGEST",
-						Value: "3123249180",
+						Value: "14574464",
 					},
 				},
 			}).WithVolume(corev1.Volume{

internal/consts/consts.go

@@ -57,6 +57,10 @@ const (
 	NLSClientTokenMountPath = "/drivers/ClientConfigToken/client_configuration_token.tok"
 	// NLSClientTokenFileName is the NLS client config token filename
 	NLSClientTokenFileName = "client_configuration_token.tok"
+	// VGPULicensingConfigDirMountPath indicates the target mount path for the directory holding the
+	// vGPU licensing configuration files. This mount does not use a subPath, so kubelet keeps its
+	// contents in sync with the backing Secret/ConfigMap while the driver pod is running.
+	VGPULicensingConfigDirMountPath = "/drivers/licensing-config"
 	// VGPUTopologyConfigMountPath indicates target mount path for vGPU topology daemon configuration file
 	VGPUTopologyConfigMountPath = "/etc/nvidia/nvidia-topologyd.conf"
 	// VGPUTopologyConfigFileName is the vGPU topology daemon configuration filename

internal/state/driver_test.go

@@ -606,6 +606,103 @@ func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) {
 	}
 }
 
+func TestDriverAdditionalConfigsVGPULicensing(t *testing.T) {
+	nlsDisabled := false
+
+	testCases := []struct {
+		description       string
+		licensingConfig   *nvidiav1alpha1.DriverLicensingConfigSpec
+		expectTokenSubcfg bool
+		expectedItemKeys  []string
+	}{
+		{
+			description: "secret with NLS enabled mounts gridd.conf and the client token",
+			licensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{
+				SecretName: "licensing-config-secret",
+			},
+			expectTokenSubcfg: true,
+			expectedItemKeys:  []string{consts.VGPULicensingFileName, consts.NLSClientTokenFileName},
+		},
+		{
+			description: "configmap with NLS disabled mounts only gridd.conf",
+			licensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{
+				Name:       "licensing-config-configmap",
+				NLSEnabled: &nlsDisabled,
+			},
+			expectTokenSubcfg: false,
+			expectedItemKeys:  []string{consts.VGPULicensingFileName},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			stateDriver := &stateDriver{
+				stateSkel: stateSkel{
+					client:    fake.NewClientBuilder().WithScheme(scheme.Scheme).Build(),
+					namespace: "test-ns",
+				},
+			}
+			driver := &nvidiav1alpha1.NVIDIADriver{}
+			driver.Spec.LicensingConfig = tc.licensingConfig
+
+			configs, err := stateDriver.getDriverAdditionalConfigs(
+				context.Background(),
+				driver,
+				testClusterInfo{runtime: consts.Containerd},
+				nodePool{osRelease: "ubuntu", osVersion: "22.04"},
+			)
+			require.NoError(t, err)
+
+			mountsByPath := map[string]corev1.VolumeMount{}
+			for _, m := range configs.VolumeMounts {
+				if m.Name == "licensing-config" {
+					mountsByPath[m.MountPath] = m
+				}
+			}
+
+			// The directory mount is what makes rotated licensing files visible to a running
+			// pod: kubelet only refreshes mounts that do not use a subPath.
+			dirMount, ok := mountsByPath[consts.VGPULicensingConfigDirMountPath]
+			require.True(t, ok, "expected a directory mount at %s", consts.VGPULicensingConfigDirMountPath)
+			assert.Empty(t, dirMount.SubPath, "directory mount must not set a subPath")
+			assert.True(t, dirMount.ReadOnly)
+
+			// The pre-existing subPath mounts are retained so that driver images which copy from
+			// the legacy paths keep working.
+			griddMount, ok := mountsByPath[consts.VGPULicensingConfigMountPath]
+			require.True(t, ok, "expected the legacy gridd.conf subPath mount to be retained")
+			assert.Equal(t, consts.VGPULicensingFileName, griddMount.SubPath)
+
+			tokenMount, ok := mountsByPath[consts.NLSClientTokenMountPath]
+			assert.Equal(t, tc.expectTokenSubcfg, ok, "NLS client token mount presence")
+			if tc.expectTokenSubcfg {
+				assert.Equal(t, consts.NLSClientTokenFileName, tokenMount.SubPath)
+			}
+
+			var items []corev1.KeyToPath
+			for _, v := range configs.Volumes {
+				if v.Name != "licensing-config" {
+					continue
+				}
+				switch {
+				case v.Secret != nil:
+					items = v.Secret.Items
+				case v.ConfigMap != nil:
+					items = v.ConfigMap.Items
+				}
+			}
+			// Keys are projected explicitly so that extra keys carried by a user's Secret
+			// (for example one synced from Vault) never land in the mounted directory.
+			require.Len(t, items, len(tc.expectedItemKeys))
+			var gotKeys []string
+			for _, item := range items {
+				gotKeys = append(gotKeys, item.Key)
+			}
+			assert.ElementsMatch(t, tc.expectedItemKeys, gotKeys)
+		})
+	}
+}
+
 func TestDriverConfigPathHelpers(t *testing.T) {
 	repoConfigPath, err := getRepoConfigPath("rhel")
 	require.NoError(t, err)
@@ -1014,6 +1111,10 @@ func TestDriverVGPULicensing(t *testing.T) {
 				MountPath: "/drivers/ClientConfigToken/client_configuration_token.tok",
 				SubPath:   "client_configuration_token.tok",
 			},
+			{
+				Name:      "licensing-config",
+				MountPath: "/drivers/licensing-config",
+			},
 		},
 		Volumes: []corev1.Volume{
 			{
@@ -1084,6 +1185,10 @@ func TestDriverVGPULicensingSecret(t *testing.T) {
 				MountPath: "/drivers/ClientConfigToken/client_configuration_token.tok",
 				SubPath:   "client_configuration_token.tok",
 			},
+			{
+				Name:      "licensing-config",
+				MountPath: "/drivers/licensing-config",
+			},
 		},
 		Volumes: []corev1.Volume{
 			{

internal/state/driver_volumes.go

@@ -279,6 +279,14 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph
 			additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, nlsTokenVolMount)
 		}
 
+		// Additionally mount the whole licensing volume as a directory. The subPath mounts above are
+		// resolved once at container start, so an in-place update of the Secret/ConfigMap is never
+		// visible to a running driver pod. A directory mount is kept in sync by kubelet, allowing the
+		// driver container to pick up a rotated license without a daemonset rollout.
+		licensingConfigDirVolMount := corev1.VolumeMount{Name: "licensing-config", ReadOnly: true,
+			MountPath: consts.VGPULicensingConfigDirMountPath}
+		additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, licensingConfigDirVolMount)
+
 		var licensingConfigVolumeSource corev1.VolumeSource
 		if cr.Spec.LicensingConfig.SecretName != "" {
 			licensingConfigVolumeSource = corev1.VolumeSource{

internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml

@@ -185,7 +185,7 @@ spec:
             fieldRef:
               fieldPath: status.hostIP
         - name: DRIVER_CONFIG_DIGEST
-          value: "1341669320"
+          value: "2473959235"
         image: nvcr.io/nvidia/driver:525.85.03-ubuntu22.04
         imagePullPolicy: IfNotPresent
         lifecycle:
@@ -253,6 +253,8 @@ spec:
         - mountPath: /drivers/ClientConfigToken/client_configuration_token.tok
           name: licensing-config
           subPath: client_configuration_token.tok
+        - mountPath: /drivers/licensing-config
+          name: licensing-config
       hostPID: true
       initContainers:
       - args:
@@ -283,7 +285,7 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
         - name: DRIVER_CONFIG_DIGEST
-          value: "1341669320"
+          value: "2473959235"
         image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel
         imagePullPolicy: IfNotPresent
         name: k8s-driver-manager

internal/state/testdata/golden/driver-vgpu-licensing.yaml

@@ -185,7 +185,7 @@ spec:
             fieldRef:
               fieldPath: status.hostIP
         - name: DRIVER_CONFIG_DIGEST
-          value: "1619279977"
+          value: "748513754"
         image: nvcr.io/nvidia/driver:525.85.03-ubuntu22.04
         imagePullPolicy: IfNotPresent
         lifecycle:
@@ -253,6 +253,8 @@ spec:
         - mountPath: /drivers/ClientConfigToken/client_configuration_token.tok
           name: licensing-config
           subPath: client_configuration_token.tok
+        - mountPath: /drivers/licensing-config
+          name: licensing-config
       hostPID: true
       initContainers:
       - args:
@@ -283,7 +285,7 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
         - name: DRIVER_CONFIG_DIGEST
-          value: "1619279977"
+          value: "748513754"
         image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel
         imagePullPolicy: IfNotPresent
         name: k8s-driver-manager