diff --git a/controllers/object_controls.go b/controllers/object_controls.go index c20387b9a..1786c3ceb 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -3471,6 +3471,13 @@ func applyLicensingConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, nlsTokenVolMount) } + // Additionally mount the whole licensing volume as a directory. The subPath mounts above are + // resolved once at container start, so an in-place update of the Secret/ConfigMap is never + // visible to a running driver pod. A directory mount is kept in sync by kubelet, allowing the + // driver container to pick up a rotated license without a daemonset rollout. + licensingConfigDirVolMount := corev1.VolumeMount{Name: "licensing-config", ReadOnly: true, MountPath: consts.VGPULicensingConfigDirMountPath} + driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, licensingConfigDirVolMount) + var licensingConfigVolumeSource corev1.VolumeSource if config.Driver.LicensingConfig.SecretName != "" { licensingConfigVolumeSource = corev1.VolumeSource{ diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 9a0881072..ffd563986 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -3443,11 +3443,16 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { MountPath: consts.VGPULicensingConfigMountPath, SubPath: consts.VGPULicensingFileName, }, + { + Name: "licensing-config", + ReadOnly: true, + MountPath: consts.VGPULicensingConfigDirMountPath, + }, }, Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "1164839178", + Value: "838227286", }, }, }).WithInitContainer(corev1.Container{ @@ -3457,7 +3462,7 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "1164839178", + Value: "838227286", }, }, }).WithVolume(corev1.Volume{ @@ -3509,11 +3514,16 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { MountPath: consts.VGPULicensingConfigMountPath, SubPath: consts.VGPULicensingFileName, }, + { + Name: "licensing-config", + ReadOnly: true, + MountPath: consts.VGPULicensingConfigDirMountPath, + }, }, Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "3123249180", + Value: "14574464", }, }, }).WithInitContainer(corev1.Container{ @@ -3523,7 +3533,7 @@ func TestTransformDriverWithLicensingConfig(t *testing.T) { Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", - Value: "3123249180", + Value: "14574464", }, }, }).WithVolume(corev1.Volume{ diff --git a/internal/consts/consts.go b/internal/consts/consts.go index f28c50794..c2ad46487 100644 --- a/internal/consts/consts.go +++ b/internal/consts/consts.go @@ -57,6 +57,10 @@ const ( NLSClientTokenMountPath = "/drivers/ClientConfigToken/client_configuration_token.tok" // NLSClientTokenFileName is the NLS client config token filename NLSClientTokenFileName = "client_configuration_token.tok" + // VGPULicensingConfigDirMountPath indicates the target mount path for the directory holding the + // vGPU licensing configuration files. This mount does not use a subPath, so kubelet keeps its + // contents in sync with the backing Secret/ConfigMap while the driver pod is running. + VGPULicensingConfigDirMountPath = "/drivers/licensing-config" // VGPUTopologyConfigMountPath indicates target mount path for vGPU topology daemon configuration file VGPUTopologyConfigMountPath = "/etc/nvidia/nvidia-topologyd.conf" // VGPUTopologyConfigFileName is the vGPU topology daemon configuration filename diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index f682de2eb..5f53aa1dc 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -606,6 +606,103 @@ func TestDriverAdditionalConfigsSubscriptionMounts(t *testing.T) { } } +func TestDriverAdditionalConfigsVGPULicensing(t *testing.T) { + nlsDisabled := false + + testCases := []struct { + description string + licensingConfig *nvidiav1alpha1.DriverLicensingConfigSpec + expectTokenSubcfg bool + expectedItemKeys []string + }{ + { + description: "secret with NLS enabled mounts gridd.conf and the client token", + licensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{ + SecretName: "licensing-config-secret", + }, + expectTokenSubcfg: true, + expectedItemKeys: []string{consts.VGPULicensingFileName, consts.NLSClientTokenFileName}, + }, + { + description: "configmap with NLS disabled mounts only gridd.conf", + licensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{ + Name: "licensing-config-configmap", + NLSEnabled: &nlsDisabled, + }, + expectTokenSubcfg: false, + expectedItemKeys: []string{consts.VGPULicensingFileName}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + stateDriver := &stateDriver{ + stateSkel: stateSkel{ + client: fake.NewClientBuilder().WithScheme(scheme.Scheme).Build(), + namespace: "test-ns", + }, + } + driver := &nvidiav1alpha1.NVIDIADriver{} + driver.Spec.LicensingConfig = tc.licensingConfig + + configs, err := stateDriver.getDriverAdditionalConfigs( + context.Background(), + driver, + testClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}, + ) + require.NoError(t, err) + + mountsByPath := map[string]corev1.VolumeMount{} + for _, m := range configs.VolumeMounts { + if m.Name == "licensing-config" { + mountsByPath[m.MountPath] = m + } + } + + // The directory mount is what makes rotated licensing files visible to a running + // pod: kubelet only refreshes mounts that do not use a subPath. + dirMount, ok := mountsByPath[consts.VGPULicensingConfigDirMountPath] + require.True(t, ok, "expected a directory mount at %s", consts.VGPULicensingConfigDirMountPath) + assert.Empty(t, dirMount.SubPath, "directory mount must not set a subPath") + assert.True(t, dirMount.ReadOnly) + + // The pre-existing subPath mounts are retained so that driver images which copy from + // the legacy paths keep working. + griddMount, ok := mountsByPath[consts.VGPULicensingConfigMountPath] + require.True(t, ok, "expected the legacy gridd.conf subPath mount to be retained") + assert.Equal(t, consts.VGPULicensingFileName, griddMount.SubPath) + + tokenMount, ok := mountsByPath[consts.NLSClientTokenMountPath] + assert.Equal(t, tc.expectTokenSubcfg, ok, "NLS client token mount presence") + if tc.expectTokenSubcfg { + assert.Equal(t, consts.NLSClientTokenFileName, tokenMount.SubPath) + } + + var items []corev1.KeyToPath + for _, v := range configs.Volumes { + if v.Name != "licensing-config" { + continue + } + switch { + case v.Secret != nil: + items = v.Secret.Items + case v.ConfigMap != nil: + items = v.ConfigMap.Items + } + } + // Keys are projected explicitly so that extra keys carried by a user's Secret + // (for example one synced from Vault) never land in the mounted directory. + require.Len(t, items, len(tc.expectedItemKeys)) + var gotKeys []string + for _, item := range items { + gotKeys = append(gotKeys, item.Key) + } + assert.ElementsMatch(t, tc.expectedItemKeys, gotKeys) + }) + } +} + func TestDriverConfigPathHelpers(t *testing.T) { repoConfigPath, err := getRepoConfigPath("rhel") require.NoError(t, err) @@ -1014,6 +1111,10 @@ func TestDriverVGPULicensing(t *testing.T) { MountPath: "/drivers/ClientConfigToken/client_configuration_token.tok", SubPath: "client_configuration_token.tok", }, + { + Name: "licensing-config", + MountPath: "/drivers/licensing-config", + }, }, Volumes: []corev1.Volume{ { @@ -1084,6 +1185,10 @@ func TestDriverVGPULicensingSecret(t *testing.T) { MountPath: "/drivers/ClientConfigToken/client_configuration_token.tok", SubPath: "client_configuration_token.tok", }, + { + Name: "licensing-config", + MountPath: "/drivers/licensing-config", + }, }, Volumes: []corev1.Volume{ { diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index 327a3c5ef..6c9a74983 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -279,6 +279,14 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, nlsTokenVolMount) } + // Additionally mount the whole licensing volume as a directory. The subPath mounts above are + // resolved once at container start, so an in-place update of the Secret/ConfigMap is never + // visible to a running driver pod. A directory mount is kept in sync by kubelet, allowing the + // driver container to pick up a rotated license without a daemonset rollout. + licensingConfigDirVolMount := corev1.VolumeMount{Name: "licensing-config", ReadOnly: true, + MountPath: consts.VGPULicensingConfigDirMountPath} + additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, licensingConfigDirVolMount) + var licensingConfigVolumeSource corev1.VolumeSource if cr.Spec.LicensingConfig.SecretName != "" { licensingConfigVolumeSource = corev1.VolumeSource{ diff --git a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml index c7e5f1f51..422770c5c 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml @@ -185,7 +185,7 @@ spec: fieldRef: fieldPath: status.hostIP - name: DRIVER_CONFIG_DIGEST - value: "1341669320" + value: "2473959235" image: nvcr.io/nvidia/driver:525.85.03-ubuntu22.04 imagePullPolicy: IfNotPresent lifecycle: @@ -253,6 +253,8 @@ spec: - mountPath: /drivers/ClientConfigToken/client_configuration_token.tok name: licensing-config subPath: client_configuration_token.tok + - mountPath: /drivers/licensing-config + name: licensing-config hostPID: true initContainers: - args: @@ -283,7 +285,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: DRIVER_CONFIG_DIGEST - value: "1341669320" + value: "2473959235" image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager diff --git a/internal/state/testdata/golden/driver-vgpu-licensing.yaml b/internal/state/testdata/golden/driver-vgpu-licensing.yaml index ba82a165f..b6cf82a79 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing.yaml @@ -185,7 +185,7 @@ spec: fieldRef: fieldPath: status.hostIP - name: DRIVER_CONFIG_DIGEST - value: "1619279977" + value: "748513754" image: nvcr.io/nvidia/driver:525.85.03-ubuntu22.04 imagePullPolicy: IfNotPresent lifecycle: @@ -253,6 +253,8 @@ spec: - mountPath: /drivers/ClientConfigToken/client_configuration_token.tok name: licensing-config subPath: client_configuration_token.tok + - mountPath: /drivers/licensing-config + name: licensing-config hostPID: true initContainers: - args: @@ -283,7 +285,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: DRIVER_CONFIG_DIGEST - value: "1619279977" + value: "748513754" image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager