Skip to content

Apply validator resources to init containers #2557

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Shivkumar13 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Apply validator resources to init containers #2557

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 24 additions & 12 deletions controllers/object_controls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -965,7 +965,7 @@ func TransformGPUDiscoveryPlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol
}

// apply plugin configuration through ConfigMap if one is provided
err = handleDevicePluginConfig(obj, config)
err = handleDevicePluginConfig(obj, config, config.GPUFeatureDiscovery.Resources)
if err != nil {
return err
}
Expand Down Expand Up @@ -1016,7 +1016,7 @@ func TransformDCGMExporterService(obj *corev1.Service, config *gpuv1.ClusterPoli
// TransformDriver transforms Nvidia driver daemonset with required config as per ClusterPolicy
func TransformDriver(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
// update driver-manager initContainer
err := transformDriverManagerInitContainer(obj, &config.Driver.Manager, config.Driver.GPUDirectRDMA)
err := transformDriverManagerInitContainer(obj, &config.Driver.Manager, config.Driver.GPUDirectRDMA, config.Driver.Resources)
if err != nil {
return err
}
Expand Down Expand Up @@ -1094,7 +1094,7 @@ func TransformDriver(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n C
// TransformVGPUManager transforms NVIDIA vGPU Manager daemonset with required config as per ClusterPolicy
func TransformVGPUManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
// update k8s-driver-manager initContainer
err := transformDriverManagerInitContainer(obj, &config.VGPUManager.DriverManager, nil)
err := transformDriverManagerInitContainer(obj, &config.VGPUManager.DriverManager, nil, config.VGPUManager.Resources)
if err != nil {
return fmt.Errorf("failed to transform k8s-driver-manager initContainer for vGPU Manager: %v", err)
}
Expand Down Expand Up @@ -1594,7 +1594,7 @@ func TransformDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
}

// apply plugin configuration through ConfigMap if one is provided
err = handleDevicePluginConfig(obj, config)
err = handleDevicePluginConfig(obj, config, config.DevicePlugin.Resources)
if err != nil {
return err
}
Expand Down Expand Up @@ -1653,6 +1653,7 @@ func TransformMPSControlDaemon(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolic
if initCtr := findContainerByName(obj.Spec.Template.Spec.InitContainers, "mps-control-daemon-mounts"); initCtr != nil {
initCtr.Image = image
initCtr.ImagePullPolicy = imagePullPolicy
applyResourceRequirements(initCtr, config.DevicePlugin.Resources)
}

// update image path and imagePullPolicy for main container
Expand All @@ -1678,7 +1679,7 @@ func TransformMPSControlDaemon(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolic
}

// apply plugin configuration through ConfigMap if one is provided
err = handleDevicePluginConfig(obj, config)
err = handleDevicePluginConfig(obj, config, config.DevicePlugin.Resources)
if err != nil {
return err
}
Expand Down Expand Up @@ -2101,7 +2102,7 @@ func TransformMIGManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,
// TransformVFIOManager transforms VFIO-PCI Manager daemonset with required config as per ClusterPolicy
func TransformVFIOManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
// update k8s-driver-manager initContainer
err := transformDriverManagerInitContainer(obj, &config.VFIOManager.DriverManager, nil)
err := transformDriverManagerInitContainer(obj, &config.VFIOManager.DriverManager, nil, config.VFIOManager.Resources)
if err != nil {
return fmt.Errorf("failed to transform k8s-driver-manager initContainer for VFIO Manager: %v", err)
}
Expand Down Expand Up @@ -2279,6 +2280,14 @@ func transformValidatorSecurityContext(ctr *corev1.Container) {
ctr.SecurityContext.RunAsUser = rootUID
}

func applyResourceRequirements(ctr *corev1.Container, resources *gpuv1.ResourceRequirements) {
if resources == nil {
return
}
ctr.Resources.Requests = resources.Requests
ctr.Resources.Limits = resources.Limits
}

// TransformValidator transforms nvidia-operator-validator daemonset with required config as per ClusterPolicy
func TransformValidator(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
err := TransformValidatorShared(obj, config)
Expand Down Expand Up @@ -2370,8 +2379,7 @@ func TransformValidatorShared(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
if config.Validator.Resources != nil {
// apply resource limits to all containers
for i := range obj.Spec.Template.Spec.Containers {
obj.Spec.Template.Spec.Containers[i].Resources.Requests = config.Validator.Resources.Requests
obj.Spec.Template.Spec.Containers[i].Resources.Limits = config.Validator.Resources.Limits
applyResourceRequirements(&obj.Spec.Template.Spec.Containers[i], config.Validator.Resources)
}
}
// set arguments if specified for validator container
Expand Down Expand Up @@ -2407,6 +2415,7 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev
if config.Validator.ImagePullPolicy != "" {
podSpec.InitContainers[i].ImagePullPolicy = gpuv1.ImagePullPolicy(config.Validator.ImagePullPolicy)
}
applyResourceRequirements(&podSpec.InitContainers[i], config.Validator.Resources)
// update the security context for the validator container
transformValidatorSecurityContext(&podSpec.InitContainers[i])

Expand Down Expand Up @@ -2783,7 +2792,7 @@ func addSharedMountsForPluginConfig(container *corev1.Container, config *gpuv1.D
}

// apply spec changes to make custom configurations provided via a ConfigMap available to all containers
func handleDevicePluginConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec) error {
func handleDevicePluginConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, resources *gpuv1.ResourceRequirements) error {
if !isCustomPluginConfigSet(config.DevicePlugin.Config) {
// remove config-manager-init container
for i, initContainer := range obj.Spec.Template.Spec.InitContainers {
Expand Down Expand Up @@ -2830,7 +2839,7 @@ func handleDevicePluginConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, createEmptyDirVolume("config"))

// apply env/volume changes to initContainer
err := transformConfigManagerInitContainer(obj, config)
err := transformConfigManagerInitContainer(obj, config, resources)
if err != nil {
return err
}
Expand All @@ -2842,7 +2851,7 @@ func handleDevicePluginConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
return nil
}

func transformConfigManagerInitContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec) error {
func transformConfigManagerInitContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, resources *gpuv1.ResourceRequirements) error {
initContainer := findContainerByName(obj.Spec.Template.Spec.InitContainers, "config-manager-init")
if initContainer == nil {
// config-manager-init container is not added to the spec, this is a no-op
Expand All @@ -2856,6 +2865,7 @@ func transformConfigManagerInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Cl
if config.DevicePlugin.ImagePullPolicy != "" {
initContainer.ImagePullPolicy = gpuv1.ImagePullPolicy(config.DevicePlugin.ImagePullPolicy)
}
applyResourceRequirements(initContainer, resources)
// setup env
setContainerEnv(initContainer, "DEFAULT_CONFIG", config.DevicePlugin.Config.Default)
setContainerEnv(initContainer, "FALLBACK_STRATEGIES", "empty")
Expand Down Expand Up @@ -2894,7 +2904,7 @@ func transformConfigManagerSidecarContainer(obj *appsv1.DaemonSet, config *gpuv1
return nil
}

func transformDriverManagerInitContainer(obj *appsv1.DaemonSet, driverManagerSpec *gpuv1.DriverManagerSpec, rdmaSpec *gpuv1.GPUDirectRDMASpec) error {
func transformDriverManagerInitContainer(obj *appsv1.DaemonSet, driverManagerSpec *gpuv1.DriverManagerSpec, rdmaSpec *gpuv1.GPUDirectRDMASpec, resources *gpuv1.ResourceRequirements) error {
container := findContainerByName(obj.Spec.Template.Spec.InitContainers, "k8s-driver-manager")

if container == nil {
Expand All @@ -2910,6 +2920,7 @@ func transformDriverManagerInitContainer(obj *appsv1.DaemonSet, driverManagerSpe
if driverManagerSpec.ImagePullPolicy != "" {
container.ImagePullPolicy = gpuv1.ImagePullPolicy(driverManagerSpec.ImagePullPolicy)
}
applyResourceRequirements(container, resources)

if rdmaSpec != nil && rdmaSpec.IsEnabled() {
setContainerEnv(container, GPUDirectRDMAEnabledEnvName, "true")
Expand Down Expand Up @@ -3899,6 +3910,7 @@ func transformValidationInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Clust
if config.Validator.ImagePullPolicy != "" {
obj.Spec.Template.Spec.InitContainers[i].ImagePullPolicy = gpuv1.ImagePullPolicy(config.Validator.ImagePullPolicy)
}
applyResourceRequirements(&obj.Spec.Template.Spec.InitContainers[i], config.Validator.Resources)
// update the security context for the validator container
transformValidatorSecurityContext(&obj.Spec.Template.Spec.InitContainers[i])
}
Expand Down
Loading
Loading