diff --git a/controllers/object_controls.go b/controllers/object_controls.go index ee38e6ef72..e35c073f41 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -1018,7 +1018,7 @@ func TransformDCGMExporterService(obj *corev1.Service, config *gpuv1.ClusterPoli // TransformDriver transforms Nvidia driver daemonset with required config as per ClusterPolicy func TransformDriver(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error { // update driver-manager initContainer - err := transformDriverManagerInitContainer(obj, &config.Driver.Manager, config.Driver.GPUDirectRDMA) + err := transformDriverManagerInitContainer(obj, &config.Driver.Manager, config.Driver.GPUDirectRDMA, config.Driver.Resources) if err != nil { return err } @@ -1096,7 +1096,7 @@ func TransformDriver(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n C // TransformVGPUManager transforms NVIDIA vGPU Manager daemonset with required config as per ClusterPolicy func TransformVGPUManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error { // update k8s-driver-manager initContainer - err := transformDriverManagerInitContainer(obj, &config.VGPUManager.DriverManager, nil) + err := transformDriverManagerInitContainer(obj, &config.VGPUManager.DriverManager, nil, config.VGPUManager.Resources) if err != nil { return fmt.Errorf("failed to transform k8s-driver-manager initContainer for vGPU Manager: %v", err) } @@ -1655,6 +1655,7 @@ func TransformMPSControlDaemon(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolic if initCtr := findContainerByName(obj.Spec.Template.Spec.InitContainers, "mps-control-daemon-mounts"); initCtr != nil { initCtr.Image = image initCtr.ImagePullPolicy = imagePullPolicy + applyResourceRequirements(initCtr, config.DevicePlugin.Resources) } // update image path and imagePullPolicy for main container @@ -2103,7 +2104,7 @@ func TransformMIGManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, // TransformVFIOManager transforms VFIO-PCI Manager daemonset with required config as per ClusterPolicy func TransformVFIOManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error { // update k8s-driver-manager initContainer - err := transformDriverManagerInitContainer(obj, &config.VFIOManager.DriverManager, nil) + err := transformDriverManagerInitContainer(obj, &config.VFIOManager.DriverManager, nil, config.VFIOManager.Resources) if err != nil { return fmt.Errorf("failed to transform k8s-driver-manager initContainer for VFIO Manager: %v", err) } @@ -2281,6 +2282,14 @@ func transformValidatorSecurityContext(ctr *corev1.Container) { ctr.SecurityContext.RunAsUser = rootUID } +func applyResourceRequirements(ctr *corev1.Container, resources *gpuv1.ResourceRequirements) { + if resources == nil { + return + } + ctr.Resources.Requests = resources.Requests + ctr.Resources.Limits = resources.Limits +} + // TransformValidator transforms nvidia-operator-validator daemonset with required config as per ClusterPolicy func TransformValidator(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error { err := TransformValidatorShared(obj, config) @@ -2372,8 +2381,7 @@ func TransformValidatorShared(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy if config.Validator.Resources != nil { // apply resource limits to all containers for i := range obj.Spec.Template.Spec.Containers { - obj.Spec.Template.Spec.Containers[i].Resources.Requests = config.Validator.Resources.Requests - obj.Spec.Template.Spec.Containers[i].Resources.Limits = config.Validator.Resources.Limits + applyResourceRequirements(&obj.Spec.Template.Spec.Containers[i], config.Validator.Resources) } } // set arguments if specified for validator container @@ -2409,6 +2417,7 @@ func TransformValidatorComponent(config *gpuv1.ClusterPolicySpec, podSpec *corev if config.Validator.ImagePullPolicy != "" { podSpec.InitContainers[i].ImagePullPolicy = gpuv1.ImagePullPolicy(config.Validator.ImagePullPolicy) } + applyResourceRequirements(&podSpec.InitContainers[i], config.Validator.Resources) // update the security context for the validator container transformValidatorSecurityContext(&podSpec.InitContainers[i]) @@ -2858,6 +2867,7 @@ func transformConfigManagerInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Cl if config.DevicePlugin.ImagePullPolicy != "" { initContainer.ImagePullPolicy = gpuv1.ImagePullPolicy(config.DevicePlugin.ImagePullPolicy) } + applyResourceRequirements(initContainer, configManagerResourceRequirements(obj, config)) // setup env setContainerEnv(initContainer, "DEFAULT_CONFIG", config.DevicePlugin.Config.Default) setContainerEnv(initContainer, "FALLBACK_STRATEGIES", "empty") @@ -2867,6 +2877,13 @@ func transformConfigManagerInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Cl return nil } +func configManagerResourceRequirements(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec) *gpuv1.ResourceRequirements { + if findContainerByName(obj.Spec.Template.Spec.Containers, "gpu-feature-discovery") != nil { + return config.GPUFeatureDiscovery.Resources + } + return config.DevicePlugin.Resources +} + func transformConfigManagerSidecarContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec) error { var container *corev1.Container for i := range obj.Spec.Template.Spec.Containers { @@ -2896,7 +2913,7 @@ func transformConfigManagerSidecarContainer(obj *appsv1.DaemonSet, config *gpuv1 return nil } -func transformDriverManagerInitContainer(obj *appsv1.DaemonSet, driverManagerSpec *gpuv1.DriverManagerSpec, rdmaSpec *gpuv1.GPUDirectRDMASpec) error { +func transformDriverManagerInitContainer(obj *appsv1.DaemonSet, driverManagerSpec *gpuv1.DriverManagerSpec, rdmaSpec *gpuv1.GPUDirectRDMASpec, resources *gpuv1.ResourceRequirements) error { container := findContainerByName(obj.Spec.Template.Spec.InitContainers, "k8s-driver-manager") if container == nil { @@ -2912,6 +2929,7 @@ func transformDriverManagerInitContainer(obj *appsv1.DaemonSet, driverManagerSpe if driverManagerSpec.ImagePullPolicy != "" { container.ImagePullPolicy = gpuv1.ImagePullPolicy(driverManagerSpec.ImagePullPolicy) } + applyResourceRequirements(container, resources) if rdmaSpec != nil && rdmaSpec.IsEnabled() { setContainerEnv(container, GPUDirectRDMAEnabledEnvName, "true") @@ -3901,6 +3919,7 @@ func transformValidationInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Clust if config.Validator.ImagePullPolicy != "" { obj.Spec.Template.Spec.InitContainers[i].ImagePullPolicy = gpuv1.ImagePullPolicy(config.Validator.ImagePullPolicy) } + applyResourceRequirements(&obj.Spec.Template.Spec.InitContainers[i], config.Validator.Resources) // update the security context for the validator container transformValidatorSecurityContext(&obj.Spec.Template.Spec.InitContainers[i]) } diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 9a0881072f..fcbb6732a1 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -1220,6 +1220,17 @@ func TestTransformDevicePlugin(t *testing.T) { } func TestTransformMPSControlDaemon(t *testing.T) { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + testCases := []struct { description string daemonset Daemonset @@ -1241,6 +1252,10 @@ func TestTransformMPSControlDaemon(t *testing.T) { ImagePullPolicy: string(corev1.PullAlways), ImagePullSecrets: []string{"secret"}, MPS: &gpuv1.MPSConfig{Root: "/var/mps"}, + Resources: &gpuv1.ResourceRequirements{ + Limits: resources.Limits, + Requests: resources.Requests, + }, }, }, expectedDaemonset: NewDaemonset(). @@ -1248,11 +1263,13 @@ func TestTransformMPSControlDaemon(t *testing.T) { Name: "mps-control-daemon-mounts", Image: "nvcr.io/mps:latest", ImagePullPolicy: corev1.PullAlways, + Resources: resources, }). WithContainer(corev1.Container{ Name: "mps-control-daemon-ctr", Image: "nvcr.io/mps:latest", ImagePullPolicy: corev1.PullAlways, + Resources: resources, Env: []corev1.EnvVar{ {Name: "NVIDIA_MIG_MONITOR_DEVICES", Value: "all"}, }, @@ -2074,6 +2091,7 @@ func TestTransformVFIOManager(t *testing.T) { Image: "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v1.0.0", ImagePullPolicy: corev1.PullIfNotPresent, Env: mockEnvCore, + Resources: resources, }). WithPullSecret(secret), }, @@ -2227,6 +2245,17 @@ func TestTransformVGPUDeviceManager(t *testing.T) { } func TestTransformValidationInitContainer(t *testing.T) { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + testCases := []struct { description string ds Daemonset @@ -2246,6 +2275,10 @@ func TestTransformValidationInitContainer(t *testing.T) { Version: "v1.0.0", ImagePullPolicy: "IfNotPresent", ImagePullSecrets: []string{"pull-secret"}, + Resources: &gpuv1.ResourceRequirements{ + Limits: resources.Limits, + Requests: resources.Requests, + }, Driver: gpuv1.DriverValidatorSpec{ Env: []gpuv1.EnvVar{{Name: "foo", Value: "bar"}}, }, @@ -2259,6 +2292,7 @@ func TestTransformValidationInitContainer(t *testing.T) { Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", ImagePullPolicy: corev1.PullIfNotPresent, Env: []corev1.EnvVar{{Name: "foo", Value: "bar"}}, + Resources: resources, SecurityContext: &corev1.SecurityContext{ RunAsUser: rootUID, }, @@ -2267,6 +2301,7 @@ func TestTransformValidationInitContainer(t *testing.T) { Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0", ImagePullPolicy: corev1.PullIfNotPresent, Env: []corev1.EnvVar{{Name: "foo", Value: "bar"}}, + Resources: resources, SecurityContext: &corev1.SecurityContext{ RunAsUser: rootUID, }, @@ -2289,7 +2324,140 @@ func newBoolPtr(b bool) *bool { return boolPtr } +func TestTransformConfigManagerInitContainerWithResources(t *testing.T) { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ds := NewDaemonset(). + WithInitContainer(corev1.Container{Name: "config-manager-init"}). + WithInitContainer(corev1.Container{Name: "dummy"}) + cpSpec := &gpuv1.ClusterPolicySpec{ + DevicePlugin: gpuv1.DevicePluginSpec{ + Repository: "nvcr.io/nvidia", + Image: "k8s-device-plugin", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Resources: &gpuv1.ResourceRequirements{ + Limits: resources.Limits, + Requests: resources.Requests, + }, + Config: &gpuv1.DevicePluginConfig{ + Name: "plugin-config", + Default: "default", + }, + }, + } + expectedDs := NewDaemonset(). + WithInitContainer(corev1.Container{ + Name: "config-manager-init", + Image: "nvcr.io/nvidia/k8s-device-plugin:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Resources: resources, + Env: []corev1.EnvVar{ + {Name: "DEFAULT_CONFIG", Value: "default"}, + {Name: "FALLBACK_STRATEGIES", Value: "empty"}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "config", MountPath: "/config"}, + {Name: "plugin-config", MountPath: "/available-configs"}, + }, + }). + WithInitContainer(corev1.Container{Name: "dummy"}) + + err := transformConfigManagerInitContainer(ds.DaemonSet, cpSpec) + require.NoError(t, err) + require.EqualValues(t, expectedDs, ds) +} + +func TestHandleDevicePluginConfigWithGPUFeatureDiscoveryResources(t *testing.T) { + gfdResources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + devicePluginResources := &gpuv1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + } + + testCases := []struct { + name string + devicePluginResources *gpuv1.ResourceRequirements + }{ + { + name: "only GPU Feature Discovery resources configured", + }, + { + name: "both component resources configured", + devicePluginResources: devicePluginResources, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ds := NewDaemonset(). + WithContainer(corev1.Container{Name: "gpu-feature-discovery"}). + WithInitContainer(corev1.Container{Name: "config-manager-init"}) + cpSpec := &gpuv1.ClusterPolicySpec{ + GPUFeatureDiscovery: gpuv1.GPUFeatureDiscoverySpec{ + Resources: &gpuv1.ResourceRequirements{ + Limits: gfdResources.Limits, + Requests: gfdResources.Requests, + }, + }, + DevicePlugin: gpuv1.DevicePluginSpec{ + Repository: "nvcr.io/nvidia", + Image: "k8s-device-plugin", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Resources: tc.devicePluginResources, + Config: &gpuv1.DevicePluginConfig{ + Name: "plugin-config", + Default: "default", + }, + }, + } + + err := handleDevicePluginConfig(ds.DaemonSet, cpSpec) + require.NoError(t, err) + + initContainer := findContainerByName(ds.Spec.Template.Spec.InitContainers, "config-manager-init") + require.NotNil(t, initContainer) + require.EqualValues(t, gfdResources, initContainer.Resources) + }) + } +} + func TestTransformDriverManagerInitContainer(t *testing.T) { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + testCases := []struct { description string ds Daemonset @@ -2315,6 +2483,10 @@ func TestTransformDriverManagerInitContainer(t *testing.T) { Enabled: newBoolPtr(true), UseHostMOFED: newBoolPtr(true), }, + Resources: &gpuv1.ResourceRequirements{ + Limits: resources.Limits, + Requests: resources.Requests, + }, }, }, expectedDs: NewDaemonset().WithInitContainer(corev1.Container{ @@ -2326,13 +2498,14 @@ func TestTransformDriverManagerInitContainer(t *testing.T) { {Name: UseHostMOFEDEnvName, Value: "true"}, {Name: "foo", Value: "bar"}, }, + Resources: resources, }).WithInitContainer(corev1.Container{Name: "dummy"}).WithPullSecret("pull-secret"), }, } for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - err := transformDriverManagerInitContainer(tc.ds.DaemonSet, &tc.cpSpec.Driver.Manager, tc.cpSpec.Driver.GPUDirectRDMA) + err := transformDriverManagerInitContainer(tc.ds.DaemonSet, &tc.cpSpec.Driver.Manager, tc.cpSpec.Driver.GPUDirectRDMA, tc.cpSpec.Driver.Resources) require.NoError(t, err) require.EqualValues(t, tc.expectedDs, tc.ds) }) @@ -2709,6 +2882,52 @@ func TestTransformValidatorComponent(t *testing.T) { } } +func TestTransformValidatorComponentWithResources(t *testing.T) { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + } + + cpSpec := &gpuv1.ClusterPolicySpec{ + Validator: gpuv1.ValidatorSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "gpu-operator-validator", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Resources: &gpuv1.ResourceRequirements{ + Limits: resources.Limits, + Requests: resources.Requests, + }, + }, + } + + testCases := []struct { + component string + containerName string + }{ + {component: "driver", containerName: "driver-validation"}, + {component: "toolkit", containerName: "toolkit-validation"}, + {component: "cuda", containerName: "cuda-validation"}, + {component: "plugin", containerName: "plugin-validation"}, + } + + for _, tc := range testCases { + t.Run(tc.containerName, func(t *testing.T) { + pod := NewPod().WithInitContainer(corev1.Container{Name: tc.containerName}) + err := TransformValidatorComponent(cpSpec, &pod.Spec, tc.component) + require.NoError(t, err) + require.Len(t, pod.Spec.InitContainers, 1) + require.EqualValues(t, resources, pod.Spec.InitContainers[0].Resources) + }) + } +} + func TestTransformValidator(t *testing.T) { testCases := []struct { description string @@ -3667,6 +3886,10 @@ func TestTransformDriverWithResources(t *testing.T) { }).WithInitContainer(corev1.Container{ Name: "k8s-driver-manager", Image: "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.8.0", + Resources: corev1.ResourceRequirements{ + Requests: resources.Requests, + Limits: resources.Limits, + }, Env: []corev1.EnvVar{ { Name: "DRIVER_CONFIG_DIGEST", @@ -4009,6 +4232,16 @@ func TestTransformVGPUManager(t *testing.T) { } mockClient := fake.NewFakeClient(node, kernelModuleConfigMap) + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } testCases := []struct { description string @@ -4028,6 +4261,10 @@ func TestTransformVGPUManager(t *testing.T) { Image: "vgpu-manager", Version: "550.90.07", ImagePullPolicy: "IfNotPresent", + Resources: &gpuv1.ResourceRequirements{ + Limits: resources.Limits, + Requests: resources.Requests, + }, DriverManager: gpuv1.DriverManagerSpec{ Repository: "nvcr.io/nvidia/cloud-native", Image: "k8s-driver-manager", @@ -4061,6 +4298,7 @@ func TestTransformVGPUManager(t *testing.T) { container := findContainerByName(tc.daemonset.Spec.Template.Spec.Containers, "nvidia-vgpu-manager-ctr") require.NotNil(t, container) require.Len(t, container.VolumeMounts, 1) + require.EqualValues(t, resources, container.Resources) require.Equal(t, corev1.VolumeMount{ Name: "vgpu-kernel-module-config", ReadOnly: true, @@ -4085,6 +4323,10 @@ func TestTransformVGPUManager(t *testing.T) { }, }, }, tc.daemonset.Spec.Template.Spec.Volumes[0]) + + driverManager := findContainerByName(tc.daemonset.Spec.Template.Spec.InitContainers, "k8s-driver-manager") + require.NotNil(t, driverManager) + require.EqualValues(t, resources, driverManager.Resources) }) } } diff --git a/internal/state/dra_driver.go b/internal/state/dra_driver.go index 4812af2e01..88621d4aec 100644 --- a/internal/state/dra_driver.go +++ b/internal/state/dra_driver.go @@ -22,10 +22,12 @@ import ( "sort" "strings" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + nvidiav1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1" nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" "github.com/NVIDIA/gpu-operator/internal/image" ) @@ -149,13 +151,67 @@ func getDRADriverSpec(spec *nvidiav1alpha1.DRADriverSpec) (*draDriverSpec, error return nil, fmt.Errorf("failed to construct DRA driver validator image path: %w", err) } + resourceRequirements := []*nvidiav1.ResourceRequirements{ + spec.GPUs.KubeletPlugin.Resources, + } + if spec.IsComputeDomainsEnabled() { + resourceRequirements = append(resourceRequirements, spec.ComputeDomains.KubeletPlugin.Resources) + } + return &draDriverSpec{ - Spec: spec, - ImagePath: imagePath, - InitImagePath: initImagePath, + Spec: spec, + ImagePath: imagePath, + InitImagePath: initImagePath, + InitContainerResources: maxResourceRequirements(resourceRequirements...), }, nil } +func maxResourceRequirements(requirements ...*nvidiav1.ResourceRequirements) *nvidiav1.ResourceRequirements { + result := &nvidiav1.ResourceRequirements{} + + for _, requirement := range requirements { + if requirement == nil { + continue + } + result.Requests = maxResourceList(result.Requests, requirement.Requests) + result.Limits = maxResourceList(result.Limits, requirement.Limits) + } + + // A request-only container can have a larger request than another container's + // explicit limit. Keep the merged requirement valid without adding limits for + // resources that were unbounded in every source container. + for name, request := range result.Requests { + limit, ok := result.Limits[name] + if ok && request.Cmp(limit) > 0 { + result.Limits[name] = request.DeepCopy() + } + } + + if len(result.Requests) == 0 && len(result.Limits) == 0 { + return nil + } + return result +} + +func maxResourceList(resourceLists ...corev1.ResourceList) corev1.ResourceList { + var result corev1.ResourceList + + for _, resourceList := range resourceLists { + for name, quantity := range resourceList { + current, ok := result[name] + if ok && quantity.Cmp(current) <= 0 { + continue + } + if result == nil { + result = corev1.ResourceList{} + } + result[name] = quantity.DeepCopy() + } + } + + return result +} + // renderDRAFeatureGates renders the feature-gate map as the FEATURE_GATES env value // (comma-separated Key=Value). Keys are sorted so the rendered value is a pure // function of the input and reconciles do not churn the pod spec. Empty when none. diff --git a/internal/state/dra_driver_test.go b/internal/state/dra_driver_test.go index ce23b68dbe..af1af168dc 100644 --- a/internal/state/dra_driver_test.go +++ b/internal/state/dra_driver_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -179,6 +180,104 @@ func TestDRADriverHealthcheckPortOverride(t *testing.T) { assert.Equal(t, int32(52000), gpus.LivenessProbe.GRPC.Port) } +func TestDRADriverValidationResources(t *testing.T) { + s := newTestDRAState(t) + cr := fullSpecGPUCluster() + + objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) + require.NoError(t, err) + + initCtr := findDaemonSet(t, objs).Spec.Template.Spec.InitContainers[0] + assert.Equal(t, "driver-validation", initCtr.Name) + assert.Equal(t, corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + }, initCtr.Resources) +} + +func TestDRADriverValidationResourcesIgnoreDisabledComputeDomains(t *testing.T) { + s := newTestDRAState(t) + cr := sampleGPUCluster() + cr.Spec.DRADriver.GPUs.KubeletPlugin.Resources = &nvidiav1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + } + cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Resources = &nvidiav1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + } + + objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) + require.NoError(t, err) + + initCtr := findDaemonSet(t, objs).Spec.Template.Spec.InitContainers[0] + assert.Equal(t, corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, initCtr.Resources) +} + +func TestMaxResourceRequirements(t *testing.T) { + t.Run("raises an explicit limit to the maximum request", func(t *testing.T) { + actual := maxResourceRequirements( + &nvidiav1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + &nvidiav1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, + ) + + assert.Equal(t, &nvidiav1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, actual) + }) + + t.Run("keeps a request-only resource unbounded", func(t *testing.T) { + actual := maxResourceRequirements(&nvidiav1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }) + + assert.Equal(t, &nvidiav1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, actual) + }) +} + func TestDRADriverHealthcheckDisabled(t *testing.T) { s := newTestDRAState(t) cr := sampleGPUCluster() diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index 6583f5df18..70b41e4b5a 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -149,6 +149,36 @@ func TestDriverRenderMinimal(t *testing.T) { require.Equal(t, string(o), actual) } +func TestDriverManagerResources(t *testing.T) { + state, err := NewStateDriver(nil, "", nil, manifestDir) + require.NoError(t, err) + stateDriver, ok := state.(*stateDriver) + require.True(t, ok) + + renderData := getMinimalDriverRenderData() + objs, err := stateDriver.renderer.RenderObjects( + &render.TemplatingData{ + Data: renderData, + }) + require.NoError(t, err) + + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + + var driverManager *corev1.Container + for i := range ds.Spec.Template.Spec.InitContainers { + if ds.Spec.Template.Spec.InitContainers[i].Name == "k8s-driver-manager" { + driverManager = &ds.Spec.Template.Spec.InitContainers[i] + break + } + } + require.NotNil(t, driverManager) + require.EqualValues(t, corev1.ResourceRequirements{ + Requests: renderData.Driver.Spec.Resources.Requests, + Limits: renderData.Driver.Spec.Resources.Limits, + }, driverManager.Resources) +} + func TestDriverHostSysDevicesSystemVolumeUsesStableParentDirectory(t *testing.T) { state, err := NewStateDriver(nil, "", nil, manifestDir) require.Nil(t, err) diff --git a/internal/state/gpucluster_render_test.go b/internal/state/gpucluster_render_test.go index 85ce06f022..98f057396d 100644 --- a/internal/state/gpucluster_render_test.go +++ b/internal/state/gpucluster_render_test.go @@ -34,12 +34,16 @@ import ( // fullSpecGPUCluster returns a CR exercising the optional DRA driver knobs: the // computeDomains capability with controller/kubelet-plugin overrides and per-container -// env/resources on the gpus kubelet plugin. +// env/resources on both kubelet plugins. func fullSpecGPUCluster() *nvidiav1alpha1.GPUCluster { cr := sampleGPUCluster() cr.Spec.DRADriver.GPUs.KubeletPlugin = nvidiav1alpha1.DRADriverKubeletPluginSpec{ Env: []nvidiav1.EnvVar{{Name: "GPUS_EXTRA", Value: "1"}}, Resources: &nvidiav1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("128Mi"), @@ -54,6 +58,16 @@ func fullSpecGPUCluster() *nvidiav1alpha1.GPUCluster { }, KubeletPlugin: nvidiav1alpha1.DRADriverKubeletPluginSpec{ Env: []nvidiav1.EnvVar{{Name: "CD_PLUGIN_EXTRA", Value: "1"}}, + Resources: &nvidiav1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + }, }, } return cr diff --git a/internal/state/testdata/golden/driver-additional-configs.yaml b/internal/state/testdata/golden/driver-additional-configs.yaml index ec1050d929..8ab73e8256 100644 --- a/internal/state/testdata/golden/driver-additional-configs.yaml +++ b/internal/state/testdata/golden/driver-additional-configs.yaml @@ -297,6 +297,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-full-spec.yaml b/internal/state/testdata/golden/driver-full-spec.yaml index 7400e6420e..ac60cba8da 100644 --- a/internal/state/testdata/golden/driver-full-spec.yaml +++ b/internal/state/testdata/golden/driver-full-spec.yaml @@ -314,6 +314,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: Always name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 200Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml b/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml index 23e62a5d59..55333dfa93 100644 --- a/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml +++ b/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml @@ -470,6 +470,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-gdrcopy.yaml b/internal/state/testdata/golden/driver-gdrcopy.yaml index 433ef59a60..4891c81db3 100644 --- a/internal/state/testdata/golden/driver-gdrcopy.yaml +++ b/internal/state/testdata/golden/driver-gdrcopy.yaml @@ -351,6 +351,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-gds.yaml b/internal/state/testdata/golden/driver-gds.yaml index 36a510e81f..e90b1f72da 100644 --- a/internal/state/testdata/golden/driver-gds.yaml +++ b/internal/state/testdata/golden/driver-gds.yaml @@ -351,6 +351,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-hostnetwork.yaml b/internal/state/testdata/golden/driver-hostnetwork.yaml index 43c22906fa..88e5fdb910 100644 --- a/internal/state/testdata/golden/driver-hostnetwork.yaml +++ b/internal/state/testdata/golden/driver-hostnetwork.yaml @@ -290,6 +290,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-minimal.yaml b/internal/state/testdata/golden/driver-minimal.yaml index 4ff360b3e6..a3c3713224 100644 --- a/internal/state/testdata/golden/driver-minimal.yaml +++ b/internal/state/testdata/golden/driver-minimal.yaml @@ -288,6 +288,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml b/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml index eceb052ad1..3a02e62dbe 100644 --- a/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml +++ b/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml @@ -406,6 +406,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-precompiled.yaml b/internal/state/testdata/golden/driver-precompiled.yaml index 5ca79c0c89..81759a8174 100644 --- a/internal/state/testdata/golden/driver-precompiled.yaml +++ b/internal/state/testdata/golden/driver-precompiled.yaml @@ -290,6 +290,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-rdma-hostmofed.yaml b/internal/state/testdata/golden/driver-rdma-hostmofed.yaml index 737965fbca..914d0eb3cd 100644 --- a/internal/state/testdata/golden/driver-rdma-hostmofed.yaml +++ b/internal/state/testdata/golden/driver-rdma-hostmofed.yaml @@ -371,6 +371,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-rdma.yaml b/internal/state/testdata/golden/driver-rdma.yaml index 87fc074372..91c339381f 100644 --- a/internal/state/testdata/golden/driver-rdma.yaml +++ b/internal/state/testdata/golden/driver-rdma.yaml @@ -365,6 +365,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-secret-env.yaml b/internal/state/testdata/golden/driver-secret-env.yaml index 99266d999d..3940e775f2 100644 --- a/internal/state/testdata/golden/driver-secret-env.yaml +++ b/internal/state/testdata/golden/driver-secret-env.yaml @@ -383,6 +383,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-vgpu-host-manager-openshift.yaml b/internal/state/testdata/golden/driver-vgpu-host-manager-openshift.yaml index 12729f37cd..7eef073fb6 100644 --- a/internal/state/testdata/golden/driver-vgpu-host-manager-openshift.yaml +++ b/internal/state/testdata/golden/driver-vgpu-host-manager-openshift.yaml @@ -368,6 +368,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-vgpu-host-manager.yaml b/internal/state/testdata/golden/driver-vgpu-host-manager.yaml index aa23b92995..69d9515e12 100644 --- a/internal/state/testdata/golden/driver-vgpu-host-manager.yaml +++ b/internal/state/testdata/golden/driver-vgpu-host-manager.yaml @@ -275,6 +275,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml index c6bf3d4a57..0e6210d0c9 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml @@ -294,6 +294,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/driver-vgpu-licensing.yaml b/internal/state/testdata/golden/driver-vgpu-licensing.yaml index 8ae457f732..a9498afd58 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing.yaml @@ -294,6 +294,13 @@ spec: image: nvcr.io/nvidia/cloud-native/k8s-driver-manager:devel imagePullPolicy: IfNotPresent name: k8s-driver-manager + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 100Mi securityContext: privileged: true volumeMounts: diff --git a/internal/state/testdata/golden/gpucluster-dra-driver-full-spec.yaml b/internal/state/testdata/golden/gpucluster-dra-driver-full-spec.yaml index ce7aaaf386..5a7d888b4c 100644 --- a/internal/state/testdata/golden/gpucluster-dra-driver-full-spec.yaml +++ b/internal/state/testdata/golden/gpucluster-dra-driver-full-spec.yaml @@ -491,6 +491,9 @@ spec: timeoutSeconds: 10 name: gpus resources: + limits: + cpu: 500m + memory: 256Mi requests: cpu: 100m memory: 128Mi @@ -582,6 +585,13 @@ spec: periodSeconds: 10 timeoutSeconds: 10 name: compute-domains + resources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 200m + memory: 64Mi securityContext: privileged: true startupProbe: @@ -634,6 +644,13 @@ spec: image: nvcr.io/nvidia/gpu-operator-validator:test imagePullPolicy: IfNotPresent name: driver-validation + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 200m + memory: 128Mi securityContext: privileged: true runAsUser: 0 diff --git a/internal/state/types.go b/internal/state/types.go index b880eba915..086b9bfb07 100644 --- a/internal/state/types.go +++ b/internal/state/types.go @@ -52,13 +52,13 @@ type gdrcopyDriverSpec struct { ImagePath string } -// draDriverSpec is a wrapper of DRADriverSpec with the fully-qualified image paths -// populated: ImagePath for the DRA driver containers and InitImagePath for the -// driver-validation init container (shipped in the gpu-operator image). +// draDriverSpec is a wrapper of DRADriverSpec with derived values used to render +// the DRA driver containers and the driver-validation init container. type draDriverSpec struct { - Spec *nvidiav1alpha1.DRADriverSpec - ImagePath string - InitImagePath string + Spec *nvidiav1alpha1.DRADriverSpec + ImagePath string + InitImagePath string + InitContainerResources *nvidiav1.ResourceRequirements } // dcgmSpec is a wrapper of DCGMSpec with the resolved image path. diff --git a/manifests/state-dra-driver/0500_daemonset.yaml b/manifests/state-dra-driver/0500_daemonset.yaml index c824d064f1..adba4fec2d 100644 --- a/manifests/state-dra-driver/0500_daemonset.yaml +++ b/manifests/state-dra-driver/0500_daemonset.yaml @@ -90,6 +90,9 @@ spec: {{- end }} command: - nvidia-validator + {{- if .DRADriver.InitContainerResources }} + resources: {{ .DRADriver.InitContainerResources | toJson }} + {{- end }} securityContext: privileged: true # The validator ships in the gpu-operator image (USER 65532); it must run diff --git a/manifests/state-driver/0500_daemonset.yaml b/manifests/state-driver/0500_daemonset.yaml index a8ff6084c3..8fc46657f2 100644 --- a/manifests/state-driver/0500_daemonset.yaml +++ b/manifests/state-driver/0500_daemonset.yaml @@ -180,6 +180,9 @@ spec: {{- end }} securityContext: privileged: true + {{- if .Driver.Spec.Resources }} + resources: {{ .Driver.Spec.Resources | yaml | nindent 12 }} + {{- end }} volumeMounts: - name: run-nvidia mountPath: /run/nvidia