Support NVIDIADriver reconciliation and driver upgrades without a ClusterPolicy

cmd/gpu-operator/main.go

@@ -193,11 +193,12 @@ func main() {
 		WithRestartOnlyPredicate(predicates.DriverPodRestartOnly(upgradeLogger))
 
 	if err = (&controllers.UpgradeReconciler{
-		Client:          mgr.GetClient(),
-		Log:             upgradeLogger,
-		Scheme:          mgr.GetScheme(),
-		StateManager:    clusterUpgradeStateManager,
-		OperatorMetrics: operatorMetrics,
+		Client:            mgr.GetClient(),
+		Log:               upgradeLogger,
+		Scheme:            mgr.GetScheme(),
+		StateManager:      clusterUpgradeStateManager,
+		OperatorMetrics:   operatorMetrics,
+		OperatorNamespace: operatorNamespace,
 	}).SetupWithManager(ctx, mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Upgrade")
 		os.Exit(1)

controllers/gpucluster_controller_test.go

@@ -71,14 +71,17 @@ func newGPUClusterReconciler(t *testing.T, objs ...client.Object) (*GPUClusterRe
 }
 
 // fakeStateManager returns canned SyncState results so the controller tests don't load
-// real manifests. GetWatchSources is promoted from the embedded (nil) interface and is
-// never called here — only SetupWithManager calls it, which these tests skip.
+// real manifests. It records the last info catalog passed to SyncState so tests can
+// assert on its entries. GetWatchSources is promoted from the embedded (nil) interface
+// and is never called here — only SetupWithManager calls it, which these tests skip.
 type fakeStateManager struct {
 	state.Manager
-	results state.Results
+	results     state.Results
+	lastCatalog state.InfoCatalog
 }
 
-func (f *fakeStateManager) SyncState(_ context.Context, _ interface{}, _ state.InfoCatalog) state.Results {
+func (f *fakeStateManager) SyncState(_ context.Context, _ interface{}, catalog state.InfoCatalog) state.Results {
+	f.lastCatalog = catalog
 	return f.results
 }
 

controllers/nvidiadriver_controller.go

@@ -64,6 +64,7 @@ type NVIDIADriverReconciler struct {
 //+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers/finalizers,verbs=update
+//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -98,45 +99,26 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return reconcile.Result{}, nil
 	}
 
-	// Get the singleton NVIDIA ClusterPolicy object in the cluster.
-	clusterPolicyList := &gpuv1.ClusterPolicyList{}
-	if err := r.List(ctx, clusterPolicyList); err != nil {
-		wrappedErr := fmt.Errorf("error getting ClusterPolicy list: %w", err)
-		logger.Error(err, "error getting ClusterPolicy list")
-		instance.Status.State = nvidiav1alpha1.NotReady
-		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
-			logger.Error(condErr, "failed to set condition")
-		}
-		return reconcile.Result{}, wrappedErr
-	}
-
-	if len(clusterPolicyList.Items) == 0 {
-		err := fmt.Errorf("no ClusterPolicy object found in the cluster")
-		logger.Error(err, "failed to get ClusterPolicy object")
+	// Source the cluster-wide host root from the active configuration: a ClusterPolicy takes
+	// precedence, otherwise the controller runs standalone against the GPUCluster.
+	clusterPolicy, gpuCluster, err := resolveActiveConfig(ctx, r.Client)
+	if err != nil {
+		logger.Error(err, "error resolving active cluster configuration")
 		instance.Status.State = nvidiav1alpha1.NotReady
 		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
 			logger.Error(condErr, "failed to set condition")
 		}
 		return reconcile.Result{}, err
 	}
-	clusterPolicyInstance := clusterPolicyList.Items[0]
 
 	// Ensure the NVIDIADriver CR has a consumer: either the ClusterPolicy delegates its
 	// driver to the NVIDIADriver CRD, or a GPUCluster exists. GPUCluster does
 	// not manage the driver itself — it is either preinstalled on the host (no NVIDIADriver
 	// CR) or installed via NVIDIADriver CRs, so any CR that exists alongside one is in use.
-	if !clusterPolicyInstance.Spec.Driver.UseNvidiaDriverCRDType() {
-		gpuClusters := &nvidiav1alpha1.GPUClusterList{}
-		if err := r.List(ctx, gpuClusters); err != nil {
-			wrappedErr := fmt.Errorf("error getting GPUCluster list: %w", err)
-			logger.Error(err, "error getting GPUCluster list")
-			instance.Status.State = nvidiav1alpha1.NotReady
-			if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
-				logger.Error(condErr, "failed to set condition")
-			}
-			return reconcile.Result{}, wrappedErr
-		}
-		if len(gpuClusters.Items) == 0 {
+	var hostRoot string
+	switch {
+	case clusterPolicy != nil:
+		if !clusterPolicy.Spec.Driver.UseNvidiaDriverCRDType() && gpuCluster == nil {
 			msg := "useNvidiaDriverCRD is not enabled in ClusterPolicy and no GPUCluster exists"
 			logger.V(consts.LogLevelWarning).Info("NVIDIADriver reconciliation skipped", "reason", msg)
 			instance.Status.State = nvidiav1alpha1.Disabled
@@ -145,6 +127,17 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			}
 			return reconcile.Result{}, nil
 		}
+		hostRoot = clusterPolicy.Spec.HostPaths.RootFS
+	case gpuCluster != nil:
+		hostRoot = gpuCluster.Spec.HostPaths.RootFS
+	default:
+		err := fmt.Errorf("no ClusterPolicy or GPUCluster object found in the cluster")
+		logger.Error(err, "failed to get a cluster-wide configuration object")
+		instance.Status.State = nvidiav1alpha1.NotReady
+		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
+			logger.Error(condErr, "failed to set condition")
+		}
+		return reconcile.Result{}, err
 	}
 
 	// Create a new InfoCatalog which is a generic interface for passing information to state managers
@@ -153,8 +146,8 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	// Add an entry for ClusterInfo, which was collected before the NVIDIADriver controller was started
 	infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)
 
-	// Add an entry for Clusterpolicy, which is needed to deploy the driver daemonset
-	infoCatalog.Add(state.InfoTypeClusterPolicyCR, clusterPolicyInstance)
+	// Add the host root, which is needed to deploy the driver daemonset
+	infoCatalog.Add(state.InfoTypeHostRoot, hostRoot)
 
 	// Verify the nodeSelector configured for this NVIDIADriver instance does
 	// not conflict with any other instances. This ensures only one driver
@@ -405,6 +398,7 @@ func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
 	gpuClusterMapFn := func(ctx context.Context, _ *nvidiav1alpha1.GPUCluster) []reconcile.Request {
 		return r.enqueueAllNVIDIADrivers(ctx)
 	}
+
 	err = c.Watch(
 		source.Kind(
 			mgr.GetCache(),

controllers/nvidiadriver_controller_test.go

@@ -240,6 +240,96 @@ func TestReconcile(t *testing.T) {
 	}
 }
 
+// TestReconcileStandalone covers the no-ClusterPolicy path: the controller falls back
+// to the GPUCluster for the cluster-wide configuration, and fails early when
+// neither object exists.
+func TestReconcileStandalone(t *testing.T) {
+	scheme := runtime.NewScheme()
+	require.NoError(t, nvidiav1alpha1.AddToScheme(scheme))
+	require.NoError(t, gpuv1.AddToScheme(scheme))
+
+	cp := &gpuv1.ClusterPolicy{
+		ObjectMeta: metav1.ObjectMeta{Name: "cluster-policy"},
+		Spec: gpuv1.ClusterPolicySpec{
+			Driver: gpuv1.DriverSpec{
+				UseNvidiaDriverCRD: ptr.To(true),
+			},
+			HostPaths: gpuv1.HostPathsSpec{RootFS: "/cp-root"},
+		},
+	}
+	gpuCluster := &nvidiav1alpha1.GPUCluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "gpu-cluster-config"},
+		Spec: nvidiav1alpha1.GPUClusterSpec{
+			HostPaths: gpuv1.HostPathsSpec{RootFS: "/gpuCluster-root"},
+		},
+	}
+
+	tests := []struct {
+		name             string
+		objects          []client.Object
+		expectedErr      string
+		expectedHostRoot string
+	}{
+		{
+			name:             "no ClusterPolicy, GPUCluster provides the host root",
+			objects:          []client.Object{gpuCluster},
+			expectedHostRoot: "/gpuCluster-root",
+		},
+		{
+			name:             "ClusterPolicy preferred over GPUCluster",
+			objects:          []client.Object{cp, gpuCluster},
+			expectedHostRoot: "/cp-root",
+		},
+		{
+			name:        "neither ClusterPolicy nor GPUCluster",
+			expectedErr: "no ClusterPolicy or GPUCluster object found in the cluster",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			driver := &nvidiav1alpha1.NVIDIADriver{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-driver"},
+			}
+
+			c := fake.NewClientBuilder().
+				WithScheme(scheme).
+				WithObjects(append([]client.Object{driver}, tc.objects...)...).
+				WithStatusSubresource(&nvidiav1alpha1.NVIDIADriver{}).
+				Build()
+
+			updater := &FakeConditionUpdater{}
+			stateManager := &fakeStateManager{results: state.Results{Status: state.SyncStateReady}}
+
+			reconciler := &NVIDIADriverReconciler{
+				Client:                c,
+				Scheme:                scheme,
+				conditionUpdater:      updater,
+				nodeSelectorValidator: &FakeNodeSelectorValidator{},
+				stateManager:          stateManager,
+			}
+
+			req := ctrl.Request{NamespacedName: types.NamespacedName{Name: driver.Name}}
+			_, err := reconciler.Reconcile(context.Background(), req)
+
+			if tc.expectedErr != "" {
+				require.ErrorContains(t, err, tc.expectedErr)
+				require.Equal(t, nvidiav1alpha1.NotReady, updater.LastErrorState)
+				return
+			}
+			require.NoError(t, err)
+
+			hostRoot, ok := stateManager.lastCatalog.Get(state.InfoTypeHostRoot).(string)
+			require.True(t, ok, "info catalog must hold a host root string")
+			require.Equal(t, tc.expectedHostRoot, hostRoot)
+
+			instance := &nvidiav1alpha1.NVIDIADriver{}
+			require.NoError(t, c.Get(context.Background(), types.NamespacedName{Name: driver.Name}, instance))
+			require.Equal(t, nvidiav1alpha1.Ready, instance.Status.State)
+		})
+	}
+}
+
 func TestReconcileConflictSetsNotReadyState(t *testing.T) {
 	scheme := runtime.NewScheme()
 	require.NoError(t, nvidiav1alpha1.AddToScheme(scheme))