29513807383

Committed 16 Jul 2026 03:58PM UTC coverage: 32.688% (+0.9%) from 31.812%

Build # 29513807383

Build Type

Pull #2571

github

Committed by

karthikvetrivel

Commit Message

Add golden render tests for GPUCluster operand manifests

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Pull Request Pull Request #2571: Add GPUCluster CRD and controller for DRA-based stack

Coverage Stats

513 of 1343 new or added lines in 31 files covered. (38.2%)

5 existing lines in 4 files now uncovered.

4663 of 14265 relevant lines covered (32.69%)

0.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

27.78

/controllers/nvidiadriver_controller.go

/*
Copyright 2021.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
        "context"
        "errors"
        "fmt"
        "maps"
        "time"

        appsv1 "k8s.io/api/apps/v1"
        corev1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/client-go/util/workqueue"
        ctrl "sigs.k8s.io/controller-runtime"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller"
        "sigs.k8s.io/controller-runtime/pkg/event"
        "sigs.k8s.io/controller-runtime/pkg/handler"
        "sigs.k8s.io/controller-runtime/pkg/log"
        "sigs.k8s.io/controller-runtime/pkg/predicate"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
        "sigs.k8s.io/controller-runtime/pkg/source"

        gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
        nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
        "github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
        "github.com/NVIDIA/gpu-operator/internal/conditions"
        "github.com/NVIDIA/gpu-operator/internal/consts"
        "github.com/NVIDIA/gpu-operator/internal/state"
        "github.com/NVIDIA/gpu-operator/internal/validator"
)

// NVIDIADriverReconciler reconciles a NVIDIADriver object
type NVIDIADriverReconciler struct {
        client.Client
        Scheme      *runtime.Scheme
        ClusterInfo clusterinfo.Interface
        Namespace   string

        stateManager          state.Manager
        nodeSelectorValidator validator.Validator
        conditionUpdater      conditions.Updater
}

//+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers/finalizers,verbs=update

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the NVIDIADriver object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile
func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
        logger := log.FromContext(ctx)
        logger.V(consts.LogLevelInfo).Info("Reconciling NVIDIADriver")

        // Get the NvidiaDriver instance from this request
        instance := &nvidiav1alpha1.NVIDIADriver{}
        if err := r.Get(ctx, req.NamespacedName, instance); err != nil {
                if apierrors.IsNotFound(err) {
                        // Request object not found, could have been deleted after reconcile request.
                        return reconcile.Result{}, nil
                }
                wrappedErr := fmt.Errorf("error getting NVIDIADriver object: %w", err)
                logger.Error(err, "error getting NVIDIADriver object")
                instance.Status.State = nvidiav1alpha1.NotReady
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, wrappedErr.Error()); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                // Error reading the object - requeue the request.
                return reconcile.Result{}, wrappedErr
        }
        if instance.HasDeletionTimestamp() {
                return reconcile.Result{}, nil
        }

        // Get the singleton NVIDIA ClusterPolicy object in the cluster.
        clusterPolicyList := &gpuv1.ClusterPolicyList{}
        if err := r.List(ctx, clusterPolicyList); err != nil {
                wrappedErr := fmt.Errorf("error getting ClusterPolicy list: %w", err)
                logger.Error(err, "error getting ClusterPolicy list")
                instance.Status.State = nvidiav1alpha1.NotReady
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return reconcile.Result{}, wrappedErr
        }

        if len(clusterPolicyList.Items) == 0 {
                err := fmt.Errorf("no ClusterPolicy object found in the cluster")
                logger.Error(err, "failed to get ClusterPolicy object")
                instance.Status.State = nvidiav1alpha1.NotReady
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return reconcile.Result{}, err
        }
        clusterPolicyInstance := clusterPolicyList.Items[0]

        // Ensure the NVIDIADriver CR has a consumer: either the ClusterPolicy delegates its
        // driver to the NVIDIADriver CRD, or a GPUCluster exists. GPUCluster does
        // not manage the driver itself — it is either preinstalled on the host (no NVIDIADriver
        // CR) or installed via NVIDIADriver CRs, so any CR that exists alongside one is in use.
        if !clusterPolicyInstance.Spec.Driver.UseNvidiaDriverCRDType() {
                gpuClusters := &nvidiav1alpha1.GPUClusterList{}
                if err := r.List(ctx, gpuClusters); err != nil {
                        wrappedErr := fmt.Errorf("error getting GPUCluster list: %w", err)
                        logger.Error(err, "error getting GPUCluster list")
                        instance.Status.State = nvidiav1alpha1.NotReady
                        if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
                                logger.Error(condErr, "failed to set condition")
                        }
                        return reconcile.Result{}, wrappedErr
                }
                if len(gpuClusters.Items) == 0 {
                        msg := "useNvidiaDriverCRD is not enabled in ClusterPolicy and no GPUCluster exists"
                        logger.V(consts.LogLevelWarning).Info("NVIDIADriver reconciliation skipped", "reason", msg)
                        instance.Status.State = nvidiav1alpha1.Disabled
                        if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.Reconciled, msg); condErr != nil {
                                logger.Error(condErr, "failed to set condition")
                        }
                        return reconcile.Result{}, nil
                }
        }

        // Create a new InfoCatalog which is a generic interface for passing information to state managers
        infoCatalog := state.NewInfoCatalog()

        // Add an entry for ClusterInfo, which was collected before the NVIDIADriver controller was started
        infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)

        // Add an entry for Clusterpolicy, which is needed to deploy the driver daemonset
        infoCatalog.Add(state.InfoTypeClusterPolicyCR, clusterPolicyInstance)

        // Verify the nodeSelector configured for this NVIDIADriver instance does
        // not conflict with any other instances. This ensures only one driver
        // is deployed per GPU node.
        if err := r.nodeSelectorValidator.Validate(ctx, instance); err != nil {
                logger.Error(err, "nodeSelector validation failed")
                instance.Status.State = nvidiav1alpha1.NotReady
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ConflictingNodeSelector, err.Error()); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return reconcile.Result{}, nil
        }

        if instance.Spec.UsePrecompiledDrivers() && (instance.Spec.IsGDSEnabled() || instance.Spec.IsGDRCopyEnabled()) {
                err := errors.New("GPUDirect Storage driver (nvidia-fs) and/or GDRCopy driver is not supported along with pre-compiled NVIDIA drivers")
                logger.Error(err, "unsupported driver combination detected")
                instance.Status.State = nvidiav1alpha1.NotReady
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return reconcile.Result{}, nil
        }

        if instance.Spec.IsGDSEnabled() && instance.Spec.IsOpenKernelModulesRequired() && !instance.Spec.IsOpenKernelModulesEnabled() {
                err := fmt.Errorf("GPUDirect Storage driver '%s' is only supported with NVIDIA OpenRM drivers. Please set 'useOpenKernelModules=true' to enable OpenRM mode", instance.Spec.GPUDirectStorage.Version)
                logger.Error(err, "unsupported driver combination detected")
                instance.Status.State = nvidiav1alpha1.NotReady
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return reconcile.Result{}, nil
        }

        // ensure that the specified K8s secret actually exists in the operator namespace
        secretName := instance.Spec.SecretEnv
        if len(secretName) > 0 {
                key := client.ObjectKey{Namespace: r.Namespace, Name: secretName}
                if err := r.Get(ctx, key, &corev1.Secret{}); err != nil {
                        logger.Error(err, "failed to get secret")
                        if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
                                logger.Error(condErr, "failed to set condition")
                        }
                        return reconcile.Result{}, nil
                }
        }

        // Sync state and update status
        managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)

        // update CR status
        if err := r.updateCrStatus(ctx, instance, managerStatus); err != nil {
                return ctrl.Result{}, err
        }

        if managerStatus.Status != state.SyncStateReady {
                logger.Info("NVIDIADriver instance is not ready")
                var errorInfo error
                for _, result := range managerStatus.StatesStatus {
                        if result.Status != state.SyncStateReady && result.ErrInfo != nil {
                                errorInfo = result.ErrInfo
                                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, errorInfo.Error())); condErr != nil {
                                        logger.Error(condErr, "failed to set condition")
                                }
                                break
                        }
                }
                // if no errors are reported from any state, then we would be waiting on driver daemonset pods
                if errorInfo == nil {
                        if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.DriverNotReady, "Waiting for driver pod to be ready"); condErr != nil {
                                logger.Error(condErr, "failed to set condition")
                        }
                }
                return reconcile.Result{RequeueAfter: time.Second * 5}, nil
        }

        if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {
                logger.Error(condErr, "failed to set condition")
                return ctrl.Result{}, condErr
        }
        return reconcile.Result{}, nil
}

func (r *NVIDIADriverReconciler) updateCrStatus(
        ctx context.Context, cr *nvidiav1alpha1.NVIDIADriver, status state.Results) error {
        reqLogger := log.FromContext(ctx)

        // Fetch latest instance and update state to avoid version mismatch
        instance := &nvidiav1alpha1.NVIDIADriver{}
        err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance)
        if err != nil {
                reqLogger.Error(err, "Failed to get NVIDIADriver instance for status update")
                return err
        }

        // Update global State
        if instance.Status.State == nvidiav1alpha1.State(status.Status) {
                return nil
        }
        instance.Status.State = nvidiav1alpha1.State(status.Status)

        // send status update request to k8s API
        reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)
        err = r.Status().Update(ctx, instance)
        if err != nil {
                reqLogger.Error(err, "Failed to update CR status")
                return err
        }
        return nil
}

// enqueueAllNVIDIADrivers lists all NVIDIADriver instances in the cluster and enqueues a reconcile
// request for each instance. This is used to trigger reconciliation for all NVIDIADriver instances
// when a relevant event occurs (e.g. ClusterPolicy/NVIDIADriver update, node label change, etc).
func (r *NVIDIADriverReconciler) enqueueAllNVIDIADrivers(ctx context.Context) []reconcile.Request {
        logger := log.FromContext(ctx)
        list := &nvidiav1alpha1.NVIDIADriverList{}

        err := r.List(ctx, list)
        if err != nil {
                logger.Error(err, "Unable to list NVIDIADriver resources")
                return []reconcile.Request{}
        }

        reconcileRequests := make([]reconcile.Request, 0, len(list.Items))
        for _, nvidiaDriver := range list.Items {
                reconcileRequests = append(reconcileRequests,
                        reconcile.Request{
                                NamespacedName: types.NamespacedName{
                                        Name:      nvidiaDriver.GetName(),
                                        Namespace: nvidiaDriver.GetNamespace(),
                                },
                        })
        }

        return reconcileRequests
}

// enqueueNVIDIADriverReconcilers enqueues the NVIDIADriver that triggered the
// event and all current NVIDIADriver instances. The triggering object is
// included even for delete events so the NotFound reconcile path can clear
// stale node owner labels.
func (r *NVIDIADriverReconciler) enqueueNVIDIADriverReconcilers(ctx context.Context, driver *nvidiav1alpha1.NVIDIADriver) []reconcile.Request {
        requests := r.enqueueAllNVIDIADrivers(ctx)
        if driver != nil {
                requests = append(requests, reconcile.Request{
                        NamespacedName: types.NamespacedName{
                                Name:      driver.GetName(),
                                Namespace: driver.GetNamespace(),
                        },
                })
        }
        return dedupeReconcileRequests(requests)
}

func dedupeReconcileRequests(requests []reconcile.Request) []reconcile.Request {
        seen := map[types.NamespacedName]struct{}{}
        deduped := make([]reconcile.Request, 0, len(requests))
        for _, request := range requests {
                if _, ok := seen[request.NamespacedName]; ok {
                        continue
                }
                seen[request.NamespacedName] = struct{}{}
                deduped = append(deduped, request)
        }
        return deduped
}

// SetupWithManager sets up the controller with the Manager.
func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
        // Create state manager
        stateManager, err := state.NewManager(
                nvidiav1alpha1.NVIDIADriverCRDName,
                r.Namespace,
                mgr.GetClient(),
                mgr.GetScheme())
        if err != nil {
                return fmt.Errorf("error creating state manager: %v", err)
        }
        r.stateManager = stateManager

        // initialize validators
        r.nodeSelectorValidator = validator.NewNodeSelectorValidator(r.Client)

        // initialize condition updater
        r.conditionUpdater = conditions.NewNvDriverUpdater(mgr.GetClient())

        // Create a new NVIDIADriver controller
        c, err := controller.New("nvidia-driver-controller", mgr, controller.Options{
                Reconciler:              r,
                MaxConcurrentReconciles: 1,
                RateLimiter:             workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),
        })
        if err != nil {
                return err
        }

        // Watch for changes to NVIDIADriver CRs. Whenever an event is generated for a NVIDIADriver CR,
        // enqueue a reconcile request for all NVIDIADriver instances.
        nvidiaDriverMapFn := func(ctx context.Context, driver *nvidiav1alpha1.NVIDIADriver) []reconcile.Request {
                return r.enqueueNVIDIADriverReconcilers(ctx, driver)
        }

        // Watch for changes to the primary resource NVIDIADriver
        err = c.Watch(source.Kind(
                mgr.GetCache(),
                &nvidiav1alpha1.NVIDIADriver{},
                handler.TypedEnqueueRequestsFromMapFunc(nvidiaDriverMapFn),
                predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.NVIDIADriver]{},
        ),
        )
        if err != nil {
                return err
        }

        // Watch for changes to ClusterPolicy. Whenever an event is generated for ClusterPolicy, enqueue
        // a reconcile request for all NVIDIADriver instances.
        mapFn := func(ctx context.Context, _ *gpuv1.ClusterPolicy) []reconcile.Request {
                return r.enqueueAllNVIDIADrivers(ctx)
        }

        // Watch for changes to the Nodes. Whenever an event is generated for a Node, enqueue
        // a reconcile request for all NVIDIADriver instances.
        nodeMapFn := func(ctx context.Context, _ *corev1.Node) []reconcile.Request {
                return r.enqueueAllNVIDIADrivers(ctx)
        }

        err = c.Watch(
                source.Kind(
                        mgr.GetCache(),
                        &gpuv1.ClusterPolicy{},
                        handler.TypedEnqueueRequestsFromMapFunc(mapFn),
                        predicate.TypedGenerationChangedPredicate[*gpuv1.ClusterPolicy]{},
                ),
        )
        if err != nil {
                return err
        }

        nodePredicate := predicate.TypedFuncs[*corev1.Node]{
                CreateFunc: func(e event.TypedCreateEvent[*corev1.Node]) bool {
                        labels := e.Object.GetLabels()
                        return hasGPULabels(labels)
                },
                UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Node]) bool {
                        logger := log.FromContext(ctx)
                        newLabels := e.ObjectNew.GetLabels()
                        oldLabels := e.ObjectOld.GetLabels()
                        nodeName := e.ObjectNew.GetName()

                        needsUpdate := hasGPULabels(newLabels) && !maps.Equal(newLabels, oldLabels)

                        if needsUpdate {
                                logger.Info("Node labels have been changed",
                                        "name", nodeName,
                                )
                        }
                        return needsUpdate
                },
                DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Node]) bool {
                        labels := e.Object.GetLabels()
                        return hasGPULabels(labels)
                },
        }

        // Watch for changes to node labels
        err = c.Watch(
                source.Kind(mgr.GetCache(),
                        &corev1.Node{},
                        handler.TypedEnqueueRequestsFromMapFunc(nodeMapFn),
                        nodePredicate,
                ),
        )
        if err != nil {
                return err
        }

        // Watch for changes to secondary resources which each state manager manages
        watchSources := stateManager.GetWatchSources(mgr)
        for _, watchSource := range watchSources {
                err = c.Watch(
                        watchSource,
                )
                if err != nil {
                        return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)
                }
        }

        // Add an index key which allows our reconciler to quickly look up DaemonSets owned by an NVIDIADriver instance
        if err := mgr.GetFieldIndexer().IndexField(ctx, &appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(rawObj client.Object) []string {
                ds := rawObj.(*appsv1.DaemonSet)
                owner := metav1.GetControllerOf(ds)
                if owner == nil {
                        return nil
                }
                if owner.APIVersion != nvidiav1alpha1.SchemeGroupVersion.String() || owner.Kind != nvidiav1alpha1.NVIDIADriverCRDName {
                        return nil
                }
                return []string{owner.Name}
        }); err != nil {
                return fmt.Errorf("failed to add index key: %w", err)
        }

        return nil
}

1	/*
2	Copyright 2021.
3
4	Licensed under the Apache License, Version 2.0 (the "License");
5	you may not use this file except in compliance with the License.
6	You may obtain a copy of the License at
7
8	http://www.apache.org/licenses/LICENSE-2.0
9
10	Unless required by applicable law or agreed to in writing, software
11	distributed under the License is distributed on an "AS IS" BASIS,
12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	See the License for the specific language governing permissions and
14	limitations under the License.
15	*/
16
17	package controllers
18
19	import (
20	"context"
21	"errors"
22	"fmt"
23	"maps"
24	"time"
25
26	appsv1 "k8s.io/api/apps/v1"
27	corev1 "k8s.io/api/core/v1"
28	apierrors "k8s.io/apimachinery/pkg/api/errors"
29	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30	"k8s.io/apimachinery/pkg/runtime"
31	"k8s.io/apimachinery/pkg/types"
32	"k8s.io/client-go/util/workqueue"
33	ctrl "sigs.k8s.io/controller-runtime"
34	"sigs.k8s.io/controller-runtime/pkg/client"
35	"sigs.k8s.io/controller-runtime/pkg/controller"
36	"sigs.k8s.io/controller-runtime/pkg/event"
37	"sigs.k8s.io/controller-runtime/pkg/handler"
38	"sigs.k8s.io/controller-runtime/pkg/log"
39	"sigs.k8s.io/controller-runtime/pkg/predicate"
40	"sigs.k8s.io/controller-runtime/pkg/reconcile"
41	"sigs.k8s.io/controller-runtime/pkg/source"
42
43	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
44	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
45	"github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
46	"github.com/NVIDIA/gpu-operator/internal/conditions"
47	"github.com/NVIDIA/gpu-operator/internal/consts"
48	"github.com/NVIDIA/gpu-operator/internal/state"
49	"github.com/NVIDIA/gpu-operator/internal/validator"
50	)
51
52	// NVIDIADriverReconciler reconciles a NVIDIADriver object
53	type NVIDIADriverReconciler struct {
54	client.Client
55	Scheme *runtime.Scheme
56	ClusterInfo clusterinfo.Interface
57	Namespace string
58
59	stateManager state.Manager
60	nodeSelectorValidator validator.Validator
61	conditionUpdater conditions.Updater
62	}
63
64	//+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers,verbs=get;list;watch;create;update;patch;delete
65	//+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers/status,verbs=get;update;patch
66	//+kubebuilder:rbac:groups=nvidia.com,resources=nvidiadrivers/finalizers,verbs=update
67
68	// Reconcile is part of the main kubernetes reconciliation loop which aims to
69	// move the current state of the cluster closer to the desired state.
70	// TODO(user): Modify the Reconcile function to compare the state specified by
71	// the NVIDIADriver object against the actual cluster state, and then
72	// perform operations to make the cluster state reflect the state specified by
73	// the user.
74	//
75	// For more details, check Reconcile and its Result here:
76	// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile
77	func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {	1✔
78	logger := log.FromContext(ctx)	1✔
79	logger.V(consts.LogLevelInfo).Info("Reconciling NVIDIADriver")	1✔
80		1✔
81	// Get the NvidiaDriver instance from this request	1✔
82	instance := &nvidiav1alpha1.NVIDIADriver{}	1✔
83	if err := r.Get(ctx, req.NamespacedName, instance); err != nil {	1✔
84	if apierrors.IsNotFound(err) {	×
85	// Request object not found, could have been deleted after reconcile request.	×
86	return reconcile.Result{}, nil	×
87	}	×
88	wrappedErr := fmt.Errorf("error getting NVIDIADriver object: %w", err)	×
89	logger.Error(err, "error getting NVIDIADriver object")	×
90	instance.Status.State = nvidiav1alpha1.NotReady	×
91	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, wrappedErr.Error()); condErr != nil {	×
92	logger.Error(condErr, "failed to set condition")	×
93	}	×
94	// Error reading the object - requeue the request.
95	return reconcile.Result{}, wrappedErr	×
96	}
97	if instance.HasDeletionTimestamp() {	1✔
98	return reconcile.Result{}, nil	×
99	}	×
100
101	// Get the singleton NVIDIA ClusterPolicy object in the cluster.
102	clusterPolicyList := &gpuv1.ClusterPolicyList{}	1✔
103	if err := r.List(ctx, clusterPolicyList); err != nil {	1✔
104	wrappedErr := fmt.Errorf("error getting ClusterPolicy list: %w", err)	×
105	logger.Error(err, "error getting ClusterPolicy list")	×
106	instance.Status.State = nvidiav1alpha1.NotReady	×
107	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {	×
108	logger.Error(condErr, "failed to set condition")	×
109	}	×
110	return reconcile.Result{}, wrappedErr	×
111	}
112
113	if len(clusterPolicyList.Items) == 0 {	1✔
114	err := fmt.Errorf("no ClusterPolicy object found in the cluster")	×
115	logger.Error(err, "failed to get ClusterPolicy object")	×
116	instance.Status.State = nvidiav1alpha1.NotReady	×
117	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {	×
118	logger.Error(condErr, "failed to set condition")	×
119	}	×
120	return reconcile.Result{}, err	×
121	}
122	clusterPolicyInstance := clusterPolicyList.Items[0]	1✔
123		1✔
124	// Ensure the NVIDIADriver CR has a consumer: either the ClusterPolicy delegates its	1✔
125	// driver to the NVIDIADriver CRD, or a GPUCluster exists. GPUCluster does	1✔
126	// not manage the driver itself — it is either preinstalled on the host (no NVIDIADriver	1✔
127	// CR) or installed via NVIDIADriver CRs, so any CR that exists alongside one is in use.	1✔
128	if !clusterPolicyInstance.Spec.Driver.UseNvidiaDriverCRDType() {	2✔
129	gpuClusters := &nvidiav1alpha1.GPUClusterList{}	1✔
130	if err := r.List(ctx, gpuClusters); err != nil {	1✔
NEW 131	wrappedErr := fmt.Errorf("error getting GPUCluster list: %w", err)	×
NEW 132	logger.Error(err, "error getting GPUCluster list")	×
NEW 133	instance.Status.State = nvidiav1alpha1.NotReady	×
NEW 134	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {	×
NEW 135	logger.Error(condErr, "failed to set condition")	×
NEW 136	}	×
NEW 137	return reconcile.Result{}, wrappedErr	×
138	}
139	if len(gpuClusters.Items) == 0 {	2✔
140	msg := "useNvidiaDriverCRD is not enabled in ClusterPolicy and no GPUCluster exists"	1✔
141	logger.V(consts.LogLevelWarning).Info("NVIDIADriver reconciliation skipped", "reason", msg)	1✔
142	instance.Status.State = nvidiav1alpha1.Disabled	1✔
143	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.Reconciled, msg); condErr != nil {	1✔
NEW 144	logger.Error(condErr, "failed to set condition")	×
NEW 145	}	×
146	return reconcile.Result{}, nil	1✔
147	}
148	}
149
150	// Create a new InfoCatalog which is a generic interface for passing information to state managers
151	infoCatalog := state.NewInfoCatalog()	1✔
152		1✔
153	// Add an entry for ClusterInfo, which was collected before the NVIDIADriver controller was started	1✔
154	infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)	1✔
155		1✔
156	// Add an entry for Clusterpolicy, which is needed to deploy the driver daemonset	1✔
157	infoCatalog.Add(state.InfoTypeClusterPolicyCR, clusterPolicyInstance)	1✔
158		1✔
159	// Verify the nodeSelector configured for this NVIDIADriver instance does	1✔
160	// not conflict with any other instances. This ensures only one driver	1✔
161	// is deployed per GPU node.	1✔
162	if err := r.nodeSelectorValidator.Validate(ctx, instance); err != nil {	2✔
163	logger.Error(err, "nodeSelector validation failed")	1✔
164	instance.Status.State = nvidiav1alpha1.NotReady	1✔
165	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ConflictingNodeSelector, err.Error()); condErr != nil {	1✔
166	logger.Error(condErr, "failed to set condition")	×
167	}	×
168	return reconcile.Result{}, nil	1✔
169	}
170
171	if instance.Spec.UsePrecompiledDrivers() && (instance.Spec.IsGDSEnabled() \|\| instance.Spec.IsGDRCopyEnabled()) {	2✔
172	err := errors.New("GPUDirect Storage driver (nvidia-fs) and/or GDRCopy driver is not supported along with pre-compiled NVIDIA drivers")	1✔
173	logger.Error(err, "unsupported driver combination detected")	1✔
174	instance.Status.State = nvidiav1alpha1.NotReady	1✔
175	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {	1✔
176	logger.Error(condErr, "failed to set condition")	×
177	}	×
178	return reconcile.Result{}, nil	1✔
179	}
180
181	if instance.Spec.IsGDSEnabled() && instance.Spec.IsOpenKernelModulesRequired() && !instance.Spec.IsOpenKernelModulesEnabled() {	×
182	err := fmt.Errorf("GPUDirect Storage driver '%s' is only supported with NVIDIA OpenRM drivers. Please set 'useOpenKernelModules=true' to enable OpenRM mode", instance.Spec.GPUDirectStorage.Version)	×
183	logger.Error(err, "unsupported driver combination detected")	×
184	instance.Status.State = nvidiav1alpha1.NotReady	×
185	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {	×
186	logger.Error(condErr, "failed to set condition")	×
187	}	×
188	return reconcile.Result{}, nil	×
189	}
190
191	// ensure that the specified K8s secret actually exists in the operator namespace
192	secretName := instance.Spec.SecretEnv	×
193	if len(secretName) > 0 {	×
194	key := client.ObjectKey{Namespace: r.Namespace, Name: secretName}	×
195	if err := r.Get(ctx, key, &corev1.Secret{}); err != nil {	×
196	logger.Error(err, "failed to get secret")	×
197	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {	×
198	logger.Error(condErr, "failed to set condition")	×
199	}	×
200	return reconcile.Result{}, nil	×
201	}
202	}
203
204	// Sync state and update status
205	managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)	×
206		×
207	// update CR status	×
208	if err := r.updateCrStatus(ctx, instance, managerStatus); err != nil {	×
209	return ctrl.Result{}, err	×
210	}	×
211
212	if managerStatus.Status != state.SyncStateReady {	×
213	logger.Info("NVIDIADriver instance is not ready")	×
214	var errorInfo error	×
215	for _, result := range managerStatus.StatesStatus {	×
216	if result.Status != state.SyncStateReady && result.ErrInfo != nil {	×
217	errorInfo = result.ErrInfo	×
218	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, errorInfo.Error())); condErr != nil {	×
219	logger.Error(condErr, "failed to set condition")	×
220	}	×
221	break	×
222	}
223	}
224	// if no errors are reported from any state, then we would be waiting on driver daemonset pods
225	if errorInfo == nil {	×
226	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.DriverNotReady, "Waiting for driver pod to be ready"); condErr != nil {	×
227	logger.Error(condErr, "failed to set condition")	×
228	}	×
229	}
230	return reconcile.Result{RequeueAfter: time.Second * 5}, nil	×
231	}
232
233	if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {	×
234	logger.Error(condErr, "failed to set condition")	×
235	return ctrl.Result{}, condErr	×
236	}	×
237	return reconcile.Result{}, nil	×
238	}
239
240	func (r *NVIDIADriverReconciler) updateCrStatus(
241	ctx context.Context, cr *nvidiav1alpha1.NVIDIADriver, status state.Results) error {	×
242	reqLogger := log.FromContext(ctx)	×
243		×
244	// Fetch latest instance and update state to avoid version mismatch	×
245	instance := &nvidiav1alpha1.NVIDIADriver{}	×
246	err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance)	×
247	if err != nil {	×
248	reqLogger.Error(err, "Failed to get NVIDIADriver instance for status update")	×
249	return err	×
250	}	×
251
252	// Update global State
253	if instance.Status.State == nvidiav1alpha1.State(status.Status) {	×
254	return nil	×
255	}	×
256	instance.Status.State = nvidiav1alpha1.State(status.Status)	×
257		×
258	// send status update request to k8s API	×
259	reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)	×
260	err = r.Status().Update(ctx, instance)	×
261	if err != nil {	×
262	reqLogger.Error(err, "Failed to update CR status")	×
263	return err	×
264	}	×
265	return nil	×
266	}
267
268	// enqueueAllNVIDIADrivers lists all NVIDIADriver instances in the cluster and enqueues a reconcile
269	// request for each instance. This is used to trigger reconciliation for all NVIDIADriver instances
270	// when a relevant event occurs (e.g. ClusterPolicy/NVIDIADriver update, node label change, etc).
271	func (r *NVIDIADriverReconciler) enqueueAllNVIDIADrivers(ctx context.Context) []reconcile.Request {	1✔
272	logger := log.FromContext(ctx)	1✔
273	list := &nvidiav1alpha1.NVIDIADriverList{}	1✔
274		1✔
275	err := r.List(ctx, list)	1✔
276	if err != nil {	1✔
277	logger.Error(err, "Unable to list NVIDIADriver resources")	×
278	return []reconcile.Request{}	×
279	}	×
280
281	reconcileRequests := make([]reconcile.Request, 0, len(list.Items))	1✔
282	for _, nvidiaDriver := range list.Items {	2✔
283	reconcileRequests = append(reconcileRequests,	1✔
284	reconcile.Request{	1✔
285	NamespacedName: types.NamespacedName{	1✔
286	Name: nvidiaDriver.GetName(),	1✔
287	Namespace: nvidiaDriver.GetNamespace(),	1✔
288	},	1✔
289	})	1✔
290	}	1✔
291
292	return reconcileRequests	1✔
293	}
294
295	// enqueueNVIDIADriverReconcilers enqueues the NVIDIADriver that triggered the
296	// event and all current NVIDIADriver instances. The triggering object is
297	// included even for delete events so the NotFound reconcile path can clear
298	// stale node owner labels.
299	func (r NVIDIADriverReconciler) enqueueNVIDIADriverReconcilers(ctx context.Context, driver nvidiav1alpha1.NVIDIADriver) []reconcile.Request {	1✔
300	requests := r.enqueueAllNVIDIADrivers(ctx)	1✔
301	if driver != nil {	2✔
302	requests = append(requests, reconcile.Request{	1✔
303	NamespacedName: types.NamespacedName{	1✔
304	Name: driver.GetName(),	1✔
305	Namespace: driver.GetNamespace(),	1✔
306	},	1✔
307	})	1✔
308	}	1✔
309	return dedupeReconcileRequests(requests)	1✔
310	}
311
312	func dedupeReconcileRequests(requests []reconcile.Request) []reconcile.Request {	1✔
313	seen := map[types.NamespacedName]struct{}{}	1✔
314	deduped := make([]reconcile.Request, 0, len(requests))	1✔
315	for _, request := range requests {	2✔
316	if _, ok := seen[request.NamespacedName]; ok {	2✔
317	continue	1✔
318	}
319	seen[request.NamespacedName] = struct{}{}	1✔
320	deduped = append(deduped, request)	1✔
321	}
322	return deduped	1✔
323	}
324
325	// SetupWithManager sets up the controller with the Manager.
326	func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {	×
327	// Create state manager	×
328	stateManager, err := state.NewManager(	×
329	nvidiav1alpha1.NVIDIADriverCRDName,	×
330	r.Namespace,	×
331	mgr.GetClient(),	×
332	mgr.GetScheme())	×
333	if err != nil {	×
334	return fmt.Errorf("error creating state manager: %v", err)	×
335	}	×
336	r.stateManager = stateManager	×
337		×
338	// initialize validators	×
339	r.nodeSelectorValidator = validator.NewNodeSelectorValidator(r.Client)	×
340		×
341	// initialize condition updater	×
342	r.conditionUpdater = conditions.NewNvDriverUpdater(mgr.GetClient())	×
343		×
344	// Create a new NVIDIADriver controller	×
345	c, err := controller.New("nvidia-driver-controller", mgr, controller.Options{	×
346	Reconciler: r,	×
347	MaxConcurrentReconciles: 1,	×
348	RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),	×
349	})	×
350	if err != nil {	×
351	return err	×
352	}	×
353
354	// Watch for changes to NVIDIADriver CRs. Whenever an event is generated for a NVIDIADriver CR,
355	// enqueue a reconcile request for all NVIDIADriver instances.
356	nvidiaDriverMapFn := func(ctx context.Context, driver *nvidiav1alpha1.NVIDIADriver) []reconcile.Request {	×
357	return r.enqueueNVIDIADriverReconcilers(ctx, driver)	×
358	}	×
359
360	// Watch for changes to the primary resource NVIDIADriver
361	err = c.Watch(source.Kind(	×
362	mgr.GetCache(),	×
363	&nvidiav1alpha1.NVIDIADriver{},	×
364	handler.TypedEnqueueRequestsFromMapFunc(nvidiaDriverMapFn),	×
365	predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.NVIDIADriver]{},	×
366	),	×
367	)	×
368	if err != nil {	×
369	return err	×
370	}	×
371
372	// Watch for changes to ClusterPolicy. Whenever an event is generated for ClusterPolicy, enqueue
373	// a reconcile request for all NVIDIADriver instances.
374	mapFn := func(ctx context.Context, _ *gpuv1.ClusterPolicy) []reconcile.Request {	×
375	return r.enqueueAllNVIDIADrivers(ctx)	×
376	}	×
377
378	// Watch for changes to the Nodes. Whenever an event is generated for a Node, enqueue
379	// a reconcile request for all NVIDIADriver instances.
380	nodeMapFn := func(ctx context.Context, _ *corev1.Node) []reconcile.Request {	×
381	return r.enqueueAllNVIDIADrivers(ctx)	×
382	}	×
383
384	err = c.Watch(	×
385	source.Kind(	×
386	mgr.GetCache(),	×
387	&gpuv1.ClusterPolicy{},	×
388	handler.TypedEnqueueRequestsFromMapFunc(mapFn),	×
389	predicate.TypedGenerationChangedPredicate[*gpuv1.ClusterPolicy]{},	×
390	),	×
391	)	×
392	if err != nil {	×
393	return err	×
394	}	×
395
396	nodePredicate := predicate.TypedFuncs[*corev1.Node]{	×
397	CreateFunc: func(e event.TypedCreateEvent[*corev1.Node]) bool {	×
398	labels := e.Object.GetLabels()	×
399	return hasGPULabels(labels)	×
400	},	×
401	UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Node]) bool {	×
402	logger := log.FromContext(ctx)	×
403	newLabels := e.ObjectNew.GetLabels()	×
404	oldLabels := e.ObjectOld.GetLabels()	×
405	nodeName := e.ObjectNew.GetName()	×
406		×
407	needsUpdate := hasGPULabels(newLabels) && !maps.Equal(newLabels, oldLabels)	×
408		×
409	if needsUpdate {	×
410	logger.Info("Node labels have been changed",	×
411	"name", nodeName,	×
412	)	×
413	}	×
414	return needsUpdate	×
415	},
416	DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Node]) bool {	×
417	labels := e.Object.GetLabels()	×
418	return hasGPULabels(labels)	×
419	},	×
420	}
421
422	// Watch for changes to node labels
423	err = c.Watch(	×
424	source.Kind(mgr.GetCache(),	×
425	&corev1.Node{},	×
426	handler.TypedEnqueueRequestsFromMapFunc(nodeMapFn),	×
427	nodePredicate,	×
428	),	×
429	)	×
430	if err != nil {	×
431	return err	×
432	}	×
433
434	// Watch for changes to secondary resources which each state manager manages
435	watchSources := stateManager.GetWatchSources(mgr)	×
436	for _, watchSource := range watchSources {	×
437	err = c.Watch(	×
438	watchSource,	×
439	)	×
440	if err != nil {	×
441	return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)	×
442	}	×
443	}
444
445	// Add an index key which allows our reconciler to quickly look up DaemonSets owned by an NVIDIADriver instance
446	if err := mgr.GetFieldIndexer().IndexField(ctx, &appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(rawObj client.Object) []string {	×
447	ds := rawObj.(*appsv1.DaemonSet)	×
448	owner := metav1.GetControllerOf(ds)	×
449	if owner == nil {	×
450	return nil	×
451	}	×
452	if owner.APIVersion != nvidiav1alpha1.SchemeGroupVersion.String() \|\| owner.Kind != nvidiav1alpha1.NVIDIADriverCRDName {	×
453	return nil	×
454	}	×
455	return []string{owner.Name}	×
456	}); err != nil {	×
457	return fmt.Errorf("failed to add index key: %w", err)	×
458	}	×
459
460	return nil	×
461	}

NVIDIA / gpu-operator / 29513807383

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous