29513807383

Committed 16 Jul 2026 03:58PM UTC coverage: 32.688% (+0.9%) from 31.812%

Build # 29513807383

Build Type

Pull #2571

github

Committed by

karthikvetrivel

Commit Message

Add golden render tests for GPUCluster operand manifests

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Pull Request Pull Request #2571: Add GPUCluster CRD and controller for DRA-based stack

Coverage Stats

513 of 1343 new or added lines in 31 files covered. (38.2%)

5 existing lines in 4 files now uncovered.

4663 of 14265 relevant lines covered (32.69%)

0.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

43.75

/controllers/gpucluster_controller.go

/*
Copyright 2025.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
        "context"
        "fmt"
        "time"

        apierrors "k8s.io/apimachinery/pkg/api/errors"
        "k8s.io/apimachinery/pkg/runtime"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/client-go/util/workqueue"
        ctrl "sigs.k8s.io/controller-runtime"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller"
        "sigs.k8s.io/controller-runtime/pkg/handler"
        "sigs.k8s.io/controller-runtime/pkg/log"
        "sigs.k8s.io/controller-runtime/pkg/predicate"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
        "sigs.k8s.io/controller-runtime/pkg/source"

        nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
        "github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
        "github.com/NVIDIA/gpu-operator/internal/conditions"
        "github.com/NVIDIA/gpu-operator/internal/consts"
        "github.com/NVIDIA/gpu-operator/internal/state"
)

// GPUClusterReconciler reconciles a GPUCluster object
type GPUClusterReconciler struct {
        client.Client
        Scheme      *runtime.Scheme
        ClusterInfo clusterinfo.Interface
        Namespace   string

        stateManager     state.Manager
        conditionUpdater conditions.Updater

        // singleton is the GPUCluster that owns reconciliation; the first instance to
        // reconcile claims it (first-wins), mirroring ClusterPolicy.
        singleton *nvidiav1alpha1.GPUCluster
}

//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/finalizers,verbs=update
//+kubebuilder:rbac:groups=nvidia.com,resources=clusterpolicies,verbs=get;list;watch
//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;update;patch
//+kubebuilder:rbac:groups=resource.k8s.io,resources=resourceclaimtemplates,verbs=get;list;watch;create;update;delete

func (r *GPUClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
        logger := log.FromContext(ctx)
        logger.V(consts.LogLevelInfo).Info("Reconciling GPUCluster")

        instance := &nvidiav1alpha1.GPUCluster{}
        if err := r.Get(ctx, req.NamespacedName, instance); err != nil {
                if apierrors.IsNotFound(err) {
                        // Deleted; owned objects are garbage-collected, so there is nothing to clean up.
                        return ctrl.Result{}, nil
                }
                // instance was not populated by the failed Get, so there is no object to
                // update status on; just surface the error for requeue.
                logger.Error(err, "error getting GPUCluster object")
                return ctrl.Result{}, fmt.Errorf("error getting GPUCluster object: %w", err)
        }

        // GPUCluster (DRA stack) may coexist with a ClusterPolicy (device-plugin
        // stack): every operand DaemonSet of both stacks gates on the per-node
        // nvidia.com/gpu-operator.resource-allocation.mode label, so each node is served by exactly one stack.

        // Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
        // claims ownership; any other instance is marked Ignored and skipped. The owner is
        // held in memory, so the choice resets on operator restart.
        if r.singleton != nil && r.singleton.Name != instance.Name {
                logger.V(consts.LogLevelWarning).Info("Multiple GPUCluster instances found, ignoring this one",
                        "name", instance.Name, "owner", r.singleton.Name)
                if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {
                        return ctrl.Result{}, err
                }
                return ctrl.Result{}, nil
        }
        r.singleton = instance

        infoCatalog := state.NewInfoCatalog()
        infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)

        managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)

        if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {
                return ctrl.Result{}, err
        }

        if managerStatus.Status != state.SyncStateReady {
                logger.Info("GPUCluster instance is not ready")
                for _, result := range managerStatus.StatesStatus {
                        if result.Status != state.SyncStateReady && result.ErrInfo != nil {
                                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, result.ErrInfo)); condErr != nil {
                                        logger.Error(condErr, "failed to set condition")
                                }
                                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
                        }
                }
                // no state reported an error, so we are waiting on operand pods
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
        }

        if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {
                logger.Error(condErr, "failed to set condition")
                return ctrl.Result{}, condErr
        }
        // Resync periodically so out-of-band changes (a deleted DeviceClass/VAP, or a
        // newly-created ClusterPolicy) are detected and reconciled even while ready;
        // only DaemonSets are watched, and the ready path is otherwise event-driven.
        return ctrl.Result{RequeueAfter: time.Minute}, nil
}

// updateCRStatus persists the given state (and the operator namespace) to the GPUCluster's
// .status subresource. It refetches the CR first to avoid resourceVersion conflicts, skips
// the API write when the status is already current, and mirrors the result back onto cr.
func (r *GPUClusterReconciler) updateCRStatus(ctx context.Context, cr *nvidiav1alpha1.GPUCluster, desired nvidiav1alpha1.State) error {
        reqLogger := log.FromContext(ctx)

        // Refetch to avoid a resourceVersion conflict.
        instance := &nvidiav1alpha1.GPUCluster{}
        if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {
                reqLogger.Error(err, "Failed to get GPUCluster instance for status update")
                return err
        }

        if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {
                return nil
        }
        instance.Status.State = desired
        instance.Status.Namespace = r.Namespace

        reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)
        if err := r.Status().Update(ctx, instance); err != nil {
                reqLogger.Error(err, "Failed to update CR status")
                return err
        }
        cr.Status.State = instance.Status.State
        cr.Status.Namespace = instance.Status.Namespace
        return nil
}

// enqueueAllGPUClusters enqueues every instance so each is reconciled when any
// instance or owned resource changes.
func (r *GPUClusterReconciler) enqueueAllGPUClusters(ctx context.Context, _ *nvidiav1alpha1.GPUCluster) []reconcile.Request {
        logger := log.FromContext(ctx)
        list := &nvidiav1alpha1.GPUClusterList{}

        if err := r.List(ctx, list); err != nil {
                logger.Error(err, "Unable to list GPUCluster resources")
                return []reconcile.Request{}
        }

        reconcileRequests := make([]reconcile.Request, 0, len(list.Items))
        for _, config := range list.Items {
                reconcileRequests = append(reconcileRequests,
                        reconcile.Request{
                                NamespacedName: types.NamespacedName{
                                        Name: config.GetName(),
                                },
                        })
        }

        return reconcileRequests
}

func (r *GPUClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
        // The state manager renders the DRA driver operand for the GPUCluster.
        stateManager, err := state.NewManager(
                nvidiav1alpha1.GPUClusterCRDName,
                r.Namespace,
                mgr.GetClient(),
                mgr.GetScheme())
        if err != nil {
                return fmt.Errorf("error creating state manager: %w", err)
        }
        r.stateManager = stateManager

        r.conditionUpdater = conditions.NewGPUClusterUpdater(mgr.GetClient())

        c, err := controller.New("gpu-cluster-controller", mgr, controller.Options{
                Reconciler:              r,
                MaxConcurrentReconciles: 1,
                RateLimiter:             workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),
        })
        if err != nil {
                return err
        }

        err = c.Watch(source.Kind(
                mgr.GetCache(),
                &nvidiav1alpha1.GPUCluster{},
                handler.TypedEnqueueRequestsFromMapFunc(r.enqueueAllGPUClusters),
                predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUCluster]{},
        ),
        )
        if err != nil {
                return err
        }

        // Watch the secondary resources each state manager owns.
        watchSources := stateManager.GetWatchSources(mgr)
        for _, watchSource := range watchSources {
                err = c.Watch(
                        watchSource,
                )
                if err != nil {
                        return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)
                }
        }

        return nil
}

1	/*
2	Copyright 2025.
3
4	Licensed under the Apache License, Version 2.0 (the "License");
5	you may not use this file except in compliance with the License.
6	You may obtain a copy of the License at
7
8	http://www.apache.org/licenses/LICENSE-2.0
9
10	Unless required by applicable law or agreed to in writing, software
11	distributed under the License is distributed on an "AS IS" BASIS,
12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	See the License for the specific language governing permissions and
14	limitations under the License.
15	*/
16
17	package controllers
18
19	import (
20	"context"
21	"fmt"
22	"time"
23
24	apierrors "k8s.io/apimachinery/pkg/api/errors"
25	"k8s.io/apimachinery/pkg/runtime"
26	"k8s.io/apimachinery/pkg/types"
27	"k8s.io/client-go/util/workqueue"
28	ctrl "sigs.k8s.io/controller-runtime"
29	"sigs.k8s.io/controller-runtime/pkg/client"
30	"sigs.k8s.io/controller-runtime/pkg/controller"
31	"sigs.k8s.io/controller-runtime/pkg/handler"
32	"sigs.k8s.io/controller-runtime/pkg/log"
33	"sigs.k8s.io/controller-runtime/pkg/predicate"
34	"sigs.k8s.io/controller-runtime/pkg/reconcile"
35	"sigs.k8s.io/controller-runtime/pkg/source"
36
37	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
38	"github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
39	"github.com/NVIDIA/gpu-operator/internal/conditions"
40	"github.com/NVIDIA/gpu-operator/internal/consts"
41	"github.com/NVIDIA/gpu-operator/internal/state"
42	)
43
44	// GPUClusterReconciler reconciles a GPUCluster object
45	type GPUClusterReconciler struct {
46	client.Client
47	Scheme *runtime.Scheme
48	ClusterInfo clusterinfo.Interface
49	Namespace string
50
51	stateManager state.Manager
52	conditionUpdater conditions.Updater
53
54	// singleton is the GPUCluster that owns reconciliation; the first instance to
55	// reconcile claims it (first-wins), mirroring ClusterPolicy.
56	singleton *nvidiav1alpha1.GPUCluster
57	}
58
59	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters,verbs=get;list;watch;create;update;patch;delete
60	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/status,verbs=get;update;patch
61	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/finalizers,verbs=update
62	//+kubebuilder:rbac:groups=nvidia.com,resources=clusterpolicies,verbs=get;list;watch
63	//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;update;patch
64	//+kubebuilder:rbac:groups=resource.k8s.io,resources=resourceclaimtemplates,verbs=get;list;watch;create;update;delete
65
66	func (r *GPUClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {	1✔
67	logger := log.FromContext(ctx)	1✔
68	logger.V(consts.LogLevelInfo).Info("Reconciling GPUCluster")	1✔
69		1✔
70	instance := &nvidiav1alpha1.GPUCluster{}	1✔
71	if err := r.Get(ctx, req.NamespacedName, instance); err != nil {	2✔
72	if apierrors.IsNotFound(err) {	2✔
73	// Deleted; owned objects are garbage-collected, so there is nothing to clean up.	1✔
74	return ctrl.Result{}, nil	1✔
75	}	1✔
76	// instance was not populated by the failed Get, so there is no object to
77	// update status on; just surface the error for requeue.
NEW 78	logger.Error(err, "error getting GPUCluster object")	×
NEW 79	return ctrl.Result{}, fmt.Errorf("error getting GPUCluster object: %w", err)	×
80	}
81
82	// GPUCluster (DRA stack) may coexist with a ClusterPolicy (device-plugin
83	// stack): every operand DaemonSet of both stacks gates on the per-node
84	// nvidia.com/gpu-operator.resource-allocation.mode label, so each node is served by exactly one stack.
85
86	// Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
87	// claims ownership; any other instance is marked Ignored and skipped. The owner is
88	// held in memory, so the choice resets on operator restart.
89	if r.singleton != nil && r.singleton.Name != instance.Name {	2✔
90	logger.V(consts.LogLevelWarning).Info("Multiple GPUCluster instances found, ignoring this one",	1✔
91	"name", instance.Name, "owner", r.singleton.Name)	1✔
92	if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {	1✔
NEW 93	return ctrl.Result{}, err	×
NEW 94	}	×
95	return ctrl.Result{}, nil	1✔
96	}
97	r.singleton = instance	1✔
98		1✔
99	infoCatalog := state.NewInfoCatalog()	1✔
100	infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)	1✔
101		1✔
102	managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)	1✔
103		1✔
104	if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {	1✔
NEW 105	return ctrl.Result{}, err	×
NEW 106	}	×
107
108	if managerStatus.Status != state.SyncStateReady {	1✔
NEW 109	logger.Info("GPUCluster instance is not ready")	×
NEW 110	for _, result := range managerStatus.StatesStatus {	×
NEW 111	if result.Status != state.SyncStateReady && result.ErrInfo != nil {	×
NEW 112	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, result.ErrInfo)); condErr != nil {	×
NEW 113	logger.Error(condErr, "failed to set condition")	×
NEW 114	}	×
NEW 115	return ctrl.Result{RequeueAfter: time.Second * 5}, nil	×
116	}
117	}
118	// no state reported an error, so we are waiting on operand pods
NEW 119	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {	×
NEW 120	logger.Error(condErr, "failed to set condition")	×
NEW 121	}	×
NEW 122	return ctrl.Result{RequeueAfter: time.Second * 5}, nil	×
123	}
124
125	if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {	1✔
NEW 126	logger.Error(condErr, "failed to set condition")	×
NEW 127	return ctrl.Result{}, condErr	×
NEW 128	}	×
129	// Resync periodically so out-of-band changes (a deleted DeviceClass/VAP, or a
130	// newly-created ClusterPolicy) are detected and reconciled even while ready;
131	// only DaemonSets are watched, and the ready path is otherwise event-driven.
132	return ctrl.Result{RequeueAfter: time.Minute}, nil	1✔
133	}
134
135	// updateCRStatus persists the given state (and the operator namespace) to the GPUCluster's
136	// .status subresource. It refetches the CR first to avoid resourceVersion conflicts, skips
137	// the API write when the status is already current, and mirrors the result back onto cr.
138	func (r GPUClusterReconciler) updateCRStatus(ctx context.Context, cr nvidiav1alpha1.GPUCluster, desired nvidiav1alpha1.State) error {	1✔
139	reqLogger := log.FromContext(ctx)	1✔
140		1✔
141	// Refetch to avoid a resourceVersion conflict.	1✔
142	instance := &nvidiav1alpha1.GPUCluster{}	1✔
143	if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {	1✔
NEW 144	reqLogger.Error(err, "Failed to get GPUCluster instance for status update")	×
NEW 145	return err	×
NEW 146	}	×
147
148	if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {	1✔
NEW 149	return nil	×
NEW 150	}	×
151	instance.Status.State = desired	1✔
152	instance.Status.Namespace = r.Namespace	1✔
153		1✔
154	reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)	1✔
155	if err := r.Status().Update(ctx, instance); err != nil {	1✔
NEW 156	reqLogger.Error(err, "Failed to update CR status")	×
NEW 157	return err	×
NEW 158	}	×
159	cr.Status.State = instance.Status.State	1✔
160	cr.Status.Namespace = instance.Status.Namespace	1✔
161	return nil	1✔
162	}
163
164	// enqueueAllGPUClusters enqueues every instance so each is reconciled when any
165	// instance or owned resource changes.
166	func (r GPUClusterReconciler) enqueueAllGPUClusters(ctx context.Context, _ nvidiav1alpha1.GPUCluster) []reconcile.Request {	1✔
167	logger := log.FromContext(ctx)	1✔
168	list := &nvidiav1alpha1.GPUClusterList{}	1✔
169		1✔
170	if err := r.List(ctx, list); err != nil {	1✔
NEW 171	logger.Error(err, "Unable to list GPUCluster resources")	×
NEW 172	return []reconcile.Request{}	×
NEW 173	}	×
174
175	reconcileRequests := make([]reconcile.Request, 0, len(list.Items))	1✔
176	for _, config := range list.Items {	2✔
177	reconcileRequests = append(reconcileRequests,	1✔
178	reconcile.Request{	1✔
179	NamespacedName: types.NamespacedName{	1✔
180	Name: config.GetName(),	1✔
181	},	1✔
182	})	1✔
183	}	1✔
184
185	return reconcileRequests	1✔
186	}
187
NEW 188	func (r *GPUClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {	×
NEW 189	// The state manager renders the DRA driver operand for the GPUCluster.	×
NEW 190	stateManager, err := state.NewManager(	×
NEW 191	nvidiav1alpha1.GPUClusterCRDName,	×
NEW 192	r.Namespace,	×
NEW 193	mgr.GetClient(),	×
NEW 194	mgr.GetScheme())	×
NEW 195	if err != nil {	×
NEW 196	return fmt.Errorf("error creating state manager: %w", err)	×
NEW 197	}	×
NEW 198	r.stateManager = stateManager	×
NEW 199		×
NEW 200	r.conditionUpdater = conditions.NewGPUClusterUpdater(mgr.GetClient())	×
NEW 201		×
NEW 202	c, err := controller.New("gpu-cluster-controller", mgr, controller.Options{	×
NEW 203	Reconciler: r,	×
NEW 204	MaxConcurrentReconciles: 1,	×
NEW 205	RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),	×
NEW 206	})	×
NEW 207	if err != nil {	×
NEW 208	return err	×
NEW 209	}	×
210
NEW 211	err = c.Watch(source.Kind(	×
NEW 212	mgr.GetCache(),	×
NEW 213	&nvidiav1alpha1.GPUCluster{},	×
NEW 214	handler.TypedEnqueueRequestsFromMapFunc(r.enqueueAllGPUClusters),	×
NEW 215	predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUCluster]{},	×
NEW 216	),	×
NEW 217	)	×
NEW 218	if err != nil {	×
NEW 219	return err	×
NEW 220	}	×
221
222	// Watch the secondary resources each state manager owns.
NEW 223	watchSources := stateManager.GetWatchSources(mgr)	×
NEW 224	for _, watchSource := range watchSources {	×
NEW 225	err = c.Watch(	×
NEW 226	watchSource,	×
NEW 227	)	×
NEW 228	if err != nil {	×
NEW 229	return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)	×
NEW 230	}	×
231	}
232
NEW 233	return nil	×
234	}

NVIDIA / gpu-operator / 29513807383

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous