27042525148

Committed 05 Jun 2026 10:04PM UTC coverage: 29.212% (+0.4%) from 28.792%

Build # 27042525148

Build Type

push

github

Committed by

karthikvetrivel

Commit Message

Add DRA driver operand to GPUClusterConfig controller

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Coverage Stats

128 of 309 new or added lines in 10 files covered. (41.42%)

4 existing lines in 3 files now uncovered.

3946 of 13508 relevant lines covered (29.21%)

0.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

40.29

/controllers/gpuclusterconfig_controller.go

/*
Copyright 2025.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
        "context"
        "fmt"
        "time"

        apierrors "k8s.io/apimachinery/pkg/api/errors"
        "k8s.io/apimachinery/pkg/runtime"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/client-go/util/workqueue"
        ctrl "sigs.k8s.io/controller-runtime"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller"
        "sigs.k8s.io/controller-runtime/pkg/handler"
        "sigs.k8s.io/controller-runtime/pkg/log"
        "sigs.k8s.io/controller-runtime/pkg/predicate"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
        "sigs.k8s.io/controller-runtime/pkg/source"

        nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
        "github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
        "github.com/NVIDIA/gpu-operator/internal/conditions"
        "github.com/NVIDIA/gpu-operator/internal/consts"
        "github.com/NVIDIA/gpu-operator/internal/state"
)

// GPUClusterConfigReconciler reconciles a GPUClusterConfig object
type GPUClusterConfigReconciler struct {
        client.Client
        Scheme      *runtime.Scheme
        ClusterInfo clusterinfo.Interface
        Namespace   string

        stateManager     state.Manager
        conditionUpdater conditions.Updater

        // singleton is the GPUClusterConfig that owns reconciliation; the first instance to
        // reconcile claims it (first-wins), mirroring ClusterPolicy.
        singleton *nvidiav1alpha1.GPUClusterConfig
}

//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/finalizers,verbs=update

func (r *GPUClusterConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
        logger := log.FromContext(ctx)
        logger.V(consts.LogLevelInfo).Info("Reconciling GPUClusterConfig")

        instance := &nvidiav1alpha1.GPUClusterConfig{}
        if err := r.Get(ctx, req.NamespacedName, instance); err != nil {
                if apierrors.IsNotFound(err) {
                        // Deleted; owned objects are garbage-collected, so there is nothing to clean up.
                        return ctrl.Result{}, nil
                }
                wrappedErr := fmt.Errorf("error getting GPUClusterConfig object: %w", err)
                logger.Error(err, "error getting GPUClusterConfig object")
                instance.Status.State = nvidiav1alpha1.NotReady
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, wrappedErr.Error()); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return ctrl.Result{}, wrappedErr
        }

        // Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
        // claims ownership; any other instance is marked Ignored and skipped. The owner is
        // held in memory, so the choice resets on operator restart.
        if r.singleton != nil && r.singleton.Name != instance.Name {
                logger.V(consts.LogLevelWarning).Info("Multiple GPUClusterConfig instances found, ignoring this one",
                        "name", instance.Name, "owner", r.singleton.Name)
                if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {
                        return ctrl.Result{}, err
                }
                return ctrl.Result{}, nil
        }
        r.singleton = instance

        infoCatalog := state.NewInfoCatalog()
        infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)

        managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)

        if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {
                return ctrl.Result{}, err
        }

        if managerStatus.Status != state.SyncStateReady {
                logger.Info("GPUClusterConfig instance is not ready")
                var errorInfo error
                for _, result := range managerStatus.StatesStatus {
                        if result.Status != state.SyncStateReady && result.ErrInfo != nil {
                                errorInfo = result.ErrInfo
                                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, errorInfo)); condErr != nil {
                                        logger.Error(condErr, "failed to set condition")
                                }
                                break
                        }
                }
                // if no errors are reported from any state, then we are waiting on operand pods
                if errorInfo == nil {
                        if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {
                                logger.Error(condErr, "failed to set condition")
                        }
                }
                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
        }

        if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {
                logger.Error(condErr, "failed to set condition")
                return ctrl.Result{}, condErr
        }
        return ctrl.Result{}, nil
}

// updateCrStatus writes desired to the CR's status, skipping the write when it is already current.
func (r *GPUClusterConfigReconciler) updateCrStatus(ctx context.Context, cr *nvidiav1alpha1.GPUClusterConfig, desired nvidiav1alpha1.State) error {
        reqLogger := log.FromContext(ctx)

        // Refetch to avoid a resourceVersion conflict.
        instance := &nvidiav1alpha1.GPUClusterConfig{}
        if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {
                reqLogger.Error(err, "Failed to get GPUClusterConfig instance for status update")
                return err
        }

        if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {
                return nil
        }
        instance.Status.State = desired
        instance.Status.Namespace = r.Namespace

        reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)
        if err := r.Status().Update(ctx, instance); err != nil {
                reqLogger.Error(err, "Failed to update CR status")
                return err
        }
        cr.Status.State = instance.Status.State
        cr.Status.Namespace = instance.Status.Namespace
        return nil
}

// enqueueAllGPUClusterConfigs enqueues every instance so each is reconciled when any
// instance or owned resource changes.
func (r *GPUClusterConfigReconciler) enqueueAllGPUClusterConfigs(ctx context.Context) []reconcile.Request {
        logger := log.FromContext(ctx)
        list := &nvidiav1alpha1.GPUClusterConfigList{}

        if err := r.List(ctx, list); err != nil {
                logger.Error(err, "Unable to list GPUClusterConfig resources")
                return []reconcile.Request{}
        }

        reconcileRequests := make([]reconcile.Request, 0, len(list.Items))
        for _, config := range list.Items {
                reconcileRequests = append(reconcileRequests,
                        reconcile.Request{
                                NamespacedName: types.NamespacedName{
                                        Name: config.GetName(),
                                },
                        })
        }

        return reconcileRequests
}

func (r *GPUClusterConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
        // The state manager renders the DRA driver operand for the GPUClusterConfig.
        stateManager, err := state.NewManager(
                nvidiav1alpha1.GPUClusterConfigCRDName,
                r.Namespace,
                mgr.GetClient(),
                mgr.GetScheme())
        if err != nil {
                return fmt.Errorf("error creating state manager: %v", err)
        }
        r.stateManager = stateManager

        r.conditionUpdater = conditions.NewGPUClusterConfigUpdater(mgr.GetClient())

        c, err := controller.New("gpu-cluster-config-controller", mgr, controller.Options{
                Reconciler:              r,
                MaxConcurrentReconciles: 1,
                RateLimiter:             workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),
        })
        if err != nil {
                return err
        }

        gpuClusterConfigMapFn := func(ctx context.Context, _ *nvidiav1alpha1.GPUClusterConfig) []reconcile.Request {
                return r.enqueueAllGPUClusterConfigs(ctx)
        }

        err = c.Watch(source.Kind(
                mgr.GetCache(),
                &nvidiav1alpha1.GPUClusterConfig{},
                handler.TypedEnqueueRequestsFromMapFunc(gpuClusterConfigMapFn),
                predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUClusterConfig]{},
        ),
        )
        if err != nil {
                return err
        }

        // Watch the secondary resources each state manager owns.
        watchSources := stateManager.GetWatchSources(mgr)
        for _, watchSource := range watchSources {
                err = c.Watch(
                        watchSource,
                )
                if err != nil {
                        return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)
                }
        }

        return nil
}

1	/*
2	Copyright 2025.
3
4	Licensed under the Apache License, Version 2.0 (the "License");
5	you may not use this file except in compliance with the License.
6	You may obtain a copy of the License at
7
8	http://www.apache.org/licenses/LICENSE-2.0
9
10	Unless required by applicable law or agreed to in writing, software
11	distributed under the License is distributed on an "AS IS" BASIS,
12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	See the License for the specific language governing permissions and
14	limitations under the License.
15	*/
16
17	package controllers
18
19	import (
20	"context"
21	"fmt"
22	"time"
23
24	apierrors "k8s.io/apimachinery/pkg/api/errors"
25	"k8s.io/apimachinery/pkg/runtime"
26	"k8s.io/apimachinery/pkg/types"
27	"k8s.io/client-go/util/workqueue"
28	ctrl "sigs.k8s.io/controller-runtime"
29	"sigs.k8s.io/controller-runtime/pkg/client"
30	"sigs.k8s.io/controller-runtime/pkg/controller"
31	"sigs.k8s.io/controller-runtime/pkg/handler"
32	"sigs.k8s.io/controller-runtime/pkg/log"
33	"sigs.k8s.io/controller-runtime/pkg/predicate"
34	"sigs.k8s.io/controller-runtime/pkg/reconcile"
35	"sigs.k8s.io/controller-runtime/pkg/source"
36
37	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
38	"github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
39	"github.com/NVIDIA/gpu-operator/internal/conditions"
40	"github.com/NVIDIA/gpu-operator/internal/consts"
41	"github.com/NVIDIA/gpu-operator/internal/state"
42	)
43
44	// GPUClusterConfigReconciler reconciles a GPUClusterConfig object
45	type GPUClusterConfigReconciler struct {
46	client.Client
47	Scheme *runtime.Scheme
48	ClusterInfo clusterinfo.Interface
49	Namespace string
50
51	stateManager state.Manager
52	conditionUpdater conditions.Updater
53
54	// singleton is the GPUClusterConfig that owns reconciliation; the first instance to
55	// reconcile claims it (first-wins), mirroring ClusterPolicy.
56	singleton *nvidiav1alpha1.GPUClusterConfig
57	}
58
59	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs,verbs=get;list;watch;create;update;patch;delete
60	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/status,verbs=get;update;patch
61	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/finalizers,verbs=update
62
63	func (r *GPUClusterConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {	1✔
64	logger := log.FromContext(ctx)	1✔
65	logger.V(consts.LogLevelInfo).Info("Reconciling GPUClusterConfig")	1✔
66		1✔
67	instance := &nvidiav1alpha1.GPUClusterConfig{}	1✔
68	if err := r.Get(ctx, req.NamespacedName, instance); err != nil {	2✔
69	if apierrors.IsNotFound(err) {	2✔
70	// Deleted; owned objects are garbage-collected, so there is nothing to clean up.	1✔
71	return ctrl.Result{}, nil	1✔
72	}	1✔
73	wrappedErr := fmt.Errorf("error getting GPUClusterConfig object: %w", err)	×
74	logger.Error(err, "error getting GPUClusterConfig object")	×
75	instance.Status.State = nvidiav1alpha1.NotReady	×
76	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, wrappedErr.Error()); condErr != nil {	×
77	logger.Error(condErr, "failed to set condition")	×
78	}	×
NEW 79	return ctrl.Result{}, wrappedErr	×
80	}
81
82	// Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
83	// claims ownership; any other instance is marked Ignored and skipped. The owner is
84	// held in memory, so the choice resets on operator restart.
85	if r.singleton != nil && r.singleton.Name != instance.Name {	2✔
86	logger.V(consts.LogLevelWarning).Info("Multiple GPUClusterConfig instances found, ignoring this one",	1✔
87	"name", instance.Name, "owner", r.singleton.Name)	1✔
88	if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {	1✔
NEW 89	return ctrl.Result{}, err	×
90	}	×
91	return ctrl.Result{}, nil	1✔
92	}
93	r.singleton = instance	1✔
94		1✔
95	infoCatalog := state.NewInfoCatalog()	1✔
96	infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)	1✔
97		1✔
98	managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)	1✔
99		1✔
100	if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {	1✔
101	return ctrl.Result{}, err	×
102	}	×
103
104	if managerStatus.Status != state.SyncStateReady {	1✔
105	logger.Info("GPUClusterConfig instance is not ready")	×
106	var errorInfo error	×
107	for _, result := range managerStatus.StatesStatus {	×
108	if result.Status != state.SyncStateReady && result.ErrInfo != nil {	×
109	errorInfo = result.ErrInfo	×
NEW 110	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, errorInfo)); condErr != nil {	×
111	logger.Error(condErr, "failed to set condition")	×
112	}	×
113	break	×
114	}
115	}
116	// if no errors are reported from any state, then we are waiting on operand pods
117	if errorInfo == nil {	×
118	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {	×
119	logger.Error(condErr, "failed to set condition")	×
120	}	×
121	}
NEW 122	return ctrl.Result{RequeueAfter: time.Second * 5}, nil	×
123	}
124
125	if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {	1✔
126	logger.Error(condErr, "failed to set condition")	×
127	return ctrl.Result{}, condErr	×
128	}	×
129	return ctrl.Result{}, nil	1✔
130	}
131
132	// updateCrStatus writes desired to the CR's status, skipping the write when it is already current.
133	func (r GPUClusterConfigReconciler) updateCrStatus(ctx context.Context, cr nvidiav1alpha1.GPUClusterConfig, desired nvidiav1alpha1.State) error {	1✔
134	reqLogger := log.FromContext(ctx)	1✔
135		1✔
136	// Refetch to avoid a resourceVersion conflict.	1✔
137	instance := &nvidiav1alpha1.GPUClusterConfig{}	1✔
138	if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {	1✔
139	reqLogger.Error(err, "Failed to get GPUClusterConfig instance for status update")	×
140	return err	×
141	}	×
142
143	if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {	1✔
144	return nil	×
145	}	×
146	instance.Status.State = desired	1✔
147	instance.Status.Namespace = r.Namespace	1✔
148		1✔
149	reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)	1✔
150	if err := r.Status().Update(ctx, instance); err != nil {	1✔
151	reqLogger.Error(err, "Failed to update CR status")	×
152	return err	×
153	}	×
154	cr.Status.State = instance.Status.State	1✔
155	cr.Status.Namespace = instance.Status.Namespace	1✔
156	return nil	1✔
157	}
158
159	// enqueueAllGPUClusterConfigs enqueues every instance so each is reconciled when any
160	// instance or owned resource changes.
161	func (r *GPUClusterConfigReconciler) enqueueAllGPUClusterConfigs(ctx context.Context) []reconcile.Request {	1✔
162	logger := log.FromContext(ctx)	1✔
163	list := &nvidiav1alpha1.GPUClusterConfigList{}	1✔
164		1✔
165	if err := r.List(ctx, list); err != nil {	1✔
166	logger.Error(err, "Unable to list GPUClusterConfig resources")	×
167	return []reconcile.Request{}	×
168	}	×
169
170	reconcileRequests := make([]reconcile.Request, 0, len(list.Items))	1✔
171	for _, config := range list.Items {	2✔
172	reconcileRequests = append(reconcileRequests,	1✔
173	reconcile.Request{	1✔
174	NamespacedName: types.NamespacedName{	1✔
175	Name: config.GetName(),	1✔
176	},	1✔
177	})	1✔
178	}	1✔
179
180	return reconcileRequests	1✔
181	}
182
183	func (r *GPUClusterConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {	×
NEW 184	// The state manager renders the DRA driver operand for the GPUClusterConfig.	×
185	stateManager, err := state.NewManager(	×
186	nvidiav1alpha1.GPUClusterConfigCRDName,	×
187	r.Namespace,	×
188	mgr.GetClient(),	×
189	mgr.GetScheme())	×
190	if err != nil {	×
191	return fmt.Errorf("error creating state manager: %v", err)	×
192	}	×
193	r.stateManager = stateManager	×
194		×
195	r.conditionUpdater = conditions.NewGPUClusterConfigUpdater(mgr.GetClient())	×
196		×
197	c, err := controller.New("gpu-cluster-config-controller", mgr, controller.Options{	×
198	Reconciler: r,	×
199	MaxConcurrentReconciles: 1,	×
200	RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),	×
201	})	×
202	if err != nil {	×
203	return err	×
204	}	×
205
206	gpuClusterConfigMapFn := func(ctx context.Context, _ *nvidiav1alpha1.GPUClusterConfig) []reconcile.Request {	×
207	return r.enqueueAllGPUClusterConfigs(ctx)	×
208	}	×
209
210	err = c.Watch(source.Kind(	×
211	mgr.GetCache(),	×
212	&nvidiav1alpha1.GPUClusterConfig{},	×
213	handler.TypedEnqueueRequestsFromMapFunc(gpuClusterConfigMapFn),	×
214	predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUClusterConfig]{},	×
215	),	×
216	)	×
217	if err != nil {	×
218	return err	×
219	}	×
220
221	// Watch the secondary resources each state manager owns.
222	watchSources := stateManager.GetWatchSources(mgr)	×
223	for _, watchSource := range watchSources {	×
224	err = c.Watch(	×
225	watchSource,	×
226	)	×
227	if err != nil {	×
228	return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)	×
229	}	×
230	}
231
232	return nil	×
233	}

NVIDIA / gpu-operator / 27042525148

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous