• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

NVIDIA / gpu-operator / 28038232541

10 Jun 2026 03:04PM UTC coverage: 29.069% (-2.5%) from 31.519%
28038232541

Pull #2571

github

karthikvetrivel
Add standalone GPU Feature Discovery operand to GPUClusterConfig

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>
Pull Request #2571: Add GPUClusterConfig CRD and controller for DRA-based stack

248 of 1061 new or added lines in 19 files covered. (23.37%)

1058 existing lines in 8 files now uncovered.

3956 of 13609 relevant lines covered (29.07%)

0.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.07
/controllers/gpuclusterconfig_controller.go
1
/*
2
Copyright 2025.
3

4
Licensed under the Apache License, Version 2.0 (the "License");
5
you may not use this file except in compliance with the License.
6
You may obtain a copy of the License at
7

8
    http://www.apache.org/licenses/LICENSE-2.0
9

10
Unless required by applicable law or agreed to in writing, software
11
distributed under the License is distributed on an "AS IS" BASIS,
12
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
See the License for the specific language governing permissions and
14
limitations under the License.
15
*/
16

17
package controllers
18

19
import (
20
        "context"
21
        "fmt"
22
        "time"
23

24
        apierrors "k8s.io/apimachinery/pkg/api/errors"
25
        "k8s.io/apimachinery/pkg/runtime"
26
        "k8s.io/apimachinery/pkg/types"
27
        "k8s.io/client-go/util/workqueue"
28
        ctrl "sigs.k8s.io/controller-runtime"
29
        "sigs.k8s.io/controller-runtime/pkg/client"
30
        "sigs.k8s.io/controller-runtime/pkg/controller"
31
        "sigs.k8s.io/controller-runtime/pkg/handler"
32
        "sigs.k8s.io/controller-runtime/pkg/log"
33
        "sigs.k8s.io/controller-runtime/pkg/predicate"
34
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
35
        "sigs.k8s.io/controller-runtime/pkg/source"
36

37
        gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
38
        nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
39
        "github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
40
        "github.com/NVIDIA/gpu-operator/internal/conditions"
41
        "github.com/NVIDIA/gpu-operator/internal/consts"
42
        "github.com/NVIDIA/gpu-operator/internal/state"
43
)
44

45
// GPUClusterConfigReconciler reconciles a GPUClusterConfig object
46
type GPUClusterConfigReconciler struct {
47
        client.Client
48
        Scheme      *runtime.Scheme
49
        ClusterInfo clusterinfo.Interface
50
        Namespace   string
51

52
        stateManager     state.Manager
53
        conditionUpdater conditions.Updater
54

55
        // singleton is the GPUClusterConfig that owns reconciliation; the first instance to
56
        // reconcile claims it (first-wins), mirroring ClusterPolicy.
57
        singleton *nvidiav1alpha1.GPUClusterConfig
58
}
59

60
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs,verbs=get;list;watch;create;update;patch;delete
61
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/status,verbs=get;update;patch
62
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusterconfigs/finalizers,verbs=update
63
//+kubebuilder:rbac:groups=nvidia.com,resources=clusterpolicies,verbs=get;list;watch
64
//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;update;patch
65
//+kubebuilder:rbac:groups=resource.k8s.io,resources=resourceclaimtemplates,verbs=get;list;watch;create;update;delete
66

67
func (r *GPUClusterConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
1✔
68
        logger := log.FromContext(ctx)
1✔
69
        logger.V(consts.LogLevelInfo).Info("Reconciling GPUClusterConfig")
1✔
70

1✔
71
        instance := &nvidiav1alpha1.GPUClusterConfig{}
1✔
72
        if err := r.Get(ctx, req.NamespacedName, instance); err != nil {
2✔
73
                if apierrors.IsNotFound(err) {
2✔
74
                        // Deleted; owned objects are garbage-collected, so there is nothing to clean up.
1✔
75
                        return ctrl.Result{}, nil
1✔
76
                }
1✔
77
                // instance was not populated by the failed Get, so there is no object to
78
                // update status on; just surface the error for requeue.
NEW
79
                logger.Error(err, "error getting GPUClusterConfig object")
×
NEW
80
                return ctrl.Result{}, fmt.Errorf("error getting GPUClusterConfig object: %w", err)
×
81
        }
82

83
        // GPUClusterConfig (DRA path) is mutually exclusive with ClusterPolicy: if one
84
        // exists, yield to it rather than deploying the DRA stack alongside it.
85
        clusterPolicies := &gpuv1.ClusterPolicyList{}
1✔
86
        if err := r.List(ctx, clusterPolicies); err != nil {
1✔
NEW
87
                return ctrl.Result{}, fmt.Errorf("error listing ClusterPolicies: %w", err)
×
NEW
88
        }
×
89
        if len(clusterPolicies.Items) > 0 {
2✔
90
                logger.V(consts.LogLevelWarning).Info("ClusterPolicy present, skipping mutually exclusive GPUClusterConfig")
1✔
91
                if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.Disabled); err != nil {
1✔
NEW
92
                        return ctrl.Result{}, err
×
NEW
93
                }
×
94
                msg := "GPUClusterConfig is mutually exclusive with ClusterPolicy; remove the ClusterPolicy or disable GPUClusterConfig"
1✔
95
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, msg); condErr != nil {
1✔
NEW
96
                        logger.Error(condErr, "failed to set condition")
×
NEW
97
                }
×
98
                return ctrl.Result{}, nil
1✔
99
        }
100

101
        // Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
102
        // claims ownership; any other instance is marked Ignored and skipped. The owner is
103
        // held in memory, so the choice resets on operator restart.
104
        if r.singleton != nil && r.singleton.Name != instance.Name {
2✔
105
                logger.V(consts.LogLevelWarning).Info("Multiple GPUClusterConfig instances found, ignoring this one",
1✔
106
                        "name", instance.Name, "owner", r.singleton.Name)
1✔
107
                if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {
1✔
NEW
108
                        return ctrl.Result{}, err
×
NEW
109
                }
×
110
                return ctrl.Result{}, nil
1✔
111
        }
112
        r.singleton = instance
1✔
113

1✔
114
        infoCatalog := state.NewInfoCatalog()
1✔
115
        infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)
1✔
116

1✔
117
        managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)
1✔
118

1✔
119
        if err := r.updateCrStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {
1✔
NEW
120
                return ctrl.Result{}, err
×
NEW
121
        }
×
122

123
        if managerStatus.Status != state.SyncStateReady {
1✔
NEW
124
                logger.Info("GPUClusterConfig instance is not ready")
×
NEW
125
                for _, result := range managerStatus.StatesStatus {
×
NEW
126
                        if result.Status != state.SyncStateReady && result.ErrInfo != nil {
×
NEW
127
                                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, result.ErrInfo)); condErr != nil {
×
NEW
128
                                        logger.Error(condErr, "failed to set condition")
×
NEW
129
                                }
×
NEW
130
                                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
×
131
                        }
132
                }
133
                // no state reported an error, so we are waiting on operand pods
NEW
134
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {
×
NEW
135
                        logger.Error(condErr, "failed to set condition")
×
NEW
136
                }
×
NEW
137
                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
×
138
        }
139

140
        if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {
1✔
NEW
141
                logger.Error(condErr, "failed to set condition")
×
NEW
142
                return ctrl.Result{}, condErr
×
NEW
143
        }
×
144
        // Resync periodically so out-of-band changes (a deleted DeviceClass/VAP, or a
145
        // newly-created ClusterPolicy) are detected and reconciled even while ready;
146
        // only DaemonSets are watched, and the ready path is otherwise event-driven.
147
        return ctrl.Result{RequeueAfter: time.Minute}, nil
1✔
148
}
149

150
// updateCrStatus writes desired to the CR's status, skipping the write when it is already current.
151
func (r *GPUClusterConfigReconciler) updateCrStatus(ctx context.Context, cr *nvidiav1alpha1.GPUClusterConfig, desired nvidiav1alpha1.State) error {
1✔
152
        reqLogger := log.FromContext(ctx)
1✔
153

1✔
154
        // Refetch to avoid a resourceVersion conflict.
1✔
155
        instance := &nvidiav1alpha1.GPUClusterConfig{}
1✔
156
        if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {
1✔
NEW
157
                reqLogger.Error(err, "Failed to get GPUClusterConfig instance for status update")
×
NEW
158
                return err
×
NEW
159
        }
×
160

161
        if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {
1✔
NEW
162
                return nil
×
NEW
163
        }
×
164
        instance.Status.State = desired
1✔
165
        instance.Status.Namespace = r.Namespace
1✔
166

1✔
167
        reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)
1✔
168
        if err := r.Status().Update(ctx, instance); err != nil {
1✔
NEW
169
                reqLogger.Error(err, "Failed to update CR status")
×
NEW
170
                return err
×
NEW
171
        }
×
172
        cr.Status.State = instance.Status.State
1✔
173
        cr.Status.Namespace = instance.Status.Namespace
1✔
174
        return nil
1✔
175
}
176

177
// enqueueAllGPUClusterConfigs enqueues every instance so each is reconciled when any
178
// instance or owned resource changes.
179
func (r *GPUClusterConfigReconciler) enqueueAllGPUClusterConfigs(ctx context.Context, _ *nvidiav1alpha1.GPUClusterConfig) []reconcile.Request {
1✔
180
        logger := log.FromContext(ctx)
1✔
181
        list := &nvidiav1alpha1.GPUClusterConfigList{}
1✔
182

1✔
183
        if err := r.List(ctx, list); err != nil {
1✔
NEW
184
                logger.Error(err, "Unable to list GPUClusterConfig resources")
×
NEW
185
                return []reconcile.Request{}
×
NEW
186
        }
×
187

188
        reconcileRequests := make([]reconcile.Request, 0, len(list.Items))
1✔
189
        for _, config := range list.Items {
2✔
190
                reconcileRequests = append(reconcileRequests,
1✔
191
                        reconcile.Request{
1✔
192
                                NamespacedName: types.NamespacedName{
1✔
193
                                        Name: config.GetName(),
1✔
194
                                },
1✔
195
                        })
1✔
196
        }
1✔
197

198
        return reconcileRequests
1✔
199
}
200

NEW
201
func (r *GPUClusterConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
×
NEW
202
        // The state manager renders the DRA driver operand for the GPUClusterConfig.
×
NEW
203
        stateManager, err := state.NewManager(
×
NEW
204
                nvidiav1alpha1.GPUClusterConfigCRDName,
×
NEW
205
                r.Namespace,
×
NEW
206
                mgr.GetClient(),
×
NEW
207
                mgr.GetScheme())
×
NEW
208
        if err != nil {
×
NEW
209
                return fmt.Errorf("error creating state manager: %v", err)
×
NEW
210
        }
×
NEW
211
        r.stateManager = stateManager
×
NEW
212

×
NEW
213
        r.conditionUpdater = conditions.NewGPUClusterConfigUpdater(mgr.GetClient())
×
NEW
214

×
NEW
215
        c, err := controller.New("gpu-cluster-config-controller", mgr, controller.Options{
×
NEW
216
                Reconciler:              r,
×
NEW
217
                MaxConcurrentReconciles: 1,
×
NEW
218
                RateLimiter:             workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),
×
NEW
219
        })
×
NEW
220
        if err != nil {
×
NEW
221
                return err
×
NEW
222
        }
×
223

NEW
224
        err = c.Watch(source.Kind(
×
NEW
225
                mgr.GetCache(),
×
NEW
226
                &nvidiav1alpha1.GPUClusterConfig{},
×
NEW
227
                handler.TypedEnqueueRequestsFromMapFunc(r.enqueueAllGPUClusterConfigs),
×
NEW
228
                predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUClusterConfig]{},
×
NEW
229
        ),
×
NEW
230
        )
×
NEW
231
        if err != nil {
×
NEW
232
                return err
×
NEW
233
        }
×
234

235
        // Watch the secondary resources each state manager owns.
NEW
236
        watchSources := stateManager.GetWatchSources(mgr)
×
NEW
237
        for _, watchSource := range watchSources {
×
NEW
238
                err = c.Watch(
×
NEW
239
                        watchSource,
×
NEW
240
                )
×
NEW
241
                if err != nil {
×
NEW
242
                        return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)
×
NEW
243
                }
×
244
        }
245

NEW
246
        return nil
×
247
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc