30285057709

Committed 27 Jul 2026 04:27PM UTC coverage: 35.976% (+1.4%) from 34.591%

Build # 30285057709

Build Type

Pull #2572

github

Committed by

karthikvetrivel

Commit Message

Support NVIDIADriver reconciliation and driver upgrades without a ClusterPolicy

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Pull Request Pull Request #2572: Support NVIDIADriver reconciliation and driver upgrades without a ClusterPolicy

Coverage Stats

47 of 88 new or added lines in 5 files covered. (53.41%)

137 existing lines in 3 files now uncovered.

5209 of 14479 relevant lines covered (35.98%)

0.41 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.29

/controllers/gpucluster_controller.go

/**
# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/

package controllers

import (
        "context"
        "fmt"
        "strings"
        "time"

        appsv1 "k8s.io/api/apps/v1"
        corev1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/client-go/tools/events"
        "k8s.io/client-go/util/workqueue"
        ctrl "sigs.k8s.io/controller-runtime"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller"
        "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
        "sigs.k8s.io/controller-runtime/pkg/handler"
        "sigs.k8s.io/controller-runtime/pkg/log"
        "sigs.k8s.io/controller-runtime/pkg/predicate"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
        "sigs.k8s.io/controller-runtime/pkg/source"

        gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
        nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
        "github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
        "github.com/NVIDIA/gpu-operator/internal/conditions"
        "github.com/NVIDIA/gpu-operator/internal/consts"
        "github.com/NVIDIA/gpu-operator/internal/state"
        "github.com/NVIDIA/gpu-operator/internal/utils"
)

// gpuClusterFinalizer holds the GPUCluster until reconcileDelete has ordered teardown.
const gpuClusterFinalizer = "gpucluster.nvidia.com/dra-resourceclaim"

// draAdminNamespaceLabelKey is the label the kube-scheduler requires on a namespace
// before it allows adminAccess: true in ResourceClaim/ResourceClaimTemplate objects.
const draAdminNamespaceLabelKey = "resource.kubernetes.io/admin-access"

// GPUClusterReconciler reconciles a GPUCluster object
type GPUClusterReconciler struct {
        client.Client
        Scheme      *runtime.Scheme
        ClusterInfo clusterinfo.Interface
        Namespace   string

        stateManager     state.Manager
        conditionUpdater conditions.Updater
        recorder         events.EventRecorder

        // singleton is the GPUCluster that owns reconciliation; the first instance to
        // reconcile claims it (first-wins), mirroring ClusterPolicy.
        singleton *nvidiav1alpha1.GPUCluster
}

//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/finalizers,verbs=update
//+kubebuilder:rbac:groups=nvidia.com,resources=clusterpolicies,verbs=get;list;watch
//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;update;patch
//+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
//+kubebuilder:rbac:groups=resource.k8s.io,resources=resourceclaimtemplates,verbs=get;list;watch;create;update;delete

func (r *GPUClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
        logger := log.FromContext(ctx)
        logger.V(consts.LogLevelInfo).Info("Reconciling GPUCluster")

        instance := &nvidiav1alpha1.GPUCluster{}
        if err := r.Get(ctx, req.NamespacedName, instance); err != nil {
                if apierrors.IsNotFound(err) {
                        // Deleted; owned objects are garbage-collected, so there is nothing to clean up.
                        return ctrl.Result{}, nil
                }
                // instance was not populated by the failed Get, so there is no object to
                // update status on; just surface the error for requeue.
                logger.Error(err, "error getting GPUCluster object")
                return ctrl.Result{}, fmt.Errorf("error getting GPUCluster object: %w", err)
        }

        if !instance.DeletionTimestamp.IsZero() {
                return r.reconcileDelete(ctx, instance)
        }

        if err := utils.EnsureFinalizer(ctx, r.Client, instance, gpuClusterFinalizer); err != nil {
                return ctrl.Result{}, fmt.Errorf("error adding finalizer to GPUCluster %s: %w", req.NamespacedName, err)
        }

        // GPUCluster (DRA stack) may coexist with a ClusterPolicy (device-plugin
        // stack): every operand DaemonSet of both stacks gates on the per-node
        // nvidia.com/gpu-operator.resource-allocation.mode label, so each node is served by exactly one stack.

        // Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
        // claims ownership; any other instance is marked Ignored and skipped. The owner is
        // held in memory, so the choice resets on operator restart.
        if r.singleton != nil && r.singleton.Name != instance.Name {
                logger.V(consts.LogLevelWarning).Info("Multiple GPUCluster instances found, ignoring this one",
                        "name", instance.Name, "owner", r.singleton.Name)
                if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {
                        return ctrl.Result{}, err
                }
                return ctrl.Result{}, nil
        }
        r.singleton = instance

        // Enabling DRA requires that all driver management is done through NVIDIADriver
        // CRs: a coexisting ClusterPolicy must delegate its driver with
        // useNvidiaDriverCRD=true. Surface the unmet prerequisite on this CR's status
        // and hold off deploying operands until it is met.
        if msg, err := r.validatePrerequisites(ctx); err != nil {
                return ctrl.Result{}, err
        } else if msg != "" {
                logger.V(consts.LogLevelWarning).Info("GPUCluster prerequisite not met", "reason", msg)
                if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.NotReady); err != nil {
                        return ctrl.Result{}, err
                }
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.PrerequisiteNotMet, msg); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return ctrl.Result{RequeueAfter: time.Minute}, nil
        }

        // The operand states render ResourceClaimTemplates with adminAccess: true, which the
        // kube-scheduler only admits from a labeled namespace; label it before syncing states.
        if err := r.ensureAdminAccessLabel(ctx); err != nil {
                return ctrl.Result{}, fmt.Errorf("failed to label namespace for admin access: %w", err)
        }

        infoCatalog := state.NewInfoCatalog()
        infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)

        managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)

        if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {
                return ctrl.Result{}, err
        }

        if managerStatus.Status != state.SyncStateReady {
                logger.Info("GPUCluster instance is not ready")
                for _, result := range managerStatus.StatesStatus {
                        if result.Status != state.SyncStateReady && result.ErrInfo != nil {
                                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, result.ErrInfo)); condErr != nil {
                                        logger.Error(condErr, "failed to set condition")
                                }
                                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
                        }
                }
                // no state reported an error, so we are waiting on operand pods
                if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {
                        logger.Error(condErr, "failed to set condition")
                }
                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
        }

        if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {
                logger.Error(condErr, "failed to set condition")
                return ctrl.Result{}, condErr
        }
        // Resync periodically so out-of-band changes (a deleted DeviceClass/VAP, or a
        // newly-created ClusterPolicy) are detected and reconciled even while ready;
        // only DaemonSets are watched, and the ready path is otherwise event-driven.
        return ctrl.Result{RequeueAfter: time.Minute}, nil
}

// validatePrerequisites checks the cross-CR rules that gate DRA enablement, returning a
// message describing the first unmet prerequisite or an empty string when all are met.
// The rules span multiple CRs, so they cannot be expressed as CRD schema validation and
// are re-evaluated on every reconcile.
func (r *GPUClusterReconciler) validatePrerequisites(ctx context.Context) (string, error) {
        clusterPolicies := &gpuv1.ClusterPolicyList{}
        if err := r.List(ctx, clusterPolicies); err != nil {
                return "", fmt.Errorf("error listing ClusterPolicy objects: %w", err)
        }
        for i := range clusterPolicies.Items {
                cp := &clusterPolicies.Items[i]
                if !cp.Spec.Driver.UseNvidiaDriverCRDType() {
                        return fmt.Sprintf("ClusterPolicy %s does not have driver.useNvidiaDriverCRD enabled; migrate driver management to NVIDIADriver CRs before enabling DRA", cp.Name), nil
                }
        }
        return "", nil
}

// ensureAdminAccessLabel patches the operator namespace with the label required by the
// kube-scheduler to allow adminAccess: true in ResourceClaim/ResourceClaimTemplate
// objects. The label is deliberately never removed: it is namespace-level configuration
// that other adminAccess consumers in the namespace may rely on.
func (r *GPUClusterReconciler) ensureAdminAccessLabel(ctx context.Context) error {
        ns := &corev1.Namespace{}
        if err := r.Get(ctx, client.ObjectKey{Name: r.Namespace}, ns); err != nil {
                return fmt.Errorf("could not get namespace %s: %w", r.Namespace, err)
        }
        if ns.Labels[draAdminNamespaceLabelKey] == "true" {
                return nil
        }
        patch := client.MergeFrom(ns.DeepCopy())
        if ns.Labels == nil {
                ns.Labels = make(map[string]string)
        }
        ns.Labels[draAdminNamespaceLabelKey] = "true"
        return r.Patch(ctx, ns, patch)
}

// reconcileDelete drains ResourceClaim-consuming DaemonSets before releasing the CR:
// garbage collection would otherwise delete the DRA kubelet plugin while their pods
// still need it to unprepare claims, leaving them stuck in Terminating. Foreground
// propagation keeps each DaemonSet present until its pods are gone, i.e. unprepared.
func (r *GPUClusterReconciler) reconcileDelete(ctx context.Context, instance *nvidiav1alpha1.GPUCluster) (ctrl.Result, error) {
        logger := log.FromContext(ctx)

        if !controllerutil.ContainsFinalizer(instance, gpuClusterFinalizer) {
                return ctrl.Result{}, nil
        }

        dsList := &appsv1.DaemonSetList{}
        if err := r.List(ctx, dsList, client.InNamespace(r.Namespace)); err != nil {
                return ctrl.Result{}, fmt.Errorf("error listing DaemonSets: %w", err)
        }
        var draining []string
        for i := range dsList.Items {
                ds := &dsList.Items[i]
                if !metav1.IsControlledBy(ds, instance) || len(ds.Spec.Template.Spec.ResourceClaims) == 0 {
                        continue
                }
                draining = append(draining, ds.Name)
                if ds.DeletionTimestamp.IsZero() {
                        logger.V(consts.LogLevelInfo).Info("Draining ResourceClaim-consuming DaemonSet before teardown", "DaemonSet", ds.Name)
                        if err := r.Delete(ctx, ds, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !apierrors.IsNotFound(err) {
                                return ctrl.Result{}, fmt.Errorf("error deleting DaemonSet %s: %w", ds.Name, err)
                        }
                }
        }
        if len(draining) > 0 {
                r.recorder.Eventf(instance, nil, corev1.EventTypeNormal, "DrainingClaimConsumers", "Delete",
                        "Waiting for ResourceClaim-consuming DaemonSet(s) to terminate: %s", strings.Join(draining, ", "))
                return ctrl.Result{RequeueAfter: time.Second * 5}, nil
        }

        controllerutil.RemoveFinalizer(instance, gpuClusterFinalizer)
        if err := r.Update(ctx, instance); err != nil {
                return ctrl.Result{}, fmt.Errorf("error removing finalizer: %w", err)
        }
        return ctrl.Result{}, nil
}

// updateCRStatus persists the given state (and the operator namespace) to the GPUCluster's
// .status subresource. It refetches the CR first to avoid resourceVersion conflicts and skips
// the API write when the status is already current. The desired status is mirrored onto cr
// up front so it is set on every non-error path.
func (r *GPUClusterReconciler) updateCRStatus(ctx context.Context, cr *nvidiav1alpha1.GPUCluster, desired nvidiav1alpha1.State) error {
        reqLogger := log.FromContext(ctx)

        // Refetch to avoid a resourceVersion conflict.
        instance := &nvidiav1alpha1.GPUCluster{}
        if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {
                reqLogger.Error(err, "Failed to get GPUCluster instance for status update")
                return err
        }
        cr.Status.State = desired
        cr.Status.Namespace = r.Namespace

        if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {
                return nil
        }
        instance.Status.State = desired
        instance.Status.Namespace = r.Namespace

        reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)
        if err := r.Status().Update(ctx, instance); err != nil {
                reqLogger.Error(err, "Failed to update CR status")
                return err
        }
        return nil
}

// enqueueAllGPUClusters enqueues every instance so each is reconciled when any
// instance or owned resource changes.
func (r *GPUClusterReconciler) enqueueAllGPUClusters(ctx context.Context, _ *nvidiav1alpha1.GPUCluster) []reconcile.Request {
        logger := log.FromContext(ctx)
        list := &nvidiav1alpha1.GPUClusterList{}

        if err := r.List(ctx, list); err != nil {
                logger.Error(err, "Unable to list GPUCluster resources")
                return []reconcile.Request{}
        }

        reconcileRequests := make([]reconcile.Request, 0, len(list.Items))
        for _, config := range list.Items {
                reconcileRequests = append(reconcileRequests,
                        reconcile.Request{
                                NamespacedName: types.NamespacedName{
                                        Name: config.GetName(),
                                },
                        })
        }

        return reconcileRequests
}

func (r *GPUClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
        // The state manager renders the DRA driver operand for the GPUCluster.
        stateManager, err := state.NewManager(
                nvidiav1alpha1.GPUClusterCRDName,
                r.Namespace,
                mgr.GetClient(),
                mgr.GetScheme())
        if err != nil {
                return fmt.Errorf("error creating state manager: %w", err)
        }
        r.stateManager = stateManager

        r.conditionUpdater = conditions.NewGPUClusterUpdater(mgr.GetClient())
        r.recorder = mgr.GetEventRecorder("nvidia-gpu-operator")

        c, err := controller.New("gpu-cluster-controller", mgr, controller.Options{
                Reconciler:              r,
                MaxConcurrentReconciles: 1,
                RateLimiter:             workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),
        })
        if err != nil {
                return err
        }

        err = c.Watch(source.Kind(
                mgr.GetCache(),
                &nvidiav1alpha1.GPUCluster{},
                handler.TypedEnqueueRequestsFromMapFunc(r.enqueueAllGPUClusters),
                predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUCluster]{},
        ),
        )
        if err != nil {
                return err
        }

        // Watch the secondary resources each state manager owns.
        watchSources := stateManager.GetWatchSources(mgr)
        for _, watchSource := range watchSources {
                err = c.Watch(
                        watchSource,
                )
                if err != nil {
                        return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)
                }
        }

        return nil
}

1	/**
2	# Copyright (c) NVIDIA CORPORATION. All rights reserved.
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	**/
16
17	package controllers
18
19	import (
20	"context"
21	"fmt"
22	"strings"
23	"time"
24
25	appsv1 "k8s.io/api/apps/v1"
26	corev1 "k8s.io/api/core/v1"
27	apierrors "k8s.io/apimachinery/pkg/api/errors"
28	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29	"k8s.io/apimachinery/pkg/runtime"
30	"k8s.io/apimachinery/pkg/types"
31	"k8s.io/client-go/tools/events"
32	"k8s.io/client-go/util/workqueue"
33	ctrl "sigs.k8s.io/controller-runtime"
34	"sigs.k8s.io/controller-runtime/pkg/client"
35	"sigs.k8s.io/controller-runtime/pkg/controller"
36	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
37	"sigs.k8s.io/controller-runtime/pkg/handler"
38	"sigs.k8s.io/controller-runtime/pkg/log"
39	"sigs.k8s.io/controller-runtime/pkg/predicate"
40	"sigs.k8s.io/controller-runtime/pkg/reconcile"
41	"sigs.k8s.io/controller-runtime/pkg/source"
42
43	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
44	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
45	"github.com/NVIDIA/gpu-operator/controllers/clusterinfo"
46	"github.com/NVIDIA/gpu-operator/internal/conditions"
47	"github.com/NVIDIA/gpu-operator/internal/consts"
48	"github.com/NVIDIA/gpu-operator/internal/state"
49	"github.com/NVIDIA/gpu-operator/internal/utils"
50	)
51
52	// gpuClusterFinalizer holds the GPUCluster until reconcileDelete has ordered teardown.
53	const gpuClusterFinalizer = "gpucluster.nvidia.com/dra-resourceclaim"
54
55	// draAdminNamespaceLabelKey is the label the kube-scheduler requires on a namespace
56	// before it allows adminAccess: true in ResourceClaim/ResourceClaimTemplate objects.
57	const draAdminNamespaceLabelKey = "resource.kubernetes.io/admin-access"
58
59	// GPUClusterReconciler reconciles a GPUCluster object
60	type GPUClusterReconciler struct {
61	client.Client
62	Scheme *runtime.Scheme
63	ClusterInfo clusterinfo.Interface
64	Namespace string
65
66	stateManager state.Manager
67	conditionUpdater conditions.Updater
68	recorder events.EventRecorder
69
70	// singleton is the GPUCluster that owns reconciliation; the first instance to
71	// reconcile claims it (first-wins), mirroring ClusterPolicy.
72	singleton *nvidiav1alpha1.GPUCluster
73	}
74
75	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters,verbs=get;list;watch;create;update;patch;delete
76	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/status,verbs=get;update;patch
77	//+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters/finalizers,verbs=update
78	//+kubebuilder:rbac:groups=nvidia.com,resources=clusterpolicies,verbs=get;list;watch
79	//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;update;patch
80	//+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
81	//+kubebuilder:rbac:groups=resource.k8s.io,resources=resourceclaimtemplates,verbs=get;list;watch;create;update;delete
82
83	func (r *GPUClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {	1✔
84	logger := log.FromContext(ctx)	1✔
85	logger.V(consts.LogLevelInfo).Info("Reconciling GPUCluster")	1✔
86		1✔
87	instance := &nvidiav1alpha1.GPUCluster{}	1✔
88	if err := r.Get(ctx, req.NamespacedName, instance); err != nil {	2✔
89	if apierrors.IsNotFound(err) {	2✔
90	// Deleted; owned objects are garbage-collected, so there is nothing to clean up.	1✔
91	return ctrl.Result{}, nil	1✔
92	}	1✔
93	// instance was not populated by the failed Get, so there is no object to
94	// update status on; just surface the error for requeue.
95	logger.Error(err, "error getting GPUCluster object")	×
96	return ctrl.Result{}, fmt.Errorf("error getting GPUCluster object: %w", err)	×
97	}
98
99	if !instance.DeletionTimestamp.IsZero() {	2✔
100	return r.reconcileDelete(ctx, instance)	1✔
101	}	1✔
102
103	if err := utils.EnsureFinalizer(ctx, r.Client, instance, gpuClusterFinalizer); err != nil {	1✔
104	return ctrl.Result{}, fmt.Errorf("error adding finalizer to GPUCluster %s: %w", req.NamespacedName, err)	×
105	}	×
106
107	// GPUCluster (DRA stack) may coexist with a ClusterPolicy (device-plugin
108	// stack): every operand DaemonSet of both stacks gates on the per-node
109	// nvidia.com/gpu-operator.resource-allocation.mode label, so each node is served by exactly one stack.
110
111	// Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
112	// claims ownership; any other instance is marked Ignored and skipped. The owner is
113	// held in memory, so the choice resets on operator restart.
114	if r.singleton != nil && r.singleton.Name != instance.Name {	2✔
115	logger.V(consts.LogLevelWarning).Info("Multiple GPUCluster instances found, ignoring this one",	1✔
116	"name", instance.Name, "owner", r.singleton.Name)	1✔
117	if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {	1✔
118	return ctrl.Result{}, err	×
119	}	×
120	return ctrl.Result{}, nil	1✔
121	}
122	r.singleton = instance	1✔
123		1✔
124	// Enabling DRA requires that all driver management is done through NVIDIADriver	1✔
125	// CRs: a coexisting ClusterPolicy must delegate its driver with	1✔
126	// useNvidiaDriverCRD=true. Surface the unmet prerequisite on this CR's status	1✔
127	// and hold off deploying operands until it is met.	1✔
128	if msg, err := r.validatePrerequisites(ctx); err != nil {	1✔
NEW 129	return ctrl.Result{}, err	×
130	} else if msg != "" {	2✔
131	logger.V(consts.LogLevelWarning).Info("GPUCluster prerequisite not met", "reason", msg)	1✔
132	if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.NotReady); err != nil {	1✔
NEW 133	return ctrl.Result{}, err	×
NEW 134	}	×
135	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.PrerequisiteNotMet, msg); condErr != nil {	1✔
NEW 136	logger.Error(condErr, "failed to set condition")	×
NEW 137	}	×
138	return ctrl.Result{RequeueAfter: time.Minute}, nil	1✔
139	}
140
141	// The operand states render ResourceClaimTemplates with adminAccess: true, which the
142	// kube-scheduler only admits from a labeled namespace; label it before syncing states.
143	if err := r.ensureAdminAccessLabel(ctx); err != nil {	1✔
UNCOV 144	return ctrl.Result{}, fmt.Errorf("failed to label namespace for admin access: %w", err)	×
UNCOV 145	}	×
146
147	infoCatalog := state.NewInfoCatalog()	1✔
148	infoCatalog.Add(state.InfoTypeClusterInfo, r.ClusterInfo)	1✔
149		1✔
150	managerStatus := r.stateManager.SyncState(ctx, instance, infoCatalog)	1✔
151		1✔
152	if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.State(managerStatus.Status)); err != nil {	1✔
UNCOV 153	return ctrl.Result{}, err	×
UNCOV 154	}	×
155
156	if managerStatus.Status != state.SyncStateReady {	1✔
157	logger.Info("GPUCluster instance is not ready")	×
158	for _, result := range managerStatus.StatesStatus {	×
159	if result.Status != state.SyncStateReady && result.ErrInfo != nil {	×
160	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, fmt.Sprintf("Error syncing state %s: %v", result.StateName, result.ErrInfo)); condErr != nil {	×
161	logger.Error(condErr, "failed to set condition")	×
UNCOV 162	}	×
UNCOV 163	return ctrl.Result{RequeueAfter: time.Second * 5}, nil	×
164	}
165	}
166	// no state reported an error, so we are waiting on operand pods
167	if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.OperandNotReady, "Waiting for operand pods to be ready"); condErr != nil {	×
168	logger.Error(condErr, "failed to set condition")	×
UNCOV 169	}	×
UNCOV 170	return ctrl.Result{RequeueAfter: time.Second * 5}, nil	×
171	}
172
173	if condErr := r.conditionUpdater.SetConditionsReady(ctx, instance, conditions.Reconciled, "All resources have been successfully reconciled"); condErr != nil {	1✔
174	logger.Error(condErr, "failed to set condition")	×
UNCOV 175	return ctrl.Result{}, condErr	×
UNCOV 176	}	×
177	// Resync periodically so out-of-band changes (a deleted DeviceClass/VAP, or a
178	// newly-created ClusterPolicy) are detected and reconciled even while ready;
179	// only DaemonSets are watched, and the ready path is otherwise event-driven.
180	return ctrl.Result{RequeueAfter: time.Minute}, nil	1✔
181	}
182
183	// validatePrerequisites checks the cross-CR rules that gate DRA enablement, returning a
184	// message describing the first unmet prerequisite or an empty string when all are met.
185	// The rules span multiple CRs, so they cannot be expressed as CRD schema validation and
186	// are re-evaluated on every reconcile.
187	func (r *GPUClusterReconciler) validatePrerequisites(ctx context.Context) (string, error) {	1✔
188	clusterPolicies := &gpuv1.ClusterPolicyList{}	1✔
189	if err := r.List(ctx, clusterPolicies); err != nil {	1✔
NEW 190	return "", fmt.Errorf("error listing ClusterPolicy objects: %w", err)	×
NEW 191	}	×
192	for i := range clusterPolicies.Items {	2✔
193	cp := &clusterPolicies.Items[i]	1✔
194	if !cp.Spec.Driver.UseNvidiaDriverCRDType() {	2✔
195	return fmt.Sprintf("ClusterPolicy %s does not have driver.useNvidiaDriverCRD enabled; migrate driver management to NVIDIADriver CRs before enabling DRA", cp.Name), nil	1✔
196	}	1✔
197	}
198	return "", nil	1✔
199	}
200
201	// ensureAdminAccessLabel patches the operator namespace with the label required by the
202	// kube-scheduler to allow adminAccess: true in ResourceClaim/ResourceClaimTemplate
203	// objects. The label is deliberately never removed: it is namespace-level configuration
204	// that other adminAccess consumers in the namespace may rely on.
205	func (r *GPUClusterReconciler) ensureAdminAccessLabel(ctx context.Context) error {	1✔
206	ns := &corev1.Namespace{}	1✔
207	if err := r.Get(ctx, client.ObjectKey{Name: r.Namespace}, ns); err != nil {	1✔
208	return fmt.Errorf("could not get namespace %s: %w", r.Namespace, err)	×
UNCOV 209	}	×
210	if ns.Labels[draAdminNamespaceLabelKey] == "true" {	1✔
UNCOV 211	return nil	×
UNCOV 212	}	×
213	patch := client.MergeFrom(ns.DeepCopy())	1✔
214	if ns.Labels == nil {	2✔
215	ns.Labels = make(map[string]string)	1✔
216	}	1✔
217	ns.Labels[draAdminNamespaceLabelKey] = "true"	1✔
218	return r.Patch(ctx, ns, patch)	1✔
219	}
220
221	// reconcileDelete drains ResourceClaim-consuming DaemonSets before releasing the CR:
222	// garbage collection would otherwise delete the DRA kubelet plugin while their pods
223	// still need it to unprepare claims, leaving them stuck in Terminating. Foreground
224	// propagation keeps each DaemonSet present until its pods are gone, i.e. unprepared.
225	func (r GPUClusterReconciler) reconcileDelete(ctx context.Context, instance nvidiav1alpha1.GPUCluster) (ctrl.Result, error) {	1✔
226	logger := log.FromContext(ctx)	1✔
227		1✔
228	if !controllerutil.ContainsFinalizer(instance, gpuClusterFinalizer) {	1✔
UNCOV 229	return ctrl.Result{}, nil	×
230	}	×
231
232	dsList := &appsv1.DaemonSetList{}	1✔
233	if err := r.List(ctx, dsList, client.InNamespace(r.Namespace)); err != nil {	1✔
UNCOV 234	return ctrl.Result{}, fmt.Errorf("error listing DaemonSets: %w", err)	×
UNCOV 235	}	×
236	var draining []string	1✔
237	for i := range dsList.Items {	2✔
238	ds := &dsList.Items[i]	1✔
239	if !metav1.IsControlledBy(ds, instance) \|\| len(ds.Spec.Template.Spec.ResourceClaims) == 0 {	2✔
240	continue	1✔
241	}
242	draining = append(draining, ds.Name)	1✔
243	if ds.DeletionTimestamp.IsZero() {	2✔
244	logger.V(consts.LogLevelInfo).Info("Draining ResourceClaim-consuming DaemonSet before teardown", "DaemonSet", ds.Name)	1✔
245	if err := r.Delete(ctx, ds, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !apierrors.IsNotFound(err) {	1✔
UNCOV 246	return ctrl.Result{}, fmt.Errorf("error deleting DaemonSet %s: %w", ds.Name, err)	×
UNCOV 247	}	×
248	}
249	}
250	if len(draining) > 0 {	2✔
251	r.recorder.Eventf(instance, nil, corev1.EventTypeNormal, "DrainingClaimConsumers", "Delete",	1✔
252	"Waiting for ResourceClaim-consuming DaemonSet(s) to terminate: %s", strings.Join(draining, ", "))	1✔
253	return ctrl.Result{RequeueAfter: time.Second * 5}, nil	1✔
254	}	1✔
255
256	controllerutil.RemoveFinalizer(instance, gpuClusterFinalizer)	1✔
257	if err := r.Update(ctx, instance); err != nil {	1✔
UNCOV 258	return ctrl.Result{}, fmt.Errorf("error removing finalizer: %w", err)	×
UNCOV 259	}	×
260	return ctrl.Result{}, nil	1✔
261	}
262
263	// updateCRStatus persists the given state (and the operator namespace) to the GPUCluster's
264	// .status subresource. It refetches the CR first to avoid resourceVersion conflicts and skips
265	// the API write when the status is already current. The desired status is mirrored onto cr
266	// up front so it is set on every non-error path.
267	func (r GPUClusterReconciler) updateCRStatus(ctx context.Context, cr nvidiav1alpha1.GPUCluster, desired nvidiav1alpha1.State) error {	1✔
268	reqLogger := log.FromContext(ctx)	1✔
269		1✔
270	// Refetch to avoid a resourceVersion conflict.	1✔
271	instance := &nvidiav1alpha1.GPUCluster{}	1✔
272	if err := r.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {	1✔
UNCOV 273	reqLogger.Error(err, "Failed to get GPUCluster instance for status update")	×
UNCOV 274	return err	×
UNCOV 275	}	×
276	cr.Status.State = desired	1✔
277	cr.Status.Namespace = r.Namespace	1✔
278		1✔
279	if instance.Status.State == desired && instance.Status.Namespace == r.Namespace {	1✔
UNCOV 280	return nil	×
UNCOV 281	}	×
282	instance.Status.State = desired	1✔
283	instance.Status.Namespace = r.Namespace	1✔
284		1✔
285	reqLogger.V(consts.LogLevelInfo).Info("Updating CR Status", "Status", instance.Status)	1✔
286	if err := r.Status().Update(ctx, instance); err != nil {	1✔
UNCOV 287	reqLogger.Error(err, "Failed to update CR status")	×
UNCOV 288	return err	×
UNCOV 289	}	×
290	return nil	1✔
291	}
292
293	// enqueueAllGPUClusters enqueues every instance so each is reconciled when any
294	// instance or owned resource changes.
295	func (r GPUClusterReconciler) enqueueAllGPUClusters(ctx context.Context, _ nvidiav1alpha1.GPUCluster) []reconcile.Request {	1✔
296	logger := log.FromContext(ctx)	1✔
297	list := &nvidiav1alpha1.GPUClusterList{}	1✔
298		1✔
299	if err := r.List(ctx, list); err != nil {	1✔
UNCOV 300	logger.Error(err, "Unable to list GPUCluster resources")	×
UNCOV 301	return []reconcile.Request{}	×
UNCOV 302	}	×
303
304	reconcileRequests := make([]reconcile.Request, 0, len(list.Items))	1✔
305	for _, config := range list.Items {	2✔
306	reconcileRequests = append(reconcileRequests,	1✔
307	reconcile.Request{	1✔
308	NamespacedName: types.NamespacedName{	1✔
309	Name: config.GetName(),	1✔
310	},	1✔
311	})	1✔
312	}	1✔
313
314	return reconcileRequests	1✔
315	}
316
317	func (r *GPUClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {	×
318	// The state manager renders the DRA driver operand for the GPUCluster.	×
319	stateManager, err := state.NewManager(	×
320	nvidiav1alpha1.GPUClusterCRDName,	×
321	r.Namespace,	×
322	mgr.GetClient(),	×
323	mgr.GetScheme())	×
324	if err != nil {	×
325	return fmt.Errorf("error creating state manager: %w", err)	×
326	}	×
327	r.stateManager = stateManager	×
328		×
329	r.conditionUpdater = conditions.NewGPUClusterUpdater(mgr.GetClient())	×
330	r.recorder = mgr.GetEventRecorder("nvidia-gpu-operator")	×
331		×
332	c, err := controller.New("gpu-cluster-controller", mgr, controller.Options{	×
333	Reconciler: r,	×
334	MaxConcurrentReconciles: 1,	×
335	RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](minDelayCR, maxDelayCR),	×
UNCOV 336	})	×
337	if err != nil {	×
338	return err	×
339	}	×
340
341	err = c.Watch(source.Kind(	×
342	mgr.GetCache(),	×
343	&nvidiav1alpha1.GPUCluster{},	×
344	handler.TypedEnqueueRequestsFromMapFunc(r.enqueueAllGPUClusters),	×
345	predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.GPUCluster]{},	×
346	),	×
UNCOV 347	)	×
UNCOV 348	if err != nil {	×
349	return err	×
350	}	×
351
352	// Watch the secondary resources each state manager owns.
353	watchSources := stateManager.GetWatchSources(mgr)	×
354	for _, watchSource := range watchSources {	×
355	err = c.Watch(	×
356	watchSource,	×
UNCOV 357	)	×
UNCOV 358	if err != nil {	×
359	return fmt.Errorf("error setting up Watch for source type %v: %w", watchSource, err)	×
UNCOV 360	}	×
361	}
362
UNCOV 363	return nil	×
364	}

NVIDIA / gpu-operator / 30285057709

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous