9903758109

Committed 12 Jul 2024 06:55AM UTC coverage: 41.0% (+0.02%) from 40.981%

Build # 9903758109

Build Type

Pull #687

github

Committed by

web-flow

Commit Message

Merge 108ef15a7 into c3ba0f3e9

Pull Request Pull Request #687: [metrics 2/x] Configure Prometheus Operator

Run Details

12 of 16 new or added lines in 1 file covered. (75.0%)

98 existing lines in 3 files now uncovered.

5599 of 13656 relevant lines covered (41.0%)

0.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.19

/controllers/sriovoperatorconfig_controller.go

/*
Copyright 2021.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
        "context"
        "fmt"
        "os"
        "sort"
        "strings"

        appsv1 "k8s.io/api/apps/v1"
        corev1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
        "k8s.io/apimachinery/pkg/runtime"
        "k8s.io/apimachinery/pkg/types"
        kscheme "k8s.io/client-go/kubernetes/scheme"
        ctrl "sigs.k8s.io/controller-runtime"
        ctrl_builder "sigs.k8s.io/controller-runtime/pkg/builder"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
        "sigs.k8s.io/controller-runtime/pkg/log"
        "sigs.k8s.io/controller-runtime/pkg/predicate"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"

        machinev1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        apply "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/apply"
        consts "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate"
        snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms"
        render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

// SriovOperatorConfigReconciler reconciles a SriovOperatorConfig object
type SriovOperatorConfigReconciler struct {
        client.Client
        Scheme         *runtime.Scheme
        PlatformHelper platforms.Interface
        FeatureGate    featuregate.FeatureGate
        GlobalClient   client.Client
}

//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs/finalizers,verbs=update

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the SriovOperatorConfig object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile
func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
        logger := log.FromContext(ctx).WithValues("sriovoperatorconfig", req.NamespacedName)
        logger.Info("Reconciling SriovOperatorConfig")

        // Note: in SetupWithManager we setup manager to enqueue only default config obj
        defaultConfig := &sriovnetworkv1.SriovOperatorConfig{}
        err := r.Get(ctx, req.NamespacedName, defaultConfig)
        if err != nil {
                if apierrors.IsNotFound(err) {
                        logger.Info("default SriovOperatorConfig object not found. waiting for creation.")
                        return reconcile.Result{}, nil
                }
                // Error reading the object - requeue the request.
                logger.Error(err, "Failed to get default SriovOperatorConfig object")
                return reconcile.Result{}, err
        }

        snolog.SetLogLevel(defaultConfig.Spec.LogLevel)

        r.FeatureGate.Init(defaultConfig.Spec.FeatureGates)
        logger.Info("enabled featureGates", "featureGates", r.FeatureGate.String())

        if !defaultConfig.Spec.EnableInjector {
                logger.Info("SR-IOV Network Resource Injector is disabled.")
        }

        if !defaultConfig.Spec.EnableOperatorWebhook {
                logger.Info("SR-IOV Network Operator Webhook is disabled.")
        }

        // Fetch the SriovNetworkNodePolicyList
        policyList := &sriovnetworkv1.SriovNetworkNodePolicyList{}
        err = r.List(ctx, policyList, &client.ListOptions{})
        if err != nil {
                // Error reading the object - requeue the request.
                return reconcile.Result{}, err
        }
        // Sort the policies with priority, higher priority ones is applied later
        // We need to use the sort so we always get the policies in the same order
        // That is needed so when we create the node Affinity for the sriov-device plugin
        // it will remain in the same order and not trigger a pod recreation
        sort.Sort(sriovnetworkv1.ByPriority(policyList.Items))

        // Render and sync webhook objects
        if err = r.syncWebhookObjs(ctx, defaultConfig); err != nil {
                return reconcile.Result{}, err
        }

        // Sync SriovNetworkConfigDaemon objects
        if err = r.syncConfigDaemonSet(ctx, defaultConfig); err != nil {
                return reconcile.Result{}, err
        }

        if err = syncPluginDaemonObjs(ctx, r.Client, r.Scheme, defaultConfig, policyList); err != nil {
                return reconcile.Result{}, err
        }

        if err = r.syncMetricsExporter(ctx, defaultConfig); err != nil {
                return reconcile.Result{}, err
        }

        // For Openshift we need to create the systemd files using a machine config
        if vars.ClusterType == consts.ClusterTypeOpenshift {
                // TODO: add support for hypershift as today there is no MCO on hypershift clusters
                if r.PlatformHelper.IsHypershift() {
                        return ctrl.Result{}, fmt.Errorf("systemd mode is not supported on hypershift")
                }

                if err = r.syncOpenShiftSystemdService(ctx, defaultConfig); err != nil {
                        return reconcile.Result{}, err
                }
        }

        logger.Info("Reconcile SriovOperatorConfig completed successfully")
        return reconcile.Result{RequeueAfter: consts.ResyncPeriod}, nil
}

// defaultConfigPredicate creates a predicate.Predicate that will return true
// only for the default sriovoperatorconfig obj.
func defaultConfigPredicate() predicate.Predicate {
        return predicate.NewPredicateFuncs(func(object client.Object) bool {
                if object.GetName() == consts.DefaultConfigName && object.GetNamespace() == vars.Namespace {
                        return true
                }
                return false
        })
}

// SetupWithManager sets up the controller with the Manager.
func (r *SriovOperatorConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
        return ctrl.NewControllerManagedBy(mgr).
                For(&sriovnetworkv1.SriovOperatorConfig{}, ctrl_builder.WithPredicates(defaultConfigPredicate())).
                Owns(&appsv1.DaemonSet{}).
                Owns(&corev1.ConfigMap{}).
                Complete(r)
}

func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context, dc *sriovnetworkv1.SriovOperatorConfig) error {
        logger := log.Log.WithName("syncConfigDaemonset")
        logger.V(1).Info("Start to sync config daemonset")

        data := render.MakeRenderData()
        data.Data["Image"] = os.Getenv("SRIOV_NETWORK_CONFIG_DAEMON_IMAGE")
        data.Data["Namespace"] = vars.Namespace
        data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE")
        data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE")
        data.Data["OVSCNIImage"] = os.Getenv("OVS_CNI_IMAGE")
        data.Data["RDMACNIImage"] = os.Getenv("RDMA_CNI_IMAGE")
        data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
        data.Data["ClusterType"] = vars.ClusterType
        data.Data["DevMode"] = os.Getenv("DEV_MODE")
        data.Data["ImagePullSecrets"] = GetImagePullSecrets()
        if dc.Spec.ConfigurationMode == sriovnetworkv1.SystemdConfigurationMode {
                data.Data["UsedSystemdMode"] = true
        } else {
                data.Data["UsedSystemdMode"] = false
        }
        data.Data["ParallelNicConfig"] = r.FeatureGate.IsEnabled(consts.ParallelNicConfigFeatureGate)

        envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH")
        if envCniBinPath == "" {
                data.Data["CNIBinPath"] = "/var/lib/cni/bin"
        } else {
                logger.V(1).Info("New cni bin found", "CNIBinPath", envCniBinPath)
                data.Data["CNIBinPath"] = envCniBinPath
        }

        if len(dc.Spec.DisablePlugins) > 0 {
                logger.V(1).Info("DisablePlugins provided", "DisablePlugins", dc.Spec.DisablePlugins)
                data.Data["DisablePlugins"] = strings.Join(dc.Spec.DisablePlugins.ToStringSlice(), ",")
        }

        objs, err := render.RenderDir(consts.ConfigDaemonPath, &data)
        if err != nil {
                logger.Error(err, "Fail to render config daemon manifests")
                return err
        }
        // Sync DaemonSets
        for _, obj := range objs {
                if obj.GetKind() == "DaemonSet" {
                        err = updateDaemonsetNodeSelector(obj, dc.Spec.ConfigDaemonNodeSelector)
                        if err != nil {
                                return err
                        }
                }

                err = r.syncK8sResource(ctx, dc, obj)
                if err != nil {
                        logger.Error(err, "Couldn't sync SR-IOV daemons objects")
                        return err
                }
        }
        return nil
}

func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context, dc *sriovnetworkv1.SriovOperatorConfig) error {
        logger := log.Log.WithName("syncMetricsExporter")
        logger.V(1).Info("Start to sync metrics exporter")

        data := render.MakeRenderData()
        data.Data["Image"] = os.Getenv("METRICS_EXPORTER_IMAGE")
        data.Data["Namespace"] = vars.Namespace
        data.Data["ImagePullSecrets"] = GetImagePullSecrets()
        data.Data["MetricsExporterSecretName"] = os.Getenv("METRICS_EXPORTER_SECRET_NAME")
        data.Data["MetricsExporterPort"] = os.Getenv("METRICS_EXPORTER_PORT")
        data.Data["MetricsExporterKubeRbacProxyImage"] = os.Getenv("METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE")
        data.Data["IsOpenshift"] = r.PlatformHelper.IsOpenshiftCluster()
        data.Data["IsPrometheusOperatorInstalled"] = isPrometheusOperatorInstalled(ctx, r.GlobalClient)
        data.Data["PrometheusOperatorServiceAccount"] = os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT")
        data.Data["PrometheusOperatorNamespace"] = os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE")
        data.Data["NodeSelectorField"] = GetDefaultNodeSelector()
        if dc.Spec.ConfigDaemonNodeSelector != nil {
                data.Data["NodeSelectorField"] = dc.Spec.ConfigDaemonNodeSelector
        }

        objs, err := render.RenderDir(consts.MetricsExporterPath, &data)
        if err != nil {
                logger.Error(err, "Fail to render metrics exporter manifests")
                return err
        }

        if r.FeatureGate.IsEnabled(consts.MetricsExporterFeatureGate) {
                for _, obj := range objs {
                        err = r.syncK8sResource(ctx, dc, obj)
                        if err != nil {
                                logger.Error(err, "Couldn't sync metrics exporter objects")
                                return err
                        }
                }

                return nil
        }

        err = r.deleteK8sResources(ctx, objs)
        if err != nil {
                return err
        }

        return nil
}

func (r *SriovOperatorConfigReconciler) syncWebhookObjs(ctx context.Context, dc *sriovnetworkv1.SriovOperatorConfig) error {
        logger := log.Log.WithName("syncWebhookObjs")
        logger.V(1).Info("Start to sync webhook objects")

        for name, path := range webhooks {
                // Render Webhook manifests
                data := render.MakeRenderData()
                data.Data["Namespace"] = vars.Namespace
                data.Data["SRIOVMutatingWebhookName"] = name
                data.Data["NetworkResourcesInjectorImage"] = os.Getenv("NETWORK_RESOURCES_INJECTOR_IMAGE")
                data.Data["SriovNetworkWebhookImage"] = os.Getenv("SRIOV_NETWORK_WEBHOOK_IMAGE")
                data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
                data.Data["ClusterType"] = vars.ClusterType
                data.Data["DevMode"] = os.Getenv("DEV_MODE")
                data.Data["ImagePullSecrets"] = GetImagePullSecrets()
                data.Data["CertManagerEnabled"] = strings.ToLower(os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_CERT_MANAGER_ENABLED")) == trueString
                data.Data["OperatorWebhookSecretName"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_SECRET_NAME")
                data.Data["OperatorWebhookCA"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT")
                data.Data["InjectorWebhookSecretName"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_SECRET_NAME")
                data.Data["InjectorWebhookCA"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT")

                data.Data["ExternalControlPlane"] = false
                if r.PlatformHelper.IsOpenshiftCluster() {
                        external := r.PlatformHelper.IsHypershift()
                        data.Data["ExternalControlPlane"] = external
                }

                // check for ResourceInjectorMatchConditionFeatureGate feature gate
                data.Data[consts.ResourceInjectorMatchConditionFeatureGate] = r.FeatureGate.IsEnabled(consts.ResourceInjectorMatchConditionFeatureGate)

                objs, err := render.RenderDir(path, &data)
                if err != nil {
                        logger.Error(err, "Fail to render webhook manifests")
                        return err
                }

                // Delete injector webhook
                if !dc.Spec.EnableInjector && path == consts.InjectorWebHookPath {
                        for _, obj := range objs {
                                err = r.deleteWebhookObject(ctx, obj)
                                if err != nil {
                                        return err
                                }
                        }
                        logger.Info("SR-IOV Admission Controller is disabled.")
                        logger.Info("To enable SR-IOV Admission Controller,")
                        logger.Info("Set 'SriovOperatorConfig.Spec.EnableInjector' to true(bool).")
                        continue
                }
                // Delete operator webhook
                if !dc.Spec.EnableOperatorWebhook && path == consts.OperatorWebHookPath {
                        for _, obj := range objs {
                                err = r.deleteWebhookObject(ctx, obj)
                                if err != nil {
                                        return err
                                }
                        }
                        logger.Info("Operator Admission Controller is disabled.")
                        logger.Info("To enable Operator Admission Controller,")
                        logger.Info("Set 'SriovOperatorConfig.Spec.EnableOperatorWebhook' to true(bool).")
                        continue
                }

                // Sync Webhook
                for _, obj := range objs {
                        err = r.syncK8sResource(ctx, dc, obj)
                        if err != nil {
                                logger.Error(err, "Couldn't sync webhook objects")
                                return err
                        }
                }
        }

        return nil
}

func (r *SriovOperatorConfigReconciler) deleteWebhookObject(ctx context.Context, obj *uns.Unstructured) error {
        if err := r.deleteK8sResource(ctx, obj); err != nil {
                return err
        }
        return nil
}

func (r *SriovOperatorConfigReconciler) deleteK8sResource(ctx context.Context, in *uns.Unstructured) error {
        if err := apply.DeleteObject(ctx, r.Client, in); err != nil {
                return fmt.Errorf("failed to delete object %v with err: %v", in, err)
        }
        return nil
}

func (r *SriovOperatorConfigReconciler) deleteK8sResources(ctx context.Context, objs []*uns.Unstructured) error {
        for _, obj := range objs {
                err := r.deleteK8sResource(ctx, obj)
                if err != nil {
                        return err
                }
        }
        return nil
}

func (r *SriovOperatorConfigReconciler) syncK8sResource(ctx context.Context, cr *sriovnetworkv1.SriovOperatorConfig, in *uns.Unstructured) error {
        switch in.GetKind() {
        case clusterRoleResourceName, clusterRoleBindingResourceName, mutatingWebhookConfigurationCRDName, validatingWebhookConfigurationCRDName, machineConfigCRDName:
        default:
                // set owner-reference only for namespaced objects
                if err := controllerutil.SetControllerReference(cr, in, r.Scheme); err != nil {
                        return err
                }
        }
        if err := apply.ApplyObject(ctx, r.Client, in); err != nil {
                return fmt.Errorf("failed to apply object %v with err: %v", in, err)
        }
        return nil
}

// syncOpenShiftSystemdService creates the Machine Config to deploy the systemd service on openshift ONLY
func (r *SriovOperatorConfigReconciler) syncOpenShiftSystemdService(ctx context.Context, cr *sriovnetworkv1.SriovOperatorConfig) error {
        logger := log.Log.WithName("syncSystemdService")

        if cr.Spec.ConfigurationMode != sriovnetworkv1.SystemdConfigurationMode {
                obj := &machinev1.MachineConfig{}
                err := r.Get(context.TODO(), types.NamespacedName{Name: consts.SystemdServiceOcpMachineConfigName}, obj)
                if err != nil {
                        if apierrors.IsNotFound(err) {
                                return nil
                        }

                        logger.Error(err, "failed to get machine config for the sriov-systemd-service")
                        return err
                }

                logger.Info("Systemd service was deployed but the operator is now operating on daemonset mode, removing the machine config")
                err = r.Delete(context.TODO(), obj)
                if err != nil {
                        logger.Error(err, "failed to remove the systemd service machine config")
                        return err
                }

                return nil
        }

        logger.Info("Start to sync config systemd machine config for openshift")
        data := render.MakeRenderData()
        data.Data["LogLevel"] = cr.Spec.LogLevel
        objs, err := render.RenderDir(consts.SystemdServiceOcpPath, &data)
        if err != nil {
                logger.Error(err, "Fail to render config daemon manifests")
                return err
        }

        // Sync machine config
        return r.setLabelInsideObject(ctx, cr, objs)
}

func (r SriovOperatorConfigReconciler) setLabelInsideObject(ctx context.Context, cr *sriovnetworkv1.SriovOperatorConfig, objs []*uns.Unstructured) error {
        logger := log.Log.WithName("setLabelInsideObject")
        for _, obj := range objs {
                if obj.GetKind() == machineConfigCRDName && len(cr.Spec.ConfigDaemonNodeSelector) > 0 {
                        scheme := kscheme.Scheme
                        mc := &machinev1.ControllerConfig{}
                        err := scheme.Convert(obj, mc, nil)
                        if err != nil {
                                logger.Error(err, "Fail to convert to MachineConfig")
                                return err
                        }
                        mc.Labels = cr.Spec.ConfigDaemonNodeSelector
                        err = scheme.Convert(mc, obj, nil)
                        if err != nil {
                                logger.Error(err, "Fail to convert to Unstructured")
                                return err
                        }
                }
                err := r.syncK8sResource(ctx, cr, obj)
                if err != nil {
                        logger.Error(err, "Couldn't sync SR-IOV daemons objects")
                        return err
                }
        }

        return nil
}

1	/*
2	Copyright 2021.
3
4	Licensed under the Apache License, Version 2.0 (the "License");
5	you may not use this file except in compliance with the License.
6	You may obtain a copy of the License at
7
8	http://www.apache.org/licenses/LICENSE-2.0
9
10	Unless required by applicable law or agreed to in writing, software
11	distributed under the License is distributed on an "AS IS" BASIS,
12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	See the License for the specific language governing permissions and
14	limitations under the License.
15	*/
16
17	package controllers
18
19	import (
20	"context"
21	"fmt"
22	"os"
23	"sort"
24	"strings"
25
26	appsv1 "k8s.io/api/apps/v1"
27	corev1 "k8s.io/api/core/v1"
28	apierrors "k8s.io/apimachinery/pkg/api/errors"
29	uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
30	"k8s.io/apimachinery/pkg/runtime"
31	"k8s.io/apimachinery/pkg/types"
32	kscheme "k8s.io/client-go/kubernetes/scheme"
33	ctrl "sigs.k8s.io/controller-runtime"
34	ctrl_builder "sigs.k8s.io/controller-runtime/pkg/builder"
35	"sigs.k8s.io/controller-runtime/pkg/client"
36	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
37	"sigs.k8s.io/controller-runtime/pkg/log"
38	"sigs.k8s.io/controller-runtime/pkg/predicate"
39	"sigs.k8s.io/controller-runtime/pkg/reconcile"
40
41	machinev1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
42
43	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
44	apply "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/apply"
45	consts "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
46	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate"
47	snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
48	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms"
49	render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render"
50	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
51	)
52
53	// SriovOperatorConfigReconciler reconciles a SriovOperatorConfig object
54	type SriovOperatorConfigReconciler struct {
55	client.Client
56	Scheme *runtime.Scheme
57	PlatformHelper platforms.Interface
58	FeatureGate featuregate.FeatureGate
59	GlobalClient client.Client
60	}
61
62	//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs,verbs=get;list;watch;create;update;patch;delete
63	//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs/status,verbs=get;update;patch
64	//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs/finalizers,verbs=update
65
66	// Reconcile is part of the main kubernetes reconciliation loop which aims to
67	// move the current state of the cluster closer to the desired state.
68	// TODO(user): Modify the Reconcile function to compare the state specified by
69	// the SriovOperatorConfig object against the actual cluster state, and then
70	// perform operations to make the cluster state reflect the state specified by
71	// the user.
72	//
73	// For more details, check Reconcile and its Result here:
74	// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile
75	func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {	1✔
76	logger := log.FromContext(ctx).WithValues("sriovoperatorconfig", req.NamespacedName)	1✔
77	logger.Info("Reconciling SriovOperatorConfig")	1✔
78		1✔
79	// Note: in SetupWithManager we setup manager to enqueue only default config obj	1✔
80	defaultConfig := &sriovnetworkv1.SriovOperatorConfig{}	1✔
81	err := r.Get(ctx, req.NamespacedName, defaultConfig)	1✔
82	if err != nil {	1✔
83	if apierrors.IsNotFound(err) {	×
84	logger.Info("default SriovOperatorConfig object not found. waiting for creation.")	×
85	return reconcile.Result{}, nil	×
UNCOV 86	}	×
87	// Error reading the object - requeue the request.
88	logger.Error(err, "Failed to get default SriovOperatorConfig object")	×
UNCOV 89	return reconcile.Result{}, err	×
90	}
91
92	snolog.SetLogLevel(defaultConfig.Spec.LogLevel)	1✔
93		1✔
94	r.FeatureGate.Init(defaultConfig.Spec.FeatureGates)	1✔
95	logger.Info("enabled featureGates", "featureGates", r.FeatureGate.String())	1✔
96		1✔
97	if !defaultConfig.Spec.EnableInjector {	2✔
98	logger.Info("SR-IOV Network Resource Injector is disabled.")	1✔
99	}	1✔
100
101	if !defaultConfig.Spec.EnableOperatorWebhook {	2✔
102	logger.Info("SR-IOV Network Operator Webhook is disabled.")	1✔
103	}	1✔
104
105	// Fetch the SriovNetworkNodePolicyList
106	policyList := &sriovnetworkv1.SriovNetworkNodePolicyList{}	1✔
107	err = r.List(ctx, policyList, &client.ListOptions{})	1✔
108	if err != nil {	1✔
109	// Error reading the object - requeue the request.	×
110	return reconcile.Result{}, err	×
UNCOV 111	}	×
112	// Sort the policies with priority, higher priority ones is applied later
113	// We need to use the sort so we always get the policies in the same order
114	// That is needed so when we create the node Affinity for the sriov-device plugin
115	// it will remain in the same order and not trigger a pod recreation
116	sort.Sort(sriovnetworkv1.ByPriority(policyList.Items))	1✔
117		1✔
118	// Render and sync webhook objects	1✔
119	if err = r.syncWebhookObjs(ctx, defaultConfig); err != nil {	1✔
120	return reconcile.Result{}, err	×
UNCOV 121	}	×
122
123	// Sync SriovNetworkConfigDaemon objects
124	if err = r.syncConfigDaemonSet(ctx, defaultConfig); err != nil {	1✔
125	return reconcile.Result{}, err	×
UNCOV 126	}	×
127
128	if err = syncPluginDaemonObjs(ctx, r.Client, r.Scheme, defaultConfig, policyList); err != nil {	1✔
129	return reconcile.Result{}, err	×
UNCOV 130	}	×
131
132	if err = r.syncMetricsExporter(ctx, defaultConfig); err != nil {	1✔
133	return reconcile.Result{}, err	×
UNCOV 134	}	×
135
136	// For Openshift we need to create the systemd files using a machine config
137	if vars.ClusterType == consts.ClusterTypeOpenshift {	2✔
138	// TODO: add support for hypershift as today there is no MCO on hypershift clusters	1✔
139	if r.PlatformHelper.IsHypershift() {	1✔
140	return ctrl.Result{}, fmt.Errorf("systemd mode is not supported on hypershift")	×
UNCOV 141	}	×
142
143	if err = r.syncOpenShiftSystemdService(ctx, defaultConfig); err != nil {	1✔
144	return reconcile.Result{}, err	×
UNCOV 145	}	×
146	}
147
148	logger.Info("Reconcile SriovOperatorConfig completed successfully")	1✔
149	return reconcile.Result{RequeueAfter: consts.ResyncPeriod}, nil	1✔
150	}
151
152	// defaultConfigPredicate creates a predicate.Predicate that will return true
153	// only for the default sriovoperatorconfig obj.
154	func defaultConfigPredicate() predicate.Predicate {	1✔
155	return predicate.NewPredicateFuncs(func(object client.Object) bool {	2✔
156	if object.GetName() == consts.DefaultConfigName && object.GetNamespace() == vars.Namespace {	2✔
157	return true	1✔
158	}	1✔
UNCOV 159	return false	×
160	})
161	}
162
163	// SetupWithManager sets up the controller with the Manager.
164	func (r *SriovOperatorConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {	1✔
165	return ctrl.NewControllerManagedBy(mgr).	1✔
166	For(&sriovnetworkv1.SriovOperatorConfig{}, ctrl_builder.WithPredicates(defaultConfigPredicate())).	1✔
167	Owns(&appsv1.DaemonSet{}).	1✔
168	Owns(&corev1.ConfigMap{}).	1✔
169	Complete(r)	1✔
170	}	1✔
171
172	func (r SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context, dc sriovnetworkv1.SriovOperatorConfig) error {	1✔
173	logger := log.Log.WithName("syncConfigDaemonset")	1✔
174	logger.V(1).Info("Start to sync config daemonset")	1✔
175		1✔
176	data := render.MakeRenderData()	1✔
177	data.Data["Image"] = os.Getenv("SRIOV_NETWORK_CONFIG_DAEMON_IMAGE")	1✔
178	data.Data["Namespace"] = vars.Namespace	1✔
179	data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE")	1✔
180	data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE")	1✔
181	data.Data["OVSCNIImage"] = os.Getenv("OVS_CNI_IMAGE")	1✔
182	data.Data["RDMACNIImage"] = os.Getenv("RDMA_CNI_IMAGE")	1✔
183	data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")	1✔
184	data.Data["ClusterType"] = vars.ClusterType	1✔
185	data.Data["DevMode"] = os.Getenv("DEV_MODE")	1✔
186	data.Data["ImagePullSecrets"] = GetImagePullSecrets()	1✔
187	if dc.Spec.ConfigurationMode == sriovnetworkv1.SystemdConfigurationMode {	1✔
UNCOV 188	data.Data["UsedSystemdMode"] = true	×
189	} else {	1✔
190	data.Data["UsedSystemdMode"] = false	1✔
191	}	1✔
192	data.Data["ParallelNicConfig"] = r.FeatureGate.IsEnabled(consts.ParallelNicConfigFeatureGate)	1✔
193		1✔
194	envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH")	1✔
195	if envCniBinPath == "" {	2✔
196	data.Data["CNIBinPath"] = "/var/lib/cni/bin"	1✔
197	} else {	1✔
198	logger.V(1).Info("New cni bin found", "CNIBinPath", envCniBinPath)	×
199	data.Data["CNIBinPath"] = envCniBinPath	×
UNCOV 200	}	×
201
202	if len(dc.Spec.DisablePlugins) > 0 {	2✔
203	logger.V(1).Info("DisablePlugins provided", "DisablePlugins", dc.Spec.DisablePlugins)	1✔
204	data.Data["DisablePlugins"] = strings.Join(dc.Spec.DisablePlugins.ToStringSlice(), ",")	1✔
205	}	1✔
206
207	objs, err := render.RenderDir(consts.ConfigDaemonPath, &data)	1✔
208	if err != nil {	1✔
209	logger.Error(err, "Fail to render config daemon manifests")	×
210	return err	×
UNCOV 211	}	×
212	// Sync DaemonSets
213	for _, obj := range objs {	2✔
214	if obj.GetKind() == "DaemonSet" {	2✔
215	err = updateDaemonsetNodeSelector(obj, dc.Spec.ConfigDaemonNodeSelector)	1✔
216	if err != nil {	1✔
217	return err	×
UNCOV 218	}	×
219	}
220
221	err = r.syncK8sResource(ctx, dc, obj)	1✔
222	if err != nil {	1✔
223	logger.Error(err, "Couldn't sync SR-IOV daemons objects")	×
224	return err	×
UNCOV 225	}	×
226	}
227	return nil	1✔
228	}
229
230	func (r SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context, dc sriovnetworkv1.SriovOperatorConfig) error {	1✔
231	logger := log.Log.WithName("syncMetricsExporter")	1✔
232	logger.V(1).Info("Start to sync metrics exporter")	1✔
233		1✔
234	data := render.MakeRenderData()	1✔
235	data.Data["Image"] = os.Getenv("METRICS_EXPORTER_IMAGE")	1✔
236	data.Data["Namespace"] = vars.Namespace	1✔
237	data.Data["ImagePullSecrets"] = GetImagePullSecrets()	1✔
238	data.Data["MetricsExporterSecretName"] = os.Getenv("METRICS_EXPORTER_SECRET_NAME")	1✔
239	data.Data["MetricsExporterPort"] = os.Getenv("METRICS_EXPORTER_PORT")	1✔
240	data.Data["MetricsExporterKubeRbacProxyImage"] = os.Getenv("METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE")	1✔
241	data.Data["IsOpenshift"] = r.PlatformHelper.IsOpenshiftCluster()	1✔
242	data.Data["IsPrometheusOperatorInstalled"] = isPrometheusOperatorInstalled(ctx, r.GlobalClient)	1✔
243	data.Data["PrometheusOperatorServiceAccount"] = os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT")	1✔
244	data.Data["PrometheusOperatorNamespace"] = os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE")	1✔
245	data.Data["NodeSelectorField"] = GetDefaultNodeSelector()	1✔
246	if dc.Spec.ConfigDaemonNodeSelector != nil {	2✔
247	data.Data["NodeSelectorField"] = dc.Spec.ConfigDaemonNodeSelector	1✔
248	}	1✔
249
250	objs, err := render.RenderDir(consts.MetricsExporterPath, &data)	1✔
251	if err != nil {	1✔
UNCOV 252	logger.Error(err, "Fail to render metrics exporter manifests")	×
253	return err	×
254	}	×
255
256	if r.FeatureGate.IsEnabled(consts.MetricsExporterFeatureGate) {	2✔
257	for _, obj := range objs {	2✔
258	err = r.syncK8sResource(ctx, dc, obj)	1✔
259	if err != nil {	1✔
UNCOV 260	logger.Error(err, "Couldn't sync metrics exporter objects")	×
261	return err	×
262	}	×
263	}
264
265	return nil	1✔
266	}
267
268	err = r.deleteK8sResources(ctx, objs)	1✔
269	if err != nil {	1✔
NEW 270	return err	×
NEW 271	}	×
272
273	return nil	1✔
274	}
275
276	func (r SriovOperatorConfigReconciler) syncWebhookObjs(ctx context.Context, dc sriovnetworkv1.SriovOperatorConfig) error {	1✔
277	logger := log.Log.WithName("syncWebhookObjs")	1✔
278	logger.V(1).Info("Start to sync webhook objects")	1✔
279		1✔
280	for name, path := range webhooks {	2✔
281	// Render Webhook manifests	1✔
282	data := render.MakeRenderData()	1✔
283	data.Data["Namespace"] = vars.Namespace	1✔
284	data.Data["SRIOVMutatingWebhookName"] = name	1✔
285	data.Data["NetworkResourcesInjectorImage"] = os.Getenv("NETWORK_RESOURCES_INJECTOR_IMAGE")	1✔
286	data.Data["SriovNetworkWebhookImage"] = os.Getenv("SRIOV_NETWORK_WEBHOOK_IMAGE")	1✔
287	data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")	1✔
288	data.Data["ClusterType"] = vars.ClusterType	1✔
289	data.Data["DevMode"] = os.Getenv("DEV_MODE")	1✔
290	data.Data["ImagePullSecrets"] = GetImagePullSecrets()	1✔
291	data.Data["CertManagerEnabled"] = strings.ToLower(os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_CERT_MANAGER_ENABLED")) == trueString	1✔
292	data.Data["OperatorWebhookSecretName"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_SECRET_NAME")	1✔
293	data.Data["OperatorWebhookCA"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT")	1✔
294	data.Data["InjectorWebhookSecretName"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_SECRET_NAME")	1✔
295	data.Data["InjectorWebhookCA"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT")	1✔
296		1✔
297	data.Data["ExternalControlPlane"] = false	1✔
298	if r.PlatformHelper.IsOpenshiftCluster() {	1✔
UNCOV 299	external := r.PlatformHelper.IsHypershift()	×
300	data.Data["ExternalControlPlane"] = external	×
301	}	×
302
303	// check for ResourceInjectorMatchConditionFeatureGate feature gate
304	data.Data[consts.ResourceInjectorMatchConditionFeatureGate] = r.FeatureGate.IsEnabled(consts.ResourceInjectorMatchConditionFeatureGate)	1✔
305		1✔
306	objs, err := render.RenderDir(path, &data)	1✔
307	if err != nil {	1✔
UNCOV 308	logger.Error(err, "Fail to render webhook manifests")	×
309	return err	×
310	}	×
311
312	// Delete injector webhook
313	if !dc.Spec.EnableInjector && path == consts.InjectorWebHookPath {	2✔
314	for _, obj := range objs {	2✔
315	err = r.deleteWebhookObject(ctx, obj)	1✔
316	if err != nil {	1✔
UNCOV 317	return err	×
318	}	×
319	}
320	logger.Info("SR-IOV Admission Controller is disabled.")	1✔
321	logger.Info("To enable SR-IOV Admission Controller,")	1✔
322	logger.Info("Set 'SriovOperatorConfig.Spec.EnableInjector' to true(bool).")	1✔
323	continue	1✔
324	}
325	// Delete operator webhook
326	if !dc.Spec.EnableOperatorWebhook && path == consts.OperatorWebHookPath {	2✔
327	for _, obj := range objs {	2✔
328	err = r.deleteWebhookObject(ctx, obj)	1✔
329	if err != nil {	1✔
UNCOV 330	return err	×
331	}	×
332	}
333	logger.Info("Operator Admission Controller is disabled.")	1✔
334	logger.Info("To enable Operator Admission Controller,")	1✔
335	logger.Info("Set 'SriovOperatorConfig.Spec.EnableOperatorWebhook' to true(bool).")	1✔
336	continue	1✔
337	}
338
339	// Sync Webhook
340	for _, obj := range objs {	2✔
341	err = r.syncK8sResource(ctx, dc, obj)	1✔
342	if err != nil {	1✔
UNCOV 343	logger.Error(err, "Couldn't sync webhook objects")	×
344	return err	×
345	}	×
346	}
347	}
348
349	return nil	1✔
350	}
351
352	func (r SriovOperatorConfigReconciler) deleteWebhookObject(ctx context.Context, obj uns.Unstructured) error {	1✔
353	if err := r.deleteK8sResource(ctx, obj); err != nil {	1✔
UNCOV 354	return err	×
355	}	×
356	return nil	1✔
357	}
358
359	func (r SriovOperatorConfigReconciler) deleteK8sResource(ctx context.Context, in uns.Unstructured) error {	1✔
360	if err := apply.DeleteObject(ctx, r.Client, in); err != nil {	1✔
UNCOV 361	return fmt.Errorf("failed to delete object %v with err: %v", in, err)	×
362	}	×
363	return nil	1✔
364	}
365
366	func (r SriovOperatorConfigReconciler) deleteK8sResources(ctx context.Context, objs []uns.Unstructured) error {	1✔
367	for _, obj := range objs {	2✔
368	err := r.deleteK8sResource(ctx, obj)	1✔
369	if err != nil {	1✔
NEW 370	return err	×
NEW 371	}	×
372	}
373	return nil	1✔
374	}
375
376	func (r SriovOperatorConfigReconciler) syncK8sResource(ctx context.Context, cr sriovnetworkv1.SriovOperatorConfig, in *uns.Unstructured) error {	1✔
377	switch in.GetKind() {	1✔
378	case clusterRoleResourceName, clusterRoleBindingResourceName, mutatingWebhookConfigurationCRDName, validatingWebhookConfigurationCRDName, machineConfigCRDName:	1✔
379	default:	1✔
380	// set owner-reference only for namespaced objects	1✔
381	if err := controllerutil.SetControllerReference(cr, in, r.Scheme); err != nil {	1✔
UNCOV 382	return err	×
383	}	×
384	}
385	if err := apply.ApplyObject(ctx, r.Client, in); err != nil {	1✔
UNCOV 386	return fmt.Errorf("failed to apply object %v with err: %v", in, err)	×
387	}	×
388	return nil	1✔
389	}
390
391	// syncOpenShiftSystemdService creates the Machine Config to deploy the systemd service on openshift ONLY
392	func (r SriovOperatorConfigReconciler) syncOpenShiftSystemdService(ctx context.Context, cr sriovnetworkv1.SriovOperatorConfig) error {	1✔
393	logger := log.Log.WithName("syncSystemdService")	1✔
394		1✔
395	if cr.Spec.ConfigurationMode != sriovnetworkv1.SystemdConfigurationMode {	2✔
396	obj := &machinev1.MachineConfig{}	1✔
397	err := r.Get(context.TODO(), types.NamespacedName{Name: consts.SystemdServiceOcpMachineConfigName}, obj)	1✔
398	if err != nil {	2✔
399	if apierrors.IsNotFound(err) {	2✔
400	return nil	1✔
401	}	1✔
402
UNCOV 403	logger.Error(err, "failed to get machine config for the sriov-systemd-service")	×
404	return err	×
405	}
406
UNCOV 407	logger.Info("Systemd service was deployed but the operator is now operating on daemonset mode, removing the machine config")	×
408	err = r.Delete(context.TODO(), obj)	×
409	if err != nil {	×
410	logger.Error(err, "failed to remove the systemd service machine config")	×
411	return err	×
412	}	×
413
UNCOV 414	return nil	×
415	}
416
UNCOV 417	logger.Info("Start to sync config systemd machine config for openshift")	×
418	data := render.MakeRenderData()	×
419	data.Data["LogLevel"] = cr.Spec.LogLevel	×
420	objs, err := render.RenderDir(consts.SystemdServiceOcpPath, &data)	×
421	if err != nil {	×
422	logger.Error(err, "Fail to render config daemon manifests")	×
423	return err	×
424	}	×
425
426	// Sync machine config
UNCOV 427	return r.setLabelInsideObject(ctx, cr, objs)	×
428	}
429
UNCOV 430	func (r SriovOperatorConfigReconciler) setLabelInsideObject(ctx context.Context, cr sriovnetworkv1.SriovOperatorConfig, objs []uns.Unstructured) error {	×
431	logger := log.Log.WithName("setLabelInsideObject")	×
432	for _, obj := range objs {	×
433	if obj.GetKind() == machineConfigCRDName && len(cr.Spec.ConfigDaemonNodeSelector) > 0 {	×
434	scheme := kscheme.Scheme	×
435	mc := &machinev1.ControllerConfig{}	×
436	err := scheme.Convert(obj, mc, nil)	×
437	if err != nil {	×
438	logger.Error(err, "Fail to convert to MachineConfig")	×
439	return err	×
440	}	×
441	mc.Labels = cr.Spec.ConfigDaemonNodeSelector	×
442	err = scheme.Convert(mc, obj, nil)	×
443	if err != nil {	×
444	logger.Error(err, "Fail to convert to Unstructured")	×
445	return err	×
446	}	×
447	}
UNCOV 448	err := r.syncK8sResource(ctx, cr, obj)	×
449	if err != nil {	×
450	logger.Error(err, "Couldn't sync SR-IOV daemons objects")	×
451	return err	×
452	}	×
453	}
454
UNCOV 455	return nil	×
456	}

k8snetworkplumbingwg / sriov-network-operator / 9903758109

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous