openshift-kni / cluster-group-upgrades-operator / #3

Committed 20 Nov 2025 11:55AM UTC coverage: 61.674% (+17.2%) from 44.472%

Build # #3

Build Type

push

Committed by

Commit Message

chore(deps): update topology-aware-lifecycle-manager-precache-4-21 to e280b86 (#5031)

Image created from 'https://github.com/openshift-kni/cluster-group-upgrades-operator?rev=e0953ccb4'

Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com>
Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com>

Run Details

4039 of 6549 relevant lines covered (61.67%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.76

/controllers/managedclusterForCGU_controller.go

/*
Copyright 2021.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
        "context"
        "encoding/json"
        "fmt"
        "os"
        "sort"
        "strconv"
        "strings"

        "github.com/go-logr/logr"
        corev1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/errors"
        "k8s.io/apimachinery/pkg/api/meta"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
        "k8s.io/apimachinery/pkg/runtime"
        "k8s.io/apimachinery/pkg/types"
        ctrl "sigs.k8s.io/controller-runtime"
        "sigs.k8s.io/controller-runtime/pkg/builder"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
        "sigs.k8s.io/controller-runtime/pkg/event"
        "sigs.k8s.io/controller-runtime/pkg/predicate"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
        yaml "sigs.k8s.io/yaml"

        "github.com/openshift-kni/cluster-group-upgrades-operator/controllers/templates"
        utils "github.com/openshift-kni/cluster-group-upgrades-operator/controllers/utils"
        ranv1alpha1 "github.com/openshift-kni/cluster-group-upgrades-operator/pkg/api/clustergroupupgrades/v1alpha1"
        clusterv1 "open-cluster-management.io/api/cluster/v1"
        policyv1 "open-cluster-management.io/config-policy-controller/api/v1"
        policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
)

const (
        ztpInstallNS             = "ztp-install"
        ztpDeployWaveAnnotation  = "ran.openshift.io/ztp-deploy-wave"
        ztpRunningLabel          = "ztp-running"
        ztpDoneLabel             = "ztp-done"
        aztpRequiredLabel        = "accelerated-ztp"
        aztpDeployServiceVariant = "full"
)

// ManagedClusterForCguReconciler reconciles a ManagedCluster object to auto create the ClusterGroupUpgrade
type ManagedClusterForCguReconciler struct {
        client.Client
        Log    logr.Logger
        Scheme *runtime.Scheme
}

//+kubebuilder:rbac:groups=cluster.open-cluster-management.io,resources=managedclusters,verbs=get;list;watch
//+kubebuilder:rbac:groups=cluster.open-cluster-management.io,resources=managedclusters/finalizers,verbs=update
//+kubebuilder:rbac:groups=ran.openshift.io,resources=clustergroupupgrades,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=ran.openshift.io,resources=precachingconfigs,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;create;delete

// Reconcile the managed cluster auto create ClusterGroupUpgrade
//   - Controller watches for create event of managed cluster object. Reconciliation
//     is triggered when a new managed cluster is created
//   - When a new managed cluster is created, create ClusterGroupUpgrade CR for the
//     cluster only when it's ready and its child policies are available
//   - As created ClusterGroupUpgrade has ownReference set to its managed cluster,
//     when the managed cluster is deleted, the ClusterGroupUpgrade will be auto-deleted
//
// Note: The Controller will requeue the Request to be processed again if the returned error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
func (r *ManagedClusterForCguReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {

        managedCluster := &clusterv1.ManagedCluster{}
        if err := r.Get(ctx, types.NamespacedName{Name: req.Name}, managedCluster); err != nil {
                if errors.IsNotFound(err) {
                        // managed cluster could have been deleted
                        return ctrl.Result{}, nil
                }
                // Error reading managed cluster, requeue the request
                return ctrl.Result{}, err
        }

        // Stop creating UOCR or AZTP if ztp of this cluster is done already
        if _, found := managedCluster.Labels[ztpDoneLabel]; found {
                r.Log.Info("ZTP for the cluster has completed. "+ztpDoneLabel+" label found.", "Name", managedCluster.Name)
                return doNotRequeue(), nil
        }

        clusterGroupUpgrade := &ranv1alpha1.ClusterGroupUpgrade{}
        if err := r.Get(ctx, types.NamespacedName{Name: req.Name, Namespace: ztpInstallNS}, clusterGroupUpgrade); err != nil {
                if !errors.IsNotFound(err) {
                        // Error reading clusterGroupUpgrade, requeue the request
                        return ctrl.Result{}, err
                }
        } else {
                // clusterGroupUpgrade for this cluster already exists, stop reconcile
                r.Log.Info("clusterGroupUpgrade found", "Name", clusterGroupUpgrade.Name, "Namespace", clusterGroupUpgrade.Namespace)
                return doNotRequeue(), nil
        }

        // clusterGroupUpgrade CR doesn't exist
        availableCondition := meta.FindStatusCondition(managedCluster.Status.Conditions, clusterv1.ManagedClusterConditionAvailable)
        if availableCondition != nil && availableCondition.Status == metav1.ConditionTrue {
                // cluster is ready
                r.Log.Info("cluster is ready", "Name", managedCluster.Name)

                // Child policies get created as soon as the placementrule/placementbinding
                // gets created and matches the parent policies to the managedcluster.
                // It takes ~45 minutes for cluster to be installed and ready.
                // At this stage, all child policies should be created.
                policies, err := utils.GetChildPolicies(ctx, r.Client, []string{managedCluster.Name})
                if err != nil {
                        return ctrl.Result{}, err
                }
                if len(policies) == 0 {
                        // likely no policies were created, so no child policies found
                        r.Log.Info("WARN: No child policies found for cluster", "Name", managedCluster.Name)
                }

                // create clusterGroupUpgrade
                if err := r.newClusterGroupUpgrade(ctx, managedCluster, policies); err != nil {
                        return ctrl.Result{}, err
                }
        } else {
                // availableCondition is false or unknown, cluster is not ready
                return r.handleNotReadyCluster(ctx, managedCluster)
        }

        return doNotRequeue(), nil
}

func (r *ManagedClusterForCguReconciler) handleNotReadyCluster(
        ctx context.Context, managedCluster *clusterv1.ManagedCluster) (reconcile.Result, error) {
        if err := r.extractAztpPolicies(ctx, managedCluster); err != nil {
                return ctrl.Result{}, err
        }
        r.Log.Info("cluster is not ready", "Name", managedCluster.Name)
        return doNotRequeue(), nil
}

// sort map[string]int by value in ascending order, return sorted keys
func sortMapByValue(sortMap map[string]int) []string {
        var keys []string
        for key := range sortMap {
                keys = append(keys, key)
        }

        sort.Slice(keys, func(i, j int) bool {
                // for equal elements, sort string alphabetically
                if sortMap[keys[i]] == sortMap[keys[j]] {
                        return keys[i] < keys[j]
                }
                return sortMap[keys[i]] < sortMap[keys[j]]
        })
        return keys
}

// Create a clusterGroupUpgrade
func (r *ManagedClusterForCguReconciler) newClusterGroupUpgrade(
        ctx context.Context, cluster *clusterv1.ManagedCluster, childPolicies []policiesv1.Policy) (err error) {

        var policyWaveMap = make(map[string]int)

        // Generate a list of ordered managed policies based on the deploy wave.
        // Deploywave is a way to order deployment of policies, it's defined
        // as a annotation in policy.
        // For example,
        //   metadata:
        //     annotations:
        //       "ran.openshift.io/ztp-deploy-wave": "1"
        // The list of policies is ordered from the lowest value to the highest.
        // Policy without a wave is not managed.
        for _, cPolicy := range childPolicies {
                // Ignore policies with remediationAction enforce
                if strings.EqualFold(string(cPolicy.Spec.RemediationAction), "enforce") {
                        r.Log.Info("Ignoring policy with remediationAction enforce", "policy", cPolicy.Name)
                        continue
                }

                deployWave, found := cPolicy.GetAnnotations()[ztpDeployWaveAnnotation]
                if found {
                        deployWaveInt, err := strconv.Atoi(deployWave)
                        if err != nil {
                                // err convert from string to int
                                return fmt.Errorf("%s in policy %s is not an interger: %s", ztpDeployWaveAnnotation, cPolicy.GetName(), err)
                        }
                        policyName, err := utils.GetParentPolicyNameAndNamespace(cPolicy.GetName())
                        if err != nil {
                                r.Log.Info("Ignoring policy with invalid name", "policy", cPolicy.Name)
                                continue
                        }
                        policyWaveMap[policyName[1]] = deployWaveInt
                }
        }

        sortedManagedPolicies := sortMapByValue(policyWaveMap)
        cguMeta := metav1.ObjectMeta{
                Name:      cluster.Name,
                Namespace: ztpInstallNS,
        }
        enable := true // default
        cguSpec := ranv1alpha1.ClusterGroupUpgradeSpec{
                Enable:          &enable,
                Clusters:        []string{cluster.Name},
                ManagedPolicies: sortedManagedPolicies,
                RemediationStrategy: &ranv1alpha1.RemediationStrategySpec{
                        MaxConcurrency: 1,
                },
                Actions: ranv1alpha1.Actions{
                        BeforeEnable: &ranv1alpha1.BeforeEnable{
                                AddClusterLabels: map[string]string{
                                        ztpRunningLabel: "",
                                },
                        },
                        AfterCompletion: &ranv1alpha1.AfterCompletion{
                                AddClusterLabels: map[string]string{
                                        ztpDoneLabel: "",
                                },
                                RemoveClusterLabels: []string{
                                        ztpRunningLabel,
                                },
                        },
                },
        }
        clusterGroupUpgrade := &ranv1alpha1.ClusterGroupUpgrade{
                ObjectMeta: cguMeta,
                Spec:       cguSpec,
        }

        // set managedcluster as the owner of its created ClusterGroupUpgrade CR, so when a cluster
        // is deleted, its dependent ClusterGroupUpgrade CR will be automatically cleaned up
        if err := controllerutil.SetControllerReference(cluster, clusterGroupUpgrade, r.Scheme); err != nil {
                return err
        }

        if err := r.Create(ctx, clusterGroupUpgrade); err != nil {
                if errors.IsNotFound(err) && strings.Contains(err.Error(), "namespace") {
                        namespace := &corev1.Namespace{
                                ObjectMeta: metav1.ObjectMeta{
                                        Name: ztpInstallNS,
                                },
                        }
                        if err := r.Create(ctx, namespace); err != nil {
                                r.Log.Error(err, "Fail to create namespace", "name", ztpInstallNS)
                                return err
                        }
                        // retry
                        if err := r.Create(ctx, clusterGroupUpgrade); err != nil {
                                r.Log.Error(err, "Fail to create clusterGroupUpgrade", "name", cluster.Name, "namespace", ztpInstallNS)
                                return err
                        }
                }

                r.Log.Error(err, "Fail to create clusterGroupUpgrade", "name", cluster.Name, "namespace", ztpInstallNS)
                return err
        }

        r.Log.Info("Found ManagedCluster "+cluster.Name+" without "+ztpDoneLabel+" label. Created clusterGroupUpgrade.",
                "name", cluster.Name, "namespace", ztpInstallNS)
        return nil
}

func (r *ManagedClusterForCguReconciler) newConfigmap(name, namespace string) *corev1.ConfigMap {
        return &corev1.ConfigMap{
                TypeMeta:   metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
                ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
                Data:       map[string]string{},
        }
}

// Extract policies for AZTP
func (r *ManagedClusterForCguReconciler) extractAztpPolicies(ctx context.Context, managedCluster *clusterv1.ManagedCluster) (err error) {
        aztpVariant, aztpRequired := managedCluster.GetLabels()[aztpRequiredLabel]
        if !aztpRequired {
                return nil
        }
        namespace := managedCluster.GetName()
        if namespace == "local-cluster" {
                return nil
        }
        configMapName := fmt.Sprintf("%s-aztp", namespace)
        innerCmNs := os.Getenv("AZTP_INNER_CONFIGMAP_NAMESPACE")
        if innerCmNs == "" {
                innerCmNs = "ztp-profile"
        }
        innerCmName := os.Getenv("AZTP_INNER_CONFIGMAP_NAME")
        if innerCmName == "" {
                innerCmName = "ztp-post-provision"
        }
        cm := r.newConfigmap(configMapName, namespace)
        if err := r.Delete(ctx, cm, &client.DeleteOptions{}); err != nil {
                if !errors.IsNotFound(err) {
                        return err
                }
        }

        policies, err := utils.GetChildPolicies(ctx, r.Client, []string{namespace})
        if err != nil {
                return err
        }
        if len(policies) == 0 {
                r.Log.Info("no child policies found for cluster", "Name", namespace)
                return nil
        }

        objects, err := r.GetConfigurationObjects(policies)
        if err != nil {
                return err
        }
        r.Log.Info(fmt.Sprintf("found %d policies and %d objects for %s", len(policies), len(objects), namespace))
        var directlyAppliedObjects []unstructured.Unstructured
        var wrappedObjects []unstructured.Unstructured

        for _, ob := range objects {
                ob.Object["status"] = map[string]interface{}{} // remove status, we can't apply it
                groupKind := ob.GroupVersionKind().GroupKind().String()
                r.Log.Info("adding object", "groupKind", groupKind, "name", ob.GetName())
                switch groupKind {
                case "PerformanceProfile.performance.openshift.io",
                        "Tuned.tuned.openshift.io",
                        "Namespace",
                        "CatalogSource.operators.coreos.com",
                        "ContainerRuntimeConfig.machineconfiguration.openshift.io":
                        directlyAppliedObjects = append(directlyAppliedObjects, ob)
                case "Subscription.operators.coreos.com":
                        ob.Object["spec"].(map[string]interface{})["installPlanApproval"] = "Automatic"
                        wrappedObjects = append(wrappedObjects, ob)
                default:
                        wrappedObjects = append(wrappedObjects, ob)
                }
        }
        if len(wrappedObjects) > 0 {
                innerCm, err := r.wrapObjects(wrappedObjects, innerCmName, innerCmNs)
                if err != nil {
                        return err
                }

                innerCmObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(innerCm)
                if err != nil {
                        return err
                }
                var innerCmUns unstructured.Unstructured
                innerCmUns.SetUnstructuredContent(innerCmObj)
                directlyAppliedObjects = append(directlyAppliedObjects, innerCmUns)
        }

        var data templates.AztpTemplateData

        data.AztpImage = os.Getenv("AZTP_IMG")
        if strings.Compare(data.AztpImage, "") == 0 {
                return fmt.Errorf("aztp failure: can't retrieve aztp image pull spec")
        }

        objects, err = templates.RenderAztpService(data, aztpVariant)
        if err != nil {
                return fmt.Errorf("failed to create aztp service manifests: %v", err)
        }
        directlyAppliedObjects = append(directlyAppliedObjects, objects...)

        cmr, err := r.wrapObjects(directlyAppliedObjects, configMapName, namespace)
        if err != nil {
                return err
        }
        // make this managedcluster owner of the configmap
        if err := controllerutil.SetControllerReference(managedCluster, cmr, r.Scheme); err != nil {
                return err
        }
        if err := r.Create(ctx, cmr); err != nil {
                return err
        }
        return nil
}

func (r *ManagedClusterForCguReconciler) wrapObjects(objects []unstructured.Unstructured, name, namespace string) (*corev1.ConfigMap, error) {
        cm := r.newConfigmap(name, namespace)
        for _, item := range objects {
                key := strings.ToLower(fmt.Sprintf("%s-%s-%s.yaml", item.GetKind(), item.GetNamespace(), item.GetName()))
                out, err := yaml.Marshal(item.Object)
                if err != nil {
                        return cm, err
                }
                cm.Data[key] = string(out)
        }
        return cm, nil
}

// GetConfigurationObjects gets encapsulated objects from configuration policies
func (r *ManagedClusterForCguReconciler) GetConfigurationObjects(policies []policiesv1.Policy) ([]unstructured.Unstructured, error) {
        var uobjects []unstructured.Unstructured

        var objects []runtime.RawExtension
        for _, pol := range policies {
                if !pol.Spec.Disabled {
                        for _, template := range pol.Spec.PolicyTemplates {
                                o := *template.ObjectDefinition.DeepCopy()
                                objects = append(objects, o)
                        }
                }
        }

        for _, ob := range objects {
                var pol policyv1.ConfigurationPolicy
                err := json.Unmarshal(ob.DeepCopy().Raw, &pol)
                if err != nil {
                        r.Log.Info(err.Error())
                        return uobjects, err
                }
                for _, ot := range pol.Spec.ObjectTemplates {
                        var object unstructured.Unstructured
                        err = object.UnmarshalJSON(ot.ObjectDefinition.DeepCopy().Raw)
                        if err != nil {
                                return uobjects, err
                        }
                        object.Object["status"] = map[string]interface{}{}
                        _, specDefined := object.Object["spec"]
                        if specDefined || object.GetKind() == "Namespace" {
                                uobjects = append(uobjects, object)
                        }
                }
        }
        return uobjects, nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *ManagedClusterForCguReconciler) SetupWithManager(mgr ctrl.Manager) error {
        namespace := &corev1.Namespace{
                ObjectMeta: metav1.ObjectMeta{
                        Name: ztpInstallNS,
                },
        }
        if err := r.Create(context.TODO(), namespace); err != nil {
                // fail to create namespace
                if !errors.IsAlreadyExists(err) {
                        return err
                }
        }

        return ctrl.NewControllerManagedBy(mgr).
                Named("managedclusterForCGU").
                For(&clusterv1.ManagedCluster{},
                        // watch for create event for managedcluster
                        builder.WithPredicates(predicate.Funcs{
                                GenericFunc: func(e event.GenericEvent) bool { return false },
                                CreateFunc:  func(e event.CreateEvent) bool { return true },
                                DeleteFunc:  func(e event.DeleteEvent) bool { return false },
                                UpdateFunc: func(e event.UpdateEvent) bool {
                                        // Check if the event was deleting the label "ztp-done"
                                        // We want to return true for that event only, and false for everything else
                                        _, doneLabelExistsInOld := e.ObjectOld.GetLabels()[ztpDoneLabel]
                                        _, doneLabelExistsInNew := e.ObjectNew.GetLabels()[ztpDoneLabel]
                                        _, aztpRequired := e.ObjectNew.GetLabels()[aztpRequiredLabel]

                                        doneLabelRemoved := doneLabelExistsInOld && !doneLabelExistsInNew

                                        var availableInNew, availableInOld bool
                                        availableCondition := meta.FindStatusCondition(e.ObjectOld.(*clusterv1.ManagedCluster).Status.Conditions, clusterv1.ManagedClusterConditionAvailable)
                                        if availableCondition != nil && availableCondition.Status == metav1.ConditionTrue {
                                                availableInOld = true
                                        }
                                        availableCondition = meta.FindStatusCondition(e.ObjectNew.(*clusterv1.ManagedCluster).Status.Conditions, clusterv1.ManagedClusterConditionAvailable)
                                        if availableCondition != nil && availableCondition.Status == metav1.ConditionTrue {
                                                availableInNew = true
                                        }

                                        var hubAccepted bool
                                        acceptedCondition := meta.FindStatusCondition(e.ObjectNew.(*clusterv1.ManagedCluster).Status.Conditions, clusterv1.ManagedClusterConditionHubAccepted)
                                        if acceptedCondition != nil && acceptedCondition.Status == metav1.ConditionTrue {
                                                hubAccepted = true
                                        }

                                        return (doneLabelRemoved && availableInNew) || (!availableInOld && availableInNew && !doneLabelExistsInNew) || (hubAccepted && aztpRequired)
                                },
                        })).
                Owns(&ranv1alpha1.ClusterGroupUpgrade{},
                        // watch for delete event for owned ClusterGroupUpgrade
                        builder.WithPredicates(predicate.Funcs{
                                GenericFunc: func(e event.GenericEvent) bool { return false },
                                CreateFunc:  func(e event.CreateEvent) bool { return false },
                                DeleteFunc:  func(e event.DeleteEvent) bool { return true },
                                UpdateFunc:  func(e event.UpdateEvent) bool { return false },
                        })).
                Complete(r)
}

1	/*
2	Copyright 2021.
3
4	Licensed under the Apache License, Version 2.0 (the "License");
5	you may not use this file except in compliance with the License.
6	You may obtain a copy of the License at
7
8	http://www.apache.org/licenses/LICENSE-2.0
9
10	Unless required by applicable law or agreed to in writing, software
11	distributed under the License is distributed on an "AS IS" BASIS,
12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	See the License for the specific language governing permissions and
14	limitations under the License.
15	*/
16
17	package controllers
18
19	import (
20	"context"
21	"encoding/json"
22	"fmt"
23	"os"
24	"sort"
25	"strconv"
26	"strings"
27
28	"github.com/go-logr/logr"
29	corev1 "k8s.io/api/core/v1"
30	"k8s.io/apimachinery/pkg/api/errors"
31	"k8s.io/apimachinery/pkg/api/meta"
32	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
34	"k8s.io/apimachinery/pkg/runtime"
35	"k8s.io/apimachinery/pkg/types"
36	ctrl "sigs.k8s.io/controller-runtime"
37	"sigs.k8s.io/controller-runtime/pkg/builder"
38	"sigs.k8s.io/controller-runtime/pkg/client"
39	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
40	"sigs.k8s.io/controller-runtime/pkg/event"
41	"sigs.k8s.io/controller-runtime/pkg/predicate"
42	"sigs.k8s.io/controller-runtime/pkg/reconcile"
43	yaml "sigs.k8s.io/yaml"
44
45	"github.com/openshift-kni/cluster-group-upgrades-operator/controllers/templates"
46	utils "github.com/openshift-kni/cluster-group-upgrades-operator/controllers/utils"
47	ranv1alpha1 "github.com/openshift-kni/cluster-group-upgrades-operator/pkg/api/clustergroupupgrades/v1alpha1"
48	clusterv1 "open-cluster-management.io/api/cluster/v1"
49	policyv1 "open-cluster-management.io/config-policy-controller/api/v1"
50	policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
51	)
52
53	const (
54	ztpInstallNS = "ztp-install"
55	ztpDeployWaveAnnotation = "ran.openshift.io/ztp-deploy-wave"
56	ztpRunningLabel = "ztp-running"
57	ztpDoneLabel = "ztp-done"
58	aztpRequiredLabel = "accelerated-ztp"
59	aztpDeployServiceVariant = "full"
60	)
61
62	// ManagedClusterForCguReconciler reconciles a ManagedCluster object to auto create the ClusterGroupUpgrade
63	type ManagedClusterForCguReconciler struct {
64	client.Client
65	Log logr.Logger
66	Scheme *runtime.Scheme
67	}
68
69	//+kubebuilder:rbac:groups=cluster.open-cluster-management.io,resources=managedclusters,verbs=get;list;watch
70	//+kubebuilder:rbac:groups=cluster.open-cluster-management.io,resources=managedclusters/finalizers,verbs=update
71	//+kubebuilder:rbac:groups=ran.openshift.io,resources=clustergroupupgrades,verbs=get;list;watch;create;update;patch;delete
72	//+kubebuilder:rbac:groups=ran.openshift.io,resources=precachingconfigs,verbs=get;list;watch;create;update;patch;delete
73	//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete
74	//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;create;delete
75
76	// Reconcile the managed cluster auto create ClusterGroupUpgrade
77	// - Controller watches for create event of managed cluster object. Reconciliation
78	// is triggered when a new managed cluster is created
79	// - When a new managed cluster is created, create ClusterGroupUpgrade CR for the
80	// cluster only when it's ready and its child policies are available
81	// - As created ClusterGroupUpgrade has ownReference set to its managed cluster,
82	// when the managed cluster is deleted, the ClusterGroupUpgrade will be auto-deleted
83	//
84	// Note: The Controller will requeue the Request to be processed again if the returned error is non-nil or
85	// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
86	func (r *ManagedClusterForCguReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {	1✔
87		1✔
88	managedCluster := &clusterv1.ManagedCluster{}	1✔
89	if err := r.Get(ctx, types.NamespacedName{Name: req.Name}, managedCluster); err != nil {	1✔
90	if errors.IsNotFound(err) {	×
91	// managed cluster could have been deleted	×
92	return ctrl.Result{}, nil	×
93	}	×
94	// Error reading managed cluster, requeue the request
95	return ctrl.Result{}, err	×
96	}
97
98	// Stop creating UOCR or AZTP if ztp of this cluster is done already
99	if _, found := managedCluster.Labels[ztpDoneLabel]; found {	2✔
100	r.Log.Info("ZTP for the cluster has completed. "+ztpDoneLabel+" label found.", "Name", managedCluster.Name)	1✔
101	return doNotRequeue(), nil	1✔
102	}	1✔
103
104	clusterGroupUpgrade := &ranv1alpha1.ClusterGroupUpgrade{}	1✔
105	if err := r.Get(ctx, types.NamespacedName{Name: req.Name, Namespace: ztpInstallNS}, clusterGroupUpgrade); err != nil {	2✔
106	if !errors.IsNotFound(err) {	1✔
107	// Error reading clusterGroupUpgrade, requeue the request	×
108	return ctrl.Result{}, err	×
109	}	×
110	} else {	1✔
111	// clusterGroupUpgrade for this cluster already exists, stop reconcile	1✔
112	r.Log.Info("clusterGroupUpgrade found", "Name", clusterGroupUpgrade.Name, "Namespace", clusterGroupUpgrade.Namespace)	1✔
113	return doNotRequeue(), nil	1✔
114	}	1✔
115
116	// clusterGroupUpgrade CR doesn't exist
117	availableCondition := meta.FindStatusCondition(managedCluster.Status.Conditions, clusterv1.ManagedClusterConditionAvailable)	1✔
118	if availableCondition != nil && availableCondition.Status == metav1.ConditionTrue {	2✔
119	// cluster is ready	1✔
120	r.Log.Info("cluster is ready", "Name", managedCluster.Name)	1✔
121		1✔
122	// Child policies get created as soon as the placementrule/placementbinding	1✔
123	// gets created and matches the parent policies to the managedcluster.	1✔
124	// It takes ~45 minutes for cluster to be installed and ready.	1✔
125	// At this stage, all child policies should be created.	1✔
126	policies, err := utils.GetChildPolicies(ctx, r.Client, []string{managedCluster.Name})	1✔
127	if err != nil {	1✔
128	return ctrl.Result{}, err	×
129	}	×
130	if len(policies) == 0 {	2✔
131	// likely no policies were created, so no child policies found	1✔
132	r.Log.Info("WARN: No child policies found for cluster", "Name", managedCluster.Name)	1✔
133	}	1✔
134
135	// create clusterGroupUpgrade
136	if err := r.newClusterGroupUpgrade(ctx, managedCluster, policies); err != nil {	2✔
137	return ctrl.Result{}, err	1✔
138	}	1✔
139	} else {	1✔
140	// availableCondition is false or unknown, cluster is not ready	1✔
141	return r.handleNotReadyCluster(ctx, managedCluster)	1✔
142	}	1✔
143
144	return doNotRequeue(), nil	1✔
145	}
146
147	func (r *ManagedClusterForCguReconciler) handleNotReadyCluster(
148	ctx context.Context, managedCluster *clusterv1.ManagedCluster) (reconcile.Result, error) {	1✔
149	if err := r.extractAztpPolicies(ctx, managedCluster); err != nil {	1✔
150	return ctrl.Result{}, err	×
151	}	×
152	r.Log.Info("cluster is not ready", "Name", managedCluster.Name)	1✔
153	return doNotRequeue(), nil	1✔
154	}
155
156	// sort map[string]int by value in ascending order, return sorted keys
157	func sortMapByValue(sortMap map[string]int) []string {	1✔
158	var keys []string	1✔
159	for key := range sortMap {	1✔
160	keys = append(keys, key)	×
161	}	×
162
163	sort.Slice(keys, func(i, j int) bool {	1✔
164	// for equal elements, sort string alphabetically	×
165	if sortMap[keys[i]] == sortMap[keys[j]] {	×
166	return keys[i] < keys[j]	×
167	}	×
168	return sortMap[keys[i]] < sortMap[keys[j]]	×
169	})
170	return keys	1✔
171	}
172
173	// Create a clusterGroupUpgrade
174	func (r *ManagedClusterForCguReconciler) newClusterGroupUpgrade(
175	ctx context.Context, cluster *clusterv1.ManagedCluster, childPolicies []policiesv1.Policy) (err error) {	1✔
176		1✔
177	var policyWaveMap = make(map[string]int)	1✔
178		1✔
179	// Generate a list of ordered managed policies based on the deploy wave.	1✔
180	// Deploywave is a way to order deployment of policies, it's defined	1✔
181	// as a annotation in policy.	1✔
182	// For example,	1✔
183	// metadata:	1✔
184	// annotations:	1✔
185	// "ran.openshift.io/ztp-deploy-wave": "1"	1✔
186	// The list of policies is ordered from the lowest value to the highest.	1✔
187	// Policy without a wave is not managed.	1✔
188	for _, cPolicy := range childPolicies {	1✔
189	// Ignore policies with remediationAction enforce	×
190	if strings.EqualFold(string(cPolicy.Spec.RemediationAction), "enforce") {	×
191	r.Log.Info("Ignoring policy with remediationAction enforce", "policy", cPolicy.Name)	×
192	continue	×
193	}
194
195	deployWave, found := cPolicy.GetAnnotations()[ztpDeployWaveAnnotation]	×
196	if found {	×
197	deployWaveInt, err := strconv.Atoi(deployWave)	×
198	if err != nil {	×
199	// err convert from string to int	×
200	return fmt.Errorf("%s in policy %s is not an interger: %s", ztpDeployWaveAnnotation, cPolicy.GetName(), err)	×
201	}	×
202	policyName, err := utils.GetParentPolicyNameAndNamespace(cPolicy.GetName())	×
203	if err != nil {	×
204	r.Log.Info("Ignoring policy with invalid name", "policy", cPolicy.Name)	×
205	continue	×
206	}
207	policyWaveMap[policyName[1]] = deployWaveInt	×
208	}
209	}
210
211	sortedManagedPolicies := sortMapByValue(policyWaveMap)	1✔
212	cguMeta := metav1.ObjectMeta{	1✔
213	Name: cluster.Name,	1✔
214	Namespace: ztpInstallNS,	1✔
215	}	1✔
216	enable := true // default	1✔
217	cguSpec := ranv1alpha1.ClusterGroupUpgradeSpec{	1✔
218	Enable: &enable,	1✔
219	Clusters: []string{cluster.Name},	1✔
220	ManagedPolicies: sortedManagedPolicies,	1✔
221	RemediationStrategy: &ranv1alpha1.RemediationStrategySpec{	1✔
222	MaxConcurrency: 1,	1✔
223	},	1✔
224	Actions: ranv1alpha1.Actions{	1✔
225	BeforeEnable: &ranv1alpha1.BeforeEnable{	1✔
226	AddClusterLabels: map[string]string{	1✔
227	ztpRunningLabel: "",	1✔
228	},	1✔
229	},	1✔
230	AfterCompletion: &ranv1alpha1.AfterCompletion{	1✔
231	AddClusterLabels: map[string]string{	1✔
232	ztpDoneLabel: "",	1✔
233	},	1✔
234	RemoveClusterLabels: []string{	1✔
235	ztpRunningLabel,	1✔
236	},	1✔
237	},	1✔
238	},	1✔
239	}	1✔
240	clusterGroupUpgrade := &ranv1alpha1.ClusterGroupUpgrade{	1✔
241	ObjectMeta: cguMeta,	1✔
242	Spec: cguSpec,	1✔
243	}	1✔
244		1✔
245	// set managedcluster as the owner of its created ClusterGroupUpgrade CR, so when a cluster	1✔
246	// is deleted, its dependent ClusterGroupUpgrade CR will be automatically cleaned up	1✔
247	if err := controllerutil.SetControllerReference(cluster, clusterGroupUpgrade, r.Scheme); err != nil {	1✔
248	return err	×
249	}	×
250
251	if err := r.Create(ctx, clusterGroupUpgrade); err != nil {	2✔
252	if errors.IsNotFound(err) && strings.Contains(err.Error(), "namespace") {	2✔
253	namespace := &corev1.Namespace{	1✔
254	ObjectMeta: metav1.ObjectMeta{	1✔
255	Name: ztpInstallNS,	1✔
256	},	1✔
257	}	1✔
258	if err := r.Create(ctx, namespace); err != nil {	1✔
259	r.Log.Error(err, "Fail to create namespace", "name", ztpInstallNS)	×
260	return err	×
261	}	×
262	// retry
263	if err := r.Create(ctx, clusterGroupUpgrade); err != nil {	1✔
264	r.Log.Error(err, "Fail to create clusterGroupUpgrade", "name", cluster.Name, "namespace", ztpInstallNS)	×
265	return err	×
266	}	×
267	}
268
269	r.Log.Error(err, "Fail to create clusterGroupUpgrade", "name", cluster.Name, "namespace", ztpInstallNS)	1✔
270	return err	1✔
271	}
272
273	r.Log.Info("Found ManagedCluster "+cluster.Name+" without "+ztpDoneLabel+" label. Created clusterGroupUpgrade.",	1✔
274	"name", cluster.Name, "namespace", ztpInstallNS)	1✔
275	return nil	1✔
276	}
277
278	func (r ManagedClusterForCguReconciler) newConfigmap(name, namespace string) corev1.ConfigMap {	×
279	return &corev1.ConfigMap{	×
280	TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},	×
281	ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},	×
282	Data: map[string]string{},	×
283	}	×
284	}	×
285
286	// Extract policies for AZTP
287	func (r ManagedClusterForCguReconciler) extractAztpPolicies(ctx context.Context, managedCluster clusterv1.ManagedCluster) (err error) {	1✔
288	aztpVariant, aztpRequired := managedCluster.GetLabels()[aztpRequiredLabel]	1✔
289	if !aztpRequired {	2✔
290	return nil	1✔
291	}	1✔
292	namespace := managedCluster.GetName()	×
293	if namespace == "local-cluster" {	×
294	return nil	×
295	}	×
296	configMapName := fmt.Sprintf("%s-aztp", namespace)	×
297	innerCmNs := os.Getenv("AZTP_INNER_CONFIGMAP_NAMESPACE")	×
298	if innerCmNs == "" {	×
299	innerCmNs = "ztp-profile"	×
300	}	×
301	innerCmName := os.Getenv("AZTP_INNER_CONFIGMAP_NAME")	×
302	if innerCmName == "" {	×
303	innerCmName = "ztp-post-provision"	×
304	}	×
305	cm := r.newConfigmap(configMapName, namespace)	×
306	if err := r.Delete(ctx, cm, &client.DeleteOptions{}); err != nil {	×
307	if !errors.IsNotFound(err) {	×
308	return err	×
309	}	×
310	}
311
312	policies, err := utils.GetChildPolicies(ctx, r.Client, []string{namespace})	×
313	if err != nil {	×
314	return err	×
315	}	×
316	if len(policies) == 0 {	×
317	r.Log.Info("no child policies found for cluster", "Name", namespace)	×
318	return nil	×
319	}	×
320
321	objects, err := r.GetConfigurationObjects(policies)	×
322	if err != nil {	×
323	return err	×
324	}	×
325	r.Log.Info(fmt.Sprintf("found %d policies and %d objects for %s", len(policies), len(objects), namespace))	×
326	var directlyAppliedObjects []unstructured.Unstructured	×
327	var wrappedObjects []unstructured.Unstructured	×
328		×
329	for _, ob := range objects {	×
330	ob.Object["status"] = map[string]interface{}{} // remove status, we can't apply it	×
331	groupKind := ob.GroupVersionKind().GroupKind().String()	×
332	r.Log.Info("adding object", "groupKind", groupKind, "name", ob.GetName())	×
333	switch groupKind {	×
334	case "PerformanceProfile.performance.openshift.io",
335	"Tuned.tuned.openshift.io",
336	"Namespace",
337	"CatalogSource.operators.coreos.com",
338	"ContainerRuntimeConfig.machineconfiguration.openshift.io":	×
339	directlyAppliedObjects = append(directlyAppliedObjects, ob)	×
340	case "Subscription.operators.coreos.com":	×
341	ob.Object["spec"].(map[string]interface{})["installPlanApproval"] = "Automatic"	×
342	wrappedObjects = append(wrappedObjects, ob)	×
343	default:	×
344	wrappedObjects = append(wrappedObjects, ob)	×
345	}
346	}
347	if len(wrappedObjects) > 0 {	×
348	innerCm, err := r.wrapObjects(wrappedObjects, innerCmName, innerCmNs)	×
349	if err != nil {	×
350	return err	×
351	}	×
352
353	innerCmObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(innerCm)	×
354	if err != nil {	×
355	return err	×
356	}	×
357	var innerCmUns unstructured.Unstructured	×
358	innerCmUns.SetUnstructuredContent(innerCmObj)	×
359	directlyAppliedObjects = append(directlyAppliedObjects, innerCmUns)	×
360	}
361
362	var data templates.AztpTemplateData	×
363		×
364	data.AztpImage = os.Getenv("AZTP_IMG")	×
365	if strings.Compare(data.AztpImage, "") == 0 {	×
366	return fmt.Errorf("aztp failure: can't retrieve aztp image pull spec")	×
367	}	×
368
369	objects, err = templates.RenderAztpService(data, aztpVariant)	×
370	if err != nil {	×
371	return fmt.Errorf("failed to create aztp service manifests: %v", err)	×
372	}	×
373	directlyAppliedObjects = append(directlyAppliedObjects, objects...)	×
374		×
375	cmr, err := r.wrapObjects(directlyAppliedObjects, configMapName, namespace)	×
376	if err != nil {	×
377	return err	×
378	}	×
379	// make this managedcluster owner of the configmap
380	if err := controllerutil.SetControllerReference(managedCluster, cmr, r.Scheme); err != nil {	×
381	return err	×
382	}	×
383	if err := r.Create(ctx, cmr); err != nil {	×
384	return err	×
385	}	×
386	return nil	×
387	}
388
389	func (r ManagedClusterForCguReconciler) wrapObjects(objects []unstructured.Unstructured, name, namespace string) (corev1.ConfigMap, error) {	×
390	cm := r.newConfigmap(name, namespace)	×
391	for _, item := range objects {	×
392	key := strings.ToLower(fmt.Sprintf("%s-%s-%s.yaml", item.GetKind(), item.GetNamespace(), item.GetName()))	×
393	out, err := yaml.Marshal(item.Object)	×
394	if err != nil {	×
395	return cm, err	×
396	}	×
397	cm.Data[key] = string(out)	×
398	}
399	return cm, nil	×
400	}
401
402	// GetConfigurationObjects gets encapsulated objects from configuration policies
403	func (r *ManagedClusterForCguReconciler) GetConfigurationObjects(policies []policiesv1.Policy) ([]unstructured.Unstructured, error) {	×
404	var uobjects []unstructured.Unstructured	×
405		×
406	var objects []runtime.RawExtension	×
407	for _, pol := range policies {	×
408	if !pol.Spec.Disabled {	×
409	for _, template := range pol.Spec.PolicyTemplates {	×
410	o := *template.ObjectDefinition.DeepCopy()	×
411	objects = append(objects, o)	×
412	}	×
413	}
414	}
415
416	for _, ob := range objects {	×
417	var pol policyv1.ConfigurationPolicy	×
418	err := json.Unmarshal(ob.DeepCopy().Raw, &pol)	×
419	if err != nil {	×
420	r.Log.Info(err.Error())	×
421	return uobjects, err	×
422	}	×
423	for _, ot := range pol.Spec.ObjectTemplates {	×
424	var object unstructured.Unstructured	×
425	err = object.UnmarshalJSON(ot.ObjectDefinition.DeepCopy().Raw)	×
426	if err != nil {	×
427	return uobjects, err	×
428	}	×
429	object.Object["status"] = map[string]interface{}{}	×
430	_, specDefined := object.Object["spec"]	×
431	if specDefined \|\| object.GetKind() == "Namespace" {	×
432	uobjects = append(uobjects, object)	×
433	}	×
434	}
435	}
436	return uobjects, nil	×
437	}
438
439	// SetupWithManager sets up the controller with the Manager.
440	func (r *ManagedClusterForCguReconciler) SetupWithManager(mgr ctrl.Manager) error {	1✔
441	namespace := &corev1.Namespace{	1✔
442	ObjectMeta: metav1.ObjectMeta{	1✔
443	Name: ztpInstallNS,	1✔
444	},	1✔
445	}	1✔
446	if err := r.Create(context.TODO(), namespace); err != nil {	2✔
447	// fail to create namespace	1✔
448	if !errors.IsAlreadyExists(err) {	1✔
449	return err	×
450	}	×
451	}
452
453	return ctrl.NewControllerManagedBy(mgr).	1✔
454	Named("managedclusterForCGU").	1✔
455	For(&clusterv1.ManagedCluster{},	1✔
456	// watch for create event for managedcluster	1✔
457	builder.WithPredicates(predicate.Funcs{	1✔
458	GenericFunc: func(e event.GenericEvent) bool { return false },	1✔
459	CreateFunc: func(e event.CreateEvent) bool { return true },	1✔
460	DeleteFunc: func(e event.DeleteEvent) bool { return false },	1✔
461	UpdateFunc: func(e event.UpdateEvent) bool {	1✔
462	// Check if the event was deleting the label "ztp-done"	1✔
463	// We want to return true for that event only, and false for everything else	1✔
464	_, doneLabelExistsInOld := e.ObjectOld.GetLabels()[ztpDoneLabel]	1✔
465	_, doneLabelExistsInNew := e.ObjectNew.GetLabels()[ztpDoneLabel]	1✔
466	_, aztpRequired := e.ObjectNew.GetLabels()[aztpRequiredLabel]	1✔
467		1✔
468	doneLabelRemoved := doneLabelExistsInOld && !doneLabelExistsInNew	1✔
469		1✔
470	var availableInNew, availableInOld bool	1✔
471	availableCondition := meta.FindStatusCondition(e.ObjectOld.(*clusterv1.ManagedCluster).Status.Conditions, clusterv1.ManagedClusterConditionAvailable)	1✔
472	if availableCondition != nil && availableCondition.Status == metav1.ConditionTrue {	2✔
473	availableInOld = true	1✔
474	}	1✔
475	availableCondition = meta.FindStatusCondition(e.ObjectNew.(*clusterv1.ManagedCluster).Status.Conditions, clusterv1.ManagedClusterConditionAvailable)	1✔
476	if availableCondition != nil && availableCondition.Status == metav1.ConditionTrue {	2✔
477	availableInNew = true	1✔
478	}	1✔
479
480	var hubAccepted bool	1✔
481	acceptedCondition := meta.FindStatusCondition(e.ObjectNew.(*clusterv1.ManagedCluster).Status.Conditions, clusterv1.ManagedClusterConditionHubAccepted)	1✔
482	if acceptedCondition != nil && acceptedCondition.Status == metav1.ConditionTrue {	1✔
483	hubAccepted = true	×
484	}	×
485
486	return (doneLabelRemoved && availableInNew) \|\| (!availableInOld && availableInNew && !doneLabelExistsInNew) \|\| (hubAccepted && aztpRequired)	1✔
487	},
488	})).
489	Owns(&ranv1alpha1.ClusterGroupUpgrade{},
490	// watch for delete event for owned ClusterGroupUpgrade
491	builder.WithPredicates(predicate.Funcs{
492	GenericFunc: func(e event.GenericEvent) bool { return false },	×
493	CreateFunc: func(e event.CreateEvent) bool { return false },	1✔
494	DeleteFunc: func(e event.DeleteEvent) bool { return true },	1✔
495	UpdateFunc: func(e event.UpdateEvent) bool { return false },	1✔
496	})).
497	Complete(r)
498	}

openshift-kni / cluster-group-upgrades-operator / #3

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous