19209619462

Committed 09 Nov 2025 02:06PM UTC coverage: 63.489% (+0.6%) from 62.904%

Build # 19209619462

Build Type

push

github

Committed by

heathcliff26

Commit Message

upgraded: Load configuration from file when changed

Instead of using node annotations, switch back to reading the configuration
from file.
This allows to use ConfigMaps together with the DaemonSets to fully manage
the upgraded daemons from the controller.
Ensure the loaded config is up-to-date by watching for file changes and
reloading the config when the file changes.

Signed-off-by: Heathcliff <heathcliff@heathcliff.eu>

Run Details

188 of 241 new or added lines in 10 files covered. (78.01%)

7 existing lines in 4 files now uncovered.

1019 of 1605 relevant lines covered (63.49%)

11.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.56

/pkg/upgrade-controller/controller/controller.go

package controller

import (
        "context"
        "fmt"
        "os"
        "time"

        "github.com/go-logr/logr"
        api "github.com/heathcliff26/kube-upgrade/pkg/apis/kubeupgrade/v1alpha3"
        "github.com/heathcliff26/kube-upgrade/pkg/client/clientset/versioned/scheme"
        "github.com/heathcliff26/kube-upgrade/pkg/constants"
        "golang.org/x/mod/semver"
        appv1 "k8s.io/api/apps/v1"
        corev1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/labels"
        "k8s.io/client-go/kubernetes"
        "k8s.io/client-go/rest"
        "k8s.io/klog/v2"
        ctrl "sigs.k8s.io/controller-runtime"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
        "sigs.k8s.io/controller-runtime/pkg/healthz"
        "sigs.k8s.io/controller-runtime/pkg/manager"
        "sigs.k8s.io/controller-runtime/pkg/manager/signals"
)

func init() {
        ctrl.SetLogger(klog.NewKlogr())
}

type controller struct {
        client.Client
        manager       manager.Manager
        client        kubernetes.Interface
        namespace     string
        upgradedImage string
}

// Run make generate when changing these comments
// +kubebuilder:rbac:groups=kubeupgrade.heathcliff.eu,resources=kubeupgradeplans,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=kubeupgrade.heathcliff.eu,resources=kubeupgradeplans/status,verbs=get;update;patch
// +kubebuilder:rbac:groups="",resources=nodes,verbs=list;update
func NewController(name string) (*controller, error) {
        config, err := rest.InClusterConfig()
        if err != nil {
                return nil, err
        }
        client, err := kubernetes.NewForConfig(config)
        if err != nil {
                return nil, err
        }

        ns, err := GetNamespace()
        if err != nil {
                return nil, err
        }

        mgr, err := ctrl.NewManager(config, manager.Options{
                Scheme:                        scheme.Scheme,
                LeaderElection:                true,
                LeaderElectionNamespace:       ns,
                LeaderElectionID:              name,
                LeaderElectionReleaseOnCancel: true,
                LeaseDuration:                 Pointer(time.Minute),
                RenewDeadline:                 Pointer(10 * time.Second),
                RetryPeriod:                   Pointer(5 * time.Second),
                HealthProbeBindAddress:        ":9090",
        })
        if err != nil {
                return nil, err
        }
        err = mgr.AddHealthzCheck("healthz", healthz.Ping)
        if err != nil {
                return nil, err
        }
        err = mgr.AddReadyzCheck("readyz", healthz.Ping)
        if err != nil {
                return nil, err
        }

        upgradedImage := os.Getenv("UPGRADED_IMAGE")
        if upgradedImage == "" {
                return nil, fmt.Errorf("UPGRADED_IMAGE environment variable is not set")
        }

        return &controller{
                Client:        mgr.GetClient(),
                manager:       mgr,
                client:        client,
                namespace:     ns,
                upgradedImage: upgradedImage,
        }, nil
}

func (c *controller) Run() error {
        err := ctrl.NewControllerManagedBy(c.manager).For(&api.KubeUpgradePlan{}).Complete(c)
        if err != nil {
                return err
        }

        err = ctrl.NewWebhookManagedBy(c.manager).
                For(&api.KubeUpgradePlan{}).
                WithDefaulter(&planMutatingHook{}).
                WithValidator(&planValidatingHook{}).
                Complete()
        if err != nil {
                return err
        }

        return c.manager.Start(signals.SetupSignalHandler())
}

func (c *controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
        logger := klog.LoggerWithValues(klog.NewKlogr(), "plan", req.Name)

        var plan api.KubeUpgradePlan
        err := c.Get(ctx, req.NamespacedName, &plan)
        if err != nil {
                logger.Error(err, "Failed to get Plan")
                return ctrl.Result{}, err
        }

        err = c.reconcile(ctx, &plan, logger)
        if err != nil {
                return ctrl.Result{}, err
        }

        err = c.Status().Update(ctx, &plan)
        if err != nil {
                logger.Error(err, "Failed to update plan status")
                return ctrl.Result{}, err
        }

        return ctrl.Result{
                Requeue:      plan.Status.Summary != api.PlanStatusComplete,
                RequeueAfter: time.Minute,
        }, nil
}

func (c *controller) reconcile(ctx context.Context, plan *api.KubeUpgradePlan, logger logr.Logger) error {
        if plan.Status.Groups == nil {
                plan.Status.Groups = make(map[string]string, len(plan.Spec.Groups))
        }

        if controllerutil.AddFinalizer(plan, constants.Finalizer) {
                err := c.Update(ctx, plan)
                if err != nil {
                        return fmt.Errorf("failed to add finalizer to plan %s: %v", plan.Name, err)
                }
        }

        cmList, err := c.client.CoreV1().ConfigMaps(c.namespace).List(ctx, metav1.ListOptions{
                LabelSelector: fmt.Sprintf("%s=%s", constants.LabelPlanName, plan.Name),
        })
        if err != nil {
                logger.WithValues("plan", plan.Name).Error(err, "Failed to fetch upgraded ConfigMaps")
                return err
        }

        daemonsList, err := c.client.AppsV1().DaemonSets(c.namespace).List(ctx, metav1.ListOptions{
                LabelSelector: fmt.Sprintf("%s=%s", constants.LabelPlanName, plan.Name),
        })
        if err != nil {
                logger.WithValues("plan", plan.Name).Error(err, "Failed to fetch upgraded DaemonSets")
                return err
        }

        if !plan.DeletionTimestamp.IsZero() {
                logger.WithValues("plan", plan.Name).Info("Plan is being deleted, cleaning up resources")
                for _, daemon := range daemonsList.Items {
                        err := c.client.AppsV1().DaemonSets(c.namespace).Delete(ctx, daemon.Name, metav1.DeleteOptions{})
                        if err != nil {
                                return fmt.Errorf("failed to delete DaemonSet %s: %v", daemon.Name, err)
                        }
                        logger.WithValues("name", daemon.Name).Info("Deleted DaemonSet")
                }
                for _, cm := range cmList.Items {
                        err := c.client.CoreV1().ConfigMaps(c.namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})
                        if err != nil {
                                return fmt.Errorf("failed to delete ConfigMap %s: %v", cm.Name, err)
                        }
                        logger.WithValues("name", cm.Name).Info("Deleted ConfigMap")
                }
                controllerutil.RemoveFinalizer(plan, constants.Finalizer)
                err := c.Update(ctx, plan)
                if err != nil {
                        return fmt.Errorf("failed to remove finalizer from plan %s: %v", plan.Name, err)
                }
                logger.WithValues("plan", plan.Name).Info("Finished cleanup of resources")
                return nil
        }

        daemons := make(map[string]appv1.DaemonSet, len(plan.Spec.Groups))
        for _, daemon := range daemonsList.Items {
                group := daemon.Labels[constants.LabelNodeGroup]
                if _, ok := plan.Spec.Groups[group]; ok {
                        daemons[group] = daemon
                } else {
                        err := c.client.AppsV1().DaemonSets(c.namespace).Delete(ctx, daemon.Name, metav1.DeleteOptions{})
                        if err != nil {
                                return fmt.Errorf("failed to delete DaemonSet %s: %v", daemon.Name, err)
                        }
                        logger.WithValues("name", daemon.Name).Info("Deleted obsolete DaemonSet")
                }
        }

        cms := make(map[string]corev1.ConfigMap, len(plan.Spec.Groups))
        for _, cm := range cmList.Items {
                group := cm.Labels[constants.LabelNodeGroup]
                if _, ok := plan.Spec.Groups[group]; ok {
                        cms[group] = cm
                } else {
                        err := c.client.CoreV1().ConfigMaps(c.namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})
                        if err != nil {
                                return fmt.Errorf("failed to delete ConfigMap %s: %v", cm.Name, err)
                        }
                        logger.WithValues("name", cm.Name).Info("Deleted obsolete ConfigMap")
                }
        }

        nodesToUpdate := make(map[string][]corev1.Node, len(plan.Spec.Groups))
        newGroupStatus := make(map[string]string, len(plan.Spec.Groups))

        for name, cfg := range plan.Spec.Groups {
                upgradedCfg := combineConfig(plan.Spec.Upgraded, plan.Spec.Groups[name].Upgraded)

                cm, ok := cms[name]
                if !ok {
                        cm = c.NewEmptyUpgradedConfigMap(plan.Name, name)
                }
                err = c.AttachUpgradedConfigMapData(&cm, upgradedCfg)
                if err != nil {
                        return fmt.Errorf("failed to attach data to ConfigMap %s: %v", cm.Name, err)
                }
                if ok {
                        _, err = c.client.CoreV1().ConfigMaps(c.namespace).Update(ctx, &cm, metav1.UpdateOptions{})
                } else {
                        logger.WithValues("group", name, "config", cm.Name).Info("Creating upgraded ConfigMap for group")
                        _, err = c.client.CoreV1().ConfigMaps(c.namespace).Create(ctx, &cm, metav1.CreateOptions{})
                }
                if err != nil {
                        return fmt.Errorf("failed to create/update ConfigMap %s: %v", cm.Name, err)
                }

                daemon, ok := daemons[name]
                if !ok {
                        daemon = c.NewEmptyUpgradedDaemonSet(plan.Name, name)
                }
                daemon.Spec = c.NewUpgradedDaemonSetSpec(plan.Name, name)
                daemon.Spec.Template.Spec.NodeSelector = cfg.Labels
                if ok {
                        _, err = c.client.AppsV1().DaemonSets(c.namespace).Update(ctx, &daemon, metav1.UpdateOptions{})
                } else {
                        logger.WithValues("group", name, "daemon", daemon.Name).Info("Creating upgraded DaemonSet for group")
                        _, err = c.client.AppsV1().DaemonSets(c.namespace).Create(ctx, &daemon, metav1.CreateOptions{})
                }
                if err != nil {
                        return fmt.Errorf("failed to create/update DaemonSet %s: %v", daemon.Name, err)
                }

                nodeList, err := c.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{
                        LabelSelector: labels.SelectorFromSet(cfg.Labels).String(),
                })
                if err != nil {
                        logger.WithValues("group", name).Error(err, "Failed to get nodes for group")
                        return err
                }

                status, update, nodes, err := c.reconcileNodes(plan.Spec.KubernetesVersion, plan.Spec.AllowDowngrade, nodeList.Items)
                if err != nil {
                        logger.WithValues("group", name).Error(err, "Failed to reconcile nodes for group")
                        return err
                }

                newGroupStatus[name] = status

                if update {
                        nodesToUpdate[name] = nodes
                } else if plan.Status.Groups[name] != newGroupStatus[name] {
                        logger.WithValues("group", name, "status", newGroupStatus[name]).Info("Group changed status")
                }
        }

        for name, nodes := range nodesToUpdate {
                if groupWaitForDependency(plan.Spec.Groups[name].DependsOn, newGroupStatus) {
                        logger.WithValues("group", name).Info("Group is waiting on dependencies")
                        newGroupStatus[name] = api.PlanStatusWaiting
                        continue
                } else if plan.Status.Groups[name] != newGroupStatus[name] {
                        logger.WithValues("group", name, "status", newGroupStatus[name]).Info("Group changed status")
                }

                for _, node := range nodes {
                        _, err := c.client.CoreV1().Nodes().Update(ctx, &node, metav1.UpdateOptions{})
                        if err != nil {
                                return fmt.Errorf("failed to update node %s: %v", node.GetName(), err)
                        }
                }
        }

        plan.Status.Groups = newGroupStatus
        plan.Status.Summary = createStatusSummary(plan.Status.Groups)

        return nil
}

func (c *controller) reconcileNodes(kubeVersion string, downgrade bool, nodes []corev1.Node) (string, bool, []corev1.Node, error) {
        if len(nodes) == 0 {
                return api.PlanStatusUnknown, false, nil, nil
        }

        completed := 0
        needUpdate := false
        errorNodes := make([]string, 0)

        for i := range nodes {
                if nodes[i].Annotations == nil {
                        nodes[i].Annotations = make(map[string]string)
                }

                // Step to cleanup after migration to v0.6.0
                // TODO: Remove in v0.7.0
                if deleteConfigAnnotations(nodes[i].Annotations) {
                        needUpdate = true
                }

                if !downgrade && semver.Compare(kubeVersion, nodes[i].Status.NodeInfo.KubeletVersion) < 0 {
                        return api.PlanStatusError, false, nil, fmt.Errorf("node %s version %s is newer than %s, but downgrade is disabled", nodes[i].GetName(), nodes[i].Status.NodeInfo.KubeletVersion, kubeVersion)
                }

                if nodes[i].Annotations[constants.NodeKubernetesVersion] == kubeVersion {
                        switch nodes[i].Annotations[constants.NodeUpgradeStatus] {
                        case constants.NodeUpgradeStatusCompleted:
                                completed++
                        case constants.NodeUpgradeStatusError:
                                errorNodes = append(errorNodes, nodes[i].GetName())
                        }
                        continue
                }

                nodes[i].Annotations[constants.NodeKubernetesVersion] = kubeVersion
                nodes[i].Annotations[constants.NodeUpgradeStatus] = constants.NodeUpgradeStatusPending

                needUpdate = true
        }

        var status string
        if len(errorNodes) > 0 {
                status = fmt.Sprintf("%s: The nodes %v are reporting errors", api.PlanStatusError, errorNodes)
        } else if len(nodes) == completed {
                status = api.PlanStatusComplete
        } else {
                status = fmt.Sprintf("%s: %d/%d nodes upgraded", api.PlanStatusProgressing, completed, len(nodes))
        }
        return status, needUpdate, nodes, nil
}

1	package controller
2
3	import (
4	"context"
5	"fmt"
6	"os"
7	"time"
8
9	"github.com/go-logr/logr"
10	api "github.com/heathcliff26/kube-upgrade/pkg/apis/kubeupgrade/v1alpha3"
11	"github.com/heathcliff26/kube-upgrade/pkg/client/clientset/versioned/scheme"
12	"github.com/heathcliff26/kube-upgrade/pkg/constants"
13	"golang.org/x/mod/semver"
14	appv1 "k8s.io/api/apps/v1"
15	corev1 "k8s.io/api/core/v1"
16	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17	"k8s.io/apimachinery/pkg/labels"
18	"k8s.io/client-go/kubernetes"
19	"k8s.io/client-go/rest"
20	"k8s.io/klog/v2"
21	ctrl "sigs.k8s.io/controller-runtime"
22	"sigs.k8s.io/controller-runtime/pkg/client"
23	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
24	"sigs.k8s.io/controller-runtime/pkg/healthz"
25	"sigs.k8s.io/controller-runtime/pkg/manager"
26	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
27	)
28
29	func init() {	2✔
30	ctrl.SetLogger(klog.NewKlogr())	2✔
31	}	2✔
32
33	type controller struct {
34	client.Client
35	manager manager.Manager
36	client kubernetes.Interface
37	namespace string
38	upgradedImage string
39	}
40
41	// Run make generate when changing these comments
42	// +kubebuilder:rbac:groups=kubeupgrade.heathcliff.eu,resources=kubeupgradeplans,verbs=get;list;watch;create;update;patch;delete
43	// +kubebuilder:rbac:groups=kubeupgrade.heathcliff.eu,resources=kubeupgradeplans/status,verbs=get;update;patch
44	// +kubebuilder:rbac:groups="",resources=nodes,verbs=list;update
45	func NewController(name string) (*controller, error) {	1✔
46	config, err := rest.InClusterConfig()	1✔
47	if err != nil {	2✔
48	return nil, err	1✔
49	}	1✔
50	client, err := kubernetes.NewForConfig(config)	×
51	if err != nil {	×
52	return nil, err	×
53	}	×
54
55	ns, err := GetNamespace()	×
56	if err != nil {	×
57	return nil, err	×
58	}	×
59
60	mgr, err := ctrl.NewManager(config, manager.Options{	×
61	Scheme: scheme.Scheme,	×
62	LeaderElection: true,	×
63	LeaderElectionNamespace: ns,	×
64	LeaderElectionID: name,	×
65	LeaderElectionReleaseOnCancel: true,	×
66	LeaseDuration: Pointer(time.Minute),	×
67	RenewDeadline: Pointer(10 * time.Second),	×
68	RetryPeriod: Pointer(5 * time.Second),	×
69	HealthProbeBindAddress: ":9090",	×
70	})	×
71	if err != nil {	×
72	return nil, err	×
73	}	×
74	err = mgr.AddHealthzCheck("healthz", healthz.Ping)	×
75	if err != nil {	×
76	return nil, err	×
77	}	×
78	err = mgr.AddReadyzCheck("readyz", healthz.Ping)	×
79	if err != nil {	×
80	return nil, err	×
81	}	×
82
83	upgradedImage := os.Getenv("UPGRADED_IMAGE")	×
84	if upgradedImage == "" {	×
85	return nil, fmt.Errorf("UPGRADED_IMAGE environment variable is not set")	×
86	}	×
87
88	return &controller{	×
89	Client: mgr.GetClient(),	×
90	manager: mgr,	×
91	client: client,	×
92	namespace: ns,	×
93	upgradedImage: upgradedImage,	×
94	}, nil	×
95	}
96
97	func (c *controller) Run() error {	×
98	err := ctrl.NewControllerManagedBy(c.manager).For(&api.KubeUpgradePlan{}).Complete(c)	×
99	if err != nil {	×
100	return err	×
101	}	×
102
103	err = ctrl.NewWebhookManagedBy(c.manager).	×
104	For(&api.KubeUpgradePlan{}).	×
105	WithDefaulter(&planMutatingHook{}).	×
106	WithValidator(&planValidatingHook{}).	×
107	Complete()	×
108	if err != nil {	×
109	return err	×
110	}	×
111
112	return c.manager.Start(signals.SetupSignalHandler())	×
113	}
114
115	func (c *controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {	×
116	logger := klog.LoggerWithValues(klog.NewKlogr(), "plan", req.Name)	×
117		×
118	var plan api.KubeUpgradePlan	×
119	err := c.Get(ctx, req.NamespacedName, &plan)	×
120	if err != nil {	×
121	logger.Error(err, "Failed to get Plan")	×
122	return ctrl.Result{}, err	×
123	}	×
124
125	err = c.reconcile(ctx, &plan, logger)	×
126	if err != nil {	×
127	return ctrl.Result{}, err	×
128	}	×
129
130	err = c.Status().Update(ctx, &plan)	×
131	if err != nil {	×
132	logger.Error(err, "Failed to update plan status")	×
133	return ctrl.Result{}, err	×
134	}	×
135
136	return ctrl.Result{	×
137	Requeue: plan.Status.Summary != api.PlanStatusComplete,	×
138	RequeueAfter: time.Minute,	×
139	}, nil	×
140	}
141
142	func (c controller) reconcile(ctx context.Context, plan api.KubeUpgradePlan, logger logr.Logger) error {	18✔
143	if plan.Status.Groups == nil {	28✔
144	plan.Status.Groups = make(map[string]string, len(plan.Spec.Groups))	10✔
145	}	10✔
146
147	if controllerutil.AddFinalizer(plan, constants.Finalizer) {	35✔
148	err := c.Update(ctx, plan)	17✔
149	if err != nil {	17✔
150	return fmt.Errorf("failed to add finalizer to plan %s: %v", plan.Name, err)	×
151	}	×
152	}
153
154	cmList, err := c.client.CoreV1().ConfigMaps(c.namespace).List(ctx, metav1.ListOptions{	18✔
155	LabelSelector: fmt.Sprintf("%s=%s", constants.LabelPlanName, plan.Name),	18✔
156	})	18✔
157	if err != nil {	18✔
NEW 158	logger.WithValues("plan", plan.Name).Error(err, "Failed to fetch upgraded ConfigMaps")	×
NEW 159	return err	×
NEW 160	}	×
161
162	daemonsList, err := c.client.AppsV1().DaemonSets(c.namespace).List(ctx, metav1.ListOptions{	18✔
163	LabelSelector: fmt.Sprintf("%s=%s", constants.LabelPlanName, plan.Name),	18✔
164	})	18✔
165	if err != nil {	18✔
NEW 166	logger.WithValues("plan", plan.Name).Error(err, "Failed to fetch upgraded DaemonSets")	×
167	return err	×
168	}	×
169
170	if !plan.DeletionTimestamp.IsZero() {	19✔
171	logger.WithValues("plan", plan.Name).Info("Plan is being deleted, cleaning up resources")	1✔
172	for _, daemon := range daemonsList.Items {	2✔
173	err := c.client.AppsV1().DaemonSets(c.namespace).Delete(ctx, daemon.Name, metav1.DeleteOptions{})	1✔
174	if err != nil {	1✔
175	return fmt.Errorf("failed to delete DaemonSet %s: %v", daemon.Name, err)	×
176	}	×
177	logger.WithValues("name", daemon.Name).Info("Deleted DaemonSet")	1✔
178	}
179	for _, cm := range cmList.Items {	2✔
180	err := c.client.CoreV1().ConfigMaps(c.namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})	1✔
181	if err != nil {	1✔
NEW 182	return fmt.Errorf("failed to delete ConfigMap %s: %v", cm.Name, err)	×
NEW 183	}	×
184	logger.WithValues("name", cm.Name).Info("Deleted ConfigMap")	1✔
185	}
186	controllerutil.RemoveFinalizer(plan, constants.Finalizer)	1✔
187	err := c.Update(ctx, plan)	1✔
188	if err != nil {	1✔
189	return fmt.Errorf("failed to remove finalizer from plan %s: %v", plan.Name, err)	×
190	}	×
191	logger.WithValues("plan", plan.Name).Info("Finished cleanup of resources")	1✔
192	return nil	1✔
193	}
194
195	daemons := make(map[string]appv1.DaemonSet, len(plan.Spec.Groups))	17✔
196	for _, daemon := range daemonsList.Items {	25✔
197	group := daemon.Labels[constants.LabelNodeGroup]	8✔
198	if _, ok := plan.Spec.Groups[group]; ok {	14✔
199	daemons[group] = daemon	6✔
200	} else {	8✔
201	err := c.client.AppsV1().DaemonSets(c.namespace).Delete(ctx, daemon.Name, metav1.DeleteOptions{})	2✔
202	if err != nil {	2✔
203	return fmt.Errorf("failed to delete DaemonSet %s: %v", daemon.Name, err)	×
204	}	×
205	logger.WithValues("name", daemon.Name).Info("Deleted obsolete DaemonSet")	2✔
206	}
207	}
208
209	cms := make(map[string]corev1.ConfigMap, len(plan.Spec.Groups))	17✔
210	for _, cm := range cmList.Items {	25✔
211	group := cm.Labels[constants.LabelNodeGroup]	8✔
212	if _, ok := plan.Spec.Groups[group]; ok {	14✔
213	cms[group] = cm	6✔
214	} else {	8✔
215	err := c.client.CoreV1().ConfigMaps(c.namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})	2✔
216	if err != nil {	2✔
NEW 217	return fmt.Errorf("failed to delete ConfigMap %s: %v", cm.Name, err)	×
NEW 218	}	×
219	logger.WithValues("name", cm.Name).Info("Deleted obsolete ConfigMap")	2✔
220	}
221	}
222
223	nodesToUpdate := make(map[string][]corev1.Node, len(plan.Spec.Groups))	17✔
224	newGroupStatus := make(map[string]string, len(plan.Spec.Groups))	17✔
225		17✔
226	for name, cfg := range plan.Spec.Groups {	54✔
227	upgradedCfg := combineConfig(plan.Spec.Upgraded, plan.Spec.Groups[name].Upgraded)	37✔
228		37✔
229	cm, ok := cms[name]	37✔
230	if !ok {	68✔
231	cm = c.NewEmptyUpgradedConfigMap(plan.Name, name)	31✔
232	}	31✔
233	err = c.AttachUpgradedConfigMapData(&cm, upgradedCfg)	37✔
234	if err != nil {	37✔
NEW 235	return fmt.Errorf("failed to attach data to ConfigMap %s: %v", cm.Name, err)	×
NEW 236	}	×
237	if ok {	43✔
238	_, err = c.client.CoreV1().ConfigMaps(c.namespace).Update(ctx, &cm, metav1.UpdateOptions{})	6✔
239	} else {	37✔
240	logger.WithValues("group", name, "config", cm.Name).Info("Creating upgraded ConfigMap for group")	31✔
241	_, err = c.client.CoreV1().ConfigMaps(c.namespace).Create(ctx, &cm, metav1.CreateOptions{})	31✔
242	}	31✔
243	if err != nil {	37✔
NEW 244	return fmt.Errorf("failed to create/update ConfigMap %s: %v", cm.Name, err)	×
NEW 245	}	×
246
247	daemon, ok := daemons[name]	37✔
248	if !ok {	68✔
249	daemon = c.NewEmptyUpgradedDaemonSet(plan.Name, name)	31✔
250	}	31✔
251	daemon.Spec = c.NewUpgradedDaemonSetSpec(plan.Name, name)	37✔
252	daemon.Spec.Template.Spec.NodeSelector = cfg.Labels	37✔
253	if ok {	43✔
254	_, err = c.client.AppsV1().DaemonSets(c.namespace).Update(ctx, &daemon, metav1.UpdateOptions{})	6✔
255	} else {	37✔
256	logger.WithValues("group", name, "daemon", daemon.Name).Info("Creating upgraded DaemonSet for group")	31✔
257	_, err = c.client.AppsV1().DaemonSets(c.namespace).Create(ctx, &daemon, metav1.CreateOptions{})	31✔
258	}	31✔
259	if err != nil {	37✔
260	return fmt.Errorf("failed to create/update DaemonSet %s: %v", daemon.Name, err)	×
261	}	×
262
263	nodeList, err := c.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{	37✔
264	LabelSelector: labels.SelectorFromSet(cfg.Labels).String(),	37✔
265	})	37✔
266	if err != nil {	37✔
267	logger.WithValues("group", name).Error(err, "Failed to get nodes for group")	×
268	return err	×
269	}	×
270
271	status, update, nodes, err := c.reconcileNodes(plan.Spec.KubernetesVersion, plan.Spec.AllowDowngrade, nodeList.Items)	37✔
272	if err != nil {	37✔
273	logger.WithValues("group", name).Error(err, "Failed to reconcile nodes for group")	×
274	return err	×
275	}	×
276
277	newGroupStatus[name] = status	37✔
278		37✔
279	if update {	60✔
280	nodesToUpdate[name] = nodes	23✔
281	} else if plan.Status.Groups[name] != newGroupStatus[name] {	47✔
282	logger.WithValues("group", name, "status", newGroupStatus[name]).Info("Group changed status")	10✔
283	}	10✔
284	}
285
286	for name, nodes := range nodesToUpdate {	40✔
287	if groupWaitForDependency(plan.Spec.Groups[name].DependsOn, newGroupStatus) {	29✔
288	logger.WithValues("group", name).Info("Group is waiting on dependencies")	6✔
289	newGroupStatus[name] = api.PlanStatusWaiting	6✔
290	continue	6✔
291	} else if plan.Status.Groups[name] != newGroupStatus[name] {	32✔
292	logger.WithValues("group", name, "status", newGroupStatus[name]).Info("Group changed status")	15✔
293	}	15✔
294
295	for _, node := range nodes {	34✔
296	_, err := c.client.CoreV1().Nodes().Update(ctx, &node, metav1.UpdateOptions{})	17✔
297	if err != nil {	17✔
298	return fmt.Errorf("failed to update node %s: %v", node.GetName(), err)	×
299	}	×
300	}
301	}
302
303	plan.Status.Groups = newGroupStatus	17✔
304	plan.Status.Summary = createStatusSummary(plan.Status.Groups)	17✔
305		17✔
306	return nil	17✔
307	}
308
309	func (c *controller) reconcileNodes(kubeVersion string, downgrade bool, nodes []corev1.Node) (string, bool, []corev1.Node, error) {	39✔
310	if len(nodes) == 0 {	43✔
311	return api.PlanStatusUnknown, false, nil, nil	4✔
312	}	4✔
313
314	completed := 0	35✔
315	needUpdate := false	35✔
316	errorNodes := make([]string, 0)	35✔
317		35✔
318	for i := range nodes {	70✔
319	if nodes[i].Annotations == nil {	55✔
320	nodes[i].Annotations = make(map[string]string)	20✔
321	}	20✔
322
323	// Step to cleanup after migration to v0.6.0
324	// TODO: Remove in v0.7.0
325	if deleteConfigAnnotations(nodes[i].Annotations) {	37✔
326	needUpdate = true	2✔
327	}	2✔
328
329	if !downgrade && semver.Compare(kubeVersion, nodes[i].Status.NodeInfo.KubeletVersion) < 0 {	36✔
330	return api.PlanStatusError, false, nil, fmt.Errorf("node %s version %s is newer than %s, but downgrade is disabled", nodes[i].GetName(), nodes[i].Status.NodeInfo.KubeletVersion, kubeVersion)	1✔
331	}	1✔
332
333	if nodes[i].Annotations[constants.NodeKubernetesVersion] == kubeVersion {	46✔
334	switch nodes[i].Annotations[constants.NodeUpgradeStatus] {	12✔
335	case constants.NodeUpgradeStatusCompleted:	11✔
336	completed++	11✔
337	case constants.NodeUpgradeStatusError:	1✔
338	errorNodes = append(errorNodes, nodes[i].GetName())	1✔
339	}
340	continue	12✔
341	}
342
343	nodes[i].Annotations[constants.NodeKubernetesVersion] = kubeVersion	22✔
344	nodes[i].Annotations[constants.NodeUpgradeStatus] = constants.NodeUpgradeStatusPending	22✔
345		22✔
346	needUpdate = true	22✔
347	}
348
349	var status string	34✔
350	if len(errorNodes) > 0 {	35✔
351	status = fmt.Sprintf("%s: The nodes %v are reporting errors", api.PlanStatusError, errorNodes)	1✔
352	} else if len(nodes) == completed {	45✔
353	status = api.PlanStatusComplete	11✔
354	} else {	33✔
355	status = fmt.Sprintf("%s: %d/%d nodes upgraded", api.PlanStatusProgressing, completed, len(nodes))	22✔
356	}	22✔
357	return status, needUpdate, nodes, nil	34✔
358	}

heathcliff26 / kube-upgrade / 19209619462

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous