5809875303

Committed 09 Aug 2023 01:45PM UTC coverage: 75.025% (+0.02%) from 75.008%

Build # 5809875303

Build Type

push

github

Committed by

web-flow

Commit Message

Add stack weight verification for HPA reconciling (#519)

* Add stack weight verification for HPA reconciling

Signed-off-by: Lucas Thiesen <lucas.thiesen@zalando.de>

* Use a more generic verification for annotations

* Fix issue of removed annotations not being caught by reconcile function

* Fix identation

Signed-off-by: Lucas Thiesen <lucas.thiesen@zalando.de>

---------

Signed-off-by: Lucas Thiesen <lucas.thiesen@zalando.de>

Run Details

15 of 15 new or added lines in 1 file covered. (100.0%)

2247 of 2995 relevant lines covered (75.03%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.4

/controller/stack_resources.go

package controller

import (
        "context"

        rgv1 "github.com/szuecs/routegroup-client/apis/zalando.org/v1"
        zv1 "github.com/zalando-incubator/stackset-controller/pkg/apis/zalando.org/v1"
        "github.com/zalando-incubator/stackset-controller/pkg/core"
        apps "k8s.io/api/apps/v1"
        v2 "k8s.io/api/autoscaling/v2"
        apiv1 "k8s.io/api/core/v1"
        networking "k8s.io/api/networking/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func pint32Equal(p1, p2 *int32) bool {
        if p1 == nil && p2 == nil {
                return true
        }
        if p1 != nil && p2 != nil {
                return *p1 == *p2
        }
        return false
}

// There are HPA metrics that depend on annotations to work properly,
// e.g. External RPS metric, this verification provides a way to verify
// all relevant annotations are actually up to date.
func areHPAAnnotationsUpToDate(updated, existing *v2.HorizontalPodAutoscaler) bool {
        if len(updated.Annotations) != len(existing.Annotations) {
                return false
        }

        for k, v := range updated.Annotations {
                if k == "stackset-controller.zalando.org/stack-generation" {
                        continue
                }

                existingValue, ok := existing.Annotations[k]
                if ok && existingValue == v {
                        continue
                }

                return false
        }

        return true
}

// syncObjectMeta copies metadata elements such as labels or annotations from source to target
func syncObjectMeta(target, source metav1.Object) {
        target.SetLabels(source.GetLabels())
        target.SetAnnotations(source.GetAnnotations())
}

func (c *StackSetController) ReconcileStackDeployment(ctx context.Context, stack *zv1.Stack, existing *apps.Deployment, generateUpdated func() *apps.Deployment) error {
        deployment := generateUpdated()

        // Create new deployment
        if existing == nil {
                _, err := c.client.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{})
                if err != nil {
                        return err
                }
                c.recorder.Eventf(
                        stack,
                        apiv1.EventTypeNormal,
                        "CreatedDeployment",
                        "Created Deployment %s",
                        deployment.Name)
                return nil
        }

        // Check if we need to update the deployment
        if core.IsResourceUpToDate(stack, existing.ObjectMeta) && pint32Equal(existing.Spec.Replicas, deployment.Spec.Replicas) {
                return nil
        }

        updated := existing.DeepCopy()
        syncObjectMeta(updated, deployment)
        updated.Spec = deployment.Spec
        updated.Spec.Selector = existing.Spec.Selector

        _, err := c.client.AppsV1().Deployments(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})
        if err != nil {
                return err
        }
        c.recorder.Eventf(
                stack,
                apiv1.EventTypeNormal,
                "UpdatedDeployment",
                "Updated Deployment %s",
                deployment.Name)
        return nil
}

func (c *StackSetController) ReconcileStackHPA(ctx context.Context, stack *zv1.Stack, existing *v2.HorizontalPodAutoscaler, generateUpdated func() (*v2.HorizontalPodAutoscaler, error)) error {
        hpa, err := generateUpdated()
        if err != nil {
                return err
        }

        // HPA removed
        if hpa == nil {
                if existing != nil {
                        err := c.client.AutoscalingV2().HorizontalPodAutoscalers(existing.Namespace).Delete(ctx, existing.Name, metav1.DeleteOptions{})
                        if err != nil {
                                return err
                        }
                        c.recorder.Eventf(
                                stack,
                                apiv1.EventTypeNormal,
                                "DeletedHPA",
                                "Deleted HPA %s",
                                existing.Namespace)
                }
                return nil
        }

        // Create new HPA
        if existing == nil {
                _, err := c.client.AutoscalingV2().HorizontalPodAutoscalers(hpa.Namespace).Create(ctx, hpa, metav1.CreateOptions{})
                if err != nil {
                        return err
                }
                c.recorder.Eventf(
                        stack,
                        apiv1.EventTypeNormal,
                        "CreatedHPA",
                        "Created HPA %s",
                        hpa.Name)
                return nil
        }

        // Check if we need to update the HPA
        if core.IsResourceUpToDate(stack, existing.ObjectMeta) &&
                pint32Equal(existing.Spec.MinReplicas, hpa.Spec.MinReplicas) &&
                areHPAAnnotationsUpToDate(hpa, existing) {
                return nil
        }

        updated := existing.DeepCopy()
        syncObjectMeta(updated, hpa)
        updated.Spec = hpa.Spec

        _, err = c.client.AutoscalingV2().HorizontalPodAutoscalers(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})
        if err != nil {
                return err
        }
        c.recorder.Eventf(
                stack,
                apiv1.EventTypeNormal,
                "UpdatedHPA",
                "Updated HPA %s",
                hpa.Name)
        return nil
}

func (c *StackSetController) ReconcileStackService(ctx context.Context, stack *zv1.Stack, existing *apiv1.Service, generateUpdated func() (*apiv1.Service, error)) error {
        service, err := generateUpdated()
        if err != nil {
                return err
        }

        // Create new service
        if existing == nil {
                _, err := c.client.CoreV1().Services(service.Namespace).Create(ctx, service, metav1.CreateOptions{})
                if err != nil {
                        return err
                }
                c.recorder.Eventf(
                        stack,
                        apiv1.EventTypeNormal,
                        "CreatedService",
                        "Created Service %s",
                        service.Name)
                return nil
        }

        // Check if we need to update the service
        if core.IsResourceUpToDate(stack, existing.ObjectMeta) {
                return nil
        }

        updated := existing.DeepCopy()
        syncObjectMeta(updated, service)
        updated.Spec = service.Spec
        updated.Spec.ClusterIP = existing.Spec.ClusterIP // ClusterIP is immutable

        _, err = c.client.CoreV1().Services(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})
        if err != nil {
                return err
        }
        c.recorder.Eventf(
                stack,
                apiv1.EventTypeNormal,
                "UpdatedService",
                "Updated Service %s",
                service.Name)
        return nil
}

func (c *StackSetController) ReconcileStackIngress(ctx context.Context, stack *zv1.Stack, existing *networking.Ingress, generateUpdated func() (*networking.Ingress, error)) error {
        ingress, err := generateUpdated()
        if err != nil {
                return err
        }

        // Ingress removed
        if ingress == nil {
                if existing != nil {
                        err := c.client.NetworkingV1().Ingresses(existing.Namespace).Delete(ctx, existing.Name, metav1.DeleteOptions{})
                        if err != nil {
                                return err
                        }
                        c.recorder.Eventf(
                                stack,
                                apiv1.EventTypeNormal,
                                "DeletedIngress",
                                "Deleted Ingress %s",
                                existing.Namespace)
                }
                return nil
        }

        // Create new Ingress
        if existing == nil {
                _, err := c.client.NetworkingV1().Ingresses(ingress.Namespace).Create(ctx, ingress, metav1.CreateOptions{})
                if err != nil {
                        return err
                }
                c.recorder.Eventf(
                        stack,
                        apiv1.EventTypeNormal,
                        "CreatedIngress",
                        "Created Ingress %s",
                        ingress.Name)
                return nil
        }

        // Check if we need to update the Ingress
        if core.IsResourceUpToDate(stack, existing.ObjectMeta) {
                return nil
        }

        updated := existing.DeepCopy()
        syncObjectMeta(updated, ingress)
        updated.Spec = ingress.Spec

        _, err = c.client.NetworkingV1().Ingresses(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})
        if err != nil {
                return err
        }
        c.recorder.Eventf(
                stack,
                apiv1.EventTypeNormal,
                "UpdatedIngress",
                "Updated Ingress %s",
                ingress.Name)
        return nil
}

func (c *StackSetController) ReconcileStackRouteGroup(ctx context.Context, stack *zv1.Stack, existing *rgv1.RouteGroup, generateUpdated func() (*rgv1.RouteGroup, error)) error {
        routegroup, err := generateUpdated()
        if err != nil {
                return err
        }

        // RouteGroup removed
        if routegroup == nil {
                if existing != nil {
                        err := c.client.RouteGroupV1().RouteGroups(existing.Namespace).Delete(ctx, existing.Name, metav1.DeleteOptions{})
                        if err != nil {
                                return err
                        }
                        c.recorder.Eventf(
                                stack,
                                apiv1.EventTypeNormal,
                                "DeletedRouteGroup",
                                "Deleted RouteGroup %s",
                                existing.Namespace)
                }
                return nil
        }

        // Create new RouteGroup
        if existing == nil {
                _, err := c.client.RouteGroupV1().RouteGroups(routegroup.Namespace).Create(ctx, routegroup, metav1.CreateOptions{})
                if err != nil {
                        return err
                }
                c.recorder.Eventf(
                        stack,
                        apiv1.EventTypeNormal,
                        "CreatedRouteGroup",
                        "Created RouteGroup %s",
                        routegroup.Name)
                return nil
        }

        // Check if we need to update the RouteGroup
        if core.IsResourceUpToDate(stack, existing.ObjectMeta) {
                return nil
        }

        updated := existing.DeepCopy()
        syncObjectMeta(updated, routegroup)
        updated.Spec = routegroup.Spec

        _, err = c.client.RouteGroupV1().RouteGroups(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})
        if err != nil {
                return err
        }
        c.recorder.Eventf(
                stack,
                apiv1.EventTypeNormal,
                "UpdatedRouteGroup",
                "Updated RouteGroup %s",
                routegroup.Name)
        return nil
}

1	package controller
2
3	import (
4	"context"
5
6	rgv1 "github.com/szuecs/routegroup-client/apis/zalando.org/v1"
7	zv1 "github.com/zalando-incubator/stackset-controller/pkg/apis/zalando.org/v1"
8	"github.com/zalando-incubator/stackset-controller/pkg/core"
9	apps "k8s.io/api/apps/v1"
10	v2 "k8s.io/api/autoscaling/v2"
11	apiv1 "k8s.io/api/core/v1"
12	networking "k8s.io/api/networking/v1"
13	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14	)
15
16	func pint32Equal(p1, p2 *int32) bool {	1✔
17	if p1 == nil && p2 == nil {	1✔
18	return true	×
19	}	×
20	if p1 != nil && p2 != nil {	2✔
21	return p1 == p2	1✔
22	}	1✔
23	return false	×
24	}
25
26	// There are HPA metrics that depend on annotations to work properly,
27	// e.g. External RPS metric, this verification provides a way to verify
28	// all relevant annotations are actually up to date.
29	func areHPAAnnotationsUpToDate(updated, existing *v2.HorizontalPodAutoscaler) bool {	1✔
30	if len(updated.Annotations) != len(existing.Annotations) {	1✔
31	return false	×
32	}	×
33
34	for k, v := range updated.Annotations {	2✔
35	if k == "stackset-controller.zalando.org/stack-generation" {	2✔
36	continue	1✔
37	}
38
39	existingValue, ok := existing.Annotations[k]	1✔
40	if ok && existingValue == v {	1✔
41	continue	×
42	}
43
44	return false	1✔
45	}
46
47	return true	1✔
48	}
49
50	// syncObjectMeta copies metadata elements such as labels or annotations from source to target
51	func syncObjectMeta(target, source metav1.Object) {	1✔
52	target.SetLabels(source.GetLabels())	1✔
53	target.SetAnnotations(source.GetAnnotations())	1✔
54	}	1✔
55
56	func (c StackSetController) ReconcileStackDeployment(ctx context.Context, stack zv1.Stack, existing apps.Deployment, generateUpdated func() apps.Deployment) error {	1✔
57	deployment := generateUpdated()	1✔
58		1✔
59	// Create new deployment	1✔
60	if existing == nil {	2✔
61	_, err := c.client.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{})	1✔
62	if err != nil {	1✔
63	return err	×
64	}	×
65	c.recorder.Eventf(	1✔
66	stack,	1✔
67	apiv1.EventTypeNormal,	1✔
68	"CreatedDeployment",	1✔
69	"Created Deployment %s",	1✔
70	deployment.Name)	1✔
71	return nil	1✔
72	}
73
74	// Check if we need to update the deployment
75	if core.IsResourceUpToDate(stack, existing.ObjectMeta) && pint32Equal(existing.Spec.Replicas, deployment.Spec.Replicas) {	2✔
76	return nil	1✔
77	}	1✔
78
79	updated := existing.DeepCopy()	1✔
80	syncObjectMeta(updated, deployment)	1✔
81	updated.Spec = deployment.Spec	1✔
82	updated.Spec.Selector = existing.Spec.Selector	1✔
83		1✔
84	_, err := c.client.AppsV1().Deployments(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})	1✔
85	if err != nil {	1✔
86	return err	×
87	}	×
88	c.recorder.Eventf(	1✔
89	stack,	1✔
90	apiv1.EventTypeNormal,	1✔
91	"UpdatedDeployment",	1✔
92	"Updated Deployment %s",	1✔
93	deployment.Name)	1✔
94	return nil	1✔
95	}
96
97	func (c StackSetController) ReconcileStackHPA(ctx context.Context, stack zv1.Stack, existing v2.HorizontalPodAutoscaler, generateUpdated func() (v2.HorizontalPodAutoscaler, error)) error {	1✔
98	hpa, err := generateUpdated()	1✔
99	if err != nil {	1✔
100	return err	×
101	}	×
102
103	// HPA removed
104	if hpa == nil {	2✔
105	if existing != nil {	2✔
106	err := c.client.AutoscalingV2().HorizontalPodAutoscalers(existing.Namespace).Delete(ctx, existing.Name, metav1.DeleteOptions{})	1✔
107	if err != nil {	1✔
108	return err	×
109	}	×
110	c.recorder.Eventf(	1✔
111	stack,	1✔
112	apiv1.EventTypeNormal,	1✔
113	"DeletedHPA",	1✔
114	"Deleted HPA %s",	1✔
115	existing.Namespace)	1✔
116	}
117	return nil	1✔
118	}
119
120	// Create new HPA
121	if existing == nil {	2✔
122	_, err := c.client.AutoscalingV2().HorizontalPodAutoscalers(hpa.Namespace).Create(ctx, hpa, metav1.CreateOptions{})	1✔
123	if err != nil {	1✔
124	return err	×
125	}	×
126	c.recorder.Eventf(	1✔
127	stack,	1✔
128	apiv1.EventTypeNormal,	1✔
129	"CreatedHPA",	1✔
130	"Created HPA %s",	1✔
131	hpa.Name)	1✔
132	return nil	1✔
133	}
134
135	// Check if we need to update the HPA
136	if core.IsResourceUpToDate(stack, existing.ObjectMeta) &&	1✔
137	pint32Equal(existing.Spec.MinReplicas, hpa.Spec.MinReplicas) &&	1✔
138	areHPAAnnotationsUpToDate(hpa, existing) {	2✔
139	return nil	1✔
140	}	1✔
141
142	updated := existing.DeepCopy()	1✔
143	syncObjectMeta(updated, hpa)	1✔
144	updated.Spec = hpa.Spec	1✔
145		1✔
146	_, err = c.client.AutoscalingV2().HorizontalPodAutoscalers(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})	1✔
147	if err != nil {	1✔
148	return err	×
149	}	×
150	c.recorder.Eventf(	1✔
151	stack,	1✔
152	apiv1.EventTypeNormal,	1✔
153	"UpdatedHPA",	1✔
154	"Updated HPA %s",	1✔
155	hpa.Name)	1✔
156	return nil	1✔
157	}
158
159	func (c StackSetController) ReconcileStackService(ctx context.Context, stack zv1.Stack, existing apiv1.Service, generateUpdated func() (apiv1.Service, error)) error {	1✔
160	service, err := generateUpdated()	1✔
161	if err != nil {	1✔
162	return err	×
163	}	×
164
165	// Create new service
166	if existing == nil {	2✔
167	_, err := c.client.CoreV1().Services(service.Namespace).Create(ctx, service, metav1.CreateOptions{})	1✔
168	if err != nil {	1✔
169	return err	×
170	}	×
171	c.recorder.Eventf(	1✔
172	stack,	1✔
173	apiv1.EventTypeNormal,	1✔
174	"CreatedService",	1✔
175	"Created Service %s",	1✔
176	service.Name)	1✔
177	return nil	1✔
178	}
179
180	// Check if we need to update the service
181	if core.IsResourceUpToDate(stack, existing.ObjectMeta) {	2✔
182	return nil	1✔
183	}	1✔
184
185	updated := existing.DeepCopy()	1✔
186	syncObjectMeta(updated, service)	1✔
187	updated.Spec = service.Spec	1✔
188	updated.Spec.ClusterIP = existing.Spec.ClusterIP // ClusterIP is immutable	1✔
189		1✔
190	_, err = c.client.CoreV1().Services(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})	1✔
191	if err != nil {	1✔
192	return err	×
193	}	×
194	c.recorder.Eventf(	1✔
195	stack,	1✔
196	apiv1.EventTypeNormal,	1✔
197	"UpdatedService",	1✔
198	"Updated Service %s",	1✔
199	service.Name)	1✔
200	return nil	1✔
201	}
202
203	func (c StackSetController) ReconcileStackIngress(ctx context.Context, stack zv1.Stack, existing networking.Ingress, generateUpdated func() (networking.Ingress, error)) error {	1✔
204	ingress, err := generateUpdated()	1✔
205	if err != nil {	1✔
206	return err	×
207	}	×
208
209	// Ingress removed
210	if ingress == nil {	2✔
211	if existing != nil {	2✔
212	err := c.client.NetworkingV1().Ingresses(existing.Namespace).Delete(ctx, existing.Name, metav1.DeleteOptions{})	1✔
213	if err != nil {	1✔
214	return err	×
215	}	×
216	c.recorder.Eventf(	1✔
217	stack,	1✔
218	apiv1.EventTypeNormal,	1✔
219	"DeletedIngress",	1✔
220	"Deleted Ingress %s",	1✔
221	existing.Namespace)	1✔
222	}
223	return nil	1✔
224	}
225
226	// Create new Ingress
227	if existing == nil {	2✔
228	_, err := c.client.NetworkingV1().Ingresses(ingress.Namespace).Create(ctx, ingress, metav1.CreateOptions{})	1✔
229	if err != nil {	1✔
230	return err	×
231	}	×
232	c.recorder.Eventf(	1✔
233	stack,	1✔
234	apiv1.EventTypeNormal,	1✔
235	"CreatedIngress",	1✔
236	"Created Ingress %s",	1✔
237	ingress.Name)	1✔
238	return nil	1✔
239	}
240
241	// Check if we need to update the Ingress
242	if core.IsResourceUpToDate(stack, existing.ObjectMeta) {	2✔
243	return nil	1✔
244	}	1✔
245
246	updated := existing.DeepCopy()	1✔
247	syncObjectMeta(updated, ingress)	1✔
248	updated.Spec = ingress.Spec	1✔
249		1✔
250	_, err = c.client.NetworkingV1().Ingresses(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})	1✔
251	if err != nil {	1✔
252	return err	×
253	}	×
254	c.recorder.Eventf(	1✔
255	stack,	1✔
256	apiv1.EventTypeNormal,	1✔
257	"UpdatedIngress",	1✔
258	"Updated Ingress %s",	1✔
259	ingress.Name)	1✔
260	return nil	1✔
261	}
262
263	func (c StackSetController) ReconcileStackRouteGroup(ctx context.Context, stack zv1.Stack, existing rgv1.RouteGroup, generateUpdated func() (rgv1.RouteGroup, error)) error {	1✔
264	routegroup, err := generateUpdated()	1✔
265	if err != nil {	1✔
266	return err	×
267	}	×
268
269	// RouteGroup removed
270	if routegroup == nil {	2✔
271	if existing != nil {	2✔
272	err := c.client.RouteGroupV1().RouteGroups(existing.Namespace).Delete(ctx, existing.Name, metav1.DeleteOptions{})	1✔
273	if err != nil {	1✔
274	return err	×
275	}	×
276	c.recorder.Eventf(	1✔
277	stack,	1✔
278	apiv1.EventTypeNormal,	1✔
279	"DeletedRouteGroup",	1✔
280	"Deleted RouteGroup %s",	1✔
281	existing.Namespace)	1✔
282	}
283	return nil	1✔
284	}
285
286	// Create new RouteGroup
287	if existing == nil {	2✔
288	_, err := c.client.RouteGroupV1().RouteGroups(routegroup.Namespace).Create(ctx, routegroup, metav1.CreateOptions{})	1✔
289	if err != nil {	1✔
290	return err	×
291	}	×
292	c.recorder.Eventf(	1✔
293	stack,	1✔
294	apiv1.EventTypeNormal,	1✔
295	"CreatedRouteGroup",	1✔
296	"Created RouteGroup %s",	1✔
297	routegroup.Name)	1✔
298	return nil	1✔
299	}
300
301	// Check if we need to update the RouteGroup
302	if core.IsResourceUpToDate(stack, existing.ObjectMeta) {	2✔
303	return nil	1✔
304	}	1✔
305
306	updated := existing.DeepCopy()	1✔
307	syncObjectMeta(updated, routegroup)	1✔
308	updated.Spec = routegroup.Spec	1✔
309		1✔
310	_, err = c.client.RouteGroupV1().RouteGroups(updated.Namespace).Update(ctx, updated, metav1.UpdateOptions{})	1✔
311	if err != nil {	1✔
312	return err	×
313	}	×
314	c.recorder.Eventf(	1✔
315	stack,	1✔
316	apiv1.EventTypeNormal,	1✔
317	"UpdatedRouteGroup",	1✔
318	"Updated RouteGroup %s",	1✔
319	routegroup.Name)	1✔
320	return nil	1✔
321	}

zalando-incubator / stackset-controller / 5809875303

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous