16782834705

Committed 06 Aug 2025 04:33PM UTC coverage: 81.413% (+0.02%) from 81.394%

Build # 16782834705

Build Type

push

github

Committed by

OliverMKing

Commit Message

fix race condition

Run Details

4078 of 5009 relevant lines covered (81.41%)

22.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.67

/pkg/controller/defaultdomaincert/default_domain_cert_controller.go

package defaultdomaincert

import (
        "context"
        "errors"
        "fmt"
        "time"

        approutingv1alpha1 "github.com/Azure/aks-app-routing-operator/api/v1alpha1"
        "github.com/Azure/aks-app-routing-operator/pkg/config"
        "github.com/Azure/aks-app-routing-operator/pkg/controller/controllername"
        "github.com/Azure/aks-app-routing-operator/pkg/controller/metrics"
        "github.com/Azure/aks-app-routing-operator/pkg/manifests"
        "github.com/Azure/aks-app-routing-operator/pkg/store"
        "github.com/Azure/aks-app-routing-operator/pkg/tls"
        "github.com/Azure/aks-app-routing-operator/pkg/util"
        corev1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/client-go/tools/record"
        "k8s.io/client-go/util/workqueue"
        ctrl "sigs.k8s.io/controller-runtime"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/log"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
        "sigs.k8s.io/controller-runtime/pkg/source"
)

var name = controllername.New("default", "domain", "cert", "reconciler")

type defaultDomainCertControllerReconciler struct {
        client client.Client
        events record.EventRecorder
        conf   *config.Config
        store  store.Store
}

func NewReconciler(conf *config.Config, mgr ctrl.Manager, store store.Store) error {
        metrics.InitControllerMetrics(name)

        if err := store.AddFile(conf.DefaultDomainCertPath); err != nil {
                return fmt.Errorf("adding default domain cert %s to store: %w", conf.DefaultDomainCertPath, err)
        }

        if err := store.AddFile(conf.DefaultDomainKeyPath); err != nil {
                return fmt.Errorf("adding default domain key %s to store: %w", conf.DefaultDomainKeyPath, err)
        }

        reconciler := &defaultDomainCertControllerReconciler{
                client: mgr.GetClient(),
                events: mgr.GetEventRecorderFor("aks-app-routing-operator"),
                conf:   conf,
                store:  store,
        }

        if _, _, err := reconciler.getAndVerifyCertAndKey(); err != nil {
                return fmt.Errorf("verifying cert and key: %w", err)
        }

        if err := name.AddToController(
                ctrl.NewControllerManagedBy(mgr).
                        For(&approutingv1alpha1.DefaultDomainCertificate{}).
                        Owns(&corev1.Secret{}).
                        WatchesRawSource(source.Func(reconciler.sendRotationEvents)),
                mgr.GetLogger(),
        ).Complete(reconciler); err != nil {
                return fmt.Errorf("building the controller: %w", err)
        }

        return nil
}

func (d *defaultDomainCertControllerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
        start := time.Now()
        lgr := log.FromContext(ctx, "name", req.Name, "namespace", req.Namespace)
        ctx = log.IntoContext(ctx, lgr)

        lgr.Info("reconciling DefaultDomainCertificate")
        defer func() {
                lgr.Info("reconcile finished", "latency", time.Since(start))
        }()

        defaultDomainCertificate := &approutingv1alpha1.DefaultDomainCertificate{}
        if err := d.client.Get(ctx, req.NamespacedName, defaultDomainCertificate); err != nil {
                if apierrors.IsNotFound(err) { // object was deleted
                        lgr.Info("DefaultDomainCertificate not found")
                        return ctrl.Result{}, nil
                }

                lgr.Error(err, "unable to get DefaultDomainCertificate")
                return ctrl.Result{}, err
        }

        if defaultDomainCertificate.Spec.Target.Secret == nil {
                err := errors.New("DefaultDomainCertificate has no target secret specified")
                lgr.Error(err, "DefaultDomainCertificate has no target specified, this should be blocked by CRD CEL validation")
                return ctrl.Result{}, err
        }

        lgr = lgr.WithValues("secretTarget", *defaultDomainCertificate.Spec.Target.Secret)
        ctx = log.IntoContext(ctx, lgr)

        lgr.Info("upserting Secret for DefaultDomainCertificate")
        secret, err := d.generateSecret(defaultDomainCertificate)
        if err != nil {
                err := fmt.Errorf("generating Secret for DefaultDomainCertificate: %w", err)
                lgr.Error(err, "failed to generate Secret for DefaultDomainCertificate")
                return ctrl.Result{}, err
        }

        if err := util.Upsert(ctx, d.client, secret); err != nil {
                d.events.Eventf(defaultDomainCertificate, corev1.EventTypeWarning, "ApplyingCertificateSecretFailed", "Failed to apply Secret for DefaultDomainCertificate: %s", err.Error())
                lgr.Error(err, "failed to upsert Secret for DefaultDomainCertificate")
                return ctrl.Result{}, err
        }

        // Update the status of the DefaultDomainCertificate
        defaultDomainCertificate.SetCondition(metav1.Condition{
                Type:    approutingv1alpha1.DefaultDomainCertificateConditionTypeAvailable,
                Status:  metav1.ConditionTrue,
                Reason:  "CertificateSecretApplied",
                Message: fmt.Sprintf("Secret %s/%s successfully applied for DefaultDomainCertificate", secret.Namespace, secret.Name),
        })
        if err := d.client.Status().Update(ctx, defaultDomainCertificate); err != nil {
                lgr.Error(err, "failed to update status for DefaultDomainCertificate")
                return ctrl.Result{}, err
        }

        return ctrl.Result{}, nil
}

func (d *defaultDomainCertControllerReconciler) generateSecret(defaultDomainCertificate *approutingv1alpha1.DefaultDomainCertificate) (*corev1.Secret, error) {
        cert, key, err := d.getAndVerifyCertAndKey()
        if err != nil {
                return nil, fmt.Errorf("getting and verifying cert and key: %w", err)
        }

        secret := &corev1.Secret{
                ObjectMeta: metav1.ObjectMeta{
                        Name:      *defaultDomainCertificate.Spec.Target.Secret,
                        Namespace: defaultDomainCertificate.Namespace,
                        Labels:    manifests.GetTopLevelLabels(),
                },
                TypeMeta: metav1.TypeMeta{
                        Kind:       "Secret",
                        APIVersion: corev1.SchemeGroupVersion.String(),
                },
                Type: corev1.SecretTypeTLS,
                Data: map[string][]byte{
                        "tls.crt": cert,
                        "tls.key": key,
                },
        }

        owner := manifests.GetOwnerRefs(defaultDomainCertificate, true)
        secret.SetOwnerReferences(owner)

        return secret, nil
}

func (d *defaultDomainCertControllerReconciler) getAndVerifyCertAndKey() ([]byte, []byte, error) {
        key, ok := d.store.GetContent(d.conf.DefaultDomainKeyPath)
        if key == nil || !ok {
                return nil, nil, fmt.Errorf("failed to get default domain key from store")
        }

        cert, ok := d.store.GetContent(d.conf.DefaultDomainCertPath)
        if cert == nil || !ok {
                return nil, nil, fmt.Errorf("failed to get default domain cert from store")
        }

        if _, err := tls.ParseTLSCertificate(cert, key); err != nil {
                return nil, nil, fmt.Errorf("validating cert and key: %w", err)
        }

        return cert, key, nil
}

// sendRotationEvents listens for store rotation events and triggers reconciles
// for all DefaultDomainCertificate resources when certificate files are rotated
func (d *defaultDomainCertControllerReconciler) sendRotationEvents(ctx context.Context, queue workqueue.TypedRateLimitingInterface[ctrl.Request]) error {
        go func() {
                logger := log.FromContext(ctx)
                logger = logger.WithName("rotation-watcher")
                logger.Info("starting rotation event watcher")
                defer func() {
                        logger.Info("rotation event watcher stopped")
                }()

                for {
                        select {
                        case <-ctx.Done():
                                return
                        case event := <-d.store.RotationEvents():
                                if event.Path != d.conf.DefaultDomainCertPath && event.Path != d.conf.DefaultDomainKeyPath {
                                        logger.Info("non-certificate file rotated " + event.Path)
                                        continue
                                }

                                ddcList := &approutingv1alpha1.DefaultDomainCertificateList{}
                                if err := d.client.List(ctx, ddcList); err != nil {
                                        // an error here is not ideal but if we are failing to list or failing to requeue controller runtime
                                        // resync period of 10 hours will catch it
                                        logger.Error(err, "failed to list DefaultDomainCertificate resources")
                                        continue
                                }

                                for _, ddc := range ddcList.Items {
                                        queue.Add(reconcile.Request{
                                                NamespacedName: types.NamespacedName{
                                                        Name:      ddc.Name,
                                                        Namespace: ddc.Namespace,
                                                },
                                        })
                                }
                        }
                }
        }()

        return nil
}

1	package defaultdomaincert
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"time"
8
9	approutingv1alpha1 "github.com/Azure/aks-app-routing-operator/api/v1alpha1"
10	"github.com/Azure/aks-app-routing-operator/pkg/config"
11	"github.com/Azure/aks-app-routing-operator/pkg/controller/controllername"
12	"github.com/Azure/aks-app-routing-operator/pkg/controller/metrics"
13	"github.com/Azure/aks-app-routing-operator/pkg/manifests"
14	"github.com/Azure/aks-app-routing-operator/pkg/store"
15	"github.com/Azure/aks-app-routing-operator/pkg/tls"
16	"github.com/Azure/aks-app-routing-operator/pkg/util"
17	corev1 "k8s.io/api/core/v1"
18	apierrors "k8s.io/apimachinery/pkg/api/errors"
19	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
20	"k8s.io/apimachinery/pkg/types"
21	"k8s.io/client-go/tools/record"
22	"k8s.io/client-go/util/workqueue"
23	ctrl "sigs.k8s.io/controller-runtime"
24	"sigs.k8s.io/controller-runtime/pkg/client"
25	"sigs.k8s.io/controller-runtime/pkg/log"
26	"sigs.k8s.io/controller-runtime/pkg/reconcile"
27	"sigs.k8s.io/controller-runtime/pkg/source"
28	)
29
30	var name = controllername.New("default", "domain", "cert", "reconciler")
31
32	type defaultDomainCertControllerReconciler struct {
33	client client.Client
34	events record.EventRecorder
35	conf *config.Config
36	store store.Store
37	}
38
39	func NewReconciler(conf *config.Config, mgr ctrl.Manager, store store.Store) error {	2✔
40	metrics.InitControllerMetrics(name)	2✔
41		2✔
42	if err := store.AddFile(conf.DefaultDomainCertPath); err != nil {	3✔
43	return fmt.Errorf("adding default domain cert %s to store: %w", conf.DefaultDomainCertPath, err)	1✔
44	}	1✔
45
46	if err := store.AddFile(conf.DefaultDomainKeyPath); err != nil {	2✔
47	return fmt.Errorf("adding default domain key %s to store: %w", conf.DefaultDomainKeyPath, err)	1✔
48	}	1✔
49
50	reconciler := &defaultDomainCertControllerReconciler{	×
51	client: mgr.GetClient(),	×
52	events: mgr.GetEventRecorderFor("aks-app-routing-operator"),	×
53	conf: conf,	×
54	store: store,	×
55	}	×
56		×
57	if _, _, err := reconciler.getAndVerifyCertAndKey(); err != nil {	×
58	return fmt.Errorf("verifying cert and key: %w", err)	×
59	}	×
60
61	if err := name.AddToController(	×
62	ctrl.NewControllerManagedBy(mgr).	×
63	For(&approutingv1alpha1.DefaultDomainCertificate{}).	×
64	Owns(&corev1.Secret{}).	×
65	WatchesRawSource(source.Func(reconciler.sendRotationEvents)),	×
66	mgr.GetLogger(),	×
67	).Complete(reconciler); err != nil {	×
68	return fmt.Errorf("building the controller: %w", err)	×
69	}	×
70
71	return nil	×
72	}
73
74	func (d *defaultDomainCertControllerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {	11✔
75	start := time.Now()	11✔
76	lgr := log.FromContext(ctx, "name", req.Name, "namespace", req.Namespace)	11✔
77	ctx = log.IntoContext(ctx, lgr)	11✔
78		11✔
79	lgr.Info("reconciling DefaultDomainCertificate")	11✔
80	defer func() {	22✔
81	lgr.Info("reconcile finished", "latency", time.Since(start))	11✔
82	}()	11✔
83
84	defaultDomainCertificate := &approutingv1alpha1.DefaultDomainCertificate{}	11✔
85	if err := d.client.Get(ctx, req.NamespacedName, defaultDomainCertificate); err != nil {	13✔
86	if apierrors.IsNotFound(err) { // object was deleted	3✔
87	lgr.Info("DefaultDomainCertificate not found")	1✔
88	return ctrl.Result{}, nil	1✔
89	}	1✔
90
91	lgr.Error(err, "unable to get DefaultDomainCertificate")	1✔
92	return ctrl.Result{}, err	1✔
93	}
94
95	if defaultDomainCertificate.Spec.Target.Secret == nil {	10✔
96	err := errors.New("DefaultDomainCertificate has no target secret specified")	1✔
97	lgr.Error(err, "DefaultDomainCertificate has no target specified, this should be blocked by CRD CEL validation")	1✔
98	return ctrl.Result{}, err	1✔
99	}	1✔
100
101	lgr = lgr.WithValues("secretTarget", *defaultDomainCertificate.Spec.Target.Secret)	8✔
102	ctx = log.IntoContext(ctx, lgr)	8✔
103		8✔
104	lgr.Info("upserting Secret for DefaultDomainCertificate")	8✔
105	secret, err := d.generateSecret(defaultDomainCertificate)	8✔
106	if err != nil {	9✔
107	err := fmt.Errorf("generating Secret for DefaultDomainCertificate: %w", err)	1✔
108	lgr.Error(err, "failed to generate Secret for DefaultDomainCertificate")	1✔
109	return ctrl.Result{}, err	1✔
110	}	1✔
111
112	if err := util.Upsert(ctx, d.client, secret); err != nil {	9✔
113	d.events.Eventf(defaultDomainCertificate, corev1.EventTypeWarning, "ApplyingCertificateSecretFailed", "Failed to apply Secret for DefaultDomainCertificate: %s", err.Error())	2✔
114	lgr.Error(err, "failed to upsert Secret for DefaultDomainCertificate")	2✔
115	return ctrl.Result{}, err	2✔
116	}	2✔
117
118	// Update the status of the DefaultDomainCertificate
119	defaultDomainCertificate.SetCondition(metav1.Condition{	5✔
120	Type: approutingv1alpha1.DefaultDomainCertificateConditionTypeAvailable,	5✔
121	Status: metav1.ConditionTrue,	5✔
122	Reason: "CertificateSecretApplied",	5✔
123	Message: fmt.Sprintf("Secret %s/%s successfully applied for DefaultDomainCertificate", secret.Namespace, secret.Name),	5✔
124	})	5✔
125	if err := d.client.Status().Update(ctx, defaultDomainCertificate); err != nil {	6✔
126	lgr.Error(err, "failed to update status for DefaultDomainCertificate")	1✔
127	return ctrl.Result{}, err	1✔
128	}	1✔
129
130	return ctrl.Result{}, nil	4✔
131	}
132
133	func (d defaultDomainCertControllerReconciler) generateSecret(defaultDomainCertificate approutingv1alpha1.DefaultDomainCertificate) (*corev1.Secret, error) {	17✔
134	cert, key, err := d.getAndVerifyCertAndKey()	17✔
135	if err != nil {	24✔
136	return nil, fmt.Errorf("getting and verifying cert and key: %w", err)	7✔
137	}	7✔
138
139	secret := &corev1.Secret{	10✔
140	ObjectMeta: metav1.ObjectMeta{	10✔
141	Name: *defaultDomainCertificate.Spec.Target.Secret,	10✔
142	Namespace: defaultDomainCertificate.Namespace,	10✔
143	Labels: manifests.GetTopLevelLabels(),	10✔
144	},	10✔
145	TypeMeta: metav1.TypeMeta{	10✔
146	Kind: "Secret",	10✔
147	APIVersion: corev1.SchemeGroupVersion.String(),	10✔
148	},	10✔
149	Type: corev1.SecretTypeTLS,	10✔
150	Data: map[string][]byte{	10✔
151	"tls.crt": cert,	10✔
152	"tls.key": key,	10✔
153	},	10✔
154	}	10✔
155		10✔
156	owner := manifests.GetOwnerRefs(defaultDomainCertificate, true)	10✔
157	secret.SetOwnerReferences(owner)	10✔
158		10✔
159	return secret, nil	10✔
160	}
161
162	func (d *defaultDomainCertControllerReconciler) getAndVerifyCertAndKey() ([]byte, []byte, error) {	21✔
163	key, ok := d.store.GetContent(d.conf.DefaultDomainKeyPath)	21✔
164	if key == nil \|\| !ok {	24✔
165	return nil, nil, fmt.Errorf("failed to get default domain key from store")	3✔
166	}	3✔
167
168	cert, ok := d.store.GetContent(d.conf.DefaultDomainCertPath)	18✔
169	if cert == nil \|\| !ok {	22✔
170	return nil, nil, fmt.Errorf("failed to get default domain cert from store")	4✔
171	}	4✔
172
173	if _, err := tls.ParseTLSCertificate(cert, key); err != nil {	17✔
174	return nil, nil, fmt.Errorf("validating cert and key: %w", err)	3✔
175	}	3✔
176
177	return cert, key, nil	11✔
178	}
179
180	// sendRotationEvents listens for store rotation events and triggers reconciles
181	// for all DefaultDomainCertificate resources when certificate files are rotated
182	func (d *defaultDomainCertControllerReconciler) sendRotationEvents(ctx context.Context, queue workqueue.TypedRateLimitingInterface[ctrl.Request]) error {	6✔
183	go func() {	12✔
184	logger := log.FromContext(ctx)	6✔
185	logger = logger.WithName("rotation-watcher")	6✔
186	logger.Info("starting rotation event watcher")	6✔
187	defer func() {	12✔
188	logger.Info("rotation event watcher stopped")	6✔
189	}()	6✔
190
191	for {	18✔
192	select {	12✔
193	case <-ctx.Done():	6✔
194	return	6✔
195	case event := <-d.store.RotationEvents():	6✔
196	if event.Path != d.conf.DefaultDomainCertPath && event.Path != d.conf.DefaultDomainKeyPath {	8✔
197	logger.Info("non-certificate file rotated " + event.Path)	2✔
198	continue	2✔
199	}
200
201	ddcList := &approutingv1alpha1.DefaultDomainCertificateList{}	4✔
202	if err := d.client.List(ctx, ddcList); err != nil {	4✔
203	// an error here is not ideal but if we are failing to list or failing to requeue controller runtime	×
204	// resync period of 10 hours will catch it	×
205	logger.Error(err, "failed to list DefaultDomainCertificate resources")	×
206	continue	×
207	}
208
209	for _, ddc := range ddcList.Items {	7✔
210	queue.Add(reconcile.Request{	3✔
211	NamespacedName: types.NamespacedName{	3✔
212	Name: ddc.Name,	3✔
213	Namespace: ddc.Namespace,	3✔
214	},	3✔
215	})	3✔
216	}	3✔
217	}
218	}
219	}()
220
221	return nil	6✔
222	}

Azure / aks-app-routing-operator / 16782834705

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous