• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

Azure / aks-app-routing-operator / 16597468726

29 Jul 2025 01:25PM UTC coverage: 81.765% (+0.01%) from 81.754%
16597468726

push

github

OliverMKing
private store

3 of 3 new or added lines in 1 file covered. (100.0%)

44 existing lines in 2 files now uncovered.

3892 of 4760 relevant lines covered (81.76%)

21.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

41.95
/pkg/controller/controller.go
1
// Copyright (c) Microsoft Corporation.
2
// Licensed under the MIT License.
3

4
package controller
5

6
import (
7
        "context"
8
        "fmt"
9
        "net/http"
10
        "time"
11

12
        approutingv1alpha1 "github.com/Azure/aks-app-routing-operator/api/v1alpha1"
13
        placeholderpod "github.com/Azure/aks-app-routing-operator/pkg/controller/keyvault/placeholderpod"
14
        "github.com/Azure/aks-app-routing-operator/pkg/controller/keyvault/spc"
15
        "github.com/Azure/aks-app-routing-operator/pkg/controller/nginxingress"
16
        "github.com/Azure/aks-app-routing-operator/pkg/controller/service"
17
        "github.com/Azure/aks-app-routing-operator/pkg/store"
18
        "github.com/Azure/aks-app-routing-operator/pkg/util"
19
        "github.com/go-logr/logr"
20
        cfgv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
21
        policyv1alpha1 "github.com/openservicemesh/osm/pkg/apis/policy/v1alpha1"
22
        ubzap "go.uber.org/zap"
23
        appsv1 "k8s.io/api/apps/v1"
24
        corev1 "k8s.io/api/core/v1"
25
        netv1 "k8s.io/api/networking/v1"
26
        apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
27
        "k8s.io/apimachinery/pkg/api/errors"
28
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29
        "k8s.io/apimachinery/pkg/runtime"
30
        utilruntime "k8s.io/apimachinery/pkg/util/runtime"
31
        "k8s.io/client-go/kubernetes"
32
        clientgoscheme "k8s.io/client-go/kubernetes/scheme"
33
        "k8s.io/client-go/rest"
34
        "k8s.io/klog/v2"
35
        ctrl "sigs.k8s.io/controller-runtime"
36
        "sigs.k8s.io/controller-runtime/pkg/cache"
37
        "sigs.k8s.io/controller-runtime/pkg/client"
38
        "sigs.k8s.io/controller-runtime/pkg/log/zap"
39
        metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
40
        gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
41
        secv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
42

43
        "github.com/Azure/aks-app-routing-operator/pkg/config"
44
        "github.com/Azure/aks-app-routing-operator/pkg/controller/dns"
45
        "github.com/Azure/aks-app-routing-operator/pkg/controller/ingress"
46
        "github.com/Azure/aks-app-routing-operator/pkg/controller/osm"
47
)
48

49
var scheme = runtime.NewScheme()
50

51
const (
52
        nicIngressClassIndex     = "spec.ingressClassName"
53
        gatewayListenerIndexName = "spec.listeners.tls.options.kubernetes.azure.com/tls-cert-service-account"
54
)
55

56
// controller runtime resync interval is 10 hours. Refreshing the store more frequently guarantees that we'll catch that.
57
// Don't need to immediately reflect rotated cert since controller won't reconcile until the next resync. Rotations
58
// aren't a customer initiated action so there's no confusion.
59
var storeRefreshInterval = 5 * time.Minute
60

61
func init() {
1✔
62
        registerSchemes(scheme)
1✔
63
        ctrl.SetLogger(getLogger())
1✔
64
        // need to set klog logger to same logger to get consistent logging format for all logs.
1✔
65
        // without this things like leader election that use klog will not have the same format.
1✔
66
        // https://github.com/kubernetes/client-go/blob/560efb3b8995da3adcec09865ca78c1ddc917cc9/tools/leaderelection/leaderelection.go#L250
1✔
67
        klog.SetLogger(getLogger())
1✔
68
}
1✔
69

70
func getLogger(opts ...zap.Opts) logr.Logger {
3✔
71
        // use raw opts to add caller info to logs
3✔
72
        rawOpts := zap.RawZapOpts(ubzap.AddCaller())
3✔
73

3✔
74
        // zap is the default recommended logger for controller-runtime when wanting json structured output
3✔
75
        return zap.New(append(opts, rawOpts)...)
3✔
76
}
3✔
77

78
func registerSchemes(s *runtime.Scheme) {
1✔
79
        utilruntime.Must(clientgoscheme.AddToScheme(s))
1✔
80
        utilruntime.Must(secv1.Install(s))
1✔
81
        utilruntime.Must(cfgv1alpha2.AddToScheme(s))
1✔
82
        utilruntime.Must(policyv1alpha1.AddToScheme(s))
1✔
83
        utilruntime.Must(approutingv1alpha1.AddToScheme(s))
1✔
84
        utilruntime.Must(apiextensionsv1.AddToScheme(s))
1✔
85
        utilruntime.Must(gatewayv1.Install(s))
1✔
86
}
1✔
87

UNCOV
88
func NewRestConfig(conf *config.Config) *rest.Config {
×
UNCOV
89
        rc := ctrl.GetConfigOrDie()
×
90
        if conf.ServiceAccountTokenPath != "" {
×
91
                rc.BearerTokenFile = conf.ServiceAccountTokenPath
×
92
        }
×
93

94
        return rc
×
95
}
96

97
func NewManagerForRestConfig(conf *config.Config, rc *rest.Config) (ctrl.Manager, error) {
×
98
        // client creates ListWatch cache for all resources that are
×
99
        // GET or LISTed. this uses too much memory on large clusters.
×
100
        var clientOpts *client.CacheOptions
×
101
        cacheOpts := cache.Options{}
×
102
        if conf.DisableExpensiveCache {
×
103
                clientOpts = &client.CacheOptions{
×
104
                        // we only interact with a small subset of pods
×
105
                        // but there's no good filter to only cache the pods we care about
×
106
                        // so we disable caching them for now.
×
107
                        DisableFor: []client.Object{&corev1.Pod{}},
×
108
                }
×
UNCOV
109

×
110
                cacheOpts.ByObject = map[client.Object]cache.ByObject{
×
111
                        &corev1.Event{}: {
×
112
                                Field: placeholderpod.EventMirrorSelector,
×
113
                        },
×
114
                }
×
115
        }
×
116

117
        m, err := ctrl.NewManager(rc, ctrl.Options{
×
118
                Metrics:                metricsserver.Options{BindAddress: conf.MetricsAddr},
×
119
                HealthProbeBindAddress: conf.ProbeAddr,
×
120
                Scheme:                 scheme,
×
121

×
122
                // we use an active-passive HA model meaning only the leader performs actions
×
123
                LeaderElection:          true,
×
124
                LeaderElectionNamespace: "kube-system",
×
125
                LeaderElectionID:        "aks-app-routing-operator-leader",
×
UNCOV
126

×
127
                Client: client.Options{Cache: clientOpts},
×
128
                Cache:  cacheOpts,
×
129
        })
×
130
        if err != nil {
×
131
                return nil, fmt.Errorf("creating manager: %w", err)
×
UNCOV
132
        }
×
133

134
        setupLog := m.GetLogger().WithName("setup")
×
135
        if err := setupProbes(conf, m, setupLog); err != nil {
×
136
                setupLog.Error(err, "failed to set up probes")
×
137
                return nil, fmt.Errorf("setting up probes: %w", err)
×
138
        }
×
139

140
        // create non-caching clients, non-caching for use before manager has started
141
        cl, err := client.New(rc, client.Options{Scheme: scheme})
×
142
        if err != nil {
×
143
                setupLog.Error(err, "unable to create non-caching client")
×
UNCOV
144
                return nil, fmt.Errorf("creating non-caching client: %w", err)
×
145
        }
×
146

147
        if err := loadCRDs(cl, conf, setupLog); err != nil {
×
148
                setupLog.Error(err, "failed to load CRDs")
×
UNCOV
149
                return nil, fmt.Errorf("loading CRDs: %w", err)
×
150
        }
×
151

152
        if err := setupIndexers(m, setupLog, conf); err != nil {
×
153
                setupLog.Error(err, "unable to setup indexers")
×
UNCOV
154
                return nil, fmt.Errorf("setting up indexers: %w", err)
×
155
        }
×
156

UNCOV
157
        storeLog := m.GetLogger().WithName("store")
×
UNCOV
158
        store := store.New(storeLog, context.Background(), storeRefreshInterval)
×
UNCOV
159
        if conf.EnableDefaultDomain {
×
UNCOV
160
                store.AddFile(conf.DefaultDomainCertPath)
×
UNCOV
161

×
UNCOV
162
                // validate that the default domain cert path exists
×
163
                _, ok := store.GetContent(conf.DefaultDomainCertPath)
×
164
                if !ok {
×
165
                        return nil, fmt.Errorf("default domain cert %s was not loaded to store", conf.DefaultDomainCertPath)
×
UNCOV
166
                }
×
167
        }
168

169
        if err := setupControllers(m, conf, setupLog, cl); err != nil {
×
170
                setupLog.Error(err, "unable to setup controllers")
×
171
                return nil, fmt.Errorf("setting up controllers: %w", err)
×
UNCOV
172
        }
×
173

UNCOV
174
        return m, nil
×
175
}
176

177
func setupIndexers(mgr ctrl.Manager, lgr logr.Logger, conf *config.Config) error {
1✔
178
        lgr.Info("setting up indexers")
1✔
179

1✔
180
        lgr.Info("adding Nginx Ingress Controller IngressClass indexer")
1✔
181
        if err := nginxingress.AddIngressClassNameIndex(mgr.GetFieldIndexer(), nicIngressClassIndex); err != nil {
1✔
UNCOV
182
                lgr.Error(err, "adding Nginx Ingress Controller IngressClass indexer")
×
UNCOV
183
                return fmt.Errorf("adding Nginx Ingress Controller IngressClass indexer: %w", err)
×
184
        }
×
185

186
        if conf.EnableGateway {
1✔
UNCOV
187
                if err := util.AddGatewayServiceAccountIndex(mgr.GetFieldIndexer(), gatewayListenerIndexName); err != nil {
×
UNCOV
188
                        lgr.Error(err, "adding Gateway Service Account indexer")
×
189
                        return fmt.Errorf("adding Gateway Service Account indexer: %w", err)
×
190
                }
×
191
        }
192

193
        lgr.Info("finished setting up indexers")
1✔
194
        return nil
1✔
195
}
196

197
func setupControllers(mgr ctrl.Manager, conf *config.Config, lgr logr.Logger, cl client.Client) error {
1✔
198
        lgr.Info("setting up controllers")
1✔
199

1✔
200
        lgr.Info("determining default IngressClass controller class")
1✔
201
        defaultCc, err := nginxingress.GetDefaultIngressClassControllerClass(cl)
1✔
202
        if err != nil {
1✔
UNCOV
203
                return fmt.Errorf("determining default IngressClass controller class: %w", err)
×
204
        }
×
205

206
        lgr.Info("setting up ExternalDNS controller")
1✔
207
        if err := dns.NewExternalDns(mgr, conf); err != nil {
1✔
UNCOV
208
                return fmt.Errorf("setting up external dns controller: %w", err)
×
209
        }
×
210

211
        lgr.Info("setting up Nginx Ingress Controller reconciler")
1✔
212
        if err := nginxingress.NewReconciler(conf, mgr, defaultCc); err != nil {
1✔
UNCOV
213
                return fmt.Errorf("setting up nginx ingress controller reconciler: %w", err)
×
214
        }
×
215

216
        lgr.Info("setting up ingress cert config reconciler")
1✔
217
        if err = osm.NewIngressCertConfigReconciler(mgr, conf); err != nil {
1✔
218
                return fmt.Errorf("setting up ingress cert config reconciler: %w", err)
×
219
        }
×
220

221
        defaultNic := nginxingress.GetDefaultNginxIngressController()
1✔
222
        if err := service.NewNginxIngressReconciler(mgr, nginxingress.ToNginxIngressConfig(&defaultNic, defaultCc)); err != nil {
1✔
223
                return fmt.Errorf("setting up nginx ingress reconciler: %w", err)
×
UNCOV
224
        }
×
225

226
        lgr.Info("setting up default Nginx Ingress Controller reconciler")
1✔
227
        if err := nginxingress.NewDefaultReconciler(mgr, conf); err != nil {
1✔
UNCOV
228
                return fmt.Errorf("setting up nginx ingress default controller reconciler: %w", err)
×
UNCOV
229
        }
×
230

231
        lgr.Info("setting up ingress concurrency watchdog")
1✔
232
        if err := ingress.NewConcurrencyWatchdog(mgr, conf, ingress.GetListNginxWatchdogTargets(mgr.GetClient(), defaultCc)); err != nil {
1✔
UNCOV
233
                return fmt.Errorf("setting up ingress concurrency watchdog: %w", err)
×
234
        }
×
235

236
        ingressManager := util.NewIngressManagerFromFn(func(ing *netv1.Ingress) (bool, error) {
1✔
UNCOV
237
                return nginxingress.IsIngressManaged(context.Background(), mgr.GetClient(), ing, nicIngressClassIndex)
×
238
        })
×
239
        lgr.Info("setting up keyvault secret provider class reconciler")
1✔
240
        if err := spc.NewIngressSecretProviderClassReconciler(mgr, conf, ingressManager); err != nil {
1✔
UNCOV
241
                return fmt.Errorf("setting up ingress secret provider class reconciler: %w", err)
×
242
        }
×
243
        lgr.Info("setting up nginx keyvault secret provider class reconciler")
1✔
244
        if err := spc.NewNginxSecretProviderClassReconciler(mgr, conf); err != nil {
1✔
UNCOV
245
                return fmt.Errorf("setting up nginx secret provider class reconciler: %w", err)
×
246
        }
×
247
        lgr.Info("setting up keyvault placeholder pod controller")
1✔
248
        if err := placeholderpod.NewPlaceholderPodController(mgr, conf, ingressManager); err != nil {
1✔
249
                return fmt.Errorf("setting up placeholder pod controller: %w", err)
×
UNCOV
250
        }
×
251
        lgr.Info("setting up keyvault event mirror")
1✔
252
        if err = placeholderpod.NewEventMirror(mgr, conf); err != nil {
1✔
UNCOV
253
                return fmt.Errorf("setting up event mirror: %w", err)
×
UNCOV
254
        }
×
255

256
        ingressSourceSpecer := osm.NewIngressControllerSourceSpecerFromFn(func(ing *netv1.Ingress) (policyv1alpha1.IngressSourceSpec, bool, error) {
1✔
UNCOV
257
                return nginxingress.IngressSource(context.Background(), mgr.GetClient(), conf, defaultCc, ing, nicIngressClassIndex)
×
UNCOV
258
        })
×
259
        lgr.Info("setting up ingress backend reconciler")
1✔
260
        if err := osm.NewIngressBackendReconciler(mgr, conf, ingressSourceSpecer); err != nil {
1✔
UNCOV
261
                return fmt.Errorf("setting up ingress backend reconciler: %w", err)
×
262
        }
×
263

264
        if conf.EnableGateway {
1✔
UNCOV
265
                lgr.Info("setting up gateway reconcilers")
×
266
                if err := spc.NewGatewaySecretClassProviderReconciler(mgr, conf, gatewayListenerIndexName); err != nil {
×
267
                        return fmt.Errorf("setting up Gateway SPC reconciler: %w", err)
×
UNCOV
268
                }
×
269
        }
270

271
        lgr.Info("finished setting up controllers")
1✔
272
        return nil
1✔
273
}
274

275
func setupProbes(conf *config.Config, mgr ctrl.Manager, log logr.Logger) error {
1✔
276
        log.Info("adding probes to manager")
1✔
277

1✔
278
        check := func(req *http.Request) error { return nil }
1✔
279

280
        if err := mgr.AddReadyzCheck("readyz", check); err != nil {
1✔
UNCOV
281
                return fmt.Errorf("adding readyz check: %w", err)
×
UNCOV
282
        }
×
283

284
        if err := mgr.AddHealthzCheck("healthz", check); err != nil {
1✔
285
                return fmt.Errorf("adding healthz check: %w", err)
×
286
        }
×
287

288
        log.Info("added probes to manager")
1✔
289
        return nil
1✔
290
}
291

292
func getSelfDeploy(kcs kubernetes.Interface, conf *config.Config, log logr.Logger) (*appsv1.Deployment, error) {
2✔
293
        // this doesn't actually work today. operator ns is not the same as resource ns which means we can't set this operator
2✔
294
        // as the owner of any child resources. https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/#owner-references-in-object-specifications
2✔
295
        // dynamic provisioning through a crd will fix this and fix our garbage collection.
2✔
296

2✔
297
        deploy, err := kcs.AppsV1().Deployments(conf.NS).Get(context.Background(), conf.OperatorDeployment, metav1.GetOptions{})
2✔
298
        if errors.IsNotFound(err) {
3✔
299
                // It's okay if we don't find the deployment - just skip setting ownership references latter
1✔
300
                log.Info("self deploy not found")
1✔
301
                return nil, nil
1✔
302
        }
1✔
303
        if err != nil {
1✔
UNCOV
304
                return nil, err
×
UNCOV
305
        }
×
306
        return deploy, nil
1✔
307
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc