20477050272

Committed 24 Dec 2025 03:04AM UTC coverage: 76.673%. First build

Build # 20477050272

Build Type

Pull #142

github

Committed by

web-flow

Commit Message

Merge 9a6f2d87a into 646711c0e

Pull Request Pull Request #142: feat: add webhook support for validation policies exist

Run Details

79 of 98 new or added lines in 6 files covered. (80.61%)

5936 of 7742 relevant lines covered (76.67%)

1.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.1

/operator/internal/controller/webhook_controller.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package controller

import (
        "bytes"
        "context"
        "fmt"
        "net/http"
        "reflect"
        "time"

        "sigs.k8s.io/controller-runtime/pkg/builder"
        "sigs.k8s.io/controller-runtime/pkg/predicate"

        "github.com/NVIDIA/skyhook/operator/api/v1alpha1"
        admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
        corev1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        ctrl "sigs.k8s.io/controller-runtime"
        runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/log"
        "sigs.k8s.io/controller-runtime/pkg/reconcile"
)

// This project used to use cert-manager to generate the webhook certificates.
// This removes the dependency on cert-manager and simplifies the deployment.
// This also removes the need to have a specific issuer, and just uses a self-signed cert.
type WebhookControllerOptions struct { // prefix these with WEBHOOK_
        SecretName  string `env:"WEBHOOK_SECRET_NAME, default=webhook-cert"`
        ServiceName string `env:"WEBHOOK_SERVICE_NAME, default=skyhook-operator-webhook-service"`
}

type WebhookController struct {
        client.Client
        cache     runtimecache.Cache
        namespace string
        certDir   string
        opts      WebhookControllerOptions
}

func NewWebhookController(client client.Client, cache runtimecache.Cache, namespace, certDir string, opts WebhookControllerOptions) (*WebhookController, error) {
        if err := ensureDummyCert(certDir); err != nil {
                return nil, err
        }

        return &WebhookController{
                Client:    client,
                cache:     cache,
                namespace: namespace,
                certDir:   certDir,
                opts:      opts,
        }, nil
}

// Start implements the Runnable interface to ensure certificates are set up before the webhook server starts
func (r *WebhookController) Start(ctx context.Context) error {
        logger := log.FromContext(ctx)
        logger.Info("Setting up webhook certificates")

        // wait for the cache to sync
        if cache := r.cache.WaitForCacheSync(ctx); !cache {
                return fmt.Errorf("failed to wait for cache to sync")
        }
        // starts the reconcile process off
        _, err := r.GetOrCreateWebhookCertSecret(ctx, r.opts.SecretName, r.namespace)
        if err != nil {
                if errors.IsAlreadyExists(err) {
                        return nil // ignore this special case, it just needs to exist
                }
                return err
        }

        logger.Info("Webhook certificates setup complete")
        return nil
}

// NeedLeaderElection implements the Runnable interface, runs only on leader
func (r *WebhookController) NeedLeaderElection() bool {
        return true
}

func (r *WebhookController) SetupWithManager(mgr ctrl.Manager) error {
        return ctrl.NewControllerManagedBy(mgr).
                For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
                        return obj.GetNamespace() == r.namespace && obj.GetName() == r.opts.SecretName
                }))).
                Complete(r)
}

// permissions
//+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations;mutatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete

// Reconcile is the main function that reconciles the webhook controller
func (r *WebhookController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
        logger := log.FromContext(ctx)
        logger.Info("Reconciling webhook controller")

        // if its deleted, skip reconciliation, this is for cleanup
        obj := &corev1.Secret{}
        if err := r.Get(ctx, req.NamespacedName, obj); err != nil {
                // handle not found, etc.
                return ctrl.Result{}, client.IgnoreNotFound(err)
        }

        // If the object is being deleted, skip reconciliation
        if !obj.ObjectMeta.DeletionTimestamp.IsZero() {
                // Optionally: handle finalizers here if you want
                return ctrl.Result{}, nil
        }

        // 1. Get or create/update the Secret with certs
        // 2. Get or create/update the webhook configurations

        // Example: check if secret exists
        secret, err := r.GetOrCreateWebhookCertSecret(ctx, r.opts.SecretName, r.namespace)
        if err != nil {
                return reconcile.Result{}, err
        }

        _, err = r.CheckOrUpdateWebhookCertSecret(ctx, secret)
        if err != nil {
                return reconcile.Result{}, err
        }

        _, err = r.CheckOrUpdateWebhookConfigurations(ctx, secret)
        if err != nil {
                return reconcile.Result{}, err
        }

        logger.Info("Reconciled webhook controller")
        return reconcile.Result{RequeueAfter: 24 * time.Hour}, nil // requeue for periodic rotation/check
}

// GetOrCreateWebhookCertSecret returns a new secret with the given name and the given CA and cert.
func (r *WebhookController) GetOrCreateWebhookCertSecret(ctx context.Context, secretName, namespace string) (*corev1.Secret, error) {

        // get the secret
        secret := &corev1.Secret{}
        err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, secret)
        if err != nil {
                if errors.IsNotFound(err) {
                        // not found, create it
                        webhookCert, err := generateCert(r.opts.ServiceName, r.namespace, 365*24*time.Hour) // TODO: this should be configured
                        if err != nil {
                                return nil, err
                        }

                        // Write cert and key to disk if CertDir is set
                        if r.certDir != "" {
                                _ = writeCertAndKey([]byte(webhookCert.TLSCert), []byte(webhookCert.TLSKey), r.certDir)
                        }

                        secret = webhookCert.ToSecret(secretName, namespace, r.opts.ServiceName)

                        if err := r.Create(ctx, secret); err != nil {
                                return nil, err
                        }

                        return secret, nil
                }
                return nil, err
        }

        // found, return it
        return secret, nil
}

func (r *WebhookController) CheckOrUpdateWebhookCertSecret(ctx context.Context, secret *corev1.Secret) (bool, error) {
        equal, err := compareCertOnDiskToSecret(r.certDir, secret)
        if err != nil {
                return false, err
        }

        // check if the secret is going to expire in the next 168 hours or if the cert on disk is different from the secret
        if !equal || secret.Annotations[fmt.Sprintf("%s/expiration", v1alpha1.METADATA_PREFIX)] < time.Now().Add(168*time.Hour).Format(time.RFC3339) {
                // expired, generate a new cert
                webhookCert, err := generateCert(r.opts.ServiceName, r.namespace, 365*24*time.Hour) // TODO: this should be configured
                if err != nil {
                        return false, err
                }

                // Write cert and key to disk if CertDir is set
                if r.certDir != "" {
                        _ = writeCertAndKey([]byte(webhookCert.TLSCert), []byte(webhookCert.TLSKey), r.certDir)
                }

                secret.Data["ca.crt"] = webhookCert.CABytes
                secret.Data["tls.crt"] = []byte(webhookCert.TLSCert)
                secret.Data["tls.key"] = []byte(webhookCert.TLSKey)
                secret.Annotations[fmt.Sprintf("%s/expiration", v1alpha1.METADATA_PREFIX)] = webhookCert.Expiration.Format(time.RFC3339)

                return true, r.Update(ctx, secret)
        }

        return false, nil
}

func (r *WebhookController) CheckOrUpdateWebhookConfigurations(ctx context.Context, secret *corev1.Secret) (bool, error) {
        // Update only CABundle fields of existing webhook configurations created by Helm
        caBundle := secret.Data["ca.crt"]
        changed := false

        // ValidatingWebhookConfiguration
        validatingName := webhookValidatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret).GetName()
        existingValidating := &admissionregistrationv1.ValidatingWebhookConfiguration{}
        if err := r.Get(ctx, types.NamespacedName{Name: validatingName}, existingValidating); err != nil {
                if errors.IsNotFound(err) {
                        return false, fmt.Errorf("ValidatingWebhookConfiguration %q not found; creation is handled by the Helm chart. Ensure the chart is installed and webhooks are enabled: %w", validatingName, err)
                }
                return false, fmt.Errorf("failed to get ValidatingWebhookConfiguration %q: %w", validatingName, err)
        }

        needUpdate := false
        expectedRules := webhookRule()
        for i := range existingValidating.Webhooks {
                if validatingWebhookNeedsUpdate(&existingValidating.Webhooks[i], caBundle, expectedRules) {
                        needUpdate = true
                }
        }
        if needUpdate {
                if err := r.Update(ctx, existingValidating); err != nil {
                        return false, err
                } else {
                        changed = true
                }
        }

        // MutatingWebhookConfiguration
        mutatingName := webhookMutatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret).GetName()
        existingMutating := &admissionregistrationv1.MutatingWebhookConfiguration{}
        if err := r.Get(ctx, types.NamespacedName{Name: mutatingName}, existingMutating); err != nil {
                if errors.IsNotFound(err) {
                        return changed, fmt.Errorf("MutatingWebhookConfiguration %q not found; creation is handled by the Helm chart. Ensure the chart is installed and webhooks are enabled: %w", mutatingName, err)
                }
                return false, fmt.Errorf("failed to get MutatingWebhookConfiguration %q: %w", mutatingName, err)
        }

        needUpdate = false
        for i := range existingMutating.Webhooks {
                if mutatingWebhookNeedsUpdate(&existingMutating.Webhooks[i], caBundle, expectedRules) {
                        needUpdate = true
                }
        }
        if needUpdate {
                if err := r.Update(ctx, existingMutating); err != nil {
                        return false, err
                } else {
                        changed = true
                }
        }

        return changed, nil
}

// webhookValidatingWebhookConfiguration returns a new validating webhook configuration.
func webhookValidatingWebhookConfiguration(namespace, serviceName string, secret *corev1.Secret) *admissionregistrationv1.ValidatingWebhookConfiguration {
        conf := admissionregistrationv1.ValidatingWebhookConfiguration{
                ObjectMeta: metav1.ObjectMeta{
                        Name: "skyhook-operator-validating-webhook",
                },
                Webhooks: []admissionregistrationv1.ValidatingWebhook{
                        {
                                Name:                    "validate-skyhook.nvidia.com",
                                ClientConfig:            webhookClient(serviceName, namespace, "/validate-skyhook-nvidia-com-v1alpha1-skyhook", secret),
                                FailurePolicy:           ptr(admissionregistrationv1.Fail),
                                Rules:                   webhookRule(),
                                SideEffects:             ptr(admissionregistrationv1.SideEffectClassNone),
                                AdmissionReviewVersions: []string{"v1"},
                        },
                },
        }

        return &conf
}

// webhookMutatingWebhookConfiguration returns a new mutating webhook configuration.
func webhookMutatingWebhookConfiguration(namespace, serviceName string, secret *corev1.Secret) *admissionregistrationv1.MutatingWebhookConfiguration {
        conf := admissionregistrationv1.MutatingWebhookConfiguration{
                ObjectMeta: metav1.ObjectMeta{
                        Name: "skyhook-operator-mutating-webhook",
                },
                Webhooks: []admissionregistrationv1.MutatingWebhook{
                        {
                                Name:                    "mutate-skyhook.nvidia.com",
                                ClientConfig:            webhookClient(serviceName, namespace, "/mutate-skyhook-nvidia-com-v1alpha1-skyhook", secret),
                                FailurePolicy:           ptr(admissionregistrationv1.Fail),
                                Rules:                   webhookRule(),
                                SideEffects:             ptr(admissionregistrationv1.SideEffectClassNone),
                                AdmissionReviewVersions: []string{"v1"},
                        },
                        {
                                Name:                    "mutate-deploymentpolicy.nvidia.com",
                                ClientConfig:            webhookClient(serviceName, namespace, "/mutate-skyhook-nvidia-com-v1alpha1-deploymentpolicy", secret),
                                FailurePolicy:           ptr(admissionregistrationv1.Fail),
                                Rules:                   webhookRule(),
                                SideEffects:             ptr(admissionregistrationv1.SideEffectClassNone),
                                AdmissionReviewVersions: []string{"v1"},
                        },
                },
        }

        return &conf
}

func compareMutatingWebhookConfigurations(a, b *admissionregistrationv1.MutatingWebhookConfiguration) bool {
        if len(a.Webhooks) != len(b.Webhooks) {
                return true
        }
        for i := range a.Webhooks {
                if !bytes.Equal(a.Webhooks[i].ClientConfig.CABundle, b.Webhooks[i].ClientConfig.CABundle) {
                        return true
                }
        }
        return false
}

func compareValidatingWebhookConfigurations(a, b *admissionregistrationv1.ValidatingWebhookConfiguration) bool {
        if len(a.Webhooks) != len(b.Webhooks) {
                return true
        }
        for i := range a.Webhooks {
                if !bytes.Equal(a.Webhooks[i].ClientConfig.CABundle, b.Webhooks[i].ClientConfig.CABundle) {
                        return true
                }
        }
        return false
}

func webhookClient(serviceName, namespace, path string, secret *corev1.Secret) admissionregistrationv1.WebhookClientConfig {
        return admissionregistrationv1.WebhookClientConfig{
                Service: &admissionregistrationv1.ServiceReference{
                        Name:      serviceName,
                        Namespace: namespace,
                        Path:      ptr(path),
                },
                CABundle: secret.Data["ca.crt"],
        }
}

func webhookRule() []admissionregistrationv1.RuleWithOperations {
        return []admissionregistrationv1.RuleWithOperations{
                {
                        Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update},
                        Rule: admissionregistrationv1.Rule{
                                APIGroups:   []string{v1alpha1.GroupVersion.Group},
                                APIVersions: []string{v1alpha1.GroupVersion.Version},
                                Resources:   []string{"skyhooks"},
                        },
                },
                {
                        Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete},
                        Rule: admissionregistrationv1.Rule{
                                APIGroups:   []string{v1alpha1.GroupVersion.Group},
                                APIVersions: []string{v1alpha1.GroupVersion.Version},
                                Resources:   []string{"deploymentpolicies"},
                        },
                },
        }
}

// validatingWebhookNeedsUpdate checks if a validating webhook needs to be updated with new CABundle or Rules
// Returns true if updates were made to the webhook
func validatingWebhookNeedsUpdate(webhook *admissionregistrationv1.ValidatingWebhook, caBundle []byte, expectedRules []admissionregistrationv1.RuleWithOperations) bool {
        needUpdate := false

        // Check if CABundle needs to be set
        if len(webhook.ClientConfig.CABundle) == 0 {
                webhook.ClientConfig.CABundle = caBundle
                needUpdate = true
        }

        // Check if rules need to be updated
        if !reflect.DeepEqual(webhook.Rules, expectedRules) {
                webhook.Rules = expectedRules
                needUpdate = true
        }

        return needUpdate
}

// mutatingWebhookNeedsUpdate checks if a mutating webhook needs to be updated
func mutatingWebhookNeedsUpdate(webhook *admissionregistrationv1.MutatingWebhook, caBundle []byte, expectedRules []admissionregistrationv1.RuleWithOperations) bool {
        needUpdate := false

        // Check if CABundle needs to be set
        if len(webhook.ClientConfig.CABundle) == 0 {
                webhook.ClientConfig.CABundle = caBundle
                needUpdate = true
        }

        // Check if rules need to be updated
        if !reflect.DeepEqual(webhook.Rules, expectedRules) {
                webhook.Rules = expectedRules
                needUpdate = true
        }

        return needUpdate
}

// WebhookSecretReadyzCheck is a readyz check for the webhook secret, if it does not exist, it will return an error
// if it exists, it will wait for the secret to be ready, this makes sure that we don't start the operator
// if the webhook secret is not ready
func (r *WebhookController) WebhookSecretReadyzCheck(_ *http.Request) error {
        secret := &corev1.Secret{}
        err := r.Client.Get(context.Background(), types.NamespacedName{
                Name:      r.opts.SecretName,
                Namespace: r.namespace,
        }, secret)

        if err != nil {
                return err
        }

        equal, err := compareCertOnDiskToSecret(r.certDir, secret)
        if err != nil {
                return err
        }

        if !equal {
                return fmt.Errorf("webhook secret is not ready")
        }

        // check for the webhook configurations
        validatingWebhookName := webhookValidatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret).GetName()
        validatingWebhookConfiguration := &admissionregistrationv1.ValidatingWebhookConfiguration{}
        err = r.Get(context.Background(), types.NamespacedName{Name: validatingWebhookName}, validatingWebhookConfiguration)
        if err != nil {
                if errors.IsNotFound(err) {
                        return fmt.Errorf("ValidatingWebhookConfiguration %q not found. Either disable webhooks (not recommended) or reinstall the operator via the Helm chart to provision webhooks", validatingWebhookName)
                }
                return err
        }

        if !bytes.Equal(validatingWebhookConfiguration.Webhooks[0].ClientConfig.CABundle, secret.Data["ca.crt"]) {
                return fmt.Errorf("webhook secret is not ready, ca bundle is not equal to the validating webhook configuration")
        }

        mutatingWebhookConfiguration := webhookMutatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret)
        err = r.Get(context.Background(), types.NamespacedName{Name: mutatingWebhookConfiguration.Name}, mutatingWebhookConfiguration)
        if err != nil {
                if errors.IsNotFound(err) {
                        return fmt.Errorf("MutatingWebhookConfiguration %q not found. Either disable webhooks (not recommended) or reinstall the operator via the Helm chart to provision webhooks", mutatingWebhookConfiguration.Name)
                }
                return err
        }

        if !bytes.Equal(mutatingWebhookConfiguration.Webhooks[0].ClientConfig.CABundle, secret.Data["ca.crt"]) {
                return fmt.Errorf("webhook secret is not ready, ca bundle is not equal to the mutating webhook configuration")
        }

        return nil
}

1	/*
2	* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3	* SPDX-License-Identifier: Apache-2.0
4	*
5	*
6	* Licensed under the Apache License, Version 2.0 (the "License");
7	* you may not use this file except in compliance with the License.
8	* You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing, software
13	* distributed under the License is distributed on an "AS IS" BASIS,
14	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	* See the License for the specific language governing permissions and
16	* limitations under the License.
17	*/
18
19	package controller
20
21	import (
22	"bytes"
23	"context"
24	"fmt"
25	"net/http"
26	"reflect"
27	"time"
28
29	"sigs.k8s.io/controller-runtime/pkg/builder"
30	"sigs.k8s.io/controller-runtime/pkg/predicate"
31
32	"github.com/NVIDIA/skyhook/operator/api/v1alpha1"
33	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
34	corev1 "k8s.io/api/core/v1"
35	"k8s.io/apimachinery/pkg/api/errors"
36	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
37	"k8s.io/apimachinery/pkg/types"
38	ctrl "sigs.k8s.io/controller-runtime"
39	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
40	"sigs.k8s.io/controller-runtime/pkg/client"
41	"sigs.k8s.io/controller-runtime/pkg/log"
42	"sigs.k8s.io/controller-runtime/pkg/reconcile"
43	)
44
45	// This project used to use cert-manager to generate the webhook certificates.
46	// This removes the dependency on cert-manager and simplifies the deployment.
47	// This also removes the need to have a specific issuer, and just uses a self-signed cert.
48	type WebhookControllerOptions struct { // prefix these with WEBHOOK_
49	SecretName string `env:"WEBHOOK_SECRET_NAME, default=webhook-cert"`
50	ServiceName string `env:"WEBHOOK_SERVICE_NAME, default=skyhook-operator-webhook-service"`
51	}
52
53	type WebhookController struct {
54	client.Client
55	cache runtimecache.Cache
56	namespace string
57	certDir string
58	opts WebhookControllerOptions
59	}
60
61	func NewWebhookController(client client.Client, cache runtimecache.Cache, namespace, certDir string, opts WebhookControllerOptions) (*WebhookController, error) {	×
62	if err := ensureDummyCert(certDir); err != nil {	×
63	return nil, err	×
64	}	×
65
66	return &WebhookController{	×
67	Client: client,	×
68	cache: cache,	×
69	namespace: namespace,	×
70	certDir: certDir,	×
71	opts: opts,	×
72	}, nil	×
73	}
74
75	// Start implements the Runnable interface to ensure certificates are set up before the webhook server starts
76	func (r *WebhookController) Start(ctx context.Context) error {	×
77	logger := log.FromContext(ctx)	×
78	logger.Info("Setting up webhook certificates")	×
79		×
80	// wait for the cache to sync	×
81	if cache := r.cache.WaitForCacheSync(ctx); !cache {	×
82	return fmt.Errorf("failed to wait for cache to sync")	×
83	}	×
84	// starts the reconcile process off
85	_, err := r.GetOrCreateWebhookCertSecret(ctx, r.opts.SecretName, r.namespace)	×
86	if err != nil {	×
87	if errors.IsAlreadyExists(err) {	×
88	return nil // ignore this special case, it just needs to exist	×
89	}	×
90	return err	×
91	}
92
93	logger.Info("Webhook certificates setup complete")	×
94	return nil	×
95	}
96
97	// NeedLeaderElection implements the Runnable interface, runs only on leader
98	func (r *WebhookController) NeedLeaderElection() bool {	×
99	return true	×
100	}	×
101
102	func (r *WebhookController) SetupWithManager(mgr ctrl.Manager) error {	×
103	return ctrl.NewControllerManagedBy(mgr).	×
104	For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {	×
105	return obj.GetNamespace() == r.namespace && obj.GetName() == r.opts.SecretName	×
106	}))).	×
107	Complete(r)
108	}
109
110	// permissions
111	//+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations;mutatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete
112	//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
113
114	// Reconcile is the main function that reconciles the webhook controller
115	func (r *WebhookController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {	×
116	logger := log.FromContext(ctx)	×
117	logger.Info("Reconciling webhook controller")	×
118		×
119	// if its deleted, skip reconciliation, this is for cleanup	×
120	obj := &corev1.Secret{}	×
121	if err := r.Get(ctx, req.NamespacedName, obj); err != nil {	×
122	// handle not found, etc.	×
123	return ctrl.Result{}, client.IgnoreNotFound(err)	×
124	}	×
125
126	// If the object is being deleted, skip reconciliation
127	if !obj.ObjectMeta.DeletionTimestamp.IsZero() {	×
128	// Optionally: handle finalizers here if you want	×
129	return ctrl.Result{}, nil	×
130	}	×
131
132	// 1. Get or create/update the Secret with certs
133	// 2. Get or create/update the webhook configurations
134
135	// Example: check if secret exists
136	secret, err := r.GetOrCreateWebhookCertSecret(ctx, r.opts.SecretName, r.namespace)	×
137	if err != nil {	×
138	return reconcile.Result{}, err	×
139	}	×
140
141	_, err = r.CheckOrUpdateWebhookCertSecret(ctx, secret)	×
142	if err != nil {	×
143	return reconcile.Result{}, err	×
144	}	×
145
146	_, err = r.CheckOrUpdateWebhookConfigurations(ctx, secret)	×
147	if err != nil {	×
148	return reconcile.Result{}, err	×
149	}	×
150
151	logger.Info("Reconciled webhook controller")	×
152	return reconcile.Result{RequeueAfter: 24 * time.Hour}, nil // requeue for periodic rotation/check	×
153	}
154
155	// GetOrCreateWebhookCertSecret returns a new secret with the given name and the given CA and cert.
156	func (r WebhookController) GetOrCreateWebhookCertSecret(ctx context.Context, secretName, namespace string) (corev1.Secret, error) {	1✔
157		1✔
158	// get the secret	1✔
159	secret := &corev1.Secret{}	1✔
160	err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, secret)	1✔
161	if err != nil {	2✔
162	if errors.IsNotFound(err) {	2✔
163	// not found, create it	1✔
164	webhookCert, err := generateCert(r.opts.ServiceName, r.namespace, 36524time.Hour) // TODO: this should be configured	1✔
165	if err != nil {	1✔
166	return nil, err	×
167	}	×
168
169	// Write cert and key to disk if CertDir is set
170	if r.certDir != "" {	2✔
171	_ = writeCertAndKey([]byte(webhookCert.TLSCert), []byte(webhookCert.TLSKey), r.certDir)	1✔
172	}	1✔
173
174	secret = webhookCert.ToSecret(secretName, namespace, r.opts.ServiceName)	1✔
175		1✔
176	if err := r.Create(ctx, secret); err != nil {	1✔
177	return nil, err	×
178	}	×
179
180	return secret, nil	1✔
181	}
182	return nil, err	×
183	}
184
185	// found, return it
186	return secret, nil	×
187	}
188
189	func (r WebhookController) CheckOrUpdateWebhookCertSecret(ctx context.Context, secret corev1.Secret) (bool, error) {	1✔
190	equal, err := compareCertOnDiskToSecret(r.certDir, secret)	1✔
191	if err != nil {	1✔
192	return false, err	×
193	}	×
194
195	// check if the secret is going to expire in the next 168 hours or if the cert on disk is different from the secret
196	if !equal \|\| secret.Annotations[fmt.Sprintf("%s/expiration", v1alpha1.METADATA_PREFIX)] < time.Now().Add(168*time.Hour).Format(time.RFC3339) {	2✔
197	// expired, generate a new cert	1✔
198	webhookCert, err := generateCert(r.opts.ServiceName, r.namespace, 36524time.Hour) // TODO: this should be configured	1✔
199	if err != nil {	1✔
200	return false, err	×
201	}	×
202
203	// Write cert and key to disk if CertDir is set
204	if r.certDir != "" {	2✔
205	_ = writeCertAndKey([]byte(webhookCert.TLSCert), []byte(webhookCert.TLSKey), r.certDir)	1✔
206	}	1✔
207
208	secret.Data["ca.crt"] = webhookCert.CABytes	1✔
209	secret.Data["tls.crt"] = []byte(webhookCert.TLSCert)	1✔
210	secret.Data["tls.key"] = []byte(webhookCert.TLSKey)	1✔
211	secret.Annotations[fmt.Sprintf("%s/expiration", v1alpha1.METADATA_PREFIX)] = webhookCert.Expiration.Format(time.RFC3339)	1✔
212		1✔
213	return true, r.Update(ctx, secret)	1✔
214	}
215
216	return false, nil	×
217	}
218
219	func (r WebhookController) CheckOrUpdateWebhookConfigurations(ctx context.Context, secret corev1.Secret) (bool, error) {	×
220	// Update only CABundle fields of existing webhook configurations created by Helm	×
221	caBundle := secret.Data["ca.crt"]	×
222	changed := false	×
223		×
224	// ValidatingWebhookConfiguration	×
225	validatingName := webhookValidatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret).GetName()	×
226	existingValidating := &admissionregistrationv1.ValidatingWebhookConfiguration{}	×
227	if err := r.Get(ctx, types.NamespacedName{Name: validatingName}, existingValidating); err != nil {	×
228	if errors.IsNotFound(err) {	×
229	return false, fmt.Errorf("ValidatingWebhookConfiguration %q not found; creation is handled by the Helm chart. Ensure the chart is installed and webhooks are enabled: %w", validatingName, err)	×
230	}	×
231	return false, fmt.Errorf("failed to get ValidatingWebhookConfiguration %q: %w", validatingName, err)	×
232	}
233
234	needUpdate := false	×
NEW 235	expectedRules := webhookRule()	×
236	for i := range existingValidating.Webhooks {	×
NEW 237	if validatingWebhookNeedsUpdate(&existingValidating.Webhooks[i], caBundle, expectedRules) {	×
238	needUpdate = true	×
239	}	×
240	}
241	if needUpdate {	×
242	if err := r.Update(ctx, existingValidating); err != nil {	×
243	return false, err	×
244	} else {	×
245	changed = true	×
246	}	×
247	}
248
249	// MutatingWebhookConfiguration
250	mutatingName := webhookMutatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret).GetName()	×
251	existingMutating := &admissionregistrationv1.MutatingWebhookConfiguration{}	×
252	if err := r.Get(ctx, types.NamespacedName{Name: mutatingName}, existingMutating); err != nil {	×
253	if errors.IsNotFound(err) {	×
254	return changed, fmt.Errorf("MutatingWebhookConfiguration %q not found; creation is handled by the Helm chart. Ensure the chart is installed and webhooks are enabled: %w", mutatingName, err)	×
255	}	×
256	return false, fmt.Errorf("failed to get MutatingWebhookConfiguration %q: %w", mutatingName, err)	×
257	}
258
259	needUpdate = false	×
260	for i := range existingMutating.Webhooks {	×
NEW 261	if mutatingWebhookNeedsUpdate(&existingMutating.Webhooks[i], caBundle, expectedRules) {	×
262	needUpdate = true	×
263	}	×
264	}
265	if needUpdate {	×
266	if err := r.Update(ctx, existingMutating); err != nil {	×
267	return false, err	×
268	} else {	×
269	changed = true	×
270	}	×
271	}
272
273	return changed, nil	×
274	}
275
276	// webhookValidatingWebhookConfiguration returns a new validating webhook configuration.
277	func webhookValidatingWebhookConfiguration(namespace, serviceName string, secret corev1.Secret) admissionregistrationv1.ValidatingWebhookConfiguration {	1✔
278	conf := admissionregistrationv1.ValidatingWebhookConfiguration{	1✔
279	ObjectMeta: metav1.ObjectMeta{	1✔
280	Name: "skyhook-operator-validating-webhook",	1✔
281	},	1✔
282	Webhooks: []admissionregistrationv1.ValidatingWebhook{	1✔
283	{	1✔
284	Name: "validate-skyhook.nvidia.com",	1✔
285	ClientConfig: webhookClient(serviceName, namespace, "/validate-skyhook-nvidia-com-v1alpha1-skyhook", secret),	1✔
286	FailurePolicy: ptr(admissionregistrationv1.Fail),	1✔
287	Rules: webhookRule(),	1✔
288	SideEffects: ptr(admissionregistrationv1.SideEffectClassNone),	1✔
289	AdmissionReviewVersions: []string{"v1"},	1✔
290	},	1✔
291	},	1✔
292	}	1✔
293		1✔
294	return &conf	1✔
295	}	1✔
296
297	// webhookMutatingWebhookConfiguration returns a new mutating webhook configuration.
298	func webhookMutatingWebhookConfiguration(namespace, serviceName string, secret corev1.Secret) admissionregistrationv1.MutatingWebhookConfiguration {	1✔
299	conf := admissionregistrationv1.MutatingWebhookConfiguration{	1✔
300	ObjectMeta: metav1.ObjectMeta{	1✔
301	Name: "skyhook-operator-mutating-webhook",	1✔
302	},	1✔
303	Webhooks: []admissionregistrationv1.MutatingWebhook{	1✔
304	{	1✔
305	Name: "mutate-skyhook.nvidia.com",	1✔
306	ClientConfig: webhookClient(serviceName, namespace, "/mutate-skyhook-nvidia-com-v1alpha1-skyhook", secret),	1✔
307	FailurePolicy: ptr(admissionregistrationv1.Fail),	1✔
308	Rules: webhookRule(),	1✔
309	SideEffects: ptr(admissionregistrationv1.SideEffectClassNone),	1✔
310	AdmissionReviewVersions: []string{"v1"},	1✔
311	},	1✔
312	{	1✔
313	Name: "mutate-deploymentpolicy.nvidia.com",	1✔
314	ClientConfig: webhookClient(serviceName, namespace, "/mutate-skyhook-nvidia-com-v1alpha1-deploymentpolicy", secret),	1✔
315	FailurePolicy: ptr(admissionregistrationv1.Fail),	1✔
316	Rules: webhookRule(),	1✔
317	SideEffects: ptr(admissionregistrationv1.SideEffectClassNone),	1✔
318	AdmissionReviewVersions: []string{"v1"},	1✔
319	},	1✔
320	},	1✔
321	}	1✔
322		1✔
323	return &conf	1✔
324	}	1✔
325
326	func compareMutatingWebhookConfigurations(a, b *admissionregistrationv1.MutatingWebhookConfiguration) bool {	1✔
327	if len(a.Webhooks) != len(b.Webhooks) {	2✔
328	return true	1✔
329	}	1✔
330	for i := range a.Webhooks {	2✔
331	if !bytes.Equal(a.Webhooks[i].ClientConfig.CABundle, b.Webhooks[i].ClientConfig.CABundle) {	2✔
332	return true	1✔
333	}	1✔
334	}
335	return false	1✔
336	}
337
338	func compareValidatingWebhookConfigurations(a, b *admissionregistrationv1.ValidatingWebhookConfiguration) bool {	1✔
339	if len(a.Webhooks) != len(b.Webhooks) {	2✔
340	return true	1✔
341	}	1✔
342	for i := range a.Webhooks {	2✔
343	if !bytes.Equal(a.Webhooks[i].ClientConfig.CABundle, b.Webhooks[i].ClientConfig.CABundle) {	2✔
344	return true	1✔
345	}	1✔
346	}
347	return false	1✔
348	}
349
350	func webhookClient(serviceName, namespace, path string, secret *corev1.Secret) admissionregistrationv1.WebhookClientConfig {	1✔
351	return admissionregistrationv1.WebhookClientConfig{	1✔
352	Service: &admissionregistrationv1.ServiceReference{	1✔
353	Name: serviceName,	1✔
354	Namespace: namespace,	1✔
355	Path: ptr(path),	1✔
356	},	1✔
357	CABundle: secret.Data["ca.crt"],	1✔
358	}	1✔
359	}	1✔
360
361	func webhookRule() []admissionregistrationv1.RuleWithOperations {	1✔
362	return []admissionregistrationv1.RuleWithOperations{	1✔
363	{	1✔
364	Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update},	1✔
365	Rule: admissionregistrationv1.Rule{	1✔
366	APIGroups: []string{v1alpha1.GroupVersion.Group},	1✔
367	APIVersions: []string{v1alpha1.GroupVersion.Version},	1✔
368	Resources: []string{"skyhooks"},	1✔
369	},	1✔
370	},	1✔
371	{	1✔
372	Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete},	1✔
373	Rule: admissionregistrationv1.Rule{	1✔
374	APIGroups: []string{v1alpha1.GroupVersion.Group},	1✔
375	APIVersions: []string{v1alpha1.GroupVersion.Version},	1✔
376	Resources: []string{"deploymentpolicies"},	1✔
377	},	1✔
378	},	1✔
379	}	1✔
380	}	1✔
381
382	// validatingWebhookNeedsUpdate checks if a validating webhook needs to be updated with new CABundle or Rules
383	// Returns true if updates were made to the webhook
384	func validatingWebhookNeedsUpdate(webhook *admissionregistrationv1.ValidatingWebhook, caBundle []byte, expectedRules []admissionregistrationv1.RuleWithOperations) bool {	1✔
385	needUpdate := false	1✔
386		1✔
387	// Check if CABundle needs to be set	1✔
388	if len(webhook.ClientConfig.CABundle) == 0 {	1✔
NEW 389	webhook.ClientConfig.CABundle = caBundle	×
NEW 390	needUpdate = true	×
NEW 391	}	×
392
393	// Check if rules need to be updated
394	if !reflect.DeepEqual(webhook.Rules, expectedRules) {	2✔
395	webhook.Rules = expectedRules	1✔
396	needUpdate = true	1✔
397	}	1✔
398
399	return needUpdate	1✔
400	}
401
402	// mutatingWebhookNeedsUpdate checks if a mutating webhook needs to be updated
403	func mutatingWebhookNeedsUpdate(webhook *admissionregistrationv1.MutatingWebhook, caBundle []byte, expectedRules []admissionregistrationv1.RuleWithOperations) bool {	1✔
404	needUpdate := false	1✔
405		1✔
406	// Check if CABundle needs to be set	1✔
407	if len(webhook.ClientConfig.CABundle) == 0 {	2✔
408	webhook.ClientConfig.CABundle = caBundle	1✔
409	needUpdate = true	1✔
410	}	1✔
411
412	// Check if rules need to be updated
413	if !reflect.DeepEqual(webhook.Rules, expectedRules) {	1✔
NEW 414	webhook.Rules = expectedRules	×
NEW 415	needUpdate = true	×
NEW 416	}	×
417
418	return needUpdate	1✔
419	}
420
421	// WebhookSecretReadyzCheck is a readyz check for the webhook secret, if it does not exist, it will return an error
422	// if it exists, it will wait for the secret to be ready, this makes sure that we don't start the operator
423	// if the webhook secret is not ready
424	func (r WebhookController) WebhookSecretReadyzCheck(_ http.Request) error {	1✔
425	secret := &corev1.Secret{}	1✔
426	err := r.Client.Get(context.Background(), types.NamespacedName{	1✔
427	Name: r.opts.SecretName,	1✔
428	Namespace: r.namespace,	1✔
429	}, secret)	1✔
430		1✔
431	if err != nil {	2✔
432	return err	1✔
433	}	1✔
434
435	equal, err := compareCertOnDiskToSecret(r.certDir, secret)	1✔
436	if err != nil {	1✔
437	return err	×
438	}	×
439
440	if !equal {	2✔
441	return fmt.Errorf("webhook secret is not ready")	1✔
442	}	1✔
443
444	// check for the webhook configurations
445	validatingWebhookName := webhookValidatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret).GetName()	1✔
446	validatingWebhookConfiguration := &admissionregistrationv1.ValidatingWebhookConfiguration{}	1✔
447	err = r.Get(context.Background(), types.NamespacedName{Name: validatingWebhookName}, validatingWebhookConfiguration)	1✔
448	if err != nil {	2✔
449	if errors.IsNotFound(err) {	2✔
450	return fmt.Errorf("ValidatingWebhookConfiguration %q not found. Either disable webhooks (not recommended) or reinstall the operator via the Helm chart to provision webhooks", validatingWebhookName)	1✔
451	}	1✔
452	return err	×
453	}
454
455	if !bytes.Equal(validatingWebhookConfiguration.Webhooks[0].ClientConfig.CABundle, secret.Data["ca.crt"]) {	2✔
456	return fmt.Errorf("webhook secret is not ready, ca bundle is not equal to the validating webhook configuration")	1✔
457	}	1✔
458
459	mutatingWebhookConfiguration := webhookMutatingWebhookConfiguration(r.namespace, r.opts.ServiceName, secret)	1✔
460	err = r.Get(context.Background(), types.NamespacedName{Name: mutatingWebhookConfiguration.Name}, mutatingWebhookConfiguration)	1✔
461	if err != nil {	1✔
462	if errors.IsNotFound(err) {	×
463	return fmt.Errorf("MutatingWebhookConfiguration %q not found. Either disable webhooks (not recommended) or reinstall the operator via the Helm chart to provision webhooks", mutatingWebhookConfiguration.Name)	×
464	}	×
465	return err	×
466	}
467
468	if !bytes.Equal(mutatingWebhookConfiguration.Webhooks[0].ClientConfig.CABundle, secret.Data["ca.crt"]) {	1✔
469	return fmt.Errorf("webhook secret is not ready, ca bundle is not equal to the mutating webhook configuration")	×
470	}	×
471
472	return nil	1✔
473	}

NVIDIA / skyhook / 20477050272

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous