25472384490

Committed 07 May 2026 02:20AM UTC coverage: 24.899% (+0.06%) from 24.835%

Build # 25472384490

Build Type

push

github

Committed by

web-flow

Commit Message

feat(servicecidr): support K8s multiple ServiceCIDR (KEP-1880) (#6690)

* feat(servicecidr): support Kubernetes multiple ServiceCIDR (KEP-1880)

Watch networking.k8s.io/v1 ServiceCIDR objects and merge them with the
--service-cluster-ip-range flag value into a single source of truth used
by every Service-CIDR consumer (U2O policy routes, vpc-lb init
containers, vpc-nat-gw routes, daemon ipset/iptables). The flag now
serves as a startup fallback that yields once the API observes any valid
entry, and re-engages if the API set ever empties — so old clusters
without the API behave exactly as before, while 1.33+ clusters converge
to the API-advertised set and pick up dynamic add/remove.

ServiceCIDR API discovery uses APIResourceExists with a 10s ticker
fallback (same shape as the NAD/KubeVirt scaffolds), so missing API on
older clusters is a no-op rather than an error. RBAC for
networking.k8s.io/servicecidrs is added to system:ovn and
system:kube-ovn-cni in install.sh and both Helm charts.

Notable behavior changes:
- vpc-lb deployment is now upserted with a ServiceCIDR hash annotation;
  changing the merged set rolls the pod via the existing Recreate
  strategy.
- U2O no-LB policy routes are pruned at the start of every reconcile
  (policy-only delete, port groups untouched) so shrinking the set no
  longer leaves stale OVN entries.
- Existing VPC NAT gateways are intentionally not re-enqueued on
  ServiceCIDR change; their routes only refresh when the pod is
  recreated by other means. Newly created NAT gateways pick up the
  current store via their own add path.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Signed-off-by: Mengxin Liu <liumengxinfly@gmail.com>

* refactor(servicecidr): consolidate duplicates and use stdlib helpers

- Promote readyServiceCIDRs to util.ReadyServiceCIDRs so controller and
  daemon share one source of truth.
- Drop custom equalStringSlice in favour of slices.Equal.
- Extract vpcLbInitContainers helpe... (continued)

Coverage Stats

124 of 467 new or added lines in 12 files covered. (26.55%)

2 existing lines in 1 file now uncovered.

14198 of 57023 relevant lines covered (24.9%)

0.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

5.41

/pkg/controller/vpc_lb.go

package controller

import (
        "context"
        "fmt"
        "strings"

        nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
        v1 "k8s.io/api/apps/v1"
        corev1 "k8s.io/api/core/v1"
        k8serrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/labels"
        "k8s.io/klog/v2"

        kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
        "github.com/kubeovn/kube-ovn/pkg/util"
)

func vpcLbDeploymentName(vpc string) string {
        return fmt.Sprintf("vpc-%s-lb", vpc)
}

func (c *Controller) createVpcLb(vpc *kubeovnv1.Vpc) error {
        desired, err := c.genVpcLbDeployment(vpc)
        if desired == nil || err != nil {
                klog.Errorf("failed to generate vpc lb deployment for %s: %v", vpc.Name, err)
                return err
        }
        deployments := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace)
        existing, err := deployments.Get(context.Background(), desired.Name, metav1.GetOptions{})
        if k8serrors.IsNotFound(err) {
                klog.Infof("create vpc lb deployment %s", desired.Name)
                if _, err = deployments.Create(context.Background(), desired, metav1.CreateOptions{}); err != nil {
                        klog.Errorf("failed to create LB deployment for VPC %s: %v", vpc.Name, err)
                        return err
                }
                return nil
        }
        if err != nil {
                klog.Errorf("failed to check LB deployment for VPC %s: %v", vpc.Name, err)
                return err
        }
        if existing.Annotations[util.ServiceCIDRHashAnnotation] == desired.Annotations[util.ServiceCIDRHashAnnotation] {
                return nil
        }
        klog.Infof("update vpc lb deployment %s for ServiceCIDR change", desired.Name)
        existing.Spec = desired.Spec
        if existing.Annotations == nil {
                existing.Annotations = map[string]string{}
        }
        existing.Annotations[util.ServiceCIDRHashAnnotation] = desired.Annotations[util.ServiceCIDRHashAnnotation]
        if _, err = deployments.Update(context.Background(), existing, metav1.UpdateOptions{}); err != nil {
                klog.Errorf("failed to update LB deployment for VPC %s: %v", vpc.Name, err)
                return err
        }
        return nil
}

func (c *Controller) deleteVpcLb(vpc *kubeovnv1.Vpc) error {
        name := vpcLbDeploymentName(vpc.Name)
        _, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Get(context.Background(), name, metav1.GetOptions{})
        if err != nil {
                if k8serrors.IsNotFound(err) {
                        return nil
                }
                klog.Errorf("failed to check LB deployment for VPC %s: %v", vpc.Name, err)
                return err
        }

        klog.Infof("delete vpc lb deployment for %s", name)
        if err = c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
                klog.Errorf("failed to delete LB deployment of VPC %s: %v", vpc.Name, err)
                return err
        }

        return nil
}

func (c *Controller) genVpcLbDeployment(vpc *kubeovnv1.Vpc) (*v1.Deployment, error) {
        if len(vpc.Status.Subnets) == 0 {
                return nil, nil
        }

        defaultSubnet, err := c.subnetsLister.Get(vpc.Status.DefaultLogicalSwitch)
        if err != nil {
                return nil, err
        }

        subnets, err := c.subnetsLister.List(labels.Everything())
        if err != nil {
                return nil, err
        }

        var gateway string
        provider := fmt.Sprintf("%s.%s", util.VpcLbNetworkAttachment, c.config.PodNamespace)
        for _, subnet := range subnets {
                if subnet.Spec.Provider == provider {
                        gateway = subnet.Spec.Gateway
                        break
                }
        }

        if gateway == "" {
                return nil, fmt.Errorf("failed to get gateway for provider %s", provider)
        }

        replicas := int32(1)
        name := vpcLbDeploymentName(vpc.Name)
        appLabel := util.NormalizeLabelValue(name)
        allowPrivilegeEscalation := true
        privileged := true
        labels := map[string]string{
                "app":           appLabel,
                util.VpcLbLabel: "true",
        }

        podAnnotations := map[string]string{
                util.VpcAnnotation:           vpc.Name,
                util.LogicalSwitchAnnotation: defaultSubnet.Name,
                nadv1.NetworkAttachmentAnnot: fmt.Sprintf(`[{"name": "%s", "default-route": ["%s"]}]`, util.VpcLbNetworkAttachment, strings.ReplaceAll(gateway, ",", `" ,"`)),
        }

        deployment := &v1.Deployment{
                ObjectMeta: metav1.ObjectMeta{
                        Name: name,
                        Labels: map[string]string{
                                util.VpcNameLabel: vpc.Name,
                                util.VpcLbLabel:   "true",
                        },
                },
                Spec: v1.DeploymentSpec{
                        Replicas: &replicas,
                        Selector: &metav1.LabelSelector{
                                MatchLabels: labels,
                        },
                        Template: corev1.PodTemplateSpec{
                                ObjectMeta: metav1.ObjectMeta{
                                        Labels:      labels,
                                        Annotations: podAnnotations,
                                },
                                Spec: corev1.PodSpec{
                                        InitContainers: []corev1.Container{},
                                        Containers: []corev1.Container{
                                                {
                                                        Name:            "vpc-lb",
                                                        Image:           vpcNatImage,
                                                        Command:         []string{"bash"},
                                                        Args:            []string{"-c", "sleep infinity"},
                                                        ImagePullPolicy: corev1.PullIfNotPresent,
                                                        SecurityContext: &corev1.SecurityContext{
                                                                Privileged:               &privileged,
                                                                AllowPrivilegeEscalation: &allowPrivilegeEscalation,
                                                        },
                                                },
                                        },
                                        TerminationGracePeriodSeconds: new(int64(0)),
                                },
                        },
                        Strategy: v1.DeploymentStrategy{
                                Type: v1.RecreateDeploymentStrategyType,
                        },
                },
        }

        v4Gw, v6Gw := util.SplitStringIP(defaultSubnet.Spec.Gateway)
        deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers,
                vpcLbInitContainers(4, v4Gw, vpcNatImage, c.serviceCIDRStore.V4CIDRs(), &privileged, &allowPrivilegeEscalation)...,
        )
        deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers,
                vpcLbInitContainers(6, v6Gw, vpcNatImage, c.serviceCIDRStore.V6CIDRs(), &privileged, &allowPrivilegeEscalation)...,
        )

        if deployment.Annotations == nil {
                deployment.Annotations = map[string]string{}
        }
        if deployment.Spec.Template.Annotations == nil {
                deployment.Spec.Template.Annotations = map[string]string{}
        }
        hash := c.serviceCIDRStore.Hash()
        deployment.Annotations[util.ServiceCIDRHashAnnotation] = hash
        deployment.Spec.Template.Annotations[util.ServiceCIDRHashAnnotation] = hash

        return deployment, nil
}

// vpcLbInitContainers returns the route + MASQUERADE init containers for one
// IP family. The container names embed the family and CIDR index so that
// adding/removing ServiceCIDRs translates into a stable, deterministic name
// list that the Deployment controller can diff cleanly.
func vpcLbInitContainers(family int, gateway, image string, cidrs []string, privileged, allowPrivEsc *bool) []corev1.Container {
        if gateway == "" || len(cidrs) == 0 {
                return nil
        }
        iptablesCmd := "iptables"
        if family == 6 {
                iptablesCmd = "ip6tables"
        }
        out := make([]corev1.Container, 0, len(cidrs)*2)
        for i, cidr := range cidrs {
                out = append(out,
                        corev1.Container{
                                Name:            fmt.Sprintf("init-ipv%d-route-%d", family, i),
                                Image:           image,
                                Command:         []string{"ip"},
                                Args:            strings.Fields(fmt.Sprintf("-%d route replace %s via %s", family, cidr, gateway)),
                                ImagePullPolicy: corev1.PullIfNotPresent,
                                SecurityContext: &corev1.SecurityContext{
                                        Privileged:               privileged,
                                        AllowPrivilegeEscalation: allowPrivEsc,
                                },
                        },
                        corev1.Container{
                                Name:            fmt.Sprintf("init-ipv%d-iptables-%d", family, i),
                                Image:           image,
                                Command:         []string{iptablesCmd},
                                Args:            strings.Fields(fmt.Sprintf("-t nat -I POSTROUTING -d %s -j MASQUERADE", cidr)),
                                ImagePullPolicy: corev1.PullIfNotPresent,
                                SecurityContext: &corev1.SecurityContext{
                                        Privileged:               privileged,
                                        AllowPrivilegeEscalation: allowPrivEsc,
                                },
                        },
                )
        }
        return out
}

1	package controller
2
3	import (
4	"context"
5	"fmt"
6	"strings"
7
8	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
9	v1 "k8s.io/api/apps/v1"
10	corev1 "k8s.io/api/core/v1"
11	k8serrors "k8s.io/apimachinery/pkg/api/errors"
12	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13	"k8s.io/apimachinery/pkg/labels"
14	"k8s.io/klog/v2"
15
16	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
17	"github.com/kubeovn/kube-ovn/pkg/util"
18	)
19
20	func vpcLbDeploymentName(vpc string) string {	1✔
21	return fmt.Sprintf("vpc-%s-lb", vpc)	1✔
22	}	1✔
23
24	func (c Controller) createVpcLb(vpc kubeovnv1.Vpc) error {	×
NEW 25	desired, err := c.genVpcLbDeployment(vpc)	×
NEW 26	if desired == nil \|\| err != nil {	×
27	klog.Errorf("failed to generate vpc lb deployment for %s: %v", vpc.Name, err)	×
28	return err	×
29	}	×
NEW 30	deployments := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace)	×
NEW 31	existing, err := deployments.Get(context.Background(), desired.Name, metav1.GetOptions{})	×
NEW 32	if k8serrors.IsNotFound(err) {	×
NEW 33	klog.Infof("create vpc lb deployment %s", desired.Name)	×
NEW 34	if _, err = deployments.Create(context.Background(), desired, metav1.CreateOptions{}); err != nil {	×
NEW 35	klog.Errorf("failed to create LB deployment for VPC %s: %v", vpc.Name, err)	×
NEW 36	return err	×
NEW 37	}	×
38	return nil	×
39	}
NEW 40	if err != nil {	×
41	klog.Errorf("failed to check LB deployment for VPC %s: %v", vpc.Name, err)	×
42	return err	×
43	}	×
NEW 44	if existing.Annotations[util.ServiceCIDRHashAnnotation] == desired.Annotations[util.ServiceCIDRHashAnnotation] {	×
NEW 45	return nil	×
NEW 46	}	×
NEW 47	klog.Infof("update vpc lb deployment %s for ServiceCIDR change", desired.Name)	×
NEW 48	existing.Spec = desired.Spec	×
NEW 49	if existing.Annotations == nil {	×
NEW 50	existing.Annotations = map[string]string{}	×
NEW 51	}	×
NEW 52	existing.Annotations[util.ServiceCIDRHashAnnotation] = desired.Annotations[util.ServiceCIDRHashAnnotation]	×
NEW 53	if _, err = deployments.Update(context.Background(), existing, metav1.UpdateOptions{}); err != nil {	×
NEW 54	klog.Errorf("failed to update LB deployment for VPC %s: %v", vpc.Name, err)	×
55	return err	×
56	}	×
UNCOV 57	return nil	×
58	}
59
60	func (c Controller) deleteVpcLb(vpc kubeovnv1.Vpc) error {	1✔
61	name := vpcLbDeploymentName(vpc.Name)	1✔
62	_, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Get(context.Background(), name, metav1.GetOptions{})	1✔
63	if err != nil {	2✔
64	if k8serrors.IsNotFound(err) {	2✔
65	return nil	1✔
66	}	1✔
67	klog.Errorf("failed to check LB deployment for VPC %s: %v", vpc.Name, err)	×
68	return err	×
69	}
70
71	klog.Infof("delete vpc lb deployment for %s", name)	×
72	if err = c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {	×
73	klog.Errorf("failed to delete LB deployment of VPC %s: %v", vpc.Name, err)	×
74	return err	×
75	}	×
76
77	return nil	×
78	}
79
80	func (c Controller) genVpcLbDeployment(vpc kubeovnv1.Vpc) (*v1.Deployment, error) {	×
81	if len(vpc.Status.Subnets) == 0 {	×
82	return nil, nil	×
83	}	×
84
85	defaultSubnet, err := c.subnetsLister.Get(vpc.Status.DefaultLogicalSwitch)	×
86	if err != nil {	×
87	return nil, err	×
88	}	×
89
90	subnets, err := c.subnetsLister.List(labels.Everything())	×
91	if err != nil {	×
92	return nil, err	×
93	}	×
94
95	var gateway string	×
96	provider := fmt.Sprintf("%s.%s", util.VpcLbNetworkAttachment, c.config.PodNamespace)	×
97	for _, subnet := range subnets {	×
98	if subnet.Spec.Provider == provider {	×
99	gateway = subnet.Spec.Gateway	×
100	break	×
101	}
102	}
103
104	if gateway == "" {	×
105	return nil, fmt.Errorf("failed to get gateway for provider %s", provider)	×
106	}	×
107
108	replicas := int32(1)	×
109	name := vpcLbDeploymentName(vpc.Name)	×
110	appLabel := util.NormalizeLabelValue(name)	×
111	allowPrivilegeEscalation := true	×
112	privileged := true	×
113	labels := map[string]string{	×
114	"app": appLabel,	×
115	util.VpcLbLabel: "true",	×
116	}	×
117		×
118	podAnnotations := map[string]string{	×
119	util.VpcAnnotation: vpc.Name,	×
120	util.LogicalSwitchAnnotation: defaultSubnet.Name,	×
121	nadv1.NetworkAttachmentAnnot: fmt.Sprintf(`[{"name": "%s", "default-route": ["%s"]}]`, util.VpcLbNetworkAttachment, strings.ReplaceAll(gateway, ",", `" ,"`)),	×
122	}	×
123		×
124	deployment := &v1.Deployment{	×
125	ObjectMeta: metav1.ObjectMeta{	×
126	Name: name,	×
127	Labels: map[string]string{	×
128	util.VpcNameLabel: vpc.Name,	×
129	util.VpcLbLabel: "true",	×
130	},	×
131	},	×
132	Spec: v1.DeploymentSpec{	×
133	Replicas: &replicas,	×
134	Selector: &metav1.LabelSelector{	×
135	MatchLabels: labels,	×
136	},	×
137	Template: corev1.PodTemplateSpec{	×
138	ObjectMeta: metav1.ObjectMeta{	×
139	Labels: labels,	×
140	Annotations: podAnnotations,	×
141	},	×
142	Spec: corev1.PodSpec{	×
143	InitContainers: []corev1.Container{},	×
144	Containers: []corev1.Container{	×
145	{	×
146	Name: "vpc-lb",	×
147	Image: vpcNatImage,	×
148	Command: []string{"bash"},	×
149	Args: []string{"-c", "sleep infinity"},	×
150	ImagePullPolicy: corev1.PullIfNotPresent,	×
151	SecurityContext: &corev1.SecurityContext{	×
152	Privileged: &privileged,	×
153	AllowPrivilegeEscalation: &allowPrivilegeEscalation,	×
154	},	×
155	},	×
156	},	×
157	TerminationGracePeriodSeconds: new(int64(0)),	×
158	},	×
159	},	×
160	Strategy: v1.DeploymentStrategy{	×
161	Type: v1.RecreateDeploymentStrategyType,	×
162	},	×
163	},	×
164	}	×
165		×
166	v4Gw, v6Gw := util.SplitStringIP(defaultSubnet.Spec.Gateway)	×
NEW 167	deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers,	×
NEW 168	vpcLbInitContainers(4, v4Gw, vpcNatImage, c.serviceCIDRStore.V4CIDRs(), &privileged, &allowPrivilegeEscalation)...,	×
NEW 169	)	×
NEW 170	deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers,	×
NEW 171	vpcLbInitContainers(6, v6Gw, vpcNatImage, c.serviceCIDRStore.V6CIDRs(), &privileged, &allowPrivilegeEscalation)...,	×
NEW 172	)	×
NEW 173		×
NEW 174	if deployment.Annotations == nil {	×
NEW 175	deployment.Annotations = map[string]string{}	×
NEW 176	}	×
NEW 177	if deployment.Spec.Template.Annotations == nil {	×
NEW 178	deployment.Spec.Template.Annotations = map[string]string{}	×
UNCOV 179	}	×
NEW 180	hash := c.serviceCIDRStore.Hash()	×
NEW 181	deployment.Annotations[util.ServiceCIDRHashAnnotation] = hash	×
NEW 182	deployment.Spec.Template.Annotations[util.ServiceCIDRHashAnnotation] = hash	×
183		×
184	return deployment, nil	×
185	}
186
187	// vpcLbInitContainers returns the route + MASQUERADE init containers for one
188	// IP family. The container names embed the family and CIDR index so that
189	// adding/removing ServiceCIDRs translates into a stable, deterministic name
190	// list that the Deployment controller can diff cleanly.
NEW 191	func vpcLbInitContainers(family int, gateway, image string, cidrs []string, privileged, allowPrivEsc *bool) []corev1.Container {	×
NEW 192	if gateway == "" \|\| len(cidrs) == 0 {	×
NEW 193	return nil	×
NEW 194	}	×
NEW 195	iptablesCmd := "iptables"	×
NEW 196	if family == 6 {	×
NEW 197	iptablesCmd = "ip6tables"	×
NEW 198	}	×
NEW 199	out := make([]corev1.Container, 0, len(cidrs)*2)	×
NEW 200	for i, cidr := range cidrs {	×
NEW 201	out = append(out,	×
NEW 202	corev1.Container{	×
NEW 203	Name: fmt.Sprintf("init-ipv%d-route-%d", family, i),	×
NEW 204	Image: image,	×
NEW 205	Command: []string{"ip"},	×
NEW 206	Args: strings.Fields(fmt.Sprintf("-%d route replace %s via %s", family, cidr, gateway)),	×
NEW 207	ImagePullPolicy: corev1.PullIfNotPresent,	×
NEW 208	SecurityContext: &corev1.SecurityContext{	×
NEW 209	Privileged: privileged,	×
NEW 210	AllowPrivilegeEscalation: allowPrivEsc,	×
NEW 211	},	×
NEW 212	},	×
NEW 213	corev1.Container{	×
NEW 214	Name: fmt.Sprintf("init-ipv%d-iptables-%d", family, i),	×
NEW 215	Image: image,	×
NEW 216	Command: []string{iptablesCmd},	×
NEW 217	Args: strings.Fields(fmt.Sprintf("-t nat -I POSTROUTING -d %s -j MASQUERADE", cidr)),	×
NEW 218	ImagePullPolicy: corev1.PullIfNotPresent,	×
NEW 219	SecurityContext: &corev1.SecurityContext{	×
NEW 220	Privileged: privileged,	×
NEW 221	AllowPrivilegeEscalation: allowPrivEsc,	×
NEW 222	},	×
NEW 223	},	×
NEW 224	)	×
NEW 225	}	×
NEW 226	return out	×
227	}

kubeovn / kube-ovn / 25472384490

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous