6837407813

Committed 09 Nov 2023 08:07PM UTC coverage: 50.459% (+0.06%) from 50.402%

Build # 6837407813

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #3964 from wizhaoredhat/dpu_ovn_ic_enablement_and_bug_fixes

Dpu OVN-IC enablement

Run Details

5 of 25 new or added lines in 2 files covered. (20.0%)

13 existing lines in 5 files now uncovered.

25540 of 50615 relevant lines covered (50.46%)

87.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.1

/go-controller/pkg/ovn/hybrid.go

package ovn

import (
        "fmt"
        "net"
        "strings"
        "sync/atomic"

        "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
        hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
        houtil "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/util"
        "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
        "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
        libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops"
        libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util"
        "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"

        ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
        "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"

        kapi "k8s.io/api/core/v1"
        "k8s.io/klog/v2"
        utilnet "k8s.io/utils/net"
)

// handleHybridOverlayPort reconciles the node's overlay port with OVN.
// It needs to handle the following cases:
//   - no subnet allocated: unset MAC annotation
//   - no MAC annotation, no lsp: configure lsp, set annotation
//   - annotation, no lsp: configure lsp
//   - annotation, lsp: ensure lsp matches annotation
//   - no annotation, lsp: set annotation from lsp
func (oc *DefaultNetworkController) handleHybridOverlayPort(node *kapi.Node, annotator kube.Annotator) error {
        var err error
        var annotationMAC, portMAC net.HardwareAddr
        var drIP net.IP
        portName := util.GetHybridOverlayPortName(node.Name)

        // retrieve mac annotation
        am, annotationOK := node.Annotations[types.HybridOverlayDRMAC]
        if annotationOK {
                annotationMAC, err = net.ParseMAC(am)
                if err != nil {
                        klog.Errorf("MAC annotation %s on node %s is invalid, ignoring.", annotationMAC, node.Name)
                        annotationOK = false
                }
        }

        // no subnet allocated? unset mac annotation, be done.
        subnets, err := util.ParseNodeHostSubnetAnnotation(node, ovntypes.DefaultNetworkName)
        if subnets == nil || err != nil {
                // No subnet allocated yet; clean up
                klog.V(5).Infof("No subnet allocation yet for %s", node.Name)
                if annotationOK {
                        oc.deleteHybridOverlayPort(node)
                        annotator.Delete(types.HybridOverlayDRMAC)
                }
                return nil
        }

        // Retrieve port MAC address; if the port isn't set up, portMAC will be nil
        lsp := &nbdb.LogicalSwitchPort{Name: portName}
        lsp, err = libovsdbops.GetLogicalSwitchPort(oc.nbClient, lsp)
        if err == nil {
                portMAC, _, _ = libovsdbutil.ExtractPortAddresses(lsp)
        }

        // compare port configuration to annotation MAC, reconcile as needed
        lspOK := false

        if drIP = net.ParseIP(node.Annotations[hotypes.HybridOverlayDRIP]); drIP == nil {
                return fmt.Errorf("cannot set up hybrid overlay ports, distributed router ip is nil")
        }
        // nothing allocated, allocate mac from HybridOverlayDRIP
        if portMAC == nil && annotationMAC == nil {
                portMAC = util.IPAddrToHWAddr(drIP)
                annotationMAC = portMAC
                klog.V(5).Infof("Allocating MAC %s to node %s", portMAC.String(), node.Name)
        } else if portMAC == nil && annotationMAC != nil { // annotation, no port
                portMAC = annotationMAC
        } else if portMAC != nil && annotationMAC == nil { // port, no annotation
                lspOK = true
                annotationMAC = portMAC
        } else if portMAC != nil && annotationMAC != nil { // port & annotation: anno wins
                if portMAC.String() != annotationMAC.String() {
                        klog.V(2).Infof("Warning: node %s lsp %s has mismatching hybrid port mac, correcting", node.Name, portName)
                        portMAC = annotationMAC
                } else {
                        lspOK = true
                }
        }

        // we need to setup a reroute policy for hybrid overlay subnet
        // this is so hybrid pod -> service -> hybrid endpoint will reroute to the DR IP
        if err := oc.setupHybridLRPolicySharedGw(subnets, node.Name, portMAC, drIP); err != nil {
                return fmt.Errorf("unable to setup Hybrid Subnet Logical Route Policy for node: %s, error: %v",
                        node.Name, err)
        }

        if !lspOK {
                klog.Infof("Creating / updating node %s hybrid overlay port with mac %s", node.Name, portMAC.String())

                // create / update lsps
                lsp := nbdb.LogicalSwitchPort{
                        Name:      portName,
                        Addresses: []string{portMAC.String()},
                }
                sw := nbdb.LogicalSwitch{Name: node.Name}

                err := libovsdbops.CreateOrUpdateLogicalSwitchPortsOnSwitch(oc.nbClient, &sw, &lsp)
                if err != nil {
                        return fmt.Errorf("failed to add hybrid overlay port %+v for node %s: %v", lsp, node.Name, err)
                }
                for _, subnet := range subnets {
                        if err := libovsdbutil.UpdateNodeSwitchExcludeIPs(oc.nbClient, node.Name, subnet); err != nil {
                                return err
                        }
                }
        }

        if !annotationOK {
                klog.Infof("Setting node %s hybrid overlay mac annotation to %s", node.Name, annotationMAC.String())
                if err := annotator.Set(types.HybridOverlayDRMAC, portMAC.String()); err != nil {
                        return fmt.Errorf("failed to set node %s hybrid overlay DRMAC annotation: %v", node.Name, err)
                }
        }

        return nil
}

func (oc *DefaultNetworkController) deleteHybridOverlayPort(node *kapi.Node) {
        klog.Infof("Removing node %s hybrid overlay port", node.Name)
        portName := util.GetHybridOverlayPortName(node.Name)
        lsp := nbdb.LogicalSwitchPort{Name: portName}
        sw := nbdb.LogicalSwitch{Name: node.Name}
        if err := libovsdbops.DeleteLogicalSwitchPorts(oc.nbClient, &sw, &lsp); err != nil {
                klog.Errorf("Failed deleting hybrind overlay port %s for node %s err: %v", portName, node.Name, err)
        }
}

func (oc *DefaultNetworkController) setupHybridLRPolicySharedGw(nodeSubnets []*net.IPNet, nodeName string, portMac net.HardwareAddr, drIPs net.IP) error {
        klog.Infof("Setting up logical route policy for hybrid subnet on node: %s", nodeName)
        var L3Prefix string
        for _, nodeSubnet := range nodeSubnets {
                if utilnet.IsIPv6CIDR(nodeSubnet) {
                        L3Prefix = "ip6"
                } else {
                        L3Prefix = "ip4"
                }
                var hybridCIDRs []*net.IPNet
                if len(config.HybridOverlay.ClusterSubnets) > 0 {
                        for _, hybridSubnet := range config.HybridOverlay.ClusterSubnets {
                                if utilnet.IsIPv6CIDR(hybridSubnet.CIDR) == utilnet.IsIPv6CIDR(nodeSubnet) {
                                        hybridCIDRs = append(hybridCIDRs, hybridSubnet.CIDR)
                                        break
                                }
                        }
                } else {
                        nodes, err := oc.kube.GetNodes()
                        if err != nil {
                                return err
                        }
                        for _, node := range nodes.Items {
                                if subnet, _ := houtil.ParseHybridOverlayHostSubnet(&node); subnet != nil {
                                        hybridCIDRs = append(hybridCIDRs, subnet)
                                }
                        }
                }

                for _, hybridCIDR := range hybridCIDRs {
                        if utilnet.IsIPv6CIDR(hybridCIDR) != utilnet.IsIPv6CIDR(nodeSubnet) {
                                // skip if the IP family is not match
                                continue
                        }
                        drIP := drIPs
                        matchStr := fmt.Sprintf(`inport == "%s%s" && %s.dst == %s`,
                                ovntypes.RouterToSwitchPrefix, nodeName, L3Prefix, hybridCIDR)

                        // Logic route policy to steer packet from pod to hybrid overlay nodes
                        logicalRouterPolicy := nbdb.LogicalRouterPolicy{
                                Priority: ovntypes.HybridOverlaySubnetPriority,
                                ExternalIDs: map[string]string{
                                        "name": ovntypes.HybridSubnetPrefix + nodeName,
                                },
                                Action:   nbdb.LogicalRouterPolicyActionReroute,
                                Nexthops: []string{drIP.String()},
                                Match:    matchStr,
                        }

                        if err := libovsdbops.CreateOrUpdateLogicalRouterPolicyWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, &logicalRouterPolicy, func(item *nbdb.LogicalRouterPolicy) bool {
                                return item.Priority == logicalRouterPolicy.Priority &&
                                        item.ExternalIDs["name"] == logicalRouterPolicy.ExternalIDs["name"]
                        }, &logicalRouterPolicy.Nexthops, &logicalRouterPolicy.Match, &logicalRouterPolicy.Action); err != nil {
                                return fmt.Errorf("failed to add policy route '%s' for host %q on %s , error: %v", matchStr, nodeName, ovntypes.OVNClusterRouter, err)
                        }

                        smb := &nbdb.StaticMACBinding{
                                LogicalPort:        ovntypes.RouterToSwitchPrefix + nodeName,
                                MAC:                portMac.String(),
                                IP:                 drIP.String(),
                                OverrideDynamicMAC: true,
                        }
                        if err := libovsdbops.CreateOrUpdateStaticMacBinding(oc.nbClient, smb); err != nil {
                                return fmt.Errorf("failed to create MAC Binding for hybrid overlay: %v", err)
                        }

                        // Logic route policy to steer packet from external to nodePort service backed by non-ovnkube pods to hybrid overlay nodes
                        gwLRPIfAddrs, err := libovsdbutil.GetLRPAddrs(oc.nbClient, ovntypes.GWRouterToJoinSwitchPrefix+ovntypes.GWRouterPrefix+nodeName)
                        if err != nil {
                                return err
                        }
                        gwLRPIfAddr, err := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(hybridCIDR), gwLRPIfAddrs)
                        if err != nil {
                                return err
                        }
                        grMatchStr := fmt.Sprintf(`%s.src == %s && %s.dst == %s`,
                                L3Prefix, gwLRPIfAddr.IP.String(), L3Prefix, hybridCIDR)
                        grLogicalRouterPolicy := nbdb.LogicalRouterPolicy{
                                Priority: ovntypes.HybridOverlaySubnetPriority,
                                ExternalIDs: map[string]string{
                                        "name": ovntypes.HybridSubnetPrefix + nodeName + "-gr",
                                },
                                Action:   nbdb.LogicalRouterPolicyActionReroute,
                                Nexthops: []string{drIP.String()},
                                Match:    grMatchStr,
                        }

                        if err := libovsdbops.CreateOrUpdateLogicalRouterPolicyWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, &grLogicalRouterPolicy, func(item *nbdb.LogicalRouterPolicy) bool {
                                return item.Priority == grLogicalRouterPolicy.Priority &&
                                        item.ExternalIDs["name"] == grLogicalRouterPolicy.ExternalIDs["name"]
                        }, &grLogicalRouterPolicy.Nexthops, &grLogicalRouterPolicy.Match, &grLogicalRouterPolicy.Action); err != nil {
                                return fmt.Errorf("failed to add policy route '%s' for host %q on %s , error: %v", matchStr, nodeName, ovntypes.OVNClusterRouter, err)
                        }
                        klog.Infof("Created hybrid overlay logical route policies for node %s", nodeName)

                        // Static route to steer packets from external to nodePort service backed by pods on hybrid overlay node.
                        // This route is to used for triggering above route policy
                        clusterRouterStaticRoutes := nbdb.LogicalRouterStaticRoute{
                                IPPrefix: hybridCIDR.String(),
                                Nexthop:  drIP.String(),
                                ExternalIDs: map[string]string{
                                        "name": ovntypes.HybridSubnetPrefix + nodeName,
                                },
                        }
                        if err := libovsdbops.CreateOrReplaceLogicalRouterStaticRouteWithPredicate(oc.nbClient,
                                ovntypes.OVNClusterRouter, &clusterRouterStaticRoutes,
                                func(item *nbdb.LogicalRouterStaticRoute) bool {
                                        return item.IPPrefix == clusterRouterStaticRoutes.IPPrefix &&
                                                item.ExternalIDs["name"] == clusterRouterStaticRoutes.ExternalIDs["name"] &&
                                                libovsdbops.PolicyEqualPredicate(clusterRouterStaticRoutes.Policy, item.Policy)
                                }, &clusterRouterStaticRoutes.Nexthop); err != nil {
                                return fmt.Errorf("failed to add policy route static '%s %s' for on %s , error: %v",
                                        clusterRouterStaticRoutes.IPPrefix, clusterRouterStaticRoutes.Nexthop,
                                        ovntypes.GWRouterPrefix+nodeName, err)
                        }
                        klog.Infof("Created hybrid overlay logical route static route at cluster router for node %s", nodeName)

                        // Static route to steer packets from external to nodePort service backed by pods on hybrid overlay node to cluster router.
                        drLRPIfAddrs, err := libovsdbutil.GetLRPAddrs(oc.nbClient, ovntypes.GWRouterToJoinSwitchPrefix+ovntypes.OVNClusterRouter)
                        if err != nil {
                                return err
                        }
                        drLRPIfAddr, err := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(hybridCIDR), drLRPIfAddrs)
                        if err != nil {
                                return fmt.Errorf("failed to match cluster router join interface IPs: %v, err: %v", drLRPIfAddr, err)
                        }
                        nodeGWRouterStaticRoutes := nbdb.LogicalRouterStaticRoute{
                                IPPrefix: hybridCIDR.String(),
                                Nexthop:  drLRPIfAddr.IP.String(),
                                ExternalIDs: map[string]string{
                                        "name": ovntypes.HybridSubnetPrefix + nodeName + "-gr",
                                },
                        }
                        if err := libovsdbops.CreateOrReplaceLogicalRouterStaticRouteWithPredicate(oc.nbClient,
                                ovntypes.GWRouterPrefix+nodeName, &nodeGWRouterStaticRoutes,
                                func(item *nbdb.LogicalRouterStaticRoute) bool {
                                        return item.IPPrefix == nodeGWRouterStaticRoutes.IPPrefix &&
                                                item.ExternalIDs["name"] == nodeGWRouterStaticRoutes.ExternalIDs["name"] &&
                                                libovsdbops.PolicyEqualPredicate(nodeGWRouterStaticRoutes.Policy, item.Policy)
                                }, &nodeGWRouterStaticRoutes.Nexthop); err != nil {
                                return fmt.Errorf("failed to add policy route static '%s %s' for on %s , error: %v",
                                        nodeGWRouterStaticRoutes.IPPrefix, nodeGWRouterStaticRoutes.Nexthop,
                                        ovntypes.GWRouterPrefix+nodeName, err)
                        }
                        klog.Infof("Created hybrid overlay logical route static route at gateway router for node %s", nodeName)
                }
        }
        return nil
}

func (oc *DefaultNetworkController) removeHybridLRPolicySharedGW(node *kapi.Node) error {
        nodeName := node.Name
        name := ovntypes.HybridSubnetPrefix + nodeName

        if err := libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, func(item *nbdb.LogicalRouterPolicy) bool {
                return item.ExternalIDs["name"] == ovntypes.HybridSubnetPrefix+nodeName || item.ExternalIDs["name"] == ovntypes.HybridSubnetPrefix+nodeName+"-gr"
        }); err != nil {
                return fmt.Errorf("failed to delete policy %s from %s, error: %v", name, ovntypes.OVNClusterRouter, err)
        }

        if err := libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, func(item *nbdb.LogicalRouterStaticRoute) bool {
                return item.ExternalIDs["name"] == name
        }); err != nil {
                return fmt.Errorf("failed to delete static route %s from %s, error: %v", name, ovntypes.OVNClusterRouter, err)
        }
        if err := libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicate(oc.nbClient, ovntypes.GWRouterPrefix+nodeName, func(item *nbdb.LogicalRouterStaticRoute) bool {
                return item.ExternalIDs["name"] == name+"-gr"
        }); err != nil {
                return fmt.Errorf("failed to delete static route %s from %s, error: %v", name+"gr", ovntypes.GWRouterPrefix+nodeName, err)
        }

        if node.Annotations[hotypes.HybridOverlayDRIP] != "" {
                smb := &nbdb.StaticMACBinding{
                        IP:          node.Annotations[hotypes.HybridOverlayDRIP],
                        LogicalPort: ovntypes.RouterToSwitchPrefix + nodeName,
                }
                err := libovsdbops.DeleteStaticMacBindings(oc.nbClient, smb)
                if err != nil {
                        return fmt.Errorf("failed to delete static mac binding %+v: %v", smb, err)
                }
        }

        return nil
}

// takes the node name and allocates the hybrid overlay distributed router ip address
func (oc *DefaultNetworkController) allocateHybridOverlayDRIP(node *kapi.Node) error {
        if atomic.LoadUint32(&oc.allInitialPodsProcessed) == 0 {
                return fmt.Errorf("cannot allocate hybrid overlay distributed router ip for nodes until all initial pods are processed")
        }

        var ips []string
        // check if there is a hybrid distributed router IP set as an annotation
        nodeHybridOverlayDRIP := node.Annotations[hotypes.HybridOverlayDRIP]
        if len(nodeHybridOverlayDRIP) > 0 {
                ips = strings.Split(nodeHybridOverlayDRIP, ",")
        }

        allocatedHybridOverlayDRIPs, err := oc.lsManager.AllocateHybridOverlay(node.Name, ips)
        if err != nil {
                return fmt.Errorf("cannot allocate hybrid overlay interface addresses on node %s: %v", node.Name, err)
        }

        var sliceHybridOverlayDRIP []string
        for _, ip := range allocatedHybridOverlayDRIPs {
                sliceHybridOverlayDRIP = append(sliceHybridOverlayDRIP, ip.IP.String())
        }

        err = oc.kube.SetAnnotationsOnNode(node.Name, map[string]interface{}{hotypes.HybridOverlayDRIP: strings.Join(sliceHybridOverlayDRIP, ",")})
        if err != nil {
                return fmt.Errorf("cannot set hybrid annotation on node %s: %v", node.Name, err)
        }

        return nil
}

1	package ovn
2
3	import (
4	"fmt"
5	"net"
6	"strings"
7	"sync/atomic"
8
9	"github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
10	hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
11	houtil "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/util"
12	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
13	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
14	libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops"
15	libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util"
16	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
17
18	ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
19	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
20
21	kapi "k8s.io/api/core/v1"
22	"k8s.io/klog/v2"
23	utilnet "k8s.io/utils/net"
24	)
25
26	// handleHybridOverlayPort reconciles the node's overlay port with OVN.
27	// It needs to handle the following cases:
28	// - no subnet allocated: unset MAC annotation
29	// - no MAC annotation, no lsp: configure lsp, set annotation
30	// - annotation, no lsp: configure lsp
31	// - annotation, lsp: ensure lsp matches annotation
32	// - no annotation, lsp: set annotation from lsp
33	func (oc DefaultNetworkController) handleHybridOverlayPort(node kapi.Node, annotator kube.Annotator) error {	22✔
34	var err error	22✔
35	var annotationMAC, portMAC net.HardwareAddr	22✔
36	var drIP net.IP	22✔
37	portName := util.GetHybridOverlayPortName(node.Name)	22✔
38		22✔
39	// retrieve mac annotation	22✔
40	am, annotationOK := node.Annotations[types.HybridOverlayDRMAC]	22✔
41	if annotationOK {	33✔
42	annotationMAC, err = net.ParseMAC(am)	11✔
43	if err != nil {	11✔
44	klog.Errorf("MAC annotation %s on node %s is invalid, ignoring.", annotationMAC, node.Name)	×
45	annotationOK = false	×
46	}	×
47	}
48
49	// no subnet allocated? unset mac annotation, be done.
50	subnets, err := util.ParseNodeHostSubnetAnnotation(node, ovntypes.DefaultNetworkName)	22✔
51	if subnets == nil \|\| err != nil {	24✔
52	// No subnet allocated yet; clean up	2✔
53	klog.V(5).Infof("No subnet allocation yet for %s", node.Name)	2✔
54	if annotationOK {	3✔
55	oc.deleteHybridOverlayPort(node)	1✔
56	annotator.Delete(types.HybridOverlayDRMAC)	1✔
57	}	1✔
58	return nil	2✔
59	}
60
61	// Retrieve port MAC address; if the port isn't set up, portMAC will be nil
62	lsp := &nbdb.LogicalSwitchPort{Name: portName}	20✔
63	lsp, err = libovsdbops.GetLogicalSwitchPort(oc.nbClient, lsp)	20✔
64	if err == nil {	34✔
65	portMAC, _, _ = libovsdbutil.ExtractPortAddresses(lsp)	14✔
66	}	14✔
67
68	// compare port configuration to annotation MAC, reconcile as needed
69	lspOK := false	20✔
70		20✔
71	if drIP = net.ParseIP(node.Annotations[hotypes.HybridOverlayDRIP]); drIP == nil {	24✔
72	return fmt.Errorf("cannot set up hybrid overlay ports, distributed router ip is nil")	4✔
73	}	4✔
74	// nothing allocated, allocate mac from HybridOverlayDRIP
75	if portMAC == nil && annotationMAC == nil {	19✔
76	portMAC = util.IPAddrToHWAddr(drIP)	3✔
77	annotationMAC = portMAC	3✔
78	klog.V(5).Infof("Allocating MAC %s to node %s", portMAC.String(), node.Name)	3✔
79	} else if portMAC == nil && annotationMAC != nil { // annotation, no port	16✔
80	portMAC = annotationMAC	×
81	} else if portMAC != nil && annotationMAC == nil { // port, no annotation	16✔
82	lspOK = true	3✔
83	annotationMAC = portMAC	3✔
84	} else if portMAC != nil && annotationMAC != nil { // port & annotation: anno wins	23✔
85	if portMAC.String() != annotationMAC.String() {	10✔
86	klog.V(2).Infof("Warning: node %s lsp %s has mismatching hybrid port mac, correcting", node.Name, portName)	×
87	portMAC = annotationMAC	×
88	} else {	10✔
89	lspOK = true	10✔
90	}	10✔
91	}
92
93	// we need to setup a reroute policy for hybrid overlay subnet
94	// this is so hybrid pod -> service -> hybrid endpoint will reroute to the DR IP
95	if err := oc.setupHybridLRPolicySharedGw(subnets, node.Name, portMAC, drIP); err != nil {	16✔
96	return fmt.Errorf("unable to setup Hybrid Subnet Logical Route Policy for node: %s, error: %v",	×
97	node.Name, err)	×
98	}	×
99
100	if !lspOK {	19✔
101	klog.Infof("Creating / updating node %s hybrid overlay port with mac %s", node.Name, portMAC.String())	3✔
102		3✔
103	// create / update lsps	3✔
104	lsp := nbdb.LogicalSwitchPort{	3✔
105	Name: portName,	3✔
106	Addresses: []string{portMAC.String()},	3✔
107	}	3✔
108	sw := nbdb.LogicalSwitch{Name: node.Name}	3✔
109		3✔
110	err := libovsdbops.CreateOrUpdateLogicalSwitchPortsOnSwitch(oc.nbClient, &sw, &lsp)	3✔
111	if err != nil {	3✔
112	return fmt.Errorf("failed to add hybrid overlay port %+v for node %s: %v", lsp, node.Name, err)	×
113	}	×
114	for _, subnet := range subnets {	6✔
115	if err := libovsdbutil.UpdateNodeSwitchExcludeIPs(oc.nbClient, node.Name, subnet); err != nil {	3✔
116	return err	×
117	}	×
118	}
119	}
120
121	if !annotationOK {	22✔
122	klog.Infof("Setting node %s hybrid overlay mac annotation to %s", node.Name, annotationMAC.String())	6✔
123	if err := annotator.Set(types.HybridOverlayDRMAC, portMAC.String()); err != nil {	6✔
124	return fmt.Errorf("failed to set node %s hybrid overlay DRMAC annotation: %v", node.Name, err)	×
125	}	×
126	}
127
128	return nil	16✔
129	}
130
131	func (oc DefaultNetworkController) deleteHybridOverlayPort(node kapi.Node) {	3✔
132	klog.Infof("Removing node %s hybrid overlay port", node.Name)	3✔
133	portName := util.GetHybridOverlayPortName(node.Name)	3✔
134	lsp := nbdb.LogicalSwitchPort{Name: portName}	3✔
135	sw := nbdb.LogicalSwitch{Name: node.Name}	3✔
136	if err := libovsdbops.DeleteLogicalSwitchPorts(oc.nbClient, &sw, &lsp); err != nil {	3✔
137	klog.Errorf("Failed deleting hybrind overlay port %s for node %s err: %v", portName, node.Name, err)	×
138	}	×
139	}
140
141	func (oc DefaultNetworkController) setupHybridLRPolicySharedGw(nodeSubnets []net.IPNet, nodeName string, portMac net.HardwareAddr, drIPs net.IP) error {	16✔
142	klog.Infof("Setting up logical route policy for hybrid subnet on node: %s", nodeName)	16✔
143	var L3Prefix string	16✔
144	for _, nodeSubnet := range nodeSubnets {	32✔
145	if utilnet.IsIPv6CIDR(nodeSubnet) {	16✔
146	L3Prefix = "ip6"	×
147	} else {	16✔
148	L3Prefix = "ip4"	16✔
149	}	16✔
150	var hybridCIDRs []*net.IPNet	16✔
151	if len(config.HybridOverlay.ClusterSubnets) > 0 {	27✔
152	for _, hybridSubnet := range config.HybridOverlay.ClusterSubnets {	22✔
153	if utilnet.IsIPv6CIDR(hybridSubnet.CIDR) == utilnet.IsIPv6CIDR(nodeSubnet) {	22✔
154	hybridCIDRs = append(hybridCIDRs, hybridSubnet.CIDR)	11✔
155	break	11✔
156	}
157	}
158	} else {	5✔
159	nodes, err := oc.kube.GetNodes()	5✔
160	if err != nil {	5✔
161	return err	×
162	}	×
163	for _, node := range nodes.Items {	14✔
164	if subnet, _ := houtil.ParseHybridOverlayHostSubnet(&node); subnet != nil {	14✔
165	hybridCIDRs = append(hybridCIDRs, subnet)	5✔
166	}	5✔
167	}
168	}
169
170	for _, hybridCIDR := range hybridCIDRs {	32✔
171	if utilnet.IsIPv6CIDR(hybridCIDR) != utilnet.IsIPv6CIDR(nodeSubnet) {	16✔
172	// skip if the IP family is not match	×
173	continue	×
174	}
175	drIP := drIPs	16✔
176	matchStr := fmt.Sprintf(`inport == "%s%s" && %s.dst == %s`,	16✔
177	ovntypes.RouterToSwitchPrefix, nodeName, L3Prefix, hybridCIDR)	16✔
178		16✔
179	// Logic route policy to steer packet from pod to hybrid overlay nodes	16✔
180	logicalRouterPolicy := nbdb.LogicalRouterPolicy{	16✔
181	Priority: ovntypes.HybridOverlaySubnetPriority,	16✔
182	ExternalIDs: map[string]string{	16✔
183	"name": ovntypes.HybridSubnetPrefix + nodeName,	16✔
184	},	16✔
185	Action: nbdb.LogicalRouterPolicyActionReroute,	16✔
186	Nexthops: []string{drIP.String()},	16✔
187	Match: matchStr,	16✔
188	}	16✔
189		16✔
190	if err := libovsdbops.CreateOrUpdateLogicalRouterPolicyWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, &logicalRouterPolicy, func(item *nbdb.LogicalRouterPolicy) bool {	74✔
191	return item.Priority == logicalRouterPolicy.Priority &&	58✔
192	item.ExternalIDs["name"] == logicalRouterPolicy.ExternalIDs["name"]	58✔
193	}, &logicalRouterPolicy.Nexthops, &logicalRouterPolicy.Match, &logicalRouterPolicy.Action); err != nil {	58✔
194	return fmt.Errorf("failed to add policy route '%s' for host %q on %s , error: %v", matchStr, nodeName, ovntypes.OVNClusterRouter, err)	×
195	}	×
196
197	smb := &nbdb.StaticMACBinding{	16✔
198	LogicalPort: ovntypes.RouterToSwitchPrefix + nodeName,	16✔
199	MAC: portMac.String(),	16✔
200	IP: drIP.String(),	16✔
201	OverrideDynamicMAC: true,	16✔
202	}	16✔
203	if err := libovsdbops.CreateOrUpdateStaticMacBinding(oc.nbClient, smb); err != nil {	16✔
204	return fmt.Errorf("failed to create MAC Binding for hybrid overlay: %v", err)	×
205	}	×
206
207	// Logic route policy to steer packet from external to nodePort service backed by non-ovnkube pods to hybrid overlay nodes
208	gwLRPIfAddrs, err := libovsdbutil.GetLRPAddrs(oc.nbClient, ovntypes.GWRouterToJoinSwitchPrefix+ovntypes.GWRouterPrefix+nodeName)	16✔
209	if err != nil {	16✔
210	return err	×
211	}	×
212	gwLRPIfAddr, err := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(hybridCIDR), gwLRPIfAddrs)	16✔
213	if err != nil {	16✔
214	return err	×
215	}	×
216	grMatchStr := fmt.Sprintf(`%s.src == %s && %s.dst == %s`,	16✔
217	L3Prefix, gwLRPIfAddr.IP.String(), L3Prefix, hybridCIDR)	16✔
218	grLogicalRouterPolicy := nbdb.LogicalRouterPolicy{	16✔
219	Priority: ovntypes.HybridOverlaySubnetPriority,	16✔
220	ExternalIDs: map[string]string{	16✔
221	"name": ovntypes.HybridSubnetPrefix + nodeName + "-gr",	16✔
222	},	16✔
223	Action: nbdb.LogicalRouterPolicyActionReroute,	16✔
224	Nexthops: []string{drIP.String()},	16✔
225	Match: grMatchStr,	16✔
226	}	16✔
227		16✔
228	if err := libovsdbops.CreateOrUpdateLogicalRouterPolicyWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, &grLogicalRouterPolicy, func(item *nbdb.LogicalRouterPolicy) bool {	77✔
229	return item.Priority == grLogicalRouterPolicy.Priority &&	61✔
230	item.ExternalIDs["name"] == grLogicalRouterPolicy.ExternalIDs["name"]	61✔
231	}, &grLogicalRouterPolicy.Nexthops, &grLogicalRouterPolicy.Match, &grLogicalRouterPolicy.Action); err != nil {	61✔
232	return fmt.Errorf("failed to add policy route '%s' for host %q on %s , error: %v", matchStr, nodeName, ovntypes.OVNClusterRouter, err)	×
233	}	×
234	klog.Infof("Created hybrid overlay logical route policies for node %s", nodeName)	16✔
235		16✔
236	// Static route to steer packets from external to nodePort service backed by pods on hybrid overlay node.	16✔
237	// This route is to used for triggering above route policy	16✔
238	clusterRouterStaticRoutes := nbdb.LogicalRouterStaticRoute{	16✔
239	IPPrefix: hybridCIDR.String(),	16✔
240	Nexthop: drIP.String(),	16✔
241	ExternalIDs: map[string]string{	16✔
242	"name": ovntypes.HybridSubnetPrefix + nodeName,	16✔
243	},	16✔
244	}	16✔
245	if err := libovsdbops.CreateOrReplaceLogicalRouterStaticRouteWithPredicate(oc.nbClient,	16✔
246	ovntypes.OVNClusterRouter, &clusterRouterStaticRoutes,	16✔
247	func(item *nbdb.LogicalRouterStaticRoute) bool {	61✔
248	return item.IPPrefix == clusterRouterStaticRoutes.IPPrefix &&	45✔
249	item.ExternalIDs["name"] == clusterRouterStaticRoutes.ExternalIDs["name"] &&	45✔
250	libovsdbops.PolicyEqualPredicate(clusterRouterStaticRoutes.Policy, item.Policy)	45✔
251	}, &clusterRouterStaticRoutes.Nexthop); err != nil {	45✔
252	return fmt.Errorf("failed to add policy route static '%s %s' for on %s , error: %v",	×
253	clusterRouterStaticRoutes.IPPrefix, clusterRouterStaticRoutes.Nexthop,	×
254	ovntypes.GWRouterPrefix+nodeName, err)	×
255	}	×
256	klog.Infof("Created hybrid overlay logical route static route at cluster router for node %s", nodeName)	16✔
257		16✔
258	// Static route to steer packets from external to nodePort service backed by pods on hybrid overlay node to cluster router.	16✔
259	drLRPIfAddrs, err := libovsdbutil.GetLRPAddrs(oc.nbClient, ovntypes.GWRouterToJoinSwitchPrefix+ovntypes.OVNClusterRouter)	16✔
260	if err != nil {	16✔
261	return err	×
262	}	×
263	drLRPIfAddr, err := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(hybridCIDR), drLRPIfAddrs)	16✔
264	if err != nil {	16✔
265	return fmt.Errorf("failed to match cluster router join interface IPs: %v, err: %v", drLRPIfAddr, err)	×
266	}	×
267	nodeGWRouterStaticRoutes := nbdb.LogicalRouterStaticRoute{	16✔
268	IPPrefix: hybridCIDR.String(),	16✔
269	Nexthop: drLRPIfAddr.IP.String(),	16✔
270	ExternalIDs: map[string]string{	16✔
271	"name": ovntypes.HybridSubnetPrefix + nodeName + "-gr",	16✔
272	},	16✔
273	}	16✔
274	if err := libovsdbops.CreateOrReplaceLogicalRouterStaticRouteWithPredicate(oc.nbClient,	16✔
275	ovntypes.GWRouterPrefix+nodeName, &nodeGWRouterStaticRoutes,	16✔
276	func(item *nbdb.LogicalRouterStaticRoute) bool {	77✔
277	return item.IPPrefix == nodeGWRouterStaticRoutes.IPPrefix &&	61✔
278	item.ExternalIDs["name"] == nodeGWRouterStaticRoutes.ExternalIDs["name"] &&	61✔
279	libovsdbops.PolicyEqualPredicate(nodeGWRouterStaticRoutes.Policy, item.Policy)	61✔
280	}, &nodeGWRouterStaticRoutes.Nexthop); err != nil {	61✔
281	return fmt.Errorf("failed to add policy route static '%s %s' for on %s , error: %v",	×
282	nodeGWRouterStaticRoutes.IPPrefix, nodeGWRouterStaticRoutes.Nexthop,	×
283	ovntypes.GWRouterPrefix+nodeName, err)	×
284	}	×
285	klog.Infof("Created hybrid overlay logical route static route at gateway router for node %s", nodeName)	16✔
286	}
287	}
288	return nil	16✔
289	}
290
291	func (oc DefaultNetworkController) removeHybridLRPolicySharedGW(node kapi.Node) error {	2✔
292	nodeName := node.Name	2✔
293	name := ovntypes.HybridSubnetPrefix + nodeName	2✔
294		2✔
295	if err := libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, func(item *nbdb.LogicalRouterPolicy) bool {	10✔
296	return item.ExternalIDs["name"] == ovntypes.HybridSubnetPrefix+nodeName \|\| item.ExternalIDs["name"] == ovntypes.HybridSubnetPrefix+nodeName+"-gr"	8✔
297	}); err != nil {	8✔
298	return fmt.Errorf("failed to delete policy %s from %s, error: %v", name, ovntypes.OVNClusterRouter, err)	×
299	}	×
300
301	if err := libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicate(oc.nbClient, ovntypes.OVNClusterRouter, func(item *nbdb.LogicalRouterStaticRoute) bool {	16✔
302	return item.ExternalIDs["name"] == name	14✔
303	}); err != nil {	14✔
304	return fmt.Errorf("failed to delete static route %s from %s, error: %v", name, ovntypes.OVNClusterRouter, err)	×
305	}	×
306	if err := libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicate(oc.nbClient, ovntypes.GWRouterPrefix+nodeName, func(item *nbdb.LogicalRouterStaticRoute) bool {	14✔
307	return item.ExternalIDs["name"] == name+"-gr"	12✔
308	}); err != nil {	12✔
309	return fmt.Errorf("failed to delete static route %s from %s, error: %v", name+"gr", ovntypes.GWRouterPrefix+nodeName, err)	×
310	}	×
311
312	if node.Annotations[hotypes.HybridOverlayDRIP] != "" {	4✔
313	smb := &nbdb.StaticMACBinding{	2✔
314	IP: node.Annotations[hotypes.HybridOverlayDRIP],	2✔
315	LogicalPort: ovntypes.RouterToSwitchPrefix + nodeName,	2✔
316	}	2✔
317	err := libovsdbops.DeleteStaticMacBindings(oc.nbClient, smb)	2✔
318	if err != nil {	2✔
319	return fmt.Errorf("failed to delete static mac binding %+v: %v", smb, err)	×
320	}	×
321	}
322
323	return nil	2✔
324	}
325
326	// takes the node name and allocates the hybrid overlay distributed router ip address
327	func (oc DefaultNetworkController) allocateHybridOverlayDRIP(node kapi.Node) error {	4✔
328	if atomic.LoadUint32(&oc.allInitialPodsProcessed) == 0 {	4✔
UNCOV 329	return fmt.Errorf("cannot allocate hybrid overlay distributed router ip for nodes until all initial pods are processed")	×
UNCOV 330	}	×
331
332	var ips []string	4✔
333	// check if there is a hybrid distributed router IP set as an annotation	4✔
334	nodeHybridOverlayDRIP := node.Annotations[hotypes.HybridOverlayDRIP]	4✔
335	if len(nodeHybridOverlayDRIP) > 0 {	4✔
336	ips = strings.Split(nodeHybridOverlayDRIP, ",")	×
337	}	×
338
339	allocatedHybridOverlayDRIPs, err := oc.lsManager.AllocateHybridOverlay(node.Name, ips)	4✔
340	if err != nil {	4✔
341	return fmt.Errorf("cannot allocate hybrid overlay interface addresses on node %s: %v", node.Name, err)	×
342	}	×
343
344	var sliceHybridOverlayDRIP []string	4✔
345	for _, ip := range allocatedHybridOverlayDRIPs {	8✔
346	sliceHybridOverlayDRIP = append(sliceHybridOverlayDRIP, ip.IP.String())	4✔
347	}	4✔
348
349	err = oc.kube.SetAnnotationsOnNode(node.Name, map[string]interface{}{hotypes.HybridOverlayDRIP: strings.Join(sliceHybridOverlayDRIP, ",")})	4✔
350	if err != nil {	4✔
351	return fmt.Errorf("cannot set hybrid annotation on node %s: %v", node.Name, err)	×
352	}	×
353
354	return nil	4✔
355	}

ovn-org / ovn-kubernetes / 6837407813

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous