22306573215

Committed 23 Feb 2026 12:42PM UTC coverage: 23.057% (+0.05%) from 23.003%

Build # 22306573215

Build Type

Pull #6311

github

Committed by

oujonny

Commit Message

fix: remove duplicate check if chart changed

Signed-off-by: oujonny <jonny@immerda.ch>

Pull Request Pull Request #6311: feat: check for pod restarts after installation

Coverage Stats

12533 of 54357 relevant lines covered (23.06%)

0.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

6.13

/pkg/controller/ip.go

package controller

import (
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "maps"
        "net"
        "reflect"
        "slices"
        "strings"

        corev1 "k8s.io/api/core/v1"
        k8serrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog/v2"
        "k8s.io/utils/ptr"
        "sigs.k8s.io/controller-runtime/pkg/client"
        "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

        kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
        "github.com/kubeovn/kube-ovn/pkg/ovs"
        "github.com/kubeovn/kube-ovn/pkg/util"
)

func (c *Controller) enqueueAddIP(obj any) {
        ipObj := obj.(*kubeovnv1.IP)
        if strings.HasPrefix(ipObj.Name, util.U2OInterconnName[0:20]) ||
                strings.HasPrefix(ipObj.Name, util.McastQuerierName[0:14]) {
                return
        }
        klog.V(3).Infof("enqueue update status subnet %s", ipObj.Spec.Subnet)
        c.updateSubnetStatusQueue.Add(ipObj.Spec.Subnet)
        for _, as := range ipObj.Spec.AttachSubnets {
                klog.V(3).Infof("enqueue update attach status for subnet %s", as)
                c.updateSubnetStatusQueue.Add(as)
        }

        key := cache.MetaObjectToName(ipObj).String()
        klog.V(3).Infof("enqueue add ip %s", key)
        c.addIPQueue.Add(key)
}

func (c *Controller) enqueueUpdateIP(oldObj, newObj any) {
        oldIP := oldObj.(*kubeovnv1.IP)
        newIP := newObj.(*kubeovnv1.IP)
        // ip can not change these specs below
        if oldIP.Spec.Subnet != "" && newIP.Spec.Subnet != oldIP.Spec.Subnet {
                klog.Errorf("ip %s subnet can not change", newIP.Name)
                return
        }
        if oldIP.Spec.Namespace != "" && newIP.Spec.Namespace != oldIP.Spec.Namespace {
                klog.Errorf("ip %s namespace can not change", newIP.Name)
                return
        }
        if oldIP.Spec.PodName != "" && newIP.Spec.PodName != oldIP.Spec.PodName {
                klog.Errorf("ip %s podName can not change", newIP.Name)
                return
        }
        if oldIP.Spec.PodType != "" && newIP.Spec.PodType != oldIP.Spec.PodType {
                klog.Errorf("ip %s podType can not change", newIP.Name)
                return
        }
        if oldIP.Spec.MacAddress != "" && newIP.Spec.MacAddress != oldIP.Spec.MacAddress {
                klog.Errorf("ip %s macAddress can not change", newIP.Name)
                return
        }
        if oldIP.Spec.V4IPAddress != "" && newIP.Spec.V4IPAddress != oldIP.Spec.V4IPAddress {
                klog.Errorf("ip %s v4IPAddress can not change", newIP.Name)
                return
        }
        if oldIP.Spec.V6IPAddress != "" {
                // v6 ip address can not use upper case
                if util.ContainsUppercase(newIP.Spec.V6IPAddress) {
                        err := fmt.Errorf("ip %s v6 ip address %s can not contain upper case", newIP.Name, newIP.Spec.V6IPAddress)
                        klog.Error(err)
                        return
                }
                if newIP.Spec.V6IPAddress != oldIP.Spec.V6IPAddress {
                        klog.Errorf("ip %s v6IPAddress can not change", newIP.Name)
                        return
                }
        }
        if !newIP.DeletionTimestamp.IsZero() {
                key := cache.MetaObjectToName(newIP).String()
                klog.V(3).Infof("enqueue update ip %s", key)
                c.updateIPQueue.Add(key)
                return
        }
        if !slices.Equal(oldIP.Spec.AttachSubnets, newIP.Spec.AttachSubnets) {
                klog.V(3).Infof("enqueue update status subnet %s", newIP.Spec.Subnet)
                for _, as := range newIP.Spec.AttachSubnets {
                        klog.V(3).Infof("enqueue update status for attach subnet %s", as)
                        c.updateSubnetStatusQueue.Add(as)
                }
        }
}

func (c *Controller) enqueueDelIP(obj any) {
        var ipObj *kubeovnv1.IP
        switch t := obj.(type) {
        case *kubeovnv1.IP:
                ipObj = t
        case cache.DeletedFinalStateUnknown:
                ip, ok := t.Obj.(*kubeovnv1.IP)
                if !ok {
                        klog.Warningf("unexpected object type: %T", t.Obj)
                        return
                }
                ipObj = ip
        default:
                klog.Warningf("unexpected type: %T", obj)
                return
        }

        if strings.HasPrefix(ipObj.Name, util.U2OInterconnName[0:20]) ||
                strings.HasPrefix(ipObj.Name, util.McastQuerierName[0:14]) {
                return
        }

        key := cache.MetaObjectToName(ipObj).String()
        klog.V(3).Infof("enqueue del ip %s", key)
        c.delIPQueue.Add(ipObj)
}

func (c *Controller) handleAddReservedIP(key string) error {
        ip, err := c.ipsLister.Get(key)
        if err != nil {
                if k8serrors.IsNotFound(err) {
                        return nil
                }
                klog.Error(err)
                return err
        }
        if !ip.DeletionTimestamp.IsZero() {
                klog.Infof("handle add process stop for deleting ip %s", ip.Name)
                return nil
        }
        if len(ip.Finalizers) != 0 {
                // finalizer already added, no need to handle it again
                return nil
        }

        klog.V(3).Infof("handle add reserved ip %s", ip.Name)

        if ip.Spec.Subnet == "" {
                err := errors.New("subnet parameter cannot be empty")
                klog.Error(err)
                return err
        }

        if ip.Spec.PodType != "" &&
                ip.Spec.PodType != util.KindVirtualMachine &&
                ip.Spec.PodType != util.KindStatefulSet {
                err := fmt.Errorf("podType %s is not supported", ip.Spec.PodType)
                klog.Error(err)
                return err
        }

        subnet, err := c.subnetsLister.Get(ip.Spec.Subnet)
        if err != nil {
                err = fmt.Errorf("failed to get subnet %s: %w", ip.Spec.Subnet, err)
                klog.Error(err)
                return err
        }

        portName := ovs.PodNameToPortName(ip.Spec.PodName, ip.Spec.Namespace, subnet.Spec.Provider)
        if portName != ip.Name {
                // invalid ip or node ip, no need to handle it here
                klog.V(3).Infof("port name %s is not equal to ip name %s", portName, ip.Name)
                return nil
        }

        // not handle add the ip, which created in pod process, lsp created before ip
        lsp, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true)
        if err != nil {
                klog.Errorf("failed to list logical switch ports %s, %v", portName, err)
                return err
        }
        if lsp != nil {
                // port already exists means the ip already created
                klog.V(3).Infof("ip %s is ready", portName)
                return nil
        }

        // v6 ip address can not use upper case
        if util.ContainsUppercase(ip.Spec.V6IPAddress) {
                err := fmt.Errorf("ip %s v6 ip address %s can not contain upper case", ip.Name, ip.Spec.V6IPAddress)
                klog.Error(err)
                return err
        }
        v4IP, v6IP, mac, err := c.ipAcquireAddress(ip, subnet)
        if err != nil {
                err = fmt.Errorf("failed to acquire ip address %w", err)
                klog.Error(err)
                return err
        }
        ipStr := util.GetStringIP(v4IP, v6IP)
        if err := c.createOrUpdateIPCR(ip.Name, ip.Spec.PodName, ipStr, mac, subnet.Name, ip.Spec.Namespace, ip.Spec.NodeName, ip.Spec.PodType); err != nil {
                err = fmt.Errorf("failed to create ips CR %s.%s: %w", ip.Spec.PodName, ip.Spec.Namespace, err)
                klog.Error(err)
                return err
        }

        ip = ip.DeepCopy()
        if ip.Labels == nil {
                ip.Labels = map[string]string{}
        }
        if ip.Labels[util.IPReservedLabel] != "false" {
                ip.Labels[util.IPReservedLabel] = "true"
                patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]`
                raw, err := json.Marshal(ip.Labels)
                if err != nil {
                        klog.Error(err)
                        return err
                }
                op := "replace"
                patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
                if _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Patch(context.Background(), ip.Name,
                        types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}); err != nil {
                        klog.Errorf("failed to patch label for ip %s, %v", ip.Name, err)
                        return err
                }
        }
        return nil
}

func (c *Controller) handleUpdateIP(key string) error {
        cachedIP, err := c.ipsLister.Get(key)
        if err != nil {
                if k8serrors.IsNotFound(err) {
                        return nil
                }
                klog.Error(err)
                return err
        }
        if !cachedIP.DeletionTimestamp.IsZero() {
                klog.Infof("handle deleting ip %s", cachedIP.Name)
                subnet, err := c.subnetsLister.Get(cachedIP.Spec.Subnet)
                if err != nil {
                        if !k8serrors.IsNotFound(err) {
                                klog.Errorf("failed to get subnet %s: %v", cachedIP.Spec.Subnet, err)
                                return err
                        }
                        // subnet not found, but ip exists, check if the ip is u2o ip or mcast querier ip
                        // if yes, remove finalizer to let ip be deleted
                        klog.Warningf("subnet %s not found for deleting ip %s", cachedIP.Spec.Subnet, cachedIP.Name)
                        if strings.HasPrefix(cachedIP.Name, util.U2OInterconnName[0:20]) ||
                                strings.HasPrefix(cachedIP.Name, util.McastQuerierName[0:14]) {
                                if err = c.handleDelIPFinalizer(cachedIP); err != nil {
                                        klog.Errorf("failed to remove finalizer for deleting ip %s: %v", cachedIP.Name, err)
                                        return err
                                }
                                return nil
                        }
                }
                portName := cachedIP.Name
                if isOvnSubnet(subnet) {
                        port, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true)
                        if err != nil {
                                klog.Errorf("failed to get logical switch port %s: %v", portName, err)
                                return err
                        }
                        if port != nil {
                                klog.Infof("delete ip cr lsp %s from switch %s", portName, subnet.Name)
                                if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil {
                                        klog.Errorf("failed to delete ip cr lsp %s from switch %s: %v", portName, subnet.Name, err)
                                        return err
                                }
                                klog.V(3).Infof("sync sg for deleted port %s", portName)
                                sgList, err := c.getPortSg(port)
                                if err != nil {
                                        klog.Errorf("get port sg failed, %v", err)
                                        return err
                                }
                                for _, sgName := range sgList {
                                        if sgName != "" {
                                                c.syncSgPortsQueue.Add(sgName)
                                        }
                                }
                        }
                }
                podKey := fmt.Sprintf("%s/%s", cachedIP.Spec.Namespace, cachedIP.Spec.PodName)
                klog.Infof("ip cr %s release ipam pod key %s from subnet %s", cachedIP.Name, podKey, cachedIP.Spec.Subnet)
                c.ipam.ReleaseAddressByNic(podKey, portName, cachedIP.Spec.Subnet)
                if err = c.handleDelIPFinalizer(cachedIP); err != nil {
                        klog.Errorf("failed to handle del ip finalizer %v", err)
                        return err
                }
                c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet)
        }
        return nil
}

func (c *Controller) handleDelIP(ip *kubeovnv1.IP) error {
        klog.Infof("deleting ip %s enqueue update status subnet %s", ip.Name, ip.Spec.Subnet)
        c.updateSubnetStatusQueue.Add(ip.Spec.Subnet)
        for _, as := range ip.Spec.AttachSubnets {
                klog.V(3).Infof("enqueue update attach status for subnet %s", as)
                c.updateSubnetStatusQueue.Add(as)
        }
        return nil
}

func (c *Controller) syncIPFinalizer(cl client.Client) error {
        // migrate deprecated finalizer to new finalizer
        ips := &kubeovnv1.IPList{}
        return migrateFinalizers(cl, ips, func(i int) (client.Object, client.Object) {
                if i < 0 || i >= len(ips.Items) {
                        return nil, nil
                }
                return ips.Items[i].DeepCopy(), ips.Items[i].DeepCopy()
        })
}

func (c *Controller) handleDelIPFinalizer(cachedIP *kubeovnv1.IP) error {
        if len(cachedIP.GetFinalizers()) == 0 {
                return nil
        }
        newIP := cachedIP.DeepCopy()
        controllerutil.RemoveFinalizer(newIP, util.DeprecatedFinalizerName)
        controllerutil.RemoveFinalizer(newIP, util.KubeOVNControllerFinalizer)
        patch, err := util.GenerateMergePatchPayload(cachedIP, newIP)
        if err != nil {
                klog.Errorf("failed to generate patch payload for ip %s, %v", cachedIP.Name, err)
                return err
        }
        if _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Patch(context.Background(), cachedIP.Name,
                types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
                if k8serrors.IsNotFound(err) {
                        return nil
                }
                klog.Errorf("failed to remove finalizer from ip %s, %v", cachedIP.Name, err)
                return err
        }
        return nil
}

func (c *Controller) acquireIPAddress(subnetName, name, nicName string) (string, string, string, error) {
        var skippedAddrs []string
        var v4ip, v6ip, mac string
        checkConflict := true
        var err error
        for {
                v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, nil, subnetName, "", skippedAddrs, checkConflict)
                if err != nil {
                        klog.Error(err)
                        return "", "", "", err
                }

                ipv4OK, ipv6OK, err := c.validatePodIP(name, subnetName, v4ip, v6ip)
                if err != nil {
                        klog.Error(err)
                        return "", "", "", err
                }

                if ipv4OK && ipv6OK {
                        return v4ip, v6ip, mac, nil
                }

                if !ipv4OK {
                        skippedAddrs = append(skippedAddrs, v4ip)
                }
                if !ipv6OK {
                        skippedAddrs = append(skippedAddrs, v6ip)
                }
        }
}

func (c *Controller) acquireStaticIPAddress(subnetName, name, nicName, ip string, macPointer *string) (string, string, string, error) {
        checkConflict := true
        var v4ip, v6ip, mac string
        var err error
        for ipStr := range strings.SplitSeq(ip, ",") {
                if net.ParseIP(ipStr) == nil {
                        return "", "", "", fmt.Errorf("failed to parse vip ip %s", ipStr)
                }
        }

        if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, macPointer, subnetName, checkConflict); err != nil {
                klog.Errorf("failed to get static virtual ip '%s', mac '%s', subnet '%s', %v", ip, mac, subnetName, err)
                return "", "", "", err
        }
        return v4ip, v6ip, mac, nil
}

func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, ns, nodeName, podType string) error {
        // `ipCRName`: pod or vm IP name must set ip CR name when creating ip CR
        var key, ipName string
        var owner *metav1.OwnerReference
        if ipCRName != "" {
                // pod IP
                key = podName
                ipName = ipCRName
        } else {
                // node IP, u2o IP or mcast querier IP
                switch {
                case subnetName == c.config.NodeSwitch:
                        node, err := c.nodesLister.Get(nodeName)
                        if err != nil {
                                err = fmt.Errorf("failed to get node %s: %w", nodeName, err)
                                klog.Error(err)
                                return err
                        }
                        key = nodeName
                        ipName = util.NodeLspName(nodeName)
                        owner = &metav1.OwnerReference{
                                APIVersion: corev1.SchemeGroupVersion.String(),
                                Kind:       util.KindNode,
                                Name:       nodeName,
                                UID:        node.UID,
                        }
                case strings.HasPrefix(podName, util.U2OInterconnName[0:20]) || strings.HasPrefix(podName, util.McastQuerierName[0:14]):
                        // u2o IP or mcast querier IP
                        subnet, err := c.subnetsLister.Get(subnetName)
                        if err != nil {
                                err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err)
                                klog.Error(err)
                                return err
                        }
                        key = podName
                        ipName = podName
                        owner = &metav1.OwnerReference{
                                APIVersion:         kubeovnv1.SchemeGroupVersion.String(),
                                Kind:               util.KindSubnet,
                                Name:               subnetName,
                                UID:                subnet.UID,
                                BlockOwnerDeletion: ptr.To(true),
                        }
                }
        }

        var err error
        var ipCR *kubeovnv1.IP
        ipCR, err = c.ipsLister.Get(ipName)
        if err != nil {
                if !k8serrors.IsNotFound(err) {
                        err := fmt.Errorf("failed to get ip CR %s: %w", ipName, err)
                        klog.Error(err)
                        return err
                }
                // the returned pointer is not nil if the CR does not exist
                ipCR = nil
        }
        if ipCR != nil && !ipCR.DeletionTimestamp.IsZero() {
                // this ip is being deleted, no need to update
                klog.Infof("enqueue update for removing finalizer to delete ip %s", ipCR.Name)
                c.updateIPQueue.Add(ipCR.Name)
                return nil
        }
        v4IP, v6IP := util.SplitStringIP(ip)
        if ipCR == nil {
                ipCR = &kubeovnv1.IP{
                        ObjectMeta: metav1.ObjectMeta{
                                Name: ipName,
                                Labels: map[string]string{
                                        util.SubnetNameLabel: subnetName,
                                        util.NodeNameLabel:   nodeName,
                                        subnetName:           "",
                                        util.IPReservedLabel: "false", // ip create with pod or node, ip not reserved
                                },
                                Finalizers: []string{util.KubeOVNControllerFinalizer},
                        },
                        Spec: kubeovnv1.IPSpec{
                                PodName:       key,
                                Subnet:        subnetName,
                                NodeName:      nodeName,
                                Namespace:     ns,
                                IPAddress:     ip,
                                V4IPAddress:   v4IP,
                                V6IPAddress:   v6IP,
                                MacAddress:    mac,
                                AttachIPs:     []string{},
                                AttachMacs:    []string{},
                                AttachSubnets: []string{},
                                PodType:       podType,
                        },
                }
                if owner != nil {
                        ipCR.OwnerReferences = []metav1.OwnerReference{*owner}
                }
                if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), ipCR, metav1.CreateOptions{}); err != nil {
                        errMsg := fmt.Errorf("failed to create ip CR %s: %w", ipName, err)
                        klog.Error(errMsg)
                        return errMsg
                }
        } else {
                newIPCR := ipCR.DeepCopy()
                if newIPCR.Labels != nil {
                        newIPCR.Labels[util.SubnetNameLabel] = subnetName
                        newIPCR.Labels[util.NodeNameLabel] = nodeName
                } else {
                        newIPCR.Labels = map[string]string{
                                util.SubnetNameLabel: subnetName,
                                util.NodeNameLabel:   nodeName,
                        }
                        // update not touch IP Reserved Label
                }
                if owner != nil {
                        // currently we only set owner for node IP, u2o IP and mcast querier ip,
                        // so it's ok to overwrite it here
                        newIPCR.OwnerReferences = []metav1.OwnerReference{*owner}
                }
                newIPCR.Spec.PodName = key
                newIPCR.Spec.Namespace = ns
                newIPCR.Spec.Subnet = subnetName
                newIPCR.Spec.NodeName = nodeName
                newIPCR.Spec.IPAddress = ip
                newIPCR.Spec.V4IPAddress = v4IP
                newIPCR.Spec.V6IPAddress = v6IP
                newIPCR.Spec.MacAddress = mac
                newIPCR.Spec.AttachIPs = []string{}
                newIPCR.Spec.AttachMacs = []string{}
                newIPCR.Spec.AttachSubnets = []string{}
                newIPCR.Spec.PodType = podType
                if maps.Equal(newIPCR.Labels, ipCR.Labels) && reflect.DeepEqual(newIPCR.Spec, ipCR.Spec) {
                        return nil
                }
                controllerutil.AddFinalizer(newIPCR, util.KubeOVNControllerFinalizer)

                if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), newIPCR, metav1.UpdateOptions{}); err != nil {
                        err := fmt.Errorf("failed to update ip CR %s: %w", ipCRName, err)
                        klog.Error(err)
                        return err
                }
        }
        return nil
}

func (c *Controller) ipAcquireAddress(ip *kubeovnv1.IP, subnet *kubeovnv1.Subnet) (string, string, string, error) {
        key := cache.NewObjectName(ip.Spec.Namespace, ip.Spec.PodName).String()
        portName := ovs.PodNameToPortName(ip.Spec.PodName, ip.Spec.Namespace, subnet.Spec.Provider)
        ipStr := util.GetStringIP(ip.Spec.V4IPAddress, ip.Spec.V6IPAddress)

        var v4IP, v6IP, mac string
        var err error
        var macPtr *string
        if isOvnSubnet(subnet) {
                if ip.Spec.MacAddress != "" {
                        macPtr = &ip.Spec.MacAddress
                }
        } else {
                macPtr = ptr.To("")
        }

        if ipStr == "" {
                // allocate address
                v4IP, v6IP, mac, err = c.acquireIPAddress(subnet.Name, ip.Name, portName)
                if err == nil {
                        return v4IP, v6IP, mac, err
                }
                err = fmt.Errorf("failed to get random address for ip %s, %w", ip.Name, err)
        } else {
                // static address
                v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipStr, macPtr, subnet.Name, true)
                if err == nil {
                        return v4IP, v6IP, mac, nil
                }
                err = fmt.Errorf("failed to get static address for ip %s, %w", ip.Name, err)
        }
        klog.Error(err)
        return "", "", "", err
}

1	package controller
2
3	import (
4	"context"
5	"encoding/json"
6	"errors"
7	"fmt"
8	"maps"
9	"net"
10	"reflect"
11	"slices"
12	"strings"
13
14	corev1 "k8s.io/api/core/v1"
15	k8serrors "k8s.io/apimachinery/pkg/api/errors"
16	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17	"k8s.io/apimachinery/pkg/types"
18	"k8s.io/client-go/tools/cache"
19	"k8s.io/klog/v2"
20	"k8s.io/utils/ptr"
21	"sigs.k8s.io/controller-runtime/pkg/client"
22	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
23
24	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
25	"github.com/kubeovn/kube-ovn/pkg/ovs"
26	"github.com/kubeovn/kube-ovn/pkg/util"
27	)
28		×
29	func (c *Controller) enqueueAddIP(obj any) {	×
30	ipObj := obj.(*kubeovnv1.IP)	×
31	if strings.HasPrefix(ipObj.Name, util.U2OInterconnName[0:20]) \|\|	×
32	strings.HasPrefix(ipObj.Name, util.McastQuerierName[0:14]) {	×
33	return	×
34	}	×
35	klog.V(3).Infof("enqueue update status subnet %s", ipObj.Spec.Subnet)	×
36	c.updateSubnetStatusQueue.Add(ipObj.Spec.Subnet)	×
37	for _, as := range ipObj.Spec.AttachSubnets {	×
38	klog.V(3).Infof("enqueue update attach status for subnet %s", as)	×
39	c.updateSubnetStatusQueue.Add(as)	×
40	}
41		×
42	key := cache.MetaObjectToName(ipObj).String()	×
43	klog.V(3).Infof("enqueue add ip %s", key)	×
44	c.addIPQueue.Add(key)
45	}
46		×
47	func (c *Controller) enqueueUpdateIP(oldObj, newObj any) {	×
48	oldIP := oldObj.(*kubeovnv1.IP)	×
49	newIP := newObj.(*kubeovnv1.IP)	×
50	// ip can not change these specs below	×
51	if oldIP.Spec.Subnet != "" && newIP.Spec.Subnet != oldIP.Spec.Subnet {	×
52	klog.Errorf("ip %s subnet can not change", newIP.Name)	×
53	return	×
54	}	×
55	if oldIP.Spec.Namespace != "" && newIP.Spec.Namespace != oldIP.Spec.Namespace {	×
56	klog.Errorf("ip %s namespace can not change", newIP.Name)	×
57	return	×
58	}	×
59	if oldIP.Spec.PodName != "" && newIP.Spec.PodName != oldIP.Spec.PodName {	×
60	klog.Errorf("ip %s podName can not change", newIP.Name)	×
61	return	×
62	}	×
63	if oldIP.Spec.PodType != "" && newIP.Spec.PodType != oldIP.Spec.PodType {	×
64	klog.Errorf("ip %s podType can not change", newIP.Name)	×
65	return	×
66	}	×
67	if oldIP.Spec.MacAddress != "" && newIP.Spec.MacAddress != oldIP.Spec.MacAddress {	×
68	klog.Errorf("ip %s macAddress can not change", newIP.Name)	×
69	return	×
70	}	×
71	if oldIP.Spec.V4IPAddress != "" && newIP.Spec.V4IPAddress != oldIP.Spec.V4IPAddress {	×
72	klog.Errorf("ip %s v4IPAddress can not change", newIP.Name)	×
73	return	×
74	}	×
75	if oldIP.Spec.V6IPAddress != "" {	×
76	// v6 ip address can not use upper case	×
77	if util.ContainsUppercase(newIP.Spec.V6IPAddress) {	×
78	err := fmt.Errorf("ip %s v6 ip address %s can not contain upper case", newIP.Name, newIP.Spec.V6IPAddress)	×
79	klog.Error(err)	×
80	return	×
81	}	×
82	if newIP.Spec.V6IPAddress != oldIP.Spec.V6IPAddress {	×
83	klog.Errorf("ip %s v6IPAddress can not change", newIP.Name)	×
84	return	×
85	}
86	}	×
87	if !newIP.DeletionTimestamp.IsZero() {	×
88	key := cache.MetaObjectToName(newIP).String()	×
89	klog.V(3).Infof("enqueue update ip %s", key)	×
90	c.updateIPQueue.Add(key)	×
91	return	×
92	}	×
93	if !slices.Equal(oldIP.Spec.AttachSubnets, newIP.Spec.AttachSubnets) {	×
94	klog.V(3).Infof("enqueue update status subnet %s", newIP.Spec.Subnet)	×
95	for _, as := range newIP.Spec.AttachSubnets {	×
96	klog.V(3).Infof("enqueue update status for attach subnet %s", as)	×
97	c.updateSubnetStatusQueue.Add(as)	×
98	}
99	}
100	}
101		×
102	func (c *Controller) enqueueDelIP(obj any) {	×
103	var ipObj *kubeovnv1.IP	×
104	switch t := obj.(type) {	×
105	case *kubeovnv1.IP:	×
106	ipObj = t	×
107	case cache.DeletedFinalStateUnknown:	×
108	ip, ok := t.Obj.(*kubeovnv1.IP)	×
109	if !ok {	×
110	klog.Warningf("unexpected object type: %T", t.Obj)	×
111	return	×
112	}	×
113	ipObj = ip	×
114	default:	×
115	klog.Warningf("unexpected type: %T", obj)	×
116	return
117	}
118		×
119	if strings.HasPrefix(ipObj.Name, util.U2OInterconnName[0:20]) \|\|	×
120	strings.HasPrefix(ipObj.Name, util.McastQuerierName[0:14]) {	×
121	return	×
122	}
123		×
124	key := cache.MetaObjectToName(ipObj).String()	×
125	klog.V(3).Infof("enqueue del ip %s", key)	×
126	c.delIPQueue.Add(ipObj)
127	}
128		×
129	func (c *Controller) handleAddReservedIP(key string) error {	×
130	ip, err := c.ipsLister.Get(key)	×
131	if err != nil {	×
132	if k8serrors.IsNotFound(err) {	×
133	return nil	×
134	}	×
135	klog.Error(err)	×
136	return err
137	}	×
138	if !ip.DeletionTimestamp.IsZero() {	×
139	klog.Infof("handle add process stop for deleting ip %s", ip.Name)	×
140	return nil	×
141	}	×
142	if len(ip.Finalizers) != 0 {	×
143	// finalizer already added, no need to handle it again	×
144	return nil	×
145	}
146		×
147	klog.V(3).Infof("handle add reserved ip %s", ip.Name)	×
148		×
149	if ip.Spec.Subnet == "" {	×
150	err := errors.New("subnet parameter cannot be empty")	×
151	klog.Error(err)	×
152	return err	×
153	}
154		×
155	if ip.Spec.PodType != "" &&	×
156	ip.Spec.PodType != util.KindVirtualMachine &&	×
157	ip.Spec.PodType != util.KindStatefulSet {	×
158	err := fmt.Errorf("podType %s is not supported", ip.Spec.PodType)	×
159	klog.Error(err)	×
160	return err	×
161	}
162		×
163	subnet, err := c.subnetsLister.Get(ip.Spec.Subnet)	×
164	if err != nil {	×
165	err = fmt.Errorf("failed to get subnet %s: %w", ip.Spec.Subnet, err)	×
166	klog.Error(err)	×
167	return err	×
168	}
169		×
170	portName := ovs.PodNameToPortName(ip.Spec.PodName, ip.Spec.Namespace, subnet.Spec.Provider)	×
171	if portName != ip.Name {	×
172	// invalid ip or node ip, no need to handle it here	×
173	klog.V(3).Infof("port name %s is not equal to ip name %s", portName, ip.Name)	×
174	return nil	×
175	}
176
177	// not handle add the ip, which created in pod process, lsp created before ip	×
178	lsp, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true)	×
179	if err != nil {	×
180	klog.Errorf("failed to list logical switch ports %s, %v", portName, err)	×
181	return err	×
182	}	×
183	if lsp != nil {	×
184	// port already exists means the ip already created	×
185	klog.V(3).Infof("ip %s is ready", portName)	×
186	return nil	×
187	}
188
189	// v6 ip address can not use upper case	×
190	if util.ContainsUppercase(ip.Spec.V6IPAddress) {	×
191	err := fmt.Errorf("ip %s v6 ip address %s can not contain upper case", ip.Name, ip.Spec.V6IPAddress)	×
192	klog.Error(err)	×
193	return err	×
194	}	×
195	v4IP, v6IP, mac, err := c.ipAcquireAddress(ip, subnet)	×
196	if err != nil {	×
197	err = fmt.Errorf("failed to acquire ip address %w", err)	×
198	klog.Error(err)	×
199	return err	×
200	}	×
201	ipStr := util.GetStringIP(v4IP, v6IP)	×
202	if err := c.createOrUpdateIPCR(ip.Name, ip.Spec.PodName, ipStr, mac, subnet.Name, ip.Spec.Namespace, ip.Spec.NodeName, ip.Spec.PodType); err != nil {	×
203	err = fmt.Errorf("failed to create ips CR %s.%s: %w", ip.Spec.PodName, ip.Spec.Namespace, err)	×
204	klog.Error(err)	×
205	return err	×
206	}
207		×
208	ip = ip.DeepCopy()	×
209	if ip.Labels == nil {	×
210	ip.Labels = map[string]string{}	×
211	}	×
212	if ip.Labels[util.IPReservedLabel] != "false" {	×
213	ip.Labels[util.IPReservedLabel] = "true"	×
214	patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]`	×
215	raw, err := json.Marshal(ip.Labels)	×
216	if err != nil {	×
217	klog.Error(err)	×
218	return err	×
219	}	×
220	op := "replace"	×
221	patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)	×
222	if _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Patch(context.Background(), ip.Name,	×
223	types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}); err != nil {	×
224	klog.Errorf("failed to patch label for ip %s, %v", ip.Name, err)	×
225	return err	×
226	}
227	}	×
228	return nil
229	}
230		1✔
231	func (c *Controller) handleUpdateIP(key string) error {	1✔
232	cachedIP, err := c.ipsLister.Get(key)	1✔
233	if err != nil {	×
234	if k8serrors.IsNotFound(err) {	×
235	return nil	×
236	}	×
237	klog.Error(err)	×
238	return err
239	}	2✔
240	if !cachedIP.DeletionTimestamp.IsZero() {	1✔
241	klog.Infof("handle deleting ip %s", cachedIP.Name)	1✔
242	subnet, err := c.subnetsLister.Get(cachedIP.Spec.Subnet)	2✔
243	if err != nil {	1✔
244	if !k8serrors.IsNotFound(err) {	×
245	klog.Errorf("failed to get subnet %s: %v", cachedIP.Spec.Subnet, err)	×
246	return err	×
247	}
248	// subnet not found, but ip exists, check if the ip is u2o ip or mcast querier ip
249	// if yes, remove finalizer to let ip be deleted	1✔
250	klog.Warningf("subnet %s not found for deleting ip %s", cachedIP.Spec.Subnet, cachedIP.Name)	1✔
251	if strings.HasPrefix(cachedIP.Name, util.U2OInterconnName[0:20]) \|\|	1✔
252	strings.HasPrefix(cachedIP.Name, util.McastQuerierName[0:14]) {	×
253	if err = c.handleDelIPFinalizer(cachedIP); err != nil {	×
254	klog.Errorf("failed to remove finalizer for deleting ip %s: %v", cachedIP.Name, err)	×
255	return err	×
256	}	×
257	return nil
258	}
259	}	1✔
260	portName := cachedIP.Name	1✔
261	if isOvnSubnet(subnet) {	×
262	port, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true)	×
263	if err != nil {	×
264	klog.Errorf("failed to get logical switch port %s: %v", portName, err)	×
265	return err	×
266	}	×
267	if port != nil {	×
268	klog.Infof("delete ip cr lsp %s from switch %s", portName, subnet.Name)	×
269	if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil {	×
270	klog.Errorf("failed to delete ip cr lsp %s from switch %s: %v", portName, subnet.Name, err)	×
271	return err	×
272	}	×
273	klog.V(3).Infof("sync sg for deleted port %s", portName)	×
274	sgList, err := c.getPortSg(port)	×
275	if err != nil {	×
276	klog.Errorf("get port sg failed, %v", err)	×
277	return err	×
278	}	×
279	for _, sgName := range sgList {	×
280	if sgName != "" {	×
281	c.syncSgPortsQueue.Add(sgName)	×
282	}
283	}
284	}
285	}	1✔
286	podKey := fmt.Sprintf("%s/%s", cachedIP.Spec.Namespace, cachedIP.Spec.PodName)	1✔
287	klog.Infof("ip cr %s release ipam pod key %s from subnet %s", cachedIP.Name, podKey, cachedIP.Spec.Subnet)	1✔
288	c.ipam.ReleaseAddressByNic(podKey, portName, cachedIP.Spec.Subnet)	1✔
289	if err = c.handleDelIPFinalizer(cachedIP); err != nil {	×
290	klog.Errorf("failed to handle del ip finalizer %v", err)	×
291	return err	×
292	}	1✔
293	c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet)
294	}	1✔
295	return nil
296	}
297		×
298	func (c Controller) handleDelIP(ip kubeovnv1.IP) error {	×
299	klog.Infof("deleting ip %s enqueue update status subnet %s", ip.Name, ip.Spec.Subnet)	×
300	c.updateSubnetStatusQueue.Add(ip.Spec.Subnet)	×
301	for _, as := range ip.Spec.AttachSubnets {	×
302	klog.V(3).Infof("enqueue update attach status for subnet %s", as)	×
303	c.updateSubnetStatusQueue.Add(as)	×
304	}	×
305	return nil
306	}
307		×
308	func (c *Controller) syncIPFinalizer(cl client.Client) error {	×
309	// migrate deprecated finalizer to new finalizer	×
310	ips := &kubeovnv1.IPList{}	×
311	return migrateFinalizers(cl, ips, func(i int) (client.Object, client.Object) {	×
312	if i < 0 \|\| i >= len(ips.Items) {	×
313	return nil, nil	×
314	}	×
315	return ips.Items[i].DeepCopy(), ips.Items[i].DeepCopy()
316	})
317	}
318		1✔
319	func (c Controller) handleDelIPFinalizer(cachedIP kubeovnv1.IP) error {	1✔
320	if len(cachedIP.GetFinalizers()) == 0 {	×
321	return nil	×
322	}	1✔
323	newIP := cachedIP.DeepCopy()	1✔
324	controllerutil.RemoveFinalizer(newIP, util.DeprecatedFinalizerName)	1✔
325	controllerutil.RemoveFinalizer(newIP, util.KubeOVNControllerFinalizer)	1✔
326	patch, err := util.GenerateMergePatchPayload(cachedIP, newIP)	1✔
327	if err != nil {	×
328	klog.Errorf("failed to generate patch payload for ip %s, %v", cachedIP.Name, err)	×
329	return err	×
330	}	1✔
331	if _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Patch(context.Background(), cachedIP.Name,	1✔
332	types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {	×
333	if k8serrors.IsNotFound(err) {	×
334	return nil	×
335	}	×
336	klog.Errorf("failed to remove finalizer from ip %s, %v", cachedIP.Name, err)	×
337	return err
338	}	1✔
339	return nil
340	}
341		×
342	func (c *Controller) acquireIPAddress(subnetName, name, nicName string) (string, string, string, error) {	×
343	var skippedAddrs []string	×
344	var v4ip, v6ip, mac string	×
345	checkConflict := true	×
346	var err error	×
347	for {	×
348	v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, nil, subnetName, "", skippedAddrs, checkConflict)	×
349	if err != nil {	×
350	klog.Error(err)	×
351	return "", "", "", err	×
352	}
353		×
354	ipv4OK, ipv6OK, err := c.validatePodIP(name, subnetName, v4ip, v6ip)	×
355	if err != nil {	×
356	klog.Error(err)	×
357	return "", "", "", err	×
358	}
359		×
360	if ipv4OK && ipv6OK {	×
361	return v4ip, v6ip, mac, nil	×
362	}
363		×
364	if !ipv4OK {	×
365	skippedAddrs = append(skippedAddrs, v4ip)	×
366	}	×
367	if !ipv6OK {	×
368	skippedAddrs = append(skippedAddrs, v6ip)	×
369	}
370	}
371	}
372		×
373	func (c Controller) acquireStaticIPAddress(subnetName, name, nicName, ip string, macPointer string) (string, string, string, error) {	×
374	checkConflict := true	×
375	var v4ip, v6ip, mac string	×
376	var err error	×
377	for ipStr := range strings.SplitSeq(ip, ",") {	×
378	if net.ParseIP(ipStr) == nil {	×
379	return "", "", "", fmt.Errorf("failed to parse vip ip %s", ipStr)	×
380	}
381	}
382		×
383	if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, macPointer, subnetName, checkConflict); err != nil {	×
384	klog.Errorf("failed to get static virtual ip '%s', mac '%s', subnet '%s', %v", ip, mac, subnetName, err)	×
385	return "", "", "", err	×
386	}	×
387	return v4ip, v6ip, mac, nil
388	}
389		×
390	func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, ns, nodeName, podType string) error {	×
391	// `ipCRName`: pod or vm IP name must set ip CR name when creating ip CR	×
392	var key, ipName string	×
393	var owner *metav1.OwnerReference	×
394	if ipCRName != "" {	×
395	// pod IP	×
396	key = podName	×
397	ipName = ipCRName	×
398	} else {	×
399	// node IP, u2o IP or mcast querier IP	×
400	switch {	×
401	case subnetName == c.config.NodeSwitch:	×
402	node, err := c.nodesLister.Get(nodeName)	×
403	if err != nil {	×
404	err = fmt.Errorf("failed to get node %s: %w", nodeName, err)	×
405	klog.Error(err)	×
406	return err	×
407	}	×
408	key = nodeName	×
409	ipName = util.NodeLspName(nodeName)	×
410	owner = &metav1.OwnerReference{	×
411	APIVersion: corev1.SchemeGroupVersion.String(),	×
412	Kind: util.KindNode,	×
413	Name: nodeName,	×
414	UID: node.UID,	×
415	}	×
416	case strings.HasPrefix(podName, util.U2OInterconnName[0:20]) \|\| strings.HasPrefix(podName, util.McastQuerierName[0:14]):	×
417	// u2o IP or mcast querier IP	×
418	subnet, err := c.subnetsLister.Get(subnetName)	×
419	if err != nil {	×
420	err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err)	×
421	klog.Error(err)	×
422	return err	×
423	}	×
424	key = podName	×
425	ipName = podName	×
426	owner = &metav1.OwnerReference{	×
427	APIVersion: kubeovnv1.SchemeGroupVersion.String(),	×
428	Kind: util.KindSubnet,	×
429	Name: subnetName,	×
430	UID: subnet.UID,	×
431	BlockOwnerDeletion: ptr.To(true),	×
432	}
433	}
434	}
435		×
436	var err error	×
437	var ipCR *kubeovnv1.IP	×
438	ipCR, err = c.ipsLister.Get(ipName)	×
439	if err != nil {	×
440	if !k8serrors.IsNotFound(err) {	×
441	err := fmt.Errorf("failed to get ip CR %s: %w", ipName, err)	×
442	klog.Error(err)	×
443	return err	×
444	}
445	// the returned pointer is not nil if the CR does not exist	×
446	ipCR = nil
447	}	×
448	if ipCR != nil && !ipCR.DeletionTimestamp.IsZero() {	×
449	// this ip is being deleted, no need to update	×
450	klog.Infof("enqueue update for removing finalizer to delete ip %s", ipCR.Name)	×
451	c.updateIPQueue.Add(ipCR.Name)	×
452	return nil	×
453	}	×
454	v4IP, v6IP := util.SplitStringIP(ip)	×
455	if ipCR == nil {	×
456	ipCR = &kubeovnv1.IP{	×
457	ObjectMeta: metav1.ObjectMeta{	×
458	Name: ipName,	×
459	Labels: map[string]string{	×
460	util.SubnetNameLabel: subnetName,	×
461	util.NodeNameLabel: nodeName,	×
462	subnetName: "",	×
463	util.IPReservedLabel: "false", // ip create with pod or node, ip not reserved	×
464	},	×
465	Finalizers: []string{util.KubeOVNControllerFinalizer},	×
466	},	×
467	Spec: kubeovnv1.IPSpec{	×
468	PodName: key,	×
469	Subnet: subnetName,	×
470	NodeName: nodeName,	×
471	Namespace: ns,	×
472	IPAddress: ip,	×
473	V4IPAddress: v4IP,	×
474	V6IPAddress: v6IP,	×
475	MacAddress: mac,	×
476	AttachIPs: []string{},	×
477	AttachMacs: []string{},	×
478	AttachSubnets: []string{},	×
479	PodType: podType,	×
480	},	×
481	}	×
482	if owner != nil {	×
483	ipCR.OwnerReferences = []metav1.OwnerReference{*owner}	×
484	}	×
485	if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), ipCR, metav1.CreateOptions{}); err != nil {	×
486	errMsg := fmt.Errorf("failed to create ip CR %s: %w", ipName, err)	×
487	klog.Error(errMsg)	×
488	return errMsg	×
489	}	×
490	} else {	×
491	newIPCR := ipCR.DeepCopy()	×
492	if newIPCR.Labels != nil {	×
493	newIPCR.Labels[util.SubnetNameLabel] = subnetName	×
494	newIPCR.Labels[util.NodeNameLabel] = nodeName	×
495	} else {	×
496	newIPCR.Labels = map[string]string{	×
497	util.SubnetNameLabel: subnetName,	×
498	util.NodeNameLabel: nodeName,	×
499	}	×
500	// update not touch IP Reserved Label	×
501	}	×
502	if owner != nil {	×
503	// currently we only set owner for node IP, u2o IP and mcast querier ip,	×
504	// so it's ok to overwrite it here	×
505	newIPCR.OwnerReferences = []metav1.OwnerReference{*owner}	×
506	}	×
507	newIPCR.Spec.PodName = key	×
508	newIPCR.Spec.Namespace = ns	×
509	newIPCR.Spec.Subnet = subnetName	×
510	newIPCR.Spec.NodeName = nodeName	×
511	newIPCR.Spec.IPAddress = ip	×
512	newIPCR.Spec.V4IPAddress = v4IP	×
513	newIPCR.Spec.V6IPAddress = v6IP	×
514	newIPCR.Spec.MacAddress = mac	×
515	newIPCR.Spec.AttachIPs = []string{}	×
516	newIPCR.Spec.AttachMacs = []string{}	×
517	newIPCR.Spec.AttachSubnets = []string{}	×
518	newIPCR.Spec.PodType = podType	×
519	if maps.Equal(newIPCR.Labels, ipCR.Labels) && reflect.DeepEqual(newIPCR.Spec, ipCR.Spec) {	×
520	return nil	×
521	}	×
522	controllerutil.AddFinalizer(newIPCR, util.KubeOVNControllerFinalizer)	×
523		×
524	if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), newIPCR, metav1.UpdateOptions{}); err != nil {	×
525	err := fmt.Errorf("failed to update ip CR %s: %w", ipCRName, err)	×
526	klog.Error(err)	×
527	return err	×
528	}
529	}	×
530	return nil
531	}
532		×
533	func (c Controller) ipAcquireAddress(ip kubeovnv1.IP, subnet *kubeovnv1.Subnet) (string, string, string, error) {	×
534	key := cache.NewObjectName(ip.Spec.Namespace, ip.Spec.PodName).String()	×
535	portName := ovs.PodNameToPortName(ip.Spec.PodName, ip.Spec.Namespace, subnet.Spec.Provider)	×
536	ipStr := util.GetStringIP(ip.Spec.V4IPAddress, ip.Spec.V6IPAddress)	×
537		×
538	var v4IP, v6IP, mac string	×
539	var err error	×
540	var macPtr *string	×
541	if isOvnSubnet(subnet) {	×
542	if ip.Spec.MacAddress != "" {	×
543	macPtr = &ip.Spec.MacAddress	×
544	}	×
545	} else {	×
546	macPtr = ptr.To("")	×
547	}
548		×
549	if ipStr == "" {	×
550	// allocate address	×
551	v4IP, v6IP, mac, err = c.acquireIPAddress(subnet.Name, ip.Name, portName)	×
552	if err == nil {	×
553	return v4IP, v6IP, mac, err	×
554	}	×
555	err = fmt.Errorf("failed to get random address for ip %s, %w", ip.Name, err)	×
556	} else {	×
557	// static address	×
558	v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipStr, macPtr, subnet.Name, true)	×
559	if err == nil {	×
560	return v4IP, v6IP, mac, nil	×
561	}	×
562	err = fmt.Errorf("failed to get static address for ip %s, %w", ip.Name, err)
563	}	×
564	klog.Error(err)	×
565	return "", "", "", err
566	}

kubeovn / kube-ovn / 22306573215

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous