kubeovn / kube-ovn / 23181819432

func (c *Controller) setIPSet() error {

        protocols := make([]string, 2)

        if c.protocol == kubeovnv1.ProtocolDual {

                protocols[0] = kubeovnv1.ProtocolIPv4

                protocols[1] = kubeovnv1.ProtocolIPv6

        } else {

                protocols[0] = c.protocol

        }

        for _, protocol := range protocols {

                if c.ipsets[protocol] == nil {

                        continue

                services := c.getServicesCIDR(protocol)

                subnets, _, err := c.getDefaultVpcSubnetsCIDR(protocol)

                if err != nil {

                        klog.Errorf("get subnets failed, %+v", err)

                        return err

                }

                subnetsNeedNat, err := c.getSubnetsNeedNAT(protocol)

                if err != nil {

                        klog.Errorf("get need nat subnets failed, %+v", err)

                        return err

                }

                subnetsDistributedGateway, err := c.getSubnetsDistributedGateway(protocol)

                if err != nil {

                        klog.Errorf("failed to get subnets with centralized gateway: %v", err)

                        return err

                }

                otherNode, err := c.getOtherNodes(protocol)

                if err != nil {

                        klog.Errorf("failed to get node, %+v", err)

                        return err

                }

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   ServiceSet,

                        Type:    ipsets.IPSetTypeHashNet,

                }, services)

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   SubnetSet,

                        Type:    ipsets.IPSetTypeHashNet,

                }, subnets)

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   LocalPodSet,

                        Type:    ipsets.IPSetTypeHashIP,

                }, nil)

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   SubnetNatSet,

                        Type:    ipsets.IPSetTypeHashNet,

                }, subnetsNeedNat)

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   SubnetDistributedGwSet,

                        Type:    ipsets.IPSetTypeHashNet,

                }, subnetsDistributedGateway)

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   OtherNodeSet,

                        Type:    ipsets.IPSetTypeHashNet,

                }, otherNode)

                c.reconcileNatOutGoingPolicyIPset(protocol)

                c.ipsets[protocol].ApplyUpdates()

        return nil

func (c *Controller) gcIPSet() {

        protocols := make([]string, 2)

        if c.protocol == kubeovnv1.ProtocolDual {

                protocols[0] = kubeovnv1.ProtocolIPv4

                protocols[1] = kubeovnv1.ProtocolIPv6

        } else {

                protocols[0] = c.protocol

        }

        for _, protocol := range protocols {

                if c.ipsets[protocol] == nil {

                        continue

                c.ipsets[protocol].ApplyDeletions()

func (c *Controller) addNatOutGoingPolicyRuleIPset(rule kubeovnv1.NatOutgoingPolicyRuleStatus, protocol string) {

        if rule.Match.SrcIPs != "" {

                ipsetName := getNatOutGoingPolicyRuleIPSetName(rule.RuleID, "src", "", false)

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   ipsetName,

                        Type:    ipsets.IPSetTypeHashNet,

                }, strings.Split(rule.Match.SrcIPs, ","))

        }

        if rule.Match.DstIPs != "" {

                ipsetName := getNatOutGoingPolicyRuleIPSetName(rule.RuleID, "dst", "", false)

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                        MaxSize: 1048576,

                        SetID:   ipsetName,

                        Type:    ipsets.IPSetTypeHashNet,

                }, strings.Split(rule.Match.DstIPs, ","))

        }

func (c *Controller) removeNatOutGoingPolicyRuleIPset(protocol string, natPolicyRuleIDs *strset.Set) {

        sets, err := c.k8sipsets.ListSets()

        if err != nil {

                klog.Errorf("failed to list ipsets: %v", err)

                return

        }

        for _, set := range sets {

                if isNatOutGoingPolicyRuleIPSet(set) {

                        ruleID, _ := getNatOutGoingPolicyRuleIPSetItem(set)

                        if !natPolicyRuleIDs.Has(ruleID) {

                                c.ipsets[protocol].RemoveIPSet(formatIPsetUnPrefix(set))

                        }

func (c *Controller) reconcileNatOutGoingPolicyIPset(protocol string) {

        subnets, err := c.getSubnetsNatOutGoingPolicy(protocol)

        if err != nil {

                klog.Errorf("failed to get subnets with NAT outgoing policy rule: %v", err)

                return

        }

        subnetCidrs := make([]string, 0, len(subnets))

        natPolicyRuleIDs := strset.New()

        for _, subnet := range subnets {

                cidrBlock, err := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)

                if err != nil {

                        klog.Errorf("failed to get subnet %s CIDR block by protocol: %v", subnet.Name, err)

                        continue

                if cidrBlock != "" {

                        subnetCidrs = append(subnetCidrs, cidrBlock)

                }

                for _, rule := range subnet.Status.NatOutgoingPolicyRules {

                        if rule.RuleID == "" {

                                klog.Errorf("unexpected empty ID for NAT outgoing rule %q of subnet %s", rule.NatOutgoingPolicyRule, subnet.Name)

                                continue

                        natPolicyRuleIDs.Add(rule.RuleID)

                        c.addNatOutGoingPolicyRuleIPset(rule, protocol)

        c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

                MaxSize: 1048576,

                SetID:   NatOutGoingPolicySubnetSet,

                Type:    ipsets.IPSetTypeHashNet,

        }, subnetCidrs)

        c.removeNatOutGoingPolicyRuleIPset(protocol, natPolicyRuleIDs)

func (c *Controller) setPolicyRouting() error {

        protocols := make([]string, 2)

        if c.protocol == kubeovnv1.ProtocolDual {

                protocols[0] = kubeovnv1.ProtocolIPv4

                protocols[1] = kubeovnv1.ProtocolIPv6

        } else {

                protocols[0] = c.protocol

        }

        for _, protocol := range protocols {

                if c.ipsets[protocol] == nil {

                        continue

                localPodIPs, err := c.getLocalPodIPsNeedPR(protocol)

                if err != nil {

                        klog.Errorf("failed to get local pod ips failed: %+v", err)

                        return err

                }

                subnetsNeedPR, err := c.getSubnetsNeedPR(protocol)

                if err != nil {

                        klog.Errorf("failed to get subnets that need policy routing: %+v", err)

                        return err

                }

                family, err := util.ProtocolToFamily(protocol)

                if err != nil {

                        klog.Error(err)

                        return err

                }

                for meta, ips := range localPodIPs {

                        if err = c.addPolicyRouting(family, meta.gateway, meta.priority, meta.tableID, ips...); err != nil {

                                klog.Errorf("failed to add policy routing for local pods: %+v", err)

                                return err

                        }

                for meta, cidr := range subnetsNeedPR {

                        if err = c.addPolicyRouting(family, meta.gateway, meta.priority, meta.tableID, cidr); err != nil {

                                klog.Errorf("failed to add policy routing for subnet: %+v", err)

                                return err

                        }

        return nil

func (c *Controller) addPodPolicyRouting(podProtocol, externalEgressGateway string, priority, tableID uint32, ips []string) error {

        egw := strings.Split(externalEgressGateway, ",")

        prMetas := make([]policyRouteMeta, 0, 2)

        if len(egw) == 1 {

                family, _ := util.ProtocolToFamily(util.CheckProtocol(egw[0]))

                if family == netlink.FAMILY_V4 || podProtocol != kubeovnv1.ProtocolDual {

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[0], gateway: egw[0]})

                } else {

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[1], gateway: egw[0]})

                }

        } else {

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V4, source: ips[0], gateway: egw[0]})

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V6, source: ips[1], gateway: egw[1]})

        }

        for _, meta := range prMetas {

                if err := c.addPolicyRouting(meta.family, meta.gateway, priority, tableID, meta.source); err != nil {

                        klog.Errorf("failed to add policy routing for pod: %+v", err)

                        return err

                }

        return nil

func (c *Controller) deletePodPolicyRouting(podProtocol, externalEgressGateway string, priority, tableID uint32, ips []string) error {

        egw := strings.Split(externalEgressGateway, ",")

        prMetas := make([]policyRouteMeta, 0, 2)

        if len(egw) == 1 {

                family, _ := util.ProtocolToFamily(util.CheckProtocol(egw[0]))

                if family == netlink.FAMILY_V4 || podProtocol != kubeovnv1.ProtocolDual {

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[0], gateway: egw[0]})

                } else {

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[1], gateway: egw[0]})

                }

        } else {

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V4, source: ips[0], gateway: egw[0]})

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V6, source: ips[1], gateway: egw[1]})

        }

        for _, meta := range prMetas {

                if err := c.deletePolicyRouting(meta.family, meta.gateway, priority, tableID, meta.source); err != nil {

                        klog.Errorf("failed to delete policy routing for pod: %+v", err)

                        return err

                }

        return nil

func (c *Controller) addPolicyRouting(family int, gateway string, priority, tableID uint32, ips ...string) error {

        route := &netlink.Route{

                Protocol: netlink.RouteProtocol(syscall.RTPROT_STATIC),

                Gw:       net.ParseIP(gateway),

                Table:    int(tableID),

        }

        if err := netlink.RouteReplace(route); err != nil && !errors.Is(err, syscall.EEXIST) {

                err = fmt.Errorf("failed to replace route in table %d: %w", tableID, err)

                klog.Error(err)

                return err

        }

        maskBits := 32

        if family == netlink.FAMILY_V6 {

                maskBits = 128

        }

        rule := netlink.NewRule()

        rule.Family = family

        rule.Table = int(tableID)

        rule.Priority = int(priority)

        mask := net.CIDRMask(maskBits, maskBits)

        for _, ip := range ips {

                if strings.ContainsRune(ip, '/') {

                        var err error

                        if rule.Src, err = netlink.ParseIPNet(ip); err != nil {

                                klog.Errorf("unexpected CIDR: %s", ip)

                                err = fmt.Errorf("failed to add route in table %d: %w", tableID, err)

                                klog.Error(err)

                                return err

                        }

                } else {

                        rule.Src = &net.IPNet{IP: net.ParseIP(ip), Mask: mask}

                }

                if err := netlink.RuleAdd(rule); err != nil && !errors.Is(err, syscall.EEXIST) {

                        err = fmt.Errorf("failed to add network rule: %w", err)

                        klog.Error(err)

                        return err

                }

        return nil

func (c *Controller) deletePolicyRouting(family int, _ string, priority, tableID uint32, ips ...string) error {

        maskBits := 32

        if family == netlink.FAMILY_V6 {

                maskBits = 128

        }

        rule := netlink.NewRule()

        rule.Family = family

        rule.Table = int(tableID)

        rule.Priority = int(priority)

        mask := net.CIDRMask(maskBits, maskBits)

        for _, ip := range ips {

                if strings.ContainsRune(ip, '/') {

                        var err error

                        if rule.Src, err = netlink.ParseIPNet(ip); err != nil {

                                klog.Errorf("unexpected CIDR: %s", ip)

                                err = fmt.Errorf("failed to delete route in table %d: %w", tableID, err)

                                klog.Error(err)

                                return err

                        }

                } else {

                        rule.Src = &net.IPNet{IP: net.ParseIP(ip), Mask: mask}

                }

                if err := netlink.RuleDel(rule); err != nil && !errors.Is(err, syscall.ENOENT) {

                        err = fmt.Errorf("failed to delete network rule: %w", err)

                        klog.Error(err)

                        return err

                }

        return nil

func findRulePositions(ipt *iptables.IPTables, rule util.IPTableRule) ([]int, error) {

        rules, err := ipt.List(rule.Table, rule.Chain)

        if err != nil {

                return nil, fmt.Errorf("failed to list iptables rules: %w", err)

        }

        positions := make([]int, 0)

        // Start from the end of the list (lowest priority) and go up to 1 (but skip index 0)

        for i := len(rules) - 1; i >= 1; i-- {

                ruleSpec := util.DoubleQuotedFields(rules[i])

                if len(ruleSpec) < 3 {

                        continue

                if slices.Equal(ruleSpec[2:], rule.Rule) {

                        positions = append(positions, i)

                }

        return positions, nil

func ensureNatPreroutingRulePosition(ipt *iptables.IPTables, rule util.IPTableRule) error {

        kubeProxyRule := util.IPTableRule{

                Table: "nat",

                Chain: "PREROUTING",

                Rule:  []string{"-m", "comment", "--comment", "kubernetes service portals", "-j", "KUBE-SERVICES"},

        }

        kubeProxyPosList, err := findRulePositions(ipt, kubeProxyRule)

        if err != nil {

                return fmt.Errorf("failed to find kube-proxy rule position: %w", err)

        }

        insertPosition := 1

        if len(kubeProxyPosList) > 0 {

                insertPosition = kubeProxyPosList[len(kubeProxyPosList)-1]

        }

        existingNatPreroutingPosList, err := findRulePositions(ipt, rule)

        if err != nil {

                return fmt.Errorf("failed to find nat prerouting rule position: %w", err)

        }

        if len(existingNatPreroutingPosList) > 0 {

                pos := existingNatPreroutingPosList[len(existingNatPreroutingPosList)-1]

                if pos <= insertPosition {

                        klog.V(5).Infof("nat prerouting rule already exists at position %d", pos)

                        return nil

                }

        klog.Infof("inserting nat prerouting rule %q at position %d", rule.Rule, insertPosition)

        return ipt.Insert(rule.Table, rule.Chain, insertPosition, rule.Rule...)

func ensureNatPreroutingRuleNoDuplicate(ipt *iptables.IPTables, rule util.IPTableRule) error {

        existingNatPreroutingPosList, err := findRulePositions(ipt, rule)

        if err != nil {

                return fmt.Errorf("failed to find nat prerouting rule position: %w", err)

        }

        if len(existingNatPreroutingPosList) == 0 {

                klog.Warningf("nat prerouting rule not found, skipping duplicate check")

                return nil

        }

        for _, pos := range existingNatPreroutingPosList[:len(existingNatPreroutingPosList)-1] {

                klog.Infof("deleting duplicate nat prerouting rule at position %d", pos)

                if err := ipt.Delete(rule.Table, rule.Chain, strconv.Itoa(pos)); err != nil {

                        return fmt.Errorf("failed to delete duplicate nat prerouting rule: %w", err)

                }

        return nil

func ensureNatPreroutingRule(ipt *iptables.IPTables, rule util.IPTableRule) error {

        klog.V(3).Infof("ensure nat prerouting rule %q", rule.Rule)

        if err := ensureNatPreroutingRulePosition(ipt, rule); err != nil {

                return fmt.Errorf("failed to ensure nat prerouting rule position: %w", err)

        }

        if err := ensureNatPreroutingRuleNoDuplicate(ipt, rule); err != nil {

                return fmt.Errorf("failed to ensure nat prerouting rule no duplicate: %w", err)

        }

        return nil

func (c *Controller) createIptablesRule(ipt *iptables.IPTables, rule util.IPTableRule) error {

        exists, err := ipt.Exists(rule.Table, rule.Chain, rule.Rule...)

        if err != nil {

                klog.Errorf("failed to check iptables rule existence: %v", err)

                return err

        }

        s := strings.Join(rule.Rule, " ")

        if exists {

                if rule.Table == NAT && rule.Chain == Prerouting {

                        return ensureNatPreroutingRule(ipt, rule)

                }

                return nil

        klog.Infof("creating iptables rule in table %s chain %s at position %d: %q", rule.Table, rule.Chain, 1, s)

        if err = ipt.Insert(rule.Table, rule.Chain, 1, rule.Rule...); err != nil {

                klog.Errorf(`failed to insert iptables rule "%s": %v`, s, err)

                return err

        }

        return nil

func (c *Controller) updateIptablesChain(ipt *iptables.IPTables, table, chain, parent string, rules []util.IPTableRule) error {

        ok, err := ipt.ChainExists(table, chain)

        if err != nil {

                klog.Errorf("failed to check existence of iptables chain %s in table %s: %v", chain, table, err)

                return err

        }

        if !ok {

                if err = ipt.NewChain(table, chain); err != nil {

                        klog.Errorf("failed to create iptables chain %s in table %s: %v", chain, table, err)

                        return err

                }

                klog.Infof("created iptables chain %s in table %s", chain, table)

        if parent != "" {

                comment := fmt.Sprintf("kube-ovn %s rules", strings.ToLower(parent))

                rule := util.IPTableRule{

                        Table: table,

                        Chain: parent,

                        Rule:  []string{"-m", "comment", "--comment", comment, "-j", chain},

                }

                if err = c.createIptablesRule(ipt, rule); err != nil {

                        klog.Errorf("failed to create iptables rule: %v", err)

                        return err

                }

        ruleList, err := ipt.List(table, chain)

        if err != nil {

                klog.Errorf("failed to list iptables rules in chain %s/%s: %v", table, chain, err)

                return err

        }

        ruleList = ruleList[1:]

        // trim prefix: "-A OVN-POSTROUTING "

        prefixLen := 4 + len(chain)

        existingRules := make([][]string, 0, len(ruleList))

        for _, r := range ruleList {

                existingRules = append(existingRules, util.DoubleQuotedFields(r[prefixLen:]))

        }

        var added int

        for i, rule := range rules {

                if i-added < len(existingRules) && slices.Equal(existingRules[i-added], rule.Rule) {

                        klog.V(5).Infof("iptables rule %v already exists", rule.Rule)

                        continue

                klog.Infof("creating iptables rule in table %s chain %s at position %d: %q", table, chain, i+1, strings.Join(rule.Rule, " "))

                if err = ipt.Insert(table, chain, i+1, rule.Rule...); err != nil {

                        klog.Errorf(`failed to insert iptables rule %v: %v`, rule.Rule, err)

                        return err

                }

                added++

        for i := len(existingRules) - 1; i >= len(rules)-added; i-- {

                if err = ipt.Delete(table, chain, strconv.Itoa(i+added+1)); err != nil {

                        klog.Errorf(`failed to delete iptables rule %v: %v`, existingRules[i], err)

                        return err

                }

                klog.Infof("deleted iptables rule in table %s chain %s: %q", table, chain, strings.Join(existingRules[i], " "))

        return nil

func (c *Controller) setIptables() error {

        klog.V(3).Infoln("start to set up iptables")

        node, err := c.nodesLister.Get(c.config.NodeName)

        if err != nil {

                klog.Errorf("failed to get node %s, %v", c.config.NodeName, err)

                return err

        }

        nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node)

        nodeIPs := map[string]string{

                kubeovnv1.ProtocolIPv4: nodeIPv4,

                kubeovnv1.ProtocolIPv6: nodeIPv6,

        }

        centralGwNatIPs, err := c.getEgressNatIPByNode(c.config.NodeName)

        if err != nil {

                klog.Errorf("failed to get centralized subnets nat ips on node %s, %v", c.config.NodeName, err)

                return err

        }

        klog.V(3).Infof("centralized subnets nat ips %v", centralGwNatIPs)

        var (

                v4Rules = []util.IPTableRule{

                        // mark packets from pod to service

                        {Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ` + util.NodeNic + ` -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000`)},

                        // nat packets marked by kube-proxy or kube-ovn

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j ` + OvnMasquerade)},

                        // nat service traffic

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j ` + OvnMasquerade)},

                        // do not nat node port service traffic with external traffic policy set to local

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN`)},

                        // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnMasquerade)},

                        // do not nat reply packets in direct routing

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},

                        // do not nat route traffic

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)},

                        // nat outgoing policy rules

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields("-m set --match-set ovn40subnets-nat-policy src -m set ! --match-set ovn40subnets dst -j " + OvnNatOutGoingPolicy)},

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j %s`, OnOutGoingNatMark, OvnMasquerade))},

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))},

                        // default nat outgoing rules

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j ` + OvnMasquerade)},

                        // clear mark

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MARK --set-xmark 0x0/0xffffffff`)},

                        // do masquerade

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MASQUERADE`)},

                        // Input Accept

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},

                        // Forward Accept

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},

                        // Output unmark to bypass kernel nat checksum issue https://github.com/flannel-io/flannel/issues/1279

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 6081 -j MARK --set-xmark 0x0`)},

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 4789 -j MARK --set-xmark 0x0`)},

                        // Drop invalid rst

                        {Table: MANGLE, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m set --match-set ovn40subnets src -m tcp --tcp-flags RST RST -m state --state INVALID -j DROP`)},

                }

                v6Rules = []util.IPTableRule{

                        // mark packets from pod to service

                        {Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ` + util.NodeNic + ` -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000`)},

                        // nat packets marked by kube-proxy or kube-ovn

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j ` + OvnMasquerade)},

                        // nat service traffic

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets src -m set --match-set ovn60subnets dst -j ` + OvnMasquerade)},

                        // do not nat node port service traffic with external traffic policy set to local

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN`)},

                        // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnMasquerade)},

                        // do not nat reply packets in direct routing

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},

                        // do not nat route traffic

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`)},

                        // nat outgoing policy rules

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields("-m set --match-set ovn60subnets-nat-policy src -m set ! --match-set ovn60subnets dst -j " + OvnNatOutGoingPolicy)},

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j %s`, OnOutGoingNatMark, OvnMasquerade))},

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))},

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j ` + OvnMasquerade)},

                        // clear mark

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MARK --set-xmark 0x0/0xffffffff`)},

                        // do masquerade

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MASQUERADE`)},

                        // Input Accept

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},

                        // Forward Accept

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},

                        // Output unmark to bypass kernel nat checksum issue https://github.com/flannel-io/flannel/issues/1279

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 6081 -j MARK --set-xmark 0x0`)},

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 4789 -j MARK --set-xmark 0x0`)},

                        // Drop invalid rst

                        {Table: MANGLE, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m set --match-set ovn60subnets src -m tcp --tcp-flags RST RST -m state --state INVALID -j DROP`)},

                }

        )

        protocols := make([]string, 0, 2)

        if c.protocol == kubeovnv1.ProtocolDual {

                protocols = append(protocols, kubeovnv1.ProtocolIPv4, kubeovnv1.ProtocolIPv6)

        } else {

                protocols = append(protocols, c.protocol)

        }

        for _, protocol := range protocols {

                ipt := c.iptables[protocol]

                if ipt == nil {

                        continue

                var kubeProxyIpsetProtocol, matchset, svcMatchset, nodeMatchSet string

                var obsoleteRules, iptablesRules []util.IPTableRule

                if protocol == kubeovnv1.ProtocolIPv4 {

                        iptablesRules = v4Rules

                        matchset, svcMatchset, nodeMatchSet = "ovn40subnets", "ovn40services", "ovn40"+OtherNodeSet

                } else {

                        iptablesRules = v6Rules

                        kubeProxyIpsetProtocol, matchset, svcMatchset, nodeMatchSet = "6-", "ovn60subnets", "ovn60services", "ovn60"+OtherNodeSet

                }

                ipset := fmt.Sprintf("KUBE-%sCLUSTER-IP", kubeProxyIpsetProtocol)

                ipsetExists, err := c.ipsetExists(ipset)

                if err != nil {

                        klog.Errorf("failed to check existence of ipset %s: %v", ipset, err)

                        return err

                }

                if ipsetExists && !c.config.DisableSvcIptablesRule {

                        iptablesRules[0].Rule = strings.Fields(fmt.Sprintf(`-i %s -m set --match-set %s src -m set --match-set %s dst,dst -j MARK --set-xmark 0x4000/0x4000`, util.NodeNic, matchset, ipset))

                        rejectRule := strings.Fields(fmt.Sprintf(`-p tcp -m mark ! --mark 0x4000/0x4000 -m set --match-set %s dst -m conntrack --ctstate NEW -j REJECT`, svcMatchset))

                        iptablesRules = append(iptablesRules,

                                util.IPTableRule{Table: "filter", Chain: "INPUT", Rule: rejectRule},

                                util.IPTableRule{Table: "filter", Chain: "OUTPUT", Rule: rejectRule},

                        )

                        // TODO: remove obsoleteRejectRule cleanup after all clusters have upgraded past v1.12

                        obsoleteRejectRule := strings.Fields(fmt.Sprintf(`-m mark ! --mark 0x4000/0x4000 -m set --match-set %s dst -m conntrack --ctstate NEW -j REJECT`, svcMatchset))

                        obsoleteRejectRules := []util.IPTableRule{

                                {Table: "filter", Chain: "INPUT", Rule: obsoleteRejectRule},

                                {Table: "filter", Chain: "OUTPUT", Rule: obsoleteRejectRule},

                        }

                        for _, rule := range obsoleteRejectRules {

                                if err = deleteIptablesRule(ipt, rule); err != nil {

                                        klog.Errorf("failed to delete obsolete iptables rule %v: %v", rule, err)

                                        return err

                                }

                if nodeIP := nodeIPs[protocol]; nodeIP != "" {

                        obsoleteRules = []util.IPTableRule{

                                {Table: NAT, Chain: Postrouting, Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset))},

                                {Table: NAT, Chain: Postrouting, Rule: strings.Fields(fmt.Sprintf(`! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, nodeIP))},

                                {Table: NAT, Chain: Postrouting, Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set ! --match-set %s src -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset, matchset))},

                        }

                }

                if c.config.DisableSvcIptablesRule {

                        // Filter out service-related rules from iptablesRules and clean up

                        // any existing service rules in standard chains.

                        // Rules in custom chains (OvnPrerouting, OvnPostrouting) are automatically

                        // reconciled by updateIptablesChain.

                        // Clean up reject rules for unmatched service traffic

                        rejectRule := strings.Fields(fmt.Sprintf(`-p tcp -m mark ! --mark 0x4000/0x4000 -m set --match-set %s dst -m conntrack --ctstate NEW -j REJECT`, svcMatchset))

                        obsoleteRejectRule := strings.Fields(fmt.Sprintf(`-m mark ! --mark 0x4000/0x4000 -m set --match-set %s dst -m conntrack --ctstate NEW -j REJECT`, svcMatchset))

                        rejectRulesToClean := []util.IPTableRule{

                                {Table: "filter", Chain: "INPUT", Rule: rejectRule},

                                {Table: "filter", Chain: "OUTPUT", Rule: rejectRule},

                                {Table: "filter", Chain: "INPUT", Rule: obsoleteRejectRule},

                                {Table: "filter", Chain: "OUTPUT", Rule: obsoleteRejectRule},

                        }

                        for _, rule := range rejectRulesToClean {

                                if err = deleteIptablesRule(ipt, rule); err != nil {

                                        klog.Errorf("failed to delete service reject iptables rule %v: %v", rule, err)

                                        return err

                                }

                        var filteredRules []util.IPTableRule

                        for _, rule := range iptablesRules {

                                ruleStr := strings.Join(rule.Rule, " ")

                                // Skip MARK 0x4000 rule in OvnPrerouting (pod → service marking)

                                if rule.Table == NAT && rule.Chain == OvnPrerouting && strings.Contains(ruleStr, "0x4000/0x4000") {

                                        continue

                                if rule.Table == NAT && rule.Chain == OvnPostrouting && strings.Contains(ruleStr, "0x4000/0x4000") && strings.Contains(ruleStr, OvnMasquerade) {

                                        continue

                                if rule.Table == "filter" && strings.Contains(ruleStr, svcMatchset) {

                                        if err = deleteIptablesRule(ipt, rule); err != nil {

                                                klog.Errorf("failed to delete service iptables rule %v: %v", rule, err)

                                                return err

                                        }

                                        continue

                                filteredRules = append(filteredRules, rule)

                        iptablesRules = filteredRules

                } else if nodeIP := nodeIPs[protocol]; nodeIP != "" {

                        rules := make([]util.IPTableRule, len(iptablesRules)+1)

                        copy(rules, iptablesRules[:1])