kubeovn/kube-ovn | Build 13853401919 | pkg/daemon/gateway_linux.go | Coveralls

78

func (c *Controller) setIPSet() error {

×

79

        protocols := make([]string, 2)

×

80

        if c.protocol == kubeovnv1.ProtocolDual {

×

81

                protocols[0] = kubeovnv1.ProtocolIPv4

×

82

                protocols[1] = kubeovnv1.ProtocolIPv6

×

83

        } else {

×

84

                protocols[0] = c.protocol

×

85

×

87

        for _, protocol := range protocols {

×

88

                if c.ipsets[protocol] == nil {

×

89

                        continue

×

91

                services := c.getServicesCIDR(protocol)

×

92

                subnets, _, err := c.getDefaultVpcSubnetsCIDR(protocol)

×

93

                if err != nil {

×

94

                        klog.Errorf("get subnets failed, %+v", err)

×

95

                        return err

×

96

×

97

                subnetsNeedNat, err := c.getSubnetsNeedNAT(protocol)

×

98

                if err != nil {

×

99

                        klog.Errorf("get need nat subnets failed, %+v", err)

×

100

                        return err

×

101

×

102

                subnetsDistributedGateway, err := c.getSubnetsDistributedGateway(protocol)

×

103

                if err != nil {

×

104

                        klog.Errorf("failed to get subnets with centralized gateway: %v", err)

×

105

                        return err

×

106

×

107

                otherNode, err := c.getOtherNodes(protocol)

×

108

                if err != nil {

×

109

                        klog.Errorf("failed to get node, %+v", err)

×

110

                        return err

×

111

×

112

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

113

                        MaxSize: 1048576,

×

114

                        SetID:   ServiceSet,

×

115

                        Type:    ipsets.IPSetTypeHashNet,

×

116

                }, services)

×

117

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

118

                        MaxSize: 1048576,

×

119

                        SetID:   SubnetSet,

×

120

                        Type:    ipsets.IPSetTypeHashNet,

×

121

                }, subnets)

×

122

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

123

                        MaxSize: 1048576,

×

124

                        SetID:   LocalPodSet,

×

125

                        Type:    ipsets.IPSetTypeHashIP,

×

126

                }, nil)

×

127

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

128

                        MaxSize: 1048576,

×

129

                        SetID:   SubnetNatSet,

×

130

                        Type:    ipsets.IPSetTypeHashNet,

×

131

                }, subnetsNeedNat)

×

132

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

133

                        MaxSize: 1048576,

×

134

                        SetID:   SubnetDistributedGwSet,

×

135

                        Type:    ipsets.IPSetTypeHashNet,

×

136

                }, subnetsDistributedGateway)

×

137

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

138

                        MaxSize: 1048576,

×

139

                        SetID:   OtherNodeSet,

×

140

                        Type:    ipsets.IPSetTypeHashNet,

×

141

                }, otherNode)

×

142

                c.reconcileNatOutGoingPolicyIPset(protocol)

×

143

                c.ipsets[protocol].ApplyUpdates()

×

145

        return nil

×

148

func (c *Controller) gcIPSet() {

×

149

        protocols := make([]string, 2)

×

150

        if c.protocol == kubeovnv1.ProtocolDual {

×

151

                protocols[0] = kubeovnv1.ProtocolIPv4

×

152

                protocols[1] = kubeovnv1.ProtocolIPv6

×

153

        } else {

×

154

                protocols[0] = c.protocol

×

155

×

157

        for _, protocol := range protocols {

×

158

                if c.ipsets[protocol] == nil {

×

159

                        continue

×

161

                c.ipsets[protocol].ApplyDeletions()

×

165

func (c *Controller) addNatOutGoingPolicyRuleIPset(rule kubeovnv1.NatOutgoingPolicyRuleStatus, protocol string) {

×

166

        if rule.Match.SrcIPs != "" {

×

167

                ipsetName := getNatOutGoingPolicyRuleIPSetName(rule.RuleID, "src", "", false)

×

168

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

169

                        MaxSize: 1048576,

×

170

                        SetID:   ipsetName,

×

171

                        Type:    ipsets.IPSetTypeHashNet,

×

172

                }, strings.Split(rule.Match.SrcIPs, ","))

×

173

×

175

        if rule.Match.DstIPs != "" {

×

176

                ipsetName := getNatOutGoingPolicyRuleIPSetName(rule.RuleID, "dst", "", false)

×

177

                c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

178

                        MaxSize: 1048576,

×

179

                        SetID:   ipsetName,

×

180

                        Type:    ipsets.IPSetTypeHashNet,

×

181

                }, strings.Split(rule.Match.DstIPs, ","))

×

182

×

185

func (c *Controller) removeNatOutGoingPolicyRuleIPset(protocol string, natPolicyRuleIDs *strset.Set) {

×

186

        sets, err := c.k8sipsets.ListSets()

×

187

        if err != nil {

×

188

                klog.Errorf("failed to list ipsets: %v", err)

×

189

                return

×

190

×

191

        for _, set := range sets {

×

192

                if isNatOutGoingPolicyRuleIPSet(set) {

×

193

                        ruleID, _ := getNatOutGoingPolicyRuleIPSetItem(set)

×

194

                        if !natPolicyRuleIDs.Has(ruleID) {

×

195

                                c.ipsets[protocol].RemoveIPSet(formatIPsetUnPrefix(set))

×

196

×

201

func (c *Controller) reconcileNatOutGoingPolicyIPset(protocol string) {

×

202

        subnets, err := c.getSubnetsNatOutGoingPolicy(protocol)

×

203

        if err != nil {

×

204

                klog.Errorf("failed to get subnets with NAT outgoing policy rule: %v", err)

×

205

                return

×

206

×

208

        subnetCidrs := make([]string, 0, len(subnets))

×

209

        natPolicyRuleIDs := strset.New()

×

210

        for _, subnet := range subnets {

×

211

                cidrBlock, err := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)

×

212

                if err != nil {

×

213

                        klog.Errorf("failed to get subnet %s CIDR block by protocol: %v", subnet.Name, err)

×

214

                        continue

×

216

                if cidrBlock != "" {

×

217

                        subnetCidrs = append(subnetCidrs, cidrBlock)

×

218

×

219

                for _, rule := range subnet.Status.NatOutgoingPolicyRules {

×

220

                        if rule.RuleID == "" {

×

221

                                klog.Errorf("unexpected empty ID for NAT outgoing rule %q of subnet %s", rule.NatOutgoingPolicyRule, subnet.Name)

×

222

                                continue

×

224

                        natPolicyRuleIDs.Add(rule.RuleID)

×

225

                        c.addNatOutGoingPolicyRuleIPset(rule, protocol)

×

229

        c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{

×

230

                MaxSize: 1048576,

×

231

                SetID:   NatOutGoingPolicySubnetSet,

×

232

                Type:    ipsets.IPSetTypeHashNet,

×

233

        }, subnetCidrs)

×

234

×

235

        c.removeNatOutGoingPolicyRuleIPset(protocol, natPolicyRuleIDs)

×

238

func (c *Controller) setPolicyRouting() error {

×

239

        protocols := make([]string, 2)

×

240

        if c.protocol == kubeovnv1.ProtocolDual {

×

241

                protocols[0] = kubeovnv1.ProtocolIPv4

×

242

                protocols[1] = kubeovnv1.ProtocolIPv6

×

243

        } else {

×

244

                protocols[0] = c.protocol

×

245

×

247

        for _, protocol := range protocols {

×

248

                if c.ipsets[protocol] == nil {

×

249

                        continue

×

252

                localPodIPs, err := c.getLocalPodIPsNeedPR(protocol)

×

253

                if err != nil {

×

254

                        klog.Errorf("failed to get local pod ips failed: %+v", err)

×

255

                        return err

×

256

×

257

                subnetsNeedPR, err := c.getSubnetsNeedPR(protocol)

×

258

                if err != nil {

×

259

                        klog.Errorf("failed to get subnets that need policy routing: %+v", err)

×

260

                        return err

×

261

×

263

                family, err := util.ProtocolToFamily(protocol)

×

264

                if err != nil {

×

265

                        klog.Error(err)

×

266

                        return err

×

267

×

269

                for meta, ips := range localPodIPs {

×

270

                        if err = c.addPolicyRouting(family, meta.gateway, meta.priority, meta.tableID, ips...); err != nil {

×

271

                                klog.Errorf("failed to add policy routing for local pods: %+v", err)

×

272

                                return err

×

273

×

275

                for meta, cidr := range subnetsNeedPR {

×

276

                        if err = c.addPolicyRouting(family, meta.gateway, meta.priority, meta.tableID, cidr); err != nil {

×

277

                                klog.Errorf("failed to add policy routing for subnet: %+v", err)

×

278

                                return err

×

279

×

283

        return nil

×

286

func (c *Controller) addPodPolicyRouting(podProtocol, externalEgressGateway string, priority, tableID uint32, ips []string) error {

×

287

        egw := strings.Split(externalEgressGateway, ",")

×

288

        prMetas := make([]policyRouteMeta, 0, 2)

×

289

        if len(egw) == 1 {

×

290

                family, _ := util.ProtocolToFamily(util.CheckProtocol(egw[0]))

×

291

                if family == netlink.FAMILY_V4 || podProtocol != kubeovnv1.ProtocolDual {

×

292

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[0], gateway: egw[0]})

×

293

                } else {

×

294

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[1], gateway: egw[0]})

×

295

×

296

        } else {

×

297

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V4, source: ips[0], gateway: egw[0]})

×

298

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V6, source: ips[1], gateway: egw[1]})

×

299

×

301

        for _, meta := range prMetas {

×

302

                if err := c.addPolicyRouting(meta.family, meta.gateway, priority, tableID, meta.source); err != nil {

×

303

                        klog.Errorf("failed to add policy routing for pod: %+v", err)

×

304

                        return err

×

305

×

308

        return nil

×

311

func (c *Controller) deletePodPolicyRouting(podProtocol, externalEgressGateway string, priority, tableID uint32, ips []string) error {

×

312

        egw := strings.Split(externalEgressGateway, ",")

×

313

        prMetas := make([]policyRouteMeta, 0, 2)

×

314

        if len(egw) == 1 {

×

315

                family, _ := util.ProtocolToFamily(util.CheckProtocol(egw[0]))

×

316

                if family == netlink.FAMILY_V4 || podProtocol != kubeovnv1.ProtocolDual {

×

317

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[0], gateway: egw[0]})

×

318

                } else {

×

319

                        prMetas = append(prMetas, policyRouteMeta{family: family, source: ips[1], gateway: egw[0]})

×

320

×

321

        } else {

×

322

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V4, source: ips[0], gateway: egw[0]})

×

323

                prMetas = append(prMetas, policyRouteMeta{family: netlink.FAMILY_V6, source: ips[1], gateway: egw[1]})

×

324

×

326

        for _, meta := range prMetas {

×

327

                if err := c.deletePolicyRouting(meta.family, meta.gateway, priority, tableID, meta.source); err != nil {

×

328

                        klog.Errorf("failed to delete policy routing for pod: %+v", err)

×

329

                        return err

×

330

×

333

        return nil

×

336

func (c *Controller) addPolicyRouting(family int, gateway string, priority, tableID uint32, ips ...string) error {

×

337

        route := &netlink.Route{

×

338

                Protocol: netlink.RouteProtocol(syscall.RTPROT_STATIC),

×

339

                Gw:       net.ParseIP(gateway),

×

340

                Table:    int(tableID),

×

341

×

342

        if err := netlink.RouteReplace(route); err != nil && !errors.Is(err, syscall.EEXIST) {

×

343

                err = fmt.Errorf("failed to replace route in table %d: %w", tableID, err)

×

344

                klog.Error(err)

×

345

                return err

×

346

×

348

        maskBits := 32

×

349

        if family == netlink.FAMILY_V6 {

×

350

                maskBits = 128

×

351

×

353

        rule := netlink.NewRule()

×

354

        rule.Family = family

×

355

        rule.Table = int(tableID)

×

356

        rule.Priority = int(priority)

×

357

        mask := net.CIDRMask(maskBits, maskBits)

×

358

×

359

        for _, ip := range ips {

×

360

                if strings.ContainsRune(ip, '/') {

×

361

                        var err error

×

362

                        if rule.Src, err = netlink.ParseIPNet(ip); err != nil {

×

363

                                klog.Errorf("unexpected CIDR: %s", ip)

×

364

                                err = fmt.Errorf("failed to add route in table %d: %w", tableID, err)

×

365

                                klog.Error(err)

×

366

                                return err

×

367

×

368

                } else {

×

369

                        rule.Src = &net.IPNet{IP: net.ParseIP(ip), Mask: mask}

×

370

×

372

                if err := netlink.RuleAdd(rule); err != nil && !errors.Is(err, syscall.EEXIST) {

×

373

                        err = fmt.Errorf("failed to add network rule: %w", err)

×

374

                        klog.Error(err)

×

375

                        return err

×

376

×

379

        return nil

×

382

func (c *Controller) deletePolicyRouting(family int, _ string, priority, tableID uint32, ips ...string) error {

×

383

        maskBits := 32

×

384

        if family == netlink.FAMILY_V6 {

×

385

                maskBits = 128

×

386

×

388

        rule := netlink.NewRule()

×

389

        rule.Family = family

×

390

        rule.Table = int(tableID)

×

391

        rule.Priority = int(priority)

×

392

        mask := net.CIDRMask(maskBits, maskBits)

×

393

×

394

        for _, ip := range ips {

×

395

                if strings.ContainsRune(ip, '/') {

×

396

                        var err error

×

397

                        if rule.Src, err = netlink.ParseIPNet(ip); err != nil {

×

398

                                klog.Errorf("unexpected CIDR: %s", ip)

×

399

                                err = fmt.Errorf("failed to delete route in table %d: %w", tableID, err)

×

400

                                klog.Error(err)

×

401

                                return err

×

402

×

403

                } else {

×

404

                        rule.Src = &net.IPNet{IP: net.ParseIP(ip), Mask: mask}

×

405

×

407

                if err := netlink.RuleDel(rule); err != nil && !errors.Is(err, syscall.ENOENT) {

×

408

                        err = fmt.Errorf("failed to delete network rule: %w", err)

×

409

                        klog.Error(err)

×

410

                        return err

×

411

×

415

        return nil

×

418

func (c *Controller) createIptablesRule(ipt *iptables.IPTables, rule util.IPTableRule) error {

×

419

        exists, err := ipt.Exists(rule.Table, rule.Chain, rule.Rule...)

×

420

        if err != nil {

×

421

                klog.Errorf("failed to check iptables rule existence: %v", err)

×

422

                return err

×

423

×

425

        s := strings.Join(rule.Rule, " ")

×

426

        if exists {

×

427

                if rule.Table == NAT && rule.Chain == Prerouting {

×

428

                        // make sure the nat prerouting iptable rule is in the first position

×

429

                        natPreroutingRules, err := ipt.List(rule.Table, rule.Chain)

×

430

                        if err != nil {

×

431

                                klog.Errorf("failed to list iptables rules: %v", err)

×

432

                                return err

×

433

×

434

                        for i, r := range natPreroutingRules {

×

435

                                ruleSpec := util.DoubleQuotedFields(r)

×

436

                                if i == 0 || len(ruleSpec) < 3 {

×

437

                                        continue

×

439

                                if i == 1 {

×

440

                                        if slices.Equal(ruleSpec[2:], rule.Rule) {

×

441

                                                klog.V(3).Infof("the first nat prerouting rule is %q", rule.Rule)

×

442

                                                continue

×

445

                                        klog.Infof("insert nat prerouting rule: %q", rule.Rule)

×

446

                                        if err = ipt.Insert(rule.Table, rule.Chain, 1, rule.Rule...); err != nil {

×

447

                                                klog.Errorf(`failed to insert iptables rule %q: %v`, s, err)

×

448

                                                return err

×

449

×

450

                                        return nil

×

452

                                if slices.Equal(ruleSpec[2:], rule.Rule) {

×

453

                                        rule.Pos = strconv.Itoa(i)

×

454

                                        klog.Warningf("delete the nat prerouting rule: %v", rule)

×

455

                                        if err = deleteIptablesRule(ipt, rule); err != nil {

×

456

                                                klog.Errorf("failed to delete rule %v: %v", rule, err)

×

457

                                                return err

×

458

×

462

                return nil

×

465

        klog.Infof("creating iptables rule in table %s chain %s at position %d: %q", rule.Table, rule.Chain, 1, s)

×

466

        if err = ipt.Insert(rule.Table, rule.Chain, 1, rule.Rule...); err != nil {

×

467

                klog.Errorf(`failed to insert iptables rule "%s": %v`, s, err)

×

468

                return err

×

469

×

471

        return nil

×

474

func (c *Controller) updateIptablesChain(ipt *iptables.IPTables, table, chain, parent string, rules []util.IPTableRule) error {

×

475

        ok, err := ipt.ChainExists(table, chain)

×

476

        if err != nil {

×

477

                klog.Errorf("failed to check existence of iptables chain %s in table %s: %v", chain, table, err)

×

478

                return err

×

479

×

480

        if !ok {

×

481

                if err = ipt.NewChain(table, chain); err != nil {

×

482

                        klog.Errorf("failed to create iptables chain %s in table %s: %v", chain, table, err)

×

483

                        return err

×

484

×

485

                klog.Infof("created iptables chain %s in table %s", chain, table)

×

487

        if parent != "" {

×

488

                comment := fmt.Sprintf("kube-ovn %s rules", strings.ToLower(parent))

×

489

                rule := util.IPTableRule{

×

490

                        Table: table,

×

491

                        Chain: parent,

×

492

                        Rule:  []string{"-m", "comment", "--comment", comment, "-j", chain},

×

493

×

494

                if err = c.createIptablesRule(ipt, rule); err != nil {

×

495

                        klog.Errorf("failed to create iptables rule: %v", err)

×

496

                        return err

×

497

×

501

        ruleList, err := ipt.List(table, chain)

×

502

        if err != nil {

×

503

                klog.Errorf("failed to list iptables rules in chain %s/%s: %v", table, chain, err)

×

504

                return err

×

505

×

508

        ruleList = ruleList[1:]

×

509

×

510

        // trim prefix: "-A OVN-POSTROUTING "

×

511

        prefixLen := 4 + len(chain)

×

512

        existingRules := make([][]string, 0, len(ruleList))

×

513

        for _, r := range ruleList {

×

514

                existingRules = append(existingRules, util.DoubleQuotedFields(r[prefixLen:]))

×

515

×

517

        var added int

×

518

        for i, rule := range rules {

×

519

                if i-added < len(existingRules) && slices.Equal(existingRules[i-added], rule.Rule) {

×

520

                        klog.V(5).Infof("iptables rule %v already exists", rule.Rule)

×

521

                        continue

×

523

                klog.Infof("creating iptables rule in table %s chain %s at position %d: %q", table, chain, i+1, strings.Join(rule.Rule, " "))

×

524

                if err = ipt.Insert(table, chain, i+1, rule.Rule...); err != nil {

×

525

                        klog.Errorf(`failed to insert iptables rule %v: %v`, rule.Rule, err)

×

526

                        return err

×

527

×

528

                added++

×

530

        for i := len(existingRules) - 1; i >= len(rules)-added; i-- {

×

531

                if err = ipt.Delete(table, chain, strconv.Itoa(i+added+1)); err != nil {

×

532

                        klog.Errorf(`failed to delete iptables rule %v: %v`, existingRules[i], err)

×

533

                        return err

×

534

×

535

                klog.Infof("deleted iptables rule in table %s chain %s: %q", table, chain, strings.Join(existingRules[i], " "))

×

538

        return nil

×

541

func (c *Controller) setIptables() error {

×

542

        klog.V(3).Infoln("start to set up iptables")

×

543

        node, err := c.nodesLister.Get(c.config.NodeName)

×

544

        if err != nil {

×

545

                klog.Errorf("failed to get node %s, %v", c.config.NodeName, err)

×

546

                return err

×

547

×

549

        nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node)

×

550

        nodeIPs := map[string]string{

×

551

                kubeovnv1.ProtocolIPv4: nodeIPv4,

×

552

                kubeovnv1.ProtocolIPv6: nodeIPv6,

×

553

×

554

×

555

        centralGwNatIPs, err := c.getEgressNatIPByNode(c.config.NodeName)

×

556

        if err != nil {

×

557

                klog.Errorf("failed to get centralized subnets nat ips on node %s, %v", c.config.NodeName, err)

×

558

                return err

×

559

×

560

        klog.V(3).Infof("centralized subnets nat ips %v", centralGwNatIPs)

×

561

×

562

        var (

×

563

                v4Rules = []util.IPTableRule{

×

564

                        // mark packets from pod to service

×

565

                        {Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000`)},

×

566

                        // nat packets marked by kube-proxy or kube-ovn

×

567

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j ` + OvnMasquerade)},

×

568

                        // nat service traffic

×

569

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j ` + OvnMasquerade)},

×

570

                        // do not nat node port service traffic with external traffic policy set to local

×

571

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN`)},

×

572

                        // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway

×

573

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnMasquerade)},

×

574

                        // do not nat reply packets in direct routing

×

575

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},

×

576

                        // do not nat route traffic

×

577

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)},

×

578

                        // nat outgoing policy rules

×

579

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m set --match-set ovn40subnets-nat-policy src -m set ! --match-set ovn40subnets dst -j %s`, OvnNatOutGoingPolicy))},

×

580

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j %s`, OnOutGoingNatMark, OvnMasquerade))},

×

581

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))},

×

582

                        // default nat outgoing rules

×

583

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j ` + OvnMasquerade)},

×

584

                        // clear mark

×

585

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MARK --set-xmark 0x0/0xffffffff`)},

×

586

                        // do masquerade

×

587

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MASQUERADE`)},

×

588

                        // Input Accept

×

589

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},

×

590

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},

×

591

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},

×

592

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},

×

593

                        // Forward Accept

×

594

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},

×

595

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},

×

596

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},

×

597

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},

×

598

                        // Output unmark to bypass kernel nat checksum issue https://github.com/flannel-io/flannel/issues/1279

×

599

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 6081 -j MARK --set-xmark 0x0`)},

×

600

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 4789 -j MARK --set-xmark 0x0`)},

×

601

                        // Drop invalid rst

×

602

                        {Table: MANGLE, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m set --match-set ovn40subnets src -m tcp --tcp-flags RST RST -m state --state INVALID -j DROP`)},

×

603

×

604

                v6Rules = []util.IPTableRule{

×

605

                        // mark packets from pod to service

×

606

                        {Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000`)},

×

607

                        // nat packets marked by kube-proxy or kube-ovn

×

608

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j ` + OvnMasquerade)},

×

609

                        // nat service traffic

×

610

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets src -m set --match-set ovn60subnets dst -j ` + OvnMasquerade)},

×

611

                        // do not nat node port service traffic with external traffic policy set to local

×

612

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN`)},

×

613

                        // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway

×

614

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnMasquerade)},

×

615

                        // do not nat reply packets in direct routing

×

616

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},

×

617

                        // do not nat route traffic

×

618

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`)},

×

619

                        // nat outgoing policy rules

×

620

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m set --match-set ovn60subnets-nat-policy src -m set ! --match-set ovn60subnets dst -j %s`, OvnNatOutGoingPolicy))},

×

621

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j %s`, OnOutGoingNatMark, OvnMasquerade))},

×

622

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))},

×

623

                        {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j ` + OvnMasquerade)},

×

624

                        // clear mark

×

625

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MARK --set-xmark 0x0/0xffffffff`)},

×

626

                        // do masquerade

×

627

                        {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MASQUERADE`)},

×

628

                        // Input Accept

×

629

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},

×

630

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},

×

631

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},

×

632

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},

×

633

                        // Forward Accept

×

634

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},

×

635

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},

×

636

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},

×

637

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},

×

638

                        // Output unmark to bypass kernel nat checksum issue https://github.com/flannel-io/flannel/issues/1279

×

639

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 6081 -j MARK --set-xmark 0x0`)},

×

640

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 4789 -j MARK --set-xmark 0x0`)},

×

641

                        // Drop invalid rst

×

642

                        {Table: MANGLE, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m set --match-set ovn60subnets src -m tcp --tcp-flags RST RST -m state --state INVALID -j DROP`)},

×

643

×

644

×

NEW

645

        protocols := make([]string, 0, 2)

×

646

        if c.protocol == kubeovnv1.ProtocolDual {

×

NEW

647

                protocols = append(protocols, kubeovnv1.ProtocolIPv4, kubeovnv1.ProtocolIPv6)

×

648

        } else {

×

NEW

649

                protocols = append(protocols, c.protocol)

×

650

×

652

        for _, protocol := range protocols {

×

653

                ipt := c.iptables[protocol]

×

654

                if ipt == nil {

×

655

                        continue

×

658

                var kubeProxyIpsetProtocol, matchset, svcMatchset, nodeMatchSet string

×

659

                var obsoleteRules, iptablesRules []util.IPTableRule

×

660

                if protocol == kubeovnv1.ProtocolIPv4 {

×

661

                        iptablesRules = v4Rules

×

662

                        matchset, svcMatchset, nodeMatchSet = "ovn40subnets", "ovn40services", "ovn40"+OtherNodeSet

×

663

                } else {

×

664

                        iptablesRules = v6Rules

×

665

                        kubeProxyIpsetProtocol, matchset, svcMatchset, nodeMatchSet = "6-", "ovn60subnets", "ovn60services", "ovn60"+OtherNodeSet

×

666

×

668

                ipset := fmt.Sprintf("KUBE-%sCLUSTER-IP", kubeProxyIpsetProtocol)

×

669

                ipsetExists, err := c.ipsetExists(ipset)

×

670

                if err != nil {

×

671

                        klog.Errorf("failed to check existence of ipset %s: %v", ipset, err)

×

672

                        return err

×

673

×

674

                if ipsetExists {

×

675

                        iptablesRules[0].Rule = strings.Fields(fmt.Sprintf(`-i ovn0 -m set --match-set %s src -m set --match-set %s dst,dst -j MARK --set-xmark 0x4000/0x4000`, matchset, ipset))

×

676

                        rejectRule := strings.Fields(fmt.Sprintf(`-p tcp -m mark ! --mark 0x4000/0x4000 -m set --match-set %s dst -m conntrack --ctstate NEW -j REJECT`, svcMatchset))

×

677

                        obsoleteRejectRule := strings.Fields(fmt.Sprintf(`-m mark ! --mark 0x4000/0x4000 -m set --match-set %s dst -m conntrack --ctstate NEW -j REJECT`, svcMatchset))

×

678

                        iptablesRules = append(iptablesRules,

×

679

                                util.IPTableRule{Table: "filter", Chain: "INPUT", Rule: rejectRule},

×

680

                                util.IPTableRule{Table: "filter", Chain: "OUTPUT", Rule: rejectRule},

×

681

×

682

                        obsoleteRejectRules := []util.IPTableRule{

×

683

                                {Table: "filter", Chain: "INPUT", Rule: obsoleteRejectRule},

×

684

                                {Table: "filter", Chain: "OUTPUT", Rule: obsoleteRejectRule},

×

685

×

686

                        for _, rule := range obsoleteRejectRules {

×

687

                                if err = deleteIptablesRule(ipt, rule); err != nil {

×

688

                                        klog.Errorf("failed to delete obsolete iptables rule %v: %v", rule, err)

×

689

                                        return err

×

690

×

694

                if nodeIP := nodeIPs[protocol]; nodeIP != "" {

×

695

                        obsoleteRules = []util.IPTableRule{

×

696

                                {Table: NAT, Chain: Postrouting, Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset))},

×

697

                                {Table: NAT, Chain: Postrouting, Rule: strings.Fields(fmt.Sprintf(`! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, nodeIP))},

×

698

                                {Table: NAT, Chain: Postrouting, Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set ! --match-set %s src -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset, matchset))},

×

699

×

700

×

701

                        rules := make([]util.IPTableRule, len(iptablesRules)+1)

×

702

                        copy(rules, iptablesRules[:1])

×

703

                        copy(rules[2:], iptablesRules[1:])

×

704

                        rules[1] = util.IPTableRule{

×

705

                                Table: NAT,

×

706

                                Chain: OvnPostrouting,

×

707

                                Rule:  strings.Fields(fmt.Sprintf(`-m set --match-set %s src -m set --match-set %s dst -m mark --mark 0x4000/0x4000 -j SNAT --to-source %s`, svcMatchset, matchset, nodeIP)),

×

708

×

709

                        iptablesRules = rules

×

710

×

711

                        for _, p := range [...]string{"tcp", "udp"} {

×

712

                                ipset := fmt.Sprintf("KUBE-%sNODE-PORT-LOCAL-%s", kubeProxyIpsetProtocol, strings.ToUpper(p))

×

713

                                ipsetExists, err := c.ipsetExists(ipset)

×

714

                                if err != nil {

×

715

                                        klog.Errorf("failed to check existence of ipset %s: %v", ipset, err)

×

716

                                        return err

×

717

×

718

                                if !ipsetExists {

×

719

                                        klog.V(5).Infof("ipset %s does not exist", ipset)

×

720

                                        continue

×

722

                                rule := fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL -m set --match-set %s dst -j MARK --set-xmark 0x80000/0x80000", p, ipset)

×

723

                                rule2 := fmt.Sprintf("-p %s -m set --match-set %s src -m set --match-set %s dst -j MARK --set-xmark 0x4000/0x4000", p, nodeMatchSet, ipset)

×

724

                                obsoleteRules = append(obsoleteRules, util.IPTableRule{Table: NAT, Chain: Prerouting, Rule: strings.Fields(rule)})

×

725

                                iptablesRules = append(iptablesRules,

×

726

                                        util.IPTableRule{Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(rule)},

×

727

                                        util.IPTableRule{Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(rule2)},

×

728

×

732

                _, subnetCidrs, err := c.getDefaultVpcSubnetsCIDR(protocol)

×

733

                if err != nil {

×

734

                        klog.Errorf("get subnets failed, %+v", err)

×

735

                        return err

×

736

×

738

                subnetNames := set.New[string]()

×

739

                for name, subnetCidr := range subnetCidrs {

×

740

                        subnetNames.Insert(name)

×

741

                        iptablesRules = append(iptablesRules,

×

742

                                util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(fmt.Sprintf(`-m comment --comment %s,%s -s %s`, util.OvnSubnetGatewayIptables, name, subnetCidr))},

×

743

                                util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(fmt.Sprintf(`-m comment --comment %s,%s -d %s`, util.OvnSubnetGatewayIptables, name, subnetCidr))},

×

744

×

745

×

747

                rules, err := ipt.List("filter", "FORWARD")

×

748

                if err != nil {

×

749

                        klog.Errorf(`failed to list iptables rule table "filter" chain "FORWARD" with err %v `, err)

×

750

                        return err

×

751

×

753

                pattern := fmt.Sprintf(`-m comment --comment "%s,`, util.OvnSubnetGatewayIptables)

×

754

                for _, rule := range rules {

×

755

                        if !strings.Contains(rule, pattern) {

×

756

                                continue

×

758

                        fields := util.DoubleQuotedFields(rule)

×

759

                        // -A FORWARD -d 10.16.0.0/16 -m comment --comment "ovn-subnet-gateway,ovn-default"

×

760

                        if len(fields) != 8 || fields[6] != "--comment" {

×

761

                                continue

×

763

                        commentFields := strings.Split(fields[7], ",")

×

764

                        if len(commentFields) != 2 {

×

765

                                continue

×

767

                        if subnetNames.Has(commentFields[1]) {

×

768

                                continue

×

772

                        if err = deleteIptablesRule(ipt, util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: fields[2:]}); err != nil {

×

773

                                klog.Error(err)

×

774

                                return err

×

775

×

778

                var natPreroutingRules, natPostroutingRules, ovnMasqueradeRules, manglePostroutingRules []util.IPTableRule

×

779

                for _, rule := range iptablesRules {

×

780

                        if rule.Table == NAT {

×

781

                                if c.k8siptables[protocol].HasRandomFully() &&

×

782

                                        (rule.Rule[len(rule.Rule)-1] == "MASQUERADE" || slices.Contains(rule.Rule, "SNAT")) {

×

783

                                        rule.Rule = append(rule.Rule, "--random-fully")

×

784

×

786

                                switch rule.Chain {

×

787

                                case OvnPrerouting:

×

788

                                        natPreroutingRules = append(natPreroutingRules, rule)

×

789

                                        continue

×

790

                                case OvnPostrouting:

×

791

                                        natPostroutingRules = append(natPostroutingRules, rule)

×

792

                                        continue

×

793

                                case OvnMasquerade:

×

794

                                        ovnMasqueradeRules = append(ovnMasqueradeRules, rule)

×

795

                                        continue

×

797

                        } else if rule.Table == MANGLE {

×

798

                                if rule.Chain == OvnPostrouting {

×

799

                                        manglePostroutingRules = append(manglePostroutingRules, rule)

×

800

                                        continue

×

804

                        if err = c.createIptablesRule(ipt, rule); err != nil {

×

805

                                klog.Errorf(`failed to create iptables rule "%s": %v`, strings.Join(rule.Rule, " "), err)

×

806

                                return err

×

807

×

810

                var randomFully string

×

811

                if c.k8siptables[protocol].HasRandomFully() {

×

812

                        randomFully = "--random-fully"

×

813

×

816

                for cidr, ip := range centralGwNatIPs {

×

817

                        if util.CheckProtocol(cidr) != protocol {

×

818

                                continue

×

821

                        s := fmt.Sprintf("-s %s -m set ! --match-set %s dst -j SNAT --to-source %s %s", cidr, matchset, ip, randomFully)

×

822

                        rule := util.IPTableRule{

×

823

                                Table: NAT,

×

824

                                Chain: OvnPostrouting,

×

825

                                Rule:  util.DoubleQuotedFields(s),

×

826

×

827

                        // insert the rule before the one for nat outgoing

×

828

                        n := len(natPostroutingRules)

×

829

                        natPostroutingRules = append(natPostroutingRules[:n-1], rule, natPostroutingRules[n-1])

×

832

                if err = c.reconcileNatOutgoingPolicyIptablesChain(protocol); err != nil {

×

833

                        klog.Error(err)

×

834

                        return err

×

835

×

NEW

837

                if err = c.reconcileTProxyIPTableRules(protocol); err != nil {

×

838

                        klog.Error(err)

×

839

                        return err

×

840

×

842

                if err = c.updateIptablesChain(ipt, NAT, OvnPrerouting, Prerouting, natPreroutingRules); err != nil {

×

843

                        klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnPrerouting, err)

×

844

                        return err

×

845

×

846

                if err = c.updateIptablesChain(ipt, NAT, OvnMasquerade, "", ovnMasqueradeRules); err != nil {

×

847

                        klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnMasquerade, err)

×

848

                        return err

×

849

×

850

                if err = c.updateIptablesChain(ipt, NAT, OvnPostrouting, Postrouting, natPostroutingRules); err != nil {

×

851

                        klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnPostrouting, err)

×

852

                        return err

×

853

×

855

                if err = c.updateIptablesChain(ipt, MANGLE, OvnPostrouting, Postrouting, manglePostroutingRules); err != nil {

×

856

                        klog.Errorf("failed to update chain %s/%s: %v", MANGLE, OvnPostrouting, err)

×

857

                        return err

×

858

×

860

                if err = c.cleanObsoleteIptablesRules(protocol, obsoleteRules); err != nil {

×

861

                        klog.Errorf("failed to clean legacy iptables rules: %v", err)

×

862

                        return err

×

863

×

865

        return nil

×

NEW

868

func (c *Controller) reconcileTProxyIPTableRules(protocol string) error {

×

869

        if !c.config.EnableTProxy {

×

870

                return nil

×

871

×

873

        ipt := c.iptables[protocol]

×

874

        tproxyPreRoutingRules := make([]util.IPTableRule, 0)

×

875

        tproxyOutputRules := make([]util.IPTableRule, 0)

×

876

×

877

        pods, err := c.getTProxyConditionPod(true)

×

878

        if err != nil {

×

879

                klog.Error(err)

×

880

                return err

×

881

×

883

        for _, pod := range pods {

×

884

                var podIP string

×

885

                for _, ip := range pod.Status.PodIPs {

×

886

                        if util.CheckProtocol(ip.IP) == protocol {

×

887

                                podIP = ip.IP

×

888

                                break

×

892

                if podIP == "" {

×

893

                        continue

×

896

                ports := getProbePorts(pod)

×

897

                if ports.Len() == 0 {

×

898

                        continue

×

901

                for _, probePort := range ports.SortedList() {

×

NEW

902

                        hostIP := c.config.NodeIPv4

×

903

                        prefixLen := 32

×

904

                        if protocol == kubeovnv1.ProtocolIPv6 {

×

905

                                prefixLen = 128

×

NEW

906

                                hostIP = c.config.NodeIPv6

×

UNCOV

907

×

909

                        tproxyOutputRules = append(tproxyOutputRules, util.IPTableRule{Table: MANGLE, Chain: OvnOutput, Rule: strings.Fields(fmt.Sprintf(`-d %s/%d -p tcp -m tcp --dport %d -j MARK --set-xmark %s`, podIP, prefixLen, probePort, tProxyOutputMarkMask))})

×

910

                        tproxyPreRoutingRules = append(tproxyPreRoutingRules, util.IPTableRule{Table: MANGLE, Chain: OvnPrerouting, Rule: strings.Fields(fmt.Sprintf(`-d %s/%d -p tcp -m tcp --dport %d -j TPROXY --on-port %d --on-ip %s --tproxy-mark %s`, podIP, prefixLen, probePort, util.TProxyListenPort, hostIP, tProxyPreRoutingMarkMask))})

×

914

        if err := c.updateIptablesChain(ipt, MANGLE, OvnPrerouting, Prerouting, tproxyPreRoutingRules); err != nil {

×

915

                klog.Errorf("failed to update chain %s with rules %v: %v", OvnPrerouting, tproxyPreRoutingRules, err)

×

916

                return err

×

917

×

919

        if err := c.updateIptablesChain(ipt, MANGLE, OvnOutput, Output, tproxyOutputRules); err != nil {

×

920

                klog.Errorf("failed to update chain %s with rules %v: %v", OvnOutput, tproxyOutputRules, err)

×

921

                return err

×

922

×

923

        return nil

×

926

func (c *Controller) cleanTProxyIPTableRules(protocol string) {

×

927

        ipt := c.iptables[protocol]

×

928

        if ipt == nil {

×

929

                return

×

930

×

931

        for _, chain := range [2]string{OvnPrerouting, OvnOutput} {

×

932

                if err := ipt.ClearChain(MANGLE, chain); err != nil {

×

933

                        klog.Errorf("failed to clear iptables chain %v in table %v, %+v", chain, MANGLE, err)

×

934

                        return

×

935

×

939

func (c *Controller) reconcileNatOutgoingPolicyIptablesChain(protocol string) error {

×

940

        ipt := c.iptables[protocol]

×

941

×

942

        natPolicySubnetIptables, natPolicyRuleIptablesMap, gcNatPolicySubnetChains, err := c.generateNatOutgoingPolicyChainRules(protocol)

×

943

        if err != nil {

×

944

                klog.Errorf(`failed to get nat policy post routing rules with err %v `, err)

×

945

                return err

×

946

×

948

        for chainName, natPolicyRuleIptableRules := range natPolicyRuleIptablesMap {

×

949

                if err = c.updateIptablesChain(ipt, NAT, chainName, "", natPolicyRuleIptableRules); err != nil {

×

950

                        klog.Errorf("failed to update chain %s with rules %v: %v", chainName, natPolicyRuleIptableRules, err)

×

951

                        return err

×

952

×

955

        if err = c.updateIptablesChain(ipt, NAT, OvnNatOutGoingPolicy, "", natPolicySubnetIptables); err != nil {

×

956

                klog.Errorf("failed to update chain %s: %v", OvnNatOutGoingPolicy, err)

×

957

                return err

×

958

×

960

        for _, gcNatPolicySubnetChain := range gcNatPolicySubnetChains {

×

961

                if err = ipt.ClearAndDeleteChain(NAT, gcNatPolicySubnetChain); err != nil {

×

962

                        klog.Errorf("failed to delete iptables chain %q in table %s: %v", gcNatPolicySubnetChain, NAT, err)

×

963

                        return err

×

964

×

965

                klog.Infof("deleted iptables chain %s in table %s", gcNatPolicySubnetChain, NAT)

×

967

        return nil

×

970

func (c *Controller) generateNatOutgoingPolicyChainRules(protocol string) ([]util.IPTableRule, map[string][]util.IPTableRule, []string, error) {

×

971

        natPolicySubnetIptables := make([]util.IPTableRule, 0)

×

972

        natPolicyRuleIptablesMap := make(map[string][]util.IPTableRule)

×

973

        natPolicySubnetUIDs := strset.New()

×

974

        gcNatPolicySubnetChains := make([]string, 0)

×

975

        subnetNames := make([]string, 0)

×

976

        subnetMap := make(map[string]*kubeovnv1.Subnet)

×

977

×

978

        subnets, err := c.getSubnetsNatOutGoingPolicy(protocol)

×

979

        if err != nil {

×

980

                klog.Errorf("failed to get subnets with NAT outgoing policy rule: %v", err)

×

981

                return nil, nil, nil, err

×

982

×

984

        for _, subnet := range subnets {

×

985

                subnetNames = append(subnetNames, subnet.Name)

×

986

                subnetMap[subnet.Name] = subnet

×

987

×

990

        sort.Strings(subnetNames)

×

991

×

992

        getMatchProtocol := func(ips string) string {

×

993

                ip := strings.Split(ips, ",")[0]

×

994

                return util.CheckProtocol(ip)

×

995

×

997

        for _, subnetName := range subnetNames {

×

998

                subnet := subnetMap[subnetName]

×

999

                var natPolicyRuleIptables []util.IPTableRule

×

1000

                natPolicySubnetUIDs.Add(util.GetTruncatedUID(string(subnet.GetUID())))

×

1001

                cidrBlock, err := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)

×

1002

                if err != nil {

×

1003

                        klog.Errorf("failed to get subnet %s cidr block with protocol: %v", subnet.Name, err)

×

1004

                        continue

×

1006

                if cidrBlock == "" {

×

1007

                        continue

×

1010

                ovnNatPolicySubnetChainName := OvnNatOutGoingPolicySubnet + util.GetTruncatedUID(string(subnet.GetUID()))

×

1011

                natPolicySubnetIptables = append(natPolicySubnetIptables, util.IPTableRule{Table: NAT, Chain: OvnNatOutGoingPolicy, Rule: strings.Fields(fmt.Sprintf(`-s %s -m comment --comment natPolicySubnet-%s -j %s`, cidrBlock, subnet.Name, ovnNatPolicySubnetChainName))})

×

1012

                for _, rule := range subnet.Status.NatOutgoingPolicyRules {

×

1013

                        var markCode string

×

1014

                        if rule.Action == util.NatPolicyRuleActionNat {

×

1015

                                markCode = OnOutGoingNatMark

×

1016

                        } else if rule.Action == util.NatPolicyRuleActionForward {

×

1017

                                markCode = OnOutGoingForwardMark

×

1018

×

1020

                        if rule.RuleID == "" {

×

1021

                                continue

×

1024

                        if rule.Match.SrcIPs != "" && getMatchProtocol(rule.Match.SrcIPs) != protocol {

×

1025

                                continue

×

1028

                        if rule.Match.DstIPs != "" && getMatchProtocol(rule.Match.DstIPs) != protocol {

×

1029

                                continue

×

1032

                        srcMatch := getNatOutGoingPolicyRuleIPSetName(rule.RuleID, "src", protocol, true)

×

1033

                        dstMatch := getNatOutGoingPolicyRuleIPSetName(rule.RuleID, "dst", protocol, true)

×

1034

×

1035

                        var ovnNatoutGoingPolicyRule util.IPTableRule

×

1036

×

1037

                        switch {

×

1038

                        case rule.Match.DstIPs != "" && rule.Match.SrcIPs != "":

×

1039

                                ovnNatoutGoingPolicyRule = util.IPTableRule{Table: NAT, Chain: ovnNatPolicySubnetChainName, Rule: strings.Fields(fmt.Sprintf(`-m set --match-set %s src -m set --match-set %s dst -j MARK --set-xmark %s`, srcMatch, dstMatch, markCode))}

×

1040

                        case rule.Match.SrcIPs != "":

×

1041

                                protocol = getMatchProtocol(rule.Match.SrcIPs)

×

1042

                                ovnNatoutGoingPolicyRule = util.IPTableRule{Table: NAT, Chain: ovnNatPolicySubnetChainName, Rule: strings.Fields(fmt.Sprintf(`-m set --match-set %s src -j MARK --set-xmark %s`, srcMatch, markCode))}

×

1043

                        case rule.Match.DstIPs != "":

×

1044

                                protocol = getMatchProtocol(rule.Match.DstIPs)

×

1045

                                ovnNatoutGoingPolicyRule = util.IPTableRule{Table: NAT, Chain: ovnNatPolicySubnetChainName, Rule: strings.Fields(fmt.Sprintf(`-m set --match-set %s dst -j MARK --set-xmark %s`, dstMatch, markCode))}

×

1046

                        default:

×

1047

                                continue

×

1049

                        natPolicyRuleIptables = append(natPolicyRuleIptables, ovnNatoutGoingPolicyRule)

×

1051

                natPolicyRuleIptablesMap[ovnNatPolicySubnetChainName] = natPolicyRuleIptables

×

1054

        existNatChains, err := c.iptables[protocol].ListChains(NAT)

×

1055

        if err != nil {

×

1056

                klog.Errorf("list chains in table nat failed")

×

1057

                return nil, nil, nil, err

×

1058

×

1060

        for _, existNatChain := range existNatChains {

×

1061

                if strings.HasPrefix(existNatChain, OvnNatOutGoingPolicySubnet) &&

×

1062

                        !natPolicySubnetUIDs.Has(getNatPolicySubnetChainUID(existNatChain)) {

×

1063

                        gcNatPolicySubnetChains = append(gcNatPolicySubnetChains, existNatChain)

×

1064

×

1067

        return natPolicySubnetIptables, natPolicyRuleIptablesMap, gcNatPolicySubnetChains, nil

×

1070

func deleteIptablesRule(ipt *iptables.IPTables, rule util.IPTableRule) error {

×

1071

        if rule.Pos != "" {

×

1072

                klog.Infof("delete iptables rule by pos %s: %v", rule.Pos, rule)

×

1073

                if err := ipt.Delete(rule.Table, rule.Chain, rule.Pos); err != nil {

×

1074

                        klog.Errorf("failed to delete iptables %s rule %q: %v", rule.Chain, strings.Join(rule.Rule, " "), err)

×

1075

                        return err

×

1076

×

1077

                return nil

×

1079

        exists, err := ipt.Exists(rule.Table, rule.Chain, rule.Rule...)

×

1080

        if err == nil && exists {

×

1081

                klog.Infof("delete iptables rule: %v", rule)

×

1082

                err = ipt.Delete(rule.Table, rule.Chain, rule.Rule...)

×

1083

×

1084

        if err != nil {

×

1085

                klog.Errorf("failed to delete iptables rule %q: %v", strings.Join(rule.Rule, " "), err)

×

1086

                return err

×

1087

×

1088

        return nil

×

1091

func clearObsoleteIptablesChain(ipt *iptables.IPTables, table, chain, parent string) error {

×

1092

        exists, err := ipt.ChainExists(table, chain)

×

1093

        if err != nil {

×

1094

                klog.Error(err)

×

1095

                return err

×

1096

×

1097

        if !exists {

×

1098

                return nil

×

1099

×

1101

        rule := fmt.Sprintf(`-m comment --comment "kube-ovn %s rules" -j %s`, strings.ToLower(parent), chain)

×

1102

        if err = deleteIptablesRule(ipt, util.IPTableRule{Table: table, Chain: parent, Rule: util.DoubleQuotedFields(rule)}); err != nil {

×

1103

                klog.Error(err)

×

1104

                return err

×

1105

×

1106

        if err = ipt.ClearAndDeleteChain(table, chain); err != nil {

×

1107

                klog.Errorf("failed to delete iptables chain %q in table %s: %v", chain, table, err)

×

1108

                return err

×

1109

×

1110

        return nil

×

1113

func (c *Controller) cleanObsoleteIptablesRules(protocol string, rules []util.IPTableRule) error {

×

1114

        if c.iptablesObsolete == nil || c.iptablesObsolete[protocol] == nil {

×

1115

                return nil

×

1116

×

1118

        var (

×

1119

                v4ObsoleteRules = []util.IPTableRule{

×

1120

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)},

×

1121

                        {Table: "mangle", Chain: Prerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x40000/0x40000`)},

×

1122

                        // legacy rules

×

1123

                        // nat packets marked by kube-proxy or kube-ovn

×

1124

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},

×

1125

                        // nat service traffic

×

1126

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE`)},

×

1127

                        // do not nat node port service traffic with external traffic policy set to local

×

1128

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN`)},

×

1129

                        // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway

×

1130

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)},

×

1131

                        // do not nat reply packets in direct routing

×

1132

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},

×

1133

                        // do not nat route traffic

×

1134

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)},

×

1135

                        // nat outgoing

×

1136

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},

×

1137

                        // mark packets from pod to service

×

1138

                        {Table: "mangle", Chain: Prerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000`)},

×

1139

                        // Input Accept

×

1140

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},

×

1141

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},

×

1142

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},

×

1143

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},

×

1144

                        // Forward Accept

×

1145

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},

×

1146

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},

×

1147

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},

×

1148

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},

×

1149

                        // Output unmark to bypass kernel nat checksum issue https://github.com/flannel-io/flannel/issues/1279

×

1150

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 6081 -j MARK --set-xmark 0x0`)},

×

1151

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 4789 -j MARK --set-xmark 0x0`)},

×

1152

×

1153

                v6ObsoleteRules = []util.IPTableRule{

×

1154

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)},

×

1155

                        {Table: "mangle", Chain: Prerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x40000/0x40000`)},

×

1156

                        // legacy rules

×

1157

                        // nat packets marked by kube-proxy or kube-ovn

×

1158

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},

×

1159

                        // nat service traffic

×

1160

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets src -m set --match-set ovn60subnets dst -j MASQUERADE`)},

×

1161

                        // do not nat node port service traffic with external traffic policy set to local

×

1162

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN`)},

×

1163

                        // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway

×

1164

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)},

×

1165

                        // do not nat reply packets in direct routing

×

1166

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},

×

1167

                        // do not nat route traffic

×

1168

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`)},

×

1169

                        // nat outgoing

×

1170

                        {Table: NAT, Chain: Postrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},

×

1171

                        // mark packets from pod to service

×

1172

                        {Table: "mangle", Chain: Prerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000`)},

×

1173

                        // Input Accept

×

1174

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},

×

1175

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},

×

1176

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},

×

1177

                        {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},

×

1178

                        // Forward Accept

×

1179

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},

×

1180

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},

×

1181

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},

×

1182

                        {Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},

×

1183

                        // Output unmark to bypass kernel nat checksum issue https://github.com/flannel-io/flannel/issues/1279

×

1184

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 6081 -j MARK --set-xmark 0x0`)},

×

1185

                        {Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 4789 -j MARK --set-xmark 0x0`)},

×

1186

×

1187

×

1188

×

1189

        var obsoleteRules []util.IPTableRule

×

1190

        if protocol == kubeovnv1.ProtocolIPv4 {

×

1191

                obsoleteRules = v4ObsoleteRules

×

1192

        } else {

×

1193

                obsoleteRules = v6ObsoleteRules

×

1194

×

1196

        ipt := c.iptablesObsolete[protocol]

×

1197

        for _, rule := range obsoleteRules {

×

1198

                if err := deleteIptablesRule(ipt, rule); err != nil {

×

1199

                        klog.Error(err)

×

1200

                        return err

×

1201

×

1203

        for _, rule := range rules {

×

1204

                if err := deleteIptablesRule(ipt, rule); err != nil {

×

1205

                        klog.Error(err)

×

1206

                        return err

×

1207

×

1210

        forwardRules, err := ipt.List("filter", "FORWARD")

×

1211

        if err != nil {

×

1212

                klog.Errorf(`failed to list legacy iptables rule in "FORWARD" chain "filter" table: %v`, err)

×

1213

                return err

×

1214

×

1215

        prefix := util.OvnSubnetGatewayIptables + ","

×

1216

        for _, rule := range forwardRules {

×

1217

                fields := util.DoubleQuotedFields(rule)

×

1218

                for _, f := range fields {

×

1219

                        if strings.HasPrefix(f, prefix) {

×

1220

                                if err = ipt.Delete("filter", "FORWARD", fields...); err != nil {

×

1221

                                        klog.Errorf("failed to delete legacy iptables rules %q: %v", rule, err)

×

1222

×

1228

        if err = c.deleteObsoleteSnatRules(ipt, NAT, Postrouting); err != nil {

×

1229

                klog.Errorf("failed to delete legacy iptables rule for SNAT: %v", err)

×

1230

                return err

×

1231

×

1233

        if err = clearObsoleteIptablesChain(ipt, NAT, OvnPrerouting, Prerouting); err != nil {

×

1234

                klog.Error(err)

×

1235

                return err

×

1236

×

1237

        if err = clearObsoleteIptablesChain(ipt, NAT, OvnPostrouting, Postrouting); err != nil {

×

1238

                klog.Error(err)

×

1239

                return err

×

1240

×

1242

        delete(c.iptablesObsolete, protocol)

×

1243

        if len(c.iptablesObsolete) == 0 {

×

1244

                c.iptablesObsolete = nil

×

1245

×

1246

        return nil

×

1249

func (c *Controller) setOvnSubnetGatewayMetric() {

×

1250

        hostname := os.Getenv(util.HostnameEnv)

×

1251

        for proto, iptables := range c.iptables {

×

1252

                rules, err := iptables.ListWithCounters("filter", "FORWARD")

×

1253

                if err != nil {

×

1254

                        klog.Errorf("get proto %s iptables failed with err %v", proto, err)

×

1255

                        continue

×

1258

                for _, rule := range rules {

×

1259

                        if !strings.Contains(rule, util.OvnSubnetGatewayIptables) {

×

1260

                                continue

×

1263

                        items := util.DoubleQuotedFields(rule)

×

1264

                        if len(items) != 11 {

×

1265

                                continue

×

1268

                        comments := strings.Split(items[7], ",")

×

1269

                        if len(comments) != 2 || comments[0] != util.OvnSubnetGatewayIptables {

×

1270

                                continue

×

1272

                        subnetName := comments[1]

×

1273

×

1274

                        var direction string

×

1275

                        switch items[2] {

×

1276

                        case "-s":

×

1277

                                direction = "egress"

×

1278

                        case "-d":

×

1279

                                direction = "ingress"

×

1280

                        default:

×

1281

                                continue

×

1284

                        cidr := items[3]

×

1285

                        proto := util.CheckProtocol(cidr)

×

1286

                        if proto == "" {

×

1287

                                klog.Errorf("failed to get protocol from cidr %q", cidr)

×

1288

                                continue

×

1291

                        currentPackets, err := strconv.Atoi(items[9])

×

1292

                        if err != nil {

×

1293

                                klog.Errorf("failed to parse packets %q: %v", items[9], err)

×

1294

                                continue

×

1296

                        currentPacketBytes, err := strconv.Atoi(items[10])

×

1297

                        if err != nil {

×

1298

                                klog.Errorf("failed to parse packet bytes %q: %v", items[10], err)

×

1299

                                continue

×

1302

                        key := strings.Join([]string{subnetName, direction, proto}, "/")

×

1303

                        if c.gwCounters[key] == nil {

×

1304

                                c.gwCounters[key] = new(util.GwIPtableCounters)

×

1305

×

1306

                        lastPackets, lastPacketBytes := c.gwCounters[key].Packets, c.gwCounters[key].PacketBytes

×

1307

                        c.gwCounters[key].Packets, c.gwCounters[key].PacketBytes = currentPackets, currentPacketBytes

×

1308

×

1309

                        if lastPackets == 0 && lastPacketBytes == 0 {

×

1310

                                // the gwCounters may just initialize don't cal the diff values,

×

1311

                                // it may loss packets to calculate during a metric period

×

1312

                                continue

×

1314

                        if currentPackets < lastPackets || currentPacketBytes < lastPacketBytes {

×

1315

                                // if currentPacketBytes < lastPacketBytes, the reason is that iptables rule is reset ,

×

1316

                                // it may loss packets to calculate during a metric period

×

1317

                                continue

×

1320

                        diffPackets := currentPackets - lastPackets

×

1321

                        diffPacketBytes := currentPacketBytes - lastPacketBytes

×

1322

                        klog.V(3).Infof(`hostname %s key %s cidr %s direction %s proto %s has diffPackets %d diffPacketBytes %d currentPackets %d currentPacketBytes %d lastPackets %d lastPacketBytes %d`,

×

1323

                                hostname, key, cidr, direction, proto, diffPackets, diffPacketBytes, currentPackets, currentPacketBytes, lastPackets, lastPacketBytes)

×

1324

                        metricOvnSubnetGatewayPackets.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPackets))

×

1325

                        metricOvnSubnetGatewayPacketBytes.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPacketBytes))

×

1330

func (c *Controller) addEgressConfig(subnet *kubeovnv1.Subnet, ip string) error {

×

1331

        if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) ||

×

1332

                subnet.Spec.GatewayType != kubeovnv1.GWDistributedType ||

×

1333

                subnet.Spec.Vpc != c.config.ClusterRouter {

×

1334

                return nil

×

1335

×

1337

        if !subnet.Spec.NatOutgoing && subnet.Spec.ExternalEgressGateway != "" {

×

1338

                podIPs := strings.Split(ip, ",")

×

1339

                protocol := util.CheckProtocol(ip)

×

1340

                return c.addPodPolicyRouting(protocol, subnet.Spec.ExternalEgressGateway, subnet.Spec.PolicyRoutingPriority, subnet.Spec.PolicyRoutingTableID, podIPs)

×

1341

×

1343

        return nil

×

1346

func (c *Controller) removeEgressConfig(subnet, ip string) error {

×

1347

        if subnet == "" || ip == "" {

×

1348

                return nil

×

1349

×

1351

        podSubnet, err := c.subnetsLister.Get(subnet)

×

1352

        if k8serrors.IsNotFound(err) {

×

1353

                return nil

×

1354

        } else if err != nil {

×

1355

                klog.Errorf("failed to get subnet %s: %+v", subnet, err)

×

1356

                return err

×

1357

×

1359

        if (podSubnet.Spec.Vlan != "" && !podSubnet.Spec.LogicalGateway) ||

×

1360

                podSubnet.Spec.GatewayType != kubeovnv1.GWDistributedType ||

×

1361

                podSubnet.Spec.Vpc != c.config.ClusterRouter {

×

1362

                return nil

×

1363

×

1365

        if !podSubnet.Spec.NatOutgoing && podSubnet.Spec.ExternalEgressGateway != "" {

×

1366

                podIPs := strings.Split(ip, ",")

×

1367

                protocol := util.CheckProtocol(ip)

×

1368

                return c.deletePodPolicyRouting(protocol, podSubnet.Spec.ExternalEgressGateway, podSubnet.Spec.PolicyRoutingPriority, podSubnet.Spec.PolicyRoutingTableID, podIPs)

×

1369

×

1371

        return nil

×

1374

func (c *Controller) setExGateway() error {

×

1375

        node, err := c.nodesLister.Get(c.config.NodeName)

×

1376

        if err != nil {

×

1377

                klog.Errorf("failed to get node, %v", err)

×

1378

                return err

×

1379

×

1380

        var isUserspaceDP bool

×

1381

        isUserspaceDP, err = ovs.IsUserspaceDataPath()

×

1382

        if err != nil {

×

1383

                klog.Error(err)

×

1384

                return err

×

1385

×

1386

        enable := node.Labels[util.ExGatewayLabel]

×

1387

        externalBridge := util.ExternalBridgeName(c.config.ExternalGatewaySwitch)

×

1388

        if enable == "true" {

×

1389

                cm, err := c.config.KubeClient.CoreV1().ConfigMaps(c.config.ExternalGatewayConfigNS).Get(context.Background(), util.ExternalGatewayConfig, metav1.GetOptions{})

×

1390

                if err != nil {

×

1391

                        klog.Errorf("failed to get ovn-external-gw-config, %v", err)

×

1392

                        return err

×

1393

×

1395

                linkName, exist := cm.Data["external-gw-nic"]

×

1396

                if !exist || len(linkName) == 0 {

×

1397

                        err = errors.New("external-gw-nic not configured in ovn-external-gw-config")

×

1398

                        klog.Error(err)

×

1399

                        return err

×

1400

×

1402

                if !isUserspaceDP {

×

1403

                        link, err := netlink.LinkByName(linkName)

×

1404

                        if err != nil {

×

1405

                                klog.Errorf("failed to get nic %s, %v", linkName, err)

×

1406

                                return err

×

1407

×

1408

                        if err := netlink.LinkSetUp(link); err != nil {

×

1409

                                klog.Errorf("failed to set gateway nic %s up, %v", linkName, err)

×

1410

                                return err

×

1411

×

1414

                externalBrReady := false

×

1415

                // if external nic already attached into another bridge

×

1416

                if existBr, err := ovs.Exec("port-to-br", linkName); err == nil {

×

1417

                        if existBr == externalBridge {

×

1418

                                externalBrReady = true

×

1419

                        } else {

×

1420

                                klog.Infof("external bridge should change from %s to %s, delete external bridge %s", existBr, externalBridge, existBr)

×

1421

                                if _, err := ovs.Exec(ovs.IfExists, "del-br", existBr); err != nil {

×

1422

                                        err = fmt.Errorf("failed to del external br %s, %w", existBr, err)

×

1423

                                        klog.Error(err)

×

1424

                                        return err

×

1425

×

1429

                if !externalBrReady {

×

1430

                        klog.Infof("create external bridge %s and add nic %s", externalBridge, linkName)

×

1431

                        if _, err := ovs.Exec(

×

1432

                                ovs.MayExist, "add-br", externalBridge, "--",

×

1433

                                ovs.MayExist, "add-port", externalBridge, linkName,

×

1434

                        ); err != nil {

×

1435

                                err = fmt.Errorf("failed to enable external gateway, %w", err)

×

1436

                                klog.Error(err)

×

1437

×

1439

                if err = addOvnMapping("ovn-bridge-mappings", c.config.ExternalGatewaySwitch, externalBridge, true); err != nil {

×

1440

                        klog.Error(err)

×

1441

                        return err

×

1442

×

1443

        } else {

×

1444

                brExists, err := ovs.BridgeExists(externalBridge)

×

1445

                if err != nil {

×

1446

                        return fmt.Errorf("failed to check OVS bridge existence: %w", err)

×

1447

×

1448

                if !brExists {

×

1449

                        return nil

×

1450

×

1452

                providerNetworks, err := c.providerNetworksLister.List(labels.Everything())

×

1453

                if err != nil && !k8serrors.IsNotFound(err) {

×

1454

                        klog.Errorf("failed to list provider networks: %v", err)

×

1455

                        return err

×

1456

×

1458

                for _, pn := range providerNetworks {

×

1459

                        // if external nic already attached into another bridge

×

1460

                        if existBr, err := ovs.Exec("port-to-br", pn.Spec.DefaultInterface); err == nil {

×

1461

                                if existBr == externalBridge {

×

1462

                                        // delete switch after related provider network not exist

×

1463

                                        return nil

×

1464

×

1468

                keepExternalSubnet := false

×

1469

                externalSubnet, err := c.subnetsLister.Get(c.config.ExternalGatewaySwitch)

×

1470

                if err != nil {

×

1471

                        if !k8serrors.IsNotFound(err) {

×

1472

                                klog.Errorf("failed to get subnet %s, %v", c.config.ExternalGatewaySwitch, err)

×

1473

                                return err

×

1474

×

1475

                } else {

×

1476

                        if externalSubnet.Spec.Vlan != "" {

×

1477

                                keepExternalSubnet = true

×

1478

×

1481

                if !isUserspaceDP && !keepExternalSubnet {

×

1482

                        klog.Infof("delete external bridge %s", externalBridge)

×

1483

                        if _, err := ovs.Exec(

×

1484

                                ovs.IfExists, "del-br", externalBridge); err != nil {

×

1485

                                err = fmt.Errorf("failed to disable external gateway, %w", err)

×

1486

                                klog.Error(err)

×

1487

                                return err

×

1488

×

1491

        return nil

×

1494

func (c *Controller) getLocalPodIPsNeedPR(protocol string) (map[policyRouteMeta][]string, error) {

×

1495

        allPods, err := c.podsLister.List(labels.Everything())

×

1496

        if err != nil {

×

1497

                klog.Errorf("failed to list pods: %+v", err)

×

1498

                return nil, err

×

1499

×

1501

        nodeName := os.Getenv(util.HostnameEnv)

×

1502

        localPodIPs := make(map[policyRouteMeta][]string)

×

1503

        for _, pod := range allPods {

×

1504

                if pod.Spec.HostNetwork ||

×

1505

                        !pod.DeletionTimestamp.IsZero() ||

×

1506

                        pod.Spec.NodeName != nodeName ||

×

1507

                        pod.Annotations[util.LogicalSwitchAnnotation] == "" ||

×

1508

                        pod.Annotations[util.IPAddressAnnotation] == "" {

×

1509

                        continue

×

1512

                subnet, err := c.subnetsLister.Get(pod.Annotations[util.LogicalSwitchAnnotation])

×

1513

                if err != nil {

×

1514

                        klog.Errorf("failed to get subnet %s: %+v", pod.Annotations[util.LogicalSwitchAnnotation], err)

×

1515

                        continue

×

1518

                if subnet.Spec.ExternalEgressGateway == "" ||

×

1519

                        subnet.Spec.Vpc != c.config.ClusterRouter ||

×

1520

                        subnet.Spec.GatewayType != kubeovnv1.GWDistributedType {

×

1521

                        continue

×

1523

                if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway {

×

1524

                        continue

×

1527

                ips := make([]string, 0, 2)

×

1528

                if len(pod.Status.PodIPs) != 0 {

×

1529

                        if len(pod.Status.PodIPs) == 2 && protocol == kubeovnv1.ProtocolIPv6 {

×

1530

                                ips = append(ips, pod.Status.PodIPs[1].IP)

×

1531

                        } else if util.CheckProtocol(pod.Status.PodIP) == protocol {

×

1532

                                ips = append(ips, pod.Status.PodIP)

×

1533

×

1534

                } else {

×

1535

                        ipv4, ipv6 := util.SplitStringIP(pod.Annotations[util.IPAddressAnnotation])

×

1536

                        if ipv4 != "" && protocol == kubeovnv1.ProtocolIPv4 {

×

1537

                                ips = append(ips, ipv4)

×

1538

×

1539

                        if ipv6 != "" && protocol == kubeovnv1.ProtocolIPv6 {

×

1540

                                ips = append(ips, ipv6)

×

1541

×

1544

                if len(ips) != 0 {

×

1545

                        meta := policyRouteMeta{

×

1546

                                priority: subnet.Spec.PolicyRoutingPriority,

×

1547

                                tableID:  subnet.Spec.PolicyRoutingTableID,

×

1548

×

1549

×

1550

                        egw := strings.Split(subnet.Spec.ExternalEgressGateway, ",")

×

1551

                        if util.CheckProtocol(egw[0]) == protocol {

×

1552

                                meta.gateway = egw[0]

×

1553

                                if util.CheckProtocol(ips[0]) == protocol {

×

1554

                                        localPodIPs[meta] = append(localPodIPs[meta], ips[0])

×

1555

                                } else {

×

1556

                                        localPodIPs[meta] = append(localPodIPs[meta], ips[1])

×

1557

×

1558

                        } else if len(egw) == 2 && len(ips) == 2 {

×

1559

                                meta.gateway = egw[1]

×

1560

                                localPodIPs[meta] = append(localPodIPs[meta], ips[1])

×

1561

×

1565

        return localPodIPs, nil

×

1568

func (c *Controller) getSubnetsNeedPR(protocol string) (map[policyRouteMeta]string, error) {

×

1569

        subnetsNeedPR := make(map[policyRouteMeta]string)

×

1570

        subnets, err := c.subnetsLister.List(labels.Everything())

×

1571

        if err != nil {

×

1572

                klog.Errorf("failed to list subnets: %v", err)

×

1573

                return nil, err

×

1574

×

1576

        for _, subnet := range subnets {

×

1577

                if subnet.DeletionTimestamp.IsZero() &&

×

1578

                        subnet.Spec.ExternalEgressGateway != "" &&

×

1579

                        (subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) &&

×

1580

                        subnet.Spec.GatewayType == kubeovnv1.GWCentralizedType &&

×

1581

                        util.GatewayContains(subnet.Spec.GatewayNode, c.config.NodeName) &&

×

1582

                        subnet.Spec.Vpc == c.config.ClusterRouter &&

×

1583

                        (subnet.Spec.Protocol == kubeovnv1.ProtocolDual || subnet.Spec.Protocol == protocol) {

×

1584

                        meta := policyRouteMeta{

×

1585

                                priority: subnet.Spec.PolicyRoutingPriority,

×

1586

                                tableID:  subnet.Spec.PolicyRoutingTableID,

×

1587

×

1588

                        egw := strings.Split(subnet.Spec.ExternalEgressGateway, ",")

×

1589

                        if util.CheckProtocol(subnet.Spec.CIDRBlock) == kubeovnv1.ProtocolDual && protocol == kubeovnv1.ProtocolIPv6 {

×

1590

                                if len(egw) == 2 {

×

1591

                                        meta.gateway = egw[1]

×

1592

                                } else if util.CheckProtocol(egw[0]) == protocol {

×

1593

                                        meta.gateway = egw[0]

×

1594

×

1595

                        } else {

×

1596

                                meta.gateway = egw[0]

×

1597

×

1598

                        if meta.gateway != "" {

×

1599

                                cidrBlock, err := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)

×

1600

                                if err == nil && cidrBlock != "" {

×

1601

                                        subnetsNeedPR[meta] = cidrBlock

×

1602

×

1607

        return subnetsNeedPR, nil

×

1610

func (c *Controller) deleteObsoleteSnatRules(ipt *iptables.IPTables, table, chain string) error {

×

1611

        rules, err := ipt.List(table, chain)

×

1612

        if err != nil {

×

1613

                klog.Errorf("failed to list iptables rules in table %v chain %v, %+v", table, chain, err)

×

1614

                return err

×

1615

×

1617

        for _, rule := range rules {

×

1618

                if !strings.Contains(rule, "--to-source") {

×

1619

                        continue

×

1623

                rule := rule[4+len(chain):]

×

1624

                spec := util.DoubleQuotedFields(rule)

×

1625

                if err = ipt.Delete(table, chain, spec...); err != nil {

×

1626

                        klog.Errorf(`failed to delete iptables rule "%s": %v`, rule, err)

×

1627

                        return err

×

1628

×

1631

        return nil

×

1634

func (c *Controller) ipsetExists(name string) (bool, error) {

×

1635

        sets, err := c.k8sipsets.ListSets()

×

1636

        if err != nil {

×

1637

                return false, fmt.Errorf("failed to list ipset names: %w", err)

×

1638

×

1640

        return slices.Contains(sets, name), nil

×

1643

func getNatOutGoingPolicyRuleIPSetName(ruleID, srcOrDst, protocol string, hasPrefix bool) string {

×

1644

        prefix := ""

×

1645

×

1646

        if hasPrefix {

×

1647

                prefix = "ovn40"

×

1648

                if protocol == kubeovnv1.ProtocolIPv6 {

×

1649

                        prefix = "ovn60"

×

1650

×

1653

        return prefix + NatOutGoingPolicyRuleSet + fmt.Sprintf("%s-%s", ruleID, srcOrDst)

×

1656

func isNatOutGoingPolicyRuleIPSet(ipsetName string) bool {

×

1657

        return strings.HasPrefix(ipsetName, "ovn40"+NatOutGoingPolicyRuleSet) ||

×

1658

                strings.HasPrefix(ipsetName, "ovn60"+NatOutGoingPolicyRuleSet)

×

1659

×

1661

func getNatOutGoingPolicyRuleIPSetItem(ipsetName string) (string, string) {

×

1662

        items := strings.Split(ipsetName[len("ovn40")+len(NatOutGoingPolicyRuleSet):], "-")

×

1663

        ruleID := items[0]

×

1664

        srcOrDst := items[1]

×

1665

        return ruleID, srcOrDst

×

1666

×

1668

func getNatPolicySubnetChainUID(chainName string) string {

×

1669

        return chainName[len(OvnNatOutGoingPolicySubnet):]

×

1670

×

1672

func formatIPsetUnPrefix(ipsetName string) string {

×

1673

        return ipsetName[len("ovn40"):]

×

1674

×

kubeovn / kube-ovn / 13853401919

Source File
Press 'n' to go to next uncovered line, 'b' for previous

kubeovn / kube-ovn / 13853401919

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous