kubeovn/kube-ovn | Build 13853401919 | pkg/daemon/ovs_linux.go | Coveralls

42

func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, netns, containerID, ifName, _ string, _ int, ip, _, ingress, egress, shortSharedDir, socketName, socketConsumption string) error {

×

43

        sharedDir := filepath.Join("/var", shortSharedDir)

×

44

        hostNicName, _ := generateNicName(containerID, ifName)

×

45

×

46

        ipStr := util.GetIPWithoutMask(ip)

×

47

        ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)

×

48

        ovs.CleanDuplicatePort(ifaceID, hostNicName)

×

49

×

50

        vhostServerPath := path.Join(sharedDir, socketName)

×

51

        if socketConsumption == util.ConsumptionKubevirt {

×

52

                vhostServerPath = path.Join(sharedDir, ifName)

×

53

×

56

        output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", hostNicName, "--",

×

57

                "set", "interface", hostNicName,

×

58

                "type=dpdkvhostuserclient",

×

59

                fmt.Sprintf("options:vhost-server-path=%s", vhostServerPath),

×

60

                fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

61

                fmt.Sprintf("external_ids:pod_name=%s", podName),

×

62

                fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

63

                fmt.Sprintf("external_ids:ip=%s", ipStr),

×

64

                fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

65

        if err != nil {

×

66

                return fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

67

×

68

        return ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress)

×

71

func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, vmMigration bool, routes []request.Route, _, _ []string, ingress, egress, deviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP, oldPodName string) ([]request.Route, error) {

×

72

        var err error

×

73

        var hostNicName, containerNicName, pfPci string

×

74

        var vfID int

×

75

        if deviceID == "" {

×

76

                hostNicName, containerNicName, err = setupVethPair(containerID, ifName, mtu)

×

77

                if err != nil {

×

78

                        klog.Errorf("failed to create veth pair %v", err)

×

79

                        return nil, err

×

80

×

81

                defer func() {

×

82

                        if err != nil {

×

83

                                if err := rollBackVethPair(hostNicName); err != nil {

×

84

                                        klog.Errorf("failed to rollback veth pair %s, %v", hostNicName, err)

×

85

                                        return

×

86

×

89

        } else {

×

90

                hostNicName, containerNicName, pfPci, vfID, err = setupSriovInterface(containerID, deviceID, vfDriver, ifName, mtu, mac)

×

91

                if err != nil {

×

92

                        klog.Errorf("failed to create sriov interfaces %v", err)

×

93

                        return nil, err

×

94

×

97

        ipStr := util.GetIPWithoutMask(ip)

×

98

        ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)

×

99

        ovs.CleanDuplicatePort(ifaceID, hostNicName)

×

100

        if yusur.IsYusurSmartNic(deviceID) {

×

101

                klog.Infof("add Yusur smartnic vfr %s to ovs", hostNicName)

×

102

                // Add yusur ovs port

×

103

                output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", hostNicName, "--",

×

104

                        "set", "interface", hostNicName, "type=dpdk",

×

105

                        fmt.Sprintf("options:dpdk-devargs=%s,representor=[%d]", pfPci, vfID),

×

106

                        fmt.Sprintf("mtu_request=%d", mtu),

×

107

                        fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

108

                        fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName),

×

109

                        fmt.Sprintf("external_ids:pod_name=%s", podName),

×

110

                        fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

111

                        fmt.Sprintf("external_ids:ip=%s", ipStr),

×

112

                        fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

113

                if err != nil {

×

114

                        return nil, fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

115

×

116

        } else {

×

117

                // Add veth pair host end to ovs port

×

118

                output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", hostNicName, "--",

×

119

                        "set", "interface", hostNicName, fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

120

                        fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName),

×

121

                        fmt.Sprintf("external_ids:pod_name=%s", podName),

×

122

                        fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

123

                        fmt.Sprintf("external_ids:ip=%s", ipStr),

×

124

                        fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

125

                if err != nil {

×

126

                        return nil, fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

127

×

129

        defer func() {

×

130

                if err != nil {

×

131

                        if err := csh.rollbackOvsPort(hostNicName, containerNicName, nicType); err != nil {

×

132

                                klog.Errorf("failed to rollback ovs port %s, %v", hostNicName, err)

×

133

                                return

×

134

×

139

        if deviceID != "" {

×

140

                var podNameNew string

×

141

                if podName != oldPodName {

×

142

                        podNameNew = oldPodName

×

143

                } else {

×

144

                        podNameNew = podName

×

145

×

146

                patch := util.KVPatch{

×

147

                        fmt.Sprintf(util.VfRepresentorNameTemplate, provider): hostNicName,

×

148

                        fmt.Sprintf(util.VfNameTemplate, provider):            containerNicName,

×

149

                        fmt.Sprintf(util.PodNicAnnotationTemplate, provider):  util.SriovNicType,

×

150

×

151

                if err = util.PatchAnnotations(csh.Config.KubeClient.CoreV1().Pods(podNamespace), podNameNew, patch); err != nil {

×

152

                        klog.Errorf("failed to patch pod %s/%s: %v", podNamespace, podNameNew, err)

×

153

                        return nil, err

×

154

×

158

        macAddr, err := net.ParseMAC(mac)

×

159

        if err != nil {

×

160

                return nil, fmt.Errorf("failed to parse mac %s %w", macAddr, err)

×

161

×

162

        if !yusur.IsYusurSmartNic(deviceID) {

×

163

                if err = configureHostNic(hostNicName); err != nil {

×

164

                        klog.Error(err)

×

165

                        return nil, err

×

166

×

168

        if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress); err != nil {

×

169

                klog.Error(err)

×

170

                return nil, err

×

171

×

173

        if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss, jitter); err != nil {

×

174

                klog.Error(err)

×

175

                return nil, err

×

176

×

178

        if containerNicName == "" {

×

179

                return nil, nil

×

180

×

181

        isUserspaceDP, err := ovs.IsUserspaceDataPath()

×

182

        if err != nil {

×

183

                klog.Error(err)

×

184

                return nil, err

×

185

×

186

        if isUserspaceDP {

×

187

                // turn off tx checksum

×

188

                if err = TurnOffNicTxChecksum(containerNicName); err != nil {

×

189

                        klog.Error(err)

×

190

                        return nil, err

×

191

×

194

        podNS, err := ns.GetNS(netns)

×

195

        if err != nil {

×

196

                err = fmt.Errorf("failed to open netns %q: %w", netns, err)

×

197

                klog.Error(err)

×

198

                return nil, err

×

199

×

200

        finalRoutes, err := csh.configureContainerNic(podName, podNamespace, containerNicName, ifName, ip, gateway, isDefaultRoute, vmMigration, routes, macAddr, podNS, mtu, nicType, gwCheckMode, u2oInterconnectionIP)

×

201

        if err != nil {

×

202

                klog.Error(err)

×

203

                return nil, err

×

204

×

205

        return finalRoutes, nil

×

208

func (csh cniServerHandler) releaseVf(podName, podNamespace, podNetns, ifName, nicType, deviceID string) error {

×

209

        // Only for SRIOV case, we'd need to move the VF from container namespace back to the host namespace

×

210

        if !(nicType == util.OffloadType && deviceID != "") {

×

211

                return nil

×

212

×

213

        podDesc := fmt.Sprintf("for pod %s/%s", podNamespace, podName)

×

214

        klog.Infof("Tear down interface %s", podDesc)

×

215

        netns, err := ns.GetNS(podNetns)

×

216

        if err != nil {

×

217

                return fmt.Errorf("failed to get container namespace %s: %w", podDesc, err)

×

218

×

219

        defer netns.Close()

×

220

×

221

        hostNS, err := ns.GetCurrentNS()

×

222

        if err != nil {

×

223

                return fmt.Errorf("failed to get host namespace %s: %w", podDesc, err)

×

224

×

225

        defer hostNS.Close()

×

226

×

227

        err = netns.Do(func(_ ns.NetNS) error {

×

228

                // container side interface deletion

×

229

                link, err := netlink.LinkByName(ifName)

×

230

                if err != nil {

×

231

                        return fmt.Errorf("failed to get container interface %s %s: %w", ifName, podDesc, err)

×

232

×

233

                if err = netlink.LinkSetDown(link); err != nil {

×

234

                        return fmt.Errorf("failed to bring down container interface %s %s: %w", ifName, podDesc, err)

×

235

×

237

                vfName := link.Attrs().Alias

×

238

                if err = netlink.LinkSetName(link, vfName); err != nil {

×

239

                        return fmt.Errorf("failed to rename container interface %s to %s %s: %w",

×

240

                                ifName, vfName, podDesc, err)

×

241

×

243

                fd := int(netns.Fd()) // #nosec G115

×

244

                if err = netlink.LinkSetNsFd(link, fd); err != nil {

×

245

                        return fmt.Errorf("failed to move container interface %s back to host namespace %s: %w",

×

246

                                ifName, podDesc, err)

×

247

×

248

                return nil

×

250

        if err != nil {

×

251

                klog.Error(err)

×

252

×

254

        return nil

×

257

func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, netns, deviceID, ifName, nicType string) error {

×

258

        if err := csh.releaseVf(podName, podNamespace, netns, ifName, nicType, deviceID); err != nil {

×

259

                return fmt.Errorf("failed to release VF %s assigned to the Pod %s/%s back to the host network namespace: "+

×

260

                        "%w", ifName, podName, podNamespace, err)

×

261

×

263

        var nicName string

×

264

        if yusur.IsYusurSmartNic(deviceID) {

×

265

                pfPci, err := yusur.GetYusurNicPfPciFromVfPci(deviceID)

×

266

                if err != nil {

×

267

                        return fmt.Errorf("failed to get pf pci %w, %s", err, deviceID)

×

268

×

270

                pfIndex, err := yusur.GetYusurNicPfIndexByPciAddress(pfPci)

×

271

                if err != nil {

×

272

                        return fmt.Errorf("failed to get pf index %w, %s", err, deviceID)

×

273

×

275

                vfIndex, err := yusur.GetYusurNicVfIndexByPciAddress(deviceID)

×

276

                if err != nil {

×

277

                        return fmt.Errorf("failed to get vf index %w, %s", err, deviceID)

×

278

×

280

                nicName = yusur.GetYusurNicVfRepresentor(pfIndex, vfIndex)

×

281

        } else {

×

282

                hostNicName, containerNicName := generateNicName(containerID, ifName)

×

283

×

284

                if nicType == util.InternalType {

×

285

                        nicName = containerNicName

×

286

                } else {

×

287

                        nicName = hostNicName

×

288

×

291

        output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", nicName)

×

292

        if err != nil {

×

293

                return fmt.Errorf("failed to delete ovs port %w, %q", err, output)

×

294

×

296

        if err = ovs.ClearPodBandwidth(podName, podNamespace, ""); err != nil {

×

297

                klog.Error(err)

×

298

                return err

×

299

×

300

        if err = ovs.ClearHtbQosQueue(podName, podNamespace, ""); err != nil {

×

301

                klog.Error(err)

×

302

                return err

×

303

×

305

        if deviceID == "" {

×

306

                hostLink, err := netlink.LinkByName(nicName)

×

307

                if err != nil {

×

308

                        // If link already not exists, return quietly

×

309

                        // E.g. Internal port had been deleted by Remove ovs port previously

×

310

                        if _, ok := err.(netlink.LinkNotFoundError); ok {

×

311

                                return nil

×

312

×

313

                        return fmt.Errorf("find host link %s failed %w", nicName, err)

×

316

                hostLinkType := hostLink.Type()

×

317

                // Sometimes no deviceID input for vf nic, avoid delete vf nic.

×

318

                if hostLinkType == "veth" {

×

319

                        if err = netlink.LinkDel(hostLink); err != nil {

×

320

                                return fmt.Errorf("delete host link %s failed %w", hostLink, err)

×

321

×

323

        } else if pciAddrRegexp.MatchString(deviceID) && !yusur.IsYusurSmartNic(deviceID) {

×

324

                // Ret VF index from PCI

×

325

                vfIndex, err := sriovnet.GetVfIndexByPciAddress(deviceID)

×

326

                if err != nil {

×

327

                        klog.Errorf("failed to get vf %s index, %v", deviceID, err)

×

328

                        return err

×

329

×

330

                if err = setVfMac(deviceID, vfIndex, "00:00:00:00:00:00"); err != nil {

×

331

                        klog.Error(err)

×

332

                        return err

×

333

×

335

        return nil

×

338

func (csh cniServerHandler) rollbackOvsPort(hostNicName, containerNicName, nicType string) (err error) {

×

339

        var nicName string

×

340

        if nicType == util.InternalType {

×

341

                nicName = containerNicName

×

342

        } else {

×

343

                nicName = hostNicName

×

344

×

345

        output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", nicName)

×

346

        if err != nil {

×

347

                klog.Warningf("failed to delete down ovs port %v, %q", err, output)

×

348

×

349

        klog.Infof("rollback ovs port success %s", nicName)

×

350

        return

×

353

func generateNicName(containerID, ifname string) (string, string) {

×

354

        if ifname == "eth0" {

×

355

                return fmt.Sprintf("%s_h", containerID[0:12]), fmt.Sprintf("%s_c", containerID[0:12])

×

356

×

358

        if strings.HasPrefix(ifname, "pod") && len(ifname) == 14 {

×

359

                ifname = ifname[3 : len(ifname)-4]

×

360

                return fmt.Sprintf("%s_%s_h", containerID[0:12-len(ifname)], ifname), fmt.Sprintf("%s_%s_c", containerID[0:12-len(ifname)], ifname)

×

361

×

362

        return fmt.Sprintf("%s_%s_h", containerID[0:12-len(ifname)], ifname), fmt.Sprintf("%s_%s_c", containerID[0:12-len(ifname)], ifname)

×

365

func configureHostNic(nicName string) error {

×

366

        hostLink, err := netlink.LinkByName(nicName)

×

367

        if err != nil {

×

368

                return fmt.Errorf("can not find host nic %s: %w", nicName, err)

×

369

×

371

        if hostLink.Attrs().OperState != netlink.OperUp {

×

372

                if err = netlink.LinkSetUp(hostLink); err != nil {

×

373

                        return fmt.Errorf("can not set host nic %s up: %w", nicName, err)

×

374

×

376

        if err = netlink.LinkSetTxQLen(hostLink, 1000); err != nil {

×

377

                return fmt.Errorf("can not set host nic %s qlen: %w", nicName, err)

×

378

×

380

        return nil

×

383

func (csh cniServerHandler) configureContainerNic(podName, podNamespace, nicName, ifName, ipAddr, gateway string, isDefaultRoute, vmMigration bool, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, gwCheckMode int, u2oInterconnectionIP string) ([]request.Route, error) {

×

384

        containerLink, err := netlink.LinkByName(nicName)

×

385

        if err != nil {

×

386

                return nil, fmt.Errorf("can not find container nic %s: %w", nicName, err)

×

387

×

390

        if err := netlink.LinkSetAlias(containerLink, nicName); err != nil {

×

391

                klog.Errorf("failed to set link alias for container nic %s: %v", nicName, err)

×

392

                return nil, err

×

393

×

395

        fd := int(netns.Fd()) // #nosec G115

×

396

        if err = netlink.LinkSetNsFd(containerLink, fd); err != nil {

×

397

                return nil, fmt.Errorf("failed to move link to netns: %w", err)

×

398

×

401

        checkIPv6DAD := !vmMigration

×

402

        detectIPv4Conflict := !vmMigration && csh.Config.EnableArpDetectIPConflict

×

403

        var finalRoutes []request.Route

×

404

        err = ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error {

×

405

                interfaceName := nicName

×

406

                if nicType != util.InternalType {

×

UNCOV

407

                        interfaceName = ifName

×

UNCOV

408

                        if err = netlink.LinkSetName(containerLink, ifName); err != nil {

×

409

                                klog.Error(err)

×

410

                                return err

×

411

×

414

                if nicType == util.InternalType {

×

415

                        if err = addAdditionalNic(ifName); err != nil {

×

416

                                klog.Error(err)

×

417

                                return err

×

418

×

419

                        if err = configureAdditionalNic(ifName, ipAddr); err != nil {

×

420

                                klog.Error(err)

×

421

                                return err

×

422

×

423

                        if err = configureNic(nicName, ipAddr, macAddr, mtu, detectIPv4Conflict, false, false); err != nil {

×

424

                                klog.Error(err)

×

425

                                return err

×

426

×

UNCOV

427

                } else {

×

UNCOV

428

                        if err = configureNic(ifName, ipAddr, macAddr, mtu, detectIPv4Conflict, true, false); err != nil {

×

429

                                klog.Error(err)

×

430

                                return err

×

431

×

434

                if isDefaultRoute {

×

UNCOV

435

                        // Only eth0 requires the default route and gateway

×

436

                        containerGw := gateway

×

437

                        if u2oInterconnectionIP != "" {

×

438

                                containerGw = u2oInterconnectionIP

×

439

×

441

                        for _, gw := range strings.Split(containerGw, ",") {

×

442

                                if err = netlink.RouteReplace(&netlink.Route{

×

443

                                        LinkIndex: containerLink.Attrs().Index,

×

UNCOV

444

                                        Scope:     netlink.SCOPE_UNIVERSE,

×

UNCOV

445

                                        Gw:        net.ParseIP(gw),

×

UNCOV

446

                                }); err != nil {

×

447

                                        return fmt.Errorf("failed to configure default gateway %s: %w", gw, err)

×

448

×

452

                for _, r := range routes {

×

UNCOV

453

                        var dst *net.IPNet

×

UNCOV

454

                        if r.Destination != "" {

×

UNCOV

455

                                if _, dst, err = net.ParseCIDR(r.Destination); err != nil {

×

456

                                        klog.Errorf("invalid route destination %s: %v", r.Destination, err)

×

457

                                        continue

×

UNCOV

461

                        var gw net.IP

×

UNCOV

462

                        if r.Gateway != "" {

×

UNCOV

463

                                if gw = net.ParseIP(r.Gateway); gw == nil {

×

464

                                        klog.Errorf("invalid route gateway %s", r.Gateway)

×

465

                                        continue

×

469

                        route := &netlink.Route{

×

470

                                Dst:       dst,

×

471

                                Gw:        gw,

×

UNCOV

472

                                LinkIndex: containerLink.Attrs().Index,

×

UNCOV

473

×

474

                        if err = netlink.RouteReplace(route); err != nil {

×

475

                                klog.Errorf("failed to add route %+v: %v", r, err)

×

476

×

479

                linkRoutes, err := netlink.RouteList(containerLink, netlink.FAMILY_ALL)

×

480

                if err != nil {

×

481

                        return fmt.Errorf("failed to get routes on interface %s: %w", ifName, err)

×

UNCOV

482

×

484

                for _, r := range linkRoutes {

×

UNCOV

485

                        if r.Family != netlink.FAMILY_V4 && r.Family != netlink.FAMILY_V6 {

×

486

                                continue

×

488

                        if r.Dst == nil && r.Gw == nil {

×

489

                                continue

×

UNCOV

491

                        if r.Dst != nil && r.Dst.IP.IsLinkLocalUnicast() {

×

UNCOV

492

                                if _, bits := r.Dst.Mask.Size(); bits == net.IPv6len*8 {

×

493

                                        // skip fe80::/10

×

494

                                        continue

×

498

                        var route request.Route

×

499

                        if r.Dst != nil {

×

500

                                route.Destination = r.Dst.String()

×

UNCOV

501

×

UNCOV

502

                        if r.Gw != nil {

×

503

                                route.Gateway = r.Gw.String()

×

504

×

505

                        finalRoutes = append(finalRoutes, route)

×

508

                if gwCheckMode != gatewayCheckModeDisabled {

×

509

                        underlayGateway := gwCheckMode == gatewayCheckModeArping || gwCheckMode == gatewayCheckModeArpingNotConcerned

×

510

                        if u2oInterconnectionIP != "" {

×

511

                                if err = csh.checkGatewayReady(podName, podNamespace, gwCheckMode, interfaceName, ipAddr, u2oInterconnectionIP, false, true); err != nil {

×

UNCOV

512

                                        klog.Error(err)

×

513

                                        return err

×

514

×

516

                        if err = csh.checkGatewayReady(podName, podNamespace, gwCheckMode, interfaceName, ipAddr, gateway, underlayGateway, true); err != nil {

×

517

                                klog.Error(err)

×

UNCOV

518

                                return err

×

519

×

522

                if checkIPv6DAD {

×

UNCOV

523

                        // check whether the ipv6 address has a dadfailed flag

×

UNCOV

524

                        addresses, err := netlink.AddrList(containerLink, netlink.FAMILY_V6)

×

525

                        if err != nil {

×

UNCOV

526

                                err = fmt.Errorf("failed to get ipv6 addresses of link %s: %w", interfaceName, err)

×

UNCOV

527

                                klog.Error(err)

×

528

                                return err

×

529

×

531

                        for _, addr := range addresses {

×

532

                                if addr.Flags&syscall.IFA_F_DADFAILED != 0 {

×

533

                                        err = fmt.Errorf("IPv6 address %s has a dadfailed flag, please check whether it has been used by another host", addr.IP.String())

×

UNCOV

534

                                        klog.Error(err)

×

535

                                        return err

×

536

×

540

                return nil

×

543

        return finalRoutes, err

×

546

func (csh cniServerHandler) checkGatewayReady(podName, podNamespace string, gwCheckMode int, intr, ipAddr, gateway string, underlayGateway, verbose bool) error {

×

547

        if gwCheckMode == gatewayCheckModeArpingNotConcerned || gwCheckMode == gatewayCheckModePingNotConcerned {

×

548

                // ignore error if disableGatewayCheck=true

×

549

                _ = waitNetworkReady(intr, ipAddr, gateway, underlayGateway, verbose, 1, nil)

×

550

                return nil

×

UNCOV

551

×

UNCOV

553

        done := make(chan struct{}, 1)

×

554

        go func() {

×

555

                interval := 5 * time.Second

×

556

                timer := time.NewTimer(interval)

×

557

                for {

×

558

                        select {

×

UNCOV

559

                        case <-done:

×

560

                                return

×

UNCOV

561

                        case <-timer.C:

×

UNCOV

564

                        pod, err := csh.KubeClient.CoreV1().Pods(podNamespace).Get(context.Background(), podName, metav1.GetOptions{})

×

UNCOV

565

                        if err != nil {

×

566

                                if !k8serrors.IsNotFound(err) {

×

UNCOV

567

                                        klog.Errorf("failed to get pod %s/%s: %v", podNamespace, podName, err)

×

UNCOV

568

                                        continue

×

570

                                pod = nil

×

572

                        if pod == nil || !pod.DeletionTimestamp.IsZero() {

×

573

                                // TODO: check pod UID

×

574

                                select {

×

575

                                case <-done:

×

576

                                case done <- struct{}{}:

×

578

                                return

×

580

                        timer.Reset(interval)

×

584

        return waitNetworkReady(intr, ipAddr, gateway, underlayGateway, verbose, gatewayCheckMaxRetry, done)

×

587

func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool, maxRetry int, done chan struct{}) error {

×

588

        ips := strings.Split(ipAddr, ",")

×

589

        for i, gw := range strings.Split(gateway, ",") {

×

590

                src := strings.Split(ips[i], "/")[0]

×

UNCOV

591

                if underlayGateway && util.CheckProtocol(gw) == kubeovnv1.ProtocolIPv4 {

×

UNCOV

592

                        mac, count, err := util.ArpResolve(nic, gw, time.Second, maxRetry, done)

×

593

                        cniConnectivityResult.WithLabelValues(nodeName).Add(float64(count))

×

UNCOV

594

                        if err != nil {

×

UNCOV

595

                                err = fmt.Errorf("network %s with gateway %s is not ready for interface %s after %d checks: %w", ips[i], gw, nic, count, err)

×

596

                                klog.Warning(err)

×

597

                                return err

×

598

×

599

                        if verbose {

×

600

                                klog.Infof("MAC addresses of gateway %s is %s", gw, mac.String())

×

601

                                klog.Infof("network %s with gateway %s is ready for interface %s after %d checks", ips[i], gw, nic, count)

×

602

×

603

                } else {

×

604

                        _, err := pingGateway(gw, src, verbose, maxRetry, done)

×

605

                        if err != nil {

×

UNCOV

606

                                klog.Error(err)

×

607

                                return err

×

608

×

UNCOV

611

        return nil

×

614

func configureNodeNic(cs kubernetes.Interface, nodeName, portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error {

×

615

        ipStr := util.GetIPWithoutMask(ip)

×

UNCOV

616

        raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",

×

617

                "set", "interface", util.NodeNic, "type=internal", "--",

×

618

                "set", "interface", util.NodeNic, fmt.Sprintf("external_ids:iface-id=%s", portName),

×

619

                fmt.Sprintf("external_ids:ip=%s", ipStr))

×

UNCOV

620

        if err != nil {

×

UNCOV

621

                klog.Errorf("failed to configure node nic %s: %v, %q", portName, err, raw)

×

622

                return errors.New(raw)

×

623

×

625

        if err = configureNic(util.NodeNic, ip, macAddr, mtu, false, false, true); err != nil {

×

626

                klog.Error(err)

×

UNCOV

627

                return err

×

628

×

630

        hostLink, err := netlink.LinkByName(util.NodeNic)

×

631

        if err != nil {

×

632

                return fmt.Errorf("can not find nic %s: %w", util.NodeNic, err)

×

633

×

UNCOV

635

        if err = netlink.LinkSetTxQLen(hostLink, 1000); err != nil {

×

UNCOV

636

                return fmt.Errorf("can not set host nic %s qlen: %w", util.NodeNic, err)

×

637

×

640

        nodeNicRoutes, err := getNicExistRoutes(hostLink, gw)

×

641

        if err != nil {

×

642

                klog.Error(err)

×

643

                return err

×

644

×

UNCOV

646

        var toAdd []netlink.Route

×

UNCOV

647

        for _, c := range strings.Split(joinCIDR, ",") {

×

648

                found := false

×

649

                for _, r := range nodeNicRoutes {

×

650

                        if r.Dst.String() == c {

×

651

                                found = true

×

652

                                break

×

655

                if !found {

×

656

                        protocol := util.CheckProtocol(c)

×

657

                        var src net.IP

×

658

                        var priority int

×

UNCOV

659

                        if protocol == kubeovnv1.ProtocolIPv4 {

×

UNCOV

660

                                for _, ip := range strings.Split(ipStr, ",") {

×

661

                                        if util.CheckProtocol(ip) == protocol {

×

662

                                                src = net.ParseIP(ip)

×

663

                                                break

×

666

                        } else {

×

667

                                priority = 256

×

668

×

669

                        _, cidr, _ := net.ParseCIDR(c)

×

670

                        toAdd = append(toAdd, netlink.Route{

×

UNCOV

671

                                Dst:      cidr,

×

UNCOV

672

                                Src:      src,

×

UNCOV

673

                                Protocol: netlink.RouteProtocol(unix.RTPROT_KERNEL),

×

674

                                Scope:    netlink.SCOPE_LINK,

×

675

                                Priority: priority,

×

676

})

×

679

        if len(toAdd) > 0 {

×

680

                klog.Infof("routes to be added on nic %s: %v", util.NodeNic, toAdd)

×

681

×

683

        for _, r := range toAdd {

×

684

                r.LinkIndex = hostLink.Attrs().Index

×

685

                klog.Infof("adding route %q on %s", r.String(), hostLink.Attrs().Name)

×

686

                if err = netlink.RouteReplace(&r); err != nil && !errors.Is(err, syscall.EEXIST) {

×

UNCOV

687

                        klog.Errorf("failed to replace route %v: %v", r, err)

×

688

×

UNCOV

692

        klog.Infof("wait ovn0 gw ready")

×

693

        status := corev1.ConditionFalse

×

694

        reason := "JoinSubnetGatewayReachable"

×

695

        message := fmt.Sprintf("ping check to gateway ip %s succeeded", gw)

×

696

        if err = waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry, nil); err != nil {

×

697

                klog.Errorf("failed to init ovn0 check: %v", err)

×

UNCOV

698

                status = corev1.ConditionTrue

×

699

                reason = "JoinSubnetGatewayUnreachable"

×

700

                message = fmt.Sprintf("ping check to gateway ip %s failed", gw)

×

701

×

UNCOV

702

        if err := util.SetNodeNetworkUnavailableCondition(cs, nodeName, status, reason, message); err != nil {

×

703

                klog.Errorf("failed to set node network unavailable condition: %v", err)

×

704

×

706

        return err

×

711

func (c *Controller) loopOvn0Check() {

×

712

        link, err := netlink.LinkByName(util.NodeNic)

×

713

        if err != nil {

×

714

                util.LogFatalAndExit(err, "failed to get ovn0 nic")

×

715

×

717

        if link.Attrs().OperState == netlink.OperDown {

×

718

                util.LogFatalAndExit(err, "ovn0 nic is down")

×

UNCOV

719

×

721

        node, err := c.nodesLister.Get(c.config.NodeName)

×

722

        if err != nil {

×

723

                klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)

×

724

                return

×

725

×

UNCOV

726

        ip := node.Annotations[util.IPAddressAnnotation]

×

UNCOV

727

        gw := node.Annotations[util.GatewayAnnotation]

×

728

        status := corev1.ConditionFalse

×

729

        reason := "JoinSubnetGatewayReachable"

×

730

        message := fmt.Sprintf("ping check to gateway ip %s succeeded", gw)

×

731

        if err = waitNetworkReady(util.NodeNic, ip, gw, false, false, 5, nil); err != nil {

×

UNCOV

732

                klog.Errorf("failed to init ovn0 check: %v", err)

×

UNCOV

733

                status = corev1.ConditionTrue

×

734

                reason = "JoinSubnetGatewayUnreachable"

×

735

                message = fmt.Sprintf("ping check to gateway ip %s failed", gw)

×

736

×

UNCOV

738

        var alreadySet bool

×

UNCOV

739

        for _, condition := range node.Status.Conditions {

×

UNCOV

740

                if condition.Type == corev1.NodeNetworkUnavailable && condition.Status == corev1.ConditionTrue &&

×

741

                        condition.Reason == reason && condition.Message == message {

×

742

                        alreadySet = true

×

743

                        break

×

746

        if !alreadySet {

×

747

                if err := util.SetNodeNetworkUnavailableCondition(c.config.KubeClient, c.config.NodeName, status, reason, message); err != nil {

×

748

                        klog.Errorf("failed to set node network unavailable condition: %v", err)

×

749

×

752

        if err != nil {

×

753

                util.LogFatalAndExit(err, "failed to ping ovn0 gateway %s", gw)

×

UNCOV

754

×

759

func (c *Controller) loopTunnelCheck() {

×

UNCOV

760

        tunnelType := c.config.NetworkType

×

761

        var tunnelNic string

×

762

        switch tunnelType {

×

763

        case "vxlan":

×

764

                tunnelNic = util.VxlanNic

×

765

        case "geneve":

×

UNCOV

766

                tunnelNic = util.GeneveNic

×

UNCOV

767

        case "stt":

×

UNCOV

768

                // TODO: tunnelNic = "stt tunnel nic name"

×

769

                return

×

770

        default:

×

771

                return

×

774

        link, err := netlink.LinkByName(tunnelNic)

×

775

        if err != nil || link == nil {

×

776

                return

×

777

×

779

        if link.Attrs().OperState == netlink.OperDown {

×

780

                klog.Errorf("nic: %s is down, attempting to bring it up", tunnelNic)

×

781

                if err := netlink.LinkSetUp(link); err != nil {

×

782

                        klog.Errorf("fail to bring up nic: %s, %v", tunnelNic, err)

×

783

×

787

func (c *Controller) checkNodeGwNicInNs(nodeExtIP, ip, gw string, gwNS ns.NetNS) error {

×

788

        exists, err := ovs.PortExists(util.NodeGwNic)

×

789

        if err != nil {

×

790

                klog.Error(err)

×

791

                return err

×

792

×

793

        filters := labels.Set{util.OvnEipTypeLabel: util.OvnEipTypeLRP}

×

794

        ovnEips, err := c.ovnEipsLister.List(labels.SelectorFromSet(filters))

×

795

        if err != nil {

×

796

                klog.Errorf("failed to list ovn eip, %v", err)

×

797

                return err

×

798

×

799

        if len(ovnEips) == 0 {

×

800

                klog.Errorf("failed to get type %s ovn eip, %v", util.OvnEipTypeLRP, err)

×

801

                // node ext gw eip need lrp eip to establish bfd session

×

802

                return nil

×

803

×

804

        if exists {

×

805

                return ns.WithNetNSPath(gwNS.Path(), func(_ ns.NetNS) error {

×

806

                        err = waitNetworkReady(util.NodeGwNic, ip, gw, true, true, 3, nil)

×

807

                        if err == nil {

×

808

                                if output, err := exec.Command("bfdd-control", "status").CombinedOutput(); err != nil {

×

809

                                        err := fmt.Errorf("failed to get bfdd status, %w, %s", err, output)

×

810

                                        klog.Error(err)

×

811

                                        return err

×

UNCOV

812

×

813

                                for _, eip := range ovnEips {

×

814

                                        if eip.Status.Ready {

×

815

                                                // #nosec G204

×

816

                                                cmd := exec.Command("bfdd-control", "status", "remote", eip.Spec.V4Ip, "local", nodeExtIP)

×

817

                                                var outb bytes.Buffer

×

UNCOV

818

                                                cmd.Stdout = &outb

×

UNCOV

819

                                                if err := cmd.Run(); err == nil {

×

UNCOV

820

                                                        out := outb.String()

×

821

                                                        klog.V(3).Info(out)

×

UNCOV

822

                                                        if strings.Contains(out, "No session") {

×

UNCOV

823

                                                                // not exist

×

UNCOV

824

                                                                cmd = exec.Command("bfdd-control", "allow", eip.Spec.V4Ip) // #nosec G204

×

825

                                                                if err := cmd.Run(); err != nil {

×

826

                                                                        err := fmt.Errorf("failed to add lrp %s ip %s into bfd listening list, %w", eip.Name, eip.Status.V4Ip, err)

×

827

                                                                        klog.Error(err)

×

UNCOV

828

                                                                        return err

×

UNCOV

829

×

831

                                                } else {

×

832

                                                        err := fmt.Errorf("faild to check bfd status remote %s local %s", eip.Spec.V4Ip, nodeExtIP)

×

833

                                                        klog.Error(err)

×

834

                                                        return err

×

835

×

839

                        return err

×

843

        err = errors.New("node external gw not ready")

×

844

        klog.Error(err)

×

845

        return err

×

848

func configureNodeGwNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int, gwNS ns.NetNS) error {

×

849

        ipStr := util.GetIPWithoutMask(ip)

×

850

        output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeGwNic, "--",

×

851

                "set", "interface", util.NodeGwNic, "type=internal", "--",

×

852

                "set", "interface", util.NodeGwNic, fmt.Sprintf("external_ids:iface-id=%s", portName),

×

853

                fmt.Sprintf("external_ids:ip=%s", ipStr),

×

854

                fmt.Sprintf("external_ids:pod_netns=%s", util.NodeGwNsPath))

×

855

        if err != nil {

×

UNCOV

856

                klog.Errorf("failed to configure node external nic %s: %v, %q", portName, err, output)

×

857

                return errors.New(output)

×

858

×

859

        gwLink, err := netlink.LinkByName(util.NodeGwNic)

×

860

        if err == nil {

×

861

                fd := int(gwNS.Fd()) // #nosec G115

×

862

                if err = netlink.LinkSetNsFd(gwLink, fd); err != nil {

×

863

                        klog.Errorf("failed to move link into netns: %v", err)

×

864

                        return err

×

865

×

866

        } else {

×

867

                klog.V(3).Infof("node external nic %q already in ns %s", util.NodeGwNic, util.NodeGwNsPath)

×

868

×

869

        return ns.WithNetNSPath(gwNS.Path(), func(_ ns.NetNS) error {

×

870

                if err = configureNic(util.NodeGwNic, ip, macAddr, mtu, true, false, false); err != nil {

×

871

                        klog.Errorf("failed to configure node gw nic %s, %v", util.NodeGwNic, err)

×

872

                        return err

×

873

×

875

                if err = configureLoNic(); err != nil {

×

876

                        klog.Errorf("failed to configure nic %s, %v", util.LoNic, err)

×

877

                        return err

×

878

×

879

                gwLink, err = netlink.LinkByName(util.NodeGwNic)

×

880

                if err != nil {

×

881

                        klog.Errorf("failed to get link %q, %v", util.NodeGwNic, err)

×

882

                        return err

×

883

×

884

                switch util.CheckProtocol(ip) {

×

885

                case kubeovnv1.ProtocolIPv4:

×

886

                        _, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")

×

887

                        err = netlink.RouteReplace(&netlink.Route{

×

888

                                LinkIndex: gwLink.Attrs().Index,

×

889

                                Scope:     netlink.SCOPE_UNIVERSE,

×

890

                                Dst:       defaultNet,

×

891

                                Gw:        net.ParseIP(gw),

×

892

})

×

893

                case kubeovnv1.ProtocolIPv6:

×

894

                        _, defaultNet, _ := net.ParseCIDR("::/0")

×

UNCOV

895

                        err = netlink.RouteReplace(&netlink.Route{

×

896

                                LinkIndex: gwLink.Attrs().Index,

×

897

                                Scope:     netlink.SCOPE_UNIVERSE,

×

898

                                Dst:       defaultNet,

×

899

                                Gw:        net.ParseIP(gw),

×

900

})

×

901

                case kubeovnv1.ProtocolDual:

×

902

                        gws := strings.Split(gw, ",")

×

UNCOV

903

                        _, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")

×

904

                        err = netlink.RouteReplace(&netlink.Route{

×

905

                                LinkIndex: gwLink.Attrs().Index,

×

906

                                Scope:     netlink.SCOPE_UNIVERSE,

×

907

                                Dst:       defaultNet,

×

908

                                Gw:        net.ParseIP(gws[0]),

×

909

})

×

910

                        if err != nil {

×

911

                                return fmt.Errorf("config v4 gateway failed: %w", err)

×

912

×

UNCOV

914

                        _, defaultNet, _ = net.ParseCIDR("::/0")

×

UNCOV

915

                        err = netlink.RouteReplace(&netlink.Route{

×

UNCOV

916

                                LinkIndex: gwLink.Attrs().Index,

×

917

                                Scope:     netlink.SCOPE_UNIVERSE,

×

918

                                Dst:       defaultNet,

×

919

                                Gw:        net.ParseIP(gws[1]),

×

920

})

×

922

                if err != nil {

×

UNCOV

923

                        return fmt.Errorf("failed to configure gateway: %w", err)

×

UNCOV

924

×

925

                cmd := exec.Command("bfdd-beacon", "--listen=0.0.0.0")

×

926

                if err := cmd.Run(); err != nil {

×

927

                        err := fmt.Errorf("failed to get start bfd listen, %w", err)

×

928

                        klog.Error(err)

×

929

                        return err

×

930

×

931

                return waitNetworkReady(util.NodeGwNic, ip, gw, true, true, 3, nil)

×

UNCOV

935

func removeNodeGwNic() error {

×

936

        if _, err := ovs.Exec(ovs.IfExists, "del-port", "br-int", util.NodeGwNic); err != nil {

×

937

                return fmt.Errorf("failed to remove ecmp external port %s from OVS bridge %s: %w", "br-int", util.NodeGwNic, err)

×

UNCOV

938

×

UNCOV

939

        klog.Infof("removed node external gw nic %q", util.NodeGwNic)

×

940

        return nil

×

943

func removeNodeGwNs() error {

×

944

        ns := netns.LoadNetNS(util.NodeGwNsPath)

×

945

        ok, err := ns.Closed()

×

UNCOV

946

        if err != nil {

×

947

                return fmt.Errorf("failed to remove node external gw ns %s: %w", util.NodeGwNs, err)

×

948

×

949

        if !ok {

×

950

                if err = ns.Remove(); err != nil {

×

951

                        return fmt.Errorf("failed to remove node external gw ns %s: %w", util.NodeGwNs, err)

×

952

×

954

        klog.Infof("node external gw ns %s removed", util.NodeGwNs)

×

955

        return nil

×

958

func (c *Controller) loopOvnExt0Check() {

×

959

        node, err := c.nodesLister.Get(c.config.NodeName)

×

960

        if err != nil {

×

961

                klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)

×

962

                return

×

963

×

965

        portName := node.Name

×

966

        needClean := false

×

967

        cachedEip, err := c.ovnEipsLister.Get(portName)

×

UNCOV

968

        if err != nil {

×

UNCOV

969

                if k8serrors.IsNotFound(err) {

×

970

                        val, ok := node.Labels[util.NodeExtGwLabel]

×

971

                        if !ok {

×

972

                                // not gw node before

×

973

                                return

×

974

×

975

                        if val == "false" {

×

976

                                // already clean

×

977

                                return

×

978

×

979

                        if val == "true" {

×

980

                                needClean = true

×

981

×

982

                } else {

×

983

                        klog.Errorf("failed to get ecmp gateway ovn eip, %v", err)

×

UNCOV

984

                        return

×

UNCOV

985

×

988

        if needClean {

×

989

                if err := removeNodeGwNic(); err != nil {

×

990

                        klog.Error(err)

×

991

                        return

×

992

×

993

                if err := removeNodeGwNs(); err != nil {

×

994

                        klog.Error(err)

×

995

                        return

×

996

×

997

                if err = c.patchNodeExternalGwLabel(false); err != nil {

×

998

                        klog.Errorf("failed to patch labels of node %s: %v", node.Name, err)

×

999

                        return

×

1000

×

1001

                return

×

1004

        if cachedEip.Status.V4Ip == "" {

×

1005

                klog.Errorf("ecmp gateway ovn eip still has no ip")

×

1006

                return

×

1007

×

1008

        ips := util.GetStringIP(cachedEip.Status.V4Ip, cachedEip.Status.V6Ip)

×

1009

        cachedSubnet, err := c.subnetsLister.Get(cachedEip.Spec.ExternalSubnet)

×

1010

        if err != nil {

×

1011

                klog.Errorf("failed to get external subnet %s, %v", cachedEip.Spec.ExternalSubnet, err)

×

1012

                return

×

1013

×

1014

        gw := cachedSubnet.Spec.Gateway

×

1015

        mac, err := net.ParseMAC(cachedEip.Status.MacAddress)

×

UNCOV

1016

        if err != nil {

×

1017

                klog.Errorf("failed to parse mac %s, %v", cachedEip.Status.MacAddress, err)

×

1018

                return

×

1019

×

1020

        gwNS, err := ns.GetNS(util.NodeGwNsPath)

×

1021

        if err != nil {

×

1022

                if _, ok := err.(ns.NSPathNotExistErr); !ok {

×

1023

                        klog.Errorf("failed to get netns from path %s: %v", util.NodeGwNsPath, err)

×

1024

                        return

×

1025

×

1026

                if err = newNetNS(util.NodeGwNsPath); err != nil {

×

1027

                        klog.Error(fmt.Errorf("failed to create gw ns %s: %w", util.NodeGwNs, err))

×

1028

                        return

×

1029

×

1030

                if gwNS, err = ns.GetNS(util.NodeGwNsPath); err != nil {

×

1031

                        klog.Errorf("failed to get netns from path %s: %v", util.NodeGwNsPath, err)

×

1032

                        return

×

1033

×

1035

        nodeExtIP := cachedEip.Spec.V4Ip

×

1036

        ipAddr, err := util.GetIPAddrWithMask(ips, cachedSubnet.Spec.CIDRBlock)

×

1037

        if err != nil {

×

1038

                klog.Errorf("failed to get ip addr with mask %s, %v", ips, err)

×

1039

                return

×

UNCOV

1040

×

UNCOV

1041

        if err := c.checkNodeGwNicInNs(nodeExtIP, ipAddr, gw, gwNS); err == nil {

×

1042

                // add all lrp ip in bfd listening list

×

1043

                return

×

1044

×

1045

        klog.Infof("setup nic ovnext0 ip %s, mac %v, mtu %d", ipAddr, mac, c.config.MTU)

×

1046

        if err := configureNodeGwNic(portName, ipAddr, gw, mac, c.config.MTU, gwNS); err != nil {

×

1047

                klog.Errorf("failed to setup ovnext0, %v", err)

×

1048

                return

×

1049

×

1050

        if err = c.patchNodeExternalGwLabel(true); err != nil {

×

1051

                klog.Errorf("failed to patch labels of node %s: %v", node.Name, err)

×

1052

                return

×

1053

×

1054

        if err = c.patchOvnEipStatus(portName, true); err != nil {

×

1055

                klog.Errorf("failed to patch status for eip %s, %v", portName, err)

×

1056

                return

×

1057

×

1060

func (c *Controller) patchOvnEipStatus(key string, ready bool) error {

×

1061

        cachedOvnEip, err := c.ovnEipsLister.Get(key)

×

1062

        if err != nil {

×

1063

                klog.Errorf("failed to get cached ovn eip '%s', %v", key, err)

×

1064

                return err

×

UNCOV

1065

×

1066

        ovnEip := cachedOvnEip.DeepCopy()

×

UNCOV

1067

        changed := false

×

UNCOV

1068

        if ovnEip.Status.Ready != ready {

×

1069

                ovnEip.Status.Ready = ready

×

1070

                changed = true

×

1071

×

1072

        if changed {

×

1073

                bytes, err := ovnEip.Status.Bytes()

×

1074

                if err != nil {

×

UNCOV

1075

                        klog.Errorf("failed to marshal ovn eip status '%s', %v", key, err)

×

1076

                        return err

×

1077

×

1078

                if _, err = c.config.KubeOvnClient.KubeovnV1().OvnEips().Patch(context.Background(), ovnEip.Name,

×

1079

                        types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil {

×

1080

                        klog.Errorf("failed to patch status for ovn eip '%s', %v", key, err)

×

UNCOV

1081

                        return err

×

1082

×

UNCOV

1084

        return nil

×

1087

func (c *Controller) patchNodeExternalGwLabel(enabled bool) error {

×

1088

        node, err := c.nodesLister.Get(c.config.NodeName)

×

1089

        if err != nil {

×

1090

                klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)

×

UNCOV

1091

                return err

×

1092

×

1094

        patch := util.KVPatch{util.NodeExtGwLabel: strconv.FormatBool(enabled)}

×

1095

        if err = util.PatchLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, patch); err != nil {

×

1096

                klog.Errorf("failed to patch labels of node %s: %v", node.Name, err)

×

UNCOV

1097

                return err

×

UNCOV

1098

×

UNCOV

1100

        return nil

×

1103

func configureMirrorLink(portName string, _ int) error {

×

1104

        mirrorLink, err := netlink.LinkByName(portName)

×

1105

        if err != nil {

×

1106

                klog.Error(err)

×

UNCOV

1107

                return fmt.Errorf("can not find mirror nic %s: %w", portName, err)

×

UNCOV

1108

×

1110

        if mirrorLink.Attrs().OperState != netlink.OperUp {

×

1111

                if err = netlink.LinkSetUp(mirrorLink); err != nil {

×

1112

                        klog.Error(err)

×

1113

                        return fmt.Errorf("can not set mirror nic %s up: %w", portName, err)

×

1114

×

1117

        return nil

×

1121

func macToLinkLocalIPv6(mac net.HardwareAddr) (net.IP, error) {

×

1122

        if len(mac) != 6 {

×

UNCOV

1123

                return nil, errors.New("invalid MAC address length")

×

UNCOV

1124

×

1127

        eui64 := make([]byte, 8)

×

1128

        copy(eui64[0:3], mac[0:3]) // Copy the first 3 bytes

×

1129

        eui64[3] = 0xff            // Insert ff

×

1130

        eui64[4] = 0xfe            // Insert fe

×

UNCOV

1131

        copy(eui64[5:], mac[3:])   // Copy the last 3 bytes

×

1132

×

1133

        // Flip the 7th bit of the first byte

×

1134

        eui64[0] ^= 0x02

×

1135

×

UNCOV

1136

        // Prepend the link-local prefix

×

1137

        linkLocalIPv6 := net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

×

1138

        copy(linkLocalIPv6[8:], eui64)

×

1139

×

1140

        return linkLocalIPv6, nil

×

1143

func configureNic(link, ip string, macAddr net.HardwareAddr, mtu int, detectIPv4Conflict, setUfoOff, ipv6LinkLocalOn bool) error {

×

1144

        nodeLink, err := netlink.LinkByName(link)

×

1145

        if err != nil {

×

UNCOV

1146

                klog.Error(err)

×

UNCOV

1147

                return fmt.Errorf("can not find nic %s: %w", link, err)

×

1148

×

1150

        if err = netlink.LinkSetHardwareAddr(nodeLink, macAddr); err != nil {

×

1151

                klog.Error(err)

×

1152

                return fmt.Errorf("can not set mac address to nic %s: %w", link, err)

×

UNCOV

1153

×

1155

        if mtu > 0 {

×

1156

                if nodeLink.Type() == "openvswitch" {

×

1157

                        _, err = ovs.Exec("set", "interface", link, fmt.Sprintf(`mtu_request=%d`, mtu))

×

1158

                } else {

×

1159

                        err = netlink.LinkSetMTU(nodeLink, mtu)

×

1160

×

1161

                if err != nil {

×

UNCOV

1162

                        return fmt.Errorf("failed to set nic %s mtu: %w", link, err)

×

1163

×

1166

        if nodeLink.Attrs().OperState != netlink.OperUp {

×

1167

                if err = netlink.LinkSetUp(nodeLink); err != nil {

×

1168

                        klog.Error(err)

×

1169

                        return fmt.Errorf("can not set node nic %s up: %w", link, err)

×

1170

×

UNCOV

1173

        ipDelMap := make(map[string]netlink.Addr)

×

UNCOV

1174

        ipAddMap := make(map[string]netlink.Addr)

×

1175

        ipAddrs, err := netlink.AddrList(nodeLink, unix.AF_UNSPEC)

×

1176

        if err != nil {

×

1177

                klog.Error(err)

×

1178

                return fmt.Errorf("can not get addr %s: %w", nodeLink, err)

×

1179

×

1181

        isIPv6LinkLocalExist := false

×

1182

        for _, ipAddr := range ipAddrs {

×

1183

                if ipAddr.IP.IsLinkLocalUnicast() {

×

1184

                        // skip 169.254.0.0/16 and fe80::/10

×

1185

                        if util.CheckProtocol(ipAddr.IP.String()) == kubeovnv1.ProtocolIPv6 {

×

UNCOV

1186

                                isIPv6LinkLocalExist = true

×

UNCOV

1187

×

1188

                        continue

×

1190

                ipDelMap[ipAddr.IPNet.String()] = ipAddr

×

UNCOV

1193

        if ipv6LinkLocalOn && !isIPv6LinkLocalExist && (util.CheckProtocol(ip) == kubeovnv1.ProtocolIPv6 || util.CheckProtocol(ip) == kubeovnv1.ProtocolDual) {

×

UNCOV

1194

                linkLocal, err := macToLinkLocalIPv6(macAddr)

×

1195

                if err != nil {

×

1196

                        return fmt.Errorf("failed to generate link-local address: %w", err)

×

1197

×

1198

                ipAddMap[linkLocal.String()] = netlink.Addr{

×

1199

                        IPNet: &net.IPNet{

×

UNCOV

1200

                                IP:   linkLocal,

×

UNCOV

1201

                                Mask: net.CIDRMask(64, 128),

×

1202

},

×

1203

×

1206

        for _, ipStr := range strings.Split(ip, ",") {

×

1207

                // Do not reassign same address for link

×

UNCOV

1208

                if _, ok := ipDelMap[ipStr]; ok {

×

1209

                        delete(ipDelMap, ipStr)

×

1210

                        continue

×

1213

                ipAddr, err := netlink.ParseAddr(ipStr)

×

1214

                if err != nil {

×

1215

                        return fmt.Errorf("can not parse address %s: %w", ipStr, err)

×

1216

×

1217

                ipAddMap[ipStr] = *ipAddr

×

1220

        for ip, addr := range ipDelMap {

×

UNCOV

1221

                klog.Infof("delete ip address %s on %s", ip, link)

×

1222

                if err = netlink.AddrDel(nodeLink, &addr); err != nil {

×

1223

                        klog.Error(err)

×

1224

                        return fmt.Errorf("delete address %s: %w", addr, err)

×

1225

×

UNCOV

1227

        for ip, addr := range ipAddMap {

×

UNCOV

1228

                if addr.IP.To4() != nil {

×

1229

                        if detectIPv4Conflict {

×

1230

                                ip := addr.IP.String()

×

1231

                                mac, err := util.ArpDetectIPConflict(link, ip, macAddr)

×

1232

                                if err != nil {

×

1233

                                        err = fmt.Errorf("failed to detect address conflict for %s on link %s: %w", ip, link, err)

×

UNCOV

1234

                                        klog.Error(err)

×

UNCOV

1235

                                        return err

×

1236

×

1237

                                if mac != nil {

×

1238

                                        return fmt.Errorf("IP address %s has already been used by host with MAC %s", ip, mac)

×

1239

×

1240

                        } else {

×

1241

                                // when detectIPConflict is true, free arp is already broadcast in the step of announcement

×

UNCOV

1242

                                if err := util.AnnounceArpAddress(link, addr.IP.String(), macAddr, 1, 1*time.Second); err != nil {

×

UNCOV

1243

                                        klog.Warningf("failed to broadcast free arp with err %v", err)

×

1244

×

1248

                klog.Infof("add ip address %s to %s", ip, link)

×

1249

                if err = netlink.AddrAdd(nodeLink, &addr); err != nil {

×

1250

                        klog.Error(err)

×

1251

                        return fmt.Errorf("can not add address %s to nic %s: %w", addr, link, err)

×

1252

×

1255

        if setUfoOff {

×

1256

                cmd := fmt.Sprintf("if ethtool -k %s | grep -q ^udp-fragmentation-offload; then ethtool -K %s ufo off; fi", link, link)

×

1257

                if output, err := exec.Command("sh", "-xc", cmd).CombinedOutput(); err != nil {

×

1258

                        klog.Error(err)

×

1259

                        return fmt.Errorf("failed to disable udp-fragmentation-offload feature of device %s to off: %w, %s", link, err, output)

×

1260

×

1263

        return nil

×

1266

func configureLoNic() error {

×

1267

        loLink, err := netlink.LinkByName(util.LoNic)

×

1268

        if err != nil {

×

1269

                err := fmt.Errorf("can not find nic %s, %w", util.LoNic, err)

×

1270

                klog.Error(err)

×

1271

                return err

×

1272

×

1274

        if loLink.Attrs().OperState != netlink.OperUp {

×

UNCOV

1275

                if err = netlink.LinkSetUp(loLink); err != nil {

×

1276

                        err := fmt.Errorf("failed to set up nic %s, %w", util.LoNic, err)

×

1277

                        klog.Error(err)

×

1278

                        return err

×

1279

×

1282

        return nil

×

1285

func (c *Controller) transferAddrsAndRoutes(nicName, brName string, delNonExistent bool) (int, error) {

×

1286

        nic, err := netlink.LinkByName(nicName)

×

1287

        if err != nil {

×

1288

                return 0, fmt.Errorf("failed to get nic by name %s: %w", nicName, err)

×

UNCOV

1289

×

1290

        bridge, err := netlink.LinkByName(brName)

×

1291

        if err != nil {

×

1292

                return 0, fmt.Errorf("failed to get bridge by name %s: %w", brName, err)

×

1293

×

1295

        addrs, err := netlink.AddrList(nic, netlink.FAMILY_ALL)

×

UNCOV

1296

        if err != nil {

×

UNCOV

1297

                return 0, fmt.Errorf("failed to get addresses on nic %s: %w", nicName, err)

×

1298

×

1299

        routes, err := netlink.RouteList(nic, netlink.FAMILY_ALL)

×

1300

        if err != nil {

×

1301

                return 0, fmt.Errorf("failed to get routes on nic %s: %w", nicName, err)

×

1302

×

UNCOV

1304

        brAddrs, err := netlink.AddrList(bridge, netlink.FAMILY_ALL)

×

1305

        if err != nil {

×

1306

                return 0, fmt.Errorf("failed to get addresses on OVS bridge %s: %w", brName, err)

×

1307

×

UNCOV

1309

        var delAddrs []netlink.Addr

×

UNCOV

1310

        if delNonExistent {

×

UNCOV

1311

                for _, addr := range brAddrs {

×

1312

                        if addr.IP.IsLinkLocalUnicast() {

×

1313

                                // skip 169.254.0.0/16 and fe80::/10

×

1314

                                continue

×

1317

                        var found bool

×

1318

                        for _, v := range addrs {

×

1319

                                if v.Equal(addr) {

×

UNCOV

1320

                                        found = true

×

1321

                                        break

×

1324

                        if !found {

×

1325

                                delAddrs = append(delAddrs, addr)

×

UNCOV

1326

×

1331

        if err = c.nmSyncer.SetManaged(nicName, false); err != nil {

×

1332

                klog.Errorf("failed to set device %s unmanaged by NetworkManager: %v", nicName, err)

×

1333

                return 0, err

×

1334

×

1335

        if err = c.nmSyncer.AddDevice(nicName, brName); err != nil {

×

1336

                klog.Errorf("failed to monitor NetworkManager event for device %s: %v", nicName, err)

×

1337

                return 0, err

×

1338

×

1340

        var count int

×

1341

        for _, addr := range addrs {

×

UNCOV

1342

                if addr.IP.IsLinkLocalUnicast() {

×

UNCOV

1343

                        // skip 169.254.0.0/16 and fe80::/10

×

1344

                        continue

×

1346

                count++

×

1347

×

1348

                if err = netlink.AddrDel(nic, &addr); err != nil {

×

1349

                        errMsg := fmt.Errorf("failed to delete address %q on nic %s: %w", addr.String(), nicName, err)

×

1350

                        klog.Error(errMsg)

×

1351

                        return 0, errMsg

×

UNCOV

1352

×

UNCOV

1353

                klog.Infof("address %q has been removed from link %s", addr.String(), nicName)

×

UNCOV

1354

×

UNCOV

1355

                addr.Label = ""

×

UNCOV

1356

                addr.PreferedLft, addr.ValidLft = 0, 0

×

1357

                if err = netlink.AddrReplace(bridge, &addr); err != nil {

×

1358

                        return 0, fmt.Errorf("failed to replace address %q on OVS bridge %s: %w", addr.String(), brName, err)

×

1359

×

1360

                klog.Infof("address %q has been added/replaced to link %s", addr.String(), brName)

×

1363

        if count != 0 {

×

1364

                for _, addr := range delAddrs {

×

UNCOV

1365

                        if err = netlink.AddrDel(bridge, &addr); err != nil {

×

UNCOV

1366

                                errMsg := fmt.Errorf("failed to delete address %q on OVS bridge %s: %w", addr.String(), brName, err)

×

1367

                                klog.Error(errMsg)

×

1368

                                return 0, errMsg

×

1369

×

UNCOV

1370

                        klog.Infof("address %q has been removed from OVS bridge %s", addr.String(), brName)

×

UNCOV

1376

        albBond, err := linkIsAlbBond(nic)

×

1377

        if err != nil {

×

1378

                return 0, err

×

1379

×

1380

        if !albBond {

×

1381

                if _, err = ovs.Exec("set", "bridge", brName, fmt.Sprintf(`other-config:hwaddr="%s"`, nic.Attrs().HardwareAddr.String())); err != nil {

×

1382

                        return 0, fmt.Errorf("failed to set MAC address of OVS bridge %s: %w", brName, err)

×

UNCOV

1383

×

UNCOV

1386

        if err = netlink.LinkSetUp(bridge); err != nil {

×

1387

                return 0, fmt.Errorf("failed to set OVS bridge %s up: %w", brName, err)

×

1388

×

1390

        for _, scope := range routeScopeOrders {

×

UNCOV

1391

                for _, route := range routes {

×

1392

                        if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() {

×

1393

                                // skip 169.254.0.0/16 and fe80::/10

×

1394

                                continue

×

1396

                        if route.Scope == scope {

×

1397

                                route.LinkIndex = bridge.Attrs().Index

×

UNCOV

1398

                                if err = netlink.RouteReplace(&route); err != nil {

×

UNCOV

1399

                                        return 0, fmt.Errorf("failed to add/replace route %s to OVS bridge %s: %w", route.String(), brName, err)

×

1400

×

1401

                                klog.Infof("route %q has been added/replaced to OVS bridge %s", route.String(), brName)

×

1406

        brRoutes, err := netlink.RouteList(bridge, netlink.FAMILY_ALL)

×

UNCOV

1407

        if err != nil {

×

UNCOV

1408

                return 0, fmt.Errorf("failed to get routes on OVS bridge %s: %w", brName, err)

×

1409

×

1411

        var delRoutes []netlink.Route

×

UNCOV

1412

        if delNonExistent && count != 0 {

×

UNCOV

1413

                for _, route := range brRoutes {

×

UNCOV

1414

                        if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() {

×

1415

                                // skip 169.254.0.0/16 and fe80::/10

×

1416

                                continue

×

1419

                        var found bool

×

1420

                        for _, v := range routes {

×

1421

                                v.LinkIndex = route.LinkIndex

×

UNCOV

1422

                                v.ILinkIndex = route.ILinkIndex

×

UNCOV

1423

                                if v.Equal(route) {

×

UNCOV

1424

                                        found = true

×

UNCOV

1425

                                        break

×

1428

                        if !found {

×

UNCOV

1429

                                delRoutes = append(delRoutes, route)

×

1430

×

UNCOV

1434

        for i := len(routeScopeOrders) - 1; i >= 0; i-- {

×

1435

                for _, route := range delRoutes {

×

1436

                        if route.Scope == routeScopeOrders[i] {

×

1437

                                if err = netlink.RouteDel(&route); err != nil {

×

1438

                                        return 0, fmt.Errorf("failed to delete route %s from OVS bridge %s: %w", route.String(), brName, err)

×

1439

×

1440

                                klog.Infof("route %q has been deleted from OVS bridge %s", route.String(), brName)

×

1445

        if err = netlink.LinkSetUp(nic); err != nil {

×

1446

                return 0, fmt.Errorf("failed to set link %s up: %w", nicName, err)

×

1447

×

UNCOV

1449

        return nic.Attrs().MTU, nil

×

1454

func (c *Controller) configProviderNic(nicName, brName string, trunks []string) (int, error) {

×

1455

        isUserspaceDP, err := ovs.IsUserspaceDataPath()

×

1456

        if err != nil {

×

1457

                klog.Error(err)

×

1458

                return 0, err

×

UNCOV

1459

×

UNCOV

1461

        var mtu int

×

UNCOV

1462

        if !isUserspaceDP {

×

1463

                mtu, err = c.transferAddrsAndRoutes(nicName, brName, false)

×

1464

                if err != nil {

×

1465

                        klog.Errorf("failed to transfer addresses and routes from %s to %s: %v", nicName, brName, err)

×

1466

                        return 0, err

×

1467

×

1469

                if _, err = ovs.Exec(ovs.MayExist, "add-port", brName, nicName,

×

1470

                        "--", "set", "port", nicName, "trunks="+strings.Join(trunks, ","), "external_ids:vendor="+util.CniTypeName); err != nil {

×

1471

                        klog.Errorf("failed to add %s to OVS bridge %s: %v", nicName, brName, err)

×

UNCOV

1472

                        return 0, err

×

1473

×

1474

                klog.V(3).Infof("ovs port %s has been added to bridge %s", nicName, brName)

×

1475

        } else {

×

1476

                mtu = c.config.MTU

×

1477

×

1479

        return mtu, nil

×

UNCOV

1482

func linkIsAlbBond(link netlink.Link) (bool, error) {

×

1483

        check := func(link netlink.Link) bool {

×

UNCOV

1484

                bond, ok := link.(*netlink.Bond)

×

UNCOV

1485

                return ok && bond.Mode == netlink.BOND_MODE_BALANCE_ALB

×

UNCOV

1486

×

1488

        if check(link) {

×

1489

                return true, nil

×

1490

×

1492

        vlan, ok := link.(*netlink.Vlan)

×

1493

        if !ok {

×

1494

                return false, nil

×

1495

×

1496

        parent, err := netlink.LinkByIndex(vlan.ParentIndex)

×

1497

        if err != nil {

×

UNCOV

1498

                klog.Errorf("failed to get link by index %d: %v", vlan.ParentIndex, err)

×

1499

                return false, err

×

1500

×

1502

        return check(parent), nil

×

1507

func (c *Controller) removeProviderNic(nicName, brName string) error {

×

1508

        c.nmSyncer.RemoveDevice(nicName)

×

1509

×

1510

        nic, err := netlink.LinkByName(nicName)

×

1511

        if err != nil {

×

UNCOV

1512

                if _, ok := err.(netlink.LinkNotFoundError); ok {

×

1513

                        klog.Warningf("failed to get nic by name %s: %v", nicName, err)

×

1514

                        return nil

×

1515

×

1516

                return fmt.Errorf("failed to get nic by name %s: %w", nicName, err)

×

1518

        bridge, err := netlink.LinkByName(brName)

×

1519

        if err != nil {

×

1520

                return fmt.Errorf("failed to get bridge by name %s: %w", brName, err)

×

1521

×

UNCOV

1523

        addrs, err := netlink.AddrList(bridge, netlink.FAMILY_ALL)

×

1524

        if err != nil {

×

1525

                return fmt.Errorf("failed to get addresses on bridge %s: %w", brName, err)

×

1526

×

1527

        routes, err := netlink.RouteList(bridge, netlink.FAMILY_ALL)

×

1528

        if err != nil {

×

1529

                return fmt.Errorf("failed to get routes on bridge %s: %w", brName, err)

×

1530

×

1532

        if _, err = ovs.Exec(ovs.IfExists, "del-port", brName, nicName); err != nil {

×

1533

                return fmt.Errorf("failed to remove %s from OVS bridge %s: %w", nicName, brName, err)

×

1534

×

1535

        klog.V(3).Infof("ovs port %s has been removed from bridge %s", nicName, brName)

×

UNCOV

1536

×

UNCOV

1537

        for _, addr := range addrs {

×

1538

                if addr.IP.IsLinkLocalUnicast() {

×

1539

                        // skip 169.254.0.0/16 and fe80::/10

×

1540

                        continue

×

1543

                if err = netlink.AddrDel(bridge, &addr); err != nil {

×

1544

                        errMsg := fmt.Errorf("failed to delete address %q on OVS bridge %s: %w", addr.String(), brName, err)

×

1545

                        klog.Error(errMsg)

×

1546

                        return errMsg

×

1547

×

1548

                klog.Infof("address %q has been deleted from link %s", addr.String(), brName)

×

1549

×

1550

                addr.Label = ""

×

1551

                if err = netlink.AddrReplace(nic, &addr); err != nil {

×

1552

                        return fmt.Errorf("failed to replace address %q on nic %s: %w", addr.String(), nicName, err)

×

1553

×

UNCOV

1554

                klog.Infof("address %q has been added/replaced to link %s", addr.String(), nicName)

×

1557

        if err = netlink.LinkSetUp(nic); err != nil {

×

1558

                klog.Errorf("failed to set link %s up: %v", nicName, err)

×

1559

                return err

×

1560

×

UNCOV

1562

        scopeOrders := [...]netlink.Scope{

×

UNCOV

1563

                netlink.SCOPE_HOST,

×

UNCOV

1564

                netlink.SCOPE_LINK,

×

1565

                netlink.SCOPE_SITE,

×

1566

                netlink.SCOPE_UNIVERSE,

×

1567

×

1568

        for _, scope := range scopeOrders {

×

1569

                for _, route := range routes {

×

1570

                        if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() {

×

UNCOV

1571

                                // skip 169.254.0.0/16 and fe80::/10

×

UNCOV

1572

                                continue

×

1574

                        if route.Scope == scope {

×

1575

                                route.LinkIndex = nic.Attrs().Index

×

1576

                                if err = netlink.RouteReplace(&route); err != nil {

×

1577

                                        return fmt.Errorf("failed to add/replace route %s: %w", route.String(), err)

×

1578

×

1579

                                klog.Infof("route %q has been added/replaced to link %s", route.String(), nicName)

×

1584

        if err = netlink.LinkSetDown(bridge); err != nil {

×

1585

                return fmt.Errorf("failed to set OVS bridge %s down: %w", brName, err)

×

1586

×

1587

        klog.V(3).Infof("link %s has been set down", brName)

×

1588

×

1589

        return nil

×

UNCOV

1592

func setupVethPair(containerID, ifName string, mtu int) (string, string, error) {

×

UNCOV

1593

        var err error

×

UNCOV

1594

        hostNicName, containerNicName := generateNicName(containerID, ifName)

×

UNCOV

1595

        // Create a veth pair, put one end to container ,the other to ovs port

×

1596

        // NOTE: DO NOT use ovs internal type interface for container.

×

1597

        // Kubernetes will detect 'eth0' nic in pod, so the nic name in pod must be 'eth0'.

×

1598

        // When renaming internal interface to 'eth0', ovs will delete and recreate this interface.

×

1599

        veth := netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostNicName}, PeerName: containerNicName}

×

1600

        if mtu > 0 {

×

1601

                veth.MTU = mtu

×

1602

×

UNCOV

1603

        if err = netlink.LinkAdd(&veth); err != nil {

×

1604

                if err := netlink.LinkDel(&veth); err != nil {

×

1605

                        klog.Errorf("failed to delete veth %v", err)

×

1606

                        return "", "", err

×

1607

×

UNCOV

1608

                return "", "", fmt.Errorf("failed to create veth for %w", err)

×

1610

        return hostNicName, containerNicName, nil

×

1615

func setupSriovInterface(containerID, deviceID, vfDriver, ifName string, mtu int, mac string) (string, string, string, int, error) {

×

1616

        isVfioPciDriver := false

×

1617

        if vfDriver == "vfio-pci" {

×

UNCOV

1618

                matches, err := filepath.Glob(filepath.Join(util.VfioSysDir, "*"))

×

UNCOV

1619

                if err != nil {

×

1620

                        return "", "", "", -1, fmt.Errorf("failed to check %s 'vfio-pci' driver path, %w", deviceID, err)

×

1621

×

1623

                for _, match := range matches {

×

1624

                        tmp, err := os.Readlink(match)

×

1625

                        if err != nil {

×

1626

                                continue

×

UNCOV

1628

                        if strings.Contains(tmp, deviceID) {

×

UNCOV

1629

                                isVfioPciDriver = true

×

1630

                                break

×

UNCOV

1634

                if !isVfioPciDriver {

×

UNCOV

1635

                        return "", "", "", -1, fmt.Errorf("driver of device %s is not 'vfio-pci'", deviceID)

×

1636

×

1639

        var vfNetdevice string

×

1640

        if !isVfioPciDriver {

×

1641

                // 1. get VF netdevice from PCI

×

UNCOV

1642

                vfNetdevices, err := sriovnet.GetNetDevicesFromPci(deviceID)

×

UNCOV

1643

                if err != nil {

×

1644

                        klog.Errorf("failed to get vf netdevice %s, %v", deviceID, err)

×

1645

                        return "", "", "", -1, err

×

1646

×

UNCOV

1649

                if len(vfNetdevices) != 1 {

×

UNCOV

1650

                        return "", "", "", -1, fmt.Errorf("failed to get one netdevice interface per %s", deviceID)

×

1651

×

1652

                vfNetdevice = vfNetdevices[0]

×

UNCOV

1655

        if yusur.IsYusurSmartNic(deviceID) {

×

UNCOV

1656

                // 2. get PF PCI

×

1657

                pfPci, err := yusur.GetYusurNicPfPciFromVfPci(deviceID)

×

1658

                if err != nil {

×

1659

                        return "", "", "", -1, err

×

1660

×

UNCOV

1663

                pfIndex, err := yusur.GetYusurNicPfIndexByPciAddress(pfPci)

×

1664

                if err != nil {

×

UNCOV

1665

                        klog.Errorf("failed to get up %s link device, %v", deviceID, err)

×

UNCOV

1666

                        return "", "", "", -1, err

×

UNCOV

1667

×

1670

                vfIndex, err := yusur.GetYusurNicVfIndexByPciAddress(deviceID)

×

1671

                if err != nil {

×

1672

                        return "", "", "", -1, err

×

UNCOV

1673

×

1676

                rep := yusur.GetYusurNicVfRepresentor(pfIndex, vfIndex)

×

1677

×

1678

                _, err = netlink.LinkByName(rep)

×

1679

                if err != nil {

×

UNCOV

1680

                        klog.Infof("vfr not exist %s", rep)

×

UNCOV

1681

×

1683

                return rep, vfNetdevice, pfPci, vfIndex, nil

×

1687

        uplink, err := sriovnet.GetUplinkRepresentor(deviceID)

×

1688

        if err != nil {

×

1689

                klog.Errorf("failed to get up %s link device, %v", deviceID, err)

×

1690

                return "", "", "", -1, err

×

1691

×

UNCOV

1694

        vfIndex, err := sriovnet.GetVfIndexByPciAddress(deviceID)

×

1695

        if err != nil {

×

1696

                klog.Errorf("failed to get vf %s index, %v", deviceID, err)

×

1697

                return "", "", "", -1, err

×

1698

×

1701

        rep, err := sriovnet.GetVfRepresentor(uplink, vfIndex)

×

1702

        if err != nil {

×

1703

                klog.Errorf("failed to get vf %d representor, %v", vfIndex, err)

×

UNCOV

1704

                return "", "", "", -1, err

×

UNCOV

1705

×

1706

        oldHostRepName := rep

×

1707

×

1708

        // 5. rename the host VF representor

×

UNCOV

1709

        hostNicName, _ := generateNicName(containerID, ifName)

×

1710

        if err = renameLink(oldHostRepName, hostNicName); err != nil {

×

UNCOV

1711

                return "", "", "", -1, fmt.Errorf("failed to rename %s to %s: %w", oldHostRepName, hostNicName, err)

×

UNCOV

1712

×

1714

        link, err := netlink.LinkByName(hostNicName)

×

1715

        if err != nil {

×

1716

                return "", "", "", -1, err

×

1717

×

1720

        if err = netlink.LinkSetMTU(link, mtu); err != nil {

×

1721

                return "", "", "", -1, fmt.Errorf("failed to set MTU on %s: %w", hostNicName, err)

×

1722

×

1725

        if err = setVfMac(deviceID, vfIndex, mac); err != nil {

×

1726

                return "", "", "", -1, err

×

1727

×

UNCOV

1729

        return hostNicName, vfNetdevice, "", -1, nil

×

1732

func renameLink(curName, newName string) error {

×

1733

        link, err := netlink.LinkByName(curName)

×

1734

        if err != nil {

×

1735

                klog.Error(err)

×

1736

                return err

×

1737

×

1739

        if err := netlink.LinkSetDown(link); err != nil {

×

1740

                klog.Error(err)

×

1741

                return err

×

1742

×

1743

        if err := netlink.LinkSetName(link, newName); err != nil {

×

1744

                klog.Error(err)

×

1745

                return err

×

1746

×

1747

        return netlink.LinkSetUp(link)

×

1750

func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, _, _ []string, ingress, egress, _, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP string) (string, []request.Route, error) {

×

1751

        _, containerNicName := generateNicName(containerID, ifName)

×

1752

        ipStr := util.GetIPWithoutMask(ip)

×

1753

        ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)

×

1754

        ovs.CleanDuplicatePort(ifaceID, containerNicName)

×

1755

×

1756

        // Add container iface to ovs port as internal port

×

UNCOV

1757

        output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", containerNicName, "--",

×

UNCOV

1758

                "set", "interface", containerNicName, "type=internal", "--",

×

UNCOV

1759

                "set", "interface", containerNicName, fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

UNCOV

1760

                fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName),

×

1761

                fmt.Sprintf("external_ids:pod_name=%s", podName),

×

1762

                fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

1763

                fmt.Sprintf("external_ids:ip=%s", ipStr),

×

1764

                fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

UNCOV

1765

        if err != nil {

×

1766

                err := fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

1767

                klog.Error(err)

×

1768

                return containerNicName, nil, err

×

UNCOV

1769

×

1770

        defer func() {

×

1771

                if err != nil {

×

1772

                        if err := csh.rollbackOvsPort("", containerNicName, nicType); err != nil {

×

UNCOV

1773

                                klog.Errorf("failed to rollback ovs port %s, %v", containerNicName, err)

×

1774

                                return

×

1775

×

UNCOV

1780

        macAddr, err := net.ParseMAC(mac)

×

UNCOV

1781

        if err != nil {

×

1782

                return containerNicName, nil, fmt.Errorf("failed to parse mac %s %w", macAddr, err)

×

1783

×

1785

        if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress); err != nil {

×

1786

                return containerNicName, nil, err

×

UNCOV

1787

×

1789

        if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss, jitter); err != nil {

×

1790

                return containerNicName, nil, err

×

1791

×

UNCOV

1793

        podNS, err := ns.GetNS(netns)

×

1794

        if err != nil {

×

1795

                return containerNicName, nil, fmt.Errorf("failed to open netns %q: %w", netns, err)

×

1796

×

1797

        routes, err = csh.configureContainerNic(podName, podNamespace, containerNicName, ifName, ip, gateway, isDefaultRoute, detectIPConflict, routes, macAddr, podNS, mtu, nicType, gwCheckMode, u2oInterconnectionIP)

×

UNCOV

1798

        return containerNicName, routes, err

×

1801

func (csh cniServerHandler) removeDefaultRoute(netns string, ipv4, ipv6 bool) error {

×

1802

        podNS, err := ns.GetNS(netns)

×

1803

        if err != nil {

×

1804

                return fmt.Errorf("failed to open netns %q: %w", netns, err)

×

1805

×

1807

        return ns.WithNetNSPath(podNS.Path(), func(_ ns.NetNS) error {

×

1808

                routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)

×

1809

                if err != nil {

×

1810

                        return fmt.Errorf("failed to get all routes: %w", err)

×

1811

×

UNCOV

1813

                for _, r := range routes {

×

1814

                        if r.Dst != nil {

×

UNCOV

1815

                                if ones, _ := r.Dst.Mask.Size(); ones != 0 {

×

UNCOV

1816

                                        continue

×

1819

                        if ipv4 && r.Family == netlink.FAMILY_V4 {

×

1820

                                klog.Infof("deleting default ipv4 route %+v", r)

×

1821

                                if err = netlink.RouteDel(&r); err != nil {

×

1822

                                        return fmt.Errorf("failed to delete route %+v: %w", r, err)

×

1823

×

UNCOV

1824

                                continue

×

1826

                        if ipv6 && r.Family == netlink.FAMILY_V6 {

×

1827

                                klog.Infof("deleting default ipv6 route %+v", r)

×

1828

                                if err = netlink.RouteDel(&r); err != nil {

×

1829

                                        return fmt.Errorf("failed to delete route %+v: %w", r, err)

×

1830

×

1833

                return nil

×

UNCOV

1838

func configureAdditionalNic(link, ip string) error {

×

1839

        nodeLink, err := netlink.LinkByName(link)

×

1840

        if err != nil {

×

1841

                return fmt.Errorf("can not find nic %s %w", link, err)

×

1842

×

UNCOV

1844

        ipDelMap := make(map[string]netlink.Addr)

×

UNCOV

1845

        ipAddMap := make(map[string]netlink.Addr)

×

1846

        ipAddrs, err := netlink.AddrList(nodeLink, 0x0)

×

1847

        if err != nil {

×

1848

                return fmt.Errorf("can not get addr %s %w", nodeLink, err)

×

1849

×

1850

        for _, ipAddr := range ipAddrs {

×

UNCOV

1851

                if ipAddr.IP.IsLinkLocalUnicast() {

×

UNCOV

1852

                        // skip 169.254.0.0/16 and fe80::/10

×

1853

                        continue

×

1855

                ipDelMap[ipAddr.IPNet.String()] = ipAddr

×

1858

        for _, ipStr := range strings.Split(ip, ",") {

×

1859

                // Do not reassign same address for link

×

1860

                if _, ok := ipDelMap[ipStr]; ok {

×

1861

                        delete(ipDelMap, ipStr)

×

UNCOV

1862

                        continue

×

UNCOV

1865

                ipAddr, err := netlink.ParseAddr(ipStr)

×

UNCOV

1866

                if err != nil {

×

1867

                        return fmt.Errorf("can not parse %s %w", ipStr, err)

×

1868

×

1869

                ipAddMap[ipStr] = *ipAddr

×

1872

        for _, addr := range ipDelMap {

×

1873

                if err = netlink.AddrDel(nodeLink, &addr); err != nil {

×

1874

                        return fmt.Errorf("delete address %s %w", addr, err)

×

1875

×

1877

        for _, addr := range ipAddMap {

×

1878

                if err = netlink.AddrAdd(nodeLink, &addr); err != nil {

×

1879

                        return fmt.Errorf("can not add address %v to nic %s, %w", addr, link, err)

×

UNCOV

1880

×

UNCOV

1883

        return nil

×

1886

func addAdditionalNic(ifName string) error {

×

1887

        dummy := &netlink.Dummy{

×

1888

                LinkAttrs: netlink.LinkAttrs{

×

UNCOV

1889

                        Name: ifName,

×

1890

},

×

1891

×

1892

×

1893

        if err := netlink.LinkAdd(dummy); err != nil {

×

UNCOV

1894

                if err := netlink.LinkDel(dummy); err != nil {

×

1895

                        klog.Errorf("failed to delete static iface %v, err %v", ifName, err)

×

1896

                        return err

×

1897

×

1898

                return fmt.Errorf("failed to create static iface %v, err %w", ifName, err)

×

UNCOV

1900

        return nil

×

1903

func setVfMac(deviceID string, vfIndex int, mac string) error {

×

1904

        macAddr, err := net.ParseMAC(mac)

×

1905

        if err != nil {

×

1906

                return fmt.Errorf("failed to parse mac %s %w", macAddr, err)

×

UNCOV

1907

×

1909

        pfPci, err := sriovnet.GetPfPciFromVfPci(deviceID)

×

1910

        if err != nil {

×

1911

                return fmt.Errorf("failed to get pf of device %s %w", deviceID, err)

×

UNCOV

1912

×

1914

        netDevs, err := sriovnet.GetNetDevicesFromPci(pfPci)

×

1915

        if err != nil {

×

1916

                return fmt.Errorf("failed to get pf of device %s %w", deviceID, err)

×

UNCOV

1917

×

1920

        var pfName string

×

1921

        for _, dev := range netDevs {

×

1922

                devicePortNameFile := filepath.Join(util.NetSysDir, dev, "phys_port_name")

×

1923

                physPortName, err := sriovutilfs.Fs.ReadFile(devicePortNameFile)

×

1924

                if err != nil {

×

1925

                        continue

×

1928

                if !strings.Contains(strings.TrimSpace(string(physPortName)), "vf") {

×

1929

                        pfName = dev

×

1930

                        break

×

1933

        if pfName == "" {

×

1934

                return fmt.Errorf("the PF device was not found in the device list, %v", netDevs)

×

1935

×

1937

        pfLink, err := netlink.LinkByName(pfName)

×

1938

        if err != nil {

×

UNCOV

1939

                return fmt.Errorf("failed to lookup pf %s: %w", pfName, err)

×

UNCOV

1940

×

1941

        if err := netlink.LinkSetVfHardwareAddr(pfLink, vfIndex, macAddr); err != nil {

×

1942

                return fmt.Errorf("can not set mac address to vf nic:%s vf:%d %w", pfName, vfIndex, err)

×

1943

×

UNCOV

1944

        return nil

×

1947

func TurnOffNicTxChecksum(nicName string) error {

×

1948

        start := time.Now()

×

1949

        args := []string{"-K", nicName, "tx", "off"}

×

1950

        output, err := exec.Command("ethtool", args...).CombinedOutput() // #nosec G204

×

UNCOV

1951

        elapsed := float64((time.Since(start)) / time.Millisecond)

×

1952

        klog.V(4).Infof("command %s %s in %vms", "ethtool", strings.Join(args, " "), elapsed)

×

UNCOV

1953

        if err != nil {

×

UNCOV

1954

                klog.Error(err)

×

1955

                return fmt.Errorf("failed to turn off nic tx checksum, output %s, err %s", string(output), err.Error())

×

1956

×

1957

        return nil

×

1960

func getShortSharedDir(uid types.UID, volumeName string) string {

×

1961

        return filepath.Clean(filepath.Join(util.DefaultHostVhostuserBaseDir, string(uid), volumeName))

×

1962

×

1964

func linkExists(name string) (bool, error) {

×

UNCOV

1965

        if _, err := netlink.LinkByName(name); err != nil {

×

UNCOV

1966

                if _, ok := err.(netlink.LinkNotFoundError); ok {

×

1967

                        return false, nil

×

1968

×

1969

                return false, err

×

1971

        return true, nil

×

UNCOV

1974

func rollBackVethPair(nicName string) error {

×

1975

        hostLink, err := netlink.LinkByName(nicName)

×

1976

        if err != nil {

×

UNCOV

1977

                // if link already not exists, return quietly

×

UNCOV

1978

                // e.g. Internal port had been deleted by Remove ovs port previously

×

UNCOV

1979

                if _, ok := err.(netlink.LinkNotFoundError); ok {

×

UNCOV

1980

                        return nil

×

UNCOV

1981

×

UNCOV

1982

                klog.Error(err)

×

UNCOV

1983

                return fmt.Errorf("find host link %s failed %w", nicName, err)

×

UNCOV

1986

        hostLinkType := hostLink.Type()

×

UNCOV

1987

        // sometimes no deviceID input for vf nic, avoid delete vf nic.

×

UNCOV

1988

        if hostLinkType == "veth" {

×

UNCOV

1989

                if err = netlink.LinkDel(hostLink); err != nil {

×

UNCOV

1990

                        klog.Error(err)

×

UNCOV

1991

                        return fmt.Errorf("delete host link %s failed %w", hostLink, err)

×

UNCOV

1992

×

UNCOV

1994

        klog.Infof("rollback veth success %s", nicName)

×

UNCOV

1995

        return nil

×

kubeovn / kube-ovn / 13853401919

Source File
Press 'n' to go to next uncovered line, 'b' for previous

kubeovn / kube-ovn / 13853401919

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous