kubeovn/kube-ovn | Build 12365336204 | pkg/daemon/ovs_linux.go | Coveralls

41

func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, netns, containerID, ifName, _ string, _ int, ip, _, ingress, egress, shortSharedDir, socketName, socketConsumption string) error {

×

42

        sharedDir := filepath.Join("/var", shortSharedDir)

×

43

        hostNicName, _ := generateNicName(containerID, ifName)

×

44

×

45

        ipStr := util.GetIPWithoutMask(ip)

×

46

        ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)

×

47

        ovs.CleanDuplicatePort(ifaceID, hostNicName)

×

48

×

49

        vhostServerPath := path.Join(sharedDir, socketName)

×

50

        if socketConsumption == util.ConsumptionKubevirt {

×

51

                vhostServerPath = path.Join(sharedDir, ifName)

×

52

×

55

        output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", hostNicName, "--",

×

56

                "set", "interface", hostNicName,

×

57

                "type=dpdkvhostuserclient",

×

58

                fmt.Sprintf("options:vhost-server-path=%s", vhostServerPath),

×

59

                fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

60

                fmt.Sprintf("external_ids:pod_name=%s", podName),

×

61

                fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

62

                fmt.Sprintf("external_ids:ip=%s", ipStr),

×

63

                fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

64

        if err != nil {

×

65

                return fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

66

×

67

        return ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress)

×

70

func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, _, _ []string, ingress, egress, deviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP, oldPodName string) ([]request.Route, error) {

×

71

        var err error

×

72

        var hostNicName, containerNicName, pfPci string

×

73

        var vfID int

×

74

        if deviceID == "" {

×

75

                hostNicName, containerNicName, err = setupVethPair(containerID, ifName, mtu)

×

76

                if err != nil {

×

77

                        klog.Errorf("failed to create veth pair %v", err)

×

78

                        return nil, err

×

79

×

80

                defer func() {

×

81

                        if err != nil {

×

82

                                if err := rollBackVethPair(hostNicName); err != nil {

×

83

                                        klog.Errorf("failed to rollback veth pair %s, %v", hostNicName, err)

×

84

                                        return

×

85

×

88

        } else {

×

89

                hostNicName, containerNicName, pfPci, vfID, err = setupSriovInterface(containerID, deviceID, vfDriver, ifName, mtu, mac)

×

90

                if err != nil {

×

91

                        klog.Errorf("failed to create sriov interfaces %v", err)

×

92

                        return nil, err

×

93

×

96

        ipStr := util.GetIPWithoutMask(ip)

×

97

        ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)

×

98

        ovs.CleanDuplicatePort(ifaceID, hostNicName)

×

99

        if yusur.IsYusurSmartNic(deviceID) {

×

100

                klog.Infof("add Yusur smartnic vfr %s to ovs", hostNicName)

×

101

                // Add yusur ovs port

×

102

                output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", hostNicName, "--",

×

103

                        "set", "interface", hostNicName, "type=dpdk",

×

104

                        fmt.Sprintf("options:dpdk-devargs=%s,representor=[%d]", pfPci, vfID),

×

105

                        fmt.Sprintf("mtu_request=%d", mtu),

×

106

                        fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

107

                        fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName),

×

108

                        fmt.Sprintf("external_ids:pod_name=%s", podName),

×

109

                        fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

110

                        fmt.Sprintf("external_ids:ip=%s", ipStr),

×

111

                        fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

112

                if err != nil {

×

113

                        return nil, fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

114

×

115

        } else {

×

116

                // Add veth pair host end to ovs port

×

117

                output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", hostNicName, "--",

×

118

                        "set", "interface", hostNicName, fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

119

                        fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName),

×

120

                        fmt.Sprintf("external_ids:pod_name=%s", podName),

×

121

                        fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

122

                        fmt.Sprintf("external_ids:ip=%s", ipStr),

×

123

                        fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

124

                if err != nil {

×

125

                        return nil, fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

126

×

128

        defer func() {

×

129

                if err != nil {

×

130

                        if err := csh.rollbackOvsPort(hostNicName, containerNicName, nicType); err != nil {

×

131

                                klog.Errorf("failed to rollback ovs port %s, %v", hostNicName, err)

×

132

                                return

×

133

×

138

        if deviceID != "" {

×

139

                var podNameNew string

×

140

                if podName != oldPodName {

×

141

                        podNameNew = oldPodName

×

142

                } else {

×

143

                        podNameNew = podName

×

144

×

145

                var pod *v1.Pod

×

146

                pod, err = csh.Controller.podsLister.Pods(podNamespace).Get(podNameNew)

×

147

                if err != nil {

×

148

                        klog.Errorf("failed to generate patch for pod %s/%s: %v", podNameNew, podNamespace, err)

×

149

                        return nil, err

×

150

×

151

                oriPod := pod.DeepCopy()

×

152

                pod.Annotations[fmt.Sprintf(util.VfRepresentorNameTemplate, provider)] = hostNicName

×

153

                pod.Annotations[fmt.Sprintf(util.VfNameTemplate, provider)] = containerNicName

×

154

                pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, provider)] = util.SriovNicType

×

155

                var patch []byte

×

156

                patch, err = util.GenerateMergePatchPayload(oriPod, pod)

×

157

                if err != nil {

×

158

                        klog.Errorf("failed to generate patch for pod %s/%s: %v", podNameNew, podNamespace, err)

×

159

                        return nil, err

×

160

×

161

                if _, err = csh.Config.KubeClient.CoreV1().Pods(podNamespace).Patch(context.Background(), podNameNew,

×

162

                        types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {

×

163

                        klog.Errorf("patch pod %s/%s failed: %v", podNameNew, podNamespace, err)

×

164

                        return nil, err

×

165

×

169

        macAddr, err := net.ParseMAC(mac)

×

170

        if err != nil {

×

171

                return nil, fmt.Errorf("failed to parse mac %s %w", macAddr, err)

×

172

×

173

        if !yusur.IsYusurSmartNic(deviceID) {

×

174

                if err = configureHostNic(hostNicName); err != nil {

×

175

                        klog.Error(err)

×

176

                        return nil, err

×

177

×

179

        if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress); err != nil {

×

180

                klog.Error(err)

×

181

                return nil, err

×

182

×

184

        if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss, jitter); err != nil {

×

185

                klog.Error(err)

×

186

                return nil, err

×

187

×

189

        if containerNicName == "" {

×

190

                return nil, nil

×

191

×

192

        isUserspaceDP, err := ovs.IsUserspaceDataPath()

×

193

        if err != nil {

×

194

                klog.Error(err)

×

195

                return nil, err

×

196

×

197

        if isUserspaceDP {

×

198

                // turn off tx checksum

×

199

                if err = TurnOffNicTxChecksum(containerNicName); err != nil {

×

200

                        klog.Error(err)

×

201

                        return nil, err

×

202

×

205

        podNS, err := ns.GetNS(netns)

×

206

        if err != nil {

×

207

                err = fmt.Errorf("failed to open netns %q: %w", netns, err)

×

208

                klog.Error(err)

×

209

                return nil, err

×

210

×

211

        finalRoutes, err := csh.configureContainerNic(podName, podNamespace, containerNicName, ifName, ip, gateway, isDefaultRoute, detectIPConflict, routes, macAddr, podNS, mtu, nicType, gwCheckMode, u2oInterconnectionIP)

×

212

        if err != nil {

×

213

                klog.Error(err)

×

214

                return nil, err

×

215

×

216

        return finalRoutes, nil

×

219

func (csh cniServerHandler) releaseVf(podName, podNamespace, podNetns, ifName, nicType, deviceID string) error {

×

220

        // Only for SRIOV case, we'd need to move the VF from container namespace back to the host namespace

×

221

        if !(nicType == util.OffloadType && deviceID != "") {

×

222

                return nil

×

223

×

224

        podDesc := fmt.Sprintf("for pod %s/%s", podNamespace, podName)

×

225

        klog.Infof("Tear down interface %s", podDesc)

×

226

        netns, err := ns.GetNS(podNetns)

×

227

        if err != nil {

×

228

                return fmt.Errorf("failed to get container namespace %s: %w", podDesc, err)

×

229

×

230

        defer netns.Close()

×

231

×

232

        hostNS, err := ns.GetCurrentNS()

×

233

        if err != nil {

×

234

                return fmt.Errorf("failed to get host namespace %s: %w", podDesc, err)

×

235

×

236

        defer hostNS.Close()

×

237

×

238

        err = netns.Do(func(_ ns.NetNS) error {

×

239

                // container side interface deletion

×

240

                link, err := netlink.LinkByName(ifName)

×

241

                if err != nil {

×

242

                        return fmt.Errorf("failed to get container interface %s %s: %w", ifName, podDesc, err)

×

243

×

244

                if err = netlink.LinkSetDown(link); err != nil {

×

245

                        return fmt.Errorf("failed to bring down container interface %s %s: %w", ifName, podDesc, err)

×

246

×

248

                vfName := link.Attrs().Alias

×

249

                if err = netlink.LinkSetName(link, vfName); err != nil {

×

250

                        return fmt.Errorf("failed to rename container interface %s to %s %s: %w",

×

251

                                ifName, vfName, podDesc, err)

×

252

×

254

                fd := int(netns.Fd()) // #nosec G115

×

255

                if err = netlink.LinkSetNsFd(link, fd); err != nil {

×

256

                        return fmt.Errorf("failed to move container interface %s back to host namespace %s: %w",

×

257

                                ifName, podDesc, err)

×

258

×

259

                return nil

×

261

        if err != nil {

×

262

                klog.Error(err)

×

263

×

265

        return nil

×

268

func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, netns, deviceID, ifName, nicType string) error {

×

269

        if err := csh.releaseVf(podName, podNamespace, netns, ifName, nicType, deviceID); err != nil {

×

270

                return fmt.Errorf("failed to release VF %s assigned to the Pod %s/%s back to the host network namespace: "+

×

271

                        "%w", ifName, podName, podNamespace, err)

×

272

×

274

        var nicName string

×

275

        if yusur.IsYusurSmartNic(deviceID) {

×

276

                pfPci, err := yusur.GetYusurNicPfPciFromVfPci(deviceID)

×

277

                if err != nil {

×

278

                        return fmt.Errorf("failed to get pf pci %w, %s", err, deviceID)

×

279

×

281

                pfIndex, err := yusur.GetYusurNicPfIndexByPciAddress(pfPci)

×

282

                if err != nil {

×

283

                        return fmt.Errorf("failed to get pf index %w, %s", err, deviceID)

×

284

×

286

                vfIndex, err := yusur.GetYusurNicVfIndexByPciAddress(deviceID)

×

287

                if err != nil {

×

288

                        return fmt.Errorf("failed to get vf index %w, %s", err, deviceID)

×

289

×

291

                nicName = yusur.GetYusurNicVfRepresentor(pfIndex, vfIndex)

×

292

        } else {

×

293

                hostNicName, containerNicName := generateNicName(containerID, ifName)

×

294

×

295

                if nicType == util.InternalType {

×

296

                        nicName = containerNicName

×

297

                } else {

×

298

                        nicName = hostNicName

×

299

×

302

        output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", nicName)

×

303

        if err != nil {

×

304

                return fmt.Errorf("failed to delete ovs port %w, %q", err, output)

×

305

×

307

        if err = ovs.ClearPodBandwidth(podName, podNamespace, ""); err != nil {

×

308

                klog.Error(err)

×

309

                return err

×

310

×

311

        if err = ovs.ClearHtbQosQueue(podName, podNamespace, ""); err != nil {

×

312

                klog.Error(err)

×

313

                return err

×

314

×

316

        if deviceID == "" {

×

317

                hostLink, err := netlink.LinkByName(nicName)

×

318

                if err != nil {

×

319

                        // If link already not exists, return quietly

×

320

                        // E.g. Internal port had been deleted by Remove ovs port previously

×

321

                        if _, ok := err.(netlink.LinkNotFoundError); ok {

×

322

                                return nil

×

323

×

324

                        return fmt.Errorf("find host link %s failed %w", nicName, err)

×

327

                hostLinkType := hostLink.Type()

×

328

                // Sometimes no deviceID input for vf nic, avoid delete vf nic.

×

329

                if hostLinkType == "veth" {

×

330

                        if err = netlink.LinkDel(hostLink); err != nil {

×

331

                                return fmt.Errorf("delete host link %s failed %w", hostLink, err)

×

332

×

334

        } else if pciAddrRegexp.MatchString(deviceID) && !yusur.IsYusurSmartNic(deviceID) {

×

335

                // Ret VF index from PCI

×

336

                vfIndex, err := sriovnet.GetVfIndexByPciAddress(deviceID)

×

337

                if err != nil {

×

338

                        klog.Errorf("failed to get vf %s index, %v", deviceID, err)

×

339

                        return err

×

340

×

341

                if err = setVfMac(deviceID, vfIndex, "00:00:00:00:00:00"); err != nil {

×

342

                        klog.Error(err)

×

343

                        return err

×

344

×

346

        return nil

×

349

func (csh cniServerHandler) rollbackOvsPort(hostNicName, containerNicName, nicType string) (err error) {

×

350

        var nicName string

×

351

        if nicType == util.InternalType {

×

352

                nicName = containerNicName

×

353

        } else {

×

354

                nicName = hostNicName

×

355

×

356

        output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", nicName)

×

357

        if err != nil {

×

358

                klog.Warningf("failed to delete down ovs port %v, %q", err, output)

×

359

×

360

        klog.Infof("rollback ovs port success %s", nicName)

×

361

        return

×

364

func generateNicName(containerID, ifname string) (string, string) {

×

365

        if ifname == "eth0" {

×

366

                return fmt.Sprintf("%s_h", containerID[0:12]), fmt.Sprintf("%s_c", containerID[0:12])

×

367

×

369

        if strings.HasPrefix(ifname, "pod") && len(ifname) == 14 {

×

370

                ifname = ifname[3 : len(ifname)-4]

×

371

                return fmt.Sprintf("%s_%s_h", containerID[0:12-len(ifname)], ifname), fmt.Sprintf("%s_%s_c", containerID[0:12-len(ifname)], ifname)

×

372

×

373

        return fmt.Sprintf("%s_%s_h", containerID[0:12-len(ifname)], ifname), fmt.Sprintf("%s_%s_c", containerID[0:12-len(ifname)], ifname)

×

376

func configureHostNic(nicName string) error {

×

377

        hostLink, err := netlink.LinkByName(nicName)

×

378

        if err != nil {

×

379

                return fmt.Errorf("can not find host nic %s: %w", nicName, err)

×

380

×

382

        if hostLink.Attrs().OperState != netlink.OperUp {

×

383

                if err = netlink.LinkSetUp(hostLink); err != nil {

×

384

                        return fmt.Errorf("can not set host nic %s up: %w", nicName, err)

×

385

×

387

        if err = netlink.LinkSetTxQLen(hostLink, 1000); err != nil {

×

388

                return fmt.Errorf("can not set host nic %s qlen: %w", nicName, err)

×

389

×

391

        return nil

×

394

func (csh cniServerHandler) configureContainerNic(podName, podNamespace, nicName, ifName, ipAddr, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, gwCheckMode int, u2oInterconnectionIP string) ([]request.Route, error) {

×

395

        containerLink, err := netlink.LinkByName(nicName)

×

396

        if err != nil {

×

397

                return nil, fmt.Errorf("can not find container nic %s: %w", nicName, err)

×

398

×

401

        if err := netlink.LinkSetAlias(containerLink, nicName); err != nil {

×

402

                klog.Errorf("failed to set link alias for container nic %s: %v", nicName, err)

×

403

                return nil, err

×

404

×

406

        fd := int(netns.Fd()) // #nosec G115

×

407

        if err = netlink.LinkSetNsFd(containerLink, fd); err != nil {

×

408

                return nil, fmt.Errorf("failed to move link to netns: %w", err)

×

409

×

411

        var finalRoutes []request.Route

×

412

        err = ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error {

×

413

                if nicType != util.InternalType {

×

414

                        if err = netlink.LinkSetName(containerLink, ifName); err != nil {

×

415

                                klog.Error(err)

×

416

                                return err

×

417

×

420

                if nicType == util.InternalType {

×

421

                        if err = addAdditionalNic(ifName); err != nil {

×

422

                                klog.Error(err)

×

423

                                return err

×

424

×

425

                        if err = configureAdditionalNic(ifName, ipAddr); err != nil {

×

426

                                klog.Error(err)

×

427

                                return err

×

428

×

429

                        if err = configureNic(nicName, ipAddr, macAddr, mtu, detectIPConflict, false, false); err != nil {

×

430

                                klog.Error(err)

×

431

                                return err

×

432

×

433

                } else {

×

434

                        if err = configureNic(ifName, ipAddr, macAddr, mtu, detectIPConflict, true, false); err != nil {

×

435

                                klog.Error(err)

×

436

                                return err

×

437

×

440

                if isDefaultRoute {

×

441

                        // Only eth0 requires the default route and gateway

×

442

                        containerGw := gateway

×

443

                        if u2oInterconnectionIP != "" {

×

444

                                containerGw = u2oInterconnectionIP

×

445

×

447

                        for _, gw := range strings.Split(containerGw, ",") {

×

448

                                if err = netlink.RouteReplace(&netlink.Route{

×

449

                                        LinkIndex: containerLink.Attrs().Index,

×

450

                                        Scope:     netlink.SCOPE_UNIVERSE,

×

451

                                        Gw:        net.ParseIP(gw),

×

452

                                }); err != nil {

×

453

                                        return fmt.Errorf("failed to configure default gateway %s: %w", gw, err)

×

454

×

458

                for _, r := range routes {

×

459

                        var dst *net.IPNet

×

460

                        if r.Destination != "" {

×

461

                                if _, dst, err = net.ParseCIDR(r.Destination); err != nil {

×

462

                                        klog.Errorf("invalid route destination %s: %v", r.Destination, err)

×

463

                                        continue

×

467

                        var gw net.IP

×

468

                        if r.Gateway != "" {

×

469

                                if gw = net.ParseIP(r.Gateway); gw == nil {

×

470

                                        klog.Errorf("invalid route gateway %s", r.Gateway)

×

471

                                        continue

×

475

                        route := &netlink.Route{

×

476

                                Dst:       dst,

×

477

                                Gw:        gw,

×

478

                                LinkIndex: containerLink.Attrs().Index,

×

479

×

480

                        if err = netlink.RouteReplace(route); err != nil {

×

481

                                klog.Errorf("failed to add route %+v: %v", r, err)

×

482

×

485

                linkRoutes, err := netlink.RouteList(containerLink, netlink.FAMILY_ALL)

×

486

                if err != nil {

×

487

                        return fmt.Errorf("failed to get routes on interface %s: %w", ifName, err)

×

488

×

490

                for _, r := range linkRoutes {

×

491

                        if r.Family != netlink.FAMILY_V4 && r.Family != netlink.FAMILY_V6 {

×

492

                                continue

×

494

                        if r.Dst == nil && r.Gw == nil {

×

495

                                continue

×

497

                        if r.Dst != nil && r.Dst.IP.IsLinkLocalUnicast() {

×

498

                                if _, bits := r.Dst.Mask.Size(); bits == net.IPv6len*8 {

×

499

                                        // skip fe80::/10

×

500

                                        continue

×

504

                        var route request.Route

×

505

                        if r.Dst != nil {

×

506

                                route.Destination = r.Dst.String()

×

507

×

508

                        if r.Gw != nil {

×

509

                                route.Gateway = r.Gw.String()

×

510

×

511

                        finalRoutes = append(finalRoutes, route)

×

514

                if gwCheckMode != gatewayCheckModeDisabled {

×

515

                        var (

×

516

                                underlayGateway = gwCheckMode == gatewayCheckModeArping || gwCheckMode == gatewayCheckModeArpingNotConcerned

×

517

                                interfaceName   = nicName

×

518

×

519

×

520

                        if nicType != util.InternalType {

×

521

                                interfaceName = ifName

×

522

×

524

                        if u2oInterconnectionIP != "" {

×

525

                                if err := csh.checkGatewayReady(podName, podNamespace, gwCheckMode, interfaceName, ipAddr, u2oInterconnectionIP, false, true); err != nil {

×

526

                                        klog.Error(err)

×

527

                                        return err

×

528

×

530

                        return csh.checkGatewayReady(podName, podNamespace, gwCheckMode, interfaceName, ipAddr, gateway, underlayGateway, true)

×

533

                return nil

×

536

        return finalRoutes, err

×

539

func (csh cniServerHandler) checkGatewayReady(podName, podNamespace string, gwCheckMode int, intr, ipAddr, gateway string, underlayGateway, verbose bool) error {

×

540

        if gwCheckMode == gatewayCheckModeArpingNotConcerned || gwCheckMode == gatewayCheckModePingNotConcerned {

×

541

                // ignore error if disableGatewayCheck=true

×

542

                _ = waitNetworkReady(intr, ipAddr, gateway, underlayGateway, verbose, 1, nil)

×

543

                return nil

×

544

×

546

        done := make(chan struct{}, 1)

×

547

        go func() {

×

548

                interval := 5 * time.Second

×

549

                timer := time.NewTimer(interval)

×

550

                for {

×

551

                        select {

×

552

                        case <-done:

×

553

                                return

×

554

                        case <-timer.C:

×

557

                        pod, err := csh.KubeClient.CoreV1().Pods(podNamespace).Get(context.Background(), podName, metav1.GetOptions{})

×

558

                        if err != nil {

×

559

                                if !k8serrors.IsNotFound(err) {

×

560

                                        klog.Errorf("failed to get pod %s/%s: %v", podNamespace, podName, err)

×

561

                                        continue

×

563

                                pod = nil

×

565

                        if pod == nil || !pod.DeletionTimestamp.IsZero() {

×

566

                                // TODO: check pod UID

×

567

                                select {

×

568

                                case <-done:

×

569

                                case done <- struct{}{}:

×

571

                                return

×

573

                        timer.Reset(interval)

×

577

        return waitNetworkReady(intr, ipAddr, gateway, underlayGateway, verbose, gatewayCheckMaxRetry, done)

×

580

func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool, maxRetry int, done chan struct{}) error {

×

581

        ips := strings.Split(ipAddr, ",")

×

582

        for i, gw := range strings.Split(gateway, ",") {

×

583

                src := strings.Split(ips[i], "/")[0]

×

584

                if underlayGateway && util.CheckProtocol(gw) == kubeovnv1.ProtocolIPv4 {

×

585

                        mac, count, err := util.ArpResolve(nic, gw, time.Second, maxRetry, done)

×

586

                        cniConnectivityResult.WithLabelValues(nodeName).Add(float64(count))

×

587

                        if err != nil {

×

588

                                err = fmt.Errorf("network %s with gateway %s is not ready for interface %s after %d checks: %w", ips[i], gw, nic, count, err)

×

589

                                klog.Warning(err)

×

590

                                return err

×

591

×

592

                        if verbose {

×

593

                                klog.Infof("MAC addresses of gateway %s is %s", gw, mac.String())

×

594

                                klog.Infof("network %s with gateway %s is ready for interface %s after %d checks", ips[i], gw, nic, count)

×

595

×

596

                } else {

×

597

                        _, err := pingGateway(gw, src, verbose, maxRetry, done)

×

598

                        if err != nil {

×

599

                                klog.Error(err)

×

600

                                return err

×

601

×

604

        return nil

×

607

func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error {

×

608

        ipStr := util.GetIPWithoutMask(ip)

×

609

        raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",

×

610

                "set", "interface", util.NodeNic, "type=internal", "--",

×

611

                "set", "interface", util.NodeNic, fmt.Sprintf("external_ids:iface-id=%s", portName),

×

612

                fmt.Sprintf("external_ids:ip=%s", ipStr))

×

613

        if err != nil {

×

614

                klog.Errorf("failed to configure node nic %s: %v, %q", portName, err, raw)

×

615

                return errors.New(raw)

×

616

×

618

        if err = configureNic(util.NodeNic, ip, macAddr, mtu, false, false, true); err != nil {

×

619

                klog.Error(err)

×

620

                return err

×

621

×

623

        hostLink, err := netlink.LinkByName(util.NodeNic)

×

624

        if err != nil {

×

625

                return fmt.Errorf("can not find nic %s: %w", util.NodeNic, err)

×

626

×

628

        if err = netlink.LinkSetTxQLen(hostLink, 1000); err != nil {

×

629

                return fmt.Errorf("can not set host nic %s qlen: %w", util.NodeNic, err)

×

630

×

633

        nodeNicRoutes, err := getNicExistRoutes(hostLink, gw)

×

634

        if err != nil {

×

635

                klog.Error(err)

×

636

                return err

×

637

×

639

        var toAdd []netlink.Route

×

640

        for _, c := range strings.Split(joinCIDR, ",") {

×

641

                found := false

×

642

                for _, r := range nodeNicRoutes {

×

643

                        if r.Dst.String() == c {

×

644

                                found = true

×

645

                                break

×

648

                if !found {

×

649

                        protocol := util.CheckProtocol(c)

×

650

                        var src net.IP

×

651

                        var priority int

×

652

                        if protocol == kubeovnv1.ProtocolIPv4 {

×

653

                                for _, ip := range strings.Split(ipStr, ",") {

×

654

                                        if util.CheckProtocol(ip) == protocol {

×

655

                                                src = net.ParseIP(ip)

×

656

                                                break

×

659

                        } else {

×

660

                                priority = 256

×

661

×

662

                        _, cidr, _ := net.ParseCIDR(c)

×

663

                        toAdd = append(toAdd, netlink.Route{

×

664

                                Dst:      cidr,

×

665

                                Src:      src,

×

666

                                Protocol: netlink.RouteProtocol(unix.RTPROT_KERNEL),

×

667

                                Scope:    netlink.SCOPE_LINK,

×

668

                                Priority: priority,

×

669

})

×

672

        if len(toAdd) > 0 {

×

673

                klog.Infof("routes to be added on nic %s: %v", util.NodeNic, toAdd)

×

674

×

676

        for _, r := range toAdd {

×

677

                r.LinkIndex = hostLink.Attrs().Index

×

678

                klog.Infof("adding route %q on %s", r.String(), hostLink.Attrs().Name)

×

679

                if err = netlink.RouteReplace(&r); err != nil && !errors.Is(err, syscall.EEXIST) {

×

680

                        klog.Errorf("failed to replace route %v: %v", r, err)

×

681

×

685

        klog.Infof("wait ovn0 gw ready")

×

686

        if err := waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry, nil); err != nil {

×

687

                klog.Errorf("failed to init ovn0 check: %v", err)

×

688

                return err

×

689

×

690

        return nil

×

695

func (c *Controller) loopOvn0Check() {

×

696

        link, err := netlink.LinkByName(util.NodeNic)

×

697

        if err != nil {

×

698

                util.LogFatalAndExit(err, "failed to get ovn0 nic")

×

699

×

701

        if link.Attrs().OperState == netlink.OperDown {

×

702

                util.LogFatalAndExit(err, "ovn0 nic is down")

×

703

×

705

        node, err := c.nodesLister.Get(c.config.NodeName)

×

706

        if err != nil {

×

707

                klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)

×

708

                return

×

709

×

710

        ip := node.Annotations[util.IPAddressAnnotation]

×

711

        gw := node.Annotations[util.GatewayAnnotation]

×

712

        if err := waitNetworkReady(util.NodeNic, ip, gw, false, false, 5, nil); err != nil {

×

713

                util.LogFatalAndExit(err, "failed to ping ovn0 gateway %s", gw)

×

714

×

719

func (c *Controller) loopTunnelCheck() {

×

720

        tunnelType := c.config.NetworkType

×

721

        var tunnelNic string

×

722

        switch tunnelType {

×

723

        case "vxlan":

×

724

                tunnelNic = util.VxlanNic

×

725

        case "geneve":

×

726

                tunnelNic = util.GeneveNic

×

727

        case "stt":

×

728

                // TODO: tunnelNic = "stt tunnel nic name"

×

729

                return

×

730

        default:

×

731

                return

×

734

        link, err := netlink.LinkByName(tunnelNic)

×

735

        if err != nil || link == nil {

×

736

                return

×

737

×

739

        if link.Attrs().OperState == netlink.OperDown {

×

740

                klog.Errorf("nic: %s is down, attempting to bring it up", tunnelNic)

×

741

                if err := netlink.LinkSetUp(link); err != nil {

×

742

                        klog.Errorf("fail to bring up nic: %s, %v", tunnelNic, err)

×

743

×

747

func (c *Controller) checkNodeGwNicInNs(nodeExtIP, ip, gw string, gwNS ns.NetNS) error {

×

748

        exists, err := ovs.PortExists(util.NodeGwNic)

×

749

        if err != nil {

×

750

                klog.Error(err)

×

751

                return err

×

752

×

753

        filters := labels.Set{util.OvnEipTypeLabel: util.OvnEipTypeLRP}

×

754

        ovnEips, err := c.ovnEipsLister.List(labels.SelectorFromSet(filters))

×

755

        if err != nil {

×

756

                klog.Errorf("failed to list ovn eip, %v", err)

×

757

                return err

×

758

×

759

        if len(ovnEips) == 0 {

×

760

                klog.Errorf("failed to get type %s ovn eip, %v", util.OvnEipTypeLRP, err)

×

761

                // node ext gw eip need lrp eip to establish bfd session

×

762

                return nil

×

763

×

764

        if exists {

×

765

                return ns.WithNetNSPath(gwNS.Path(), func(_ ns.NetNS) error {

×

766

                        err = waitNetworkReady(util.NodeGwNic, ip, gw, true, true, 3, nil)

×

767

                        if err == nil {

×

768

                                if output, err := exec.Command("bfdd-control", "status").CombinedOutput(); err != nil {

×

769

                                        err := fmt.Errorf("failed to get bfdd status, %w, %s", err, output)

×

770

                                        klog.Error(err)

×

771

                                        return err

×

772

×

773

                                for _, eip := range ovnEips {

×

774

                                        if eip.Status.Ready {

×

775

                                                // #nosec G204

×

776

                                                cmd := exec.Command("bfdd-control", "status", "remote", eip.Spec.V4Ip, "local", nodeExtIP)

×

777

                                                var outb bytes.Buffer

×

778

                                                cmd.Stdout = &outb

×

779

                                                if err := cmd.Run(); err == nil {

×

780

                                                        out := outb.String()

×

781

                                                        klog.V(3).Info(out)

×

782

                                                        if strings.Contains(out, "No session") {

×

783

                                                                // not exist

×

784

                                                                cmd = exec.Command("bfdd-control", "allow", eip.Spec.V4Ip) // #nosec G204

×

785

                                                                if err := cmd.Run(); err != nil {

×

786

                                                                        err := fmt.Errorf("failed to add lrp %s ip %s into bfd listening list, %w", eip.Name, eip.Status.V4Ip, err)

×

787

                                                                        klog.Error(err)

×

788

                                                                        return err

×

789

×

791

                                                } else {

×

792

                                                        err := fmt.Errorf("faild to check bfd status remote %s local %s", eip.Spec.V4Ip, nodeExtIP)

×

793

                                                        klog.Error(err)

×

794

                                                        return err

×

795

×

799

                        return err

×

803

        err = errors.New("node external gw not ready")

×

804

        klog.Error(err)

×

805

        return err

×

808

func configureNodeGwNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int, gwNS ns.NetNS) error {

×

809

        ipStr := util.GetIPWithoutMask(ip)

×

810

        output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeGwNic, "--",

×

811

                "set", "interface", util.NodeGwNic, "type=internal", "--",

×

812

                "set", "interface", util.NodeGwNic, fmt.Sprintf("external_ids:iface-id=%s", portName),

×

813

                fmt.Sprintf("external_ids:ip=%s", ipStr),

×

814

                fmt.Sprintf("external_ids:pod_netns=%s", util.NodeGwNsPath))

×

815

        if err != nil {

×

816

                klog.Errorf("failed to configure node external nic %s: %v, %q", portName, err, output)

×

817

                return errors.New(output)

×

818

×

819

        gwLink, err := netlink.LinkByName(util.NodeGwNic)

×

820

        if err == nil {

×

821

                fd := int(gwNS.Fd()) // #nosec G115

×

822

                if err = netlink.LinkSetNsFd(gwLink, fd); err != nil {

×

823

                        klog.Errorf("failed to move link into netns: %v", err)

×

824

                        return err

×

825

×

826

        } else {

×

827

                klog.V(3).Infof("node external nic %q already in ns %s", util.NodeGwNic, util.NodeGwNsPath)

×

828

×

829

        return ns.WithNetNSPath(gwNS.Path(), func(_ ns.NetNS) error {

×

830

                if err = configureNic(util.NodeGwNic, ip, macAddr, mtu, true, false, false); err != nil {

×

831

                        klog.Errorf("failed to configure node gw nic %s, %v", util.NodeGwNic, err)

×

832

                        return err

×

833

×

835

                if err = configureLoNic(); err != nil {

×

836

                        klog.Errorf("failed to configure nic %s, %v", util.LoNic, err)

×

837

                        return err

×

838

×

839

                gwLink, err = netlink.LinkByName(util.NodeGwNic)

×

840

                if err != nil {

×

841

                        klog.Errorf("failed to get link %q, %v", util.NodeGwNic, err)

×

842

                        return err

×

843

×

844

                switch util.CheckProtocol(ip) {

×

845

                case kubeovnv1.ProtocolIPv4:

×

846

                        _, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")

×

847

                        err = netlink.RouteReplace(&netlink.Route{

×

848

                                LinkIndex: gwLink.Attrs().Index,

×

849

                                Scope:     netlink.SCOPE_UNIVERSE,

×

850

                                Dst:       defaultNet,

×

851

                                Gw:        net.ParseIP(gw),

×

852

})

×

853

                case kubeovnv1.ProtocolIPv6:

×

854

                        _, defaultNet, _ := net.ParseCIDR("::/0")

×

855

                        err = netlink.RouteReplace(&netlink.Route{

×

856

                                LinkIndex: gwLink.Attrs().Index,

×

857

                                Scope:     netlink.SCOPE_UNIVERSE,

×

858

                                Dst:       defaultNet,

×

859

                                Gw:        net.ParseIP(gw),

×

860

})

×

861

                case kubeovnv1.ProtocolDual:

×

862

                        gws := strings.Split(gw, ",")

×

863

                        _, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")

×

864

                        err = netlink.RouteReplace(&netlink.Route{

×

865

                                LinkIndex: gwLink.Attrs().Index,

×

866

                                Scope:     netlink.SCOPE_UNIVERSE,

×

867

                                Dst:       defaultNet,

×

868

                                Gw:        net.ParseIP(gws[0]),

×

869

})

×

870

                        if err != nil {

×

871

                                return fmt.Errorf("config v4 gateway failed: %w", err)

×

872

×

874

                        _, defaultNet, _ = net.ParseCIDR("::/0")

×

875

                        err = netlink.RouteReplace(&netlink.Route{

×

876

                                LinkIndex: gwLink.Attrs().Index,

×

877

                                Scope:     netlink.SCOPE_UNIVERSE,

×

878

                                Dst:       defaultNet,

×

879

                                Gw:        net.ParseIP(gws[1]),

×

880

})

×

882

                if err != nil {

×

883

                        return fmt.Errorf("failed to configure gateway: %w", err)

×

884

×

885

                cmd := exec.Command("bfdd-beacon", "--listen=0.0.0.0")

×

886

                if err := cmd.Run(); err != nil {

×

887

                        err := fmt.Errorf("failed to get start bfd listen, %w", err)

×

888

                        klog.Error(err)

×

889

                        return err

×

890

×

891

                return waitNetworkReady(util.NodeGwNic, ip, gw, true, true, 3, nil)

×

895

func removeNodeGwNic() error {

×

896

        if _, err := ovs.Exec(ovs.IfExists, "del-port", "br-int", util.NodeGwNic); err != nil {

×

897

                return fmt.Errorf("failed to remove ecmp external port %s from OVS bridge %s: %w", "br-int", util.NodeGwNic, err)

×

898

×

899

        klog.Infof("removed node external gw nic %q", util.NodeGwNic)

×

900

        return nil

×

903

func removeNodeGwNs() error {

×

904

        ns := netns.LoadNetNS(util.NodeGwNsPath)

×

905

        ok, err := ns.Closed()

×

906

        if err != nil {

×

907

                return fmt.Errorf("failed to remove node external gw ns %s: %w", util.NodeGwNs, err)

×

908

×

909

        if !ok {

×

910

                if err = ns.Remove(); err != nil {

×

911

                        return fmt.Errorf("failed to remove node external gw ns %s: %w", util.NodeGwNs, err)

×

912

×

914

        klog.Infof("node external gw ns %s removed", util.NodeGwNs)

×

915

        return nil

×

918

func (c *Controller) loopOvnExt0Check() {

×

919

        node, err := c.nodesLister.Get(c.config.NodeName)

×

920

        if err != nil {

×

921

                klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)

×

922

                return

×

923

×

925

        portName := node.Name

×

926

        needClean := false

×

927

        cachedEip, err := c.ovnEipsLister.Get(portName)

×

928

        if err != nil {

×

929

                if k8serrors.IsNotFound(err) {

×

930

                        val, ok := node.Labels[util.NodeExtGwLabel]

×

931

                        if !ok {

×

932

                                // not gw node before

×

933

                                return

×

934

×

935

                        if val == "false" {

×

936

                                // already clean

×

937

                                return

×

938

×

939

                        if val == "true" {

×

940

                                needClean = true

×

941

×

942

                } else {

×

943

                        klog.Errorf("failed to get ecmp gateway ovn eip, %v", err)

×

944

                        return

×

945

×

948

        if needClean {

×

949

                if err := removeNodeGwNic(); err != nil {

×

950

                        klog.Error(err)

×

951

                        return

×

952

×

953

                if err := removeNodeGwNs(); err != nil {

×

954

                        klog.Error(err)

×

955

                        return

×

956

×

957

                if err = c.patchNodeExternalGwLabel(false); err != nil {

×

958

                        klog.Errorf("failed to patch labels of node %s: %v", node.Name, err)

×

959

                        return

×

960

×

961

                return

×

964

        if cachedEip.Status.V4Ip == "" {

×

965

                klog.Errorf("ecmp gateway ovn eip still has no ip")

×

966

                return

×

967

×

968

        ips := util.GetStringIP(cachedEip.Status.V4Ip, cachedEip.Status.V6Ip)

×

969

        cachedSubnet, err := c.subnetsLister.Get(cachedEip.Spec.ExternalSubnet)

×

970

        if err != nil {

×

971

                klog.Errorf("failed to get external subnet %s, %v", cachedEip.Spec.ExternalSubnet, err)

×

972

                return

×

973

×

974

        gw := cachedSubnet.Spec.Gateway

×

975

        mac, err := net.ParseMAC(cachedEip.Status.MacAddress)

×

976

        if err != nil {

×

977

                klog.Errorf("failed to parse mac %s, %v", cachedEip.Status.MacAddress, err)

×

978

                return

×

979

×

980

        gwNS, err := ns.GetNS(util.NodeGwNsPath)

×

981

        if err != nil {

×

982

                if _, ok := err.(ns.NSPathNotExistErr); !ok {

×

983

                        klog.Errorf("failed to get netns from path %s: %v", util.NodeGwNsPath, err)

×

984

                        return

×

985

×

986

                if err = newNetNS(util.NodeGwNsPath); err != nil {

×

987

                        klog.Error(fmt.Errorf("failed to create gw ns %s: %w", util.NodeGwNs, err))

×

988

                        return

×

989

×

990

                if gwNS, err = ns.GetNS(util.NodeGwNsPath); err != nil {

×

991

                        klog.Errorf("failed to get netns from path %s: %v", util.NodeGwNsPath, err)

×

992

                        return

×

993

×

995

        nodeExtIP := cachedEip.Spec.V4Ip

×

996

        ipAddr, err := util.GetIPAddrWithMask(ips, cachedSubnet.Spec.CIDRBlock)

×

997

        if err != nil {

×

998

                klog.Errorf("failed to get ip addr with mask %s, %v", ips, err)

×

999

                return

×

1000

×

1001

        if err := c.checkNodeGwNicInNs(nodeExtIP, ipAddr, gw, gwNS); err == nil {

×

1002

                // add all lrp ip in bfd listening list

×

1003

                return

×

1004

×

1005

        klog.Infof("setup nic ovnext0 ip %s, mac %v, mtu %d", ipAddr, mac, c.config.MTU)

×

1006

        if err := configureNodeGwNic(portName, ipAddr, gw, mac, c.config.MTU, gwNS); err != nil {

×

1007

                klog.Errorf("failed to setup ovnext0, %v", err)

×

1008

                return

×

1009

×

1010

        if err = c.patchNodeExternalGwLabel(true); err != nil {

×

1011

                klog.Errorf("failed to patch labels of node %s: %v", node.Name, err)

×

1012

                return

×

1013

×

1014

        if err = c.patchOvnEipStatus(portName, true); err != nil {

×

1015

                klog.Errorf("failed to patch status for eip %s, %v", portName, err)

×

1016

                return

×

1017

×

1020

func (c *Controller) patchOvnEipStatus(key string, ready bool) error {

×

1021

        cachedOvnEip, err := c.ovnEipsLister.Get(key)

×

1022

        if err != nil {

×

1023

                klog.Errorf("failed to get cached ovn eip '%s', %v", key, err)

×

1024

                return err

×

1025

×

1026

        ovnEip := cachedOvnEip.DeepCopy()

×

1027

        changed := false

×

1028

        if ovnEip.Status.Ready != ready {

×

1029

                ovnEip.Status.Ready = ready

×

1030

                changed = true

×

1031

×

1032

        if changed {

×

1033

                bytes, err := ovnEip.Status.Bytes()

×

1034

                if err != nil {

×

1035

                        klog.Errorf("failed to marshal ovn eip status '%s', %v", key, err)

×

1036

                        return err

×

1037

×

1038

                if _, err = c.config.KubeOvnClient.KubeovnV1().OvnEips().Patch(context.Background(), ovnEip.Name,

×

1039

                        types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil {

×

1040

                        klog.Errorf("failed to patch status for ovn eip '%s', %v", key, err)

×

1041

                        return err

×

1042

×

1044

        return nil

×

1047

func (c *Controller) patchNodeExternalGwLabel(enabled bool) error {

×

1048

        node, err := c.nodesLister.Get(c.config.NodeName)

×

1049

        if err != nil {

×

1050

                klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)

×

1051

                return err

×

1052

×

1054

        labels := map[string]any{util.NodeExtGwLabel: strconv.FormatBool(enabled)}

×

NEW

1055

        if err = util.PatchLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err != nil {

×

1056

                klog.Errorf("failed to update labels of node %s: %v", node.Name, err)

×

1057

                return err

×

1058

×

1060

        return nil

×

1063

func configureMirrorLink(portName string, _ int) error {

×

1064

        mirrorLink, err := netlink.LinkByName(portName)

×

1065

        if err != nil {

×

1066

                klog.Error(err)

×

1067

                return fmt.Errorf("can not find mirror nic %s: %w", portName, err)

×

1068

×

1070

        if mirrorLink.Attrs().OperState != netlink.OperUp {

×

1071

                if err = netlink.LinkSetUp(mirrorLink); err != nil {

×

1072

                        klog.Error(err)

×

1073

                        return fmt.Errorf("can not set mirror nic %s up: %w", portName, err)

×

1074

×

1077

        return nil

×

1081

func macToLinkLocalIPv6(mac net.HardwareAddr) (net.IP, error) {

×

1082

        if len(mac) != 6 {

×

1083

                return nil, errors.New("invalid MAC address length")

×

1084

×

1087

        eui64 := make([]byte, 8)

×

1088

        copy(eui64[0:3], mac[0:3]) // Copy the first 3 bytes

×

1089

        eui64[3] = 0xff            // Insert ff

×

1090

        eui64[4] = 0xfe            // Insert fe

×

1091

        copy(eui64[5:], mac[3:])   // Copy the last 3 bytes

×

1092

×

1093

        // Flip the 7th bit of the first byte

×

1094

        eui64[0] ^= 0x02

×

1095

×

1096

        // Prepend the link-local prefix

×

1097

        linkLocalIPv6 := net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

×

1098

        copy(linkLocalIPv6[8:], eui64)

×

1099

×

1100

        return linkLocalIPv6, nil

×

1103

func configureNic(link, ip string, macAddr net.HardwareAddr, mtu int, detectIPConflict, setUfoOff, ipv6LinkLocalOn bool) error {

×

1104

        nodeLink, err := netlink.LinkByName(link)

×

1105

        if err != nil {

×

1106

                klog.Error(err)

×

1107

                return fmt.Errorf("can not find nic %s: %w", link, err)

×

1108

×

1110

        if err = netlink.LinkSetHardwareAddr(nodeLink, macAddr); err != nil {

×

1111

                klog.Error(err)

×

1112

                return fmt.Errorf("can not set mac address to nic %s: %w", link, err)

×

1113

×

1115

        if mtu > 0 {

×

1116

                if nodeLink.Type() == "openvswitch" {

×

1117

                        _, err = ovs.Exec("set", "interface", link, fmt.Sprintf(`mtu_request=%d`, mtu))

×

1118

                } else {

×

1119

                        err = netlink.LinkSetMTU(nodeLink, mtu)

×

1120

×

1121

                if err != nil {

×

1122

                        return fmt.Errorf("failed to set nic %s mtu: %w", link, err)

×

1123

×

1126

        if nodeLink.Attrs().OperState != netlink.OperUp {

×

1127

                if err = netlink.LinkSetUp(nodeLink); err != nil {

×

1128

                        klog.Error(err)

×

1129

                        return fmt.Errorf("can not set node nic %s up: %w", link, err)

×

1130

×

1133

        ipDelMap := make(map[string]netlink.Addr)

×

1134

        ipAddMap := make(map[string]netlink.Addr)

×

1135

        ipAddrs, err := netlink.AddrList(nodeLink, unix.AF_UNSPEC)

×

1136

        if err != nil {

×

1137

                klog.Error(err)

×

1138

                return fmt.Errorf("can not get addr %s: %w", nodeLink, err)

×

1139

×

1141

        isIPv6LinkLocalExist := false

×

1142

        for _, ipAddr := range ipAddrs {

×

1143

                if ipAddr.IP.IsLinkLocalUnicast() {

×

1144

                        // skip 169.254.0.0/16 and fe80::/10

×

1145

                        if util.CheckProtocol(ipAddr.IP.String()) == kubeovnv1.ProtocolIPv6 {

×

1146

                                isIPv6LinkLocalExist = true

×

1147

×

1148

                        continue

×

1150

                ipDelMap[ipAddr.IPNet.String()] = ipAddr

×

1153

        if ipv6LinkLocalOn && !isIPv6LinkLocalExist && (util.CheckProtocol(ip) == kubeovnv1.ProtocolIPv6 || util.CheckProtocol(ip) == kubeovnv1.ProtocolDual) {

×

1154

                linkLocal, err := macToLinkLocalIPv6(macAddr)

×

1155

                if err != nil {

×

1156

                        return fmt.Errorf("failed to generate link-local address: %w", err)

×

1157

×

1158

                ipAddMap[linkLocal.String()] = netlink.Addr{

×

1159

                        IPNet: &net.IPNet{

×

1160

                                IP:   linkLocal,

×

1161

                                Mask: net.CIDRMask(64, 128),

×

1162

},

×

1163

×

1166

        for _, ipStr := range strings.Split(ip, ",") {

×

1167

                // Do not reassign same address for link

×

1168

                if _, ok := ipDelMap[ipStr]; ok {

×

1169

                        delete(ipDelMap, ipStr)

×

1170

                        continue

×

1173

                ipAddr, err := netlink.ParseAddr(ipStr)

×

1174

                if err != nil {

×

1175

                        return fmt.Errorf("can not parse address %s: %w", ipStr, err)

×

1176

×

1177

                ipAddMap[ipStr] = *ipAddr

×

1180

        for ip, addr := range ipDelMap {

×

1181

                klog.Infof("delete ip address %s on %s", ip, link)

×

1182

                if err = netlink.AddrDel(nodeLink, &addr); err != nil {

×

1183

                        klog.Error(err)

×

1184

                        return fmt.Errorf("delete address %s: %w", addr, err)

×

1185

×

1187

        for ip, addr := range ipAddMap {

×

1188

                if detectIPConflict && addr.IP.To4() != nil {

×

1189

                        ip := addr.IP.String()

×

1190

                        mac, err := util.ArpDetectIPConflict(link, ip, macAddr)

×

1191

                        if err != nil {

×

1192

                                err = fmt.Errorf("failed to detect address conflict for %s on link %s: %w", ip, link, err)

×

1193

                                klog.Error(err)

×

1194

                                return err

×

1195

×

1196

                        if mac != nil {

×

1197

                                return fmt.Errorf("IP address %s has already been used by host with MAC %s", ip, mac)

×

1198

×

1200

                if addr.IP.To4() != nil && !detectIPConflict {

×

1201

                        // when detectIPConflict is true, free arp is already broadcast in the step of announcement

×

1202

                        if err := util.AnnounceArpAddress(link, addr.IP.String(), macAddr, 1, 1*time.Second); err != nil {

×

1203

                                klog.Warningf("failed to broadcast free arp with err %v", err)

×

1204

×

1207

                klog.Infof("add ip address %s to %s", ip, link)

×

1208

                if err = netlink.AddrAdd(nodeLink, &addr); err != nil {

×

1209

                        klog.Error(err)

×

1210

                        return fmt.Errorf("can not add address %s to nic %s: %w", addr, link, err)

×

1211

×

1214

        if setUfoOff {

×

1215

                cmd := fmt.Sprintf("if ethtool -k %s | grep -q ^udp-fragmentation-offload; then ethtool -K %s ufo off; fi", link, link)

×

1216

                if output, err := exec.Command("sh", "-xc", cmd).CombinedOutput(); err != nil {

×

1217

                        klog.Error(err)

×

1218

                        return fmt.Errorf("failed to disable udp-fragmentation-offload feature of device %s to off: %w, %s", link, err, output)

×

1219

×

1222

        return nil

×

1225

func configureLoNic() error {

×

1226

        loLink, err := netlink.LinkByName(util.LoNic)

×

1227

        if err != nil {

×

1228

                err := fmt.Errorf("can not find nic %s, %w", util.LoNic, err)

×

1229

                klog.Error(err)

×

1230

                return err

×

1231

×

1233

        if loLink.Attrs().OperState != netlink.OperUp {

×

1234

                if err = netlink.LinkSetUp(loLink); err != nil {

×

1235

                        err := fmt.Errorf("failed to set up nic %s, %w", util.LoNic, err)

×

1236

                        klog.Error(err)

×

1237

                        return err

×

1238

×

1241

        return nil

×

1244

func (c *Controller) transferAddrsAndRoutes(nicName, brName string, delNonExistent bool) (int, error) {

×

1245

        nic, err := netlink.LinkByName(nicName)

×

1246

        if err != nil {

×

1247

                return 0, fmt.Errorf("failed to get nic by name %s: %w", nicName, err)

×

1248

×

1249

        bridge, err := netlink.LinkByName(brName)

×

1250

        if err != nil {

×

1251

                return 0, fmt.Errorf("failed to get bridge by name %s: %w", brName, err)

×

1252

×

1254

        addrs, err := netlink.AddrList(nic, netlink.FAMILY_ALL)

×

1255

        if err != nil {

×

1256

                return 0, fmt.Errorf("failed to get addresses on nic %s: %w", nicName, err)

×

1257

×

1258

        routes, err := netlink.RouteList(nic, netlink.FAMILY_ALL)

×

1259

        if err != nil {

×

1260

                return 0, fmt.Errorf("failed to get routes on nic %s: %w", nicName, err)

×

1261

×

1263

        brAddrs, err := netlink.AddrList(bridge, netlink.FAMILY_ALL)

×

1264

        if err != nil {

×

1265

                return 0, fmt.Errorf("failed to get addresses on OVS bridge %s: %w", brName, err)

×

1266

×

1268

        var delAddrs []netlink.Addr

×

1269

        if delNonExistent {

×

1270

                for _, addr := range brAddrs {

×

1271

                        if addr.IP.IsLinkLocalUnicast() {

×

1272

                                // skip 169.254.0.0/16 and fe80::/10

×

1273

                                continue

×

1276

                        var found bool

×

1277

                        for _, v := range addrs {

×

1278

                                if v.Equal(addr) {

×

1279

                                        found = true

×

1280

                                        break

×

1283

                        if !found {

×

1284

                                delAddrs = append(delAddrs, addr)

×

1285

×

1290

        if err = c.nmSyncer.SetManaged(nicName, false); err != nil {

×

1291

                klog.Errorf("failed to set device %s unmanaged by NetworkManager: %v", nicName, err)

×

1292

                return 0, err

×

1293

×

1294

        if err = c.nmSyncer.AddDevice(nicName, brName); err != nil {

×

1295

                klog.Errorf("failed to monitor NetworkManager event for device %s: %v", nicName, err)

×

1296

                return 0, err

×

1297

×

1299

        var count int

×

1300

        for _, addr := range addrs {

×

1301

                if addr.IP.IsLinkLocalUnicast() {

×

1302

                        // skip 169.254.0.0/16 and fe80::/10

×

1303

                        continue

×

1305

                count++

×

1306

×

1307

                if err = netlink.AddrDel(nic, &addr); err != nil {

×

1308

                        errMsg := fmt.Errorf("failed to delete address %q on nic %s: %w", addr.String(), nicName, err)

×

1309

                        klog.Error(errMsg)

×

1310

                        return 0, errMsg

×

1311

×

1312

                klog.Infof("address %q has been removed from link %s", addr.String(), nicName)

×

1313

×

1314

                addr.Label = ""

×

1315

                addr.PreferedLft, addr.ValidLft = 0, 0

×

1316

                if err = netlink.AddrReplace(bridge, &addr); err != nil {

×

1317

                        return 0, fmt.Errorf("failed to replace address %q on OVS bridge %s: %w", addr.String(), brName, err)

×

1318

×

1319

                klog.Infof("address %q has been added/replaced to link %s", addr.String(), brName)

×

1322

        if count != 0 {

×

1323

                for _, addr := range delAddrs {

×

1324

                        if err = netlink.AddrDel(bridge, &addr); err != nil {

×

1325

                                errMsg := fmt.Errorf("failed to delete address %q on OVS bridge %s: %w", addr.String(), brName, err)

×

1326

                                klog.Error(errMsg)

×

1327

                                return 0, errMsg

×

1328

×

1329

                        klog.Infof("address %q has been removed from OVS bridge %s", addr.String(), brName)

×

1335

        albBond, err := linkIsAlbBond(nic)

×

1336

        if err != nil {

×

1337

                return 0, err

×

1338

×

1339

        if !albBond {

×

1340

                if _, err = ovs.Exec("set", "bridge", brName, fmt.Sprintf(`other-config:hwaddr="%s"`, nic.Attrs().HardwareAddr.String())); err != nil {

×

1341

                        return 0, fmt.Errorf("failed to set MAC address of OVS bridge %s: %w", brName, err)

×

1342

×

1345

        if err = netlink.LinkSetUp(bridge); err != nil {

×

1346

                return 0, fmt.Errorf("failed to set OVS bridge %s up: %w", brName, err)

×

1347

×

1349

        for _, scope := range routeScopeOrders {

×

1350

                for _, route := range routes {

×

1351

                        if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() {

×

1352

                                // skip 169.254.0.0/16 and fe80::/10

×

1353

                                continue

×

1355

                        if route.Scope == scope {

×

1356

                                route.LinkIndex = bridge.Attrs().Index

×

1357

                                if err = netlink.RouteReplace(&route); err != nil {

×

1358

                                        return 0, fmt.Errorf("failed to add/replace route %s to OVS bridge %s: %w", route.String(), brName, err)

×

1359

×

1360

                                klog.Infof("route %q has been added/replaced to OVS bridge %s", route.String(), brName)

×

1365

        brRoutes, err := netlink.RouteList(bridge, netlink.FAMILY_ALL)

×

1366

        if err != nil {

×

1367

                return 0, fmt.Errorf("failed to get routes on OVS bridge %s: %w", brName, err)

×

1368

×

1370

        var delRoutes []netlink.Route

×

1371

        if delNonExistent && count != 0 {

×

1372

                for _, route := range brRoutes {

×

1373

                        if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() {

×

1374

                                // skip 169.254.0.0/16 and fe80::/10

×

1375

                                continue

×

1378

                        var found bool

×

1379

                        for _, v := range routes {

×

1380

                                v.LinkIndex = route.LinkIndex

×

1381

                                v.ILinkIndex = route.ILinkIndex

×

1382

                                if v.Equal(route) {

×

1383

                                        found = true

×

1384

                                        break

×

1387

                        if !found {

×

1388

                                delRoutes = append(delRoutes, route)

×

1389

×

1393

        for i := len(routeScopeOrders) - 1; i >= 0; i-- {

×

1394

                for _, route := range delRoutes {

×

1395

                        if route.Scope == routeScopeOrders[i] {

×

1396

                                if err = netlink.RouteDel(&route); err != nil {

×

1397

                                        return 0, fmt.Errorf("failed to delete route %s from OVS bridge %s: %w", route.String(), brName, err)

×

1398

×

1399

                                klog.Infof("route %q has been deleted from OVS bridge %s", route.String(), brName)

×

1404

        if err = netlink.LinkSetUp(nic); err != nil {

×

1405

                return 0, fmt.Errorf("failed to set link %s up: %w", nicName, err)

×

1406

×

1408

        return nic.Attrs().MTU, nil

×

1413

func (c *Controller) configProviderNic(nicName, brName string, trunks []string) (int, error) {

×

1414

        isUserspaceDP, err := ovs.IsUserspaceDataPath()

×

1415

        if err != nil {

×

1416

                klog.Error(err)

×

1417

                return 0, err

×

1418

×

1420

        var mtu int

×

1421

        if !isUserspaceDP {

×

1422

                mtu, err = c.transferAddrsAndRoutes(nicName, brName, false)

×

1423

                if err != nil {

×

1424

                        klog.Errorf("failed to transfer addresses and routes from %s to %s: %v", nicName, brName, err)

×

1425

                        return 0, err

×

1426

×

1428

                if _, err = ovs.Exec(ovs.MayExist, "add-port", brName, nicName,

×

1429

                        "--", "set", "port", nicName, "trunks="+strings.Join(trunks, ","), "external_ids:vendor="+util.CniTypeName); err != nil {

×

1430

                        klog.Errorf("failed to add %s to OVS bridge %s: %v", nicName, brName, err)

×

1431

                        return 0, err

×

1432

×

1433

                klog.V(3).Infof("ovs port %s has been added to bridge %s", nicName, brName)

×

1434

        } else {

×

1435

                mtu = c.config.MTU

×

1436

×

1438

        return mtu, nil

×

1441

func linkIsAlbBond(link netlink.Link) (bool, error) {

×

1442

        check := func(link netlink.Link) bool {

×

1443

                bond, ok := link.(*netlink.Bond)

×

1444

                return ok && bond.Mode == netlink.BOND_MODE_BALANCE_ALB

×

1445

×

1447

        if check(link) {

×

1448

                return true, nil

×

1449

×

1451

        vlan, ok := link.(*netlink.Vlan)

×

1452

        if !ok {

×

1453

                return false, nil

×

1454

×

1455

        parent, err := netlink.LinkByIndex(vlan.ParentIndex)

×

1456

        if err != nil {

×

1457

                klog.Errorf("failed to get link by index %d: %v", vlan.ParentIndex, err)

×

1458

                return false, err

×

1459

×

1461

        return check(parent), nil

×

1466

func (c *Controller) removeProviderNic(nicName, brName string) error {

×

1467

        c.nmSyncer.RemoveDevice(nicName)

×

1468

×

1469

        nic, err := netlink.LinkByName(nicName)

×

1470

        if err != nil {

×

1471

                if _, ok := err.(netlink.LinkNotFoundError); ok {

×

1472

                        klog.Warningf("failed to get nic by name %s: %v", nicName, err)

×

1473

                        return nil

×

1474

×

1475

                return fmt.Errorf("failed to get nic by name %s: %w", nicName, err)

×

1477

        bridge, err := netlink.LinkByName(brName)

×

1478

        if err != nil {

×

1479

                return fmt.Errorf("failed to get bridge by name %s: %w", brName, err)

×

1480

×

1482

        addrs, err := netlink.AddrList(bridge, netlink.FAMILY_ALL)

×

1483

        if err != nil {

×

1484

                return fmt.Errorf("failed to get addresses on bridge %s: %w", brName, err)

×

1485

×

1486

        routes, err := netlink.RouteList(bridge, netlink.FAMILY_ALL)

×

1487

        if err != nil {

×

1488

                return fmt.Errorf("failed to get routes on bridge %s: %w", brName, err)

×

1489

×

1491

        if _, err = ovs.Exec(ovs.IfExists, "del-port", brName, nicName); err != nil {

×

1492

                return fmt.Errorf("failed to remove %s from OVS bridge %s: %w", nicName, brName, err)

×

1493

×

1494

        klog.V(3).Infof("ovs port %s has been removed from bridge %s", nicName, brName)

×

1495

×

1496

        for _, addr := range addrs {

×

1497

                if addr.IP.IsLinkLocalUnicast() {

×

1498

                        // skip 169.254.0.0/16 and fe80::/10

×

1499

                        continue

×

1502

                if err = netlink.AddrDel(bridge, &addr); err != nil {

×

1503

                        errMsg := fmt.Errorf("failed to delete address %q on OVS bridge %s: %w", addr.String(), brName, err)

×

1504

                        klog.Error(errMsg)

×

1505

                        return errMsg

×

1506

×

1507

                klog.Infof("address %q has been deleted from link %s", addr.String(), brName)

×

1508

×

1509

                addr.Label = ""

×

1510

                if err = netlink.AddrReplace(nic, &addr); err != nil {

×

1511

                        return fmt.Errorf("failed to replace address %q on nic %s: %w", addr.String(), nicName, err)

×

1512

×

1513

                klog.Infof("address %q has been added/replaced to link %s", addr.String(), nicName)

×

1516

        if err = netlink.LinkSetUp(nic); err != nil {

×

1517

                klog.Errorf("failed to set link %s up: %v", nicName, err)

×

1518

                return err

×

1519

×

1521

        scopeOrders := [...]netlink.Scope{

×

1522

                netlink.SCOPE_HOST,

×

1523

                netlink.SCOPE_LINK,

×

1524

                netlink.SCOPE_SITE,

×

1525

                netlink.SCOPE_UNIVERSE,

×

1526

×

1527

        for _, scope := range scopeOrders {

×

1528

                for _, route := range routes {

×

1529

                        if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() {

×

1530

                                // skip 169.254.0.0/16 and fe80::/10

×

1531

                                continue

×

1533

                        if route.Scope == scope {

×

1534

                                route.LinkIndex = nic.Attrs().Index

×

1535

                                if err = netlink.RouteReplace(&route); err != nil {

×

1536

                                        return fmt.Errorf("failed to add/replace route %s: %w", route.String(), err)

×

1537

×

1538

                                klog.Infof("route %q has been added/replaced to link %s", route.String(), nicName)

×

1543

        if err = netlink.LinkSetDown(bridge); err != nil {

×

1544

                return fmt.Errorf("failed to set OVS bridge %s down: %w", brName, err)

×

1545

×

1546

        klog.V(3).Infof("link %s has been set down", brName)

×

1547

×

1548

        return nil

×

1551

func setupVethPair(containerID, ifName string, mtu int) (string, string, error) {

×

1552

        var err error

×

1553

        hostNicName, containerNicName := generateNicName(containerID, ifName)

×

1554

        // Create a veth pair, put one end to container ,the other to ovs port

×

1555

        // NOTE: DO NOT use ovs internal type interface for container.

×

1556

        // Kubernetes will detect 'eth0' nic in pod, so the nic name in pod must be 'eth0'.

×

1557

        // When renaming internal interface to 'eth0', ovs will delete and recreate this interface.

×

1558

        veth := netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostNicName}, PeerName: containerNicName}

×

1559

        if mtu > 0 {

×

1560

                veth.MTU = mtu

×

1561

×

1562

        if err = netlink.LinkAdd(&veth); err != nil {

×

1563

                if err := netlink.LinkDel(&veth); err != nil {

×

1564

                        klog.Errorf("failed to delete veth %v", err)

×

1565

                        return "", "", err

×

1566

×

1567

                return "", "", fmt.Errorf("failed to create veth for %w", err)

×

1569

        return hostNicName, containerNicName, nil

×

1574

func setupSriovInterface(containerID, deviceID, vfDriver, ifName string, mtu int, mac string) (string, string, string, int, error) {

×

1575

        isVfioPciDriver := false

×

1576

        if vfDriver == "vfio-pci" {

×

1577

                matches, err := filepath.Glob(filepath.Join(util.VfioSysDir, "*"))

×

1578

                if err != nil {

×

1579

                        return "", "", "", -1, fmt.Errorf("failed to check %s 'vfio-pci' driver path, %w", deviceID, err)

×

1580

×

1582

                for _, match := range matches {

×

1583

                        tmp, err := os.Readlink(match)

×

1584

                        if err != nil {

×

1585

                                continue

×

1587

                        if strings.Contains(tmp, deviceID) {

×

1588

                                isVfioPciDriver = true

×

1589

                                break

×

1593

                if !isVfioPciDriver {

×

1594

                        return "", "", "", -1, fmt.Errorf("driver of device %s is not 'vfio-pci'", deviceID)

×

1595

×

1598

        var vfNetdevice string

×

1599

        if !isVfioPciDriver {

×

1600

                // 1. get VF netdevice from PCI

×

1601

                vfNetdevices, err := sriovnet.GetNetDevicesFromPci(deviceID)

×

1602

                if err != nil {

×

1603

                        klog.Errorf("failed to get vf netdevice %s, %v", deviceID, err)

×

1604

                        return "", "", "", -1, err

×

1605

×

1608

                if len(vfNetdevices) != 1 {

×

1609

                        return "", "", "", -1, fmt.Errorf("failed to get one netdevice interface per %s", deviceID)

×

1610

×

1611

                vfNetdevice = vfNetdevices[0]

×

1614

        if yusur.IsYusurSmartNic(deviceID) {

×

1615

                // 2. get PF PCI

×

1616

                pfPci, err := yusur.GetYusurNicPfPciFromVfPci(deviceID)

×

1617

                if err != nil {

×

1618

                        return "", "", "", -1, err

×

1619

×

1622

                pfIndex, err := yusur.GetYusurNicPfIndexByPciAddress(pfPci)

×

1623

                if err != nil {

×

1624

                        klog.Errorf("failed to get up %s link device, %v", deviceID, err)

×

1625

                        return "", "", "", -1, err

×

1626

×

1629

                vfIndex, err := yusur.GetYusurNicVfIndexByPciAddress(deviceID)

×

1630

                if err != nil {

×

1631

                        return "", "", "", -1, err

×

1632

×

1635

                rep := yusur.GetYusurNicVfRepresentor(pfIndex, vfIndex)

×

1636

×

1637

                _, err = netlink.LinkByName(rep)

×

1638

                if err != nil {

×

1639

                        klog.Infof("vfr not exist %s", rep)

×

1640

×

1642

                return rep, vfNetdevice, pfPci, vfIndex, nil

×

1646

        uplink, err := sriovnet.GetUplinkRepresentor(deviceID)

×

1647

        if err != nil {

×

1648

                klog.Errorf("failed to get up %s link device, %v", deviceID, err)

×

1649

                return "", "", "", -1, err

×

1650

×

1653

        vfIndex, err := sriovnet.GetVfIndexByPciAddress(deviceID)

×

1654

        if err != nil {

×

1655

                klog.Errorf("failed to get vf %s index, %v", deviceID, err)

×

1656

                return "", "", "", -1, err

×

1657

×

1660

        rep, err := sriovnet.GetVfRepresentor(uplink, vfIndex)

×

1661

        if err != nil {

×

1662

                klog.Errorf("failed to get vf %d representor, %v", vfIndex, err)

×

1663

                return "", "", "", -1, err

×

1664

×

1665

        oldHostRepName := rep

×

1666

×

1667

        // 5. rename the host VF representor

×

1668

        hostNicName, _ := generateNicName(containerID, ifName)

×

1669

        if err = renameLink(oldHostRepName, hostNicName); err != nil {

×

1670

                return "", "", "", -1, fmt.Errorf("failed to rename %s to %s: %w", oldHostRepName, hostNicName, err)

×

1671

×

1673

        link, err := netlink.LinkByName(hostNicName)

×

1674

        if err != nil {

×

1675

                return "", "", "", -1, err

×

1676

×

1679

        if err = netlink.LinkSetMTU(link, mtu); err != nil {

×

1680

                return "", "", "", -1, fmt.Errorf("failed to set MTU on %s: %w", hostNicName, err)

×

1681

×

1684

        if err = setVfMac(deviceID, vfIndex, mac); err != nil {

×

1685

                return "", "", "", -1, err

×

1686

×

1688

        return hostNicName, vfNetdevice, "", -1, nil

×

1691

func renameLink(curName, newName string) error {

×

1692

        link, err := netlink.LinkByName(curName)

×

1693

        if err != nil {

×

1694

                klog.Error(err)

×

1695

                return err

×

1696

×

1698

        if err := netlink.LinkSetDown(link); err != nil {

×

1699

                klog.Error(err)

×

1700

                return err

×

1701

×

1702

        if err := netlink.LinkSetName(link, newName); err != nil {

×

1703

                klog.Error(err)

×

1704

                return err

×

1705

×

1706

        return netlink.LinkSetUp(link)

×

1709

func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, _, _ []string, ingress, egress, _, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP string) (string, []request.Route, error) {

×

1710

        _, containerNicName := generateNicName(containerID, ifName)

×

1711

        ipStr := util.GetIPWithoutMask(ip)

×

1712

        ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)

×

1713

        ovs.CleanDuplicatePort(ifaceID, containerNicName)

×

1714

×

1715

        // Add container iface to ovs port as internal port

×

1716

        output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", containerNicName, "--",

×

1717

                "set", "interface", containerNicName, "type=internal", "--",

×

1718

                "set", "interface", containerNicName, fmt.Sprintf("external_ids:iface-id=%s", ifaceID),

×

1719

                fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName),

×

1720

                fmt.Sprintf("external_ids:pod_name=%s", podName),

×

1721

                fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),

×

1722

                fmt.Sprintf("external_ids:ip=%s", ipStr),

×

1723

                fmt.Sprintf("external_ids:pod_netns=%s", netns))

×

1724

        if err != nil {

×

1725

                err := fmt.Errorf("add nic to ovs failed %w: %q", err, output)

×

1726

                klog.Error(err)

×

1727

                return containerNicName, nil, err

×

1728

×

1729

        defer func() {

×

1730

                if err != nil {

×

1731

                        if err := csh.rollbackOvsPort("", containerNicName, nicType); err != nil {

×

1732

                                klog.Errorf("failed to rollback ovs port %s, %v", containerNicName, err)

×

1733

                                return

×

1734

×

1739

        macAddr, err := net.ParseMAC(mac)

×

1740

        if err != nil {

×

1741

                return containerNicName, nil, fmt.Errorf("failed to parse mac %s %w", macAddr, err)

×

1742

×

1744

        if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress); err != nil {

×

1745

                return containerNicName, nil, err

×

1746

×

1748

        if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss, jitter); err != nil {

×

1749

                return containerNicName, nil, err

×

1750

×

1752

        podNS, err := ns.GetNS(netns)

×

1753

        if err != nil {

×

1754

                return containerNicName, nil, fmt.Errorf("failed to open netns %q: %w", netns, err)

×

1755

×

1756

        routes, err = csh.configureContainerNic(podName, podNamespace, containerNicName, ifName, ip, gateway, isDefaultRoute, detectIPConflict, routes, macAddr, podNS, mtu, nicType, gwCheckMode, u2oInterconnectionIP)

×

1757

        return containerNicName, routes, err

×

1760

func (csh cniServerHandler) removeDefaultRoute(netns string, ipv4, ipv6 bool) error {

×

1761

        podNS, err := ns.GetNS(netns)

×

1762

        if err != nil {

×

1763

                return fmt.Errorf("failed to open netns %q: %w", netns, err)

×

1764

×

1766

        return ns.WithNetNSPath(podNS.Path(), func(_ ns.NetNS) error {

×

1767

                routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)

×

1768

                if err != nil {

×

1769

                        return fmt.Errorf("failed to get all routes: %w", err)

×

1770

×

1772

                for _, r := range routes {

×

1773

                        if r.Dst != nil {

×

1774

                                if ones, _ := r.Dst.Mask.Size(); ones != 0 {

×

1775

                                        continue

×

1778

                        if ipv4 && r.Family == netlink.FAMILY_V4 {

×

1779

                                klog.Infof("deleting default ipv4 route %+v", r)

×

1780

                                if err = netlink.RouteDel(&r); err != nil {

×

1781

                                        return fmt.Errorf("failed to delete route %+v: %w", r, err)

×

1782

×

1783

                                continue

×

1785

                        if ipv6 && r.Family == netlink.FAMILY_V6 {

×

1786

                                klog.Infof("deleting default ipv6 route %+v", r)

×

1787

                                if err = netlink.RouteDel(&r); err != nil {

×

1788

                                        return fmt.Errorf("failed to delete route %+v: %w", r, err)

×

1789

×

1792

                return nil

×

1797

func configureAdditionalNic(link, ip string) error {

×

1798

        nodeLink, err := netlink.LinkByName(link)

×

1799

        if err != nil {

×

1800

                return fmt.Errorf("can not find nic %s %w", link, err)

×

1801

×

1803

        ipDelMap := make(map[string]netlink.Addr)

×

1804

        ipAddMap := make(map[string]netlink.Addr)

×

1805

        ipAddrs, err := netlink.AddrList(nodeLink, 0x0)

×

1806

        if err != nil {

×

1807

                return fmt.Errorf("can not get addr %s %w", nodeLink, err)

×

1808

×

1809

        for _, ipAddr := range ipAddrs {

×

1810

                if ipAddr.IP.IsLinkLocalUnicast() {

×

1811

                        // skip 169.254.0.0/16 and fe80::/10

×

1812

                        continue

×

1814

                ipDelMap[ipAddr.IPNet.String()] = ipAddr

×

1817

        for _, ipStr := range strings.Split(ip, ",") {

×

1818

                // Do not reassign same address for link

×

1819

                if _, ok := ipDelMap[ipStr]; ok {

×

1820

                        delete(ipDelMap, ipStr)

×

1821

                        continue

×

1824

                ipAddr, err := netlink.ParseAddr(ipStr)

×

1825

                if err != nil {

×

1826

                        return fmt.Errorf("can not parse %s %w", ipStr, err)

×

1827

×

1828

                ipAddMap[ipStr] = *ipAddr

×

1831

        for _, addr := range ipDelMap {

×

1832

                if err = netlink.AddrDel(nodeLink, &addr); err != nil {

×

1833

                        return fmt.Errorf("delete address %s %w", addr, err)

×

1834

×

1836

        for _, addr := range ipAddMap {

×

1837

                if err = netlink.AddrAdd(nodeLink, &addr); err != nil {

×

1838

                        return fmt.Errorf("can not add address %v to nic %s, %w", addr, link, err)

×

1839

×

1842

        return nil

×

1845

func addAdditionalNic(ifName string) error {

×

1846

        dummy := &netlink.Dummy{

×

1847

                LinkAttrs: netlink.LinkAttrs{

×

1848

                        Name: ifName,

×

1849

},

×

1850

×

1851

×

1852

        if err := netlink.LinkAdd(dummy); err != nil {

×

1853

                if err := netlink.LinkDel(dummy); err != nil {

×

1854

                        klog.Errorf("failed to delete static iface %v, err %v", ifName, err)

×

1855

                        return err

×

1856

×

1857

                return fmt.Errorf("failed to create static iface %v, err %w", ifName, err)

×

1859

        return nil

×

1862

func setVfMac(deviceID string, vfIndex int, mac string) error {

×

1863

        macAddr, err := net.ParseMAC(mac)

×

1864

        if err != nil {

×

1865

                return fmt.Errorf("failed to parse mac %s %w", macAddr, err)

×

1866

×

1868

        pfPci, err := sriovnet.GetPfPciFromVfPci(deviceID)

×

1869

        if err != nil {

×

1870

                return fmt.Errorf("failed to get pf of device %s %w", deviceID, err)

×

1871

×

1873

        netDevs, err := sriovnet.GetNetDevicesFromPci(pfPci)

×

1874

        if err != nil {

×

1875

                return fmt.Errorf("failed to get pf of device %s %w", deviceID, err)

×

1876

×

1879

        var pfName string

×

1880

        for _, dev := range netDevs {

×

1881

                devicePortNameFile := filepath.Join(util.NetSysDir, dev, "phys_port_name")

×

1882

                physPortName, err := sriovutilfs.Fs.ReadFile(devicePortNameFile)

×

1883

                if err != nil {

×

1884

                        continue

×

1887

                if !strings.Contains(strings.TrimSpace(string(physPortName)), "vf") {

×

1888

                        pfName = dev

×

1889

                        break

×

1892

        if pfName == "" {

×

1893

                return fmt.Errorf("the PF device was not found in the device list, %v", netDevs)

×

1894

×

1896

        pfLink, err := netlink.LinkByName(pfName)

×

1897

        if err != nil {

×

1898

                return fmt.Errorf("failed to lookup pf %s: %w", pfName, err)

×

1899

×

1900

        if err := netlink.LinkSetVfHardwareAddr(pfLink, vfIndex, macAddr); err != nil {

×

1901

                return fmt.Errorf("can not set mac address to vf nic:%s vf:%d %w", pfName, vfIndex, err)

×

1902

×

1903

        return nil

×

1906

func TurnOffNicTxChecksum(nicName string) error {

×

1907

        start := time.Now()

×

1908

        args := []string{"-K", nicName, "tx", "off"}

×

1909

        output, err := exec.Command("ethtool", args...).CombinedOutput() // #nosec G204

×

1910

        elapsed := float64((time.Since(start)) / time.Millisecond)

×

1911

        klog.V(4).Infof("command %s %s in %vms", "ethtool", strings.Join(args, " "), elapsed)

×

1912

        if err != nil {

×

1913

                klog.Error(err)

×

1914

                return fmt.Errorf("failed to turn off nic tx checksum, output %s, err %s", string(output), err.Error())

×

1915

×

1916

        return nil

×

1919

func getShortSharedDir(uid types.UID, volumeName string) string {

×

1920

        return filepath.Clean(filepath.Join(util.DefaultHostVhostuserBaseDir, string(uid), volumeName))

×

1921

×

1923

func linkExists(name string) (bool, error) {

×

1924

        if _, err := netlink.LinkByName(name); err != nil {

×

1925

                if _, ok := err.(netlink.LinkNotFoundError); ok {

×

1926

                        return false, nil

×

1927

×

1928

                return false, err

×

1930

        return true, nil

×

1933

func rollBackVethPair(nicName string) error {

×

1934

        hostLink, err := netlink.LinkByName(nicName)

×

1935

        if err != nil {

×

1936

                // if link already not exists, return quietly

×

1937

                // e.g. Internal port had been deleted by Remove ovs port previously

×

1938

                if _, ok := err.(netlink.LinkNotFoundError); ok {

×

1939

                        return nil

×

1940

×

1941

                klog.Error(err)

×

1942

                return fmt.Errorf("find host link %s failed %w", nicName, err)

×

1945

        hostLinkType := hostLink.Type()

×

1946

        // sometimes no deviceID input for vf nic, avoid delete vf nic.

×

1947

        if hostLinkType == "veth" {

×

1948

                if err = netlink.LinkDel(hostLink); err != nil {

×

1949

                        klog.Error(err)

×

1950

                        return fmt.Errorf("delete host link %s failed %w", hostLink, err)

×

1951

×

1953

        klog.Infof("rollback veth success %s", nicName)

×

1954

        return nil

×

kubeovn / kube-ovn / 12365336204

Source File
Press 'n' to go to next uncovered line, 'b' for previous

kubeovn / kube-ovn / 12365336204

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous