10179580977

Committed 31 Jul 2024 11:45AM UTC coverage: 43.816% (-0.2%) from 43.968%

Build # 10179580977

Build Type

Pull #733

github

Committed by

web-flow

Commit Message

Merge d80add92c into 57e1e9056

Pull Request Pull Request #733: feat: implement MlxResetFW to reset the FW on VF changes

Run Details

1 of 63 new or added lines in 6 files covered. (1.59%)

6 existing lines in 2 files now uncovered.

6536 of 14917 relevant lines covered (43.82%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/pkg/vendors/mellanox/mellanox.go

package mlxutils

import (
        "fmt"
        "regexp"
        "strconv"
        "strings"

        kerrors "k8s.io/apimachinery/pkg/util/errors"
        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
)

// BlueField mode representation
type BlueFieldMode int

const (
        BluefieldDpu BlueFieldMode = iota
        BluefieldConnectXMode

        internalCPUPageSupplier   = "INTERNAL_CPU_PAGE_SUPPLIER"
        internalCPUEswitchManager = "INTERNAL_CPU_ESWITCH_MANAGER"
        internalCPUIbVporto       = "INTERNAL_CPU_IB_VPORT0"
        internalCPUOffloadEngine  = "INTERNAL_CPU_OFFLOAD_ENGINE"
        internalCPUModel          = "INTERNAL_CPU_MODEL"

        ecpf        = "ECPF"
        extHostPf   = "EXT_HOST_PF"
        embeddedCPU = "EMBEDDED_CPU"

        disabled = "DISABLED"
        enabled  = "ENABLED"

        VendorMellanox = "15b3"
        DeviceBF2      = "a2d6"
        DeviceBF3      = "a2dc"

        PreconfiguredLinkType = "Preconfigured"
        UknownLinkType        = "Uknown"
        TotalVfs              = "NUM_OF_VFS"
        EnableSriov           = "SRIOV_EN"
        LinkTypeP1            = "LINK_TYPE_P1"
        LinkTypeP2            = "LINK_TYPE_P2"
        MellanoxVendorID      = "15b3"
)

type MlxNic struct {
        EnableSriov bool
        TotalVfs    int
        LinkTypeP1  string
        LinkTypeP2  string
}

//go:generate ../../../bin/mockgen -destination mock/mock_mellanox.go -source mellanox.go
type MellanoxInterface interface {
        MstConfigReadData(string) (string, string, error)
        GetMellanoxBlueFieldMode(string) (BlueFieldMode, error)
        GetMlxNicFwData(pciAddress string) (current, next *MlxNic, err error)

        MlxConfigFW(attributesToChange map[string]MlxNic) error
        MlxResetFW(pciAddresses []string) error
}

type mellanoxHelper struct {
        utils utils.CmdInterface
}

func New(utilsHelper utils.CmdInterface) MellanoxInterface {
        return &mellanoxHelper{
                utils: utilsHelper,
        }
}

func (m *mellanoxHelper) MstConfigReadData(pciAddress string) (string, string, error) {
        log.Log.Info("MstConfigReadData()", "device", pciAddress)
        args := []string{"-e", "-d", pciAddress, "q"}
        stdout, stderr, err := m.utils.RunCommand("mstconfig", args...)
        return stdout, stderr, err
}

func (m *mellanoxHelper) GetMellanoxBlueFieldMode(PciAddress string) (BlueFieldMode, error) {
        log.Log.V(2).Info("MellanoxBlueFieldMode(): checking mode for device", "device", PciAddress)
        stdout, stderr, err := m.MstConfigReadData(PciAddress)
        if err != nil {
                log.Log.Error(err, "MellanoxBlueFieldMode(): failed to get mlx nic fw data", "stderr", stderr)
                return -1, fmt.Errorf("failed to get mlx nic fw data %w", err)
        }

        attrs := []string{internalCPUPageSupplier,
                internalCPUEswitchManager,
                internalCPUIbVporto,
                internalCPUOffloadEngine,
                internalCPUModel}
        mstCurrentData, _ := ParseMstconfigOutput(stdout, attrs)

        internalCPUPageSupplierstatus, exist := mstCurrentData[internalCPUPageSupplier]
        if !exist {
                return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUPageSupplier)
        }

        internalCPUEswitchManagerStatus, exist := mstCurrentData[internalCPUEswitchManager]
        if !exist {
                return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUEswitchManager)
        }

        internalCPUIbVportoStatus, exist := mstCurrentData[internalCPUIbVporto]
        if !exist {
                return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUIbVporto)
        }

        internalCPUOffloadEngineStatus, exist := mstCurrentData[internalCPUOffloadEngine]
        if !exist {
                return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUOffloadEngine)
        }

        internalCPUModelStatus, exist := mstCurrentData[internalCPUModel]
        if !exist {
                return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUModel)
        }

        // check for DPU
        if strings.Contains(internalCPUPageSupplierstatus, ecpf) &&
                strings.Contains(internalCPUEswitchManagerStatus, ecpf) &&
                strings.Contains(internalCPUIbVportoStatus, ecpf) &&
                strings.Contains(internalCPUOffloadEngineStatus, enabled) &&
                strings.Contains(internalCPUModelStatus, embeddedCPU) {
                log.Log.V(2).Info("MellanoxBlueFieldMode(): device in DPU mode", "device", PciAddress)
                return BluefieldDpu, nil
        } else if strings.Contains(internalCPUPageSupplierstatus, extHostPf) &&
                strings.Contains(internalCPUEswitchManagerStatus, extHostPf) &&
                strings.Contains(internalCPUIbVportoStatus, extHostPf) &&
                strings.Contains(internalCPUOffloadEngineStatus, disabled) &&
                strings.Contains(internalCPUModelStatus, embeddedCPU) {
                log.Log.V(2).Info("MellanoxBlueFieldMode(): device in ConnectX mode", "device", PciAddress)
                return BluefieldConnectXMode, nil
        }

        log.Log.Error(err, "MellanoxBlueFieldMode(): unknown device status",
                "device", PciAddress, "mstconfig-output", stdout)
        return -1, fmt.Errorf("MellanoxBlueFieldMode(): unknown device status for %s", PciAddress)
}

func (m *mellanoxHelper) MlxResetFW(pciAddresses []string) error {
        log.Log.Info("mellanox-plugin resetFW()")
        var errs []error
        for _, pciAddress := range pciAddresses {
                cmdArgs := []string{"-d", pciAddress, "--skip_driver", "-l", "3", "-y", "reset"}
                log.Log.Info("mellanox-plugin: resetFW()", "cmd-args", cmdArgs)
                // We have to ensure that pciutils is installed into the container image Dockerfile.sriov-network-config-daemon
                _, stderr, err := m.utils.RunCommand("mstfwreset", cmdArgs...)
                if err != nil {
                        log.Log.Error(err, "mellanox-plugin resetFW(): failed", "stderr", stderr)
                        errs = append(errs, err)
                }
        }
        return kerrors.NewAggregate(errs)
}

func (m *mellanoxHelper) MlxConfigFW(attributesToChange map[string]MlxNic) error {
        log.Log.Info("mellanox-plugin configFW()")
        for pciAddr, fwArgs := range attributesToChange {
                cmdArgs := []string{"-d", pciAddr, "-y", "set"}
                if fwArgs.EnableSriov {
                        cmdArgs = append(cmdArgs, fmt.Sprintf("%s=True", EnableSriov))
                } else if fwArgs.TotalVfs == 0 {
                        cmdArgs = append(cmdArgs, fmt.Sprintf("%s=False", EnableSriov))
                }
                if fwArgs.TotalVfs > -1 {
                        cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%d", TotalVfs, fwArgs.TotalVfs))
                }
                if len(fwArgs.LinkTypeP1) > 0 {
                        cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP1, fwArgs.LinkTypeP1))
                }
                if len(fwArgs.LinkTypeP2) > 0 {
                        cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP2, fwArgs.LinkTypeP2))
                }

                log.Log.V(2).Info("mellanox-plugin: configFW()", "cmd-args", cmdArgs)
                if len(cmdArgs) <= 4 {
                        continue
                }
                _, strerr, err := m.utils.RunCommand("mstconfig", cmdArgs...)
                if err != nil {
                        log.Log.Error(err, "mellanox-plugin configFW(): failed", "stderr", strerr)
                        return err
                }
        }
        return nil
}

func (m *mellanoxHelper) GetMlxNicFwData(pciAddress string) (current, next *MlxNic, err error) {
        log.Log.Info("mellanox-plugin getMlnxNicFwData()", "device", pciAddress)
        attrs := []string{TotalVfs, EnableSriov, LinkTypeP1, LinkTypeP2}

        out, stderr, err := m.MstConfigReadData(pciAddress)
        if err != nil {
                log.Log.Error(err, "mellanox-plugin getMlnxNicFwData(): failed", "stderr", stderr)
                return
        }
        mstCurrentData, mstNextData := ParseMstconfigOutput(out, attrs)
        current, err = mlnxNicFromMap(mstCurrentData)
        if err != nil {
                log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for current mstconfig data failed")
                return
        }
        next, err = mlnxNicFromMap(mstNextData)
        if err != nil {
                log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for next mstconfig data failed")
        }
        return
}

func ParseMstconfigOutput(mstOutput string, attributes []string) (fwCurrent, fwNext map[string]string) {
        log.Log.Info("ParseMstconfigOutput()", "attributes", attributes)
        fwCurrent = map[string]string{}
        fwNext = map[string]string{}
        formatRegex := regexp.MustCompile(`(?P<Attribute>\w+)\s+(?P<Default>\S+)\s+(?P<Current>\S+)\s+(?P<Next>\S+)`)
        mstOutputLines := strings.Split(mstOutput, "\n")
        for _, attr := range attributes {
                for _, line := range mstOutputLines {
                        if strings.Contains(line, attr) {
                                regexResult := formatRegex.FindStringSubmatch(line)
                                fwCurrent[attr] = regexResult[3]
                                fwNext[attr] = regexResult[4]
                                break
                        }
                }
        }
        return
}

func HasMellanoxInterfacesInSpec(ifaceStatuses sriovnetworkv1.InterfaceExts, ifaceSpecs sriovnetworkv1.Interfaces) bool {
        for _, ifaceStatus := range ifaceStatuses {
                if ifaceStatus.Vendor == VendorMellanox {
                        for _, iface := range ifaceSpecs {
                                if iface.PciAddress == ifaceStatus.PciAddress {
                                        log.Log.V(2).Info("hasMellanoxInterfacesInSpec(): Mellanox device specified in SriovNetworkNodeState spec",
                                                "name", ifaceStatus.Name,
                                                "address", ifaceStatus.PciAddress)
                                        return true
                                }
                        }
                }
        }
        return false
}

func GetPciAddressPrefix(pciAddress string) string {
        return pciAddress[:len(pciAddress)-1]
}

func IsDualPort(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) bool {
        log.Log.Info("mellanox-plugin IsDualPort()", "address", pciAddress)
        pciAddressPrefix := GetPciAddressPrefix(pciAddress)
        return len(mellanoxNicsStatus[pciAddressPrefix]) > 1
}

// handleTotalVfs return required total VFs or max (required VFs for dual port NIC) and needReboot if totalVfs will change
func HandleTotalVfs(fwCurrent, fwNext, attrs *MlxNic, ifaceSpec sriovnetworkv1.Interface, isDualPort bool, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) (
        totalVfs int, needReboot, changeWithoutReboot bool) {
        totalVfs = ifaceSpec.NumVfs
        // Check if the other port is changing theGetMlnxNicFwData number of VF
        if isDualPort {
                otherIfaceSpec := getOtherPortSpec(ifaceSpec.PciAddress, mellanoxNicsSpec)
                if otherIfaceSpec != nil {
                        if otherIfaceSpec.NumVfs > totalVfs {
                                totalVfs = otherIfaceSpec.NumVfs
                        }
                }
        }

        // if the PF is externally managed we just need to check the totalVfs requested in the policy is not higher than
        // the configured amount
        if ifaceSpec.ExternallyManaged {
                if totalVfs > fwCurrent.TotalVfs {
                        log.Log.Error(nil, "The nic is externallyManaged and TotalVfs configured on the system is lower then requested VFs, failing configuration",
                                "current", fwCurrent.TotalVfs, "requested", totalVfs)
                        attrs.TotalVfs = totalVfs
                        needReboot = true
                        changeWithoutReboot = false
                }
                return
        }

        if fwCurrent.TotalVfs != totalVfs {
                log.Log.V(2).Info("Changing TotalVfs, needs reboot", "current", fwCurrent.TotalVfs, "requested", totalVfs)
                attrs.TotalVfs = totalVfs
                needReboot = true
        }

        // Remove policy then re-apply it
        if !needReboot && fwNext.TotalVfs != totalVfs {
                log.Log.V(2).Info("Changing TotalVfs to same as Next Boot value, doesn't require rebooting",
                        "current", fwCurrent.TotalVfs, "next", fwNext.TotalVfs, "requested", totalVfs)
                attrs.TotalVfs = totalVfs
                changeWithoutReboot = true
        }

        return
}

// handleEnableSriov based on totalVfs it decide to disable (totalVfs=0) or enable (totalVfs changed from 0) sriov
// and need reboot if enableSriov will change
func HandleEnableSriov(totalVfs int, fwCurrent, fwNext, attrs *MlxNic) (needReboot, changeWithoutReboot bool) {
        if totalVfs == 0 && fwCurrent.EnableSriov {
                log.Log.V(2).Info("disabling Sriov, needs reboot")
                attrs.EnableSriov = false
                return true, false
        } else if totalVfs > 0 && !fwCurrent.EnableSriov {
                log.Log.V(2).Info("enabling Sriov, needs reboot")
                attrs.EnableSriov = true
                return true, false
        } else if totalVfs > 0 && !fwNext.EnableSriov {
                attrs.EnableSriov = true
                return false, true
        }

        return false, false
}

// handleLinkType based on existing linkType and requested link
func HandleLinkType(pciPrefix string, fwData, attr *MlxNic,
        mellanoxNicsSpec map[string]sriovnetworkv1.Interface,
        mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) (bool, error) {
        needReboot := false

        pciAddress := pciPrefix + "0"
        if firstPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok {
                ifaceStatus := getIfaceStatus(pciAddress, mellanoxNicsStatus)
                needChange, err := isLinkTypeRequireChange(firstPortSpec, ifaceStatus, fwData.LinkTypeP1)
                if err != nil {
                        return false, err
                }

                if needChange {
                        log.Log.V(2).Info("Changing LinkTypeP1, needs reboot",
                                "from", fwData.LinkTypeP1, "to", firstPortSpec.LinkType)
                        attr.LinkTypeP1 = firstPortSpec.LinkType
                        needReboot = true
                }
        }

        pciAddress = pciPrefix + "1"
        if secondPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok {
                ifaceStatus := getIfaceStatus(pciAddress, mellanoxNicsStatus)
                needChange, err := isLinkTypeRequireChange(secondPortSpec, ifaceStatus, fwData.LinkTypeP2)
                if err != nil {
                        return false, err
                }

                if needChange {
                        log.Log.V(2).Info("Changing LinkTypeP2, needs reboot",
                                "from", fwData.LinkTypeP2, "to", secondPortSpec.LinkType)
                        attr.LinkTypeP2 = secondPortSpec.LinkType
                        needReboot = true
                }
        }

        return needReboot, nil
}

func mlnxNicFromMap(mstData map[string]string) (*MlxNic, error) {
        log.Log.Info("mellanox-plugin mlnxNicFromMap()", "data", mstData)
        fwData := &MlxNic{}
        if strings.Contains(mstData[EnableSriov], "True") {
                fwData.EnableSriov = true
        }
        i, err := strconv.Atoi(mstData[TotalVfs])
        if err != nil {
                return nil, err
        }

        fwData.TotalVfs = i
        fwData.LinkTypeP1 = getLinkType(mstData[LinkTypeP1])
        if linkTypeP2, ok := mstData[LinkTypeP2]; ok {
                fwData.LinkTypeP2 = getLinkType(linkTypeP2)
        }

        return fwData, nil
}

func getLinkType(linkType string) string {
        log.Log.Info("mellanox-plugin getLinkType()", "link-type", linkType)
        if strings.Contains(linkType, consts.LinkTypeETH) {
                return consts.LinkTypeETH
        } else if strings.Contains(linkType, consts.LinkTypeIB) {
                return consts.LinkTypeIB
        } else if len(linkType) > 0 {
                log.Log.Error(nil, "mellanox-plugin getLinkType(): link type is not one of [ETH, IB], treating as unknown",
                        "link-type", linkType)
                return UknownLinkType
        } else {
                log.Log.Info("mellanox-plugin getLinkType(): LINK_TYPE_P* attribute was not found, treating as preconfigured link type")
                return PreconfiguredLinkType
        }
}

func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovnetworkv1.InterfaceExt, fwLinkType string) (bool, error) {
        log.Log.Info("mellanox-plugin isLinkTypeRequireChange()", "device", iface.PciAddress)
        if iface.LinkType != "" && !strings.EqualFold(ifaceStatus.LinkType, iface.LinkType) {
                if !strings.EqualFold(iface.LinkType, consts.LinkTypeETH) && !strings.EqualFold(iface.LinkType, consts.LinkTypeIB) {
                        return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Not supported link type: %s,"+
                                " supported link types: [eth, ETH, ib, and IB]", iface.LinkType)
                }
                if fwLinkType == UknownLinkType {
                        return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Unknown link type: %s", fwLinkType)
                }
                if fwLinkType == PreconfiguredLinkType {
                        return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Network card %s does not support link type change", iface.PciAddress)
                }

                return true, nil
        }

        return false, nil
}

func getOtherPortSpec(pciAddress string, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) *sriovnetworkv1.Interface {
        log.Log.Info("mellanox-plugin getOtherPortSpec()", "pciAddress", pciAddress)
        pciAddrPrefix := GetPciAddressPrefix(pciAddress)
        pciAddrSuffix := pciAddress[len(pciAddrPrefix):]

        if pciAddrSuffix == "0" {
                iface := mellanoxNicsSpec[pciAddrPrefix+"1"]
                return &iface
        }

        iface := mellanoxNicsSpec[pciAddrPrefix+"0"]
        return &iface
}

func getIfaceStatus(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) sriovnetworkv1.InterfaceExt {
        return mellanoxNicsStatus[GetPciAddressPrefix(pciAddress)][pciAddress]
}

1	package mlxutils
2
3	import (
4	"fmt"
5	"regexp"
6	"strconv"
7	"strings"
8
9	kerrors "k8s.io/apimachinery/pkg/util/errors"
10	"sigs.k8s.io/controller-runtime/pkg/log"
11
12	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
13	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
14	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
15	)
16
17	// BlueField mode representation
18	type BlueFieldMode int
19
20	const (
21	BluefieldDpu BlueFieldMode = iota
22	BluefieldConnectXMode
23
24	internalCPUPageSupplier = "INTERNAL_CPU_PAGE_SUPPLIER"
25	internalCPUEswitchManager = "INTERNAL_CPU_ESWITCH_MANAGER"
26	internalCPUIbVporto = "INTERNAL_CPU_IB_VPORT0"
27	internalCPUOffloadEngine = "INTERNAL_CPU_OFFLOAD_ENGINE"
28	internalCPUModel = "INTERNAL_CPU_MODEL"
29
30	ecpf = "ECPF"
31	extHostPf = "EXT_HOST_PF"
32	embeddedCPU = "EMBEDDED_CPU"
33
34	disabled = "DISABLED"
35	enabled = "ENABLED"
36
37	VendorMellanox = "15b3"
38	DeviceBF2 = "a2d6"
39	DeviceBF3 = "a2dc"
40
41	PreconfiguredLinkType = "Preconfigured"
42	UknownLinkType = "Uknown"
43	TotalVfs = "NUM_OF_VFS"
44	EnableSriov = "SRIOV_EN"
45	LinkTypeP1 = "LINK_TYPE_P1"
46	LinkTypeP2 = "LINK_TYPE_P2"
47	MellanoxVendorID = "15b3"
48	)
49
50	type MlxNic struct {
51	EnableSriov bool
52	TotalVfs int
53	LinkTypeP1 string
54	LinkTypeP2 string
55	}
56
57	//go:generate ../../../bin/mockgen -destination mock/mock_mellanox.go -source mellanox.go
58	type MellanoxInterface interface {
59	MstConfigReadData(string) (string, string, error)
60	GetMellanoxBlueFieldMode(string) (BlueFieldMode, error)
61	GetMlxNicFwData(pciAddress string) (current, next *MlxNic, err error)
62
63	MlxConfigFW(attributesToChange map[string]MlxNic) error
64	MlxResetFW(pciAddresses []string) error
65	}
66
67	type mellanoxHelper struct {
68	utils utils.CmdInterface
69	}
70
71	func New(utilsHelper utils.CmdInterface) MellanoxInterface {	×
72	return &mellanoxHelper{	×
73	utils: utilsHelper,	×
74	}	×
75	}	×
76
77	func (m *mellanoxHelper) MstConfigReadData(pciAddress string) (string, string, error) {	×
78	log.Log.Info("MstConfigReadData()", "device", pciAddress)	×
79	args := []string{"-e", "-d", pciAddress, "q"}	×
80	stdout, stderr, err := m.utils.RunCommand("mstconfig", args...)	×
81	return stdout, stderr, err	×
82	}	×
83
84	func (m *mellanoxHelper) GetMellanoxBlueFieldMode(PciAddress string) (BlueFieldMode, error) {	×
85	log.Log.V(2).Info("MellanoxBlueFieldMode(): checking mode for device", "device", PciAddress)	×
86	stdout, stderr, err := m.MstConfigReadData(PciAddress)	×
87	if err != nil {	×
88	log.Log.Error(err, "MellanoxBlueFieldMode(): failed to get mlx nic fw data", "stderr", stderr)	×
89	return -1, fmt.Errorf("failed to get mlx nic fw data %w", err)	×
90	}	×
91
92	attrs := []string{internalCPUPageSupplier,	×
93	internalCPUEswitchManager,	×
94	internalCPUIbVporto,	×
95	internalCPUOffloadEngine,	×
96	internalCPUModel}	×
97	mstCurrentData, _ := ParseMstconfigOutput(stdout, attrs)	×
98		×
99	internalCPUPageSupplierstatus, exist := mstCurrentData[internalCPUPageSupplier]	×
100	if !exist {	×
101	return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUPageSupplier)	×
102	}	×
103
104	internalCPUEswitchManagerStatus, exist := mstCurrentData[internalCPUEswitchManager]	×
105	if !exist {	×
106	return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUEswitchManager)	×
107	}	×
108
109	internalCPUIbVportoStatus, exist := mstCurrentData[internalCPUIbVporto]	×
110	if !exist {	×
111	return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUIbVporto)	×
112	}	×
113
114	internalCPUOffloadEngineStatus, exist := mstCurrentData[internalCPUOffloadEngine]	×
115	if !exist {	×
116	return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUOffloadEngine)	×
117	}	×
118
119	internalCPUModelStatus, exist := mstCurrentData[internalCPUModel]	×
120	if !exist {	×
121	return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUModel)	×
122	}	×
123
124	// check for DPU
125	if strings.Contains(internalCPUPageSupplierstatus, ecpf) &&	×
126	strings.Contains(internalCPUEswitchManagerStatus, ecpf) &&	×
127	strings.Contains(internalCPUIbVportoStatus, ecpf) &&	×
128	strings.Contains(internalCPUOffloadEngineStatus, enabled) &&	×
129	strings.Contains(internalCPUModelStatus, embeddedCPU) {	×
130	log.Log.V(2).Info("MellanoxBlueFieldMode(): device in DPU mode", "device", PciAddress)	×
131	return BluefieldDpu, nil	×
132	} else if strings.Contains(internalCPUPageSupplierstatus, extHostPf) &&	×
133	strings.Contains(internalCPUEswitchManagerStatus, extHostPf) &&	×
134	strings.Contains(internalCPUIbVportoStatus, extHostPf) &&	×
135	strings.Contains(internalCPUOffloadEngineStatus, disabled) &&	×
136	strings.Contains(internalCPUModelStatus, embeddedCPU) {	×
137	log.Log.V(2).Info("MellanoxBlueFieldMode(): device in ConnectX mode", "device", PciAddress)	×
138	return BluefieldConnectXMode, nil	×
139	}	×
140
141	log.Log.Error(err, "MellanoxBlueFieldMode(): unknown device status",	×
142	"device", PciAddress, "mstconfig-output", stdout)	×
143	return -1, fmt.Errorf("MellanoxBlueFieldMode(): unknown device status for %s", PciAddress)	×
144	}
145
NEW 146	func (m *mellanoxHelper) MlxResetFW(pciAddresses []string) error {	×
NEW 147	log.Log.Info("mellanox-plugin resetFW()")	×
NEW 148	var errs []error	×
NEW 149	for _, pciAddress := range pciAddresses {	×
NEW 150	cmdArgs := []string{"-d", pciAddress, "--skip_driver", "-l", "3", "-y", "reset"}	×
NEW 151	log.Log.Info("mellanox-plugin: resetFW()", "cmd-args", cmdArgs)	×
NEW 152	// We have to ensure that pciutils is installed into the container image Dockerfile.sriov-network-config-daemon	×
NEW 153	_, stderr, err := m.utils.RunCommand("mstfwreset", cmdArgs...)	×
NEW 154	if err != nil {	×
NEW 155	log.Log.Error(err, "mellanox-plugin resetFW(): failed", "stderr", stderr)	×
NEW 156	errs = append(errs, err)	×
NEW 157	}	×
158	}
NEW 159	return kerrors.NewAggregate(errs)	×
160	}
161
162	func (m *mellanoxHelper) MlxConfigFW(attributesToChange map[string]MlxNic) error {	×
163	log.Log.Info("mellanox-plugin configFW()")	×
164	for pciAddr, fwArgs := range attributesToChange {	×
165	cmdArgs := []string{"-d", pciAddr, "-y", "set"}	×
166	if fwArgs.EnableSriov {	×
167	cmdArgs = append(cmdArgs, fmt.Sprintf("%s=True", EnableSriov))	×
168	} else if fwArgs.TotalVfs == 0 {	×
169	cmdArgs = append(cmdArgs, fmt.Sprintf("%s=False", EnableSriov))	×
170	}	×
171	if fwArgs.TotalVfs > -1 {	×
172	cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%d", TotalVfs, fwArgs.TotalVfs))	×
173	}	×
174	if len(fwArgs.LinkTypeP1) > 0 {	×
175	cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP1, fwArgs.LinkTypeP1))	×
176	}	×
177	if len(fwArgs.LinkTypeP2) > 0 {	×
178	cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP2, fwArgs.LinkTypeP2))	×
179	}	×
180
181	log.Log.V(2).Info("mellanox-plugin: configFW()", "cmd-args", cmdArgs)	×
182	if len(cmdArgs) <= 4 {	×
183	continue	×
184	}
185	_, strerr, err := m.utils.RunCommand("mstconfig", cmdArgs...)	×
186	if err != nil {	×
187	log.Log.Error(err, "mellanox-plugin configFW(): failed", "stderr", strerr)	×
188	return err	×
189	}	×
190	}
191	return nil	×
192	}
193
194	func (m mellanoxHelper) GetMlxNicFwData(pciAddress string) (current, next MlxNic, err error) {	×
195	log.Log.Info("mellanox-plugin getMlnxNicFwData()", "device", pciAddress)	×
196	attrs := []string{TotalVfs, EnableSriov, LinkTypeP1, LinkTypeP2}	×
197		×
198	out, stderr, err := m.MstConfigReadData(pciAddress)	×
199	if err != nil {	×
200	log.Log.Error(err, "mellanox-plugin getMlnxNicFwData(): failed", "stderr", stderr)	×
201	return	×
202	}	×
203	mstCurrentData, mstNextData := ParseMstconfigOutput(out, attrs)	×
204	current, err = mlnxNicFromMap(mstCurrentData)	×
205	if err != nil {	×
206	log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for current mstconfig data failed")	×
207	return	×
208	}	×
209	next, err = mlnxNicFromMap(mstNextData)	×
210	if err != nil {	×
211	log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for next mstconfig data failed")	×
212	}	×
213	return	×
214	}
215
216	func ParseMstconfigOutput(mstOutput string, attributes []string) (fwCurrent, fwNext map[string]string) {	×
217	log.Log.Info("ParseMstconfigOutput()", "attributes", attributes)	×
218	fwCurrent = map[string]string{}	×
219	fwNext = map[string]string{}	×
220	formatRegex := regexp.MustCompile(`(?P<Attribute>\w+)\s+(?P<Default>\S+)\s+(?P<Current>\S+)\s+(?P<Next>\S+)`)	×
221	mstOutputLines := strings.Split(mstOutput, "\n")	×
222	for _, attr := range attributes {	×
223	for _, line := range mstOutputLines {	×
224	if strings.Contains(line, attr) {	×
225	regexResult := formatRegex.FindStringSubmatch(line)	×
226	fwCurrent[attr] = regexResult[3]	×
227	fwNext[attr] = regexResult[4]	×
228	break	×
229	}
230	}
231	}
232	return	×
233	}
234
235	func HasMellanoxInterfacesInSpec(ifaceStatuses sriovnetworkv1.InterfaceExts, ifaceSpecs sriovnetworkv1.Interfaces) bool {	×
236	for _, ifaceStatus := range ifaceStatuses {	×
237	if ifaceStatus.Vendor == VendorMellanox {	×
238	for _, iface := range ifaceSpecs {	×
239	if iface.PciAddress == ifaceStatus.PciAddress {	×
240	log.Log.V(2).Info("hasMellanoxInterfacesInSpec(): Mellanox device specified in SriovNetworkNodeState spec",	×
241	"name", ifaceStatus.Name,	×
242	"address", ifaceStatus.PciAddress)	×
243	return true	×
244	}	×
245	}
246	}
247	}
248	return false	×
249	}
250
251	func GetPciAddressPrefix(pciAddress string) string {	×
252	return pciAddress[:len(pciAddress)-1]	×
253	}	×
254
255	func IsDualPort(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) bool {	×
256	log.Log.Info("mellanox-plugin IsDualPort()", "address", pciAddress)	×
257	pciAddressPrefix := GetPciAddressPrefix(pciAddress)	×
258	return len(mellanoxNicsStatus[pciAddressPrefix]) > 1	×
259	}	×
260
261	// handleTotalVfs return required total VFs or max (required VFs for dual port NIC) and needReboot if totalVfs will change
262	func HandleTotalVfs(fwCurrent, fwNext, attrs *MlxNic, ifaceSpec sriovnetworkv1.Interface, isDualPort bool, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) (
263	totalVfs int, needReboot, changeWithoutReboot bool) {	×
264	totalVfs = ifaceSpec.NumVfs	×
265	// Check if the other port is changing theGetMlnxNicFwData number of VF	×
266	if isDualPort {	×
267	otherIfaceSpec := getOtherPortSpec(ifaceSpec.PciAddress, mellanoxNicsSpec)	×
268	if otherIfaceSpec != nil {	×
269	if otherIfaceSpec.NumVfs > totalVfs {	×
270	totalVfs = otherIfaceSpec.NumVfs	×
271	}	×
272	}
273	}
274
275	// if the PF is externally managed we just need to check the totalVfs requested in the policy is not higher than
276	// the configured amount
277	if ifaceSpec.ExternallyManaged {	×
278	if totalVfs > fwCurrent.TotalVfs {	×
279	log.Log.Error(nil, "The nic is externallyManaged and TotalVfs configured on the system is lower then requested VFs, failing configuration",	×
280	"current", fwCurrent.TotalVfs, "requested", totalVfs)	×
281	attrs.TotalVfs = totalVfs	×
282	needReboot = true	×
283	changeWithoutReboot = false	×
284	}	×
285	return	×
286	}
287
288	if fwCurrent.TotalVfs != totalVfs {	×
289	log.Log.V(2).Info("Changing TotalVfs, needs reboot", "current", fwCurrent.TotalVfs, "requested", totalVfs)	×
290	attrs.TotalVfs = totalVfs	×
291	needReboot = true	×
292	}	×
293
294	// Remove policy then re-apply it
295	if !needReboot && fwNext.TotalVfs != totalVfs {	×
296	log.Log.V(2).Info("Changing TotalVfs to same as Next Boot value, doesn't require rebooting",	×
297	"current", fwCurrent.TotalVfs, "next", fwNext.TotalVfs, "requested", totalVfs)	×
298	attrs.TotalVfs = totalVfs	×
299	changeWithoutReboot = true	×
300	}	×
301
302	return	×
303	}
304
305	// handleEnableSriov based on totalVfs it decide to disable (totalVfs=0) or enable (totalVfs changed from 0) sriov
306	// and need reboot if enableSriov will change
307	func HandleEnableSriov(totalVfs int, fwCurrent, fwNext, attrs *MlxNic) (needReboot, changeWithoutReboot bool) {	×
308	if totalVfs == 0 && fwCurrent.EnableSriov {	×
309	log.Log.V(2).Info("disabling Sriov, needs reboot")	×
310	attrs.EnableSriov = false	×
311	return true, false	×
312	} else if totalVfs > 0 && !fwCurrent.EnableSriov {	×
313	log.Log.V(2).Info("enabling Sriov, needs reboot")	×
314	attrs.EnableSriov = true	×
315	return true, false	×
316	} else if totalVfs > 0 && !fwNext.EnableSriov {	×
317	attrs.EnableSriov = true	×
318	return false, true	×
319	}	×
320
321	return false, false	×
322	}
323
324	// handleLinkType based on existing linkType and requested link
325	func HandleLinkType(pciPrefix string, fwData, attr *MlxNic,
326	mellanoxNicsSpec map[string]sriovnetworkv1.Interface,
327	mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) (bool, error) {	×
328	needReboot := false	×
329		×
330	pciAddress := pciPrefix + "0"	×
331	if firstPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok {	×
332	ifaceStatus := getIfaceStatus(pciAddress, mellanoxNicsStatus)	×
333	needChange, err := isLinkTypeRequireChange(firstPortSpec, ifaceStatus, fwData.LinkTypeP1)	×
334	if err != nil {	×
335	return false, err	×
336	}	×
337
338	if needChange {	×
339	log.Log.V(2).Info("Changing LinkTypeP1, needs reboot",	×
340	"from", fwData.LinkTypeP1, "to", firstPortSpec.LinkType)	×
341	attr.LinkTypeP1 = firstPortSpec.LinkType	×
342	needReboot = true	×
343	}	×
344	}
345
346	pciAddress = pciPrefix + "1"	×
347	if secondPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok {	×
348	ifaceStatus := getIfaceStatus(pciAddress, mellanoxNicsStatus)	×
349	needChange, err := isLinkTypeRequireChange(secondPortSpec, ifaceStatus, fwData.LinkTypeP2)	×
350	if err != nil {	×
351	return false, err	×
352	}	×
353
354	if needChange {	×
355	log.Log.V(2).Info("Changing LinkTypeP2, needs reboot",	×
356	"from", fwData.LinkTypeP2, "to", secondPortSpec.LinkType)	×
357	attr.LinkTypeP2 = secondPortSpec.LinkType	×
358	needReboot = true	×
359	}	×
360	}
361
362	return needReboot, nil	×
363	}
364
365	func mlnxNicFromMap(mstData map[string]string) (*MlxNic, error) {	×
366	log.Log.Info("mellanox-plugin mlnxNicFromMap()", "data", mstData)	×
367	fwData := &MlxNic{}	×
368	if strings.Contains(mstData[EnableSriov], "True") {	×
369	fwData.EnableSriov = true	×
370	}	×
371	i, err := strconv.Atoi(mstData[TotalVfs])	×
372	if err != nil {	×
373	return nil, err	×
374	}	×
375
376	fwData.TotalVfs = i	×
377	fwData.LinkTypeP1 = getLinkType(mstData[LinkTypeP1])	×
378	if linkTypeP2, ok := mstData[LinkTypeP2]; ok {	×
379	fwData.LinkTypeP2 = getLinkType(linkTypeP2)	×
380	}	×
381
382	return fwData, nil	×
383	}
384
385	func getLinkType(linkType string) string {	×
386	log.Log.Info("mellanox-plugin getLinkType()", "link-type", linkType)	×
387	if strings.Contains(linkType, consts.LinkTypeETH) {	×
388	return consts.LinkTypeETH	×
389	} else if strings.Contains(linkType, consts.LinkTypeIB) {	×
390	return consts.LinkTypeIB	×
391	} else if len(linkType) > 0 {	×
392	log.Log.Error(nil, "mellanox-plugin getLinkType(): link type is not one of [ETH, IB], treating as unknown",	×
393	"link-type", linkType)	×
394	return UknownLinkType	×
395	} else {	×
396	log.Log.Info("mellanox-plugin getLinkType(): LINK_TYPE_P* attribute was not found, treating as preconfigured link type")	×
397	return PreconfiguredLinkType	×
398	}	×
399	}
400
401	func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovnetworkv1.InterfaceExt, fwLinkType string) (bool, error) {	×
402	log.Log.Info("mellanox-plugin isLinkTypeRequireChange()", "device", iface.PciAddress)	×
403	if iface.LinkType != "" && !strings.EqualFold(ifaceStatus.LinkType, iface.LinkType) {	×
404	if !strings.EqualFold(iface.LinkType, consts.LinkTypeETH) && !strings.EqualFold(iface.LinkType, consts.LinkTypeIB) {	×
405	return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Not supported link type: %s,"+	×
406	" supported link types: [eth, ETH, ib, and IB]", iface.LinkType)	×
407	}	×
408	if fwLinkType == UknownLinkType {	×
409	return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Unknown link type: %s", fwLinkType)	×
410	}	×
411	if fwLinkType == PreconfiguredLinkType {	×
412	return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Network card %s does not support link type change", iface.PciAddress)	×
413	}	×
414
415	return true, nil	×
416	}
417
418	return false, nil	×
419	}
420
421	func getOtherPortSpec(pciAddress string, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) *sriovnetworkv1.Interface {	×
422	log.Log.Info("mellanox-plugin getOtherPortSpec()", "pciAddress", pciAddress)	×
423	pciAddrPrefix := GetPciAddressPrefix(pciAddress)	×
424	pciAddrSuffix := pciAddress[len(pciAddrPrefix):]	×
425		×
426	if pciAddrSuffix == "0" {	×
427	iface := mellanoxNicsSpec[pciAddrPrefix+"1"]	×
428	return &iface	×
429	}	×
430
431	iface := mellanoxNicsSpec[pciAddrPrefix+"0"]	×
432	return &iface	×
433	}
434
435	func getIfaceStatus(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) sriovnetworkv1.InterfaceExt {	×
436	return mellanoxNicsStatus[GetPciAddressPrefix(pciAddress)][pciAddress]	×
437	}	×

k8snetworkplumbingwg / sriov-network-operator / 10179580977

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous