10179580977

Committed 31 Jul 2024 11:45AM UTC coverage: 43.816% (-0.2%) from 43.968%

Build # 10179580977

Build Type

Pull #733

github

Committed by

web-flow

Commit Message

Merge d80add92c into 57e1e9056

Pull Request Pull Request #733: feat: implement MlxResetFW to reset the FW on VF changes

Run Details

1 of 63 new or added lines in 6 files covered. (1.59%)

6 existing lines in 2 files now uncovered.

6536 of 14917 relevant lines covered (43.82%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

7.64

/pkg/plugins/mellanox/mellanox_plugin.go

package mellanox

import (
        "fmt"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
        mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox"
)

var PluginName = "mellanox"

type MellanoxPlugin struct {
        PluginName  string
        SpecVersion string
        helpers     helper.HostHelpersInterface
}

var pciAddressesToReset []string
var attributesToChange map[string]mlx.MlxNic
var mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt
var mellanoxNicsSpec map[string]sriovnetworkv1.Interface

// Initialize our plugin and set up initial values
func NewMellanoxPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) {
        mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}

        return &MellanoxPlugin{
                PluginName:  PluginName,
                SpecVersion: "1.0",
                helpers:     helpers,
        }, nil
}

// Name returns the name of the plugin
func (p *MellanoxPlugin) Name() string {
        return p.PluginName
}

// SpecVersion returns the version of the spec expected by the plugin
func (p *MellanoxPlugin) Spec() string {
        return p.SpecVersion
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        log.Log.Info("mellanox plugin OnNodeStateChange()")

        needDrain = false
        needReboot = false
        err = nil
        pciAddressesToReset = []string{}
        attributesToChange = map[string]mlx.MlxNic{}
        mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}
        mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{}
        processedNics := map[string]bool{}

        // fill mellanoxNicsStatus
        for _, iface := range new.Status.Interfaces {
                if iface.Vendor != mlx.MellanoxVendorID {
                        continue
                }

                pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)
                if ifaces, ok := mellanoxNicsStatus[pciPrefix]; ok {
                        ifaces[iface.PciAddress] = iface
                } else {
                        mellanoxNicsStatus[pciPrefix] = map[string]sriovnetworkv1.InterfaceExt{iface.PciAddress: iface}
                }
        }

        // Add only mellanox cards that required changes in the map, to help track dual port NICs
        for _, iface := range new.Spec.Interfaces {
                pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)
                if _, ok := mellanoxNicsStatus[pciPrefix]; !ok {
                        continue
                }
                mellanoxNicsSpec[iface.PciAddress] = iface
        }

        if p.helpers.IsKernelLockdownMode() {
                if len(mellanoxNicsSpec) > 0 {
                        log.Log.Info("Lockdown mode detected, failing on interface update for mellanox devices")
                        return false, false, fmt.Errorf("mellanox device detected when in lockdown mode")
                }
                log.Log.Info("Lockdown mode detected, skpping mellanox nic processing")
                return
        }

        for _, ifaceSpec := range mellanoxNicsSpec {
                pciPrefix := mlx.GetPciAddressPrefix(ifaceSpec.PciAddress)
                // skip processed nics, help not running the same logic 2 times for dual port NICs
                if _, ok := processedNics[pciPrefix]; ok {
                        continue
                }
                processedNics[pciPrefix] = true
                fwCurrent, fwNext, err := p.helpers.GetMlxNicFwData(ifaceSpec.PciAddress)
                if err != nil {
                        return false, false, err
                }

                isDualPort := mlx.IsDualPort(ifaceSpec.PciAddress, mellanoxNicsStatus)
                // Attributes to change
                attrs := &mlx.MlxNic{TotalVfs: -1}
                var changeWithoutReboot bool

                totalVfs, totalVfsNeedReboot, totalVfsChangeWithoutReboot := mlx.HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, isDualPort, mellanoxNicsSpec)
                sriovEnNeedReboot, sriovEnChangeWithoutReboot := mlx.HandleEnableSriov(totalVfs, fwCurrent, fwNext, attrs)
                needReboot = totalVfsNeedReboot || sriovEnNeedReboot
                changeWithoutReboot = totalVfsChangeWithoutReboot || sriovEnChangeWithoutReboot

                needLinkChange, err := mlx.HandleLinkType(pciPrefix, fwCurrent, attrs, mellanoxNicsSpec, mellanoxNicsStatus)
                if err != nil {
                        return false, false, err
                }
                needReboot = needReboot || needLinkChange

                // no FW changes allowed when NIC is externally managed
                if ifaceSpec.ExternallyManaged {
                        if totalVfsNeedReboot || totalVfsChangeWithoutReboot {
                                return false, false, fmt.Errorf(
                                        "interface %s required a change in the TotalVfs but the policy is externally managed failing: firmware TotalVf %d requested TotalVf %d",
                                        ifaceSpec.PciAddress, fwCurrent.TotalVfs, totalVfs)
                        }
                        if needLinkChange {
                                return false, false, fmt.Errorf("change required for link type but the policy is externally managed, failing")
                        }
                }

                if needReboot || changeWithoutReboot {
                        attributesToChange[ifaceSpec.PciAddress] = *attrs
                }

                if needReboot {
                        pciAddressesToReset = append(pciAddressesToReset, ifaceSpec.PciAddress)
                }
        }

        // Set total VFs to 0 for mellanox interfaces with no spec
        for pciPrefix, portsMap := range mellanoxNicsStatus {
                if _, ok := processedNics[pciPrefix]; ok {
                        continue
                }

                // Add the nic to processed Nics to not repeat the process for dual nic ports
                processedNics[pciPrefix] = true
                pciAddress := pciPrefix + "0"

                // Skip devices not configured by the operator
                if p.nicNotConfiguredByOperator(portsMap) {
                        continue
                }

                // Skip externally managed NICs
                if p.nicHasExternallyManagedPFs(portsMap) {
                        continue
                }

                // Skip unsupported devices
                if id := sriovnetworkv1.GetVfDeviceID(portsMap[pciAddress].DeviceID); id == "" {
                        continue
                }

                _, fwNext, err := p.helpers.GetMlxNicFwData(pciAddress)
                if err != nil {
                        return false, false, err
                }

                if fwNext.TotalVfs > 0 || fwNext.EnableSriov {
                        attributesToChange[pciAddress] = mlx.MlxNic{TotalVfs: 0}
                        log.Log.V(2).Info("Changing TotalVfs to 0, doesn't require rebooting", "fwNext.totalVfs", fwNext.TotalVfs)
                }
        }

        if needReboot {
                needDrain = true
        }
        log.Log.V(2).Info("mellanox plugin", "need-drain", needDrain, "need-reboot", needReboot)
        return
}

// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/631
// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs.
func (p *MellanoxPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        return false, nil
}

// Apply config change
func (p *MellanoxPlugin) Apply() error {
        if p.helpers.IsKernelLockdownMode() {
                log.Log.Info("mellanox plugin Apply() - skipping due to lockdown mode")
                return nil
        }
        log.Log.Info("mellanox plugin Apply()")
        if err := p.helpers.MlxConfigFW(attributesToChange); err != nil {
                return err
        }
        if vars.MlxPluginFwReset {
                return p.helpers.MlxResetFW(pciAddressesToReset)
        }
        return nil
}

// nicHasExternallyManagedPFs returns true if one of the ports(interface) of the NIC is marked as externally managed
// in StoreManagerInterface.
func (p *MellanoxPlugin) nicHasExternallyManagedPFs(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {
        for _, iface := range nicPortsMap {
                pfStatus, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)
                if err != nil {
                        log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)
                        continue
                }
                if !exist {
                        continue
                }
                if pfStatus.ExternallyManaged {
                        log.Log.V(2).Info("PF is extenally managed, skip FW TotalVfs reset")
                        return true
                }
        }
        return false
}

// nicNotConfiguredByOperator returns true if one of the ports(interface) of the NIC is not configured by operator
func (p *MellanoxPlugin) nicNotConfiguredByOperator(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {
        for _, iface := range nicPortsMap {
                _, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)
                if err != nil {
                        log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)
                        continue
                }
                if exist {
                        log.Log.V(2).Info("PF configured by the operator", "interface", iface)
                        return true
                }
        }
        return false
}

1	package mellanox
2
3	import (
4	"fmt"
5
6	"sigs.k8s.io/controller-runtime/pkg/log"
7
8	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
9	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
10	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
11	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
12	mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox"
13	)
14
15	var PluginName = "mellanox"
16
17	type MellanoxPlugin struct {
18	PluginName string
19	SpecVersion string
20	helpers helper.HostHelpersInterface
21	}
22
23	var pciAddressesToReset []string
24	var attributesToChange map[string]mlx.MlxNic
25	var mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt
26	var mellanoxNicsSpec map[string]sriovnetworkv1.Interface
27
28	// Initialize our plugin and set up initial values
29	func NewMellanoxPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) {	1✔
30	mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}	1✔
31		1✔
32	return &MellanoxPlugin{	1✔
33	PluginName: PluginName,	1✔
34	SpecVersion: "1.0",	1✔
35	helpers: helpers,	1✔
36	}, nil	1✔
37	}	1✔
38
39	// Name returns the name of the plugin
40	func (p *MellanoxPlugin) Name() string {	1✔
41	return p.PluginName	1✔
42	}	1✔
43
44	// SpecVersion returns the version of the spec expected by the plugin
45	func (p *MellanoxPlugin) Spec() string {	×
46	return p.SpecVersion	×
47	}	×
48
49	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
50	func (p MellanoxPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	×
51	log.Log.Info("mellanox plugin OnNodeStateChange()")	×
52		×
53	needDrain = false	×
54	needReboot = false	×
55	err = nil	×
56	pciAddressesToReset = []string{}	×
NEW 57	attributesToChange = map[string]mlx.MlxNic{}	×
58	mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}	×
59	mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{}	×
60	processedNics := map[string]bool{}	×
61		×
62	// fill mellanoxNicsStatus	×
63	for _, iface := range new.Status.Interfaces {	×
64	if iface.Vendor != mlx.MellanoxVendorID {	×
65	continue	×
66	}
67
68	pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)	×
69	if ifaces, ok := mellanoxNicsStatus[pciPrefix]; ok {	×
70	ifaces[iface.PciAddress] = iface	×
71	} else {	×
72	mellanoxNicsStatus[pciPrefix] = map[string]sriovnetworkv1.InterfaceExt{iface.PciAddress: iface}	×
73	}	×
74	}
75
76	// Add only mellanox cards that required changes in the map, to help track dual port NICs
77	for _, iface := range new.Spec.Interfaces {	×
78	pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)	×
79	if _, ok := mellanoxNicsStatus[pciPrefix]; !ok {	×
80	continue	×
81	}
82	mellanoxNicsSpec[iface.PciAddress] = iface	×
83	}
84
85	if p.helpers.IsKernelLockdownMode() {	×
86	if len(mellanoxNicsSpec) > 0 {	×
87	log.Log.Info("Lockdown mode detected, failing on interface update for mellanox devices")	×
88	return false, false, fmt.Errorf("mellanox device detected when in lockdown mode")	×
89	}	×
90	log.Log.Info("Lockdown mode detected, skpping mellanox nic processing")	×
91	return	×
92	}
93
94	for _, ifaceSpec := range mellanoxNicsSpec {	×
95	pciPrefix := mlx.GetPciAddressPrefix(ifaceSpec.PciAddress)	×
96	// skip processed nics, help not running the same logic 2 times for dual port NICs	×
97	if _, ok := processedNics[pciPrefix]; ok {	×
98	continue	×
99	}
100	processedNics[pciPrefix] = true	×
101	fwCurrent, fwNext, err := p.helpers.GetMlxNicFwData(ifaceSpec.PciAddress)	×
102	if err != nil {	×
103	return false, false, err	×
104	}	×
105
106	isDualPort := mlx.IsDualPort(ifaceSpec.PciAddress, mellanoxNicsStatus)	×
107	// Attributes to change	×
108	attrs := &mlx.MlxNic{TotalVfs: -1}	×
109	var changeWithoutReboot bool	×
110		×
111	totalVfs, totalVfsNeedReboot, totalVfsChangeWithoutReboot := mlx.HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, isDualPort, mellanoxNicsSpec)	×
112	sriovEnNeedReboot, sriovEnChangeWithoutReboot := mlx.HandleEnableSriov(totalVfs, fwCurrent, fwNext, attrs)	×
113	needReboot = totalVfsNeedReboot \|\| sriovEnNeedReboot	×
114	changeWithoutReboot = totalVfsChangeWithoutReboot \|\| sriovEnChangeWithoutReboot	×
115		×
116	needLinkChange, err := mlx.HandleLinkType(pciPrefix, fwCurrent, attrs, mellanoxNicsSpec, mellanoxNicsStatus)	×
117	if err != nil {	×
118	return false, false, err	×
119	}	×
120	needReboot = needReboot \|\| needLinkChange	×
121		×
122	// no FW changes allowed when NIC is externally managed	×
123	if ifaceSpec.ExternallyManaged {	×
124	if totalVfsNeedReboot \|\| totalVfsChangeWithoutReboot {	×
125	return false, false, fmt.Errorf(	×
126	"interface %s required a change in the TotalVfs but the policy is externally managed failing: firmware TotalVf %d requested TotalVf %d",	×
127	ifaceSpec.PciAddress, fwCurrent.TotalVfs, totalVfs)	×
128	}	×
129	if needLinkChange {	×
130	return false, false, fmt.Errorf("change required for link type but the policy is externally managed, failing")	×
131	}	×
132	}
133
134	if needReboot \|\| changeWithoutReboot {	×
135	attributesToChange[ifaceSpec.PciAddress] = *attrs	×
136	}	×
137
NEW 138	if needReboot {	×
NEW 139	pciAddressesToReset = append(pciAddressesToReset, ifaceSpec.PciAddress)	×
NEW 140	}	×
141	}
142
143	// Set total VFs to 0 for mellanox interfaces with no spec
144	for pciPrefix, portsMap := range mellanoxNicsStatus {	×
145	if _, ok := processedNics[pciPrefix]; ok {	×
146	continue	×
147	}
148
149	// Add the nic to processed Nics to not repeat the process for dual nic ports
150	processedNics[pciPrefix] = true	×
151	pciAddress := pciPrefix + "0"	×
152		×
153	// Skip devices not configured by the operator	×
154	if p.nicNotConfiguredByOperator(portsMap) {	×
155	continue	×
156	}
157
158	// Skip externally managed NICs
159	if p.nicHasExternallyManagedPFs(portsMap) {	×
160	continue	×
161	}
162
163	// Skip unsupported devices
164	if id := sriovnetworkv1.GetVfDeviceID(portsMap[pciAddress].DeviceID); id == "" {	×
165	continue	×
166	}
167
168	_, fwNext, err := p.helpers.GetMlxNicFwData(pciAddress)	×
169	if err != nil {	×
170	return false, false, err	×
171	}	×
172
173	if fwNext.TotalVfs > 0 \|\| fwNext.EnableSriov {	×
174	attributesToChange[pciAddress] = mlx.MlxNic{TotalVfs: 0}	×
175	log.Log.V(2).Info("Changing TotalVfs to 0, doesn't require rebooting", "fwNext.totalVfs", fwNext.TotalVfs)	×
176	}	×
177	}
178
179	if needReboot {	×
180	needDrain = true	×
181	}	×
182	log.Log.V(2).Info("mellanox plugin", "need-drain", needDrain, "need-reboot", needReboot)	×
183	return	×
184	}
185
186	// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/631
187	// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs.
188	func (p MellanoxPlugin) CheckStatusChanges(sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	×
189	return false, nil	×
190	}	×
191
192	// Apply config change
193	func (p *MellanoxPlugin) Apply() error {	×
194	if p.helpers.IsKernelLockdownMode() {	×
195	log.Log.Info("mellanox plugin Apply() - skipping due to lockdown mode")	×
196	return nil	×
197	}	×
198	log.Log.Info("mellanox plugin Apply()")	×
199	if err := p.helpers.MlxConfigFW(attributesToChange); err != nil {	×
UNCOV 200	return err	×
UNCOV 201	}	×
UNCOV 202	if vars.MlxPluginFwReset {	×
UNCOV 203	return p.helpers.MlxResetFW(pciAddressesToReset)	×
204	}	×
205	return nil	×
206	}
207
208	// nicHasExternallyManagedPFs returns true if one of the ports(interface) of the NIC is marked as externally managed
209	// in StoreManagerInterface.
UNCOV 210	func (p *MellanoxPlugin) nicHasExternallyManagedPFs(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {	×
211	for _, iface := range nicPortsMap {	×
NEW 212	pfStatus, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)	×
NEW 213	if err != nil {	×
NEW 214	log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)	×
NEW 215	continue	×
216	}
NEW 217	if !exist {	×
NEW 218	continue	×
219	}
220	if pfStatus.ExternallyManaged {	×
221	log.Log.V(2).Info("PF is extenally managed, skip FW TotalVfs reset")	×
222	return true	×
223	}	×
224	}
225	return false	×
226	}
227
228	// nicNotConfiguredByOperator returns true if one of the ports(interface) of the NIC is not configured by operator
229	func (p *MellanoxPlugin) nicNotConfiguredByOperator(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {	×
230	for _, iface := range nicPortsMap {	×
231	_, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)	×
232	if err != nil {	×
233	log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)	×
234	continue	×
235	}
236	if exist {	×
237	log.Log.V(2).Info("PF configured by the operator", "interface", iface)	×
238	return true	×
239	}	×
240	}
241	return false	×
242	}

k8snetworkplumbingwg / sriov-network-operator / 10179580977

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous