9939865280

Committed 15 Jul 2024 01:00PM UTC coverage: 40.901% (-0.08%) from 40.981%

Build # 9939865280

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #734 from SchSeba/fix_mlx_plugin

Skip firmware reset on devices the operator doesn't control

Run Details

16 of 53 new or added lines in 5 files covered. (30.19%)

5 existing lines in 1 file now uncovered.

5599 of 13689 relevant lines covered (40.9%)

0.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.16

/pkg/plugins/mellanox/mellanox_plugin.go

package mellanox

import (
        "fmt"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox"
)

var PluginName = "mellanox"

type MellanoxPlugin struct {
        PluginName  string
        SpecVersion string
        helpers     helper.HostHelpersInterface
}

var attributesToChange map[string]mlx.MlxNic
var mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt
var mellanoxNicsSpec map[string]sriovnetworkv1.Interface

// Initialize our plugin and set up initial values
func NewMellanoxPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) {
        mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}

        return &MellanoxPlugin{
                PluginName:  PluginName,
                SpecVersion: "1.0",
                helpers:     helpers,
        }, nil
}

// Name returns the name of the plugin
func (p *MellanoxPlugin) Name() string {
        return p.PluginName
}

// SpecVersion returns the version of the spec expected by the plugin
func (p *MellanoxPlugin) Spec() string {
        return p.SpecVersion
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        log.Log.Info("mellanox plugin OnNodeStateChange()")

        needDrain = false
        needReboot = false
        err = nil
        attributesToChange = map[string]mlx.MlxNic{}
        mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}
        mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{}
        processedNics := map[string]bool{}

        // fill mellanoxNicsStatus
        for _, iface := range new.Status.Interfaces {
                if iface.Vendor != mlx.MellanoxVendorID {
                        continue
                }

                pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)
                if ifaces, ok := mellanoxNicsStatus[pciPrefix]; ok {
                        ifaces[iface.PciAddress] = iface
                } else {
                        mellanoxNicsStatus[pciPrefix] = map[string]sriovnetworkv1.InterfaceExt{iface.PciAddress: iface}
                }
        }

        // Add only mellanox cards that required changes in the map, to help track dual port NICs
        for _, iface := range new.Spec.Interfaces {
                pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)
                if _, ok := mellanoxNicsStatus[pciPrefix]; !ok {
                        continue
                }
                mellanoxNicsSpec[iface.PciAddress] = iface
        }

        if p.helpers.IsKernelLockdownMode() {
                if len(mellanoxNicsSpec) > 0 {
                        log.Log.Info("Lockdown mode detected, failing on interface update for mellanox devices")
                        return false, false, fmt.Errorf("mellanox device detected when in lockdown mode")
                }
                log.Log.Info("Lockdown mode detected, skpping mellanox nic processing")
                return
        }

        for _, ifaceSpec := range mellanoxNicsSpec {
                pciPrefix := mlx.GetPciAddressPrefix(ifaceSpec.PciAddress)
                // skip processed nics, help not running the same logic 2 times for dual port NICs
                if _, ok := processedNics[pciPrefix]; ok {
                        continue
                }
                processedNics[pciPrefix] = true
                fwCurrent, fwNext, err := p.helpers.GetMlxNicFwData(ifaceSpec.PciAddress)
                if err != nil {
                        return false, false, err
                }

                isDualPort := mlx.IsDualPort(ifaceSpec.PciAddress, mellanoxNicsStatus)
                // Attributes to change
                attrs := &mlx.MlxNic{TotalVfs: -1}
                var changeWithoutReboot bool

                totalVfs, totalVfsNeedReboot, totalVfsChangeWithoutReboot := mlx.HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, isDualPort, mellanoxNicsSpec)
                sriovEnNeedReboot, sriovEnChangeWithoutReboot := mlx.HandleEnableSriov(totalVfs, fwCurrent, fwNext, attrs)
                needReboot = totalVfsNeedReboot || sriovEnNeedReboot
                changeWithoutReboot = totalVfsChangeWithoutReboot || sriovEnChangeWithoutReboot

                needLinkChange, err := mlx.HandleLinkType(pciPrefix, fwCurrent, attrs, mellanoxNicsSpec, mellanoxNicsStatus)
                if err != nil {
                        return false, false, err
                }
                needReboot = needReboot || needLinkChange

                // no FW changes allowed when NIC is externally managed
                if ifaceSpec.ExternallyManaged {
                        if totalVfsNeedReboot || totalVfsChangeWithoutReboot {
                                return false, false, fmt.Errorf(
                                        "interface %s required a change in the TotalVfs but the policy is externally managed failing: firmware TotalVf %d requested TotalVf %d",
                                        ifaceSpec.PciAddress, fwCurrent.TotalVfs, totalVfs)
                        }
                        if needLinkChange {
                                return false, false, fmt.Errorf("change required for link type but the policy is externally managed, failing")
                        }
                }

                if needReboot || changeWithoutReboot {
                        attributesToChange[ifaceSpec.PciAddress] = *attrs
                }
        }

        // Set total VFs to 0 for mellanox interfaces with no spec
        for pciPrefix, portsMap := range mellanoxNicsStatus {
                if _, ok := processedNics[pciPrefix]; ok {
                        continue
                }

                // Add the nic to processed Nics to not repeat the process for dual nic ports
                processedNics[pciPrefix] = true
                pciAddress := pciPrefix + "0"

                // Skip devices not configured by the operator
                if p.nicNotConfiguredByOperator(portsMap) {
                        continue
                }

                // Skip externally managed NICs
                if p.nicHasExternallyManagedPFs(portsMap) {
                        continue
                }

                // Skip unsupported devices
                if id := sriovnetworkv1.GetVfDeviceID(portsMap[pciAddress].DeviceID); id == "" {
                        continue
                }

                _, fwNext, err := p.helpers.GetMlxNicFwData(pciAddress)
                if err != nil {
                        return false, false, err
                }

                if fwNext.TotalVfs > 0 || fwNext.EnableSriov {
                        attributesToChange[pciAddress] = mlx.MlxNic{TotalVfs: 0}
                        log.Log.V(2).Info("Changing TotalVfs to 0, doesn't require rebooting", "fwNext.totalVfs", fwNext.TotalVfs)
                }
        }

        if needReboot {
                needDrain = true
        }
        log.Log.V(2).Info("mellanox plugin", "need-drain", needDrain, "need-reboot", needReboot)
        return
}

// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/631
// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs.
func (p *MellanoxPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        return false, nil
}

// Apply config change
func (p *MellanoxPlugin) Apply() error {
        if p.helpers.IsKernelLockdownMode() {
                log.Log.Info("mellanox plugin Apply() - skipping due to lockdown mode")
                return nil
        }
        log.Log.Info("mellanox plugin Apply()")
        return p.helpers.MlxConfigFW(attributesToChange)
}

// nicHasExternallyManagedPFs returns true if one of the ports(interface) of the NIC is marked as externally managed
// in StoreManagerInterface.
func (p *MellanoxPlugin) nicHasExternallyManagedPFs(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {
        for _, iface := range nicPortsMap {
                pfStatus, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)
                if err != nil {
                        log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)
                        continue
                }
                if !exist {
                        continue
                }
                if pfStatus.ExternallyManaged {
                        log.Log.V(2).Info("PF is extenally managed, skip FW TotalVfs reset")
                        return true
                }
        }
        return false
}

// nicNotConfiguredByOperator returns true if one of the ports(interface) of the NIC is not configured by operator
func (p *MellanoxPlugin) nicNotConfiguredByOperator(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {
        for _, iface := range nicPortsMap {
                _, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)
                if err != nil {
                        log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)
                        continue
                }
                if exist {
                        log.Log.V(2).Info("PF configured by the operator", "interface", iface)
                        return true
                }
        }
        return false
}

1	package mellanox
2
3	import (
4	"fmt"
5
6	"sigs.k8s.io/controller-runtime/pkg/log"
7
8	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
9	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
10	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
11	mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox"
12	)
13
14	var PluginName = "mellanox"
15
16	type MellanoxPlugin struct {
17	PluginName string
18	SpecVersion string
19	helpers helper.HostHelpersInterface
20	}
21
22	var attributesToChange map[string]mlx.MlxNic
23	var mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt
24	var mellanoxNicsSpec map[string]sriovnetworkv1.Interface
25
26	// Initialize our plugin and set up initial values
27	func NewMellanoxPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) {	1✔
28	mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}	1✔
29		1✔
30	return &MellanoxPlugin{	1✔
31	PluginName: PluginName,	1✔
32	SpecVersion: "1.0",	1✔
33	helpers: helpers,	1✔
34	}, nil	1✔
35	}	1✔
36
37	// Name returns the name of the plugin
38	func (p *MellanoxPlugin) Name() string {	1✔
39	return p.PluginName	1✔
40	}	1✔
41
42	// SpecVersion returns the version of the spec expected by the plugin
43	func (p *MellanoxPlugin) Spec() string {	×
44	return p.SpecVersion	×
45	}	×
46
47	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
48	func (p MellanoxPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	×
49	log.Log.Info("mellanox plugin OnNodeStateChange()")	×
50		×
51	needDrain = false	×
52	needReboot = false	×
53	err = nil	×
54	attributesToChange = map[string]mlx.MlxNic{}	×
55	mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}	×
56	mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{}	×
57	processedNics := map[string]bool{}	×
58		×
59	// fill mellanoxNicsStatus	×
60	for _, iface := range new.Status.Interfaces {	×
61	if iface.Vendor != mlx.MellanoxVendorID {	×
62	continue	×
63	}
64
65	pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)	×
66	if ifaces, ok := mellanoxNicsStatus[pciPrefix]; ok {	×
67	ifaces[iface.PciAddress] = iface	×
68	} else {	×
69	mellanoxNicsStatus[pciPrefix] = map[string]sriovnetworkv1.InterfaceExt{iface.PciAddress: iface}	×
70	}	×
71	}
72
73	// Add only mellanox cards that required changes in the map, to help track dual port NICs
74	for _, iface := range new.Spec.Interfaces {	×
75	pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress)	×
76	if _, ok := mellanoxNicsStatus[pciPrefix]; !ok {	×
77	continue	×
78	}
79	mellanoxNicsSpec[iface.PciAddress] = iface	×
80	}
81
82	if p.helpers.IsKernelLockdownMode() {	×
83	if len(mellanoxNicsSpec) > 0 {	×
84	log.Log.Info("Lockdown mode detected, failing on interface update for mellanox devices")	×
85	return false, false, fmt.Errorf("mellanox device detected when in lockdown mode")	×
86	}	×
87	log.Log.Info("Lockdown mode detected, skpping mellanox nic processing")	×
88	return	×
89	}
90
91	for _, ifaceSpec := range mellanoxNicsSpec {	×
92	pciPrefix := mlx.GetPciAddressPrefix(ifaceSpec.PciAddress)	×
93	// skip processed nics, help not running the same logic 2 times for dual port NICs	×
94	if _, ok := processedNics[pciPrefix]; ok {	×
95	continue	×
96	}
97	processedNics[pciPrefix] = true	×
98	fwCurrent, fwNext, err := p.helpers.GetMlxNicFwData(ifaceSpec.PciAddress)	×
99	if err != nil {	×
100	return false, false, err	×
101	}	×
102
103	isDualPort := mlx.IsDualPort(ifaceSpec.PciAddress, mellanoxNicsStatus)	×
104	// Attributes to change	×
105	attrs := &mlx.MlxNic{TotalVfs: -1}	×
106	var changeWithoutReboot bool	×
107		×
108	totalVfs, totalVfsNeedReboot, totalVfsChangeWithoutReboot := mlx.HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, isDualPort, mellanoxNicsSpec)	×
109	sriovEnNeedReboot, sriovEnChangeWithoutReboot := mlx.HandleEnableSriov(totalVfs, fwCurrent, fwNext, attrs)	×
110	needReboot = totalVfsNeedReboot \|\| sriovEnNeedReboot	×
111	changeWithoutReboot = totalVfsChangeWithoutReboot \|\| sriovEnChangeWithoutReboot	×
112		×
113	needLinkChange, err := mlx.HandleLinkType(pciPrefix, fwCurrent, attrs, mellanoxNicsSpec, mellanoxNicsStatus)	×
114	if err != nil {	×
115	return false, false, err	×
116	}	×
117	needReboot = needReboot \|\| needLinkChange	×
118		×
119	// no FW changes allowed when NIC is externally managed	×
120	if ifaceSpec.ExternallyManaged {	×
121	if totalVfsNeedReboot \|\| totalVfsChangeWithoutReboot {	×
122	return false, false, fmt.Errorf(	×
123	"interface %s required a change in the TotalVfs but the policy is externally managed failing: firmware TotalVf %d requested TotalVf %d",	×
124	ifaceSpec.PciAddress, fwCurrent.TotalVfs, totalVfs)	×
125	}	×
126	if needLinkChange {	×
127	return false, false, fmt.Errorf("change required for link type but the policy is externally managed, failing")	×
128	}	×
129	}
130
131	if needReboot \|\| changeWithoutReboot {	×
132	attributesToChange[ifaceSpec.PciAddress] = *attrs	×
133	}	×
134	}
135
136	// Set total VFs to 0 for mellanox interfaces with no spec
137	for pciPrefix, portsMap := range mellanoxNicsStatus {	×
138	if _, ok := processedNics[pciPrefix]; ok {	×
139	continue	×
140	}
141
142	// Add the nic to processed Nics to not repeat the process for dual nic ports
143	processedNics[pciPrefix] = true	×
144	pciAddress := pciPrefix + "0"	×
145		×
NEW 146	// Skip devices not configured by the operator	×
NEW 147	if p.nicNotConfiguredByOperator(portsMap) {	×
NEW 148	continue	×
149	}
150
151	// Skip externally managed NICs
152	if p.nicHasExternallyManagedPFs(portsMap) {	×
153	continue	×
154	}
155
156	// Skip unsupported devices
157	if id := sriovnetworkv1.GetVfDeviceID(portsMap[pciAddress].DeviceID); id == "" {	×
158	continue	×
159	}
160
161	_, fwNext, err := p.helpers.GetMlxNicFwData(pciAddress)	×
162	if err != nil {	×
163	return false, false, err	×
164	}	×
165
166	if fwNext.TotalVfs > 0 \|\| fwNext.EnableSriov {	×
167	attributesToChange[pciAddress] = mlx.MlxNic{TotalVfs: 0}	×
168	log.Log.V(2).Info("Changing TotalVfs to 0, doesn't require rebooting", "fwNext.totalVfs", fwNext.TotalVfs)	×
169	}	×
170	}
171
172	if needReboot {	×
173	needDrain = true	×
174	}	×
175	log.Log.V(2).Info("mellanox plugin", "need-drain", needDrain, "need-reboot", needReboot)	×
176	return	×
177	}
178
179	// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/631
180	// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs.
181	func (p MellanoxPlugin) CheckStatusChanges(sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	×
182	return false, nil	×
183	}	×
184
185	// Apply config change
186	func (p *MellanoxPlugin) Apply() error {	×
187	if p.helpers.IsKernelLockdownMode() {	×
188	log.Log.Info("mellanox plugin Apply() - skipping due to lockdown mode")	×
189	return nil	×
190	}	×
191	log.Log.Info("mellanox plugin Apply()")	×
192	return p.helpers.MlxConfigFW(attributesToChange)	×
193	}
194
195	// nicHasExternallyManagedPFs returns true if one of the ports(interface) of the NIC is marked as externally managed
196	// in StoreManagerInterface.
197	func (p *MellanoxPlugin) nicHasExternallyManagedPFs(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {	×
198	for _, iface := range nicPortsMap {	×
199	pfStatus, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)	×
200	if err != nil {	×
201	log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)	×
202	continue	×
203	}
204	if !exist {	×
205	continue	×
206	}
207	if pfStatus.ExternallyManaged {	×
208	log.Log.V(2).Info("PF is extenally managed, skip FW TotalVfs reset")	×
209	return true	×
210	}	×
211	}
212	return false	×
213	}
214
215	// nicNotConfiguredByOperator returns true if one of the ports(interface) of the NIC is not configured by operator
NEW 216	func (p *MellanoxPlugin) nicNotConfiguredByOperator(nicPortsMap map[string]sriovnetworkv1.InterfaceExt) bool {	×
NEW 217	for _, iface := range nicPortsMap {	×
NEW 218	_, exist, err := p.helpers.LoadPfsStatus(iface.PciAddress)	×
NEW 219	if err != nil {	×
NEW 220	log.Log.Error(err, "failed to load PF status from disk", "address", iface.PciAddress)	×
NEW 221	continue	×
222	}
NEW 223	if exist {	×
NEW 224	log.Log.V(2).Info("PF configured by the operator", "interface", iface)	×
NEW 225	return true	×
NEW 226	}	×
227	}
NEW 228	return false	×
229	}

k8snetworkplumbingwg / sriov-network-operator / 9939865280

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous