11601588912

Committed 30 Oct 2024 09:02PM UTC coverage: 45.66% (+0.1%) from 45.538%

Build # 11601588912

Build Type

Pull #799

github

Committed by

web-flow

Commit Message

Merge 88fc3f674 into 5009e9914

Pull Request Pull Request #799: Rdma subsytem mode

Run Details

149 of 289 new or added lines in 12 files covered. (51.56%)

3 existing lines in 2 files now uncovered.

6859 of 15022 relevant lines covered (45.66%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.78

/pkg/plugins/generic/generic_plugin.go

package generic

import (
        "errors"
        "fmt"
        "syscall"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

var PluginName = "generic"

// driver id
const (
        Vfio = iota
        VirtioVdpa
        VhostVdpa
)

// driver name
const (
        vfioPciDriver    = "vfio_pci"
        virtioVdpaDriver = "virtio_vdpa"
        vhostVdpaDriver  = "vhost_vdpa"
)

// function type for determining if a given driver has to be loaded in the kernel
type needDriver func(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool

type DriverState struct {
        DriverName     string
        DeviceType     string
        VdpaType       string
        NeedDriverFunc needDriver
        DriverLoaded   bool
}

type DriverStateMapType map[uint]*DriverState

type KargStateMapType map[string]bool

type GenericPlugin struct {
        PluginName              string
        SpecVersion             string
        DesireState             *sriovnetworkv1.SriovNetworkNodeState
        DriverStateMap          DriverStateMapType
        DesiredKernelArgs       KargStateMapType
        helpers                 helper.HostHelpersInterface
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

type Option = func(c *genericPluginOptions)

// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
func WithSkipVFConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipVFConfiguration = true
        }
}

// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
func WithSkipBridgeConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipBridgeConfiguration = true
        }
}

type genericPluginOptions struct {
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

const scriptsPath = "bindata/scripts/kargs.sh"

// Initialize our plugin and set up initial values
func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {
        cfg := &genericPluginOptions{}
        for _, o := range options {
                o(cfg)
        }
        driverStateMap := make(map[uint]*DriverState)
        driverStateMap[Vfio] = &DriverState{
                DriverName:     vfioPciDriver,
                DeviceType:     consts.DeviceTypeVfioPci,
                VdpaType:       "",
                NeedDriverFunc: needDriverCheckDeviceType,
                DriverLoaded:   false,
        }
        driverStateMap[VirtioVdpa] = &DriverState{
                DriverName:     virtioVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVirtio,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }
        driverStateMap[VhostVdpa] = &DriverState{
                DriverName:     vhostVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVhost,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }

        // To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc
        // kernel args if they are configured
        kargs, err := helpers.GetCurrentKernelArgs()
        if err != nil {
                return nil, err
        }
        desiredKernelArgs := KargStateMapType{
                consts.KernelArgPciRealloc:    helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc),
                consts.KernelArgIntelIommu:    helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu),
                consts.KernelArgIommuPt:       helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt),
                consts.KernelArgRdmaShared:    false,
                consts.KernelArgRdmaExclusive: false,
        }

        return &GenericPlugin{
                PluginName:              PluginName,
                SpecVersion:             "1.0",
                DriverStateMap:          driverStateMap,
                DesiredKernelArgs:       desiredKernelArgs,
                helpers:                 helpers,
                skipVFConfiguration:     cfg.skipVFConfiguration,
                skipBridgeConfiguration: cfg.skipBridgeConfiguration,
        }, nil
}

// Name returns the name of the plugin
func (p *GenericPlugin) Name() string {
        return p.PluginName
}

// Spec returns the version of the spec expected by the plugin
func (p *GenericPlugin) Spec() string {
        return p.SpecVersion
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        log.Log.Info("generic plugin OnNodeStateChange()")
        p.DesireState = new

        needDrain = p.needDrainNode(new.Spec, new.Status)
        needReboot, err = p.needRebootNode(new)
        if err != nil {
                return needDrain, needReboot, err
        }

        if needReboot {
                needDrain = true
        }
        return
}

// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        log.Log.Info("generic-plugin CheckStatusChanges()")

        for _, iface := range current.Spec.Interfaces {
                found := false
                for _, ifaceStatus := range current.Status.Interfaces {
                        // TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632
                        if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {
                                found = true
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)
                                        return true, nil
                                }
                                break
                        }
                }
                if !found {
                        log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)
                }
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {
                        log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")
                        return true, nil
                }
        }

        shouldUpdate, err := p.shouldUpdateKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")
                return false, err
        }

        return shouldUpdate, nil
}

func (p *GenericPlugin) syncDriverState() error {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {
                        log.Log.V(2).Info("loading driver", "name", driverState.DriverName)
                        if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {
                                log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)
                                return err
                        }
                        driverState.DriverLoaded = true
                }
        }
        return nil
}

// Apply config change
func (p *GenericPlugin) Apply() error {
        log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)

        if err := p.syncDriverState(); err != nil {
                return err
        }

        // When calling from systemd do not try to chroot
        if !vars.UsingSystemdMode {
                exit, err := p.helpers.Chroot(consts.Host)
                if err != nil {
                        return err
                }
                defer exit()
        }

        if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,
                p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {
                // Catch the "cannot allocate memory" error and try to use PCI realloc
                if errors.Is(err, syscall.ENOMEM) {
                        p.enableDesiredKernelArgs(consts.KernelArgPciRealloc)
                }
                return err
        }

        if p.shouldConfigureBridges() {
                if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {
                        return err
                }
        }

        return nil
}

func needDriverCheckDeviceType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].DeviceType == driverState.DeviceType {
                                return true
                        }
                }
        }
        return false
}

func needDriverCheckVdpaType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].VdpaType == driverState.VdpaType {
                                return true
                        }
                }
        }
        return false
}

// setKernelArg Tries to add the kernel args via ostree or grubby.
func setKernelArg(helper helper.HostHelpersInterface, mode, karg string) error {
        log.Log.Info("generic plugin setKernelArg()")
        _, _, err := helper.RunCommand("/bin/sh", scriptsPath, mode, karg)
        if err != nil {
                // if grubby is not there log and assume kernel args are set correctly.
                if utils.IsCommandNotFound(err) {
                        log.Log.Error(err, "generic plugin setKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are set",
                                "kargs", karg)
                        return nil
                }
                log.Log.Error(err, "generic plugin setKernelArg(): fail to enable kernel arg", "karg", karg)
                return err
        }
        return nil
}

// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled.
func (p *GenericPlugin) enableDesiredKernelArgs(karg string) {
        log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg)
        p.DesiredKernelArgs[karg] = true
}

// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled.
func (p *GenericPlugin) disableDesiredKernelArgs(karg string) {
        log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg)
        p.DesiredKernelArgs[karg] = false
}

// getKernelArgs gets Kernel arguments on the running sy.
func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) {
        kargs, err := p.helpers.GetCurrentKernelArgs()
        if err != nil {
                return false, err
        }

        for karg, kargState := range p.DesiredKernelArgs {
                if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) {
                        return true, nil
                }

                if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) {
                        return true, nil
                }
        }
        return false, nil
}

// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) {
        kargs, err := p.helpers.GetCurrentKernelArgs()
        if err != nil {
                return false, err
        }

        needReboot := false
        for karg, kargState := range p.DesiredKernelArgs {
                if kargState {
                        err = setKernelArg(p.helpers, "add", karg)
                        if err != nil {
                                log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)
                                return false, err
                        }

                        if !p.helpers.IsKernelArgsSet(kargs, karg) {
                                needReboot = true
                        }
                } else {
                        err = setKernelArg(p.helpers, "remove", karg)
                        if err != nil {
                                log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg)
                                return false, err
                        }

                        if p.helpers.IsKernelArgsSet(kargs, karg) {
                                needReboot = true
                        }
                }
        }
        return needReboot, nil
}

func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)

        if p.needToUpdateVFs(desired, current) {
                return true
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {
                        log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        for _, ifaceStatus := range current.Interfaces {
                configured := false
                for _, iface := range desired.Interfaces {
                        if iface.PciAddress == ifaceStatus.PciAddress {
                                configured = true
                                if ifaceStatus.NumVfs == 0 {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",
                                                "address", iface.PciAddress)
                                        break
                                }
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",
                                                "address", iface.PciAddress)
                                        return true
                                }
                                log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",
                                        "address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)
                        }
                }
                if !configured && ifaceStatus.NumVfs > 0 {
                        // load the PF info
                        pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)
                        if err != nil {
                                log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if !exist {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if pfStatus.ExternallyManaged {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",
                                "interface", ifaceStatus)
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) shouldConfigureBridges() bool {
        return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration
}

func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) {
        driverState := p.DriverStateMap[Vfio]
        if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                p.enableDesiredKernelArgs(consts.KernelArgIntelIommu)
                p.enableDesiredKernelArgs(consts.KernelArgIommuPt)
        }
}

func (p *GenericPlugin) configRdmaKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) error {
        if state.Spec.System.RdmaMode == "" {
                p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)
        } else if state.Spec.System.RdmaMode == "shared" {
                p.enableDesiredKernelArgs(consts.KernelArgRdmaShared)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
        } else if state.Spec.System.RdmaMode == "exclusive" {
                p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)
        } else {
                err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode)
                log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma")
                return err
        }

        return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode)
}

func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        needReboot := false

        p.addVfioDesiredKernelArg(state)
        err := p.configRdmaKernelArg(state)
        if err != nil {
                return false, err
        }

        needReboot, err = p.syncDesiredKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")
                return false, err
        }
        if needReboot {
                log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")
        }

        return needReboot, nil
}

// ////////////// for testing purposes only ///////////////////////
func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {
        return p.DriverStateMap
}

func (p *GenericPlugin) loadDriverForTests(state *sriovnetworkv1.SriovNetworkNodeState) {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                        driverState.DriverLoaded = true
                }
        }
}

//////////////////////////////////////////////////////////////////

1	package generic
2
3	import (
4	"errors"
5	"fmt"
6	"syscall"
7
8	"sigs.k8s.io/controller-runtime/pkg/log"
9
10	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
11	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
12	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
13	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
14	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
15	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
16	)
17
18	var PluginName = "generic"
19
20	// driver id
21	const (
22	Vfio = iota
23	VirtioVdpa
24	VhostVdpa
25	)
26
27	// driver name
28	const (
29	vfioPciDriver = "vfio_pci"
30	virtioVdpaDriver = "virtio_vdpa"
31	vhostVdpaDriver = "vhost_vdpa"
32	)
33
34	// function type for determining if a given driver has to be loaded in the kernel
35	type needDriver func(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool
36
37	type DriverState struct {
38	DriverName string
39	DeviceType string
40	VdpaType string
41	NeedDriverFunc needDriver
42	DriverLoaded bool
43	}
44
45	type DriverStateMapType map[uint]*DriverState
46
47	type KargStateMapType map[string]bool
48
49	type GenericPlugin struct {
50	PluginName string
51	SpecVersion string
52	DesireState *sriovnetworkv1.SriovNetworkNodeState
53	DriverStateMap DriverStateMapType
54	DesiredKernelArgs KargStateMapType
55	helpers helper.HostHelpersInterface
56	skipVFConfiguration bool
57	skipBridgeConfiguration bool
58	}
59
60	type Option = func(c *genericPluginOptions)
61
62	// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
63	// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
64	// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
65	func WithSkipVFConfiguration() Option {	1✔
66	return func(c *genericPluginOptions) {	1✔
67	c.skipVFConfiguration = true	×
68	}	×
69	}
70
71	// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
72	func WithSkipBridgeConfiguration() Option {	1✔
73	return func(c *genericPluginOptions) {	1✔
74	c.skipBridgeConfiguration = true	×
75	}	×
76	}
77
78	type genericPluginOptions struct {
79	skipVFConfiguration bool
80	skipBridgeConfiguration bool
81	}
82
83	const scriptsPath = "bindata/scripts/kargs.sh"
84
85	// Initialize our plugin and set up initial values
86	func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {	1✔
87	cfg := &genericPluginOptions{}	1✔
88	for _, o := range options {	1✔
89	o(cfg)	×
90	}	×
91	driverStateMap := make(map[uint]*DriverState)	1✔
92	driverStateMap[Vfio] = &DriverState{	1✔
93	DriverName: vfioPciDriver,	1✔
94	DeviceType: consts.DeviceTypeVfioPci,	1✔
95	VdpaType: "",	1✔
96	NeedDriverFunc: needDriverCheckDeviceType,	1✔
97	DriverLoaded: false,	1✔
98	}	1✔
99	driverStateMap[VirtioVdpa] = &DriverState{	1✔
100	DriverName: virtioVdpaDriver,	1✔
101	DeviceType: consts.DeviceTypeNetDevice,	1✔
102	VdpaType: consts.VdpaTypeVirtio,	1✔
103	NeedDriverFunc: needDriverCheckVdpaType,	1✔
104	DriverLoaded: false,	1✔
105	}	1✔
106	driverStateMap[VhostVdpa] = &DriverState{	1✔
107	DriverName: vhostVdpaDriver,	1✔
108	DeviceType: consts.DeviceTypeNetDevice,	1✔
109	VdpaType: consts.VdpaTypeVhost,	1✔
110	NeedDriverFunc: needDriverCheckVdpaType,	1✔
111	DriverLoaded: false,	1✔
112	}	1✔
113		1✔
114	// To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc	1✔
115	// kernel args if they are configured	1✔
116	kargs, err := helpers.GetCurrentKernelArgs()	1✔
117	if err != nil {	1✔
NEW 118	return nil, err	×
NEW 119	}	×
120	desiredKernelArgs := KargStateMapType{	1✔
121	consts.KernelArgPciRealloc: helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc),	1✔
122	consts.KernelArgIntelIommu: helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu),	1✔
123	consts.KernelArgIommuPt: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt),	1✔
124	consts.KernelArgRdmaShared: false,	1✔
125	consts.KernelArgRdmaExclusive: false,	1✔
126	}	1✔
127		1✔
128	return &GenericPlugin{	1✔
129	PluginName: PluginName,	1✔
130	SpecVersion: "1.0",	1✔
131	DriverStateMap: driverStateMap,	1✔
132	DesiredKernelArgs: desiredKernelArgs,	1✔
133	helpers: helpers,	1✔
134	skipVFConfiguration: cfg.skipVFConfiguration,	1✔
135	skipBridgeConfiguration: cfg.skipBridgeConfiguration,	1✔
136	}, nil	1✔
137	}
138
139	// Name returns the name of the plugin
140	func (p *GenericPlugin) Name() string {	1✔
141	return p.PluginName	1✔
142	}	1✔
143
144	// Spec returns the version of the spec expected by the plugin
145	func (p *GenericPlugin) Spec() string {	×
146	return p.SpecVersion	×
147	}	×
148
149	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
150	func (p GenericPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	1✔
151	log.Log.Info("generic plugin OnNodeStateChange()")	1✔
152	p.DesireState = new	1✔
153		1✔
154	needDrain = p.needDrainNode(new.Spec, new.Status)	1✔
155	needReboot, err = p.needRebootNode(new)	1✔
156	if err != nil {	1✔
157	return needDrain, needReboot, err	×
158	}	×
159
160	if needReboot {	1✔
161	needDrain = true	×
162	}	×
163	return	1✔
164	}
165
166	// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
167	func (p GenericPlugin) CheckStatusChanges(current sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
168	log.Log.Info("generic-plugin CheckStatusChanges()")	1✔
169		1✔
170	for _, iface := range current.Spec.Interfaces {	2✔
171	found := false	1✔
172	for _, ifaceStatus := range current.Status.Interfaces {	2✔
173	// TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632	1✔
174	if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {	2✔
175	found = true	1✔
176	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
177	log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)	1✔
178	return true, nil	1✔
179	}	1✔
180	break	1✔
181	}
182	}
183	if !found {	1✔
184	log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)	×
185	}	×
186	}
187
188	if p.shouldConfigureBridges() {	2✔
189	if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {	2✔
190	log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")	1✔
191	return true, nil	1✔
192	}	1✔
193	}
194
195	shouldUpdate, err := p.shouldUpdateKernelArgs()	1✔
196	if err != nil {	1✔
197	log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")	×
198	return false, err	×
199	}	×
200
201	return shouldUpdate, nil	1✔
202	}
203
204	func (p *GenericPlugin) syncDriverState() error {	×
205	for _, driverState := range p.DriverStateMap {	×
206	if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {	×
207	log.Log.V(2).Info("loading driver", "name", driverState.DriverName)	×
208	if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {	×
209	log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)	×
210	return err	×
211	}	×
212	driverState.DriverLoaded = true	×
213	}
214	}
215	return nil	×
216	}
217
218	// Apply config change
219	func (p *GenericPlugin) Apply() error {	×
220	log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)	×
221		×
222	if err := p.syncDriverState(); err != nil {	×
223	return err	×
224	}	×
225
226	// When calling from systemd do not try to chroot
227	if !vars.UsingSystemdMode {	×
228	exit, err := p.helpers.Chroot(consts.Host)	×
229	if err != nil {	×
230	return err	×
231	}	×
232	defer exit()	×
233	}
234
235	if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,	×
236	p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {	×
237	// Catch the "cannot allocate memory" error and try to use PCI realloc	×
238	if errors.Is(err, syscall.ENOMEM) {	×
NEW 239	p.enableDesiredKernelArgs(consts.KernelArgPciRealloc)	×
240	}	×
241	return err	×
242	}
243
244	if p.shouldConfigureBridges() {	×
245	if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {	×
246	return err	×
247	}	×
248	}
249
250	return nil	×
251	}
252
253	func needDriverCheckDeviceType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
254	for _, iface := range state.Spec.Interfaces {	2✔
255	for i := range iface.VfGroups {	2✔
256	if iface.VfGroups[i].DeviceType == driverState.DeviceType {	2✔
257	return true	1✔
258	}	1✔
259	}
260	}
261	return false	1✔
262	}
263
264	func needDriverCheckVdpaType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
265	for _, iface := range state.Spec.Interfaces {	2✔
266	for i := range iface.VfGroups {	2✔
267	if iface.VfGroups[i].VdpaType == driverState.VdpaType {	2✔
268	return true	1✔
269	}	1✔
270	}
271	}
272	return false	1✔
273	}
274
275	// setKernelArg Tries to add the kernel args via ostree or grubby.
276	func setKernelArg(helper helper.HostHelpersInterface, mode, karg string) error {	1✔
277	log.Log.Info("generic plugin setKernelArg()")	1✔
278	_, _, err := helper.RunCommand("/bin/sh", scriptsPath, mode, karg)	1✔
279	if err != nil {	1✔
280	// if grubby is not there log and assume kernel args are set correctly.	×
281	if utils.IsCommandNotFound(err) {	×
282	log.Log.Error(err, "generic plugin setKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are set",	×
283	"kargs", karg)	×
NEW 284	return nil	×
285	}	×
286	log.Log.Error(err, "generic plugin setKernelArg(): fail to enable kernel arg", "karg", karg)	×
NEW 287	return err	×
288	}
289	return nil	1✔
290	}
291
292	// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled.
293	func (p *GenericPlugin) enableDesiredKernelArgs(karg string) {	1✔
294	log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg)	1✔
295	p.DesiredKernelArgs[karg] = true	1✔
296	}	1✔
297
298	// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled.
299	func (p *GenericPlugin) disableDesiredKernelArgs(karg string) {	1✔
300	log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg)	1✔
301	p.DesiredKernelArgs[karg] = false	1✔
302	}	1✔
303
304	// getKernelArgs gets Kernel arguments on the running sy.
305	func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) {	1✔
306	kargs, err := p.helpers.GetCurrentKernelArgs()	1✔
307	if err != nil {	1✔
NEW 308	return false, err	×
309	}	×
310
311	for karg, kargState := range p.DesiredKernelArgs {	2✔
312	if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) {	2✔
313	return true, nil	1✔
314	}	1✔
315
316	if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) {	1✔
NEW 317	return true, nil	×
UNCOV 318	}	×
319	}
320	return false, nil	1✔
321	}
322
323	// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
324	func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) {	1✔
325	kargs, err := p.helpers.GetCurrentKernelArgs()	1✔
326	if err != nil {	1✔
NEW 327	return false, err	×
NEW 328	}	×
329
330	needReboot := false	1✔
331	for karg, kargState := range p.DesiredKernelArgs {	2✔
332	if kargState {	1✔
NEW 333	err = setKernelArg(p.helpers, "add", karg)	×
NEW 334	if err != nil {	×
NEW 335	log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)	×
NEW 336	return false, err	×
NEW 337	}	×
338
NEW 339	if !p.helpers.IsKernelArgsSet(kargs, karg) {	×
NEW 340	needReboot = true	×
NEW 341	}	×
342	} else {	1✔
343	err = setKernelArg(p.helpers, "remove", karg)	1✔
344	if err != nil {	1✔
NEW 345	log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg)	×
NEW 346	return false, err	×
NEW 347	}	×
348
349	if p.helpers.IsKernelArgsSet(kargs, karg) {	1✔
NEW 350	needReboot = true	×
NEW 351	}	×
352	}
353	}
354	return needReboot, nil	1✔
355	}
356
357	func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
358	log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)	1✔
359		1✔
360	if p.needToUpdateVFs(desired, current) {	2✔
361	return true	1✔
362	}	1✔
363
364	if p.shouldConfigureBridges() {	2✔
365	if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {	2✔
366	log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")	1✔
367	return true	1✔
368	}	1✔
369	}
370	return false	1✔
371	}
372
373	func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
374	for _, ifaceStatus := range current.Interfaces {	2✔
375	configured := false	1✔
376	for _, iface := range desired.Interfaces {	2✔
377	if iface.PciAddress == ifaceStatus.PciAddress {	2✔
378	configured = true	1✔
379	if ifaceStatus.NumVfs == 0 {	1✔
380	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",	×
381	"address", iface.PciAddress)	×
382	break	×
383	}
384	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
385	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",	1✔
386	"address", iface.PciAddress)	1✔
387	return true	1✔
388	}	1✔
389	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",	1✔
390	"address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)	1✔
391	}
392	}
393	if !configured && ifaceStatus.NumVfs > 0 {	1✔
394	// load the PF info	×
395	pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)	×
396	if err != nil {	×
397	log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",	×
398	"address", ifaceStatus.PciAddress)	×
399	continue	×
400	}
401
402	if !exist {	×
403	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",	×
404	"name", ifaceStatus.Name,	×
405	"address", ifaceStatus.PciAddress)	×
406	continue	×
407	}
408
409	if pfStatus.ExternallyManaged {	×
410	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",	×
411	"name", ifaceStatus.Name,	×
412	"address", ifaceStatus.PciAddress)	×
413	continue	×
414	}
415
416	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",	×
417	"interface", ifaceStatus)	×
418	return true	×
419	}
420	}
421	return false	1✔
422	}
423
424	func (p *GenericPlugin) shouldConfigureBridges() bool {	1✔
425	return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration	1✔
426	}	1✔
427
428	func (p GenericPlugin) addVfioDesiredKernelArg(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
429	driverState := p.DriverStateMap[Vfio]	1✔
430	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
431	p.enableDesiredKernelArgs(consts.KernelArgIntelIommu)	1✔
432	p.enableDesiredKernelArgs(consts.KernelArgIommuPt)	1✔
433	}	1✔
434	}
435
436	func (p GenericPlugin) configRdmaKernelArg(state sriovnetworkv1.SriovNetworkNodeState) error {	1✔
437	if state.Spec.System.RdmaMode == "" {	2✔
438	p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	1✔
439	p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)	1✔
440	} else if state.Spec.System.RdmaMode == "shared" {	1✔
NEW 441	p.enableDesiredKernelArgs(consts.KernelArgRdmaShared)	×
NEW 442	p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	×
NEW 443	} else if state.Spec.System.RdmaMode == "exclusive" {	×
NEW 444	p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	×
NEW 445	p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)	×
NEW 446	} else {	×
NEW 447	err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode)	×
NEW 448	log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma")	×
NEW 449	return err	×
NEW 450	}	×
451
452	return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode)	1✔
453	}
454
455	func (p GenericPlugin) needRebootNode(state sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
456	needReboot := false	1✔
457		1✔
458	p.addVfioDesiredKernelArg(state)	1✔
459	err := p.configRdmaKernelArg(state)	1✔
460	if err != nil {	1✔
461	return false, err	×
462	}	×
463
464	needReboot, err = p.syncDesiredKernelArgs()	1✔
465	if err != nil {	1✔
NEW 466	log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")	×
NEW 467	return false, err	×
NEW 468	}	×
469	if needReboot {	1✔
NEW 470	log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")	×
UNCOV 471	}	×
472
473	return needReboot, nil	1✔
474	}
475
476	// ////////////// for testing purposes only ///////////////////////
477	func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {	1✔
478	return p.DriverStateMap	1✔
479	}	1✔
480
481	func (p GenericPlugin) loadDriverForTests(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
482	for _, driverState := range p.DriverStateMap {	2✔
483	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
484	driverState.DriverLoaded = true	1✔
485	}	1✔
486	}
487	}
488
489	//////////////////////////////////////////////////////////////////

k8snetworkplumbingwg / sriov-network-operator / 11601588912

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous