11723997336

Committed 07 Nov 2024 01:29PM UTC coverage: 45.623% (+0.2%) from 45.458%

Build # 11723997336

Build Type

Pull #799

github

Committed by

web-flow

Commit Message

Merge 02c6b009c into 2b02ba1fe

Pull Request Pull Request #799: Rdma subsytem mode

Run Details

158 of 293 new or added lines in 12 files covered. (53.92%)

3 existing lines in 2 files now uncovered.

6880 of 15080 relevant lines covered (45.62%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.86

/pkg/plugins/generic/generic_plugin.go

package generic

import (
        "errors"
        "fmt"
        "syscall"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
        hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

var PluginName = "generic"

// driver id
const (
        Vfio = iota
        VirtioVdpa
        VhostVdpa
)

// driver name
const (
        vfioPciDriver    = "vfio_pci"
        virtioVdpaDriver = "virtio_vdpa"
        vhostVdpaDriver  = "vhost_vdpa"
)

// function type for determining if a given driver has to be loaded in the kernel
type needDriver func(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool

type DriverState struct {
        DriverName     string
        DeviceType     string
        VdpaType       string
        NeedDriverFunc needDriver
        DriverLoaded   bool
}

type DriverStateMapType map[uint]*DriverState

type KargStateMapType map[string]bool

type GenericPlugin struct {
        PluginName              string
        SpecVersion             string
        DesireState             *sriovnetworkv1.SriovNetworkNodeState
        DriverStateMap          DriverStateMapType
        DesiredKernelArgs       KargStateMapType
        helpers                 helper.HostHelpersInterface
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

type Option = func(c *genericPluginOptions)

// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
func WithSkipVFConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipVFConfiguration = true
        }
}

// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
func WithSkipBridgeConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipBridgeConfiguration = true
        }
}

type genericPluginOptions struct {
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

const scriptsPath = "bindata/scripts/kargs.sh"

// Initialize our plugin and set up initial values
func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {
        cfg := &genericPluginOptions{}
        for _, o := range options {
                o(cfg)
        }
        driverStateMap := make(map[uint]*DriverState)
        driverStateMap[Vfio] = &DriverState{
                DriverName:     vfioPciDriver,
                DeviceType:     consts.DeviceTypeVfioPci,
                VdpaType:       "",
                NeedDriverFunc: needDriverCheckDeviceType,
                DriverLoaded:   false,
        }
        driverStateMap[VirtioVdpa] = &DriverState{
                DriverName:     virtioVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVirtio,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }
        driverStateMap[VhostVdpa] = &DriverState{
                DriverName:     vhostVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVhost,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }

        // To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc
        // kernel args if they are configured
        kargs, err := helpers.GetCurrentKernelArgs()
        if err != nil {
                return nil, err
        }
        desiredKernelArgs := KargStateMapType{
                consts.KernelArgPciRealloc:       helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc),
                consts.KernelArgIntelIommu:       helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu),
                consts.KernelArgIommuPt:          helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt),
                consts.KernelArgIommuPassthrough: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPassthrough),
                consts.KernelArgRdmaShared:       false,
                consts.KernelArgRdmaExclusive:    false,
        }

        return &GenericPlugin{
                PluginName:              PluginName,
                SpecVersion:             "1.0",
                DriverStateMap:          driverStateMap,
                DesiredKernelArgs:       desiredKernelArgs,
                helpers:                 helpers,
                skipVFConfiguration:     cfg.skipVFConfiguration,
                skipBridgeConfiguration: cfg.skipBridgeConfiguration,
        }, nil
}

// Name returns the name of the plugin
func (p *GenericPlugin) Name() string {
        return p.PluginName
}

// Spec returns the version of the spec expected by the plugin
func (p *GenericPlugin) Spec() string {
        return p.SpecVersion
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        log.Log.Info("generic plugin OnNodeStateChange()")
        p.DesireState = new

        needDrain = p.needDrainNode(new.Spec, new.Status)
        needReboot, err = p.needRebootNode(new)
        if err != nil {
                return needDrain, needReboot, err
        }

        if needReboot {
                needDrain = true
        }
        return
}

// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        log.Log.Info("generic-plugin CheckStatusChanges()")

        for _, iface := range current.Spec.Interfaces {
                found := false
                for _, ifaceStatus := range current.Status.Interfaces {
                        // TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632
                        if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {
                                found = true
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)
                                        return true, nil
                                }
                                break
                        }
                }
                if !found {
                        log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)
                }
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {
                        log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")
                        return true, nil
                }
        }

        shouldUpdate, err := p.shouldUpdateKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")
                return false, err
        }

        return shouldUpdate, nil
}

func (p *GenericPlugin) syncDriverState() error {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {
                        log.Log.V(2).Info("loading driver", "name", driverState.DriverName)
                        if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {
                                log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)
                                return err
                        }
                        driverState.DriverLoaded = true
                }
        }
        return nil
}

// Apply config change
func (p *GenericPlugin) Apply() error {
        log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)

        if err := p.syncDriverState(); err != nil {
                return err
        }

        // When calling from systemd do not try to chroot
        if !vars.UsingSystemdMode {
                exit, err := p.helpers.Chroot(consts.Host)
                if err != nil {
                        return err
                }
                defer exit()
        }

        if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,
                p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {
                // Catch the "cannot allocate memory" error and try to use PCI realloc
                if errors.Is(err, syscall.ENOMEM) {
                        p.enableDesiredKernelArgs(consts.KernelArgPciRealloc)
                }
                return err
        }

        if p.shouldConfigureBridges() {
                if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {
                        return err
                }
        }

        return nil
}

func needDriverCheckDeviceType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].DeviceType == driverState.DeviceType {
                                return true
                        }
                }
        }
        return false
}

func needDriverCheckVdpaType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].VdpaType == driverState.VdpaType {
                                return true
                        }
                }
        }
        return false
}

// editKernelArg Tries to add the kernel args via ostree or grubby.
func editKernelArg(helper helper.HostHelpersInterface, mode, karg string) error {
        log.Log.Info("generic plugin editKernelArg()", "mode", mode, "karg", karg)
        _, _, err := helper.RunCommand("/bin/sh", scriptsPath, mode, karg)
        if err != nil {
                // if grubby is not there log and assume kernel args are set correctly.
                if utils.IsCommandNotFound(err) {
                        log.Log.Error(err, "generic plugin editKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are correct",
                                "kargs", karg)
                        return nil
                }
                log.Log.Error(err, "generic plugin editKernelArg(): fail to edit kernel arg", "karg", karg)
                return err
        }
        return nil
}

// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled.
func (p *GenericPlugin) enableDesiredKernelArgs(karg string) {
        log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg)
        p.DesiredKernelArgs[karg] = true
}

// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled.
func (p *GenericPlugin) disableDesiredKernelArgs(karg string) {
        log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg)
        p.DesiredKernelArgs[karg] = false
}

// shouldUpdateKernelArgs returns true if the DesiredKernelArgs state is not equal to the running kernel args in the system
func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) {
        kargs, err := p.helpers.GetCurrentKernelArgs()
        if err != nil {
                return false, err
        }

        for karg, kargState := range p.DesiredKernelArgs {
                if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) {
                        return true, nil
                }

                if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) {
                        return true, nil
                }
        }
        return false, nil
}

// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) {
        kargs, err := p.helpers.GetCurrentKernelArgs()
        if err != nil {
                return false, err
        }

        needReboot := false
        for karg, kargState := range p.DesiredKernelArgs {
                if kargState {
                        err = editKernelArg(p.helpers, "add", karg)
                        if err != nil {
                                log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)
                                return false, err
                        }

                        if !p.helpers.IsKernelArgsSet(kargs, karg) {
                                needReboot = true
                        }
                } else {
                        err = editKernelArg(p.helpers, "remove", karg)
                        if err != nil {
                                log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg)
                                return false, err
                        }

                        if p.helpers.IsKernelArgsSet(kargs, karg) {
                                needReboot = true
                        }
                }
        }
        return needReboot, nil
}

func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)

        if p.needToUpdateVFs(desired, current) {
                return true
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {
                        log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        for _, ifaceStatus := range current.Interfaces {
                configured := false
                for _, iface := range desired.Interfaces {
                        if iface.PciAddress == ifaceStatus.PciAddress {
                                configured = true
                                if ifaceStatus.NumVfs == 0 {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",
                                                "address", iface.PciAddress)
                                        break
                                }
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",
                                                "address", iface.PciAddress)
                                        return true
                                }
                                log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",
                                        "address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)
                        }
                }
                if !configured && ifaceStatus.NumVfs > 0 {
                        // load the PF info
                        pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)
                        if err != nil {
                                log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if !exist {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if pfStatus.ExternallyManaged {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",
                                "interface", ifaceStatus)
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) shouldConfigureBridges() bool {
        return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration
}

func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) {
        driverState := p.DriverStateMap[Vfio]

        kernelArgFnByCPUVendor := map[hostTypes.CPUVendor]func(){
                hostTypes.CPUVendorIntel: func() {
                        p.enableDesiredKernelArgs(consts.KernelArgIntelIommu)
                        p.enableDesiredKernelArgs(consts.KernelArgIommuPt)
                },
                hostTypes.CPUVendorAMD: func() {
                        p.enableDesiredKernelArgs(consts.KernelArgIommuPt)
                },
                hostTypes.CPUVendorARM: func() {
                        p.enableDesiredKernelArgs(consts.KernelArgIommuPassthrough)
                },
        }

        if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                cpuVendor, err := p.helpers.GetCPUVendor()
                if err != nil {
                        log.Log.Error(err, "can't get CPU vendor, falling back to Intel")
                        cpuVendor = hostTypes.CPUVendorIntel
                }

                addKernelArgFn := kernelArgFnByCPUVendor[cpuVendor]
                if addKernelArgFn != nil {
                        addKernelArgFn()
                }
        }
}

func (p *GenericPlugin) configRdmaKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) error {
        if state.Spec.System.RdmaMode == "" {
                p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)
        } else if state.Spec.System.RdmaMode == "shared" {
                p.enableDesiredKernelArgs(consts.KernelArgRdmaShared)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
        } else if state.Spec.System.RdmaMode == "exclusive" {
                p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)
        } else {
                err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode)
                log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma")
                return err
        }

        return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode)
}

func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        needReboot := false

        p.addVfioDesiredKernelArg(state)
        err := p.configRdmaKernelArg(state)
        if err != nil {
                return false, err
        }

        needReboot, err = p.syncDesiredKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")
                return false, err
        }
        if needReboot {
                log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")
        }

        return needReboot, nil
}

// ////////////// for testing purposes only ///////////////////////
func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {
        return p.DriverStateMap
}

func (p *GenericPlugin) loadDriverForTests(state *sriovnetworkv1.SriovNetworkNodeState) {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                        driverState.DriverLoaded = true
                }
        }
}

//////////////////////////////////////////////////////////////////

1	package generic
2
3	import (
4	"errors"
5	"fmt"
6	"syscall"
7
8	"sigs.k8s.io/controller-runtime/pkg/log"
9
10	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
11	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
12	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
13	hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
14	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
15	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
16	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
17	)
18
19	var PluginName = "generic"
20
21	// driver id
22	const (
23	Vfio = iota
24	VirtioVdpa
25	VhostVdpa
26	)
27
28	// driver name
29	const (
30	vfioPciDriver = "vfio_pci"
31	virtioVdpaDriver = "virtio_vdpa"
32	vhostVdpaDriver = "vhost_vdpa"
33	)
34
35	// function type for determining if a given driver has to be loaded in the kernel
36	type needDriver func(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool
37
38	type DriverState struct {
39	DriverName string
40	DeviceType string
41	VdpaType string
42	NeedDriverFunc needDriver
43	DriverLoaded bool
44	}
45
46	type DriverStateMapType map[uint]*DriverState
47
48	type KargStateMapType map[string]bool
49
50	type GenericPlugin struct {
51	PluginName string
52	SpecVersion string
53	DesireState *sriovnetworkv1.SriovNetworkNodeState
54	DriverStateMap DriverStateMapType
55	DesiredKernelArgs KargStateMapType
56	helpers helper.HostHelpersInterface
57	skipVFConfiguration bool
58	skipBridgeConfiguration bool
59	}
60
61	type Option = func(c *genericPluginOptions)
62
63	// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
64	// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
65	// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
66	func WithSkipVFConfiguration() Option {	1✔
67	return func(c *genericPluginOptions) {	1✔
68	c.skipVFConfiguration = true	×
69	}	×
70	}
71
72	// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
73	func WithSkipBridgeConfiguration() Option {	1✔
74	return func(c *genericPluginOptions) {	1✔
75	c.skipBridgeConfiguration = true	×
76	}	×
77	}
78
79	type genericPluginOptions struct {
80	skipVFConfiguration bool
81	skipBridgeConfiguration bool
82	}
83
84	const scriptsPath = "bindata/scripts/kargs.sh"
85
86	// Initialize our plugin and set up initial values
87	func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {	1✔
88	cfg := &genericPluginOptions{}	1✔
89	for _, o := range options {	1✔
90	o(cfg)	×
91	}	×
92	driverStateMap := make(map[uint]*DriverState)	1✔
93	driverStateMap[Vfio] = &DriverState{	1✔
94	DriverName: vfioPciDriver,	1✔
95	DeviceType: consts.DeviceTypeVfioPci,	1✔
96	VdpaType: "",	1✔
97	NeedDriverFunc: needDriverCheckDeviceType,	1✔
98	DriverLoaded: false,	1✔
99	}	1✔
100	driverStateMap[VirtioVdpa] = &DriverState{	1✔
101	DriverName: virtioVdpaDriver,	1✔
102	DeviceType: consts.DeviceTypeNetDevice,	1✔
103	VdpaType: consts.VdpaTypeVirtio,	1✔
104	NeedDriverFunc: needDriverCheckVdpaType,	1✔
105	DriverLoaded: false,	1✔
106	}	1✔
107	driverStateMap[VhostVdpa] = &DriverState{	1✔
108	DriverName: vhostVdpaDriver,	1✔
109	DeviceType: consts.DeviceTypeNetDevice,	1✔
110	VdpaType: consts.VdpaTypeVhost,	1✔
111	NeedDriverFunc: needDriverCheckVdpaType,	1✔
112	DriverLoaded: false,	1✔
113	}	1✔
114		1✔
115	// To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc	1✔
116	// kernel args if they are configured	1✔
117	kargs, err := helpers.GetCurrentKernelArgs()	1✔
118	if err != nil {	1✔
NEW 119	return nil, err	×
NEW 120	}	×
121	desiredKernelArgs := KargStateMapType{	1✔
122	consts.KernelArgPciRealloc: helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc),	1✔
123	consts.KernelArgIntelIommu: helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu),	1✔
124	consts.KernelArgIommuPt: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt),	1✔
125	consts.KernelArgIommuPassthrough: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPassthrough),	1✔
126	consts.KernelArgRdmaShared: false,	1✔
127	consts.KernelArgRdmaExclusive: false,	1✔
128	}	1✔
129		1✔
130	return &GenericPlugin{	1✔
131	PluginName: PluginName,	1✔
132	SpecVersion: "1.0",	1✔
133	DriverStateMap: driverStateMap,	1✔
134	DesiredKernelArgs: desiredKernelArgs,	1✔
135	helpers: helpers,	1✔
136	skipVFConfiguration: cfg.skipVFConfiguration,	1✔
137	skipBridgeConfiguration: cfg.skipBridgeConfiguration,	1✔
138	}, nil	1✔
139	}
140
141	// Name returns the name of the plugin
142	func (p *GenericPlugin) Name() string {	1✔
143	return p.PluginName	1✔
144	}	1✔
145
146	// Spec returns the version of the spec expected by the plugin
147	func (p *GenericPlugin) Spec() string {	×
148	return p.SpecVersion	×
149	}	×
150
151	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
152	func (p GenericPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	1✔
153	log.Log.Info("generic plugin OnNodeStateChange()")	1✔
154	p.DesireState = new	1✔
155		1✔
156	needDrain = p.needDrainNode(new.Spec, new.Status)	1✔
157	needReboot, err = p.needRebootNode(new)	1✔
158	if err != nil {	1✔
159	return needDrain, needReboot, err	×
160	}	×
161
162	if needReboot {	1✔
163	needDrain = true	×
164	}	×
165	return	1✔
166	}
167
168	// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
169	func (p GenericPlugin) CheckStatusChanges(current sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
170	log.Log.Info("generic-plugin CheckStatusChanges()")	1✔
171		1✔
172	for _, iface := range current.Spec.Interfaces {	2✔
173	found := false	1✔
174	for _, ifaceStatus := range current.Status.Interfaces {	2✔
175	// TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632	1✔
176	if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {	2✔
177	found = true	1✔
178	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
179	log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)	1✔
180	return true, nil	1✔
181	}	1✔
182	break	1✔
183	}
184	}
185	if !found {	1✔
186	log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)	×
187	}	×
188	}
189
190	if p.shouldConfigureBridges() {	2✔
191	if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {	2✔
192	log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")	1✔
193	return true, nil	1✔
194	}	1✔
195	}
196
197	shouldUpdate, err := p.shouldUpdateKernelArgs()	1✔
198	if err != nil {	1✔
199	log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")	×
200	return false, err	×
201	}	×
202
203	return shouldUpdate, nil	1✔
204	}
205
206	func (p *GenericPlugin) syncDriverState() error {	×
207	for _, driverState := range p.DriverStateMap {	×
208	if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {	×
209	log.Log.V(2).Info("loading driver", "name", driverState.DriverName)	×
210	if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {	×
211	log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)	×
212	return err	×
213	}	×
214	driverState.DriverLoaded = true	×
215	}
216	}
217	return nil	×
218	}
219
220	// Apply config change
221	func (p *GenericPlugin) Apply() error {	×
222	log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)	×
223		×
224	if err := p.syncDriverState(); err != nil {	×
225	return err	×
226	}	×
227
228	// When calling from systemd do not try to chroot
229	if !vars.UsingSystemdMode {	×
230	exit, err := p.helpers.Chroot(consts.Host)	×
231	if err != nil {	×
232	return err	×
233	}	×
234	defer exit()	×
235	}
236
237	if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,	×
238	p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {	×
239	// Catch the "cannot allocate memory" error and try to use PCI realloc	×
240	if errors.Is(err, syscall.ENOMEM) {	×
NEW 241	p.enableDesiredKernelArgs(consts.KernelArgPciRealloc)	×
242	}	×
243	return err	×
244	}
245
246	if p.shouldConfigureBridges() {	×
247	if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {	×
248	return err	×
249	}	×
250	}
251
252	return nil	×
253	}
254
255	func needDriverCheckDeviceType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
256	for _, iface := range state.Spec.Interfaces {	2✔
257	for i := range iface.VfGroups {	2✔
258	if iface.VfGroups[i].DeviceType == driverState.DeviceType {	2✔
259	return true	1✔
260	}	1✔
261	}
262	}
263	return false	1✔
264	}
265
266	func needDriverCheckVdpaType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
267	for _, iface := range state.Spec.Interfaces {	2✔
268	for i := range iface.VfGroups {	2✔
269	if iface.VfGroups[i].VdpaType == driverState.VdpaType {	2✔
270	return true	1✔
271	}	1✔
272	}
273	}
274	return false	1✔
275	}
276
277	// editKernelArg Tries to add the kernel args via ostree or grubby.
278	func editKernelArg(helper helper.HostHelpersInterface, mode, karg string) error {	1✔
279	log.Log.Info("generic plugin editKernelArg()", "mode", mode, "karg", karg)	1✔
280	_, _, err := helper.RunCommand("/bin/sh", scriptsPath, mode, karg)	1✔
281	if err != nil {	1✔
282	// if grubby is not there log and assume kernel args are set correctly.	×
283	if utils.IsCommandNotFound(err) {	×
NEW 284	log.Log.Error(err, "generic plugin editKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are correct",	×
285	"kargs", karg)	×
NEW 286	return nil	×
287	}	×
NEW 288	log.Log.Error(err, "generic plugin editKernelArg(): fail to edit kernel arg", "karg", karg)	×
NEW 289	return err	×
290	}
291	return nil	1✔
292	}
293
294	// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled.
295	func (p *GenericPlugin) enableDesiredKernelArgs(karg string) {	1✔
296	log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg)	1✔
297	p.DesiredKernelArgs[karg] = true	1✔
298	}	1✔
299
300	// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled.
301	func (p *GenericPlugin) disableDesiredKernelArgs(karg string) {	1✔
302	log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg)	1✔
303	p.DesiredKernelArgs[karg] = false	1✔
304	}	1✔
305
306	// shouldUpdateKernelArgs returns true if the DesiredKernelArgs state is not equal to the running kernel args in the system
307	func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) {	1✔
308	kargs, err := p.helpers.GetCurrentKernelArgs()	1✔
309	if err != nil {	1✔
NEW 310	return false, err	×
311	}	×
312
313	for karg, kargState := range p.DesiredKernelArgs {	2✔
314	if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) {	2✔
315	return true, nil	1✔
316	}	1✔
317
318	if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) {	1✔
NEW 319	return true, nil	×
UNCOV 320	}	×
321	}
322	return false, nil	1✔
323	}
324
325	// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
326	func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) {	1✔
327	kargs, err := p.helpers.GetCurrentKernelArgs()	1✔
328	if err != nil {	1✔
NEW 329	return false, err	×
NEW 330	}	×
331
332	needReboot := false	1✔
333	for karg, kargState := range p.DesiredKernelArgs {	2✔
334	if kargState {	1✔
NEW 335	err = editKernelArg(p.helpers, "add", karg)	×
NEW 336	if err != nil {	×
NEW 337	log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)	×
NEW 338	return false, err	×
NEW 339	}	×
340
NEW 341	if !p.helpers.IsKernelArgsSet(kargs, karg) {	×
NEW 342	needReboot = true	×
NEW 343	}	×
344	} else {	1✔
345	err = editKernelArg(p.helpers, "remove", karg)	1✔
346	if err != nil {	1✔
NEW 347	log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg)	×
NEW 348	return false, err	×
NEW 349	}	×
350
351	if p.helpers.IsKernelArgsSet(kargs, karg) {	1✔
NEW 352	needReboot = true	×
NEW 353	}	×
354	}
355	}
356	return needReboot, nil	1✔
357	}
358
359	func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
360	log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)	1✔
361		1✔
362	if p.needToUpdateVFs(desired, current) {	2✔
363	return true	1✔
364	}	1✔
365
366	if p.shouldConfigureBridges() {	2✔
367	if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {	2✔
368	log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")	1✔
369	return true	1✔
370	}	1✔
371	}
372	return false	1✔
373	}
374
375	func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
376	for _, ifaceStatus := range current.Interfaces {	2✔
377	configured := false	1✔
378	for _, iface := range desired.Interfaces {	2✔
379	if iface.PciAddress == ifaceStatus.PciAddress {	2✔
380	configured = true	1✔
381	if ifaceStatus.NumVfs == 0 {	1✔
382	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",	×
383	"address", iface.PciAddress)	×
384	break	×
385	}
386	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
387	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",	1✔
388	"address", iface.PciAddress)	1✔
389	return true	1✔
390	}	1✔
391	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",	1✔
392	"address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)	1✔
393	}
394	}
395	if !configured && ifaceStatus.NumVfs > 0 {	1✔
396	// load the PF info	×
397	pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)	×
398	if err != nil {	×
399	log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",	×
400	"address", ifaceStatus.PciAddress)	×
401	continue	×
402	}
403
404	if !exist {	×
405	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",	×
406	"name", ifaceStatus.Name,	×
407	"address", ifaceStatus.PciAddress)	×
408	continue	×
409	}
410
411	if pfStatus.ExternallyManaged {	×
412	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",	×
413	"name", ifaceStatus.Name,	×
414	"address", ifaceStatus.PciAddress)	×
415	continue	×
416	}
417
418	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",	×
419	"interface", ifaceStatus)	×
420	return true	×
421	}
422	}
423	return false	1✔
424	}
425
426	func (p *GenericPlugin) shouldConfigureBridges() bool {	1✔
427	return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration	1✔
428	}	1✔
429
430	func (p GenericPlugin) addVfioDesiredKernelArg(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
431	driverState := p.DriverStateMap[Vfio]	1✔
432		1✔
433	kernelArgFnByCPUVendor := map[hostTypes.CPUVendor]func(){	1✔
434	hostTypes.CPUVendorIntel: func() {	2✔
435	p.enableDesiredKernelArgs(consts.KernelArgIntelIommu)	1✔
436	p.enableDesiredKernelArgs(consts.KernelArgIommuPt)	1✔
437	},	1✔
438	hostTypes.CPUVendorAMD: func() {	1✔
439	p.enableDesiredKernelArgs(consts.KernelArgIommuPt)	1✔
440	},	1✔
441	hostTypes.CPUVendorARM: func() {	1✔
442	p.enableDesiredKernelArgs(consts.KernelArgIommuPassthrough)	1✔
443	},	1✔
444	}
445
446	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
447	cpuVendor, err := p.helpers.GetCPUVendor()	1✔
448	if err != nil {	1✔
449	log.Log.Error(err, "can't get CPU vendor, falling back to Intel")	×
450	cpuVendor = hostTypes.CPUVendorIntel	×
451	}	×
452
453	addKernelArgFn := kernelArgFnByCPUVendor[cpuVendor]	1✔
454	if addKernelArgFn != nil {	2✔
455	addKernelArgFn()	1✔
456	}	1✔
457	}
458	}
459
460	func (p GenericPlugin) configRdmaKernelArg(state sriovnetworkv1.SriovNetworkNodeState) error {	1✔
461	if state.Spec.System.RdmaMode == "" {	2✔
462	p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	1✔
463	p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)	1✔
464	} else if state.Spec.System.RdmaMode == "shared" {	3✔
465	p.enableDesiredKernelArgs(consts.KernelArgRdmaShared)	1✔
466	p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	1✔
467	} else if state.Spec.System.RdmaMode == "exclusive" {	3✔
468	p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	1✔
469	p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)	1✔
470	} else {	1✔
NEW 471	err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode)	×
NEW 472	log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma")	×
NEW 473	return err	×
NEW 474	}	×
475
476	return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode)	1✔
477	}
478
479	func (p GenericPlugin) needRebootNode(state sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
480	needReboot := false	1✔
481		1✔
482	p.addVfioDesiredKernelArg(state)	1✔
483	err := p.configRdmaKernelArg(state)	1✔
484	if err != nil {	1✔
485	return false, err	×
486	}	×
487
488	needReboot, err = p.syncDesiredKernelArgs()	1✔
489	if err != nil {	1✔
NEW 490	log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")	×
NEW 491	return false, err	×
NEW 492	}	×
493	if needReboot {	1✔
NEW 494	log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")	×
UNCOV 495	}	×
496
497	return needReboot, nil	1✔
498	}
499
500	// ////////////// for testing purposes only ///////////////////////
501	func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {	1✔
502	return p.DriverStateMap	1✔
503	}	1✔
504
505	func (p GenericPlugin) loadDriverForTests(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
506	for _, driverState := range p.DriverStateMap {	2✔
507	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
508	driverState.DriverLoaded = true	1✔
509	}	1✔
510	}
511	}
512
513	//////////////////////////////////////////////////////////////////

k8snetworkplumbingwg / sriov-network-operator / 11723997336

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous