14725181549

Committed 29 Apr 2025 06:54AM UTC coverage: 61.624% (+0.01%) from 61.612%

Build # 14725181549

Build Type

Pull #847

github

Committed by

web-flow

Commit Message

Merge 9d4fd607c into b416aa462

Pull Request Pull Request #847: Make kargs.sh work with systemd service and for Ubuntu

Run Details

3 of 5 new or added lines in 1 file covered. (60.0%)

2 existing lines in 1 file now uncovered.

8525 of 13834 relevant lines covered (61.62%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.95

/pkg/plugins/generic/generic_plugin.go

package generic

import (
        "errors"
        "fmt"
        "syscall"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
        hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

var PluginName = "generic"

// driver id
const (
        Vfio = iota
        VirtioVdpa
        VhostVdpa
)

// driver name
const (
        vfioPciDriver    = "vfio_pci"
        virtioVdpaDriver = "virtio_vdpa"
        vhostVdpaDriver  = "vhost_vdpa"
)

// function type for determining if a given driver has to be loaded in the kernel
type needDriver func(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool

type DriverState struct {
        DriverName     string
        DeviceType     string
        VdpaType       string
        NeedDriverFunc needDriver
        DriverLoaded   bool
}

type DriverStateMapType map[uint]*DriverState

type KargStateMapType map[string]bool

type GenericPlugin struct {
        PluginName              string
        DesireState             *sriovnetworkv1.SriovNetworkNodeState
        DriverStateMap          DriverStateMapType
        DesiredKernelArgs       KargStateMapType
        helpers                 helper.HostHelpersInterface
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

type Option = func(c *genericPluginOptions)

// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
func WithSkipVFConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipVFConfiguration = true
        }
}

// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
func WithSkipBridgeConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipBridgeConfiguration = true
        }
}

type genericPluginOptions struct {
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

const daemonScriptsPath = "/bindata/scripts/kargs.sh"
const systemdScriptsPath = "/var/lib/sriov/kargs.sh"

// Initialize our plugin and set up initial values
func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {
        cfg := &genericPluginOptions{}
        for _, o := range options {
                o(cfg)
        }
        driverStateMap := make(map[uint]*DriverState)
        driverStateMap[Vfio] = &DriverState{
                DriverName:     vfioPciDriver,
                DeviceType:     consts.DeviceTypeVfioPci,
                VdpaType:       "",
                NeedDriverFunc: needDriverCheckDeviceType,
                DriverLoaded:   false,
        }
        driverStateMap[VirtioVdpa] = &DriverState{
                DriverName:     virtioVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVirtio,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }
        driverStateMap[VhostVdpa] = &DriverState{
                DriverName:     vhostVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVhost,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }

        // To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc
        // kernel args if they are configured
        kargs, err := helpers.GetCurrentKernelArgs()
        if err != nil {
                return nil, err
        }
        desiredKernelArgs := KargStateMapType{
                consts.KernelArgPciRealloc:    helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc),
                consts.KernelArgIntelIommu:    helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu),
                consts.KernelArgIommuPt:       helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt),
                consts.KernelArgRdmaShared:    false,
                consts.KernelArgRdmaExclusive: false,
        }

        return &GenericPlugin{
                PluginName:              PluginName,
                DriverStateMap:          driverStateMap,
                DesiredKernelArgs:       desiredKernelArgs,
                helpers:                 helpers,
                skipVFConfiguration:     cfg.skipVFConfiguration,
                skipBridgeConfiguration: cfg.skipBridgeConfiguration,
        }, nil
}

// Name returns the name of the plugin
func (p *GenericPlugin) Name() string {
        return p.PluginName
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        log.Log.Info("generic plugin OnNodeStateChange()")
        p.DesireState = new

        needDrain = p.needDrainNode(new.Spec, new.Status)
        needReboot, err = p.needRebootNode(new)
        if err != nil {
                return needDrain, needReboot, err
        }

        if needReboot {
                needDrain = true
        }
        return
}

// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        log.Log.Info("generic-plugin CheckStatusChanges()")

        for _, iface := range current.Spec.Interfaces {
                found := false
                for _, ifaceStatus := range current.Status.Interfaces {
                        // TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632
                        if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {
                                found = true
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)
                                        return true, nil
                                }
                                break
                        }
                }
                if !found {
                        log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)
                }
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {
                        log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")
                        return true, nil
                }
        }

        shouldUpdate, err := p.shouldUpdateKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")
                return false, err
        }

        return shouldUpdate, nil
}

func (p *GenericPlugin) syncDriverState() error {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {
                        log.Log.V(2).Info("loading driver", "name", driverState.DriverName)
                        if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {
                                log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)
                                return err
                        }
                        driverState.DriverLoaded = true
                }
        }
        return nil
}

// Apply config change
func (p *GenericPlugin) Apply() error {
        log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)

        if err := p.syncDriverState(); err != nil {
                return err
        }

        // When calling from systemd do not try to chroot
        if !vars.UsingSystemdMode {
                exit, err := p.helpers.Chroot(consts.Host)
                if err != nil {
                        return err
                }
                defer exit()
        }

        if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,
                p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {
                // Catch the "cannot allocate memory" error and try to use PCI realloc
                if errors.Is(err, syscall.ENOMEM) {
                        p.enableDesiredKernelArgs(consts.KernelArgPciRealloc)
                }
                return err
        }

        if p.shouldConfigureBridges() {
                if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {
                        return err
                }
        }

        return nil
}

func needDriverCheckDeviceType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].DeviceType == driverState.DeviceType {
                                return true
                        }
                }
        }
        return false
}

func needDriverCheckVdpaType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].VdpaType == driverState.VdpaType {
                                return true
                        }
                }
        }
        return false
}

// editKernelArg Tries to add the kernel args via ostree or grubby.
func editKernelArg(helper helper.HostHelpersInterface, mode, karg string) error {
        log.Log.Info("generic plugin editKernelArg()", "mode", mode, "karg", karg)
        script := daemonScriptsPath
        if vars.UsingSystemdMode {
                script = systemdScriptsPath
        }
        _, _, err := helper.RunCommand("/bin/bash", script, mode, karg)
        if err != nil {
                // if grubby is not there log and assume kernel args are set correctly.
                if utils.IsCommandNotFound(err) {
                        log.Log.Error(err, "generic plugin editKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are correct",
                                "kargs", karg)
                        return nil
                }
                log.Log.Error(err, "generic plugin editKernelArg(): fail to edit kernel arg", "karg", karg)
                return err
        }
        return nil
}

// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled.
func (p *GenericPlugin) enableDesiredKernelArgs(karg string) {
        log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg)
        p.DesiredKernelArgs[karg] = true
}

// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled.
func (p *GenericPlugin) disableDesiredKernelArgs(karg string) {
        log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg)
        p.DesiredKernelArgs[karg] = false
}

// shouldUpdateKernelArgs returns true if the DesiredKernelArgs state is not equal to the running kernel args in the system
func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) {
        kargs, err := p.helpers.GetCurrentKernelArgs()
        if err != nil {
                return false, err
        }

        for karg, kargState := range p.DesiredKernelArgs {
                if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) {
                        return true, nil
                }

                if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) {
                        return true, nil
                }
        }
        return false, nil
}

// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) {
        kargs, err := p.helpers.GetCurrentKernelArgs()
        if err != nil {
                return false, err
        }

        needReboot := false
        for karg, kargState := range p.DesiredKernelArgs {
                if kargState {
                        err = editKernelArg(p.helpers, "add", karg)
                        if err != nil {
                                log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)
                                return false, err
                        }

                        if !p.helpers.IsKernelArgsSet(kargs, karg) {
                                needReboot = true
                        }
                } else {
                        err = editKernelArg(p.helpers, "remove", karg)
                        if err != nil {
                                log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg)
                                return false, err
                        }

                        if p.helpers.IsKernelArgsSet(kargs, karg) {
                                needReboot = true
                        }
                }
        }
        return needReboot, nil
}

func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)

        if p.needToUpdateVFs(desired, current) {
                return true
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {
                        log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        for _, ifaceStatus := range current.Interfaces {
                configured := false
                for _, iface := range desired.Interfaces {
                        if iface.PciAddress == ifaceStatus.PciAddress {
                                configured = true
                                if ifaceStatus.NumVfs == 0 {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",
                                                "address", iface.PciAddress)
                                        break
                                }
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",
                                                "address", iface.PciAddress)
                                        return true
                                }
                                log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",
                                        "address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)
                        }
                }
                if !configured && ifaceStatus.NumVfs > 0 {
                        // load the PF info
                        pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)
                        if err != nil {
                                log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if !exist {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if pfStatus.ExternallyManaged {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",
                                "interface", ifaceStatus)
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) shouldConfigureBridges() bool {
        return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration
}

func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) {
        driverState := p.DriverStateMap[Vfio]

        kernelArgFnByCPUVendor := map[hostTypes.CPUVendor]func(){
                hostTypes.CPUVendorIntel: func() {
                        p.enableDesiredKernelArgs(consts.KernelArgIntelIommu)
                        p.enableDesiredKernelArgs(consts.KernelArgIommuPt)
                },
                hostTypes.CPUVendorAMD: func() {
                        p.enableDesiredKernelArgs(consts.KernelArgIommuPt)
                },
        }

        if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                cpuVendor, err := p.helpers.GetCPUVendor()
                if err != nil {
                        log.Log.Error(err, "can't get CPU vendor, falling back to Intel")
                        cpuVendor = hostTypes.CPUVendorIntel
                }

                addKernelArgFn := kernelArgFnByCPUVendor[cpuVendor]
                if addKernelArgFn != nil {
                        addKernelArgFn()
                }
        }
}

func (p *GenericPlugin) configRdmaKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) error {
        if state.Spec.System.RdmaMode == "" {
                p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)
        } else if state.Spec.System.RdmaMode == "shared" {
                p.enableDesiredKernelArgs(consts.KernelArgRdmaShared)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
        } else if state.Spec.System.RdmaMode == "exclusive" {
                p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive)
                p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)
        } else {
                err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode)
                log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma")
                return err
        }

        return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode)
}

func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        needReboot := false

        p.addVfioDesiredKernelArg(state)
        err := p.configRdmaKernelArg(state)
        if err != nil {
                return false, err
        }

        needReboot, err = p.syncDesiredKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")
                return false, err
        }
        if needReboot {
                log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")
        }

        return needReboot, nil
}

// ////////////// for testing purposes only ///////////////////////
func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {
        return p.DriverStateMap
}

func (p *GenericPlugin) loadDriverForTests(state *sriovnetworkv1.SriovNetworkNodeState) {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                        driverState.DriverLoaded = true
                }
        }
}

//////////////////////////////////////////////////////////////////

1	package generic
2
3	import (
4	"errors"
5	"fmt"
6	"syscall"
7
8	"sigs.k8s.io/controller-runtime/pkg/log"
9
10	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
11	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
12	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
13	hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
14	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
15	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
16	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
17	)
18
19	var PluginName = "generic"
20
21	// driver id
22	const (
23	Vfio = iota
24	VirtioVdpa
25	VhostVdpa
26	)
27
28	// driver name
29	const (
30	vfioPciDriver = "vfio_pci"
31	virtioVdpaDriver = "virtio_vdpa"
32	vhostVdpaDriver = "vhost_vdpa"
33	)
34
35	// function type for determining if a given driver has to be loaded in the kernel
36	type needDriver func(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool
37
38	type DriverState struct {
39	DriverName string
40	DeviceType string
41	VdpaType string
42	NeedDriverFunc needDriver
43	DriverLoaded bool
44	}
45
46	type DriverStateMapType map[uint]*DriverState
47
48	type KargStateMapType map[string]bool
49
50	type GenericPlugin struct {
51	PluginName string
52	DesireState *sriovnetworkv1.SriovNetworkNodeState
53	DriverStateMap DriverStateMapType
54	DesiredKernelArgs KargStateMapType
55	helpers helper.HostHelpersInterface
56	skipVFConfiguration bool
57	skipBridgeConfiguration bool
58	}
59
60	type Option = func(c *genericPluginOptions)
61
62	// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
63	// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
64	// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
65	func WithSkipVFConfiguration() Option {	1✔
66	return func(c *genericPluginOptions) {	1✔
67	c.skipVFConfiguration = true	×
68	}	×
69	}
70
71	// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
72	func WithSkipBridgeConfiguration() Option {	1✔
73	return func(c *genericPluginOptions) {	1✔
74	c.skipBridgeConfiguration = true	×
75	}	×
76	}
77
78	type genericPluginOptions struct {
79	skipVFConfiguration bool
80	skipBridgeConfiguration bool
81	}
82
83	const daemonScriptsPath = "/bindata/scripts/kargs.sh"
84	const systemdScriptsPath = "/var/lib/sriov/kargs.sh"
85
86	// Initialize our plugin and set up initial values
87	func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {	1✔
88	cfg := &genericPluginOptions{}	1✔
89	for _, o := range options {	1✔
90	o(cfg)	×
91	}	×
92	driverStateMap := make(map[uint]*DriverState)	1✔
93	driverStateMap[Vfio] = &DriverState{	1✔
94	DriverName: vfioPciDriver,	1✔
95	DeviceType: consts.DeviceTypeVfioPci,	1✔
96	VdpaType: "",	1✔
97	NeedDriverFunc: needDriverCheckDeviceType,	1✔
98	DriverLoaded: false,	1✔
99	}	1✔
100	driverStateMap[VirtioVdpa] = &DriverState{	1✔
101	DriverName: virtioVdpaDriver,	1✔
102	DeviceType: consts.DeviceTypeNetDevice,	1✔
103	VdpaType: consts.VdpaTypeVirtio,	1✔
104	NeedDriverFunc: needDriverCheckVdpaType,	1✔
105	DriverLoaded: false,	1✔
106	}	1✔
107	driverStateMap[VhostVdpa] = &DriverState{	1✔
108	DriverName: vhostVdpaDriver,	1✔
109	DeviceType: consts.DeviceTypeNetDevice,	1✔
110	VdpaType: consts.VdpaTypeVhost,	1✔
111	NeedDriverFunc: needDriverCheckVdpaType,	1✔
112	DriverLoaded: false,	1✔
113	}	1✔
114		1✔
115	// To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc	1✔
116	// kernel args if they are configured	1✔
117	kargs, err := helpers.GetCurrentKernelArgs()	1✔
118	if err != nil {	1✔
119	return nil, err	×
120	}	×
121	desiredKernelArgs := KargStateMapType{	1✔
122	consts.KernelArgPciRealloc: helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc),	1✔
123	consts.KernelArgIntelIommu: helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu),	1✔
124	consts.KernelArgIommuPt: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt),	1✔
125	consts.KernelArgRdmaShared: false,	1✔
126	consts.KernelArgRdmaExclusive: false,	1✔
127	}	1✔
128		1✔
129	return &GenericPlugin{	1✔
130	PluginName: PluginName,	1✔
131	DriverStateMap: driverStateMap,	1✔
132	DesiredKernelArgs: desiredKernelArgs,	1✔
133	helpers: helpers,	1✔
134	skipVFConfiguration: cfg.skipVFConfiguration,	1✔
135	skipBridgeConfiguration: cfg.skipBridgeConfiguration,	1✔
136	}, nil	1✔
137	}
138
139	// Name returns the name of the plugin
140	func (p *GenericPlugin) Name() string {	1✔
141	return p.PluginName	1✔
142	}	1✔
143
144	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
145	func (p GenericPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	1✔
146	log.Log.Info("generic plugin OnNodeStateChange()")	1✔
147	p.DesireState = new	1✔
148		1✔
149	needDrain = p.needDrainNode(new.Spec, new.Status)	1✔
150	needReboot, err = p.needRebootNode(new)	1✔
151	if err != nil {	1✔
152	return needDrain, needReboot, err	×
153	}	×
154
155	if needReboot {	1✔
156	needDrain = true	×
157	}	×
158	return	1✔
159	}
160
161	// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
162	func (p GenericPlugin) CheckStatusChanges(current sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
163	log.Log.Info("generic-plugin CheckStatusChanges()")	1✔
164		1✔
165	for _, iface := range current.Spec.Interfaces {	2✔
166	found := false	1✔
167	for _, ifaceStatus := range current.Status.Interfaces {	2✔
168	// TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632	1✔
169	if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {	2✔
170	found = true	1✔
171	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
172	log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)	1✔
173	return true, nil	1✔
174	}	1✔
175	break	1✔
176	}
177	}
178	if !found {	1✔
179	log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)	×
180	}	×
181	}
182
183	if p.shouldConfigureBridges() {	2✔
184	if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {	2✔
185	log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")	1✔
186	return true, nil	1✔
187	}	1✔
188	}
189
190	shouldUpdate, err := p.shouldUpdateKernelArgs()	1✔
191	if err != nil {	1✔
192	log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")	×
193	return false, err	×
194	}	×
195
196	return shouldUpdate, nil	1✔
197	}
198
199	func (p *GenericPlugin) syncDriverState() error {	1✔
200	for _, driverState := range p.DriverStateMap {	2✔
201	if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {	1✔
202	log.Log.V(2).Info("loading driver", "name", driverState.DriverName)	×
203	if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {	×
204	log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)	×
205	return err	×
206	}	×
207	driverState.DriverLoaded = true	×
208	}
209	}
210	return nil	1✔
211	}
212
213	// Apply config change
214	func (p *GenericPlugin) Apply() error {	1✔
215	log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)	1✔
216		1✔
217	if err := p.syncDriverState(); err != nil {	1✔
218	return err	×
219	}	×
220
221	// When calling from systemd do not try to chroot
222	if !vars.UsingSystemdMode {	2✔
223	exit, err := p.helpers.Chroot(consts.Host)	1✔
224	if err != nil {	1✔
225	return err	×
226	}	×
227	defer exit()	1✔
228	}
229
230	if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,	1✔
231	p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {	1✔
232	// Catch the "cannot allocate memory" error and try to use PCI realloc	×
233	if errors.Is(err, syscall.ENOMEM) {	×
234	p.enableDesiredKernelArgs(consts.KernelArgPciRealloc)	×
235	}	×
236	return err	×
237	}
238
239	if p.shouldConfigureBridges() {	1✔
240	if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {	×
241	return err	×
242	}	×
243	}
244
245	return nil	1✔
246	}
247
248	func needDriverCheckDeviceType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
249	for _, iface := range state.Spec.Interfaces {	2✔
250	for i := range iface.VfGroups {	2✔
251	if iface.VfGroups[i].DeviceType == driverState.DeviceType {	2✔
252	return true	1✔
253	}	1✔
254	}
255	}
256	return false	1✔
257	}
258
259	func needDriverCheckVdpaType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
260	for _, iface := range state.Spec.Interfaces {	2✔
261	for i := range iface.VfGroups {	2✔
262	if iface.VfGroups[i].VdpaType == driverState.VdpaType {	2✔
263	return true	1✔
264	}	1✔
265	}
266	}
267	return false	1✔
268	}
269
270	// editKernelArg Tries to add the kernel args via ostree or grubby.
271	func editKernelArg(helper helper.HostHelpersInterface, mode, karg string) error {	1✔
272	log.Log.Info("generic plugin editKernelArg()", "mode", mode, "karg", karg)	1✔
273	script := daemonScriptsPath	1✔
274	if vars.UsingSystemdMode {	1✔
NEW 275	script = systemdScriptsPath	×
NEW 276	}	×
277	_, _, err := helper.RunCommand("/bin/bash", script, mode, karg)	1✔
278	if err != nil {	1✔
279	// if grubby is not there log and assume kernel args are set correctly.	×
280	if utils.IsCommandNotFound(err) {	×
281	log.Log.Error(err, "generic plugin editKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are correct",	×
282	"kargs", karg)	×
283	return nil	×
284	}	×
285	log.Log.Error(err, "generic plugin editKernelArg(): fail to edit kernel arg", "karg", karg)	×
286	return err	×
287	}
288	return nil	1✔
289	}
290
291	// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled.
292	func (p *GenericPlugin) enableDesiredKernelArgs(karg string) {	1✔
293	log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg)	1✔
294	p.DesiredKernelArgs[karg] = true	1✔
295	}	1✔
296
297	// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled.
298	func (p *GenericPlugin) disableDesiredKernelArgs(karg string) {	1✔
299	log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg)	1✔
300	p.DesiredKernelArgs[karg] = false	1✔
301	}	1✔
302
303	// shouldUpdateKernelArgs returns true if the DesiredKernelArgs state is not equal to the running kernel args in the system
304	func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) {	1✔
305	kargs, err := p.helpers.GetCurrentKernelArgs()	1✔
306	if err != nil {	1✔
307	return false, err	×
308	}	×
309
310	for karg, kargState := range p.DesiredKernelArgs {	2✔
311	if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) {	2✔
312	return true, nil	1✔
313	}	1✔
314
315	if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) {	1✔
316	return true, nil	×
317	}	×
318	}
319	return false, nil	1✔
320	}
321
322	// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
323	func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) {	1✔
324	kargs, err := p.helpers.GetCurrentKernelArgs()	1✔
325	if err != nil {	1✔
326	return false, err	×
327	}	×
328
329	needReboot := false	1✔
330	for karg, kargState := range p.DesiredKernelArgs {	2✔
331	if kargState {	2✔
332	err = editKernelArg(p.helpers, "add", karg)	1✔
333	if err != nil {	1✔
334	log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)	×
335	return false, err	×
336	}	×
337
338	if !p.helpers.IsKernelArgsSet(kargs, karg) {	1✔
339	needReboot = true	×
340	}	×
341	} else {	1✔
342	err = editKernelArg(p.helpers, "remove", karg)	1✔
343	if err != nil {	1✔
344	log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg)	×
345	return false, err	×
346	}	×
347
348	if p.helpers.IsKernelArgsSet(kargs, karg) {	1✔
349	needReboot = true	×
350	}	×
351	}
352	}
353	return needReboot, nil	1✔
354	}
355
356	func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
357	log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)	1✔
358		1✔
359	if p.needToUpdateVFs(desired, current) {	2✔
360	return true	1✔
361	}	1✔
362
363	if p.shouldConfigureBridges() {	2✔
364	if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {	2✔
365	log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")	1✔
366	return true	1✔
367	}	1✔
368	}
369	return false	1✔
370	}
371
372	func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
373	for _, ifaceStatus := range current.Interfaces {	2✔
374	configured := false	1✔
375	for _, iface := range desired.Interfaces {	2✔
376	if iface.PciAddress == ifaceStatus.PciAddress {	2✔
377	configured = true	1✔
378	if ifaceStatus.NumVfs == 0 {	1✔
379	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",	×
380	"address", iface.PciAddress)	×
381	break	×
382	}
383	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
384	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",	1✔
385	"address", iface.PciAddress)	1✔
386	return true	1✔
387	}	1✔
388	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",	1✔
389	"address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)	1✔
390	}
391	}
392	if !configured && ifaceStatus.NumVfs > 0 {	2✔
393	// load the PF info	1✔
394	pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)	1✔
395	if err != nil {	1✔
396	log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",	×
397	"address", ifaceStatus.PciAddress)	×
398	continue	×
399	}
400
401	if !exist {	1✔
402	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",	×
403	"name", ifaceStatus.Name,	×
404	"address", ifaceStatus.PciAddress)	×
405	continue	×
406	}
407
408	if pfStatus.ExternallyManaged {	1✔
409	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",	×
410	"name", ifaceStatus.Name,	×
411	"address", ifaceStatus.PciAddress)	×
412	continue	×
413	}
414
415	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",	1✔
416	"interface", ifaceStatus)	1✔
417	return true	1✔
418	}
419	}
420	return false	1✔
421	}
422
423	func (p *GenericPlugin) shouldConfigureBridges() bool {	1✔
424	return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration	1✔
425	}	1✔
426
427	func (p GenericPlugin) addVfioDesiredKernelArg(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
428	driverState := p.DriverStateMap[Vfio]	1✔
429		1✔
430	kernelArgFnByCPUVendor := map[hostTypes.CPUVendor]func(){	1✔
431	hostTypes.CPUVendorIntel: func() {	2✔
432	p.enableDesiredKernelArgs(consts.KernelArgIntelIommu)	1✔
433	p.enableDesiredKernelArgs(consts.KernelArgIommuPt)	1✔
434	},	1✔
435	hostTypes.CPUVendorAMD: func() {	1✔
436	p.enableDesiredKernelArgs(consts.KernelArgIommuPt)	1✔
437	},	1✔
438	}
439
440	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
441	cpuVendor, err := p.helpers.GetCPUVendor()	1✔
442	if err != nil {	1✔
443	log.Log.Error(err, "can't get CPU vendor, falling back to Intel")	×
444	cpuVendor = hostTypes.CPUVendorIntel	×
445	}	×
446
447	addKernelArgFn := kernelArgFnByCPUVendor[cpuVendor]	1✔
448	if addKernelArgFn != nil {	2✔
449	addKernelArgFn()	1✔
450	}	1✔
451	}
452	}
453
454	func (p GenericPlugin) configRdmaKernelArg(state sriovnetworkv1.SriovNetworkNodeState) error {	1✔
455	if state.Spec.System.RdmaMode == "" {	2✔
456	p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	1✔
457	p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)	1✔
458	} else if state.Spec.System.RdmaMode == "shared" {	3✔
459	p.enableDesiredKernelArgs(consts.KernelArgRdmaShared)	1✔
460	p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	1✔
461	} else if state.Spec.System.RdmaMode == "exclusive" {	3✔
462	p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive)	1✔
463	p.disableDesiredKernelArgs(consts.KernelArgRdmaShared)	1✔
464	} else {	1✔
465	err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode)	×
466	log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma")	×
467	return err	×
468	}	×
469
470	return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode)	1✔
471	}
472
473	func (p GenericPlugin) needRebootNode(state sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
474	needReboot := false	1✔
475		1✔
476	p.addVfioDesiredKernelArg(state)	1✔
477	err := p.configRdmaKernelArg(state)	1✔
478	if err != nil {	1✔
479	return false, err	×
480	}	×
481
482	needReboot, err = p.syncDesiredKernelArgs()	1✔
483	if err != nil {	1✔
484	log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")	×
485	return false, err	×
486	}	×
487	if needReboot {	1✔
488	log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")	×
489	}	×
490
491	return needReboot, nil	1✔
492	}
493
494	// ////////////// for testing purposes only ///////////////////////
495	func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {	1✔
496	return p.DriverStateMap	1✔
497	}	1✔
498
499	func (p GenericPlugin) loadDriverForTests(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
500	for _, driverState := range p.DriverStateMap {	2✔
501	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
502	driverState.DriverLoaded = true	1✔
503	}	1✔
504	}
505	}
506
507	//////////////////////////////////////////////////////////////////

k8snetworkplumbingwg / sriov-network-operator / 14725181549

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous