11591346372

Committed 30 Oct 2024 10:29AM UTC coverage: 45.465% (-0.07%) from 45.538%

Build # 11591346372

Build Type

Pull #796

github

Committed by

web-flow

Commit Message

Merge 5522c9610 into 5009e9914

Pull Request Pull Request #796: kernel: Set arguments based on CPU architecture

Run Details

44 of 85 new or added lines in 9 files covered. (51.76%)

10 existing lines in 2 files now uncovered.

6792 of 14939 relevant lines covered (45.46%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.28

/pkg/plugins/generic/generic_plugin.go

package generic

import (
        "bytes"
        "errors"
        "os/exec"
        "strconv"
        "strings"
        "syscall"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
        hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

var PluginName = "generic"

// driver id
const (
        Vfio = iota
        VirtioVdpa
        VhostVdpa
)

// driver name
const (
        vfioPciDriver    = "vfio_pci"
        virtioVdpaDriver = "virtio_vdpa"
        vhostVdpaDriver  = "vhost_vdpa"
)

// function type for determining if a given driver has to be loaded in the kernel
type needDriver func(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool

type DriverState struct {
        DriverName     string
        DeviceType     string
        VdpaType       string
        NeedDriverFunc needDriver
        DriverLoaded   bool
}

type DriverStateMapType map[uint]*DriverState

type GenericPlugin struct {
        PluginName              string
        SpecVersion             string
        DesireState             *sriovnetworkv1.SriovNetworkNodeState
        DriverStateMap          DriverStateMapType
        DesiredKernelArgs       map[string]bool
        helpers                 helper.HostHelpersInterface
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

type Option = func(c *genericPluginOptions)

// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
func WithSkipVFConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipVFConfiguration = true
        }
}

// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
func WithSkipBridgeConfiguration() Option {
        return func(c *genericPluginOptions) {
                c.skipBridgeConfiguration = true
        }
}

type genericPluginOptions struct {
        skipVFConfiguration     bool
        skipBridgeConfiguration bool
}

const scriptsPath = "bindata/scripts/enable-kargs.sh"

// Initialize our plugin and set up initial values
func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {
        cfg := &genericPluginOptions{}
        for _, o := range options {
                o(cfg)
        }
        driverStateMap := make(map[uint]*DriverState)
        driverStateMap[Vfio] = &DriverState{
                DriverName:     vfioPciDriver,
                DeviceType:     consts.DeviceTypeVfioPci,
                VdpaType:       "",
                NeedDriverFunc: needDriverCheckDeviceType,
                DriverLoaded:   false,
        }
        driverStateMap[VirtioVdpa] = &DriverState{
                DriverName:     virtioVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVirtio,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }
        driverStateMap[VhostVdpa] = &DriverState{
                DriverName:     vhostVdpaDriver,
                DeviceType:     consts.DeviceTypeNetDevice,
                VdpaType:       consts.VdpaTypeVhost,
                NeedDriverFunc: needDriverCheckVdpaType,
                DriverLoaded:   false,
        }
        return &GenericPlugin{
                PluginName:              PluginName,
                SpecVersion:             "1.0",
                DriverStateMap:          driverStateMap,
                DesiredKernelArgs:       make(map[string]bool),
                helpers:                 helpers,
                skipVFConfiguration:     cfg.skipVFConfiguration,
                skipBridgeConfiguration: cfg.skipBridgeConfiguration,
        }, nil
}

// Name returns the name of the plugin
func (p *GenericPlugin) Name() string {
        return p.PluginName
}

// Spec returns the version of the spec expected by the plugin
func (p *GenericPlugin) Spec() string {
        return p.SpecVersion
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        log.Log.Info("generic plugin OnNodeStateChange()")
        p.DesireState = new

        needDrain = p.needDrainNode(new.Spec, new.Status)
        needReboot, err = p.needRebootNode(new)
        if err != nil {
                return needDrain, needReboot, err
        }

        if needReboot {
                needDrain = true
        }
        return
}

// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        log.Log.Info("generic-plugin CheckStatusChanges()")

        for _, iface := range current.Spec.Interfaces {
                found := false
                for _, ifaceStatus := range current.Status.Interfaces {
                        // TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632
                        if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {
                                found = true
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)
                                        return true, nil
                                }
                                break
                        }
                }
                if !found {
                        log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)
                }
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {
                        log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")
                        return true, nil
                }
        }

        missingKernelArgs, err := p.getMissingKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")
                return false, err
        }

        if len(missingKernelArgs) != 0 {
                log.Log.V(0).Info("generic-plugin CheckStatusChanges(): kernel args missing",
                        "kernelArgs", missingKernelArgs)
        }

        return len(missingKernelArgs) != 0, nil
}

func (p *GenericPlugin) syncDriverState() error {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {
                        log.Log.V(2).Info("loading driver", "name", driverState.DriverName)
                        if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {
                                log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)
                                return err
                        }
                        driverState.DriverLoaded = true
                }
        }
        return nil
}

// Apply config change
func (p *GenericPlugin) Apply() error {
        log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)

        if err := p.syncDriverState(); err != nil {
                return err
        }

        // When calling from systemd do not try to chroot
        if !vars.UsingSystemdMode {
                exit, err := p.helpers.Chroot(consts.Host)
                if err != nil {
                        return err
                }
                defer exit()
        }

        if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,
                p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {
                // Catch the "cannot allocate memory" error and try to use PCI realloc
                if errors.Is(err, syscall.ENOMEM) {
                        p.addToDesiredKernelArgs(consts.KernelArgPciRealloc)
                }
                return err
        }

        if p.shouldConfigureBridges() {
                if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {
                        return err
                }
        }

        return nil
}

func needDriverCheckDeviceType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].DeviceType == driverState.DeviceType {
                                return true
                        }
                }
        }
        return false
}

func needDriverCheckVdpaType(state *sriovnetworkv1.SriovNetworkNodeState, driverState *DriverState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].VdpaType == driverState.VdpaType {
                                return true
                        }
                }
        }
        return false
}

// setKernelArg Tries to add the kernel args via ostree or grubby.
func setKernelArg(karg string) (bool, error) {
        log.Log.Info("generic plugin setKernelArg()")
        var stdout, stderr bytes.Buffer
        cmd := exec.Command("/bin/sh", scriptsPath, karg)
        cmd.Stdout = &stdout
        cmd.Stderr = &stderr

        if err := cmd.Run(); err != nil {
                // if grubby is not there log and assume kernel args are set correctly.
                if utils.IsCommandNotFound(err) {
                        log.Log.Error(err, "generic plugin setKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are set",
                                "kargs", karg)
                        return false, nil
                }
                log.Log.Error(err, "generic plugin setKernelArg(): fail to enable kernel arg", "karg", karg)
                return false, err
        }

        i, err := strconv.Atoi(strings.TrimSpace(stdout.String()))
        if err == nil {
                if i > 0 {
                        log.Log.Info("generic plugin setKernelArg(): need to reboot node for kernel arg", "karg", karg)
                        return true, nil
                }
        }
        return false, err
}

// addToDesiredKernelArgs Should be called to queue a kernel arg to be added to the node.
func (p *GenericPlugin) addToDesiredKernelArgs(karg string) {
        if _, ok := p.DesiredKernelArgs[karg]; !ok {
                log.Log.Info("generic plugin addToDesiredKernelArgs(): Adding to desired kernel arg", "karg", karg)
                p.DesiredKernelArgs[karg] = false
        }
}

// getMissingKernelArgs gets Kernel arguments that have not been set.
func (p *GenericPlugin) getMissingKernelArgs() ([]string, error) {
        missingArgs := make([]string, 0, len(p.DesiredKernelArgs))
        if len(p.DesiredKernelArgs) == 0 {
                return nil, nil
        }

        kargs, err := p.helpers.GetCurrentKernelArgs()
        if err != nil {
                return nil, err
        }

        for desiredKarg := range p.DesiredKernelArgs {
                if !p.helpers.IsKernelArgsSet(kargs, desiredKarg) {
                        missingArgs = append(missingArgs, desiredKarg)
                }
        }
        return missingArgs, nil
}

// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
func (p *GenericPlugin) syncDesiredKernelArgs(kargs []string) (bool, error) {
        needReboot := false

        for _, karg := range kargs {
                if p.DesiredKernelArgs[karg] {
                        log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): previously attempted to set kernel arg",
                                "karg", karg)
                }
                // There is a case when we try to set the kernel argument here, the daemon could decide to not reboot because
                // the daemon encountered a potentially one-time error. However we always want to make sure that the kernel
                // argument is set once the daemon goes through node state sync again.
                update, err := setKernelArg(karg)
                if err != nil {
                        log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)
                        return false, err
                }
                if update {
                        needReboot = true
                        log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): need reboot for setting kernel arg", "karg", karg)
                }
                p.DesiredKernelArgs[karg] = true
        }
        return needReboot, nil
}

func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)

        if p.needToUpdateVFs(desired, current) {
                return true
        }

        if p.shouldConfigureBridges() {
                if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {
                        log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {
        for _, ifaceStatus := range current.Interfaces {
                configured := false
                for _, iface := range desired.Interfaces {
                        if iface.PciAddress == ifaceStatus.PciAddress {
                                configured = true
                                if ifaceStatus.NumVfs == 0 {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",
                                                "address", iface.PciAddress)
                                        break
                                }
                                if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {
                                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",
                                                "address", iface.PciAddress)
                                        return true
                                }
                                log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",
                                        "address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)
                        }
                }
                if !configured && ifaceStatus.NumVfs > 0 {
                        // load the PF info
                        pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)
                        if err != nil {
                                log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if !exist {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        if pfStatus.ExternallyManaged {
                                log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",
                                        "name", ifaceStatus.Name,
                                        "address", ifaceStatus.PciAddress)
                                continue
                        }

                        log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",
                                "interface", ifaceStatus)
                        return true
                }
        }
        return false
}

func (p *GenericPlugin) shouldConfigureBridges() bool {
        return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration
}

func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) {
        driverState := p.DriverStateMap[Vfio]

        kernelArgFnByCPUVendor := map[hostTypes.CPUVendor]func(){
                hostTypes.CPUVendorIntel: func() {
                        p.addToDesiredKernelArgs(consts.KernelArgIntelIommu)
                        p.addToDesiredKernelArgs(consts.KernelArgIommuPt)
                },
                hostTypes.CPUVendorAMD: func() {
                        p.addToDesiredKernelArgs(consts.KernelArgIommuPt)
                },
                hostTypes.CPUVendorARM: func() {
                        p.addToDesiredKernelArgs(consts.KernelArgIommuPassthrough)
                },
        }

        if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                cpuVendor, err := p.helpers.GetCPUVendor()
                if err != nil {
                        log.Log.Error(err, "can't get CPU vendor, falling back to Intel")
                        cpuVendor = hostTypes.CPUVendorIntel
                }

                addKernelArgFn := kernelArgFnByCPUVendor[cpuVendor]
                if addKernelArgFn != nil {
                        addKernelArgFn()
                }
        }
}

func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        needReboot := false

        p.addVfioDesiredKernelArg(state)

        missingKernelArgs, err := p.getMissingKernelArgs()
        if err != nil {
                log.Log.Error(err, "generic-plugin needRebootNode(): failed to verify missing kernel arguments")
                return false, err
        }

        if len(missingKernelArgs) != 0 {
                needReboot, err = p.syncDesiredKernelArgs(missingKernelArgs)
                if err != nil {
                        log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")
                        return false, err
                }
                if needReboot {
                        log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")
                }
        }

        return needReboot, nil
}

// ////////////// for testing purposes only ///////////////////////
func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {
        return p.DriverStateMap
}

func (p *GenericPlugin) loadDriverForTests(state *sriovnetworkv1.SriovNetworkNodeState) {
        for _, driverState := range p.DriverStateMap {
                if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {
                        driverState.DriverLoaded = true
                }
        }
}

//////////////////////////////////////////////////////////////////

1	package generic
2
3	import (
4	"bytes"
5	"errors"
6	"os/exec"
7	"strconv"
8	"strings"
9	"syscall"
10
11	"sigs.k8s.io/controller-runtime/pkg/log"
12
13	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
14	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
15	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
16	hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
17	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
18	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
19	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
20	)
21
22	var PluginName = "generic"
23
24	// driver id
25	const (
26	Vfio = iota
27	VirtioVdpa
28	VhostVdpa
29	)
30
31	// driver name
32	const (
33	vfioPciDriver = "vfio_pci"
34	virtioVdpaDriver = "virtio_vdpa"
35	vhostVdpaDriver = "vhost_vdpa"
36	)
37
38	// function type for determining if a given driver has to be loaded in the kernel
39	type needDriver func(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool
40
41	type DriverState struct {
42	DriverName string
43	DeviceType string
44	VdpaType string
45	NeedDriverFunc needDriver
46	DriverLoaded bool
47	}
48
49	type DriverStateMapType map[uint]*DriverState
50
51	type GenericPlugin struct {
52	PluginName string
53	SpecVersion string
54	DesireState *sriovnetworkv1.SriovNetworkNodeState
55	DriverStateMap DriverStateMapType
56	DesiredKernelArgs map[string]bool
57	helpers helper.HostHelpersInterface
58	skipVFConfiguration bool
59	skipBridgeConfiguration bool
60	}
61
62	type Option = func(c *genericPluginOptions)
63
64	// WithSkipVFConfiguration configures generic plugin to skip configuration of the VFs.
65	// In this case PFs will be configured and VFs are created only, VF configuration phase is skipped.
66	// VFs on the PF (if the PF is not ExternallyManaged) will have no driver after the plugin execution completes.
67	func WithSkipVFConfiguration() Option {	1✔
68	return func(c *genericPluginOptions) {	1✔
69	c.skipVFConfiguration = true	×
70	}	×
71	}
72
73	// WithSkipBridgeConfiguration configures generic_plugin to skip configuration of the managed bridges
74	func WithSkipBridgeConfiguration() Option {	1✔
75	return func(c *genericPluginOptions) {	1✔
76	c.skipBridgeConfiguration = true	×
77	}	×
78	}
79
80	type genericPluginOptions struct {
81	skipVFConfiguration bool
82	skipBridgeConfiguration bool
83	}
84
85	const scriptsPath = "bindata/scripts/enable-kargs.sh"
86
87	// Initialize our plugin and set up initial values
88	func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) {	1✔
89	cfg := &genericPluginOptions{}	1✔
90	for _, o := range options {	1✔
91	o(cfg)	×
92	}	×
93	driverStateMap := make(map[uint]*DriverState)	1✔
94	driverStateMap[Vfio] = &DriverState{	1✔
95	DriverName: vfioPciDriver,	1✔
96	DeviceType: consts.DeviceTypeVfioPci,	1✔
97	VdpaType: "",	1✔
98	NeedDriverFunc: needDriverCheckDeviceType,	1✔
99	DriverLoaded: false,	1✔
100	}	1✔
101	driverStateMap[VirtioVdpa] = &DriverState{	1✔
102	DriverName: virtioVdpaDriver,	1✔
103	DeviceType: consts.DeviceTypeNetDevice,	1✔
104	VdpaType: consts.VdpaTypeVirtio,	1✔
105	NeedDriverFunc: needDriverCheckVdpaType,	1✔
106	DriverLoaded: false,	1✔
107	}	1✔
108	driverStateMap[VhostVdpa] = &DriverState{	1✔
109	DriverName: vhostVdpaDriver,	1✔
110	DeviceType: consts.DeviceTypeNetDevice,	1✔
111	VdpaType: consts.VdpaTypeVhost,	1✔
112	NeedDriverFunc: needDriverCheckVdpaType,	1✔
113	DriverLoaded: false,	1✔
114	}	1✔
115	return &GenericPlugin{	1✔
116	PluginName: PluginName,	1✔
117	SpecVersion: "1.0",	1✔
118	DriverStateMap: driverStateMap,	1✔
119	DesiredKernelArgs: make(map[string]bool),	1✔
120	helpers: helpers,	1✔
121	skipVFConfiguration: cfg.skipVFConfiguration,	1✔
122	skipBridgeConfiguration: cfg.skipBridgeConfiguration,	1✔
123	}, nil	1✔
124	}
125
126	// Name returns the name of the plugin
127	func (p *GenericPlugin) Name() string {	1✔
128	return p.PluginName	1✔
129	}	1✔
130
131	// Spec returns the version of the spec expected by the plugin
132	func (p *GenericPlugin) Spec() string {	×
133	return p.SpecVersion	×
134	}	×
135
136	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node
137	func (p GenericPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	1✔
138	log.Log.Info("generic plugin OnNodeStateChange()")	1✔
139	p.DesireState = new	1✔
140		1✔
141	needDrain = p.needDrainNode(new.Spec, new.Status)	1✔
142	needReboot, err = p.needRebootNode(new)	1✔
143	if err != nil {	1✔
144	return needDrain, needReboot, err	×
145	}	×
146
147	if needReboot {	1✔
148	needDrain = true	×
149	}	×
150	return	1✔
151	}
152
153	// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs.
154	func (p GenericPlugin) CheckStatusChanges(current sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
155	log.Log.Info("generic-plugin CheckStatusChanges()")	1✔
156		1✔
157	for _, iface := range current.Spec.Interfaces {	2✔
158	found := false	1✔
159	for _, ifaceStatus := range current.Status.Interfaces {	2✔
160	// TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632	1✔
161	if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged {	2✔
162	found = true	1✔
163	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
164	log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress)	1✔
165	return true, nil	1✔
166	}	1✔
167	break	1✔
168	}
169	}
170	if !found {	1✔
171	log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress)	×
172	}	×
173	}
174
175	if p.shouldConfigureBridges() {	2✔
176	if sriovnetworkv1.NeedToUpdateBridges(&current.Spec.Bridges, &current.Status.Bridges) {	2✔
177	log.Log.Info("CheckStatusChanges(): bridge configuration needs to be updated")	1✔
178	return true, nil	1✔
179	}	1✔
180	}
181
182	missingKernelArgs, err := p.getMissingKernelArgs()	1✔
183	if err != nil {	1✔
184	log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments")	×
185	return false, err	×
186	}	×
187
188	if len(missingKernelArgs) != 0 {	2✔
189	log.Log.V(0).Info("generic-plugin CheckStatusChanges(): kernel args missing",	1✔
190	"kernelArgs", missingKernelArgs)	1✔
191	}	1✔
192
193	return len(missingKernelArgs) != 0, nil	1✔
194	}
195
196	func (p *GenericPlugin) syncDriverState() error {	×
197	for _, driverState := range p.DriverStateMap {	×
198	if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) {	×
199	log.Log.V(2).Info("loading driver", "name", driverState.DriverName)	×
200	if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil {	×
201	log.Log.Error(err, "generic plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName)	×
202	return err	×
203	}	×
204	driverState.DriverLoaded = true	×
205	}
206	}
207	return nil	×
208	}
209
210	// Apply config change
211	func (p *GenericPlugin) Apply() error {	×
212	log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec)	×
213		×
214	if err := p.syncDriverState(); err != nil {	×
215	return err	×
216	}	×
217
218	// When calling from systemd do not try to chroot
219	if !vars.UsingSystemdMode {	×
220	exit, err := p.helpers.Chroot(consts.Host)	×
221	if err != nil {	×
222	return err	×
223	}	×
224	defer exit()	×
225	}
226
227	if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces,	×
228	p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil {	×
229	// Catch the "cannot allocate memory" error and try to use PCI realloc	×
230	if errors.Is(err, syscall.ENOMEM) {	×
231	p.addToDesiredKernelArgs(consts.KernelArgPciRealloc)	×
232	}	×
233	return err	×
234	}
235
236	if p.shouldConfigureBridges() {	×
237	if err := p.helpers.ConfigureBridges(p.DesireState.Spec.Bridges, p.DesireState.Status.Bridges); err != nil {	×
238	return err	×
239	}	×
240	}
241
242	return nil	×
243	}
244
245	func needDriverCheckDeviceType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
246	for _, iface := range state.Spec.Interfaces {	2✔
247	for i := range iface.VfGroups {	2✔
248	if iface.VfGroups[i].DeviceType == driverState.DeviceType {	2✔
249	return true	1✔
250	}	1✔
251	}
252	}
253	return false	1✔
254	}
255
256	func needDriverCheckVdpaType(state sriovnetworkv1.SriovNetworkNodeState, driverState DriverState) bool {	1✔
257	for _, iface := range state.Spec.Interfaces {	2✔
258	for i := range iface.VfGroups {	2✔
259	if iface.VfGroups[i].VdpaType == driverState.VdpaType {	2✔
260	return true	1✔
261	}	1✔
262	}
263	}
264	return false	1✔
265	}
266
267	// setKernelArg Tries to add the kernel args via ostree or grubby.
268	func setKernelArg(karg string) (bool, error) {	×
269	log.Log.Info("generic plugin setKernelArg()")	×
270	var stdout, stderr bytes.Buffer	×
271	cmd := exec.Command("/bin/sh", scriptsPath, karg)	×
272	cmd.Stdout = &stdout	×
273	cmd.Stderr = &stderr	×
274		×
275	if err := cmd.Run(); err != nil {	×
276	// if grubby is not there log and assume kernel args are set correctly.	×
277	if utils.IsCommandNotFound(err) {	×
278	log.Log.Error(err, "generic plugin setKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are set",	×
279	"kargs", karg)	×
280	return false, nil	×
281	}	×
282	log.Log.Error(err, "generic plugin setKernelArg(): fail to enable kernel arg", "karg", karg)	×
283	return false, err	×
284	}
285
286	i, err := strconv.Atoi(strings.TrimSpace(stdout.String()))	×
287	if err == nil {	×
288	if i > 0 {	×
289	log.Log.Info("generic plugin setKernelArg(): need to reboot node for kernel arg", "karg", karg)	×
290	return true, nil	×
291	}	×
292	}
293	return false, err	×
294	}
295
296	// addToDesiredKernelArgs Should be called to queue a kernel arg to be added to the node.
297	func (p *GenericPlugin) addToDesiredKernelArgs(karg string) {	1✔
298	if _, ok := p.DesiredKernelArgs[karg]; !ok {	2✔
299	log.Log.Info("generic plugin addToDesiredKernelArgs(): Adding to desired kernel arg", "karg", karg)	1✔
300	p.DesiredKernelArgs[karg] = false	1✔
301	}	1✔
302	}
303
304	// getMissingKernelArgs gets Kernel arguments that have not been set.
305	func (p *GenericPlugin) getMissingKernelArgs() ([]string, error) {	1✔
306	missingArgs := make([]string, 0, len(p.DesiredKernelArgs))	1✔
307	if len(p.DesiredKernelArgs) == 0 {	2✔
308	return nil, nil	1✔
309	}	1✔
310
311	kargs, err := p.helpers.GetCurrentKernelArgs()	1✔
312	if err != nil {	1✔
313	return nil, err	×
314	}	×
315
316	for desiredKarg := range p.DesiredKernelArgs {	2✔
317	if !p.helpers.IsKernelArgsSet(kargs, desiredKarg) {	2✔
318	missingArgs = append(missingArgs, desiredKarg)	1✔
319	}	1✔
320	}
321	return missingArgs, nil	1✔
322	}
323
324	// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed.
325	func (p *GenericPlugin) syncDesiredKernelArgs(kargs []string) (bool, error) {	×
326	needReboot := false	×
327		×
328	for _, karg := range kargs {	×
329	if p.DesiredKernelArgs[karg] {	×
330	log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): previously attempted to set kernel arg",	×
331	"karg", karg)	×
332	}	×
333	// There is a case when we try to set the kernel argument here, the daemon could decide to not reboot because
334	// the daemon encountered a potentially one-time error. However we always want to make sure that the kernel
335	// argument is set once the daemon goes through node state sync again.
336	update, err := setKernelArg(karg)	×
337	if err != nil {	×
338	log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg)	×
339	return false, err	×
340	}	×
341	if update {	×
342	needReboot = true	×
343	log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): need reboot for setting kernel arg", "karg", karg)	×
344	}	×
345	p.DesiredKernelArgs[karg] = true	×
346	}
347	return needReboot, nil	×
348	}
349
350	func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
351	log.Log.V(2).Info("generic plugin needDrainNode()", "current", current, "desired", desired)	1✔
352		1✔
353	if p.needToUpdateVFs(desired, current) {	2✔
354	return true	1✔
355	}	1✔
356
357	if p.shouldConfigureBridges() {	2✔
358	if sriovnetworkv1.NeedToUpdateBridges(&desired.Bridges, &current.Bridges) {	2✔
359	log.Log.V(2).Info("generic plugin needDrainNode(): need drain since bridge configuration needs to be updated")	1✔
360	return true	1✔
361	}	1✔
362	}
363	return false	1✔
364	}
365
366	func (p *GenericPlugin) needToUpdateVFs(desired sriovnetworkv1.SriovNetworkNodeStateSpec, current sriovnetworkv1.SriovNetworkNodeStateStatus) bool {	1✔
367	for _, ifaceStatus := range current.Interfaces {	2✔
368	configured := false	1✔
369	for _, iface := range desired.Interfaces {	2✔
370	if iface.PciAddress == ifaceStatus.PciAddress {	2✔
371	configured = true	1✔
372	if ifaceStatus.NumVfs == 0 {	1✔
373	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0",	×
374	"address", iface.PciAddress)	×
375	break	×
376	}
377	if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) {	2✔
378	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain, for PCI address request update",	1✔
379	"address", iface.PciAddress)	1✔
380	return true	1✔
381	}	1✔
382	log.Log.V(2).Info("generic plugin needToUpdateVFs(): no need drain,for PCI address",	1✔
383	"address", iface.PciAddress, "expected-vfs", iface.NumVfs, "current-vfs", ifaceStatus.NumVfs)	1✔
384	}
385	}
386	if !configured && ifaceStatus.NumVfs > 0 {	1✔
387	// load the PF info	×
388	pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress)	×
389	if err != nil {	×
390	log.Log.Error(err, "generic plugin needToUpdateVFs(): failed to load info about PF status for pci device",	×
391	"address", ifaceStatus.PciAddress)	×
392	continue	×
393	}
394
395	if !exist {	×
396	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping drain",	×
397	"name", ifaceStatus.Name,	×
398	"address", ifaceStatus.PciAddress)	×
399	continue	×
400	}
401
402	if pfStatus.ExternallyManaged {	×
403	log.Log.Info("generic plugin needToUpdateVFs(): PF name with pci address was externally created. Skipping drain",	×
404	"name", ifaceStatus.Name,	×
405	"address", ifaceStatus.PciAddress)	×
406	continue	×
407	}
408
409	log.Log.V(2).Info("generic plugin needToUpdateVFs(): need drain since interface needs to be reset",	×
410	"interface", ifaceStatus)	×
411	return true	×
412	}
413	}
414	return false	1✔
415	}
416
417	func (p *GenericPlugin) shouldConfigureBridges() bool {	1✔
418	return vars.ManageSoftwareBridges && !p.skipBridgeConfiguration	1✔
419	}	1✔
420
421	func (p GenericPlugin) addVfioDesiredKernelArg(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
422	driverState := p.DriverStateMap[Vfio]	1✔
423		1✔
424	kernelArgFnByCPUVendor := map[hostTypes.CPUVendor]func(){	1✔
425	hostTypes.CPUVendorIntel: func() {	2✔
426	p.addToDesiredKernelArgs(consts.KernelArgIntelIommu)	1✔
427	p.addToDesiredKernelArgs(consts.KernelArgIommuPt)	1✔
428	},	1✔
429	hostTypes.CPUVendorAMD: func() {	1✔
430	p.addToDesiredKernelArgs(consts.KernelArgIommuPt)	1✔
431	},	1✔
432	hostTypes.CPUVendorARM: func() {	1✔
433	p.addToDesiredKernelArgs(consts.KernelArgIommuPassthrough)	1✔
434	},	1✔
435	}
436
437	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
438	cpuVendor, err := p.helpers.GetCPUVendor()	1✔
439	if err != nil {	1✔
NEW 440	log.Log.Error(err, "can't get CPU vendor, falling back to Intel")	×
NEW 441	cpuVendor = hostTypes.CPUVendorIntel	×
NEW 442	}	×
443
444	addKernelArgFn := kernelArgFnByCPUVendor[cpuVendor]	1✔
445	if addKernelArgFn != nil {	2✔
446	addKernelArgFn()	1✔
447	}	1✔
448	}
449	}
450
451	func (p GenericPlugin) needRebootNode(state sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	1✔
452	needReboot := false	1✔
453		1✔
454	p.addVfioDesiredKernelArg(state)	1✔
455		1✔
456	missingKernelArgs, err := p.getMissingKernelArgs()	1✔
457	if err != nil {	1✔
458	log.Log.Error(err, "generic-plugin needRebootNode(): failed to verify missing kernel arguments")	×
459	return false, err	×
460	}	×
461
462	if len(missingKernelArgs) != 0 {	1✔
463	needReboot, err = p.syncDesiredKernelArgs(missingKernelArgs)	×
464	if err != nil {	×
465	log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments")	×
466	return false, err	×
467	}	×
468	if needReboot {	×
469	log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments")	×
470	}	×
471	}
472
473	return needReboot, nil	1✔
474	}
475
476	// ////////////// for testing purposes only ///////////////////////
477	func (p *GenericPlugin) getDriverStateMap() DriverStateMapType {	1✔
478	return p.DriverStateMap	1✔
479	}	1✔
480
481	func (p GenericPlugin) loadDriverForTests(state sriovnetworkv1.SriovNetworkNodeState) {	1✔
482	for _, driverState := range p.DriverStateMap {	2✔
483	if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) {	2✔
484	driverState.DriverLoaded = true	1✔
485	}	1✔
486	}
487	}
488
489	//////////////////////////////////////////////////////////////////

k8snetworkplumbingwg / sriov-network-operator / 11591346372

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous