10194187385

Committed 01 Aug 2024 07:44AM UTC coverage: 45.22% (+1.3%) from 43.968%

Build # 10194187385

Build Type

Pull #746

github

Committed by

web-flow

Commit Message

Merge f8cc3364c into 57e1e9056

Pull Request Pull Request #746: Remove logic that installs rdma-core package

Run Details

13 of 38 new or added lines in 5 files covered. (34.21%)

3 existing lines in 1 file now uncovered.

6528 of 14436 relevant lines covered (45.22%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.93

/pkg/host/internal/kernel/kernel.go

package kernel

import (
        "errors"
        "fmt"
        "os"
        "path/filepath"
        "strings"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

type kernel struct {
        utilsHelper utils.CmdInterface
}

func New(utilsHelper utils.CmdInterface) types.KernelInterface {
        return &kernel{utilsHelper: utilsHelper}
}

func (k *kernel) LoadKernelModule(name string, args ...string) error {
        log.Log.Info("LoadKernelModule(): try to load kernel module", "name", name, "args", args)
        chrootDefinition := utils.GetChrootExtension()
        cmdArgs := strings.Join(args, " ")

        // check if the driver is already loaded in to the system
        isLoaded, err := k.IsKernelModuleLoaded(name)
        if err != nil {
                log.Log.Error(err, "LoadKernelModule(): failed to check if kernel module is already loaded", "name", name)
        }
        if isLoaded {
                log.Log.Info("LoadKernelModule(): kernel module already loaded", "name", name)
                return nil
        }

        _, _, err = k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s modprobe %s %s", chrootDefinition, name, cmdArgs))
        if err != nil {
                log.Log.Error(err, "LoadKernelModule(): failed to load kernel module with arguments", "name", name, "args", args)
                return err
        }
        return nil
}

func (k *kernel) IsKernelModuleLoaded(kernelModuleName string) (bool, error) {
        log.Log.Info("IsKernelModuleLoaded(): check if kernel module is loaded", "name", kernelModuleName)
        chrootDefinition := utils.GetChrootExtension()

        stdout, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s lsmod | grep \"^%s\"", chrootDefinition, kernelModuleName))
        if err != nil && len(stderr) != 0 {
                log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded",
                        "name", kernelModuleName, "stderr", stderr)
                return false, err
        }
        log.Log.V(2).Info("IsKernelModuleLoaded():", "stdout", stdout)
        if len(stderr) != 0 {
                log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded", "name", kernelModuleName, "stderr", stderr)
                return false, fmt.Errorf(stderr)
        }

        if len(stdout) != 0 {
                log.Log.Info("IsKernelModuleLoaded(): kernel module already loaded", "name", kernelModuleName)
                return true, nil
        }

        return false, nil
}

func (k *kernel) TryEnableTun() {
        if err := k.LoadKernelModule("tun"); err != nil {
                log.Log.Error(err, "tryEnableTun(): TUN kernel module not loaded")
        }
}

func (k *kernel) TryEnableVhostNet() {
        if err := k.LoadKernelModule("vhost_net"); err != nil {
                log.Log.Error(err, "tryEnableVhostNet(): VHOST_NET kernel module not loaded")
        }
}

// GetCurrentKernelArgs This retrieves the kernel cmd line arguments
func (k *kernel) GetCurrentKernelArgs() (string, error) {
        path := consts.ProcKernelCmdLine
        if !vars.UsingSystemdMode {
                path = filepath.Join(consts.Host, path)
        }

        path = filepath.Join(vars.FilesystemRoot, path)
        cmdLine, err := os.ReadFile(path)
        if err != nil {
                return "", fmt.Errorf("GetCurrentKernelArgs(): Error reading %s: %v", path, err)
        }
        return string(cmdLine), nil
}

// IsKernelArgsSet This checks if the kernel cmd line is set properly. Please note that the same key could be repeated
// several times in the kernel cmd line. We can only ensure that the kernel cmd line has the key/val kernel arg that we set.
func (k *kernel) IsKernelArgsSet(cmdLine string, karg string) bool {
        elements := strings.Fields(cmdLine)
        for _, element := range elements {
                if element == karg {
                        return true
                }
        }
        return false
}

// Unbind unbind driver for one device
func (k *kernel) Unbind(pciAddr string) error {
        log.Log.V(2).Info("Unbind(): unbind device driver for device", "device", pciAddr)
        return k.UnbindDriverByBusAndDevice(consts.BusPci, pciAddr)
}

// BindDpdkDriver bind dpdk driver for one device
// Bind the device given by "pciAddr" to the driver "driver"
func (k *kernel) BindDpdkDriver(pciAddr, driver string) error {
        log.Log.V(2).Info("BindDpdkDriver(): bind device to driver",
                "device", pciAddr, "driver", driver)
        if err := k.BindDriverByBusAndDevice(consts.BusPci, pciAddr, driver); err != nil {
                _, innerErr := os.Readlink(filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "iommu_group"))
                if innerErr != nil {
                        log.Log.Error(err, "Could not read IOMMU group for device", "device", pciAddr)
                        return fmt.Errorf(
                                "cannot bind driver %s to device %s, make sure IOMMU is enabled in BIOS. %w", driver, pciAddr, innerErr)
                }
                return err
        }
        return nil
}

// BindDefaultDriver bind driver for one device
// Bind the device given by "pciAddr" to the default driver
func (k *kernel) BindDefaultDriver(pciAddr string) error {
        log.Log.V(2).Info("BindDefaultDriver(): bind device to default driver", "device", pciAddr)

        curDriver, err := getDriverByBusAndDevice(consts.BusPci, pciAddr)
        if err != nil {
                return err
        }
        if curDriver != "" {
                if !sriovnetworkv1.StringInArray(curDriver, vars.DpdkDrivers) {
                        log.Log.V(2).Info("BindDefaultDriver(): device already bound to default driver",
                                "device", pciAddr, "driver", curDriver)
                        return nil
                }
                if err := k.UnbindDriverByBusAndDevice(consts.BusPci, pciAddr); err != nil {
                        return err
                }
        }
        if err := setDriverOverride(consts.BusPci, pciAddr, ""); err != nil {
                return err
        }
        if err := probeDriver(consts.BusPci, pciAddr); err != nil {
                return err
        }
        return nil
}

// BindDriverByBusAndDevice binds device to the provided driver
// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
// driver - the name of the driver, e.g. vfio-pci or vhost_vdpa.
func (k *kernel) BindDriverByBusAndDevice(bus, device, driver string) error {
        log.Log.V(2).Info("BindDriverByBusAndDevice(): bind device to driver",
                "bus", bus, "device", device, "driver", driver)

        curDriver, err := getDriverByBusAndDevice(bus, device)
        if err != nil {
                return err
        }
        if curDriver != "" {
                if curDriver == driver {
                        log.Log.V(2).Info("BindDriverByBusAndDevice(): device already bound to driver",
                                "bus", bus, "device", device, "driver", driver)
                        return nil
                }
                if err := k.UnbindDriverByBusAndDevice(bus, device); err != nil {
                        return err
                }
        }
        if err := setDriverOverride(bus, device, driver); err != nil {
                return err
        }
        if err := bindDriver(bus, device, driver); err != nil {
                return err
        }
        return setDriverOverride(bus, device, "")
}

// Workaround function to handle a case where the vf default driver is stuck and not able to create the vf kernel interface.
// This function unbind the VF from the default driver and try to bind it again
// bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2045087
func (k *kernel) RebindVfToDefaultDriver(vfAddr string) error {
        log.Log.Info("RebindVfToDefaultDriver()", "vf", vfAddr)
        if err := k.Unbind(vfAddr); err != nil {
                return err
        }
        if err := k.BindDefaultDriver(vfAddr); err != nil {
                log.Log.Error(err, "RebindVfToDefaultDriver(): fail to bind default driver", "device", vfAddr)
                return err
        }

        log.Log.Info("RebindVfToDefaultDriver(): workaround implemented", "vf", vfAddr)
        return nil
}

func (k *kernel) UnbindDriverIfNeeded(vfAddr string, isRdma bool) error {
        if isRdma {
                log.Log.Info("UnbindDriverIfNeeded(): unbinding driver", "device", vfAddr)
                if err := k.Unbind(vfAddr); err != nil {
                        return err
                }
                log.Log.Info("UnbindDriverIfNeeded(): unbounded driver", "device", vfAddr)
        }
        return nil
}

// UnbindDriverByBusAndDevice unbind device identified by bus and device ID from the driver
// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
func (k *kernel) UnbindDriverByBusAndDevice(bus, device string) error {
        log.Log.V(2).Info("UnbindDriverByBusAndDevice(): unbind device driver for device", "bus", bus, "device", device)
        driver, err := getDriverByBusAndDevice(bus, device)
        if err != nil {
                return err
        }
        if driver == "" {
                log.Log.V(2).Info("UnbindDriverByBusAndDevice(): device has no driver", "bus", bus, "device", device)
                return nil
        }
        return unbindDriver(bus, device, driver)
}

func (k *kernel) HasDriver(pciAddr string) (bool, string) {
        driver, err := getDriverByBusAndDevice(consts.BusPci, pciAddr)
        if err != nil {
                log.Log.V(2).Info("HasDriver(): device driver is empty for device", "device", pciAddr)
                return false, ""
        }
        if driver != "" {
                log.Log.V(2).Info("HasDriver(): device driver for device", "device", pciAddr, "driver", driver)
                return true, driver
        }
        return false, ""
}

// GetDriverByBusAndDevice returns driver for the device or error.
// returns "", nil if the device has no driver.
// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
func (k *kernel) GetDriverByBusAndDevice(bus, device string) (string, error) {
        log.Log.V(2).Info("GetDriverByBusAndDevice(): get driver for device", "bus", bus, "device", device)
        return getDriverByBusAndDevice(bus, device)
}

// CheckRDMAEnabled returns true if RDMA modules are loaded on host
func (k *kernel) CheckRDMAEnabled() (bool, error) {
        log.Log.V(2).Info("CheckRDMAEnabled()")
        chrootDefinition := utils.GetChrootExtension()

        _, stderr, mlx5Err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s lsmod | grep --quiet 'mlx5_core'", chrootDefinition))
        if mlx5Err != nil && len(stderr) != 0 {
                log.Log.Error(mlx5Err, "CheckRDMAEnabled(): failed to check for kernel module 'mlx5_core'", "stderr", stderr)
                return false, fmt.Errorf(stderr)
        }

        if mlx5Err != nil {
                log.Log.Error(nil, "CheckRDMAEnabled(): no RDMA capable devices")
                return false, nil
        }
        return k.rdmaModulesAreLoaded()
}

func (k *kernel) rdmaModulesAreLoaded() (bool, error) {
        log.Log.V(2).Info("rdmaModulesAreLoaded()")
        chrootDefinition := utils.GetChrootExtension()

        // check if the driver is already loaded in to the system
        _, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s lsmod | grep --quiet '\\(^ib\\|^rdma\\)'", chrootDefinition))
        if err != nil && len(stderr) != 0 {
                log.Log.Error(err, "rdmaModulesAreLoaded(): fail to check if ib and rdma kernel modules are loaded", "stderr", stderr)
                return false, fmt.Errorf(stderr)
        }

        if err != nil {
                log.Log.Error(nil, "rdmaModulesAreLoaded(): RDMA modules are not loaded, you may need to install rdma-core package")
                return false, nil
        }
        log.Log.V(2).Info("rdmaModulesAreLoaded(): RDMA modules are loaded")
        return true, nil
}

// IsKernelLockdownMode returns true when kernel lockdown mode is enabled
// TODO: change this to return error
func (k *kernel) IsKernelLockdownMode() bool {
        path := utils.GetHostExtension()
        path = filepath.Join(path, "/sys/kernel/security/lockdown")

        stdout, stderr, err := k.utilsHelper.RunCommand("cat", path)
        log.Log.V(2).Info("IsKernelLockdownMode()", "output", stdout, "error", err)
        if err != nil {
                log.Log.Error(err, "IsKernelLockdownMode(): failed to check for lockdown file", "stderr", stderr)
                return false
        }
        return strings.Contains(stdout, "[integrity]") || strings.Contains(stdout, "[confidentiality]")
}

// returns driver for device on the bus
func getDriverByBusAndDevice(bus, device string) (string, error) {
        driverLink := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "devices", device, "driver")
        driverInfo, err := os.Readlink(driverLink)
        if err != nil {
                if errors.Is(err, os.ErrNotExist) {
                        log.Log.V(2).Info("getDriverByBusAndDevice(): driver path for device not exist", "bus", bus, "device", device, "driver", driverInfo)
                        return "", nil
                }
                log.Log.Error(err, "getDriverByBusAndDevice(): error getting driver info for device", "bus", bus, "device", device)
                return "", err
        }
        log.Log.V(2).Info("getDriverByBusAndDevice(): driver for device", "bus", bus, "device", device, "driver", driverInfo)
        return filepath.Base(driverInfo), nil
}

// binds device to the provide driver
func bindDriver(bus, device, driver string) error {
        log.Log.V(2).Info("bindDriver(): bind to driver", "bus", bus, "device", device, "driver", driver)
        bindPath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers", driver, "bind")
        err := os.WriteFile(bindPath, []byte(device), os.ModeAppend)
        if err != nil {
                log.Log.Error(err, "bindDriver(): failed to bind driver", "bus", bus, "device", device, "driver", driver)
                return err
        }
        return nil
}

// unbind device from the driver
func unbindDriver(bus, device, driver string) error {
        log.Log.V(2).Info("unbindDriver(): unbind from driver", "bus", bus, "device", device, "driver", driver)
        unbindPath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers", driver, "unbind")
        err := os.WriteFile(unbindPath, []byte(device), os.ModeAppend)
        if err != nil {
                log.Log.Error(err, "unbindDriver(): failed to unbind driver", "bus", bus, "device", device, "driver", driver)
                return err
        }
        return nil
}

// probes driver for device on the bus
func probeDriver(bus, device string) error {
        log.Log.V(2).Info("probeDriver(): drivers probe", "bus", bus, "device", device)
        probePath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers_probe")
        err := os.WriteFile(probePath, []byte(device), os.ModeAppend)
        if err != nil {
                log.Log.Error(err, "probeDriver(): failed to trigger driver probe", "bus", bus, "device", device)
                return err
        }
        return nil
}

// set driver override for the bus/device,
// resets override if override arg is "",
// if device doesn't support overriding (has no driver_override path), does nothing
func setDriverOverride(bus, device, override string) error {
        driverOverridePath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "devices", device, "driver_override")
        if _, err := os.Stat(driverOverridePath); err != nil {
                if os.IsNotExist(err) {
                        log.Log.V(2).Info("setDriverOverride(): device doesn't support driver override, skip", "bus", bus, "device", device)
                        return nil
                }
                return err
        }
        var overrideData []byte
        if override != "" {
                log.Log.V(2).Info("setDriverOverride(): configure driver override for device", "bus", bus, "device", device, "driver", override)
                overrideData = []byte(override)
        } else {
                log.Log.V(2).Info("setDriverOverride(): reset driver override for device", "bus", bus, "device", device)
                overrideData = []byte("\x00")
        }
        err := os.WriteFile(driverOverridePath, overrideData, os.ModeAppend)
        if err != nil {
                log.Log.Error(err, "setDriverOverride(): fail to write driver_override for device",
                        "bus", bus, "device", device, "driver", override)
                return err
        }
        return nil
}

1	package kernel
2
3	import (
4	"errors"
5	"fmt"
6	"os"
7	"path/filepath"
8	"strings"
9
10	"sigs.k8s.io/controller-runtime/pkg/log"
11
12	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
13	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
14	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
15	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
16	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
17	)
18
19	type kernel struct {
20	utilsHelper utils.CmdInterface
21	}
22
23	func New(utilsHelper utils.CmdInterface) types.KernelInterface {	1✔
24	return &kernel{utilsHelper: utilsHelper}	1✔
25	}	1✔
26
27	func (k *kernel) LoadKernelModule(name string, args ...string) error {	×
28	log.Log.Info("LoadKernelModule(): try to load kernel module", "name", name, "args", args)	×
29	chrootDefinition := utils.GetChrootExtension()	×
30	cmdArgs := strings.Join(args, " ")	×
31		×
32	// check if the driver is already loaded in to the system	×
33	isLoaded, err := k.IsKernelModuleLoaded(name)	×
34	if err != nil {	×
35	log.Log.Error(err, "LoadKernelModule(): failed to check if kernel module is already loaded", "name", name)	×
36	}	×
37	if isLoaded {	×
38	log.Log.Info("LoadKernelModule(): kernel module already loaded", "name", name)	×
39	return nil	×
40	}	×
41
42	_, _, err = k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s modprobe %s %s", chrootDefinition, name, cmdArgs))	×
43	if err != nil {	×
44	log.Log.Error(err, "LoadKernelModule(): failed to load kernel module with arguments", "name", name, "args", args)	×
45	return err	×
46	}	×
47	return nil	×
48	}
49
50	func (k *kernel) IsKernelModuleLoaded(kernelModuleName string) (bool, error) {	×
51	log.Log.Info("IsKernelModuleLoaded(): check if kernel module is loaded", "name", kernelModuleName)	×
52	chrootDefinition := utils.GetChrootExtension()	×
53		×
54	stdout, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s lsmod \| grep \"^%s\"", chrootDefinition, kernelModuleName))	×
55	if err != nil && len(stderr) != 0 {	×
56	log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded",	×
57	"name", kernelModuleName, "stderr", stderr)	×
58	return false, err	×
59	}	×
60	log.Log.V(2).Info("IsKernelModuleLoaded():", "stdout", stdout)	×
61	if len(stderr) != 0 {	×
62	log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded", "name", kernelModuleName, "stderr", stderr)	×
63	return false, fmt.Errorf(stderr)	×
64	}	×
65
66	if len(stdout) != 0 {	×
67	log.Log.Info("IsKernelModuleLoaded(): kernel module already loaded", "name", kernelModuleName)	×
68	return true, nil	×
69	}	×
70
71	return false, nil	×
72	}
73
74	func (k *kernel) TryEnableTun() {	×
75	if err := k.LoadKernelModule("tun"); err != nil {	×
76	log.Log.Error(err, "tryEnableTun(): TUN kernel module not loaded")	×
77	}	×
78	}
79
80	func (k *kernel) TryEnableVhostNet() {	×
81	if err := k.LoadKernelModule("vhost_net"); err != nil {	×
82	log.Log.Error(err, "tryEnableVhostNet(): VHOST_NET kernel module not loaded")	×
83	}	×
84	}
85
86	// GetCurrentKernelArgs This retrieves the kernel cmd line arguments
87	func (k *kernel) GetCurrentKernelArgs() (string, error) {	×
88	path := consts.ProcKernelCmdLine	×
89	if !vars.UsingSystemdMode {	×
90	path = filepath.Join(consts.Host, path)	×
91	}	×
92
93	path = filepath.Join(vars.FilesystemRoot, path)	×
94	cmdLine, err := os.ReadFile(path)	×
95	if err != nil {	×
96	return "", fmt.Errorf("GetCurrentKernelArgs(): Error reading %s: %v", path, err)	×
97	}	×
98	return string(cmdLine), nil	×
99	}
100
101	// IsKernelArgsSet This checks if the kernel cmd line is set properly. Please note that the same key could be repeated
102	// several times in the kernel cmd line. We can only ensure that the kernel cmd line has the key/val kernel arg that we set.
103	func (k *kernel) IsKernelArgsSet(cmdLine string, karg string) bool {	×
104	elements := strings.Fields(cmdLine)	×
105	for _, element := range elements {	×
106	if element == karg {	×
107	return true	×
108	}	×
109	}
110	return false	×
111	}
112
113	// Unbind unbind driver for one device
114	func (k *kernel) Unbind(pciAddr string) error {	1✔
115	log.Log.V(2).Info("Unbind(): unbind device driver for device", "device", pciAddr)	1✔
116	return k.UnbindDriverByBusAndDevice(consts.BusPci, pciAddr)	1✔
117	}	1✔
118
119	// BindDpdkDriver bind dpdk driver for one device
120	// Bind the device given by "pciAddr" to the driver "driver"
121	func (k *kernel) BindDpdkDriver(pciAddr, driver string) error {	1✔
122	log.Log.V(2).Info("BindDpdkDriver(): bind device to driver",	1✔
123	"device", pciAddr, "driver", driver)	1✔
124	if err := k.BindDriverByBusAndDevice(consts.BusPci, pciAddr, driver); err != nil {	2✔
125	_, innerErr := os.Readlink(filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "iommu_group"))	1✔
126	if innerErr != nil {	2✔
127	log.Log.Error(err, "Could not read IOMMU group for device", "device", pciAddr)	1✔
128	return fmt.Errorf(	1✔
129	"cannot bind driver %s to device %s, make sure IOMMU is enabled in BIOS. %w", driver, pciAddr, innerErr)	1✔
130	}	1✔
131	return err	×
132	}
133	return nil	1✔
134	}
135
136	// BindDefaultDriver bind driver for one device
137	// Bind the device given by "pciAddr" to the default driver
138	func (k *kernel) BindDefaultDriver(pciAddr string) error {	1✔
139	log.Log.V(2).Info("BindDefaultDriver(): bind device to default driver", "device", pciAddr)	1✔
140		1✔
141	curDriver, err := getDriverByBusAndDevice(consts.BusPci, pciAddr)	1✔
142	if err != nil {	1✔
143	return err	×
144	}	×
145	if curDriver != "" {	2✔
146	if !sriovnetworkv1.StringInArray(curDriver, vars.DpdkDrivers) {	2✔
147	log.Log.V(2).Info("BindDefaultDriver(): device already bound to default driver",	1✔
148	"device", pciAddr, "driver", curDriver)	1✔
149	return nil	1✔
150	}	1✔
151	if err := k.UnbindDriverByBusAndDevice(consts.BusPci, pciAddr); err != nil {	1✔
152	return err	×
153	}	×
154	}
155	if err := setDriverOverride(consts.BusPci, pciAddr, ""); err != nil {	1✔
156	return err	×
157	}	×
158	if err := probeDriver(consts.BusPci, pciAddr); err != nil {	2✔
159	return err	1✔
160	}	1✔
161	return nil	1✔
162	}
163
164	// BindDriverByBusAndDevice binds device to the provided driver
165	// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
166	// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
167	// driver - the name of the driver, e.g. vfio-pci or vhost_vdpa.
168	func (k *kernel) BindDriverByBusAndDevice(bus, device, driver string) error {	1✔
169	log.Log.V(2).Info("BindDriverByBusAndDevice(): bind device to driver",	1✔
170	"bus", bus, "device", device, "driver", driver)	1✔
171		1✔
172	curDriver, err := getDriverByBusAndDevice(bus, device)	1✔
173	if err != nil {	1✔
174	return err	×
175	}	×
176	if curDriver != "" {	2✔
177	if curDriver == driver {	2✔
178	log.Log.V(2).Info("BindDriverByBusAndDevice(): device already bound to driver",	1✔
179	"bus", bus, "device", device, "driver", driver)	1✔
180	return nil	1✔
181	}	1✔
182	if err := k.UnbindDriverByBusAndDevice(bus, device); err != nil {	1✔
183	return err	×
184	}	×
185	}
186	if err := setDriverOverride(bus, device, driver); err != nil {	1✔
187	return err	×
188	}	×
189	if err := bindDriver(bus, device, driver); err != nil {	2✔
190	return err	1✔
191	}	1✔
192	return setDriverOverride(bus, device, "")	1✔
193	}
194
195	// Workaround function to handle a case where the vf default driver is stuck and not able to create the vf kernel interface.
196	// This function unbind the VF from the default driver and try to bind it again
197	// bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2045087
198	func (k *kernel) RebindVfToDefaultDriver(vfAddr string) error {	×
199	log.Log.Info("RebindVfToDefaultDriver()", "vf", vfAddr)	×
200	if err := k.Unbind(vfAddr); err != nil {	×
201	return err	×
202	}	×
203	if err := k.BindDefaultDriver(vfAddr); err != nil {	×
204	log.Log.Error(err, "RebindVfToDefaultDriver(): fail to bind default driver", "device", vfAddr)	×
205	return err	×
206	}	×
207
208	log.Log.Info("RebindVfToDefaultDriver(): workaround implemented", "vf", vfAddr)	×
209	return nil	×
210	}
211
212	func (k *kernel) UnbindDriverIfNeeded(vfAddr string, isRdma bool) error {	×
213	if isRdma {	×
214	log.Log.Info("UnbindDriverIfNeeded(): unbinding driver", "device", vfAddr)	×
215	if err := k.Unbind(vfAddr); err != nil {	×
216	return err	×
217	}	×
218	log.Log.Info("UnbindDriverIfNeeded(): unbounded driver", "device", vfAddr)	×
219	}
220	return nil	×
221	}
222
223	// UnbindDriverByBusAndDevice unbind device identified by bus and device ID from the driver
224	// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
225	// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
226	func (k *kernel) UnbindDriverByBusAndDevice(bus, device string) error {	1✔
227	log.Log.V(2).Info("UnbindDriverByBusAndDevice(): unbind device driver for device", "bus", bus, "device", device)	1✔
228	driver, err := getDriverByBusAndDevice(bus, device)	1✔
229	if err != nil {	1✔
230	return err	×
231	}	×
232	if driver == "" {	2✔
233	log.Log.V(2).Info("UnbindDriverByBusAndDevice(): device has no driver", "bus", bus, "device", device)	1✔
234	return nil	1✔
235	}	1✔
236	return unbindDriver(bus, device, driver)	1✔
237	}
238
239	func (k *kernel) HasDriver(pciAddr string) (bool, string) {	1✔
240	driver, err := getDriverByBusAndDevice(consts.BusPci, pciAddr)	1✔
241	if err != nil {	1✔
242	log.Log.V(2).Info("HasDriver(): device driver is empty for device", "device", pciAddr)	×
243	return false, ""	×
244	}	×
245	if driver != "" {	2✔
246	log.Log.V(2).Info("HasDriver(): device driver for device", "device", pciAddr, "driver", driver)	1✔
247	return true, driver	1✔
248	}	1✔
249	return false, ""	1✔
250	}
251
252	// GetDriverByBusAndDevice returns driver for the device or error.
253	// returns "", nil if the device has no driver.
254	// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
255	// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
256	func (k *kernel) GetDriverByBusAndDevice(bus, device string) (string, error) {	1✔
257	log.Log.V(2).Info("GetDriverByBusAndDevice(): get driver for device", "bus", bus, "device", device)	1✔
258	return getDriverByBusAndDevice(bus, device)	1✔
259	}	1✔
260
261	// CheckRDMAEnabled returns true if RDMA modules are loaded on host
NEW 262	func (k *kernel) CheckRDMAEnabled() (bool, error) {	×
NEW 263	log.Log.V(2).Info("CheckRDMAEnabled()")	×
264	chrootDefinition := utils.GetChrootExtension()	×
265		×
NEW 266	_, stderr, mlx5Err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s lsmod \| grep --quiet 'mlx5_core'", chrootDefinition))	×
267	if mlx5Err != nil && len(stderr) != 0 {	×
NEW 268	log.Log.Error(mlx5Err, "CheckRDMAEnabled(): failed to check for kernel module 'mlx5_core'", "stderr", stderr)	×
269	return false, fmt.Errorf(stderr)	×
270	}	×
271
NEW 272	if mlx5Err != nil {	×
NEW 273	log.Log.Error(nil, "CheckRDMAEnabled(): no RDMA capable devices")	×
274	return false, nil	×
275	}	×
NEW 276	return k.rdmaModulesAreLoaded()	×
277	}
278
NEW 279	func (k *kernel) rdmaModulesAreLoaded() (bool, error) {	×
NEW 280	log.Log.V(2).Info("rdmaModulesAreLoaded()")	×
281	chrootDefinition := utils.GetChrootExtension()	×
282		×
283	// check if the driver is already loaded in to the system	×
NEW 284	_, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s lsmod \| grep --quiet '\\(^ib\\\|^rdma\\)'", chrootDefinition))	×
285	if err != nil && len(stderr) != 0 {	×
NEW 286	log.Log.Error(err, "rdmaModulesAreLoaded(): fail to check if ib and rdma kernel modules are loaded", "stderr", stderr)	×
287	return false, fmt.Errorf(stderr)	×
288	}	×
289
290	if err != nil {	×
NEW 291	log.Log.Error(nil, "rdmaModulesAreLoaded(): RDMA modules are not loaded, you may need to install rdma-core package")	×
292	return false, nil	×
293	}	×
NEW 294	log.Log.V(2).Info("rdmaModulesAreLoaded(): RDMA modules are loaded")	×
295	return true, nil	×
296	}
297
298	// IsKernelLockdownMode returns true when kernel lockdown mode is enabled
299	// TODO: change this to return error
300	func (k *kernel) IsKernelLockdownMode() bool {	1✔
301	path := utils.GetHostExtension()	1✔
302	path = filepath.Join(path, "/sys/kernel/security/lockdown")	1✔
303		1✔
304	stdout, stderr, err := k.utilsHelper.RunCommand("cat", path)	1✔
305	log.Log.V(2).Info("IsKernelLockdownMode()", "output", stdout, "error", err)	1✔
306	if err != nil {	1✔
307	log.Log.Error(err, "IsKernelLockdownMode(): failed to check for lockdown file", "stderr", stderr)	×
308	return false	×
309	}	×
310	return strings.Contains(stdout, "[integrity]") \|\| strings.Contains(stdout, "[confidentiality]")	1✔
311	}
312
313	// returns driver for device on the bus
314	func getDriverByBusAndDevice(bus, device string) (string, error) {	1✔
315	driverLink := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "devices", device, "driver")	1✔
316	driverInfo, err := os.Readlink(driverLink)	1✔
317	if err != nil {	2✔
318	if errors.Is(err, os.ErrNotExist) {	2✔
319	log.Log.V(2).Info("getDriverByBusAndDevice(): driver path for device not exist", "bus", bus, "device", device, "driver", driverInfo)	1✔
320	return "", nil	1✔
321	}	1✔
322	log.Log.Error(err, "getDriverByBusAndDevice(): error getting driver info for device", "bus", bus, "device", device)	×
323	return "", err	×
324	}
325	log.Log.V(2).Info("getDriverByBusAndDevice(): driver for device", "bus", bus, "device", device, "driver", driverInfo)	1✔
326	return filepath.Base(driverInfo), nil	1✔
327	}
328
329	// binds device to the provide driver
330	func bindDriver(bus, device, driver string) error {	1✔
331	log.Log.V(2).Info("bindDriver(): bind to driver", "bus", bus, "device", device, "driver", driver)	1✔
332	bindPath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers", driver, "bind")	1✔
333	err := os.WriteFile(bindPath, []byte(device), os.ModeAppend)	1✔
334	if err != nil {	2✔
335	log.Log.Error(err, "bindDriver(): failed to bind driver", "bus", bus, "device", device, "driver", driver)	1✔
336	return err	1✔
337	}	1✔
338	return nil	1✔
339	}
340
341	// unbind device from the driver
342	func unbindDriver(bus, device, driver string) error {	1✔
343	log.Log.V(2).Info("unbindDriver(): unbind from driver", "bus", bus, "device", device, "driver", driver)	1✔
344	unbindPath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers", driver, "unbind")	1✔
345	err := os.WriteFile(unbindPath, []byte(device), os.ModeAppend)	1✔
346	if err != nil {	2✔
347	log.Log.Error(err, "unbindDriver(): failed to unbind driver", "bus", bus, "device", device, "driver", driver)	1✔
348	return err	1✔
349	}	1✔
350	return nil	1✔
351	}
352
353	// probes driver for device on the bus
354	func probeDriver(bus, device string) error {	1✔
355	log.Log.V(2).Info("probeDriver(): drivers probe", "bus", bus, "device", device)	1✔
356	probePath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers_probe")	1✔
357	err := os.WriteFile(probePath, []byte(device), os.ModeAppend)	1✔
358	if err != nil {	2✔
359	log.Log.Error(err, "probeDriver(): failed to trigger driver probe", "bus", bus, "device", device)	1✔
360	return err	1✔
361	}	1✔
362	return nil	1✔
363	}
364
365	// set driver override for the bus/device,
366	// resets override if override arg is "",
367	// if device doesn't support overriding (has no driver_override path), does nothing
368	func setDriverOverride(bus, device, override string) error {	1✔
369	driverOverridePath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "devices", device, "driver_override")	1✔
370	if _, err := os.Stat(driverOverridePath); err != nil {	2✔
371	if os.IsNotExist(err) {	2✔
372	log.Log.V(2).Info("setDriverOverride(): device doesn't support driver override, skip", "bus", bus, "device", device)	1✔
373	return nil	1✔
374	}	1✔
375	return err	×
376	}
377	var overrideData []byte	1✔
378	if override != "" {	2✔
379	log.Log.V(2).Info("setDriverOverride(): configure driver override for device", "bus", bus, "device", device, "driver", override)	1✔
380	overrideData = []byte(override)	1✔
381	} else {	2✔
382	log.Log.V(2).Info("setDriverOverride(): reset driver override for device", "bus", bus, "device", device)	1✔
383	overrideData = []byte("\x00")	1✔
384	}	1✔
385	err := os.WriteFile(driverOverridePath, overrideData, os.ModeAppend)	1✔
386	if err != nil {	1✔
387	log.Log.Error(err, "setDriverOverride(): fail to write driver_override for device",	×
388	"bus", bus, "device", device, "driver", override)	×
389	return err	×
390	}	×
391	return nil	1✔
392	}

k8snetworkplumbingwg / sriov-network-operator / 10194187385

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous