1460f023-f8c8-4587-b36b-5e6fe90fca99

Committed 01 Jul 2026 08:10PM UTC coverage: 72.234% (+0.05%) from 72.188%

Build # 1460f023-f8c8-4587-b36b-5e6fe90fca99

Build Type

push

prow

Committed by

web-flow

Commit Message

Merge pull request #18095 from fra2404/vep-168

VEP 168: Masquerade Port Ranges support (Alpha)

Coverage Stats

171 of 191 new or added lines in 6 files covered. (89.53%)

229 existing lines in 11 files now uncovered.

83352 of 115392 relevant lines covered (72.23%)

418.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.41

/pkg/virt-handler/device-manager/device_controller.go

/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright The KubeVirt Authors.
 *
 */

package device_manager

import (
        "fmt"
        "math"
        "os"
        "path"
        "strings"
        "sync"
        "time"

        k8sv1 "k8s.io/api/core/v1"
        "k8s.io/client-go/tools/cache"

        "kubevirt.io/kubevirt/pkg/virt-controller/services"

        "kubevirt.io/kubevirt/pkg/virt-handler/cgroup"

        "kubevirt.io/client-go/log"

        "kubevirt.io/kubevirt/pkg/storage/reservation"
        virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
        "kubevirt.io/kubevirt/pkg/virt-handler/selinux"
)

var defaultBackoffTime = []time.Duration{1 * time.Second, 2 * time.Second, 5 * time.Second, 10 * time.Second}

type controlledDevice struct {
        devicePlugin Device
        started      bool
        stopChan     chan struct{}
        backoff      []time.Duration
}

func (c *controlledDevice) Start() {
        if c.started {
                return
        }

        stop := make(chan struct{})

        logger := log.DefaultLogger()
        dev := c.devicePlugin
        deviceName := dev.GetDeviceName()
        logger.Infof("Starting a device plugin for device: %s", deviceName)
        retries := 0

        backoff := c.backoff
        if backoff == nil {
                backoff = defaultBackoffTime
        }

        go func() {
                for {
                        err := dev.Start(stop)
                        if err != nil {
                                logger.Reason(err).Errorf("Error starting %s device plugin", deviceName)
                                retries = int(math.Min(float64(retries+1), float64(len(backoff)-1)))
                        } else {
                                retries = 0
                        }

                        select {
                        case <-stop:
                                // Ok we don't want to re-register
                                return
                        case <-time.After(backoff[retries]):
                                // Wait a little and re-register
                                continue
                        }
                }
        }()

        c.stopChan = stop
        c.started = true
}

func (c *controlledDevice) Stop() {
        if !c.started {
                return
        }
        close(c.stopChan)

        c.stopChan = nil
        c.started = false
}

func (c *controlledDevice) GetName() string {
        return c.devicePlugin.GetDeviceName()
}

func PermanentHostDevicePlugins(hypervisorDevice string, maxDevices int, permissions string) []Device {
        var permanentDevicePluginPaths = map[string]string{
                hypervisorDevice: "/dev/" + hypervisorDevice,
                "tun":            "/dev/net/tun",
                "vhost-net":      "/dev/vhost-net",
        }

        ret := make([]Device, 0, len(permanentDevicePluginPaths))
        for name, path := range permanentDevicePluginPaths {
                ret = append(ret, NewGenericDevicePlugin(name, path, maxDevices, permissions, name != hypervisorDevice))
        }
        return ret
}

type DeviceControllerInterface interface {
        Initialized() bool
        RefreshMediatedDeviceTypes()
}

type DeviceController struct {
        permanentPlugins         map[string]Device
        startedPlugins           map[string]controlledDevice
        startedPluginsMutex      sync.Mutex
        host                     string
        maxDevices               int
        permissions              string
        backoff                  []time.Duration
        virtConfig               *virtconfig.ClusterConfig
        mdevTypesManager         *MDEVTypesManager
        nodeStore                cache.Store
        mdevRefreshWG            *sync.WaitGroup
        lastTDXAttestationConfig *tdxConfigState
}

type tdxConfigState struct {
        socketPath string
        requireQGS bool
}

func NewDeviceController(
        host string,
        maxDevices int,
        permissions string,
        permanentPlugins []Device,
        clusterConfig *virtconfig.ClusterConfig,
        nodeStore cache.Store,
) *DeviceController {
        permanentPluginsMap := make(map[string]Device, len(permanentPlugins))
        for i := range permanentPlugins {
                permanentPluginsMap[permanentPlugins[i].GetDeviceName()] = permanentPlugins[i]
        }

        controller := &DeviceController{
                permanentPlugins: permanentPluginsMap,
                startedPlugins:   map[string]controlledDevice{},
                host:             host,
                maxDevices:       maxDevices,
                permissions:      permissions,
                backoff:          defaultBackoffTime,
                virtConfig:       clusterConfig,
                mdevTypesManager: NewMDEVTypesManager(),
                nodeStore:        nodeStore,
                mdevRefreshWG:    &sync.WaitGroup{},
        }

        return controller
}

func (c *DeviceController) NodeHasDevice(devicePath string) bool {
        _, err := os.Stat(devicePath)
        // Since this is a boolean question, any error means "no"
        return err == nil
}

func (c *DeviceController) updateTdxDevice() (Device, error) {
        maxTDXVMs, err := cgroup.GetMiscCapacity("tdx")
        if err != nil {
                return nil, fmt.Errorf("failed to get TDX capacity from misc.capacity: %v", err)
        } else if maxTDXVMs > 0 {
                var selinuxExecutor selinux.SELinuxExecutor
                socketPath := c.virtConfig.GetQGSSocketPath()
                socketDir := path.Dir(socketPath)
                socketFile := path.Base(socketPath)
                var tdxPlugin Device
                var err error
                if c.virtConfig.RequireQGS() {
                        tdxPlugin, err = NewSocketDevicePlugin(services.TdxDeviceName, socketDir, socketFile, maxTDXVMs, selinuxExecutor, nil, true)
                } else {
                        tdxPlugin = NewOptionalSocketDevicePlugin(services.TdxDeviceName, socketDir, socketFile, maxTDXVMs, selinuxExecutor, nil, true)
                }
                return tdxPlugin, err
        } else {
                return nil, fmt.Errorf("an invalid device capacity of %d was report for tdx", maxTDXVMs)
        }
}

// updatePermittedHostDevicePlugins returns a slice of device plugins for permitted devices which are present on the node
func (c *DeviceController) updatePermittedHostDevicePlugins() []Device {
        var permittedDevices []Device

        if c.virtConfig.WorkloadEncryptionTDXEnabled() {
                tdxPlugin, err := c.updateTdxDevice()
                if err != nil {
                        log.Log.Reason(err).Errorf("failed to configure the TDX-QGS device plugin")
                } else {
                        permittedDevices = append(permittedDevices, tdxPlugin)
                }
        }

        var featureGatedGenericDevices = []struct {
                Name      string
                Path      string
                IsAllowed func() bool
        }{
                {"sev", "/dev/sev", c.virtConfig.WorkloadEncryptionSEVEnabled},
                {"vhost-vsock", "/dev/vhost-vsock", c.virtConfig.VSOCKEnabled},
        }

        for _, dev := range featureGatedGenericDevices {
                if dev.IsAllowed() {
                        permittedDevices = append(
                                permittedDevices,
                                NewGenericDevicePlugin(dev.Name, dev.Path, c.maxDevices, c.permissions, true),
                        )
                }
        }

        if c.virtConfig.IOMMUFDEnabled() {
                permittedDevices = append(permittedDevices, NewIOMMUFDDevicePlugin(c.maxDevices))
        }

        if c.virtConfig.PersistentReservationEnabled() {
                d, err := NewSocketDevicePlugin(reservation.GetPrResourceName(), reservation.GetPrHelperSocketDir(), reservation.GetPrHelperSocket(), c.maxDevices, selinux.SELinuxExecutor{}, NewPermissionManager(), false)
                if err != nil {
                        log.Log.Reason(err).Errorf("failed to configure the desired mdev types, failed to get node details")
                } else {
                        permittedDevices = append(permittedDevices, d)
                }
        }

        hostDevs := c.virtConfig.GetPermittedHostDevices()
        if hostDevs == nil {
                return permittedDevices
        }

        if len(hostDevs.PciHostDevices) != 0 {
                supportedPCIDeviceMap := make(map[string]string)
                for _, pciDev := range hostDevs.PciHostDevices {
                        log.Log.V(4).Infof("Permitted PCI device in the cluster, ID: %s, resourceName: %s, externalProvider: %t",
                                strings.ToLower(pciDev.PCIVendorSelector),
                                pciDev.ResourceName,
                                pciDev.ExternalResourceProvider)
                        // do not add a device plugin for this resource if it's being provided via an external device plugin
                        if !pciDev.ExternalResourceProvider {
                                supportedPCIDeviceMap[strings.ToLower(pciDev.PCIVendorSelector)] = pciDev.ResourceName
                        }
                }
                for pciResourceName, pciDevices := range discoverPermittedHostPCIDevices(supportedPCIDeviceMap) {
                        log.Log.V(4).Infof("Discovered PCIs %d devices on the node for the resource: %s", len(pciDevices), pciResourceName)
                        // add a device plugin only for new devices
                        permittedDevices = append(permittedDevices, NewPCIDevicePlugin(pciDevices, pciResourceName))
                }
        }
        if len(hostDevs.MediatedDevices) != 0 {
                supportedMdevsMap := make(map[string]string)
                for _, supportedMdev := range hostDevs.MediatedDevices {
                        log.Log.V(4).Infof("Permitted mediated device in the cluster, ID: %s, resourceName: %s",
                                supportedMdev.MDEVNameSelector,
                                supportedMdev.ResourceName)
                        // do not add a device plugin for this resource if it's being provided via an external device plugin
                        if !supportedMdev.ExternalResourceProvider {
                                selector := removeSelectorSpaces(supportedMdev.MDEVNameSelector)
                                supportedMdevsMap[selector] = supportedMdev.ResourceName
                        }
                }
                for mdevTypeName, mdevUUIDs := range discoverPermittedHostMediatedDevices(supportedMdevsMap) {
                        mdevResourceName := supportedMdevsMap[mdevTypeName]
                        log.Log.V(4).Infof("Discovered mediated device on the node, type: %s, resourceName: %s", mdevTypeName, mdevResourceName)

                        permittedDevices = append(permittedDevices, NewMediatedDevicePlugin(mdevUUIDs, mdevResourceName))
                }
        }

        for resourceName, pluginDevices := range discoverAllowedUSBDevices(hostDevs.USB) {
                permittedDevices = append(permittedDevices, NewUSBDevicePlugin(resourceName, pluginDevices))
        }

        return permittedDevices
}

func removeSelectorSpaces(selectorName string) string {
        // The name usually contain spaces which should be replaced with _
        // Such as GRID T4-1Q
        typeNameStr := strings.Replace(selectorName, " ", "_", -1)
        typeNameStr = strings.TrimSpace(typeNameStr)
        return typeNameStr
}

func (c *DeviceController) splitPermittedDevices(devices []Device) (map[string]Device, map[string]struct{}) {
        devicePluginsToRun := make(map[string]Device)
        devicePluginsToStop := make(map[string]struct{})

        // generate a map of currently started device plugins
        for resourceName := range c.startedPlugins {
                _, isPermanent := c.permanentPlugins[resourceName]
                if !isPermanent {
                        devicePluginsToStop[resourceName] = struct{}{}
                }
        }

        for _, device := range devices {
                if _, isRunning := c.startedPlugins[device.GetDeviceName()]; !isRunning {
                        devicePluginsToRun[device.GetDeviceName()] = device
                } else {
                        delete(devicePluginsToStop, device.GetDeviceName())
                }
        }

        return devicePluginsToRun, devicePluginsToStop
}

func (c *DeviceController) RefreshMediatedDeviceTypes() {
        go func() {
                if c.refreshMediatedDeviceTypes() {
                        c.refreshPermittedDevices()
                }
        }()
}

func (c *DeviceController) getExternallyProvidedMdevs() map[string]struct{} {
        externalMdevResourcesMap := make(map[string]struct{})
        if hostDevs := c.virtConfig.GetPermittedHostDevices(); hostDevs != nil {
                for _, supportedMdev := range hostDevs.MediatedDevices {
                        if supportedMdev.ExternalResourceProvider {
                                selector := removeSelectorSpaces(supportedMdev.MDEVNameSelector)
                                externalMdevResourcesMap[selector] = struct{}{}
                        }
                }
        }
        return externalMdevResourcesMap
}

func (c *DeviceController) refreshMediatedDeviceTypes() bool {
        // the handling of mediated device is disabled
        if c.virtConfig.MediatedDevicesHandlingDisabled() {
                return false
        }

        node, err := c.getNode()
        if err != nil {
                log.Log.Reason(err).Errorf("failed to configure the desired mdev types, failed to get node details")
                return false
        }
        externallyProvidedMdevMap := c.getExternallyProvidedMdevs()

        nodeDesiredMdevTypesList := c.virtConfig.GetDesiredMDEVTypes(node)
        requiresDevicePluginsUpdate, err := c.mdevTypesManager.updateMDEVTypesConfiguration(nodeDesiredMdevTypesList, externallyProvidedMdevMap)
        if err != nil {
                log.Log.Reason(err).Errorf("failed to configure the desired mdev types: %s", strings.Join(nodeDesiredMdevTypesList, ", "))
        }
        return requiresDevicePluginsUpdate
}

func (c *DeviceController) getNode() (*k8sv1.Node, error) {
        nodeObj, exists, err := c.nodeStore.GetByKey(c.host)
        if err != nil {
                log.DefaultLogger().Errorf("Unable to get node: %s", err.Error())
                return nil, err
        }
        if !exists {
                log.DefaultLogger().Errorf("node %s does not exist", c.host)
                return nil, fmt.Errorf("node %s does not exist", c.host)
        }

        node, ok := nodeObj.(*k8sv1.Node)
        if !ok {
                return nil, fmt.Errorf("unknown object type found in node informer")
        }

        return node, nil
}

func (c *DeviceController) refreshTDXConfig() bool {
        if !c.virtConfig.WorkloadEncryptionTDXEnabled() {
                // TDX not enabled, reset tracking
                c.lastTDXAttestationConfig = nil
                return false
        }

        currentTDXAttestationConfig := tdxConfigState{
                socketPath: c.virtConfig.GetQGSSocketPath(),
                requireQGS: c.virtConfig.RequireQGS(),
        }

        changed := c.lastTDXAttestationConfig == nil || *c.lastTDXAttestationConfig != currentTDXAttestationConfig

        if changed {
                c.lastTDXAttestationConfig = &currentTDXAttestationConfig
        }

        return changed
}

func (c *DeviceController) refreshPermittedDevices() {
        c.mdevRefreshWG.Add(1)
        logger := log.DefaultLogger()
        var debugDevAdded []string
        var debugDevRemoved []string

        // This function can be called multiple times in parallel, either because of multiple
        //   informer callbacks for the same event, or because the configmap was quickly updated
        //   multiple times in a row. To avoid starting/stopping device plugins multiple times,
        //   we need to protect c.startedPlugins, which we read from in
        //   c.updatePermittedHostDevicePlugins() and write to below.
        c.startedPluginsMutex.Lock()
        defer c.startedPluginsMutex.Unlock()

        // Check if QGS config changed and restart the QGS device plugin if needed
        if changed := c.refreshTDXConfig(); changed {
                if _, exists := c.startedPlugins[services.TdxDevice]; exists {
                        logger.Infof("QGS config changed, restarting QGS device plugin")
                        // only call stopDevice here,
                        // startDevice will be called when updatePermittedHostDevicePlugins() is called
                        c.stopDevice(services.TdxDevice)
                }
        }

        enabledDevicePlugins, disabledDevicePlugins := c.splitPermittedDevices(
                c.updatePermittedHostDevicePlugins(),
        )

        // start device plugin for newly permitted devices
        for resourceName, dev := range enabledDevicePlugins {
                c.startDevice(resourceName, dev)
                debugDevAdded = append(debugDevAdded, resourceName)
        }
        // remove device plugin for now forbidden devices
        for resourceName := range disabledDevicePlugins {
                c.stopDevice(resourceName)
                debugDevRemoved = append(debugDevRemoved, resourceName)
        }

        logger.V(3).Info("refreshed device plugins for permitted/forbidden host devices")
        if len(debugDevAdded) > 0 {
                logger.Infof("enabled device-plugins for: %v", debugDevAdded)
        }
        if len(debugDevRemoved) > 0 {
                logger.Infof("disabled device-plugins for: %v", debugDevRemoved)
        }
        c.mdevRefreshWG.Done()
}

func (c *DeviceController) startDevice(resourceName string, dev Device) {
        c.stopDevice(resourceName)
        controlledDev := controlledDevice{
                devicePlugin: dev,
                backoff:      c.backoff,
        }
        controlledDev.Start()
        c.startedPlugins[resourceName] = controlledDev
}

func (c *DeviceController) stopDevice(resourceName string) {
        dev, exists := c.startedPlugins[resourceName]
        if exists {
                dev.Stop()
                delete(c.startedPlugins, resourceName)
        }
}

func (c *DeviceController) Run(stop chan struct{}) {
        logger := log.DefaultLogger()

        // start the permanent DevicePlugins
        func() {
                c.startedPluginsMutex.Lock()
                defer c.startedPluginsMutex.Unlock()
                for name, dev := range c.permanentPlugins {
                        c.startDevice(name, dev)
                }
        }()

        refreshMediatedDeviceTypesFn := func() {
                c.refreshMediatedDeviceTypes()
        }
        c.virtConfig.SetConfigModifiedCallback(refreshMediatedDeviceTypesFn)
        c.virtConfig.SetConfigModifiedCallback(c.refreshPermittedDevices)
        c.refreshPermittedDevices()

        // keep running until stop
        <-stop

        // stop all device plugins
        func() {
                c.startedPluginsMutex.Lock()
                defer c.startedPluginsMutex.Unlock()
                for name := range c.startedPlugins {
                        c.stopDevice(name)
                }
        }()

        // wait for any concurrent mdev refreshes to finish
        c.mdevRefreshWG.Wait()

        logger.Info("Shutting down device plugin controller")
}

func (c *DeviceController) Initialized() bool {
        c.startedPluginsMutex.Lock()
        defer c.startedPluginsMutex.Unlock()
        for _, dev := range c.startedPlugins {
                if !dev.devicePlugin.GetInitialized() {
                        return false
                }
        }

        return true
}

1	/*
2	* This file is part of the KubeVirt project
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*
16	* Copyright The KubeVirt Authors.
17	*
18	*/
19
20	package device_manager
21
22	import (
23	"fmt"
24	"math"
25	"os"
26	"path"
27	"strings"
28	"sync"
29	"time"
30
31	k8sv1 "k8s.io/api/core/v1"
32	"k8s.io/client-go/tools/cache"
33
34	"kubevirt.io/kubevirt/pkg/virt-controller/services"
35
36	"kubevirt.io/kubevirt/pkg/virt-handler/cgroup"
37
38	"kubevirt.io/client-go/log"
39
40	"kubevirt.io/kubevirt/pkg/storage/reservation"
41	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
42	"kubevirt.io/kubevirt/pkg/virt-handler/selinux"
43	)
44
45	var defaultBackoffTime = []time.Duration{1 * time.Second, 2 * time.Second, 5 * time.Second, 10 * time.Second}
46
47	type controlledDevice struct {
48	devicePlugin Device
49	started bool
50	stopChan chan struct{}
51	backoff []time.Duration
52	}
53
54	func (c *controlledDevice) Start() {	13✔
55	if c.started {	13✔
56	return	×
57	}	×
58
59	stop := make(chan struct{})	13✔
60		13✔
61	logger := log.DefaultLogger()	13✔
62	dev := c.devicePlugin	13✔
63	deviceName := dev.GetDeviceName()	13✔
64	logger.Infof("Starting a device plugin for device: %s", deviceName)	13✔
65	retries := 0	13✔
66		13✔
67	backoff := c.backoff	13✔
68	if backoff == nil {	13✔
69	backoff = defaultBackoffTime	×
70	}	×
71
72	go func() {	26✔
73	for {	29✔
74	err := dev.Start(stop)	16✔
75	if err != nil {	24✔
76	logger.Reason(err).Errorf("Error starting %s device plugin", deviceName)	8✔
77	retries = int(math.Min(float64(retries+1), float64(len(backoff)-1)))	8✔
78	} else {	16✔
79	retries = 0	8✔
80	}	8✔
81
82	select {	16✔
83	case <-stop:	13✔
84	// Ok we don't want to re-register	13✔
85	return	13✔
86	case <-time.After(backoff[retries]):	3✔
87	// Wait a little and re-register	3✔
88	continue	3✔
89	}
90	}
91	}()
92
93	c.stopChan = stop	13✔
94	c.started = true	13✔
95	}
96
97	func (c *controlledDevice) Stop() {	13✔
98	if !c.started {	13✔
99	return	×
100	}	×
101	close(c.stopChan)	13✔
102		13✔
103	c.stopChan = nil	13✔
104	c.started = false	13✔
105	}
106
107	func (c *controlledDevice) GetName() string {	×
108	return c.devicePlugin.GetDeviceName()	×
109	}	×
110
111	func PermanentHostDevicePlugins(hypervisorDevice string, maxDevices int, permissions string) []Device {	136✔
112	var permanentDevicePluginPaths = map[string]string{	136✔
113	hypervisorDevice: "/dev/" + hypervisorDevice,	136✔
114	"tun": "/dev/net/tun",	136✔
115	"vhost-net": "/dev/vhost-net",	136✔
116	}	136✔
117		136✔
118	ret := make([]Device, 0, len(permanentDevicePluginPaths))	136✔
119	for name, path := range permanentDevicePluginPaths {	544✔
120	ret = append(ret, NewGenericDevicePlugin(name, path, maxDevices, permissions, name != hypervisorDevice))	408✔
121	}	408✔
122	return ret	136✔
123	}
124
125	type DeviceControllerInterface interface {
126	Initialized() bool
127	RefreshMediatedDeviceTypes()
128	}
129
130	type DeviceController struct {
131	permanentPlugins map[string]Device
132	startedPlugins map[string]controlledDevice
133	startedPluginsMutex sync.Mutex
134	host string
135	maxDevices int
136	permissions string
137	backoff []time.Duration
138	virtConfig *virtconfig.ClusterConfig
139	mdevTypesManager *MDEVTypesManager
140	nodeStore cache.Store
141	mdevRefreshWG *sync.WaitGroup
142	lastTDXAttestationConfig *tdxConfigState
143	}
144
145	type tdxConfigState struct {
146	socketPath string
147	requireQGS bool
148	}
149
150	func NewDeviceController(
151	host string,
152	maxDevices int,
153	permissions string,
154	permanentPlugins []Device,
155	clusterConfig *virtconfig.ClusterConfig,
156	nodeStore cache.Store,
157	) *DeviceController {	149✔
158	permanentPluginsMap := make(map[string]Device, len(permanentPlugins))	149✔
159	for i := range permanentPlugins {	563✔
160	permanentPluginsMap[permanentPlugins[i].GetDeviceName()] = permanentPlugins[i]	414✔
161	}	414✔
162
163	controller := &DeviceController{	149✔
164	permanentPlugins: permanentPluginsMap,	149✔
165	startedPlugins: map[string]controlledDevice{},	149✔
166	host: host,	149✔
167	maxDevices: maxDevices,	149✔
168	permissions: permissions,	149✔
169	backoff: defaultBackoffTime,	149✔
170	virtConfig: clusterConfig,	149✔
171	mdevTypesManager: NewMDEVTypesManager(),	149✔
172	nodeStore: nodeStore,	149✔
173	mdevRefreshWG: &sync.WaitGroup{},	149✔
174	}	149✔
175		149✔
176	return controller	149✔
177	}
178
179	func (c *DeviceController) NodeHasDevice(devicePath string) bool {	4✔
180	_, err := os.Stat(devicePath)	4✔
181	// Since this is a boolean question, any error means "no"	4✔
182	return err == nil	4✔
183	}	4✔
184
185	func (c *DeviceController) updateTdxDevice() (Device, error) {	×
186	maxTDXVMs, err := cgroup.GetMiscCapacity("tdx")	×
187	if err != nil {	×
188	return nil, fmt.Errorf("failed to get TDX capacity from misc.capacity: %v", err)	×
189	} else if maxTDXVMs > 0 {	×
190	var selinuxExecutor selinux.SELinuxExecutor	×
191	socketPath := c.virtConfig.GetQGSSocketPath()	×
192	socketDir := path.Dir(socketPath)	×
193	socketFile := path.Base(socketPath)	×
194	var tdxPlugin Device	×
195	var err error	×
196	if c.virtConfig.RequireQGS() {	×
197	tdxPlugin, err = NewSocketDevicePlugin(services.TdxDeviceName, socketDir, socketFile, maxTDXVMs, selinuxExecutor, nil, true)	×
198	} else {	×
199	tdxPlugin = NewOptionalSocketDevicePlugin(services.TdxDeviceName, socketDir, socketFile, maxTDXVMs, selinuxExecutor, nil, true)	×
200	}	×
201	return tdxPlugin, err	×
202	} else {	×
203	return nil, fmt.Errorf("an invalid device capacity of %d was report for tdx", maxTDXVMs)	×
204	}	×
205	}
206
207	// updatePermittedHostDevicePlugins returns a slice of device plugins for permitted devices which are present on the node
208	func (c *DeviceController) updatePermittedHostDevicePlugins() []Device {	14✔
209	var permittedDevices []Device	14✔
210		14✔
211	if c.virtConfig.WorkloadEncryptionTDXEnabled() {	14✔
212	tdxPlugin, err := c.updateTdxDevice()	×
213	if err != nil {	×
214	log.Log.Reason(err).Errorf("failed to configure the TDX-QGS device plugin")	×
215	} else {	×
216	permittedDevices = append(permittedDevices, tdxPlugin)	×
217	}	×
218	}
219
220	var featureGatedGenericDevices = []struct {	14✔
221	Name string	14✔
222	Path string	14✔
223	IsAllowed func() bool	14✔
224	}{	14✔
225	{"sev", "/dev/sev", c.virtConfig.WorkloadEncryptionSEVEnabled},	14✔
226	{"vhost-vsock", "/dev/vhost-vsock", c.virtConfig.VSOCKEnabled},	14✔
227	}	14✔
228		14✔
229	for _, dev := range featureGatedGenericDevices {	42✔
230	if dev.IsAllowed() {	42✔
231	permittedDevices = append(	14✔
232	permittedDevices,	14✔
233	NewGenericDevicePlugin(dev.Name, dev.Path, c.maxDevices, c.permissions, true),	14✔
234	)	14✔
235	}	14✔
236	}
237
238	if c.virtConfig.IOMMUFDEnabled() {	14✔
239	permittedDevices = append(permittedDevices, NewIOMMUFDDevicePlugin(c.maxDevices))	×
240	}	×
241
242	if c.virtConfig.PersistentReservationEnabled() {	14✔
243	d, err := NewSocketDevicePlugin(reservation.GetPrResourceName(), reservation.GetPrHelperSocketDir(), reservation.GetPrHelperSocket(), c.maxDevices, selinux.SELinuxExecutor{}, NewPermissionManager(), false)	×
244	if err != nil {	×
245	log.Log.Reason(err).Errorf("failed to configure the desired mdev types, failed to get node details")	×
246	} else {	×
247	permittedDevices = append(permittedDevices, d)	×
248	}	×
249	}
250
251	hostDevs := c.virtConfig.GetPermittedHostDevices()	14✔
252	if hostDevs == nil {	18✔
253	return permittedDevices	4✔
254	}	4✔
255
256	if len(hostDevs.PciHostDevices) != 0 {	17✔
257	supportedPCIDeviceMap := make(map[string]string)	7✔
258	for _, pciDev := range hostDevs.PciHostDevices {	20✔
259	log.Log.V(4).Infof("Permitted PCI device in the cluster, ID: %s, resourceName: %s, externalProvider: %t",	13✔
260	strings.ToLower(pciDev.PCIVendorSelector),	13✔
261	pciDev.ResourceName,	13✔
262	pciDev.ExternalResourceProvider)	13✔
263	// do not add a device plugin for this resource if it's being provided via an external device plugin	13✔
264	if !pciDev.ExternalResourceProvider {	14✔
265	supportedPCIDeviceMap[strings.ToLower(pciDev.PCIVendorSelector)] = pciDev.ResourceName	1✔
266	}	1✔
267	}
268	for pciResourceName, pciDevices := range discoverPermittedHostPCIDevices(supportedPCIDeviceMap) {	8✔
269	log.Log.V(4).Infof("Discovered PCIs %d devices on the node for the resource: %s", len(pciDevices), pciResourceName)	1✔
270	// add a device plugin only for new devices	1✔
271	permittedDevices = append(permittedDevices, NewPCIDevicePlugin(pciDevices, pciResourceName))	1✔
272	}	1✔
273	}
274	if len(hostDevs.MediatedDevices) != 0 {	11✔
275	supportedMdevsMap := make(map[string]string)	1✔
276	for _, supportedMdev := range hostDevs.MediatedDevices {	2✔
277	log.Log.V(4).Infof("Permitted mediated device in the cluster, ID: %s, resourceName: %s",	1✔
278	supportedMdev.MDEVNameSelector,	1✔
279	supportedMdev.ResourceName)	1✔
280	// do not add a device plugin for this resource if it's being provided via an external device plugin	1✔
281	if !supportedMdev.ExternalResourceProvider {	2✔
282	selector := removeSelectorSpaces(supportedMdev.MDEVNameSelector)	1✔
283	supportedMdevsMap[selector] = supportedMdev.ResourceName	1✔
284	}	1✔
285	}
286	for mdevTypeName, mdevUUIDs := range discoverPermittedHostMediatedDevices(supportedMdevsMap) {	2✔
287	mdevResourceName := supportedMdevsMap[mdevTypeName]	1✔
288	log.Log.V(4).Infof("Discovered mediated device on the node, type: %s, resourceName: %s", mdevTypeName, mdevResourceName)	1✔
289		1✔
290	permittedDevices = append(permittedDevices, NewMediatedDevicePlugin(mdevUUIDs, mdevResourceName))	1✔
291	}	1✔
292	}
293
294	for resourceName, pluginDevices := range discoverAllowedUSBDevices(hostDevs.USB) {	10✔
295	permittedDevices = append(permittedDevices, NewUSBDevicePlugin(resourceName, pluginDevices))	×
296	}	×
297
298	return permittedDevices	10✔
299	}
300
301	func removeSelectorSpaces(selectorName string) string {	5✔
302	// The name usually contain spaces which should be replaced with _	5✔
303	// Such as GRID T4-1Q	5✔
304	typeNameStr := strings.Replace(selectorName, " ", "_", -1)	5✔
305	typeNameStr = strings.TrimSpace(typeNameStr)	5✔
306	return typeNameStr	5✔
307	}	5✔
308
309	func (c *DeviceController) splitPermittedDevices(devices []Device) (map[string]Device, map[string]struct{}) {	14✔
310	devicePluginsToRun := make(map[string]Device)	14✔
311	devicePluginsToStop := make(map[string]struct{})	14✔
312		14✔
313	// generate a map of currently started device plugins	14✔
314	for resourceName := range c.startedPlugins {	35✔
315	_, isPermanent := c.permanentPlugins[resourceName]	21✔
316	if !isPermanent {	30✔
317	devicePluginsToStop[resourceName] = struct{}{}	9✔
318	}	9✔
319	}
320
321	for _, device := range devices {	30✔
322	if _, isRunning := c.startedPlugins[device.GetDeviceName()]; !isRunning {	27✔
323	devicePluginsToRun[device.GetDeviceName()] = device	11✔
324	} else {	16✔
325	delete(devicePluginsToStop, device.GetDeviceName())	5✔
326	}	5✔
327	}
328
329	return devicePluginsToRun, devicePluginsToStop	14✔
330	}
331
332	func (c *DeviceController) RefreshMediatedDeviceTypes() {	×
333	go func() {	×
334	if c.refreshMediatedDeviceTypes() {	×
335	c.refreshPermittedDevices()	×
336	}	×
337	}()
338	}
339
340	func (c *DeviceController) getExternallyProvidedMdevs() map[string]struct{} {	11✔
341	externalMdevResourcesMap := make(map[string]struct{})	11✔
342	if hostDevs := c.virtConfig.GetPermittedHostDevices(); hostDevs != nil {	11✔
343	for _, supportedMdev := range hostDevs.MediatedDevices {	×
344	if supportedMdev.ExternalResourceProvider {	×
345	selector := removeSelectorSpaces(supportedMdev.MDEVNameSelector)	×
346	externalMdevResourcesMap[selector] = struct{}{}	×
347	}	×
348	}
349	}
350	return externalMdevResourcesMap	11✔
351	}
352
353	func (c *DeviceController) refreshMediatedDeviceTypes() bool {	16✔
354	// the handling of mediated device is disabled	16✔
355	if c.virtConfig.MediatedDevicesHandlingDisabled() {	16✔
356	return false	×
357	}	×
358
359	node, err := c.getNode()	16✔
360	if err != nil {	21✔
361	log.Log.Reason(err).Errorf("failed to configure the desired mdev types, failed to get node details")	5✔
362	return false	5✔
363	}	5✔
364	externallyProvidedMdevMap := c.getExternallyProvidedMdevs()	11✔
365		11✔
366	nodeDesiredMdevTypesList := c.virtConfig.GetDesiredMDEVTypes(node)	11✔
367	requiresDevicePluginsUpdate, err := c.mdevTypesManager.updateMDEVTypesConfiguration(nodeDesiredMdevTypesList, externallyProvidedMdevMap)	11✔
368	if err != nil {	11✔
369	log.Log.Reason(err).Errorf("failed to configure the desired mdev types: %s", strings.Join(nodeDesiredMdevTypesList, ", "))	×
370	}	×
371	return requiresDevicePluginsUpdate	11✔
372	}
373
374	func (c DeviceController) getNode() (k8sv1.Node, error) {	16✔
375	nodeObj, exists, err := c.nodeStore.GetByKey(c.host)	16✔
376	if err != nil {	16✔
377	log.DefaultLogger().Errorf("Unable to get node: %s", err.Error())	×
378	return nil, err	×
379	}	×
380	if !exists {	21✔
381	log.DefaultLogger().Errorf("node %s does not exist", c.host)	5✔
382	return nil, fmt.Errorf("node %s does not exist", c.host)	5✔
383	}	5✔
384
385	node, ok := nodeObj.(*k8sv1.Node)	11✔
386	if !ok {	11✔
387	return nil, fmt.Errorf("unknown object type found in node informer")	×
388	}	×
389
390	return node, nil	11✔
391	}
392
393	func (c *DeviceController) refreshTDXConfig() bool {	10✔
394	if !c.virtConfig.WorkloadEncryptionTDXEnabled() {	20✔
395	// TDX not enabled, reset tracking	10✔
396	c.lastTDXAttestationConfig = nil	10✔
397	return false	10✔
398	}	10✔
399
400	currentTDXAttestationConfig := tdxConfigState{	×
401	socketPath: c.virtConfig.GetQGSSocketPath(),	×
402	requireQGS: c.virtConfig.RequireQGS(),	×
403	}	×
404		×
405	changed := c.lastTDXAttestationConfig == nil \|\| *c.lastTDXAttestationConfig != currentTDXAttestationConfig	×
406		×
407	if changed {	×
408	c.lastTDXAttestationConfig = &currentTDXAttestationConfig	×
409	}	×
410
411	return changed	×
412	}
413
414	func (c *DeviceController) refreshPermittedDevices() {	10✔
415	c.mdevRefreshWG.Add(1)	10✔
416	logger := log.DefaultLogger()	10✔
417	var debugDevAdded []string	10✔
418	var debugDevRemoved []string	10✔
419		10✔
420	// This function can be called multiple times in parallel, either because of multiple	10✔
421	// informer callbacks for the same event, or because the configmap was quickly updated	10✔
422	// multiple times in a row. To avoid starting/stopping device plugins multiple times,	10✔
423	// we need to protect c.startedPlugins, which we read from in	10✔
424	// c.updatePermittedHostDevicePlugins() and write to below.	10✔
425	c.startedPluginsMutex.Lock()	10✔
426	defer c.startedPluginsMutex.Unlock()	10✔
427		10✔
428	// Check if QGS config changed and restart the QGS device plugin if needed	10✔
429	if changed := c.refreshTDXConfig(); changed {	10✔
430	if _, exists := c.startedPlugins[services.TdxDevice]; exists {	×
431	logger.Infof("QGS config changed, restarting QGS device plugin")	×
432	// only call stopDevice here,	×
433	// startDevice will be called when updatePermittedHostDevicePlugins() is called	×
434	c.stopDevice(services.TdxDevice)	×
435	}	×
436	}
437
438	enabledDevicePlugins, disabledDevicePlugins := c.splitPermittedDevices(	10✔
439	c.updatePermittedHostDevicePlugins(),	10✔
440	)	10✔
441		10✔
442	// start device plugin for newly permitted devices	10✔
443	for resourceName, dev := range enabledDevicePlugins {	15✔
444	c.startDevice(resourceName, dev)	5✔
445	debugDevAdded = append(debugDevAdded, resourceName)	5✔
446	}	5✔
447	// remove device plugin for now forbidden devices
448	for resourceName := range disabledDevicePlugins {	12✔
449	c.stopDevice(resourceName)	2✔
450	debugDevRemoved = append(debugDevRemoved, resourceName)	2✔
451	}	2✔
452
453	logger.V(3).Info("refreshed device plugins for permitted/forbidden host devices")	10✔
454	if len(debugDevAdded) > 0 {	15✔
455	logger.Infof("enabled device-plugins for: %v", debugDevAdded)	5✔
456	}	5✔
457	if len(debugDevRemoved) > 0 {	11✔
458	logger.Infof("disabled device-plugins for: %v", debugDevRemoved)	1✔
459	}	1✔
460	c.mdevRefreshWG.Done()	10✔
461	}
462
463	func (c *DeviceController) startDevice(resourceName string, dev Device) {	13✔
464	c.stopDevice(resourceName)	13✔
465	controlledDev := controlledDevice{	13✔
466	devicePlugin: dev,	13✔
467	backoff: c.backoff,	13✔
468	}	13✔
469	controlledDev.Start()	13✔
470	c.startedPlugins[resourceName] = controlledDev	13✔
471	}	13✔
472
473	func (c *DeviceController) stopDevice(resourceName string) {	26✔
474	dev, exists := c.startedPlugins[resourceName]	26✔
475	if exists {	39✔
476	dev.Stop()	13✔
477	delete(c.startedPlugins, resourceName)	13✔
478	}	13✔
479	}
480
481	func (c *DeviceController) Run(stop chan struct{}) {	5✔
482	logger := log.DefaultLogger()	5✔
483		5✔
484	// start the permanent DevicePlugins	5✔
485	func() {	10✔
486	c.startedPluginsMutex.Lock()	5✔
487	defer c.startedPluginsMutex.Unlock()	5✔
488	for name, dev := range c.permanentPlugins {	11✔
489	c.startDevice(name, dev)	6✔
490	}	6✔
491	}()
492
493	refreshMediatedDeviceTypesFn := func() {	10✔
494	c.refreshMediatedDeviceTypes()	5✔
495	}	5✔
496	c.virtConfig.SetConfigModifiedCallback(refreshMediatedDeviceTypesFn)	5✔
497	c.virtConfig.SetConfigModifiedCallback(c.refreshPermittedDevices)	5✔
498	c.refreshPermittedDevices()	5✔
499		5✔
500	// keep running until stop	5✔
501	<-stop	5✔
502		5✔
503	// stop all device plugins	5✔
504	func() {	10✔
505	c.startedPluginsMutex.Lock()	5✔
506	defer c.startedPluginsMutex.Unlock()	5✔
507	for name := range c.startedPlugins {	16✔
508	c.stopDevice(name)	11✔
509	}	11✔
510	}()
511
512	// wait for any concurrent mdev refreshes to finish
513	c.mdevRefreshWG.Wait()	5✔
514		5✔
515	logger.Info("Shutting down device plugin controller")	5✔
516	}
517
UNCOV 518	func (c *DeviceController) Initialized() bool {	×
UNCOV 519	c.startedPluginsMutex.Lock()	×
UNCOV 520	defer c.startedPluginsMutex.Unlock()	×
UNCOV 521	for _, dev := range c.startedPlugins {	×
UNCOV 522	if !dev.devicePlugin.GetInitialized() {	×
523	return false	×
524	}	×
525	}
526
UNCOV 527	return true	×
528	}

kubevirt / kubevirt / 1460f023-f8c8-4587-b36b-5e6fe90fca99

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous