3751025296

Build Type

github

Committed by GitHub

Commit Message

Merge 421284b55 into 788d76f7e

Pull Request Pull Request #365: Implementation for new systemd configuration method

Run Details

958 of 958 new or added lines in 18 files covered. (100.0%)

1971 of 8330 relevant lines covered (23.66%)

0.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

38.13

/pkg/plugins/generic/generic_plugin.go

package generic

import (
        "bytes"
        "os/exec"
        "reflect"
        "strconv"
        "strings"
        "syscall"

        "github.com/golang/glog"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
)

var PluginName = "generic_plugin"

type GenericPlugin struct {
        PluginName     string
        SpecVersion    string
        DesireState    *sriovnetworkv1.SriovNetworkNodeState
        LastState      *sriovnetworkv1.SriovNetworkNodeState
        LoadVfioDriver uint
        RunningOnHost  bool
        HostManager    host.HostManagerInterface
}

const scriptsPath = "bindata/scripts/enable-kargs.sh"

const (
        unloaded = iota
        loading
        loaded
)

// Initialize our plugin and set up initial values
func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) {
        return &GenericPlugin{
                PluginName:     PluginName,
                SpecVersion:    "1.0",
                LoadVfioDriver: unloaded,
                RunningOnHost:  runningOnHost,
                HostManager:    host.NewHostManager(runningOnHost),
        }, nil
}

// Name returns the name of the plugin
func (p *GenericPlugin) Name() string {
        return p.PluginName
}

// Spec returns the version of the spec expected by the plugin
func (p *GenericPlugin) Spec() string {
        return p.SpecVersion
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        glog.Info("generic-plugin OnNodeStateChange()")
        needDrain = false
        needReboot = false
        err = nil
        p.DesireState = new

        needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces)
        needReboot = needRebootNode(new, &p.LoadVfioDriver)

        if needReboot {
                needDrain = true
        }
        return
}

// Apply config change
func (p *GenericPlugin) Apply() error {
        glog.Infof("generic-plugin Apply(): desiredState=%v", p.DesireState.Spec)
        if p.LoadVfioDriver == loading {
                if err := p.HostManager.LoadKernelModule("vfio_pci"); err != nil {
                        glog.Errorf("generic-plugin Apply(): fail to load vfio_pci kmod: %v", err)
                        return err
                }
                p.LoadVfioDriver = loaded
        }

        if p.LastState != nil {
                glog.Infof("generic-plugin Apply(): lastStat=%v", p.LastState.Spec)
                if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) {
                        glog.Info("generic-plugin Apply(): nothing to apply")
                        return nil
                }
        }

        // Create a map with all the PFs we will need to configure
        // we need to create it here before we access the host file system using the chroot function
        // because the skipConfigVf needs the mstconfig package that exist only inside the sriov-config-daemon file system
        pfsToSkip, err := utils.GetPfsToSkip(p.DesireState)
        if err != nil {
                return err
        }

        // When calling from systemd do not try to chroot
        if !p.RunningOnHost {
                exit, err := utils.Chroot("/host")
                if err != nil {
                        return err
                }
                defer exit()
        }

        if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil {
                return err
        }
        p.LastState = &sriovnetworkv1.SriovNetworkNodeState{}
        *p.LastState = *p.DesireState
        return nil
}

func needVfioDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].DeviceType == constants.DeviceTypeVfioPci {
                                return true
                        }
                }
        }
        return false
}

func tryEnableIommuInKernelArgs() (bool, error) {
        glog.Info("generic-plugin tryEnableIommuInKernelArgs()")
        args := [2]string{"intel_iommu=on", "iommu=pt"}
        var stdout, stderr bytes.Buffer
        cmd := exec.Command("/bin/sh", scriptsPath, args[0], args[1])
        cmd.Stdout = &stdout
        cmd.Stderr = &stderr

        if err := cmd.Run(); err != nil {
                // if grubby is not there log and assume kernel args are set correctly.
                if isCommandNotFound(err) {
                        glog.Error("generic-plugin tryEnableIommuInKernelArgs(): grubby command not found. Please ensure that kernel args intel_iommu=on iommu=pt are set")
                        return false, nil
                }
                glog.Errorf("generic-plugin tryEnableIommuInKernelArgs(): fail to enable iommu %s: %v", args, err)
                return false, err
        }

        i, err := strconv.Atoi(strings.TrimSpace(stdout.String()))
        if err == nil {
                if i > 0 {
                        glog.Infof("generic-plugin tryEnableIommuInKernelArgs(): need to reboot node")
                        return true, nil
                }
        }
        return false, err
}

func isCommandNotFound(err error) bool {
        if exitErr, ok := err.(*exec.ExitError); ok {
                if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 {
                        return true
                }
        }
        return false
}

func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.InterfaceExts) (needDrain bool) {
        glog.V(2).Infof("generic-plugin needDrainNode(): current state '%+v', desired state '%+v'", current, desired)
        needDrain = false
        for _, ifaceStatus := range current {
                configured := false
                for _, iface := range desired {
                        if iface.PciAddress == ifaceStatus.PciAddress {
                                // TODO: no need to perform further checks if ifaceStatus.NumVfs equals to 0
                                // once https://github.com/kubernetes/kubernetes/issues/109595 will be fixed
                                configured = true
                                if utils.NeedUpdate(&iface, &ifaceStatus) {
                                        glog.V(2).Infof("generic-plugin needDrainNode(): need drain, PF %s request update", iface.PciAddress)
                                        needDrain = true
                                        return
                                }
                                glog.V(2).Infof("generic-plugin needDrainNode(): no need drain, expect NumVfs %v, current NumVfs %v", iface.NumVfs, ifaceStatus.NumVfs)
                        }
                }
                if !configured && ifaceStatus.NumVfs > 0 {
                        glog.V(2).Infof("generic-plugin needDrainNode(): need drain, %v needs to be reset", ifaceStatus)
                        needDrain = true
                        return
                }
        }
        return
}

func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint) (needReboot bool) {
        needReboot = false
        if *loadVfioDriver != loaded {
                if needVfioDriver(state) {
                        *loadVfioDriver = loading
                        update, err := tryEnableIommuInKernelArgs()
                        if err != nil {
                                glog.Errorf("generic-plugin needRebootNode():fail to enable iommu in kernel args: %v", err)
                        }
                        if update {
                                glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for enabling iommu kernel args")
                        }
                        needReboot = needReboot || update
                }
        }

        update, err := utils.WriteSwitchdevConfFile(state)
        if err != nil {
                glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file")
        }
        if update {
                glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for updating switchdev device configuration")
        }
        needReboot = needReboot || update
        return
}

1	package generic
2
3	import (
4	"bytes"
5	"os/exec"
6	"reflect"
7	"strconv"
8	"strings"
9	"syscall"
10
11	"github.com/golang/glog"
12
13	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
14	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
15	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host"
16	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
17	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
18	)
19
20	var PluginName = "generic_plugin"
21
22	type GenericPlugin struct {
23	PluginName string
24	SpecVersion string
25	DesireState *sriovnetworkv1.SriovNetworkNodeState
26	LastState *sriovnetworkv1.SriovNetworkNodeState
27	LoadVfioDriver uint
28	RunningOnHost bool
29	HostManager host.HostManagerInterface
30	}
31
32	const scriptsPath = "bindata/scripts/enable-kargs.sh"
33
34	const (
35	unloaded = iota
36	loading
37	loaded
38	)
39
40	// Initialize our plugin and set up initial values
41	func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) {	1✔
42	return &GenericPlugin{	1✔
43	PluginName: PluginName,	1✔
44	SpecVersion: "1.0",	1✔
45	LoadVfioDriver: unloaded,	1✔
46	RunningOnHost: runningOnHost,	1✔
47	HostManager: host.NewHostManager(runningOnHost),	1✔
48	}, nil	1✔
49	}	1✔
50
51	// Name returns the name of the plugin
52	func (p *GenericPlugin) Name() string {	×
53	return p.PluginName	×
54	}	×
55
56	// Spec returns the version of the spec expected by the plugin
57	func (p *GenericPlugin) Spec() string {	×
58	return p.SpecVersion	×
59	}	×
60
61	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
62	func (p GenericPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	1✔
63	glog.Info("generic-plugin OnNodeStateChange()")	1✔
64	needDrain = false	1✔
65	needReboot = false	1✔
66	err = nil	1✔
67	p.DesireState = new	1✔
68		1✔
69	needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces)	1✔
70	needReboot = needRebootNode(new, &p.LoadVfioDriver)	1✔
71		1✔
72	if needReboot {	1✔
73	needDrain = true	×
74	}	×
75	return	1✔
76	}
77
78	// Apply config change
79	func (p *GenericPlugin) Apply() error {	×
80	glog.Infof("generic-plugin Apply(): desiredState=%v", p.DesireState.Spec)	×
81	if p.LoadVfioDriver == loading {	×
82	if err := p.HostManager.LoadKernelModule("vfio_pci"); err != nil {	×
83	glog.Errorf("generic-plugin Apply(): fail to load vfio_pci kmod: %v", err)	×
84	return err	×
85	}	×
86	p.LoadVfioDriver = loaded	×
87	}
88
89	if p.LastState != nil {	×
90	glog.Infof("generic-plugin Apply(): lastStat=%v", p.LastState.Spec)	×
91	if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) {	×
92	glog.Info("generic-plugin Apply(): nothing to apply")	×
93	return nil	×
94	}	×
95	}
96
97	// Create a map with all the PFs we will need to configure
98	// we need to create it here before we access the host file system using the chroot function
99	// because the skipConfigVf needs the mstconfig package that exist only inside the sriov-config-daemon file system
100	pfsToSkip, err := utils.GetPfsToSkip(p.DesireState)	×
101	if err != nil {	×
102	return err	×
103	}	×
104
105	// When calling from systemd do not try to chroot
106	if !p.RunningOnHost {	×
107	exit, err := utils.Chroot("/host")	×
108	if err != nil {	×
109	return err	×
110	}	×
111	defer exit()	×
112	}
113
114	if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil {	×
115	return err	×
116	}	×
117	p.LastState = &sriovnetworkv1.SriovNetworkNodeState{}	×
118	p.LastState = p.DesireState	×
119	return nil	×
120	}
121
122	func needVfioDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {	1✔
123	for _, iface := range state.Spec.Interfaces {	2✔
124	for i := range iface.VfGroups {	2✔
125	if iface.VfGroups[i].DeviceType == constants.DeviceTypeVfioPci {	1✔
126	return true	×
127	}	×
128	}
129	}
130	return false	1✔
131	}
132
133	func tryEnableIommuInKernelArgs() (bool, error) {	×
134	glog.Info("generic-plugin tryEnableIommuInKernelArgs()")	×
135	args := [2]string{"intel_iommu=on", "iommu=pt"}	×
136	var stdout, stderr bytes.Buffer	×
137	cmd := exec.Command("/bin/sh", scriptsPath, args[0], args[1])	×
138	cmd.Stdout = &stdout	×
139	cmd.Stderr = &stderr	×
140		×
141	if err := cmd.Run(); err != nil {	×
142	// if grubby is not there log and assume kernel args are set correctly.	×
143	if isCommandNotFound(err) {	×
144	glog.Error("generic-plugin tryEnableIommuInKernelArgs(): grubby command not found. Please ensure that kernel args intel_iommu=on iommu=pt are set")	×
145	return false, nil	×
146	}	×
147	glog.Errorf("generic-plugin tryEnableIommuInKernelArgs(): fail to enable iommu %s: %v", args, err)	×
148	return false, err	×
149	}
150
151	i, err := strconv.Atoi(strings.TrimSpace(stdout.String()))	×
152	if err == nil {	×
153	if i > 0 {	×
154	glog.Infof("generic-plugin tryEnableIommuInKernelArgs(): need to reboot node")	×
155	return true, nil	×
156	}	×
157	}
158	return false, err	×
159	}
160
161	func isCommandNotFound(err error) bool {	×
162	if exitErr, ok := err.(*exec.ExitError); ok {	×
163	if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 {	×
164	return true	×
165	}	×
166	}
167	return false	×
168	}
169
170	func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.InterfaceExts) (needDrain bool) {	1✔
171	glog.V(2).Infof("generic-plugin needDrainNode(): current state '%+v', desired state '%+v'", current, desired)	1✔
172	needDrain = false	1✔
173	for _, ifaceStatus := range current {	2✔
174	configured := false	1✔
175	for _, iface := range desired {	2✔
176	if iface.PciAddress == ifaceStatus.PciAddress {	2✔
177	// TODO: no need to perform further checks if ifaceStatus.NumVfs equals to 0	1✔
178	// once https://github.com/kubernetes/kubernetes/issues/109595 will be fixed	1✔
179	configured = true	1✔
180	if utils.NeedUpdate(&iface, &ifaceStatus) {	2✔
181	glog.V(2).Infof("generic-plugin needDrainNode(): need drain, PF %s request update", iface.PciAddress)	1✔
182	needDrain = true	1✔
183	return	1✔
184	}	1✔
185	glog.V(2).Infof("generic-plugin needDrainNode(): no need drain, expect NumVfs %v, current NumVfs %v", iface.NumVfs, ifaceStatus.NumVfs)	1✔
186	}
187	}
188	if !configured && ifaceStatus.NumVfs > 0 {	1✔
189	glog.V(2).Infof("generic-plugin needDrainNode(): need drain, %v needs to be reset", ifaceStatus)	×
190	needDrain = true	×
191	return	×
192	}	×
193	}
194	return	1✔
195	}
196
197	func needRebootNode(state sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver uint) (needReboot bool) {	1✔
198	needReboot = false	1✔
199	if *loadVfioDriver != loaded {	2✔
200	if needVfioDriver(state) {	1✔
201	*loadVfioDriver = loading	×
202	update, err := tryEnableIommuInKernelArgs()	×
203	if err != nil {	×
204	glog.Errorf("generic-plugin needRebootNode():fail to enable iommu in kernel args: %v", err)	×
205	}	×
206	if update {	×
207	glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for enabling iommu kernel args")	×
208	}	×
209	needReboot = needReboot \|\| update	×
210	}
211	}
212
213	update, err := utils.WriteSwitchdevConfFile(state)	1✔
214	if err != nil {	1✔
215	glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file")	×
216	}	×
217	if update {	1✔
218	glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for updating switchdev device configuration")	×
219	}	×
220	needReboot = needReboot \|\| update	1✔
221	return	1✔
222	}

k8snetworkplumbingwg / sriov-network-operator / 3751025296

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous