4253702965

Build Type

github

Committed by GitHub

Commit Message

Merge f01bcd9a4 into 331d1ae13

Pull Request Pull Request #377: Vdpa introduction

Run Details

36 of 36 new or added lines in 5 files covered. (100.0%)

1949 of 7564 relevant lines covered (25.77%)

0.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

38.31

/pkg/plugins/generic/generic_plugin.go

package generic

import (
        "bytes"
        "os/exec"
        "reflect"
        "strconv"
        "strings"
        "syscall"

        "github.com/golang/glog"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
)

var PluginName = "generic_plugin"

type GenericPlugin struct {
        PluginName           string
        SpecVersion          string
        DesireState          *sriovnetworkv1.SriovNetworkNodeState
        LastState            *sriovnetworkv1.SriovNetworkNodeState
        LoadVfioDriver       uint
        LoadVirtioVdpaDriver uint
}

const scriptsPath = "bindata/scripts/enable-kargs.sh"

const (
        unloaded = iota
        loading
        loaded
)

// Initialize our plugin and set up initial values
func NewGenericPlugin() (plugin.VendorPlugin, error) {
        return &GenericPlugin{
                PluginName:           PluginName,
                SpecVersion:          "1.0",
                LoadVfioDriver:       unloaded,
                LoadVirtioVdpaDriver: unloaded,
        }, nil
}

// Name returns the name of the plugin
func (p *GenericPlugin) Name() string {
        return p.PluginName
}

// Spec returns the version of the spec expected by the plugin
func (p *GenericPlugin) Spec() string {
        return p.SpecVersion
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        glog.Info("generic-plugin OnNodeStateChange()")
        needDrain = false
        needReboot = false
        err = nil
        p.DesireState = new

        needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces)
        needReboot = needRebootNode(new, &p.LoadVfioDriver, &p.LoadVirtioVdpaDriver)

        if needReboot {
                needDrain = true
        }
        return
}

// Apply config change
func (p *GenericPlugin) Apply() error {
        glog.Infof("generic-plugin Apply(): desiredState=%v", p.DesireState.Spec)
        if p.LoadVfioDriver == loading {
                if err := utils.LoadKernelModule("vfio_pci"); err != nil {
                        glog.Errorf("generic-plugin Apply(): fail to load vfio_pci kmod: %v", err)
                        return err
                }
                p.LoadVfioDriver = loaded
        }

        if p.LoadVirtioVdpaDriver == loading {
                if err := utils.LoadKernelModule("virtio_vdpa"); err != nil {
                        glog.Errorf("generic-plugin Apply(): fail to load virtio_vdpa kmod: %v", err)
                        return err
                }
                p.LoadVirtioVdpaDriver = loaded
        }

        if p.LastState != nil {
                glog.Infof("generic-plugin Apply(): lastStat=%v", p.LastState.Spec)
                if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) {
                        glog.Info("generic-plugin Apply(): nothing to apply")
                        return nil
                }
        }

        // Create a map with all the PFs we will need to configure
        // we need to create it here before we access the host file system using the chroot function
        // because the skipConfigVf needs the mstconfig package that exist only inside the sriov-config-daemon file system
        pfsToSkip, err := utils.GetPfsToSkip(p.DesireState)
        if err != nil {
                return err
        }

        exit, err := utils.Chroot("/host")
        if err != nil {
                return err
        }
        defer exit()
        if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil {
                return err
        }
        p.LastState = &sriovnetworkv1.SriovNetworkNodeState{}
        *p.LastState = *p.DesireState
        return nil
}

func needVfioDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].DeviceType == constants.DeviceTypeVfioPci {
                                return true
                        }
                }
        }
        return false
}

func needVirtioVdpaDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {
        for _, iface := range state.Spec.Interfaces {
                for i := range iface.VfGroups {
                        if iface.VfGroups[i].VdpaType == constants.VdpaTypeVirtio {
                                return true
                        }
                }
        }
        return false
}

func tryEnableIommuInKernelArgs() (bool, error) {
        glog.Info("generic-plugin tryEnableIommuInKernelArgs()")
        args := [2]string{"intel_iommu=on", "iommu=pt"}
        var stdout, stderr bytes.Buffer
        cmd := exec.Command("/bin/sh", scriptsPath, args[0], args[1])
        cmd.Stdout = &stdout
        cmd.Stderr = &stderr

        if err := cmd.Run(); err != nil {
                // if grubby is not there log and assume kernel args are set correctly.
                if isCommandNotFound(err) {
                        glog.Error("generic-plugin tryEnableIommuInKernelArgs(): grubby command not found. Please ensure that kernel args intel_iommu=on iommu=pt are set")
                        return false, nil
                }
                glog.Errorf("generic-plugin tryEnableIommuInKernelArgs(): fail to enable iommu %s: %v", args, err)
                return false, err
        }

        i, err := strconv.Atoi(strings.TrimSpace(stdout.String()))
        if err == nil {
                if i > 0 {
                        glog.Infof("generic-plugin tryEnableIommuInKernelArgs(): need to reboot node")
                        return true, nil
                }
        }
        return false, err
}

func isCommandNotFound(err error) bool {
        if exitErr, ok := err.(*exec.ExitError); ok {
                if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 {
                        return true
                }
        }
        return false
}

func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.InterfaceExts) (needDrain bool) {
        glog.V(2).Infof("generic-plugin needDrainNode(): current state '%+v', desired state '%+v'", current, desired)
        needDrain = false
        for _, ifaceStatus := range current {
                configured := false
                for _, iface := range desired {
                        if iface.PciAddress == ifaceStatus.PciAddress {
                                // TODO: no need to perform further checks if ifaceStatus.NumVfs equals to 0
                                // once https://github.com/kubernetes/kubernetes/issues/109595 will be fixed
                                configured = true
                                if utils.NeedUpdate(&iface, &ifaceStatus) {
                                        glog.V(2).Infof("generic-plugin needDrainNode(): need drain, PF %s request update", iface.PciAddress)
                                        needDrain = true
                                        return
                                }
                                glog.V(2).Infof("generic-plugin needDrainNode(): no need drain, expect NumVfs %v, current NumVfs %v", iface.NumVfs, ifaceStatus.NumVfs)
                        }
                }
                if !configured && ifaceStatus.NumVfs > 0 {
                        glog.V(2).Infof("generic-plugin needDrainNode(): need drain, %v needs to be reset", ifaceStatus)
                        needDrain = true
                        return
                }
        }
        return
}

func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint, loadVirtioVdpaDriver *uint) (needReboot bool) {
        needReboot = false
        if *loadVfioDriver != loaded {
                if needVfioDriver(state) {
                        *loadVfioDriver = loading
                        update, err := tryEnableIommuInKernelArgs()
                        if err != nil {
                                glog.Errorf("generic-plugin needRebootNode():fail to enable iommu in kernel args: %v", err)
                        }
                        if update {
                                glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for enabling iommu kernel args")
                        }
                        needReboot = needReboot || update
                }
        }

        if *loadVirtioVdpaDriver != loaded {
                if needVirtioVdpaDriver(state) {
                        *loadVirtioVdpaDriver = loading
                }
        }

        update, err := utils.WriteSwitchdevConfFile(state)
        if err != nil {
                glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file")
        }
        if update {
                glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for updating switchdev device configuration")
        }
        needReboot = needReboot || update
        return
}

1	package generic
2
3	import (
4	"bytes"
5	"os/exec"
6	"reflect"
7	"strconv"
8	"strings"
9	"syscall"
10
11	"github.com/golang/glog"
12
13	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
14	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
15	plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
16	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
17	)
18
19	var PluginName = "generic_plugin"
20
21	type GenericPlugin struct {
22	PluginName string
23	SpecVersion string
24	DesireState *sriovnetworkv1.SriovNetworkNodeState
25	LastState *sriovnetworkv1.SriovNetworkNodeState
26	LoadVfioDriver uint
27	LoadVirtioVdpaDriver uint
28	}
29
30	const scriptsPath = "bindata/scripts/enable-kargs.sh"
31
32	const (
33	unloaded = iota
34	loading
35	loaded
36	)
37
38	// Initialize our plugin and set up initial values
39	func NewGenericPlugin() (plugin.VendorPlugin, error) {	1✔
40	return &GenericPlugin{	1✔
41	PluginName: PluginName,	1✔
42	SpecVersion: "1.0",	1✔
43	LoadVfioDriver: unloaded,	1✔
44	LoadVirtioVdpaDriver: unloaded,	1✔
45	}, nil	1✔
46	}	1✔
47
48	// Name returns the name of the plugin
49	func (p *GenericPlugin) Name() string {	×
50	return p.PluginName	×
51	}	×
52
53	// Spec returns the version of the spec expected by the plugin
54	func (p *GenericPlugin) Spec() string {	×
55	return p.SpecVersion	×
56	}	×
57
58	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
59	func (p GenericPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	1✔
60	glog.Info("generic-plugin OnNodeStateChange()")	1✔
61	needDrain = false	1✔
62	needReboot = false	1✔
63	err = nil	1✔
64	p.DesireState = new	1✔
65		1✔
66	needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces)	1✔
67	needReboot = needRebootNode(new, &p.LoadVfioDriver, &p.LoadVirtioVdpaDriver)	1✔
68		1✔
69	if needReboot {	1✔
70	needDrain = true	×
71	}	×
72	return	1✔
73	}
74
75	// Apply config change
76	func (p *GenericPlugin) Apply() error {	×
77	glog.Infof("generic-plugin Apply(): desiredState=%v", p.DesireState.Spec)	×
78	if p.LoadVfioDriver == loading {	×
79	if err := utils.LoadKernelModule("vfio_pci"); err != nil {	×
80	glog.Errorf("generic-plugin Apply(): fail to load vfio_pci kmod: %v", err)	×
81	return err	×
82	}	×
83	p.LoadVfioDriver = loaded	×
84	}
85
86	if p.LoadVirtioVdpaDriver == loading {	×
87	if err := utils.LoadKernelModule("virtio_vdpa"); err != nil {	×
88	glog.Errorf("generic-plugin Apply(): fail to load virtio_vdpa kmod: %v", err)	×
89	return err	×
90	}	×
91	p.LoadVirtioVdpaDriver = loaded	×
92	}
93
94	if p.LastState != nil {	×
95	glog.Infof("generic-plugin Apply(): lastStat=%v", p.LastState.Spec)	×
96	if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) {	×
97	glog.Info("generic-plugin Apply(): nothing to apply")	×
98	return nil	×
99	}	×
100	}
101
102	// Create a map with all the PFs we will need to configure
103	// we need to create it here before we access the host file system using the chroot function
104	// because the skipConfigVf needs the mstconfig package that exist only inside the sriov-config-daemon file system
105	pfsToSkip, err := utils.GetPfsToSkip(p.DesireState)	×
106	if err != nil {	×
107	return err	×
108	}	×
109
110	exit, err := utils.Chroot("/host")	×
111	if err != nil {	×
112	return err	×
113	}	×
114	defer exit()	×
115	if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil {	×
116	return err	×
117	}	×
118	p.LastState = &sriovnetworkv1.SriovNetworkNodeState{}	×
119	p.LastState = p.DesireState	×
120	return nil	×
121	}
122
123	func needVfioDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {	1✔
124	for _, iface := range state.Spec.Interfaces {	2✔
125	for i := range iface.VfGroups {	2✔
126	if iface.VfGroups[i].DeviceType == constants.DeviceTypeVfioPci {	1✔
127	return true	×
128	}	×
129	}
130	}
131	return false	1✔
132	}
133
134	func needVirtioVdpaDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool {	1✔
135	for _, iface := range state.Spec.Interfaces {	2✔
136	for i := range iface.VfGroups {	2✔
137	if iface.VfGroups[i].VdpaType == constants.VdpaTypeVirtio {	1✔
138	return true	×
139	}	×
140	}
141	}
142	return false	1✔
143	}
144
145	func tryEnableIommuInKernelArgs() (bool, error) {	×
146	glog.Info("generic-plugin tryEnableIommuInKernelArgs()")	×
147	args := [2]string{"intel_iommu=on", "iommu=pt"}	×
148	var stdout, stderr bytes.Buffer	×
149	cmd := exec.Command("/bin/sh", scriptsPath, args[0], args[1])	×
150	cmd.Stdout = &stdout	×
151	cmd.Stderr = &stderr	×
152		×
153	if err := cmd.Run(); err != nil {	×
154	// if grubby is not there log and assume kernel args are set correctly.	×
155	if isCommandNotFound(err) {	×
156	glog.Error("generic-plugin tryEnableIommuInKernelArgs(): grubby command not found. Please ensure that kernel args intel_iommu=on iommu=pt are set")	×
157	return false, nil	×
158	}	×
159	glog.Errorf("generic-plugin tryEnableIommuInKernelArgs(): fail to enable iommu %s: %v", args, err)	×
160	return false, err	×
161	}
162
163	i, err := strconv.Atoi(strings.TrimSpace(stdout.String()))	×
164	if err == nil {	×
165	if i > 0 {	×
166	glog.Infof("generic-plugin tryEnableIommuInKernelArgs(): need to reboot node")	×
167	return true, nil	×
168	}	×
169	}
170	return false, err	×
171	}
172
173	func isCommandNotFound(err error) bool {	×
174	if exitErr, ok := err.(*exec.ExitError); ok {	×
175	if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 {	×
176	return true	×
177	}	×
178	}
179	return false	×
180	}
181
182	func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.InterfaceExts) (needDrain bool) {	1✔
183	glog.V(2).Infof("generic-plugin needDrainNode(): current state '%+v', desired state '%+v'", current, desired)	1✔
184	needDrain = false	1✔
185	for _, ifaceStatus := range current {	2✔
186	configured := false	1✔
187	for _, iface := range desired {	2✔
188	if iface.PciAddress == ifaceStatus.PciAddress {	2✔
189	// TODO: no need to perform further checks if ifaceStatus.NumVfs equals to 0	1✔
190	// once https://github.com/kubernetes/kubernetes/issues/109595 will be fixed	1✔
191	configured = true	1✔
192	if utils.NeedUpdate(&iface, &ifaceStatus) {	2✔
193	glog.V(2).Infof("generic-plugin needDrainNode(): need drain, PF %s request update", iface.PciAddress)	1✔
194	needDrain = true	1✔
195	return	1✔
196	}	1✔
197	glog.V(2).Infof("generic-plugin needDrainNode(): no need drain, expect NumVfs %v, current NumVfs %v", iface.NumVfs, ifaceStatus.NumVfs)	1✔
198	}
199	}
200	if !configured && ifaceStatus.NumVfs > 0 {	1✔
201	glog.V(2).Infof("generic-plugin needDrainNode(): need drain, %v needs to be reset", ifaceStatus)	×
202	needDrain = true	×
203	return	×
204	}	×
205	}
206	return	1✔
207	}
208
209	func needRebootNode(state sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver uint, loadVirtioVdpaDriver *uint) (needReboot bool) {	1✔
210	needReboot = false	1✔
211	if *loadVfioDriver != loaded {	2✔
212	if needVfioDriver(state) {	1✔
213	*loadVfioDriver = loading	×
214	update, err := tryEnableIommuInKernelArgs()	×
215	if err != nil {	×
216	glog.Errorf("generic-plugin needRebootNode():fail to enable iommu in kernel args: %v", err)	×
217	}	×
218	if update {	×
219	glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for enabling iommu kernel args")	×
220	}	×
221	needReboot = needReboot \|\| update	×
222	}
223	}
224
225	if *loadVirtioVdpaDriver != loaded {	2✔
226	if needVirtioVdpaDriver(state) {	1✔
227	*loadVirtioVdpaDriver = loading	×
228	}	×
229	}
230
231	update, err := utils.WriteSwitchdevConfFile(state)	1✔
232	if err != nil {	1✔
233	glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file")	×
234	}	×
235	if update {	1✔
236	glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for updating switchdev device configuration")	×
237	}	×
238	needReboot = needReboot \|\| update	1✔
239	return	1✔
240	}

k8snetworkplumbingwg / sriov-network-operator / 4253702965

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous