3751025296

Build Type

github

Committed by GitHub

Commit Message

Merge 421284b55 into 788d76f7e

Pull Request Pull Request #365: Implementation for new systemd configuration method

Run Details

958 of 958 new or added lines in 18 files covered. (100.0%)

1971 of 8330 relevant lines covered (23.66%)

0.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/pkg/utils/utils_virtual.go

package utils

import (
        "encoding/json"
        "errors"
        "fmt"
        "io"
        "io/ioutil"
        "os"
        "path/filepath"
        "strconv"

        "github.com/golang/glog"
        "github.com/hashicorp/go-retryablehttp"
        "github.com/jaypipes/ghw"

        dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
)

// PlatformType ...
type PlatformType int

const (
        // Baremetal platform
        Baremetal PlatformType = iota
        // VirtualOpenStack ...
        VirtualOpenStack
)

func (e PlatformType) String() string {
        switch e {
        case Baremetal:
                return "Baremetal"
        case VirtualOpenStack:
                return "Virtual/Openstack"
        default:
                return fmt.Sprintf("%d", int(e))
        }
}

var (
        // PlatformMap contains supported platforms for virtual VF
        PlatformMap = map[string]PlatformType{
                "openstack": VirtualOpenStack,
        }
)

const (
        ospHostMetaDataDir     = "/host/var/config/openstack/2018-08-27"
        ospMetaDataDir         = "/var/config/openstack/2018-08-27"
        ospMetaDataBaseURL     = "http://169.254.169.254/openstack/2018-08-27"
        ospHostNetworkDataFile = ospHostMetaDataDir + "/network_data.json"
        ospHostMetaDataFile    = ospHostMetaDataDir + "/meta_data.json"
        ospNetworkDataFile     = ospMetaDataDir + "/network_data.json"
        ospMetaDataFile        = ospMetaDataDir + "/meta_data.json"
        ospNetworkDataURL      = ospMetaDataBaseURL + "/network_data.json"
        ospMetaDataURL         = ospMetaDataBaseURL + "/meta_data.json"
)

// OSPMetaDataDevice -- Device structure within meta_data.json
type OSPMetaDataDevice struct {
        Vlan      int      `json:"vlan,omitempty"`
        VfTrusted bool     `json:"vf_trusted,omitempty"`
        Type      string   `json:"type,omitempty"`
        Mac       string   `json:"mac,omitempty"`
        Bus       string   `json:"bus,omitempty"`
        Address   string   `json:"address,omitempty"`
        Tags      []string `json:"tags,omitempty"`
}

// OSPMetaData -- Openstack meta_data.json format
type OSPMetaData struct {
        UUID             string              `json:"uuid,omitempty"`
        AdminPass        string              `json:"admin_pass,omitempty"`
        Name             string              `json:"name,omitempty"`
        LaunchIndex      int                 `json:"launch_index,omitempty"`
        AvailabilityZone string              `json:"availability_zone,omitempty"`
        ProjectID        string              `json:"project_id,omitempty"`
        Devices          []OSPMetaDataDevice `json:"devices,omitempty"`
}

// OSPNetworkLink OSP Link metadata
type OSPNetworkLink struct {
        ID          string `json:"id"`
        VifID       string `json:"vif_id,omitempty"`
        Type        string `json:"type"`
        Mtu         int    `json:"mtu,omitempty"`
        EthernetMac string `json:"ethernet_mac_address"`
}

// OSPNetwork OSP Network metadata
type OSPNetwork struct {
        ID        string `json:"id"`
        Type      string `json:"type"`
        Link      string `json:"link"`
        NetworkID string `json:"network_id"`
}

// OSPNetworkData OSP Network metadata
type OSPNetworkData struct {
        Links    []OSPNetworkLink `json:"links,omitempty"`
        Networks []OSPNetwork     `json:"networks,omitempty"`
        // Omit Services
}

type OSPDevicesInfo map[string]*OSPDeviceInfo

type OSPDeviceInfo struct {
        MacAddress string
        NetworkID  string
}

// GetOpenstackData gets the metadata and network_data
func GetOpenstackData(useHostPath bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
        metaData, networkData, err = getOpenstackDataFromConfigDrive(useHostPath)
        if err != nil {
                metaData, networkData, err = getOpenstackDataFromMetadataService()
        }
        return metaData, networkData, err
}

// getOpenstackDataFromConfigDrive reads the meta_data and network_data files
func getOpenstackDataFromConfigDrive(useHostPath bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
        metaData = &OSPMetaData{}
        networkData = &OSPNetworkData{}
        glog.Infof("reading OpenStack meta_data from config-drive")
        var metadataf *os.File
        ospMetaDataFilePath := ospMetaDataFile
        if useHostPath {
                ospMetaDataFilePath = ospHostMetaDataFile
        }
        metadataf, err = os.Open(ospMetaDataFilePath)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospHostMetaDataFile, err)
        }
        defer func() {
                if e := metadataf.Close(); err == nil && e != nil {
                        err = fmt.Errorf("error closing file %s: %w", ospHostMetaDataFile, e)
                }
        }()
        if err = json.NewDecoder(metadataf).Decode(&metaData); err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospHostMetaDataFile, err)
        }

        glog.Infof("reading OpenStack network_data from config-drive")
        var networkDataf *os.File
        ospNetworkDataFilePath := ospNetworkDataFile
        if useHostPath {
                ospNetworkDataFilePath = ospHostNetworkDataFile
        }
        networkDataf, err = os.Open(ospNetworkDataFilePath)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospHostNetworkDataFile, err)
        }
        defer func() {
                if e := networkDataf.Close(); err == nil && e != nil {
                        err = fmt.Errorf("error closing file %s: %w", ospHostNetworkDataFile, e)
                }
        }()
        if err = json.NewDecoder(networkDataf).Decode(&networkData); err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospHostNetworkDataFile, err)
        }
        return metaData, networkData, err
}

func getBodyFromURL(url string) ([]byte, error) {
        glog.V(2).Infof("Getting body from %s", url)
        resp, err := retryablehttp.Get(url)
        if err != nil {
                return nil, err
        }
        rawBytes, err := io.ReadAll(resp.Body)
        if err != nil {
                return nil, err
        }
        defer resp.Body.Close()
        return rawBytes, nil
}

// getOpenstackDataFromMetadataService fetchs the metadata and network_data from the metadata service
func getOpenstackDataFromMetadataService() (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
        metaData = &OSPMetaData{}
        networkData = &OSPNetworkData{}
        glog.Infof("getting OpenStack meta_data from metadata server")
        metaDataRawBytes, err := getBodyFromURL(ospMetaDataURL)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error getting OpenStack meta_data from %s: %v", ospMetaDataURL, err)
        }
        err = json.Unmarshal(metaDataRawBytes, metaData)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospMetaDataURL)
        }

        glog.Infof("getting OpenStack network_data from metadata server")
        networkDataRawBytes, err := getBodyFromURL(ospNetworkDataURL)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error getting OpenStack network_data from %s: %v", ospNetworkDataURL, err)
        }
        err = json.Unmarshal(networkDataRawBytes, networkData)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospNetworkDataURL)
        }
        return metaData, networkData, nil
}

// CreateOpenstackDevicesInfo create the openstack device info map
func CreateOpenstackDevicesInfo(metaData *OSPMetaData, networkData *OSPNetworkData) (OSPDevicesInfo, error) {
        glog.Infof("CreateOpenstackDevicesInfo()")
        devicesInfo := make(OSPDevicesInfo)
        if metaData == nil || networkData == nil {
                return nil, nil
        }

        // use this for hw pass throw interfaces
        for _, device := range metaData.Devices {
                for _, link := range networkData.Links {
                        if device.Mac == link.EthernetMac {
                                for _, network := range networkData.Networks {
                                        if network.Link == link.ID {
                                                networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID
                                                devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: device.Mac, NetworkID: networkID}
                                        }
                                }
                        }
                }
        }

        // for vhostuser interface type we check the interfaces on the node
        pci, err := ghw.PCI()
        if err != nil {
                return nil, fmt.Errorf("CreateOpenstackDevicesInfo(): error getting PCI info: %v", err)
        }

        devices := pci.ListDevices()
        if len(devices) == 0 {
                return nil, fmt.Errorf("CreateOpenstackDevicesInfo(): could not retrieve PCI devices")
        }

        for _, device := range devices {
                if _, exist := devicesInfo[device.Address]; exist {
                        //we already discover the device via openstack metadata
                        continue
                }

                devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)
                if err != nil {
                        glog.Warningf("CreateOpenstackDevicesInfo(): unable to parse device class for device %+v %q", device, err)
                        continue
                }
                if devClass != netClass {
                        // Not network device
                        continue
                }

                macAddress := ""
                if name := tryToGetVirtualInterfaceName(device.Address); name != "" {
                        if mac := getNetDevMac(name); mac != "" {
                                macAddress = mac
                        }
                }
                if macAddress == "" {
                        // we didn't manage to find a mac address for the nic skipping
                        continue
                }

                for _, link := range networkData.Links {
                        if macAddress == link.EthernetMac {
                                for _, network := range networkData.Networks {
                                        if network.Link == link.ID {
                                                networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID
                                                devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: macAddress, NetworkID: networkID}
                                        }
                                }
                        }
                }
        }

        return devicesInfo, err
}

// DiscoverSriovDevicesVirtual discovers VFs on a virtual platform
func DiscoverSriovDevicesVirtual(devicesInfo OSPDevicesInfo) ([]sriovnetworkv1.InterfaceExt, error) {
        glog.V(2).Info("DiscoverSriovDevicesVirtual()")
        pfList := []sriovnetworkv1.InterfaceExt{}

        pci, err := ghw.PCI()
        if err != nil {
                return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): error getting PCI info: %v", err)
        }

        devices := pci.ListDevices()
        if len(devices) == 0 {
                return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): could not retrieve PCI devices")
        }

        for _, device := range devices {
                devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)
                if err != nil {
                        glog.Warningf("DiscoverSriovDevicesVirtual(): unable to parse device class for device %+v %q", device, err)
                        continue
                }
                if devClass != netClass {
                        // Not network device
                        continue
                }

                deviceInfo, exist := devicesInfo[device.Address]
                if !exist {
                        glog.Warningf("DiscoverSriovDevicesVirtual(): unable to find device in devicesInfo list for pci %s", device.Address)
                        continue
                }
                netFilter := deviceInfo.NetworkID
                metaMac := deviceInfo.MacAddress

                driver, err := dputils.GetDriverName(device.Address)
                if err != nil {
                        glog.Warningf("DiscoverSriovDevicesVirtual(): unable to parse device driver for device %+v %q", device, err)
                        continue
                }
                iface := sriovnetworkv1.InterfaceExt{
                        PciAddress: device.Address,
                        Driver:     driver,
                        Vendor:     device.Vendor.ID,
                        DeviceID:   device.Product.ID,
                        NetFilter:  netFilter,
                }
                if mtu := getNetdevMTU(device.Address); mtu > 0 {
                        iface.Mtu = mtu
                }
                if name := tryToGetVirtualInterfaceName(device.Address); name != "" {
                        iface.Name = name
                        if iface.Mac = getNetDevMac(name); iface.Mac == "" {
                                iface.Mac = metaMac
                        }
                        iface.LinkSpeed = getNetDevLinkSpeed(name)
                }
                iface.LinkType = getLinkType(iface)

                iface.TotalVfs = 1
                iface.NumVfs = 1

                vf := sriovnetworkv1.VirtualFunction{
                        PciAddress: device.Address,
                        Driver:     driver,
                        VfID:       0,
                        Vendor:     iface.Vendor,
                        DeviceID:   iface.DeviceID,
                        Mtu:        iface.Mtu,
                        Mac:        iface.Mac,
                }
                iface.VFs = append(iface.VFs, vf)

                pfList = append(pfList, iface)
        }
        return pfList, nil
}

func CreateOpenstackDevicesInfoFromNodeStatus(networkState *sriovnetworkv1.SriovNetworkNodeState) OSPDevicesInfo {
        devicesInfo := make(OSPDevicesInfo)
        for _, iface := range networkState.Status.Interfaces {
                devicesInfo[iface.PciAddress] = &OSPDeviceInfo{MacAddress: iface.Mac, NetworkID: iface.NetFilter}
        }

        return devicesInfo
}

// tryToGetVirtualInterfaceName get the interface name of a virtio interface
func tryToGetVirtualInterfaceName(pciAddr string) string {
        glog.Infof("tryToGetVirtualInterfaceName() get interface name for device %s", pciAddr)

        // To support different driver that is not virtio-pci like mlx
        name := tryGetInterfaceName(pciAddr)
        if name != "" {
                return name
        }

        netDir, err := filepath.Glob(filepath.Join(sysBusPciDevices, pciAddr, "virtio*", "net"))
        if err != nil || len(netDir) < 1 {
                return ""
        }

        fInfos, err := ioutil.ReadDir(netDir[0])
        if err != nil {
                glog.Warningf("tryToGetVirtualInterfaceName(): failed to read net directory %s: %q", netDir, err)
                return ""
        }

        names := make([]string, 0)
        for _, f := range fInfos {
                names = append(names, f.Name())
        }

        if len(names) < 1 {
                return ""
        }

        return names[0]
}

// SyncNodeStateVirtual attempt to update the node state to match the desired state
//  in virtual platforms
func SyncNodeStateVirtual(newState *sriovnetworkv1.SriovNetworkNodeState) error {
        var err error
        for _, ifaceStatus := range newState.Status.Interfaces {
                for _, iface := range newState.Spec.Interfaces {
                        if iface.PciAddress == ifaceStatus.PciAddress {
                                if !needUpdateVirtual(&iface, &ifaceStatus) {
                                        glog.V(2).Infof("SyncNodeStateVirtual(): no need update interface %s", iface.PciAddress)
                                        break
                                }
                                if err = configSriovDeviceVirtual(&iface, &ifaceStatus); err != nil {
                                        glog.Errorf("SyncNodeStateVirtual(): fail to config sriov interface %s: %v", iface.PciAddress, err)
                                        return err
                                }
                                break
                        }
                }
        }
        return nil
}

func needUpdateVirtual(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) bool {
        // The device MTU is set by the platorm
        // The NumVfs is always 1
        if iface.NumVfs > 0 {
                for _, vf := range ifaceStatus.VFs {
                        ingroup := false
                        for _, group := range iface.VfGroups {
                                if sriovnetworkv1.IndexInRange(vf.VfID, group.VfRange) {
                                        ingroup = true
                                        if group.DeviceType != constants.DeviceTypeNetDevice {
                                                if group.DeviceType != vf.Driver {
                                                        glog.V(2).Infof("needUpdateVirtual(): Driver needs update, desired=%s, current=%s", group.DeviceType, vf.Driver)
                                                        return true
                                                }
                                        } else {
                                                if sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) {
                                                        glog.V(2).Infof("needUpdateVirtual(): Driver needs update, desired=%s, current=%s", group.DeviceType, vf.Driver)
                                                        return true
                                                }
                                        }
                                        break
                                }
                        }
                        if !ingroup && sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) {
                                // VF which has DPDK driver loaded but not in any group, needs to be reset to default driver.
                                return true
                        }
                }
        }
        return false
}

func configSriovDeviceVirtual(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) error {
        glog.V(2).Infof("configSriovDeviceVirtual(): config interface %s with %v", iface.PciAddress, iface)
        // Config VFs
        if iface.NumVfs > 0 {
                if iface.NumVfs > 1 {
                        glog.Warningf("configSriovDeviceVirtual(): in a virtual environment, only one VF per interface (NumVfs: %d)", iface.NumVfs)
                        return errors.New("NumVfs > 1")
                }
                if len(iface.VfGroups) != 1 {
                        glog.Warningf("configSriovDeviceVirtual(): missing VFGroup")
                        return errors.New("NumVfs != 1")
                }
                addr := iface.PciAddress
                glog.V(2).Infof("configSriovDeviceVirtual(): addr %s", addr)
                driver := ""
                vfID := 0
                for _, group := range iface.VfGroups {
                        glog.V(2).Infof("configSriovDeviceVirtual(): group %v", group)
                        if sriovnetworkv1.IndexInRange(vfID, group.VfRange) {
                                glog.V(2).Infof("configSriovDeviceVirtual(): indexInRange %d", vfID)
                                if sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) {
                                        glog.V(2).Infof("configSriovDeviceVirtual(): driver %s", group.DeviceType)
                                        driver = group.DeviceType
                                }
                                break
                        }
                }
                if driver == "" {
                        glog.V(2).Infof("configSriovDeviceVirtual(): bind default")
                        if err := BindDefaultDriver(addr); err != nil {
                                glog.Warningf("configSriovDeviceVirtual(): fail to bind default driver for device %s", addr)
                                return err
                        }
                } else {
                        glog.V(2).Infof("configSriovDeviceVirtual(): bind driver %s", driver)
                        if err := BindDpdkDriver(addr, driver); err != nil {
                                glog.Warningf("configSriovDeviceVirtual(): fail to bind driver %s for device %s", driver, addr)
                                return err
                        }
                }
        }
        return nil
}

1	package utils
2
3	import (
4	"encoding/json"
5	"errors"
6	"fmt"
7	"io"
8	"io/ioutil"
9	"os"
10	"path/filepath"
11	"strconv"
12
13	"github.com/golang/glog"
14	"github.com/hashicorp/go-retryablehttp"
15	"github.com/jaypipes/ghw"
16
17	dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
18
19	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
20	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
21	)
22
23	// PlatformType ...
24	type PlatformType int
25
26	const (
27	// Baremetal platform
28	Baremetal PlatformType = iota
29	// VirtualOpenStack ...
30	VirtualOpenStack
31	)
32
33	func (e PlatformType) String() string {	×
34	switch e {	×
35	case Baremetal:	×
36	return "Baremetal"	×
37	case VirtualOpenStack:	×
38	return "Virtual/Openstack"	×
39	default:	×
40	return fmt.Sprintf("%d", int(e))	×
41	}
42	}
43
44	var (
45	// PlatformMap contains supported platforms for virtual VF
46	PlatformMap = map[string]PlatformType{
47	"openstack": VirtualOpenStack,
48	}
49	)
50
51	const (
52	ospHostMetaDataDir = "/host/var/config/openstack/2018-08-27"
53	ospMetaDataDir = "/var/config/openstack/2018-08-27"
54	ospMetaDataBaseURL = "http://169.254.169.254/openstack/2018-08-27"
55	ospHostNetworkDataFile = ospHostMetaDataDir + "/network_data.json"
56	ospHostMetaDataFile = ospHostMetaDataDir + "/meta_data.json"
57	ospNetworkDataFile = ospMetaDataDir + "/network_data.json"
58	ospMetaDataFile = ospMetaDataDir + "/meta_data.json"
59	ospNetworkDataURL = ospMetaDataBaseURL + "/network_data.json"
60	ospMetaDataURL = ospMetaDataBaseURL + "/meta_data.json"
61	)
62
63	// OSPMetaDataDevice -- Device structure within meta_data.json
64	type OSPMetaDataDevice struct {
65	Vlan int `json:"vlan,omitempty"`
66	VfTrusted bool `json:"vf_trusted,omitempty"`
67	Type string `json:"type,omitempty"`
68	Mac string `json:"mac,omitempty"`
69	Bus string `json:"bus,omitempty"`
70	Address string `json:"address,omitempty"`
71	Tags []string `json:"tags,omitempty"`
72	}
73
74	// OSPMetaData -- Openstack meta_data.json format
75	type OSPMetaData struct {
76	UUID string `json:"uuid,omitempty"`
77	AdminPass string `json:"admin_pass,omitempty"`
78	Name string `json:"name,omitempty"`
79	LaunchIndex int `json:"launch_index,omitempty"`
80	AvailabilityZone string `json:"availability_zone,omitempty"`
81	ProjectID string `json:"project_id,omitempty"`
82	Devices []OSPMetaDataDevice `json:"devices,omitempty"`
83	}
84
85	// OSPNetworkLink OSP Link metadata
86	type OSPNetworkLink struct {
87	ID string `json:"id"`
88	VifID string `json:"vif_id,omitempty"`
89	Type string `json:"type"`
90	Mtu int `json:"mtu,omitempty"`
91	EthernetMac string `json:"ethernet_mac_address"`
92	}
93
94	// OSPNetwork OSP Network metadata
95	type OSPNetwork struct {
96	ID string `json:"id"`
97	Type string `json:"type"`
98	Link string `json:"link"`
99	NetworkID string `json:"network_id"`
100	}
101
102	// OSPNetworkData OSP Network metadata
103	type OSPNetworkData struct {
104	Links []OSPNetworkLink `json:"links,omitempty"`
105	Networks []OSPNetwork `json:"networks,omitempty"`
106	// Omit Services
107	}
108
109	type OSPDevicesInfo map[string]*OSPDeviceInfo
110
111	type OSPDeviceInfo struct {
112	MacAddress string
113	NetworkID string
114	}
115
116	// GetOpenstackData gets the metadata and network_data
117	func GetOpenstackData(useHostPath bool) (metaData OSPMetaData, networkData OSPNetworkData, err error) {	×
118	metaData, networkData, err = getOpenstackDataFromConfigDrive(useHostPath)	×
119	if err != nil {	×
120	metaData, networkData, err = getOpenstackDataFromMetadataService()	×
121	}	×
122	return metaData, networkData, err	×
123	}
124
125	// getOpenstackDataFromConfigDrive reads the meta_data and network_data files
126	func getOpenstackDataFromConfigDrive(useHostPath bool) (metaData OSPMetaData, networkData OSPNetworkData, err error) {	×
127	metaData = &OSPMetaData{}	×
128	networkData = &OSPNetworkData{}	×
129	glog.Infof("reading OpenStack meta_data from config-drive")	×
130	var metadataf *os.File	×
131	ospMetaDataFilePath := ospMetaDataFile	×
132	if useHostPath {	×
133	ospMetaDataFilePath = ospHostMetaDataFile	×
134	}	×
135	metadataf, err = os.Open(ospMetaDataFilePath)	×
136	if err != nil {	×
137	return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospHostMetaDataFile, err)	×
138	}	×
139	defer func() {	×
140	if e := metadataf.Close(); err == nil && e != nil {	×
141	err = fmt.Errorf("error closing file %s: %w", ospHostMetaDataFile, e)	×
142	}	×
143	}()
144	if err = json.NewDecoder(metadataf).Decode(&metaData); err != nil {	×
145	return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospHostMetaDataFile, err)	×
146	}	×
147
148	glog.Infof("reading OpenStack network_data from config-drive")	×
149	var networkDataf *os.File	×
150	ospNetworkDataFilePath := ospNetworkDataFile	×
151	if useHostPath {	×
152	ospNetworkDataFilePath = ospHostNetworkDataFile	×
153	}	×
154	networkDataf, err = os.Open(ospNetworkDataFilePath)	×
155	if err != nil {	×
156	return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospHostNetworkDataFile, err)	×
157	}	×
158	defer func() {	×
159	if e := networkDataf.Close(); err == nil && e != nil {	×
160	err = fmt.Errorf("error closing file %s: %w", ospHostNetworkDataFile, e)	×
161	}	×
162	}()
163	if err = json.NewDecoder(networkDataf).Decode(&networkData); err != nil {	×
164	return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospHostNetworkDataFile, err)	×
165	}	×
166	return metaData, networkData, err	×
167	}
168
169	func getBodyFromURL(url string) ([]byte, error) {	×
170	glog.V(2).Infof("Getting body from %s", url)	×
171	resp, err := retryablehttp.Get(url)	×
172	if err != nil {	×
173	return nil, err	×
174	}	×
175	rawBytes, err := io.ReadAll(resp.Body)	×
176	if err != nil {	×
177	return nil, err	×
178	}	×
179	defer resp.Body.Close()	×
180	return rawBytes, nil	×
181	}
182
183	// getOpenstackDataFromMetadataService fetchs the metadata and network_data from the metadata service
184	func getOpenstackDataFromMetadataService() (metaData OSPMetaData, networkData OSPNetworkData, err error) {	×
185	metaData = &OSPMetaData{}	×
186	networkData = &OSPNetworkData{}	×
187	glog.Infof("getting OpenStack meta_data from metadata server")	×
188	metaDataRawBytes, err := getBodyFromURL(ospMetaDataURL)	×
189	if err != nil {	×
190	return metaData, networkData, fmt.Errorf("error getting OpenStack meta_data from %s: %v", ospMetaDataURL, err)	×
191	}	×
192	err = json.Unmarshal(metaDataRawBytes, metaData)	×
193	if err != nil {	×
194	return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospMetaDataURL)	×
195	}	×
196
197	glog.Infof("getting OpenStack network_data from metadata server")	×
198	networkDataRawBytes, err := getBodyFromURL(ospNetworkDataURL)	×
199	if err != nil {	×
200	return metaData, networkData, fmt.Errorf("error getting OpenStack network_data from %s: %v", ospNetworkDataURL, err)	×
201	}	×
202	err = json.Unmarshal(networkDataRawBytes, networkData)	×
203	if err != nil {	×
204	return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospNetworkDataURL)	×
205	}	×
206	return metaData, networkData, nil	×
207	}
208
209	// CreateOpenstackDevicesInfo create the openstack device info map
210	func CreateOpenstackDevicesInfo(metaData OSPMetaData, networkData OSPNetworkData) (OSPDevicesInfo, error) {	×
211	glog.Infof("CreateOpenstackDevicesInfo()")	×
212	devicesInfo := make(OSPDevicesInfo)	×
213	if metaData == nil \|\| networkData == nil {	×
214	return nil, nil	×
215	}	×
216
217	// use this for hw pass throw interfaces
218	for _, device := range metaData.Devices {	×
219	for _, link := range networkData.Links {	×
220	if device.Mac == link.EthernetMac {	×
221	for _, network := range networkData.Networks {	×
222	if network.Link == link.ID {	×
223	networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID	×
224	devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: device.Mac, NetworkID: networkID}	×
225	}	×
226	}
227	}
228	}
229	}
230
231	// for vhostuser interface type we check the interfaces on the node
232	pci, err := ghw.PCI()	×
233	if err != nil {	×
234	return nil, fmt.Errorf("CreateOpenstackDevicesInfo(): error getting PCI info: %v", err)	×
235	}	×
236
237	devices := pci.ListDevices()	×
238	if len(devices) == 0 {	×
239	return nil, fmt.Errorf("CreateOpenstackDevicesInfo(): could not retrieve PCI devices")	×
240	}	×
241
242	for _, device := range devices {	×
243	if _, exist := devicesInfo[device.Address]; exist {	×
244	//we already discover the device via openstack metadata	×
245	continue	×
246	}
247
248	devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)	×
249	if err != nil {	×
250	glog.Warningf("CreateOpenstackDevicesInfo(): unable to parse device class for device %+v %q", device, err)	×
251	continue	×
252	}
253	if devClass != netClass {	×
254	// Not network device	×
255	continue	×
256	}
257
258	macAddress := ""	×
259	if name := tryToGetVirtualInterfaceName(device.Address); name != "" {	×
260	if mac := getNetDevMac(name); mac != "" {	×
261	macAddress = mac	×
262	}	×
263	}
264	if macAddress == "" {	×
265	// we didn't manage to find a mac address for the nic skipping	×
266	continue	×
267	}
268
269	for _, link := range networkData.Links {	×
270	if macAddress == link.EthernetMac {	×
271	for _, network := range networkData.Networks {	×
272	if network.Link == link.ID {	×
273	networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID	×
274	devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: macAddress, NetworkID: networkID}	×
275	}	×
276	}
277	}
278	}
279	}
280
281	return devicesInfo, err	×
282	}
283
284	// DiscoverSriovDevicesVirtual discovers VFs on a virtual platform
285	func DiscoverSriovDevicesVirtual(devicesInfo OSPDevicesInfo) ([]sriovnetworkv1.InterfaceExt, error) {	×
286	glog.V(2).Info("DiscoverSriovDevicesVirtual()")	×
287	pfList := []sriovnetworkv1.InterfaceExt{}	×
288		×
289	pci, err := ghw.PCI()	×
290	if err != nil {	×
291	return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): error getting PCI info: %v", err)	×
292	}	×
293
294	devices := pci.ListDevices()	×
295	if len(devices) == 0 {	×
296	return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): could not retrieve PCI devices")	×
297	}	×
298
299	for _, device := range devices {	×
300	devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)	×
301	if err != nil {	×
302	glog.Warningf("DiscoverSriovDevicesVirtual(): unable to parse device class for device %+v %q", device, err)	×
303	continue	×
304	}
305	if devClass != netClass {	×
306	// Not network device	×
307	continue	×
308	}
309
310	deviceInfo, exist := devicesInfo[device.Address]	×
311	if !exist {	×
312	glog.Warningf("DiscoverSriovDevicesVirtual(): unable to find device in devicesInfo list for pci %s", device.Address)	×
313	continue	×
314	}
315	netFilter := deviceInfo.NetworkID	×
316	metaMac := deviceInfo.MacAddress	×
317		×
318	driver, err := dputils.GetDriverName(device.Address)	×
319	if err != nil {	×
320	glog.Warningf("DiscoverSriovDevicesVirtual(): unable to parse device driver for device %+v %q", device, err)	×
321	continue	×
322	}
323	iface := sriovnetworkv1.InterfaceExt{	×
324	PciAddress: device.Address,	×
325	Driver: driver,	×
326	Vendor: device.Vendor.ID,	×
327	DeviceID: device.Product.ID,	×
328	NetFilter: netFilter,	×
329	}	×
330	if mtu := getNetdevMTU(device.Address); mtu > 0 {	×
331	iface.Mtu = mtu	×
332	}	×
333	if name := tryToGetVirtualInterfaceName(device.Address); name != "" {	×
334	iface.Name = name	×
335	if iface.Mac = getNetDevMac(name); iface.Mac == "" {	×
336	iface.Mac = metaMac	×
337	}	×
338	iface.LinkSpeed = getNetDevLinkSpeed(name)	×
339	}
340	iface.LinkType = getLinkType(iface)	×
341		×
342	iface.TotalVfs = 1	×
343	iface.NumVfs = 1	×
344		×
345	vf := sriovnetworkv1.VirtualFunction{	×
346	PciAddress: device.Address,	×
347	Driver: driver,	×
348	VfID: 0,	×
349	Vendor: iface.Vendor,	×
350	DeviceID: iface.DeviceID,	×
351	Mtu: iface.Mtu,	×
352	Mac: iface.Mac,	×
353	}	×
354	iface.VFs = append(iface.VFs, vf)	×
355		×
356	pfList = append(pfList, iface)	×
357	}
358	return pfList, nil	×
359	}
360
361	func CreateOpenstackDevicesInfoFromNodeStatus(networkState *sriovnetworkv1.SriovNetworkNodeState) OSPDevicesInfo {	×
362	devicesInfo := make(OSPDevicesInfo)	×
363	for _, iface := range networkState.Status.Interfaces {	×
364	devicesInfo[iface.PciAddress] = &OSPDeviceInfo{MacAddress: iface.Mac, NetworkID: iface.NetFilter}	×
365	}	×
366
367	return devicesInfo	×
368	}
369
370	// tryToGetVirtualInterfaceName get the interface name of a virtio interface
371	func tryToGetVirtualInterfaceName(pciAddr string) string {	×
372	glog.Infof("tryToGetVirtualInterfaceName() get interface name for device %s", pciAddr)	×
373		×
374	// To support different driver that is not virtio-pci like mlx	×
375	name := tryGetInterfaceName(pciAddr)	×
376	if name != "" {	×
377	return name	×
378	}	×
379
380	netDir, err := filepath.Glob(filepath.Join(sysBusPciDevices, pciAddr, "virtio*", "net"))	×
381	if err != nil \|\| len(netDir) < 1 {	×
382	return ""	×
383	}	×
384
385	fInfos, err := ioutil.ReadDir(netDir[0])	×
386	if err != nil {	×
387	glog.Warningf("tryToGetVirtualInterfaceName(): failed to read net directory %s: %q", netDir, err)	×
388	return ""	×
389	}	×
390
391	names := make([]string, 0)	×
392	for _, f := range fInfos {	×
393	names = append(names, f.Name())	×
394	}	×
395
396	if len(names) < 1 {	×
397	return ""	×
398	}	×
399
400	return names[0]	×
401	}
402
403	// SyncNodeStateVirtual attempt to update the node state to match the desired state
404	// in virtual platforms
405	func SyncNodeStateVirtual(newState *sriovnetworkv1.SriovNetworkNodeState) error {	×
406	var err error	×
407	for _, ifaceStatus := range newState.Status.Interfaces {	×
408	for _, iface := range newState.Spec.Interfaces {	×
409	if iface.PciAddress == ifaceStatus.PciAddress {	×
410	if !needUpdateVirtual(&iface, &ifaceStatus) {	×
411	glog.V(2).Infof("SyncNodeStateVirtual(): no need update interface %s", iface.PciAddress)	×
412	break	×
413	}
414	if err = configSriovDeviceVirtual(&iface, &ifaceStatus); err != nil {	×
415	glog.Errorf("SyncNodeStateVirtual(): fail to config sriov interface %s: %v", iface.PciAddress, err)	×
416	return err	×
417	}	×
418	break	×
419	}
420	}
421	}
422	return nil	×
423	}
424
425	func needUpdateVirtual(iface sriovnetworkv1.Interface, ifaceStatus sriovnetworkv1.InterfaceExt) bool {	×
426	// The device MTU is set by the platorm	×
427	// The NumVfs is always 1	×
428	if iface.NumVfs > 0 {	×
429	for _, vf := range ifaceStatus.VFs {	×
430	ingroup := false	×
431	for _, group := range iface.VfGroups {	×
432	if sriovnetworkv1.IndexInRange(vf.VfID, group.VfRange) {	×
433	ingroup = true	×
434	if group.DeviceType != constants.DeviceTypeNetDevice {	×
435	if group.DeviceType != vf.Driver {	×
436	glog.V(2).Infof("needUpdateVirtual(): Driver needs update, desired=%s, current=%s", group.DeviceType, vf.Driver)	×
437	return true	×
438	}	×
439	} else {	×
440	if sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) {	×
441	glog.V(2).Infof("needUpdateVirtual(): Driver needs update, desired=%s, current=%s", group.DeviceType, vf.Driver)	×
442	return true	×
443	}	×
444	}
445	break	×
446	}
447	}
448	if !ingroup && sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) {	×
449	// VF which has DPDK driver loaded but not in any group, needs to be reset to default driver.	×
450	return true	×
451	}	×
452	}
453	}
454	return false	×
455	}
456
457	func configSriovDeviceVirtual(iface sriovnetworkv1.Interface, ifaceStatus sriovnetworkv1.InterfaceExt) error {	×
458	glog.V(2).Infof("configSriovDeviceVirtual(): config interface %s with %v", iface.PciAddress, iface)	×
459	// Config VFs	×
460	if iface.NumVfs > 0 {	×
461	if iface.NumVfs > 1 {	×
462	glog.Warningf("configSriovDeviceVirtual(): in a virtual environment, only one VF per interface (NumVfs: %d)", iface.NumVfs)	×
463	return errors.New("NumVfs > 1")	×
464	}	×
465	if len(iface.VfGroups) != 1 {	×
466	glog.Warningf("configSriovDeviceVirtual(): missing VFGroup")	×
467	return errors.New("NumVfs != 1")	×
468	}	×
469	addr := iface.PciAddress	×
470	glog.V(2).Infof("configSriovDeviceVirtual(): addr %s", addr)	×
471	driver := ""	×
472	vfID := 0	×
473	for _, group := range iface.VfGroups {	×
474	glog.V(2).Infof("configSriovDeviceVirtual(): group %v", group)	×
475	if sriovnetworkv1.IndexInRange(vfID, group.VfRange) {	×
476	glog.V(2).Infof("configSriovDeviceVirtual(): indexInRange %d", vfID)	×
477	if sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) {	×
478	glog.V(2).Infof("configSriovDeviceVirtual(): driver %s", group.DeviceType)	×
479	driver = group.DeviceType	×
480	}	×
481	break	×
482	}
483	}
484	if driver == "" {	×
485	glog.V(2).Infof("configSriovDeviceVirtual(): bind default")	×
486	if err := BindDefaultDriver(addr); err != nil {	×
487	glog.Warningf("configSriovDeviceVirtual(): fail to bind default driver for device %s", addr)	×
488	return err	×
489	}	×
490	} else {	×
491	glog.V(2).Infof("configSriovDeviceVirtual(): bind driver %s", driver)	×
492	if err := BindDpdkDriver(addr, driver); err != nil {	×
493	glog.Warningf("configSriovDeviceVirtual(): fail to bind driver %s for device %s", driver, addr)	×
494	return err	×
495	}	×
496	}
497	}
498	return nil	×
499	}

k8snetworkplumbingwg / sriov-network-operator / 3751025296

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous