11613650922

Committed 31 Oct 2024 02:17PM UTC coverage: 45.458% (-0.08%) from 45.538%

Build # 11613650922

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #796 from zeeke/us/OCPBUGS-43654

kernel: Set arguments based on CPU architecture

Run Details

44 of 85 new or added lines in 9 files covered. (51.76%)

6 existing lines in 1 file now uncovered.

6791 of 14939 relevant lines covered (45.46%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

15.89

/pkg/platforms/openstack/openstack.go

package openstack

import (
        "encoding/json"
        "fmt"
        "io"
        "os"
        "os/exec"
        "path/filepath"
        "strconv"
        "strings"

        "github.com/hashicorp/go-retryablehttp"
        "github.com/jaypipes/ghw"
        "github.com/jaypipes/ghw/pkg/net"
        "sigs.k8s.io/controller-runtime/pkg/log"

        dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host"
)

const (
        varConfigPath      = "/var/config"
        ospMetaDataBaseDir = "/openstack/2018-08-27"
        ospMetaDataDir     = varConfigPath + ospMetaDataBaseDir
        ospMetaDataBaseURL = "http://169.254.169.254" + ospMetaDataBaseDir
        ospNetworkDataJSON = "network_data.json"
        ospMetaDataJSON    = "meta_data.json"
        ospNetworkDataURL  = ospMetaDataBaseURL + "/" + ospNetworkDataJSON
        ospMetaDataURL     = ospMetaDataBaseURL + "/" + ospMetaDataJSON
        // Config drive is defined as an iso9660 or vfat (deprecated) drive
        // with the "config-2" label.
        //https://docs.openstack.org/nova/latest/user/config-drive.html
        configDriveLabel = "config-2"
)

var (
        ospNetworkDataFile = ospMetaDataDir + "/" + ospNetworkDataJSON
        ospMetaDataFile    = ospMetaDataDir + "/" + ospMetaDataJSON
)

//go:generate ../../../bin/mockgen -destination mock/mock_openstack.go -source openstack.go
type OpenstackInterface interface {
        CreateOpenstackDevicesInfo() error
        CreateOpenstackDevicesInfoFromNodeStatus(*sriovnetworkv1.SriovNetworkNodeState)
        DiscoverSriovDevicesVirtual() ([]sriovnetworkv1.InterfaceExt, error)
}

type openstackContext struct {
        hostManager          host.HostManagerInterface
        openStackDevicesInfo OSPDevicesInfo
}

// OSPMetaDataDevice -- Device structure within meta_data.json
type OSPMetaDataDevice struct {
        Vlan      int      `json:"vlan,omitempty"`
        VfTrusted bool     `json:"vf_trusted,omitempty"`
        Type      string   `json:"type,omitempty"`
        Mac       string   `json:"mac,omitempty"`
        Bus       string   `json:"bus,omitempty"`
        Address   string   `json:"address,omitempty"`
        Tags      []string `json:"tags,omitempty"`
}

// OSPMetaData -- Openstack meta_data.json format
type OSPMetaData struct {
        UUID             string              `json:"uuid,omitempty"`
        AdminPass        string              `json:"admin_pass,omitempty"`
        Name             string              `json:"name,omitempty"`
        LaunchIndex      int                 `json:"launch_index,omitempty"`
        AvailabilityZone string              `json:"availability_zone,omitempty"`
        ProjectID        string              `json:"project_id,omitempty"`
        Devices          []OSPMetaDataDevice `json:"devices,omitempty"`
}

// OSPNetworkLink OSP Link metadata
type OSPNetworkLink struct {
        ID          string `json:"id"`
        VifID       string `json:"vif_id,omitempty"`
        Type        string `json:"type"`
        Mtu         int    `json:"mtu,omitempty"`
        EthernetMac string `json:"ethernet_mac_address"`
}

// OSPNetwork OSP Network metadata
type OSPNetwork struct {
        ID        string `json:"id"`
        Type      string `json:"type"`
        Link      string `json:"link"`
        NetworkID string `json:"network_id"`
}

// OSPNetworkData OSP Network metadata
type OSPNetworkData struct {
        Links    []OSPNetworkLink `json:"links,omitempty"`
        Networks []OSPNetwork     `json:"networks,omitempty"`
        // Omit Services
}

type OSPDevicesInfo map[string]*OSPDeviceInfo

type OSPDeviceInfo struct {
        MacAddress string
        NetworkID  string
}

func New(hostManager host.HostManagerInterface) OpenstackInterface {
        return &openstackContext{
                hostManager: hostManager,
        }
}

// GetOpenstackData gets the metadata and network_data
func getOpenstackData(mountConfigDrive bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
        metaData, networkData, err = getOpenstackDataFromConfigDrive(mountConfigDrive)
        if err != nil {
                log.Log.Error(err, "GetOpenStackData(): non-fatal error getting OpenStack data from config drive")
                metaData, networkData, err = getOpenstackDataFromMetadataService()
                if err != nil {
                        return metaData, networkData, fmt.Errorf("GetOpenStackData(): error getting OpenStack data: %w", err)
                }
        }

        // We can't rely on the PCI address from the metadata so we will lookup the real PCI address
        // for the NIC that matches the MAC address.
        //
        // Libvirt/QEMU cannot guarantee that the address specified in the XML will match the address seen by the guest.
        // This is a well known limitation: https://libvirt.org/pci-addresses.html
        // When using the q35 machine type, it highlights this issue due to the change from using PCI to PCI-E bus for virtual devices.
        //
        // With that said, the PCI value in Nova Metadata is a best effort hint due to the limitations mentioned above. Therefore
        // we will lookup the real PCI address for the NIC that matches the MAC address.
        netInfo, err := ghw.Network()
        if err != nil {
                return metaData, networkData, fmt.Errorf("GetOpenStackData(): error getting network info: %w", err)
        }
        for i, device := range metaData.Devices {
                realPCIAddr, err := getPCIAddressFromMACAddress(device.Mac, netInfo.NICs)
                if err != nil {
                        // If we can't find the PCI address, we will just print a warning, return the data as is with no error.
                        // In the future, we'll want to drain the node if sno-initial-node-state.json doesn't exist when daemon is restarted and when we have SR-IOV
                        // allocated devices already.
                        log.Log.Error(err, "Warning GetOpenstackData(): error getting PCI address for device",
                                "device-mac", device.Mac)
                        return metaData, networkData, nil
                }
                if realPCIAddr != device.Address {
                        log.Log.V(2).Info("GetOpenstackData(): PCI address for device does not match Nova metadata value, it'll be overwritten",
                                "device-mac", device.Mac,
                                "current-address", device.Address,
                                "overwrite-address", realPCIAddr)
                        metaData.Devices[i].Address = realPCIAddr
                }
        }

        return metaData, networkData, err
}

// getConfigDriveDevice returns the config drive device which was found
func getConfigDriveDevice() (string, error) {
        dev := "/dev/disk/by-label/" + configDriveLabel
        if _, err := os.Stat(dev); os.IsNotExist(err) {
                out, err := exec.Command(
                        "blkid", "-l",
                        "-t", "LABEL="+configDriveLabel,
                        "-o", "device",
                ).CombinedOutput()
                if err != nil {
                        return "", fmt.Errorf("unable to run blkid: %v", err)
                }
                dev = strings.TrimSpace(string(out))
        }
        log.Log.Info("found config drive device", "device", dev)
        return dev, nil
}

// mountConfigDriveDevice mounts the config drive and return the path
func mountConfigDriveDevice(device string) (string, error) {
        if device == "" {
                return "", fmt.Errorf("device is empty")
        }
        tmpDir, err := os.MkdirTemp("", "sriov-configdrive")
        if err != nil {
                return "", fmt.Errorf("error creating temp directory: %w", err)
        }
        cmd := exec.Command("mount", "-o", "ro", "-t", "auto", device, tmpDir)
        if err := cmd.Run(); err != nil {
                return "", fmt.Errorf("error mounting config drive: %w", err)
        }
        log.Log.V(2).Info("mounted config drive device", "device", device, "path", tmpDir)
        return tmpDir, nil
}

// ummountConfigDriveDevice ummounts the config drive device
func ummountConfigDriveDevice(path string) error {
        if path == "" {
                return fmt.Errorf("path is empty")
        }
        cmd := exec.Command("umount", path)
        if err := cmd.Run(); err != nil {
                return fmt.Errorf("error umounting config drive: %w", err)
        }
        log.Log.V(2).Info("umounted config drive", "path", path)
        return nil
}

// getOpenstackDataFromConfigDrive reads the meta_data and network_data files
func getOpenstackDataFromConfigDrive(mountConfigDrive bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
        metaData = &OSPMetaData{}
        networkData = &OSPNetworkData{}
        var configDrivePath string
        log.Log.Info("reading OpenStack meta_data from config-drive")
        var metadataf *os.File
        ospMetaDataFilePath := ospMetaDataFile
        if mountConfigDrive {
                configDriveDevice, err := getConfigDriveDevice()
                if err != nil {
                        return metaData, networkData, fmt.Errorf("error finding config drive device: %w", err)
                }
                configDrivePath, err = mountConfigDriveDevice(configDriveDevice)
                if err != nil {
                        return metaData, networkData, fmt.Errorf("error mounting config drive device: %w", err)
                }
                defer func() {
                        if e := ummountConfigDriveDevice(configDrivePath); err == nil && e != nil {
                                err = fmt.Errorf("error umounting config drive device: %w", e)
                        }
                        if e := os.Remove(configDrivePath); err == nil && e != nil {
                                err = fmt.Errorf("error removing temp directory %s: %w", configDrivePath, e)
                        }
                }()
                ospMetaDataFilePath = filepath.Join(configDrivePath, ospMetaDataBaseDir, ospMetaDataJSON)
                ospNetworkDataFile = filepath.Join(configDrivePath, ospMetaDataBaseDir, ospNetworkDataJSON)
        }
        metadataf, err = os.Open(ospMetaDataFilePath)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospMetaDataFilePath, err)
        }
        defer func() {
                if e := metadataf.Close(); err == nil && e != nil {
                        err = fmt.Errorf("error closing file %s: %w", ospMetaDataFilePath, e)
                }
        }()
        if err = json.NewDecoder(metadataf).Decode(&metaData); err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospMetaDataFilePath, err)
        }

        log.Log.Info("reading OpenStack network_data from config-drive")
        var networkDataf *os.File
        ospNetworkDataFilePath := ospNetworkDataFile
        networkDataf, err = os.Open(ospNetworkDataFilePath)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospNetworkDataFilePath, err)
        }
        defer func() {
                if e := networkDataf.Close(); err == nil && e != nil {
                        err = fmt.Errorf("error closing file %s: %w", ospNetworkDataFilePath, e)
                }
        }()
        if err = json.NewDecoder(networkDataf).Decode(&networkData); err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospNetworkDataFilePath, err)
        }
        return metaData, networkData, err
}

func getBodyFromURL(url string) ([]byte, error) {
        log.Log.V(2).Info("Getting body from", "url", url)
        resp, err := retryablehttp.Get(url)
        if err != nil {
                return nil, err
        }
        rawBytes, err := io.ReadAll(resp.Body)
        if err != nil {
                return nil, err
        }
        defer resp.Body.Close()
        return rawBytes, nil
}

// getOpenstackDataFromMetadataService fetchs the metadata and network_data from the metadata service
func getOpenstackDataFromMetadataService() (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
        metaData = &OSPMetaData{}
        networkData = &OSPNetworkData{}
        log.Log.Info("getting OpenStack meta_data from metadata server")
        metaDataRawBytes, err := getBodyFromURL(ospMetaDataURL)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error getting OpenStack meta_data from %s: %v", ospMetaDataURL, err)
        }
        err = json.Unmarshal(metaDataRawBytes, metaData)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospMetaDataURL)
        }

        log.Log.Info("getting OpenStack network_data from metadata server")
        networkDataRawBytes, err := getBodyFromURL(ospNetworkDataURL)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error getting OpenStack network_data from %s: %v", ospNetworkDataURL, err)
        }
        err = json.Unmarshal(networkDataRawBytes, networkData)
        if err != nil {
                return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospNetworkDataURL)
        }
        return metaData, networkData, nil
}

// getPCIAddressFromMACAddress returns the PCI address of a device given its MAC address
func getPCIAddressFromMACAddress(macAddress string, nics []*net.NIC) (string, error) {
        var pciAddress string
        for _, nic := range nics {
                if strings.EqualFold(nic.MacAddress, macAddress) {
                        if pciAddress == "" {
                                pciAddress = *nic.PCIAddress
                        } else {
                                return "", fmt.Errorf("more than one device found with MAC address %s is unsupported", macAddress)
                        }
                }
        }

        if pciAddress != "" {
                return pciAddress, nil
        }

        return "", fmt.Errorf("no device found with MAC address %s", macAddress)
}

// CreateOpenstackDevicesInfo create the openstack device info map
func (o *openstackContext) CreateOpenstackDevicesInfo() error {
        log.Log.Info("CreateOpenstackDevicesInfo()")
        devicesInfo := make(OSPDevicesInfo)

        metaData, networkData, err := getOpenstackData(true)
        if err != nil {
                log.Log.Error(err, "failed to read OpenStack data")
                return err
        }

        if metaData == nil || networkData == nil {
                o.openStackDevicesInfo = make(OSPDevicesInfo)
                return nil
        }

        // use this for hw pass throw interfaces
        for _, device := range metaData.Devices {
                for _, link := range networkData.Links {
                        if device.Mac == link.EthernetMac {
                                for _, network := range networkData.Networks {
                                        if network.Link == link.ID {
                                                networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID
                                                devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: device.Mac, NetworkID: networkID}
                                        }
                                }
                        }
                }
        }

        // for vhostuser interface type we check the interfaces on the node
        pci, err := ghw.PCI()
        if err != nil {
                return fmt.Errorf("CreateOpenstackDevicesInfo(): error getting PCI info: %v", err)
        }

        devices := pci.Devices
        if len(devices) == 0 {
                return fmt.Errorf("CreateOpenstackDevicesInfo(): could not retrieve PCI devices")
        }

        for _, device := range devices {
                if _, exist := devicesInfo[device.Address]; exist {
                        //we already discover the device via openstack metadata
                        continue
                }

                devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)
                if err != nil {
                        log.Log.Error(err, "CreateOpenstackDevicesInfo(): unable to parse device class for device, skipping",
                                "device", device)
                        continue
                }
                if devClass != consts.NetClass {
                        // Not network device
                        continue
                }

                macAddress := ""
                if name := o.hostManager.TryToGetVirtualInterfaceName(device.Address); name != "" {
                        if mac := o.hostManager.GetNetDevMac(name); mac != "" {
                                macAddress = mac
                        }
                }
                if macAddress == "" {
                        // we didn't manage to find a mac address for the nic skipping
                        continue
                }

                for _, link := range networkData.Links {
                        if macAddress == link.EthernetMac {
                                for _, network := range networkData.Networks {
                                        if network.Link == link.ID {
                                                networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID
                                                devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: macAddress, NetworkID: networkID}
                                        }
                                }
                        }
                }
        }

        o.openStackDevicesInfo = devicesInfo
        return nil
}

// DiscoverSriovDevicesVirtual discovers VFs on a virtual platform
func (o *openstackContext) DiscoverSriovDevicesVirtual() ([]sriovnetworkv1.InterfaceExt, error) {
        log.Log.V(2).Info("DiscoverSriovDevicesVirtual()")
        pfList := []sriovnetworkv1.InterfaceExt{}

        pci, err := ghw.PCI()
        if err != nil {
                return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): error getting PCI info: %v", err)
        }

        devices := pci.Devices
        if len(devices) == 0 {
                return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): could not retrieve PCI devices")
        }

        for _, device := range devices {
                devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)
                if err != nil {
                        log.Log.Error(err, "DiscoverSriovDevicesVirtual(): unable to parse device class for device, skipping",
                                "device", device)
                        continue
                }
                if devClass != consts.NetClass {
                        // Not network device
                        continue
                }

                deviceInfo, exist := o.openStackDevicesInfo[device.Address]
                if !exist {
                        log.Log.Error(nil, "DiscoverSriovDevicesVirtual(): unable to find device in devicesInfo list, skipping",
                                "device", device.Address)
                        continue
                }
                netFilter := deviceInfo.NetworkID
                metaMac := deviceInfo.MacAddress

                driver, err := dputils.GetDriverName(device.Address)
                if err != nil {
                        log.Log.Error(err, "DiscoverSriovDevicesVirtual(): unable to parse device driver for device, skipping",
                                "device", device)
                        continue
                }
                iface := sriovnetworkv1.InterfaceExt{
                        PciAddress: device.Address,
                        Driver:     driver,
                        Vendor:     device.Vendor.ID,
                        DeviceID:   device.Product.ID,
                        NetFilter:  netFilter,
                }
                if mtu := o.hostManager.GetNetdevMTU(device.Address); mtu > 0 {
                        iface.Mtu = mtu
                }
                if name := o.hostManager.TryToGetVirtualInterfaceName(device.Address); name != "" {
                        iface.Name = name
                        if iface.Mac = o.hostManager.GetNetDevMac(name); iface.Mac == "" {
                                iface.Mac = metaMac
                        }
                        iface.LinkSpeed = o.hostManager.GetNetDevLinkSpeed(name)
                        iface.LinkType = o.hostManager.GetLinkType(name)
                }

                iface.TotalVfs = 1
                iface.NumVfs = 1

                vf := sriovnetworkv1.VirtualFunction{
                        PciAddress: device.Address,
                        Driver:     driver,
                        VfID:       0,
                        Vendor:     iface.Vendor,
                        DeviceID:   iface.DeviceID,
                        Mtu:        iface.Mtu,
                        Mac:        iface.Mac,
                }
                iface.VFs = append(iface.VFs, vf)

                pfList = append(pfList, iface)
        }
        return pfList, nil
}

func (o *openstackContext) CreateOpenstackDevicesInfoFromNodeStatus(networkState *sriovnetworkv1.SriovNetworkNodeState) {
        devicesInfo := make(OSPDevicesInfo)
        for _, iface := range networkState.Status.Interfaces {
                devicesInfo[iface.PciAddress] = &OSPDeviceInfo{MacAddress: iface.Mac, NetworkID: iface.NetFilter}
        }

        o.openStackDevicesInfo = devicesInfo
}

1	package openstack
2
3	import (
4	"encoding/json"
5	"fmt"
6	"io"
7	"os"
8	"os/exec"
9	"path/filepath"
10	"strconv"
11	"strings"
12
13	"github.com/hashicorp/go-retryablehttp"
14	"github.com/jaypipes/ghw"
15	"github.com/jaypipes/ghw/pkg/net"
16	"sigs.k8s.io/controller-runtime/pkg/log"
17
18	dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
19
20	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
21	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
22	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host"
23	)
24
25	const (
26	varConfigPath = "/var/config"
27	ospMetaDataBaseDir = "/openstack/2018-08-27"
28	ospMetaDataDir = varConfigPath + ospMetaDataBaseDir
29	ospMetaDataBaseURL = "http://169.254.169.254" + ospMetaDataBaseDir
30	ospNetworkDataJSON = "network_data.json"
31	ospMetaDataJSON = "meta_data.json"
32	ospNetworkDataURL = ospMetaDataBaseURL + "/" + ospNetworkDataJSON
33	ospMetaDataURL = ospMetaDataBaseURL + "/" + ospMetaDataJSON
34	// Config drive is defined as an iso9660 or vfat (deprecated) drive
35	// with the "config-2" label.
36	//https://docs.openstack.org/nova/latest/user/config-drive.html
37	configDriveLabel = "config-2"
38	)
39
40	var (
41	ospNetworkDataFile = ospMetaDataDir + "/" + ospNetworkDataJSON
42	ospMetaDataFile = ospMetaDataDir + "/" + ospMetaDataJSON
43	)
44
45	//go:generate ../../../bin/mockgen -destination mock/mock_openstack.go -source openstack.go
46	type OpenstackInterface interface {
47	CreateOpenstackDevicesInfo() error
48	CreateOpenstackDevicesInfoFromNodeStatus(*sriovnetworkv1.SriovNetworkNodeState)
49	DiscoverSriovDevicesVirtual() ([]sriovnetworkv1.InterfaceExt, error)
50	}
51
52	type openstackContext struct {
53	hostManager host.HostManagerInterface
54	openStackDevicesInfo OSPDevicesInfo
55	}
56
57	// OSPMetaDataDevice -- Device structure within meta_data.json
58	type OSPMetaDataDevice struct {
59	Vlan int `json:"vlan,omitempty"`
60	VfTrusted bool `json:"vf_trusted,omitempty"`
61	Type string `json:"type,omitempty"`
62	Mac string `json:"mac,omitempty"`
63	Bus string `json:"bus,omitempty"`
64	Address string `json:"address,omitempty"`
65	Tags []string `json:"tags,omitempty"`
66	}
67
68	// OSPMetaData -- Openstack meta_data.json format
69	type OSPMetaData struct {
70	UUID string `json:"uuid,omitempty"`
71	AdminPass string `json:"admin_pass,omitempty"`
72	Name string `json:"name,omitempty"`
73	LaunchIndex int `json:"launch_index,omitempty"`
74	AvailabilityZone string `json:"availability_zone,omitempty"`
75	ProjectID string `json:"project_id,omitempty"`
76	Devices []OSPMetaDataDevice `json:"devices,omitempty"`
77	}
78
79	// OSPNetworkLink OSP Link metadata
80	type OSPNetworkLink struct {
81	ID string `json:"id"`
82	VifID string `json:"vif_id,omitempty"`
83	Type string `json:"type"`
84	Mtu int `json:"mtu,omitempty"`
85	EthernetMac string `json:"ethernet_mac_address"`
86	}
87
88	// OSPNetwork OSP Network metadata
89	type OSPNetwork struct {
90	ID string `json:"id"`
91	Type string `json:"type"`
92	Link string `json:"link"`
93	NetworkID string `json:"network_id"`
94	}
95
96	// OSPNetworkData OSP Network metadata
97	type OSPNetworkData struct {
98	Links []OSPNetworkLink `json:"links,omitempty"`
99	Networks []OSPNetwork `json:"networks,omitempty"`
100	// Omit Services
101	}
102
103	type OSPDevicesInfo map[string]*OSPDeviceInfo
104
105	type OSPDeviceInfo struct {
106	MacAddress string
107	NetworkID string
108	}
109
110	func New(hostManager host.HostManagerInterface) OpenstackInterface {	×
111	return &openstackContext{	×
112	hostManager: hostManager,	×
113	}	×
114	}	×
115
116	// GetOpenstackData gets the metadata and network_data
117	func getOpenstackData(mountConfigDrive bool) (metaData OSPMetaData, networkData OSPNetworkData, err error) {	1✔
118	metaData, networkData, err = getOpenstackDataFromConfigDrive(mountConfigDrive)	1✔
119	if err != nil {	1✔
120	log.Log.Error(err, "GetOpenStackData(): non-fatal error getting OpenStack data from config drive")	×
121	metaData, networkData, err = getOpenstackDataFromMetadataService()	×
122	if err != nil {	×
123	return metaData, networkData, fmt.Errorf("GetOpenStackData(): error getting OpenStack data: %w", err)	×
124	}	×
125	}
126
127	// We can't rely on the PCI address from the metadata so we will lookup the real PCI address
128	// for the NIC that matches the MAC address.
129	//
130	// Libvirt/QEMU cannot guarantee that the address specified in the XML will match the address seen by the guest.
131	// This is a well known limitation: https://libvirt.org/pci-addresses.html
132	// When using the q35 machine type, it highlights this issue due to the change from using PCI to PCI-E bus for virtual devices.
133	//
134	// With that said, the PCI value in Nova Metadata is a best effort hint due to the limitations mentioned above. Therefore
135	// we will lookup the real PCI address for the NIC that matches the MAC address.
136	netInfo, err := ghw.Network()	1✔
137	if err != nil {	1✔
138	return metaData, networkData, fmt.Errorf("GetOpenStackData(): error getting network info: %w", err)	×
139	}	×
140	for i, device := range metaData.Devices {	2✔
141	realPCIAddr, err := getPCIAddressFromMACAddress(device.Mac, netInfo.NICs)	1✔
142	if err != nil {	1✔
143	// If we can't find the PCI address, we will just print a warning, return the data as is with no error.	×
144	// In the future, we'll want to drain the node if sno-initial-node-state.json doesn't exist when daemon is restarted and when we have SR-IOV	×
145	// allocated devices already.	×
146	log.Log.Error(err, "Warning GetOpenstackData(): error getting PCI address for device",	×
147	"device-mac", device.Mac)	×
148	return metaData, networkData, nil	×
149	}	×
150	if realPCIAddr != device.Address {	2✔
151	log.Log.V(2).Info("GetOpenstackData(): PCI address for device does not match Nova metadata value, it'll be overwritten",	1✔
152	"device-mac", device.Mac,	1✔
153	"current-address", device.Address,	1✔
154	"overwrite-address", realPCIAddr)	1✔
155	metaData.Devices[i].Address = realPCIAddr	1✔
156	}	1✔
157	}
158
159	return metaData, networkData, err	1✔
160	}
161
162	// getConfigDriveDevice returns the config drive device which was found
163	func getConfigDriveDevice() (string, error) {	×
164	dev := "/dev/disk/by-label/" + configDriveLabel	×
165	if _, err := os.Stat(dev); os.IsNotExist(err) {	×
166	out, err := exec.Command(	×
167	"blkid", "-l",	×
168	"-t", "LABEL="+configDriveLabel,	×
169	"-o", "device",	×
170	).CombinedOutput()	×
171	if err != nil {	×
172	return "", fmt.Errorf("unable to run blkid: %v", err)	×
173	}	×
174	dev = strings.TrimSpace(string(out))	×
175	}
176	log.Log.Info("found config drive device", "device", dev)	×
177	return dev, nil	×
178	}
179
180	// mountConfigDriveDevice mounts the config drive and return the path
181	func mountConfigDriveDevice(device string) (string, error) {	×
182	if device == "" {	×
183	return "", fmt.Errorf("device is empty")	×
184	}	×
185	tmpDir, err := os.MkdirTemp("", "sriov-configdrive")	×
186	if err != nil {	×
187	return "", fmt.Errorf("error creating temp directory: %w", err)	×
188	}	×
189	cmd := exec.Command("mount", "-o", "ro", "-t", "auto", device, tmpDir)	×
190	if err := cmd.Run(); err != nil {	×
191	return "", fmt.Errorf("error mounting config drive: %w", err)	×
192	}	×
193	log.Log.V(2).Info("mounted config drive device", "device", device, "path", tmpDir)	×
194	return tmpDir, nil	×
195	}
196
197	// ummountConfigDriveDevice ummounts the config drive device
198	func ummountConfigDriveDevice(path string) error {	×
199	if path == "" {	×
200	return fmt.Errorf("path is empty")	×
201	}	×
202	cmd := exec.Command("umount", path)	×
203	if err := cmd.Run(); err != nil {	×
204	return fmt.Errorf("error umounting config drive: %w", err)	×
205	}	×
206	log.Log.V(2).Info("umounted config drive", "path", path)	×
207	return nil	×
208	}
209
210	// getOpenstackDataFromConfigDrive reads the meta_data and network_data files
211	func getOpenstackDataFromConfigDrive(mountConfigDrive bool) (metaData OSPMetaData, networkData OSPNetworkData, err error) {	1✔
212	metaData = &OSPMetaData{}	1✔
213	networkData = &OSPNetworkData{}	1✔
214	var configDrivePath string	1✔
215	log.Log.Info("reading OpenStack meta_data from config-drive")	1✔
216	var metadataf *os.File	1✔
217	ospMetaDataFilePath := ospMetaDataFile	1✔
218	if mountConfigDrive {	1✔
219	configDriveDevice, err := getConfigDriveDevice()	×
220	if err != nil {	×
221	return metaData, networkData, fmt.Errorf("error finding config drive device: %w", err)	×
222	}	×
223	configDrivePath, err = mountConfigDriveDevice(configDriveDevice)	×
224	if err != nil {	×
225	return metaData, networkData, fmt.Errorf("error mounting config drive device: %w", err)	×
226	}	×
227	defer func() {	×
228	if e := ummountConfigDriveDevice(configDrivePath); err == nil && e != nil {	×
229	err = fmt.Errorf("error umounting config drive device: %w", e)	×
230	}	×
231	if e := os.Remove(configDrivePath); err == nil && e != nil {	×
232	err = fmt.Errorf("error removing temp directory %s: %w", configDrivePath, e)	×
233	}	×
234	}()
235	ospMetaDataFilePath = filepath.Join(configDrivePath, ospMetaDataBaseDir, ospMetaDataJSON)	×
236	ospNetworkDataFile = filepath.Join(configDrivePath, ospMetaDataBaseDir, ospNetworkDataJSON)	×
237	}
238	metadataf, err = os.Open(ospMetaDataFilePath)	1✔
239	if err != nil {	1✔
240	return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospMetaDataFilePath, err)	×
241	}	×
242	defer func() {	2✔
243	if e := metadataf.Close(); err == nil && e != nil {	1✔
244	err = fmt.Errorf("error closing file %s: %w", ospMetaDataFilePath, e)	×
245	}	×
246	}()
247	if err = json.NewDecoder(metadataf).Decode(&metaData); err != nil {	1✔
248	return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospMetaDataFilePath, err)	×
249	}	×
250
251	log.Log.Info("reading OpenStack network_data from config-drive")	1✔
252	var networkDataf *os.File	1✔
253	ospNetworkDataFilePath := ospNetworkDataFile	1✔
254	networkDataf, err = os.Open(ospNetworkDataFilePath)	1✔
255	if err != nil {	1✔
256	return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospNetworkDataFilePath, err)	×
257	}	×
258	defer func() {	2✔
259	if e := networkDataf.Close(); err == nil && e != nil {	1✔
260	err = fmt.Errorf("error closing file %s: %w", ospNetworkDataFilePath, e)	×
261	}	×
262	}()
263	if err = json.NewDecoder(networkDataf).Decode(&networkData); err != nil {	1✔
264	return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospNetworkDataFilePath, err)	×
265	}	×
266	return metaData, networkData, err	1✔
267	}
268
269	func getBodyFromURL(url string) ([]byte, error) {	×
270	log.Log.V(2).Info("Getting body from", "url", url)	×
271	resp, err := retryablehttp.Get(url)	×
272	if err != nil {	×
273	return nil, err	×
274	}	×
275	rawBytes, err := io.ReadAll(resp.Body)	×
276	if err != nil {	×
277	return nil, err	×
278	}	×
279	defer resp.Body.Close()	×
280	return rawBytes, nil	×
281	}
282
283	// getOpenstackDataFromMetadataService fetchs the metadata and network_data from the metadata service
284	func getOpenstackDataFromMetadataService() (metaData OSPMetaData, networkData OSPNetworkData, err error) {	×
285	metaData = &OSPMetaData{}	×
286	networkData = &OSPNetworkData{}	×
287	log.Log.Info("getting OpenStack meta_data from metadata server")	×
288	metaDataRawBytes, err := getBodyFromURL(ospMetaDataURL)	×
289	if err != nil {	×
290	return metaData, networkData, fmt.Errorf("error getting OpenStack meta_data from %s: %v", ospMetaDataURL, err)	×
291	}	×
292	err = json.Unmarshal(metaDataRawBytes, metaData)	×
293	if err != nil {	×
294	return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospMetaDataURL)	×
295	}	×
296
297	log.Log.Info("getting OpenStack network_data from metadata server")	×
298	networkDataRawBytes, err := getBodyFromURL(ospNetworkDataURL)	×
299	if err != nil {	×
300	return metaData, networkData, fmt.Errorf("error getting OpenStack network_data from %s: %v", ospNetworkDataURL, err)	×
301	}	×
302	err = json.Unmarshal(networkDataRawBytes, networkData)	×
303	if err != nil {	×
304	return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospNetworkDataURL)	×
305	}	×
306	return metaData, networkData, nil	×
307	}
308
309	// getPCIAddressFromMACAddress returns the PCI address of a device given its MAC address
310	func getPCIAddressFromMACAddress(macAddress string, nics []*net.NIC) (string, error) {	1✔
311	var pciAddress string	1✔
312	for _, nic := range nics {	2✔
313	if strings.EqualFold(nic.MacAddress, macAddress) {	2✔
314	if pciAddress == "" {	2✔
315	pciAddress = *nic.PCIAddress	1✔
316	} else {	1✔
317	return "", fmt.Errorf("more than one device found with MAC address %s is unsupported", macAddress)	×
318	}	×
319	}
320	}
321
322	if pciAddress != "" {	2✔
323	return pciAddress, nil	1✔
324	}	1✔
325
326	return "", fmt.Errorf("no device found with MAC address %s", macAddress)	×
327	}
328
329	// CreateOpenstackDevicesInfo create the openstack device info map
330	func (o *openstackContext) CreateOpenstackDevicesInfo() error {	×
331	log.Log.Info("CreateOpenstackDevicesInfo()")	×
332	devicesInfo := make(OSPDevicesInfo)	×
333		×
334	metaData, networkData, err := getOpenstackData(true)	×
335	if err != nil {	×
336	log.Log.Error(err, "failed to read OpenStack data")	×
337	return err	×
338	}	×
339
340	if metaData == nil \|\| networkData == nil {	×
341	o.openStackDevicesInfo = make(OSPDevicesInfo)	×
342	return nil	×
343	}	×
344
345	// use this for hw pass throw interfaces
346	for _, device := range metaData.Devices {	×
347	for _, link := range networkData.Links {	×
348	if device.Mac == link.EthernetMac {	×
349	for _, network := range networkData.Networks {	×
350	if network.Link == link.ID {	×
351	networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID	×
352	devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: device.Mac, NetworkID: networkID}	×
353	}	×
354	}
355	}
356	}
357	}
358
359	// for vhostuser interface type we check the interfaces on the node
360	pci, err := ghw.PCI()	×
361	if err != nil {	×
362	return fmt.Errorf("CreateOpenstackDevicesInfo(): error getting PCI info: %v", err)	×
363	}	×
364
NEW 365	devices := pci.Devices	×
366	if len(devices) == 0 {	×
367	return fmt.Errorf("CreateOpenstackDevicesInfo(): could not retrieve PCI devices")	×
368	}	×
369
370	for _, device := range devices {	×
371	if _, exist := devicesInfo[device.Address]; exist {	×
372	//we already discover the device via openstack metadata	×
373	continue	×
374	}
375
376	devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)	×
377	if err != nil {	×
378	log.Log.Error(err, "CreateOpenstackDevicesInfo(): unable to parse device class for device, skipping",	×
379	"device", device)	×
380	continue	×
381	}
382	if devClass != consts.NetClass {	×
383	// Not network device	×
384	continue	×
385	}
386
387	macAddress := ""	×
388	if name := o.hostManager.TryToGetVirtualInterfaceName(device.Address); name != "" {	×
389	if mac := o.hostManager.GetNetDevMac(name); mac != "" {	×
390	macAddress = mac	×
391	}	×
392	}
393	if macAddress == "" {	×
394	// we didn't manage to find a mac address for the nic skipping	×
395	continue	×
396	}
397
398	for _, link := range networkData.Links {	×
399	if macAddress == link.EthernetMac {	×
400	for _, network := range networkData.Networks {	×
401	if network.Link == link.ID {	×
402	networkID := sriovnetworkv1.OpenstackNetworkID.String() + ":" + network.NetworkID	×
403	devicesInfo[device.Address] = &OSPDeviceInfo{MacAddress: macAddress, NetworkID: networkID}	×
404	}	×
405	}
406	}
407	}
408	}
409
410	o.openStackDevicesInfo = devicesInfo	×
411	return nil	×
412	}
413
414	// DiscoverSriovDevicesVirtual discovers VFs on a virtual platform
415	func (o *openstackContext) DiscoverSriovDevicesVirtual() ([]sriovnetworkv1.InterfaceExt, error) {	×
416	log.Log.V(2).Info("DiscoverSriovDevicesVirtual()")	×
417	pfList := []sriovnetworkv1.InterfaceExt{}	×
418		×
419	pci, err := ghw.PCI()	×
420	if err != nil {	×
421	return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): error getting PCI info: %v", err)	×
422	}	×
423
NEW 424	devices := pci.Devices	×
425	if len(devices) == 0 {	×
426	return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): could not retrieve PCI devices")	×
427	}	×
428
429	for _, device := range devices {	×
430	devClass, err := strconv.ParseInt(device.Class.ID, 16, 64)	×
431	if err != nil {	×
432	log.Log.Error(err, "DiscoverSriovDevicesVirtual(): unable to parse device class for device, skipping",	×
433	"device", device)	×
434	continue	×
435	}
436	if devClass != consts.NetClass {	×
437	// Not network device	×
438	continue	×
439	}
440
441	deviceInfo, exist := o.openStackDevicesInfo[device.Address]	×
442	if !exist {	×
443	log.Log.Error(nil, "DiscoverSriovDevicesVirtual(): unable to find device in devicesInfo list, skipping",	×
444	"device", device.Address)	×
445	continue	×
446	}
447	netFilter := deviceInfo.NetworkID	×
448	metaMac := deviceInfo.MacAddress	×
449		×
450	driver, err := dputils.GetDriverName(device.Address)	×
451	if err != nil {	×
452	log.Log.Error(err, "DiscoverSriovDevicesVirtual(): unable to parse device driver for device, skipping",	×
453	"device", device)	×
454	continue	×
455	}
456	iface := sriovnetworkv1.InterfaceExt{	×
457	PciAddress: device.Address,	×
458	Driver: driver,	×
459	Vendor: device.Vendor.ID,	×
460	DeviceID: device.Product.ID,	×
461	NetFilter: netFilter,	×
462	}	×
463	if mtu := o.hostManager.GetNetdevMTU(device.Address); mtu > 0 {	×
464	iface.Mtu = mtu	×
465	}	×
466	if name := o.hostManager.TryToGetVirtualInterfaceName(device.Address); name != "" {	×
467	iface.Name = name	×
468	if iface.Mac = o.hostManager.GetNetDevMac(name); iface.Mac == "" {	×
469	iface.Mac = metaMac	×
470	}	×
471	iface.LinkSpeed = o.hostManager.GetNetDevLinkSpeed(name)	×
472	iface.LinkType = o.hostManager.GetLinkType(name)	×
473	}
474
475	iface.TotalVfs = 1	×
476	iface.NumVfs = 1	×
477		×
478	vf := sriovnetworkv1.VirtualFunction{	×
479	PciAddress: device.Address,	×
480	Driver: driver,	×
481	VfID: 0,	×
482	Vendor: iface.Vendor,	×
483	DeviceID: iface.DeviceID,	×
484	Mtu: iface.Mtu,	×
485	Mac: iface.Mac,	×
486	}	×
487	iface.VFs = append(iface.VFs, vf)	×
488		×
489	pfList = append(pfList, iface)	×
490	}
491	return pfList, nil	×
492	}
493
494	func (o openstackContext) CreateOpenstackDevicesInfoFromNodeStatus(networkState sriovnetworkv1.SriovNetworkNodeState) {	×
495	devicesInfo := make(OSPDevicesInfo)	×
496	for _, iface := range networkState.Status.Interfaces {	×
497	devicesInfo[iface.PciAddress] = &OSPDeviceInfo{MacAddress: iface.Mac, NetworkID: iface.NetFilter}	×
498	}	×
499
500	o.openStackDevicesInfo = devicesInfo	×
501	}

k8snetworkplumbingwg / sriov-network-operator / 11613650922

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous