17385686062

Committed 01 Sep 2025 07:20PM UTC coverage: 18.016% (-0.1%) from 18.149%

Build # 17385686062

Build Type

Pull #281

github

Committed by

web-flow

Commit Message

Merge f6f978f6a into 6ec9a6d6b

Pull Request Pull Request #281: Port to Apple Hypervisor framework

Run Details

0 of 152 new or added lines in 11 files covered. (0.0%)

1323 existing lines in 30 files now uncovered.

1362 of 7560 relevant lines covered (18.02%)

18.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/alioth/src/virtio/dev/net/net.rs

// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

pub mod tap;

use std::fmt::Debug;
use std::fs::{File, OpenOptions};
use std::io::{ErrorKind, IoSlice};
use std::mem::MaybeUninit;
use std::num::NonZeroU16;
use std::os::fd::{AsFd, AsRawFd};
use std::os::unix::prelude::OpenOptionsExt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::mpsc::Receiver;
use std::thread::JoinHandle;

use bitflags::bitflags;
use io_uring::cqueue::Entry as Cqe;
use io_uring::opcode;
use io_uring::types::Fd;
use libc::{IFF_MULTI_QUEUE, IFF_NO_PI, IFF_TAP, IFF_VNET_HDR, O_NONBLOCK};
use mio::event::Event;
use mio::unix::SourceFd;
use mio::{Interest, Registry, Token};
use serde::Deserialize;
use serde_aco::Help;
use zerocopy::{FromBytes, Immutable, IntoBytes};

use crate::hv::IoeventFd;
use crate::mem::mapped::RamBus;
use crate::net::MacAddr;
use crate::virtio::dev::{DevParam, DeviceId, Result, Virtio, WakeEvent};
use crate::virtio::queue::{
    DescChain, QueueReg, Status, VirtQueue, copy_from_reader, copy_to_writer,
};
use crate::virtio::worker::io_uring::{ActiveIoUring, BufferAction, IoUring, VirtioIoUring};
use crate::virtio::worker::mio::{ActiveMio, Mio, VirtioMio};
use crate::virtio::worker::{Waker, WorkerApi};
use crate::virtio::{FEATURE_BUILT_IN, IrqSender, error};
use crate::{c_enum, impl_mmio_for_zerocopy};

use self::tap::{TunFeature, tun_set_iff, tun_set_offload, tun_set_vnet_hdr_sz};

#[repr(C, align(8))]
#[derive(Debug, Default, FromBytes, Immutable, IntoBytes)]
pub struct NetConfig {
    mac: MacAddr,
    status: u16,
    max_queue_pairs: u16,
    mtu: u16,
    speed: u32,
    duplex: u8,
    rss_max_key_size: u8,
    rss_max_indirection_table_length: u16,
    supported_hash_types: u32,
}

impl_mmio_for_zerocopy!(NetConfig);

c_enum! {
    #[derive(Default, FromBytes, Immutable, IntoBytes)]
    struct CtrlAck(u8);
    {
        OK = 0;
        ERR = 1;
    }
}

c_enum! {
    #[derive(Default, FromBytes, Immutable, IntoBytes)]
    struct CtrlClass(u8);
    {
        MQ = 4;
    }
}

c_enum! {
    #[derive(Default, FromBytes, Immutable, IntoBytes)]
    struct CtrlMq(u8);
    {
        VQ_PARIS_SET = 0;
    }
}

#[repr(C)]
#[derive(Debug, Default, FromBytes, Immutable, IntoBytes)]
struct CtrlMqParisSet {
    virtq_pairs: u16,
}

#[repr(C)]
#[derive(Debug, Default, FromBytes, Immutable, IntoBytes)]
struct CtrlHdr {
    class: CtrlClass,
    command: u8,
}

bitflags! {
    #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash)]
    pub struct NetFeature: u128 {
        const CSUM = 1 << 0;
        const GUEST_CSUM = 1 << 1;
        const CTRL_GUEST_OFFLOADS = 1 << 2;
        const MTU = 1 << 3;
        const MAC = 1 << 5;
        const GUEST_TSO4 = 1 << 7;
        const GUEST_TSO6 = 1 << 8;
        const GUEST_ECN = 1 << 9;
        const GUEST_UFO = 1 << 10;
        const HOST_TSO4 = 1 << 11;
        const HOST_TSO6 = 1 << 12;
        const HOST_ECN = 1 << 13;
        const HOST_UFO = 1 << 14;
        const MRG_RXBUF = 1 << 15;
        const STATUS = 1 << 16;
        const CTRL_VQ = 1 << 17;
        const CTRL_RX = 1 << 18;
        const CTRL_VLAN = 1 << 19;
        const GUEST_ANNOUNCE = 1 << 21;
        const MQ = 1 << 22;
        const CTRL_MAC_ADDR = 1 << 23;
        const GUEST_USO4 = 1 << 54;
        const GUEST_USO6 = 1 << 55;
        const HOST_USO = 1 << 56;
        const HASH_REPORT = 1 << 57;
        const GUEST_HDRLEN = 1 << 59;
        const RSS = 1 << 60;
        const RSC_EXT = 1 << 61;
        const STANDBY = 1 << 62;
        const SPEED_DUPLEX = 1 << 63;
        const INDIRECT_DESC = 1 << 28;
    }
}

#[derive(Debug)]
pub struct Net {
    name: Arc<str>,
    config: Arc<NetConfig>,
    tap_sockets: Vec<File>,
    feature: NetFeature,
    driver_feature: NetFeature,
    dev_tap: Option<PathBuf>,
    if_name: Option<String>,
    api: WorkerApi,
}

#[derive(Debug, Deserialize, Clone, Help)]
pub struct NetTapParam {
    /// MAC address of the virtual NIC, e.g. 06:3a:76:53:da:3d.
    pub mac: MacAddr,
    /// Maximum transmission unit.
    pub mtu: u16,
    /// Number of pairs of transmit/receive queues. [default: 1]
    #[serde(alias = "qp")]
    pub queue_pairs: Option<NonZeroU16>,
    /// Path to the character device file of a tap interface.
    ///
    /// Required for MacVTap and IPVTap, e.g. /dev/tapX.
    /// Optional for TUN/TAP. [default: /dev/net/tun]
    pub tap: Option<PathBuf>,
    /// Name of a tap interface, e.g. tapX.
    ///
    /// Required for TUN/TAP. Optional for MacVTap and IPVTap.
    #[serde(alias = "if")]
    pub if_name: Option<String>,
    /// System API for asynchronous IO.
    #[serde(default)]
    pub api: WorkerApi,
}

impl DevParam for NetTapParam {
    type Device = Net;

    fn build(self, name: impl Into<Arc<str>>) -> Result<Net> {
        Net::new(self, name)
    }
}

fn new_socket(dev_tap: Option<&Path>, blocking: bool) -> Result<File> {
    let tap_dev = dev_tap.unwrap_or(Path::new("/dev/net/tun"));
    let mut opt = OpenOptions::new();
    opt.read(true).write(true);
    if !blocking {
        opt.custom_flags(O_NONBLOCK);
    }
    let socket = opt.open(tap_dev)?;
    Ok(socket)
}

impl Net {
    pub fn new(param: NetTapParam, name: impl Into<Arc<str>>) -> Result<Self> {
        let mut socket = new_socket(
            param.tap.as_deref(),
            matches!(param.api, WorkerApi::IoUring),
        )?;
        let max_queue_pairs = param.queue_pairs.map(From::from).unwrap_or(1);
        setup_socket(&mut socket, param.if_name.as_deref(), max_queue_pairs > 1)?;
        let mut dev_feat = NetFeature::MAC
            | NetFeature::MTU
            | NetFeature::CSUM
            | NetFeature::HOST_TSO4
            | NetFeature::HOST_TSO6
            | NetFeature::HOST_ECN
            | NetFeature::HOST_UFO
            | NetFeature::HOST_USO
            | NetFeature::CTRL_VQ
            | detect_tap_offload(&socket);
        if max_queue_pairs > 1 {
            dev_feat |= NetFeature::MQ;
        }
        let net = Net {
            name: name.into(),
            config: Arc::new(NetConfig {
                mac: param.mac,
                max_queue_pairs,
                mtu: param.mtu,
                ..Default::default()
            }),
            tap_sockets: vec![socket],
            feature: dev_feat,
            driver_feature: NetFeature::empty(),
            dev_tap: param.tap,
            if_name: param.if_name,
            api: param.api,
        };
        Ok(net)
    }

    fn handle_ctrl_queue(
        &mut self,
        desc: &mut DescChain,
        registry: Option<&Registry>,
    ) -> Result<u32> {
        let Some(header) = desc
            .readable
            .first()
            .and_then(|b| CtrlHdr::read_from_bytes(b).ok())
        else {
            return error::InvalidBuffer.fail();
        };
        let Some(ack_byte) = desc.writable.first_mut().and_then(|v| v.first_mut()) else {
            return error::InvalidBuffer.fail();
        };
        let ack = match header.class {
            CtrlClass::MQ => match CtrlMq(header.command) {
                CtrlMq::VQ_PARIS_SET => {
                    let to_set = |b: &IoSlice| CtrlMqParisSet::read_from_bytes(b).ok();
                    let Some(data) = desc.readable.get(1).and_then(to_set) else {
                        return error::InvalidBuffer.fail();
                    };
                    let pairs = data.virtq_pairs as usize;
                    self.tap_sockets.truncate(pairs);
                    for index in self.tap_sockets.len()..pairs {
                        let mut socket = new_socket(
                            self.dev_tap.as_deref(),
                            matches!(self.api, WorkerApi::IoUring),
                        )?;
                        setup_socket(&mut socket, self.if_name.as_deref(), true)?;
                        enable_tap_offload(&mut socket, self.driver_feature)?;
                        if let Some(r) = registry {
                            r.register(
                                &mut SourceFd(&socket.as_raw_fd()),
                                Token(index),
                                Interest::READABLE | Interest::WRITABLE,
                            )?;
                        }
                        self.tap_sockets.push(socket);
                    }
                    log::info!("{}: using {pairs} pairs of queues", self.name);
                    CtrlAck::OK
                }
                _ => CtrlAck::ERR,
            },
            _ => CtrlAck::ERR,
        };
        *ack_byte = ack.raw();
        Ok(1)
    }
}

impl Virtio for Net {
    type Config = NetConfig;
    type Feature = NetFeature;

    fn id(&self) -> DeviceId {
        DeviceId::Net
    }

    fn name(&self) -> &str {
        &self.name
    }

    fn num_queues(&self) -> u16 {
        let data_queues = self.config.max_queue_pairs << 1;
        if self.feature.contains(NetFeature::CTRL_VQ) {
            data_queues + 1
        } else {
            data_queues
        }
    }

    fn config(&self) -> Arc<NetConfig> {
        self.config.clone()
    }

    fn feature(&self) -> u128 {
        self.feature.bits() | FEATURE_BUILT_IN
    }

    fn spawn_worker<S, E>(
        self,
        event_rx: Receiver<WakeEvent<S, E>>,
        memory: Arc<RamBus>,
        queue_regs: Arc<[QueueReg]>,
    ) -> Result<(JoinHandle<()>, Arc<Waker>)>
    where
        S: IrqSender,
        E: IoeventFd,
    {
        match self.api {
            WorkerApi::Mio => Mio::spawn_worker(self, event_rx, memory, queue_regs),
            WorkerApi::IoUring => IoUring::spawn_worker(self, event_rx, memory, queue_regs),
        }
    }
}

impl VirtioMio for Net {
    fn reset(&mut self, registry: &Registry) {
        self.tap_sockets.truncate(1);
        let _ = registry.deregister(&mut SourceFd(&self.tap_sockets[0].as_raw_fd()));
    }

    fn activate<'m, Q, S, E>(
        &mut self,
        feature: u128,
        active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        Q: VirtQueue<'m>,
        S: IrqSender,
        E: IoeventFd,
    {
        self.driver_feature = NetFeature::from_bits_retain(feature);
        let socket = &mut self.tap_sockets[0];
        enable_tap_offload(socket, self.driver_feature)?;
        active_mio.poll.registry().register(
            &mut SourceFd(&socket.as_raw_fd()),
            Token(0),
            Interest::READABLE | Interest::WRITABLE,
        )?;
        Ok(())
    }

    fn handle_event<'a, 'm, Q, S, E>(
        &mut self,
        event: &Event,
        active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        Q: VirtQueue<'m>,
        S: IrqSender,
        E: IoeventFd,
    {
        let token = event.token().0;
        let irq_sender = active_mio.irq_sender;
        if event.is_readable() {
            let rx_queue_index = token << 1;
            let Some(Some(queue)) = active_mio.queues.get_mut(rx_queue_index) else {
                log::error!("{}: cannot find rx queue {rx_queue_index}", self.name);
                return Ok(());
            };
            let Some(socket) = self.tap_sockets.get(token) else {
                log::error!("{}: cannot find tap queue {token}", self.name);
                return Ok(());
            };
            queue.handle_desc(rx_queue_index as u16, irq_sender, copy_from_reader(socket))?;
        }
        if event.is_writable() {
            let tx_queue_index = (token << 1) + 1;
            let Some(Some(queue)) = active_mio.queues.get_mut(tx_queue_index) else {
                log::error!("{}: cannot find tx queue {tx_queue_index}", self.name);
                return Ok(());
            };
            let Some(socket) = self.tap_sockets.get(token) else {
                log::error!("{}: cannot find tap queue {token}", self.name);
                return Ok(());
            };
            queue.handle_desc(tx_queue_index as u16, irq_sender, copy_to_writer(socket))?;
        }
        Ok(())
    }

    fn handle_queue<'m, Q, S, E>(
        &mut self,
        index: u16,
        active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        Q: VirtQueue<'m>,
        S: IrqSender,
        E: IoeventFd,
    {
        let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else {
            log::error!("{}: invalid queue index {index}", self.name);
            return Ok(());
        };
        let irq_sender = active_mio.irq_sender;
        let registry = active_mio.poll.registry();
        if index == self.config.max_queue_pairs * 2 {
            return queue.handle_desc(index, irq_sender, |chain| {
                let len = self.handle_ctrl_queue(chain, Some(registry))?;
                Ok(Status::Done { len })
            });
        }
        let Some(socket) = self.tap_sockets.get(index as usize >> 1) else {
            log::error!("{}: invalid tap queue {}", self.name, index >> 1);
            return Ok(());
        };
        if index & 1 == 0 {
            queue.handle_desc(index, irq_sender, copy_from_reader(socket))
        } else {
            queue.handle_desc(index, irq_sender, copy_to_writer(socket))
        }
    }
}

impl VirtioIoUring for Net {
    fn activate<'m, Q, S, E>(
        &mut self,
        feature: u128,
        _ring: &mut ActiveIoUring<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        S: IrqSender,
        Q: VirtQueue<'m>,
        E: IoeventFd,
    {
        self.driver_feature = NetFeature::from_bits_retain(feature);
        let socket = &mut self.tap_sockets[0];
        enable_tap_offload(socket, self.driver_feature)?;
        Ok(())
    }

    fn handle_desc(&mut self, q_index: u16, chain: &mut DescChain) -> Result<BufferAction> {
        if q_index == self.config.max_queue_pairs * 2 {
            let len = self.handle_ctrl_queue(chain, None)?;
            return Ok(BufferAction::Written(len));
        }
        let Some(socket) = self.tap_sockets.get(q_index as usize >> 1) else {
            log::error!("{}: invalid tap queue {}", self.name, q_index >> 1);
            return Ok(BufferAction::Written(0));
        };
        let entry = if q_index & 1 == 0 {
            let writable = &chain.writable;
            opcode::Readv::new(
                Fd(socket.as_raw_fd()),
                writable.as_ptr() as *const _,
                writable.len() as _,
            )
            .build()
        } else {
            let readable = &chain.readable;
            opcode::Writev::new(
                Fd(socket.as_raw_fd()),
                readable.as_ptr() as *const _,
                readable.len() as _,
            )
            .build()
        };
        Ok(BufferAction::Sqe(entry))
    }

    fn complete_desc(&mut self, q_index: u16, _chain: &mut DescChain, cqe: &Cqe) -> Result<u32> {
        let ret = cqe.result();
        if ret < 0 {
            let err = std::io::Error::from_raw_os_error(-ret);
            log::error!("{}: failed to send/receive packet: {err}", self.name,);
            return Ok(0);
        }
        if q_index & 1 == 0 {
            Ok(ret as u32)
        } else {
            Ok(0)
        }
    }
}

const VNET_HEADER_SIZE: i32 = 12;

fn setup_socket(file: &mut File, if_name: Option<&str>, mq: bool) -> Result<()> {
    let mut tap_ifconfig = unsafe { MaybeUninit::<libc::ifreq>::zeroed().assume_init() };

    if let Some(name) = if_name {
        let name_len = std::cmp::min(tap_ifconfig.ifr_name.len() - 1, name.len());
        tap_ifconfig.ifr_name.as_mut_bytes()[0..name_len]
            .copy_from_slice(&name.as_bytes()[0..name_len]);
    }

    let mut flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR;
    if mq {
        flags |= IFF_MULTI_QUEUE;
    }
    tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;

    unsafe { tun_set_iff(file, &tap_ifconfig) }.or_else(|e| {
        if e.kind() == ErrorKind::InvalidInput && !mq {
            flags |= IFF_MULTI_QUEUE;
            tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;
            unsafe { tun_set_iff(file, &tap_ifconfig) }
        } else {
            Err(e)
        }
    })?;

    unsafe { tun_set_vnet_hdr_sz(file, &VNET_HEADER_SIZE) }?;
    Ok(())
}

fn detect_tap_offload(tap: &impl AsFd) -> NetFeature {
    let mut tap_feature = TunFeature::all();
    let mut dev_feat = NetFeature::GUEST_CSUM
        | NetFeature::GUEST_TSO4
        | NetFeature::GUEST_TSO6
        | NetFeature::GUEST_ECN
        | NetFeature::GUEST_UFO
        | NetFeature::GUEST_USO4
        | NetFeature::GUEST_USO6;
    if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
        return dev_feat;
    }
    tap_feature &= !(TunFeature::USO4 | TunFeature::USO6);
    dev_feat &= !(NetFeature::GUEST_USO4 | NetFeature::GUEST_USO6);
    if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
        return dev_feat;
    }
    tap_feature &= !(TunFeature::UFO);
    dev_feat &= !NetFeature::GUEST_UFO;
    if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
        return dev_feat;
    }
    NetFeature::empty()
}

fn enable_tap_offload(tap: &mut File, feature: NetFeature) -> Result<()> {
    let mut tap_feature = TunFeature::empty();
    if feature.contains(NetFeature::GUEST_CSUM) {
        tap_feature |= TunFeature::CSUM;
    }
    if feature.contains(NetFeature::GUEST_TSO4) {
        tap_feature |= TunFeature::TSO4;
    }
    if feature.contains(NetFeature::GUEST_TSO6) {
        tap_feature |= TunFeature::TSO6;
    }
    if feature.contains(NetFeature::GUEST_ECN) {
        tap_feature |= TunFeature::TSO_ECN;
    }
    if feature.contains(NetFeature::GUEST_UFO) {
        tap_feature |= TunFeature::UFO;
    }
    if feature.contains(NetFeature::GUEST_USO4) {
        tap_feature |= TunFeature::USO4;
    }
    if feature.contains(NetFeature::GUEST_USO6) {
        tap_feature |= TunFeature::USO6;
    }
    unsafe { tun_set_offload(tap, tap_feature.bits()) }?;
    Ok(())
}

1	// Copyright 2024 Google LLC
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// https://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	pub mod tap;
16
17	use std::fmt::Debug;
18	use std::fs::{File, OpenOptions};
19	use std::io::{ErrorKind, IoSlice};
20	use std::mem::MaybeUninit;
21	use std::num::NonZeroU16;
22	use std::os::fd::{AsFd, AsRawFd};
23	use std::os::unix::prelude::OpenOptionsExt;
24	use std::path::{Path, PathBuf};
25	use std::sync::Arc;
26	use std::sync::mpsc::Receiver;
27	use std::thread::JoinHandle;
28
29	use bitflags::bitflags;
30	use io_uring::cqueue::Entry as Cqe;
31	use io_uring::opcode;
32	use io_uring::types::Fd;
33	use libc::{IFF_MULTI_QUEUE, IFF_NO_PI, IFF_TAP, IFF_VNET_HDR, O_NONBLOCK};
34	use mio::event::Event;
35	use mio::unix::SourceFd;
36	use mio::{Interest, Registry, Token};
37	use serde::Deserialize;
38	use serde_aco::Help;
39	use zerocopy::{FromBytes, Immutable, IntoBytes};
40
41	use crate::hv::IoeventFd;
42	use crate::mem::mapped::RamBus;
43	use crate::net::MacAddr;
44	use crate::virtio::dev::{DevParam, DeviceId, Result, Virtio, WakeEvent};
45	use crate::virtio::queue::{
46	DescChain, QueueReg, Status, VirtQueue, copy_from_reader, copy_to_writer,
47	};
48	use crate::virtio::worker::io_uring::{ActiveIoUring, BufferAction, IoUring, VirtioIoUring};
49	use crate::virtio::worker::mio::{ActiveMio, Mio, VirtioMio};
50	use crate::virtio::worker::{Waker, WorkerApi};
51	use crate::virtio::{FEATURE_BUILT_IN, IrqSender, error};
52	use crate::{c_enum, impl_mmio_for_zerocopy};
53
54	use self::tap::{TunFeature, tun_set_iff, tun_set_offload, tun_set_vnet_hdr_sz};
55
56	#[repr(C, align(8))]
57	#[derive(Debug, Default, FromBytes, Immutable, IntoBytes)]
58	pub struct NetConfig {
59	mac: MacAddr,
60	status: u16,
61	max_queue_pairs: u16,
62	mtu: u16,
63	speed: u32,
64	duplex: u8,
65	rss_max_key_size: u8,
66	rss_max_indirection_table_length: u16,
67	supported_hash_types: u32,
68	}
69
70	impl_mmio_for_zerocopy!(NetConfig);
71
72	c_enum! {
73	#[derive(Default, FromBytes, Immutable, IntoBytes)]
74	struct CtrlAck(u8);
75	{
76	OK = 0;
77	ERR = 1;
78	}
79	}
80
81	c_enum! {
82	#[derive(Default, FromBytes, Immutable, IntoBytes)]
83	struct CtrlClass(u8);
84	{
85	MQ = 4;
86	}
87	}
88
89	c_enum! {
90	#[derive(Default, FromBytes, Immutable, IntoBytes)]
91	struct CtrlMq(u8);
92	{
93	VQ_PARIS_SET = 0;
94	}
95	}
96
97	#[repr(C)]
98	#[derive(Debug, Default, FromBytes, Immutable, IntoBytes)]
99	struct CtrlMqParisSet {
100	virtq_pairs: u16,
101	}
102
103	#[repr(C)]
104	#[derive(Debug, Default, FromBytes, Immutable, IntoBytes)]
105	struct CtrlHdr {
106	class: CtrlClass,
107	command: u8,
108	}
109
110	bitflags! {
111	#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash)]
112	pub struct NetFeature: u128 {
113	const CSUM = 1 << 0;
114	const GUEST_CSUM = 1 << 1;
115	const CTRL_GUEST_OFFLOADS = 1 << 2;
116	const MTU = 1 << 3;
117	const MAC = 1 << 5;
118	const GUEST_TSO4 = 1 << 7;
119	const GUEST_TSO6 = 1 << 8;
120	const GUEST_ECN = 1 << 9;
121	const GUEST_UFO = 1 << 10;
122	const HOST_TSO4 = 1 << 11;
123	const HOST_TSO6 = 1 << 12;
124	const HOST_ECN = 1 << 13;
125	const HOST_UFO = 1 << 14;
126	const MRG_RXBUF = 1 << 15;
127	const STATUS = 1 << 16;
128	const CTRL_VQ = 1 << 17;
129	const CTRL_RX = 1 << 18;
130	const CTRL_VLAN = 1 << 19;
131	const GUEST_ANNOUNCE = 1 << 21;
132	const MQ = 1 << 22;
133	const CTRL_MAC_ADDR = 1 << 23;
134	const GUEST_USO4 = 1 << 54;
135	const GUEST_USO6 = 1 << 55;
136	const HOST_USO = 1 << 56;
137	const HASH_REPORT = 1 << 57;
138	const GUEST_HDRLEN = 1 << 59;
139	const RSS = 1 << 60;
140	const RSC_EXT = 1 << 61;
141	const STANDBY = 1 << 62;
142	const SPEED_DUPLEX = 1 << 63;
143	const INDIRECT_DESC = 1 << 28;
144	}
145	}
146
147	#[derive(Debug)]
148	pub struct Net {
149	name: Arc<str>,
150	config: Arc<NetConfig>,
151	tap_sockets: Vec<File>,
152	feature: NetFeature,
153	driver_feature: NetFeature,
154	dev_tap: Option<PathBuf>,
155	if_name: Option<String>,
156	api: WorkerApi,
157	}
158
159	#[derive(Debug, Deserialize, Clone, Help)]
160	pub struct NetTapParam {
161	/// MAC address of the virtual NIC, e.g. 06:3a:76:53:da:3d.
162	pub mac: MacAddr,
163	/// Maximum transmission unit.
164	pub mtu: u16,
165	/// Number of pairs of transmit/receive queues. [default: 1]
166	#[serde(alias = "qp")]
167	pub queue_pairs: Option<NonZeroU16>,
168	/// Path to the character device file of a tap interface.
169	///
170	/// Required for MacVTap and IPVTap, e.g. /dev/tapX.
171	/// Optional for TUN/TAP. [default: /dev/net/tun]
172	pub tap: Option<PathBuf>,
173	/// Name of a tap interface, e.g. tapX.
174	///
175	/// Required for TUN/TAP. Optional for MacVTap and IPVTap.
176	#[serde(alias = "if")]
177	pub if_name: Option<String>,
178	/// System API for asynchronous IO.
179	#[serde(default)]
180	pub api: WorkerApi,
181	}
182
183	impl DevParam for NetTapParam {
184	type Device = Net;
185
186	fn build(self, name: impl Into<Arc<str>>) -> Result<Net> {	×
187	Net::new(self, name)	×
188	}
189	}
190
UNCOV 191	fn new_socket(dev_tap: Option<&Path>, blocking: bool) -> Result<File> {
UNCOV 192	let tap_dev = dev_tap.unwrap_or(Path::new("/dev/net/tun"));
UNCOV 193	let mut opt = OpenOptions::new();
UNCOV 194	opt.read(true).write(true);
UNCOV 195	if !blocking {
UNCOV 196	opt.custom_flags(O_NONBLOCK);
197	}
UNCOV 198	let socket = opt.open(tap_dev)?;
UNCOV 199	Ok(socket)
200	}
201
202	impl Net {
203	pub fn new(param: NetTapParam, name: impl Into<Arc<str>>) -> Result<Self> {	×
204	let mut socket = new_socket(
205	param.tap.as_deref(),	×
206	matches!(param.api, WorkerApi::IoUring),	×
207	)?;
208	let max_queue_pairs = param.queue_pairs.map(From::from).unwrap_or(1);	×
209	setup_socket(&mut socket, param.if_name.as_deref(), max_queue_pairs > 1)?;	×
210	let mut dev_feat = NetFeature::MAC	×
211	\| NetFeature::MTU	×
212	\| NetFeature::CSUM	×
213	\| NetFeature::HOST_TSO4	×
214	\| NetFeature::HOST_TSO6	×
215	\| NetFeature::HOST_ECN	×
216	\| NetFeature::HOST_UFO	×
217	\| NetFeature::HOST_USO	×
218	\| NetFeature::CTRL_VQ	×
219	\| detect_tap_offload(&socket);	×
220	if max_queue_pairs > 1 {	×
221	dev_feat \|= NetFeature::MQ;	×
222	}
223	let net = Net {
224	name: name.into(),	×
225	config: Arc::new(NetConfig {	×
226	mac: param.mac,
227	max_queue_pairs,
228	mtu: param.mtu,
229	..Default::default()
230	}),
231	tap_sockets: vec![socket],	×
232	feature: dev_feat,
233	driver_feature: NetFeature::empty(),	×
234	dev_tap: param.tap,	×
235	if_name: param.if_name,	×
236	api: param.api,	×
237	};
238	Ok(net)	×
239	}
240
UNCOV 241	fn handle_ctrl_queue(
242	&mut self,
243	desc: &mut DescChain,
244	registry: Option<&Registry>,
245	) -> Result<u32> {
UNCOV 246	let Some(header) = desc
UNCOV 247	.readable
248	.first()
UNCOV 249	.and_then(\|b\| CtrlHdr::read_from_bytes(b).ok())
250	else {
UNCOV 251	return error::InvalidBuffer.fail();
252	};
UNCOV 253	let Some(ack_byte) = desc.writable.first_mut().and_then(\|v\| v.first_mut()) else {
UNCOV 254	return error::InvalidBuffer.fail();
255	};
UNCOV 256	let ack = match header.class {
UNCOV 257	CtrlClass::MQ => match CtrlMq(header.command) {
258	CtrlMq::VQ_PARIS_SET => {
UNCOV 259	let to_set = \|b: &IoSlice\| CtrlMqParisSet::read_from_bytes(b).ok();
UNCOV 260	let Some(data) = desc.readable.get(1).and_then(to_set) else {
UNCOV 261	return error::InvalidBuffer.fail();
262	};
UNCOV 263	let pairs = data.virtq_pairs as usize;
UNCOV 264	self.tap_sockets.truncate(pairs);
UNCOV 265	for index in self.tap_sockets.len()..pairs {
266	let mut socket = new_socket(
UNCOV 267	self.dev_tap.as_deref(),
UNCOV 268	matches!(self.api, WorkerApi::IoUring),
269	)?;
UNCOV 270	setup_socket(&mut socket, self.if_name.as_deref(), true)?;
UNCOV 271	enable_tap_offload(&mut socket, self.driver_feature)?;
UNCOV 272	if let Some(r) = registry {
UNCOV 273	r.register(
UNCOV 274	&mut SourceFd(&socket.as_raw_fd()),
UNCOV 275	Token(index),
UNCOV 276	Interest::READABLE \| Interest::WRITABLE,
277	)?;
278	}
UNCOV 279	self.tap_sockets.push(socket);
280	}
UNCOV 281	log::info!("{}: using {pairs} pairs of queues", self.name);
UNCOV 282	CtrlAck::OK
283	}
UNCOV 284	_ => CtrlAck::ERR,
285	},
UNCOV 286	_ => CtrlAck::ERR,
287	};
UNCOV 288	*ack_byte = ack.raw();
UNCOV 289	Ok(1)
290	}
291	}
292
293	impl Virtio for Net {
294	type Config = NetConfig;
295	type Feature = NetFeature;
296
UNCOV 297	fn id(&self) -> DeviceId {
UNCOV 298	DeviceId::Net
299	}
300
UNCOV 301	fn name(&self) -> &str {
UNCOV 302	&self.name
303	}
304
UNCOV 305	fn num_queues(&self) -> u16 {
UNCOV 306	let data_queues = self.config.max_queue_pairs << 1;
UNCOV 307	if self.feature.contains(NetFeature::CTRL_VQ) {
UNCOV 308	data_queues + 1
309	} else {
UNCOV 310	data_queues
311	}
312	}
313
UNCOV 314	fn config(&self) -> Arc<NetConfig> {
UNCOV 315	self.config.clone()
316	}
317
UNCOV 318	fn feature(&self) -> u128 {
UNCOV 319	self.feature.bits() \| FEATURE_BUILT_IN
320	}
321
UNCOV 322	fn spawn_worker<S, E>(
323	self,
324	event_rx: Receiver<WakeEvent<S, E>>,
325	memory: Arc<RamBus>,
326	queue_regs: Arc<[QueueReg]>,
327	) -> Result<(JoinHandle<()>, Arc<Waker>)>
328	where
329	S: IrqSender,
330	E: IoeventFd,
331	{
332	match self.api {	×
333	WorkerApi::Mio => Mio::spawn_worker(self, event_rx, memory, queue_regs),	×
334	WorkerApi::IoUring => IoUring::spawn_worker(self, event_rx, memory, queue_regs),	×
335	}
336	}
337	}
338
339	impl VirtioMio for Net {
UNCOV 340	fn reset(&mut self, registry: &Registry) {
UNCOV 341	self.tap_sockets.truncate(1);
UNCOV 342	let _ = registry.deregister(&mut SourceFd(&self.tap_sockets[0].as_raw_fd()));
343	}
344
UNCOV 345	fn activate<'m, Q, S, E>(
346	&mut self,
347	feature: u128,
348	active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
349	) -> Result<()>
350	where
351	Q: VirtQueue<'m>,
352	S: IrqSender,
353	E: IoeventFd,
354	{
355	self.driver_feature = NetFeature::from_bits_retain(feature);	×
356	let socket = &mut self.tap_sockets[0];	×
357	enable_tap_offload(socket, self.driver_feature)?;	×
358	active_mio.poll.registry().register(	×
359	&mut SourceFd(&socket.as_raw_fd()),	×
360	Token(0),	×
361	Interest::READABLE \| Interest::WRITABLE,	×
362	)?;
363	Ok(())	×
364	}
365
UNCOV 366	fn handle_event<'a, 'm, Q, S, E>(
367	&mut self,
368	event: &Event,
369	active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
370	) -> Result<()>
371	where
372	Q: VirtQueue<'m>,
373	S: IrqSender,
374	E: IoeventFd,
375	{
376	let token = event.token().0;	×
377	let irq_sender = active_mio.irq_sender;	×
378	if event.is_readable() {	×
379	let rx_queue_index = token << 1;	×
380	let Some(Some(queue)) = active_mio.queues.get_mut(rx_queue_index) else {	×
381	log::error!("{}: cannot find rx queue {rx_queue_index}", self.name);	×
382	return Ok(());	×
383	};
384	let Some(socket) = self.tap_sockets.get(token) else {	×
385	log::error!("{}: cannot find tap queue {token}", self.name);	×
386	return Ok(());	×
387	};
388	queue.handle_desc(rx_queue_index as u16, irq_sender, copy_from_reader(socket))?;	×
389	}
390	if event.is_writable() {	×
391	let tx_queue_index = (token << 1) + 1;	×
392	let Some(Some(queue)) = active_mio.queues.get_mut(tx_queue_index) else {	×
393	log::error!("{}: cannot find tx queue {tx_queue_index}", self.name);	×
394	return Ok(());	×
395	};
396	let Some(socket) = self.tap_sockets.get(token) else {	×
397	log::error!("{}: cannot find tap queue {token}", self.name);	×
398	return Ok(());	×
399	};
400	queue.handle_desc(tx_queue_index as u16, irq_sender, copy_to_writer(socket))?;	×
401	}
402	Ok(())	×
403	}
404
UNCOV 405	fn handle_queue<'m, Q, S, E>(
406	&mut self,
407	index: u16,
408	active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
409	) -> Result<()>
410	where
411	Q: VirtQueue<'m>,
412	S: IrqSender,
413	E: IoeventFd,
414	{
415	let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else {	×
416	log::error!("{}: invalid queue index {index}", self.name);	×
417	return Ok(());	×
418	};
419	let irq_sender = active_mio.irq_sender;	×
420	let registry = active_mio.poll.registry();	×
421	if index == self.config.max_queue_pairs * 2 {	×
422	return queue.handle_desc(index, irq_sender, \|chain\| {	×
423	let len = self.handle_ctrl_queue(chain, Some(registry))?;	×
424	Ok(Status::Done { len })	×
425	});
426	}
427	let Some(socket) = self.tap_sockets.get(index as usize >> 1) else {	×
428	log::error!("{}: invalid tap queue {}", self.name, index >> 1);	×
429	return Ok(());	×
430	};
431	if index & 1 == 0 {	×
432	queue.handle_desc(index, irq_sender, copy_from_reader(socket))	×
433	} else {
434	queue.handle_desc(index, irq_sender, copy_to_writer(socket))	×
435	}
436	}
437	}
438
439	impl VirtioIoUring for Net {
UNCOV 440	fn activate<'m, Q, S, E>(
441	&mut self,
442	feature: u128,
443	_ring: &mut ActiveIoUring<'_, '_, 'm, Q, S, E>,
444	) -> Result<()>
445	where
446	S: IrqSender,
447	Q: VirtQueue<'m>,
448	E: IoeventFd,
449	{
450	self.driver_feature = NetFeature::from_bits_retain(feature);	×
451	let socket = &mut self.tap_sockets[0];	×
452	enable_tap_offload(socket, self.driver_feature)?;	×
453	Ok(())	×
454	}
455
UNCOV 456	fn handle_desc(&mut self, q_index: u16, chain: &mut DescChain) -> Result<BufferAction> {
UNCOV 457	if q_index == self.config.max_queue_pairs * 2 {
UNCOV 458	let len = self.handle_ctrl_queue(chain, None)?;
UNCOV 459	return Ok(BufferAction::Written(len));
460	}
UNCOV 461	let Some(socket) = self.tap_sockets.get(q_index as usize >> 1) else {
UNCOV 462	log::error!("{}: invalid tap queue {}", self.name, q_index >> 1);
UNCOV 463	return Ok(BufferAction::Written(0));
464	};
UNCOV 465	let entry = if q_index & 1 == 0 {
UNCOV 466	let writable = &chain.writable;
467	opcode::Readv::new(
UNCOV 468	Fd(socket.as_raw_fd()),
UNCOV 469	writable.as_ptr() as *const _,
UNCOV 470	writable.len() as _,
471	)
472	.build()
473	} else {
UNCOV 474	let readable = &chain.readable;
475	opcode::Writev::new(
UNCOV 476	Fd(socket.as_raw_fd()),
UNCOV 477	readable.as_ptr() as *const _,
UNCOV 478	readable.len() as _,
479	)
480	.build()
481	};
UNCOV 482	Ok(BufferAction::Sqe(entry))
483	}
484
UNCOV 485	fn complete_desc(&mut self, q_index: u16, _chain: &mut DescChain, cqe: &Cqe) -> Result<u32> {
UNCOV 486	let ret = cqe.result();
UNCOV 487	if ret < 0 {
UNCOV 488	let err = std::io::Error::from_raw_os_error(-ret);
UNCOV 489	log::error!("{}: failed to send/receive packet: {err}", self.name,);
UNCOV 490	return Ok(0);
491	}
UNCOV 492	if q_index & 1 == 0 {
UNCOV 493	Ok(ret as u32)
494	} else {
UNCOV 495	Ok(0)
496	}
497	}
498	}
499
500	const VNET_HEADER_SIZE: i32 = 12;
501
UNCOV 502	fn setup_socket(file: &mut File, if_name: Option<&str>, mq: bool) -> Result<()> {
UNCOV 503	let mut tap_ifconfig = unsafe { MaybeUninit::<libc::ifreq>::zeroed().assume_init() };
504
UNCOV 505	if let Some(name) = if_name {
UNCOV 506	let name_len = std::cmp::min(tap_ifconfig.ifr_name.len() - 1, name.len());
UNCOV 507	tap_ifconfig.ifr_name.as_mut_bytes()[0..name_len]
UNCOV 508	.copy_from_slice(&name.as_bytes()[0..name_len]);
509	}
510
UNCOV 511	let mut flags = IFF_TAP \| IFF_NO_PI \| IFF_VNET_HDR;
UNCOV 512	if mq {
UNCOV 513	flags \|= IFF_MULTI_QUEUE;
514	}
UNCOV 515	tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;
516
UNCOV 517	unsafe { tun_set_iff(file, &tap_ifconfig) }.or_else(\|e\| {
UNCOV 518	if e.kind() == ErrorKind::InvalidInput && !mq {
UNCOV 519	flags \|= IFF_MULTI_QUEUE;
UNCOV 520	tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;
UNCOV 521	unsafe { tun_set_iff(file, &tap_ifconfig) }
522	} else {
UNCOV 523	Err(e)
524	}
525	})?;
526
UNCOV 527	unsafe { tun_set_vnet_hdr_sz(file, &VNET_HEADER_SIZE) }?;
UNCOV 528	Ok(())
529	}
530
UNCOV 531	fn detect_tap_offload(tap: &impl AsFd) -> NetFeature {
UNCOV 532	let mut tap_feature = TunFeature::all();
UNCOV 533	let mut dev_feat = NetFeature::GUEST_CSUM
UNCOV 534	\| NetFeature::GUEST_TSO4
UNCOV 535	\| NetFeature::GUEST_TSO6
UNCOV 536	\| NetFeature::GUEST_ECN
UNCOV 537	\| NetFeature::GUEST_UFO
UNCOV 538	\| NetFeature::GUEST_USO4
UNCOV 539	\| NetFeature::GUEST_USO6;
UNCOV 540	if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
UNCOV 541	return dev_feat;
542	}
UNCOV 543	tap_feature &= !(TunFeature::USO4 \| TunFeature::USO6);
UNCOV 544	dev_feat &= !(NetFeature::GUEST_USO4 \| NetFeature::GUEST_USO6);
UNCOV 545	if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
UNCOV 546	return dev_feat;
547	}
UNCOV 548	tap_feature &= !(TunFeature::UFO);
UNCOV 549	dev_feat &= !NetFeature::GUEST_UFO;
UNCOV 550	if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
UNCOV 551	return dev_feat;
552	}
UNCOV 553	NetFeature::empty()
554	}
555
UNCOV 556	fn enable_tap_offload(tap: &mut File, feature: NetFeature) -> Result<()> {
UNCOV 557	let mut tap_feature = TunFeature::empty();
UNCOV 558	if feature.contains(NetFeature::GUEST_CSUM) {
UNCOV 559	tap_feature \|= TunFeature::CSUM;
560	}
UNCOV 561	if feature.contains(NetFeature::GUEST_TSO4) {
UNCOV 562	tap_feature \|= TunFeature::TSO4;
563	}
UNCOV 564	if feature.contains(NetFeature::GUEST_TSO6) {
UNCOV 565	tap_feature \|= TunFeature::TSO6;
566	}
UNCOV 567	if feature.contains(NetFeature::GUEST_ECN) {
UNCOV 568	tap_feature \|= TunFeature::TSO_ECN;
569	}
UNCOV 570	if feature.contains(NetFeature::GUEST_UFO) {
UNCOV 571	tap_feature \|= TunFeature::UFO;
572	}
UNCOV 573	if feature.contains(NetFeature::GUEST_USO4) {
UNCOV 574	tap_feature \|= TunFeature::USO4;
575	}
UNCOV 576	if feature.contains(NetFeature::GUEST_USO6) {
UNCOV 577	tap_feature \|= TunFeature::USO6;
578	}
UNCOV 579	unsafe { tun_set_offload(tap, tap_feature.bits()) }?;
UNCOV 580	Ok(())
581	}

google / alioth / 17385686062

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous