18638570188

Committed 20 Oct 2025 12:38AM UTC coverage: 20.202% (-0.01%) from 20.213%

Build # 18638570188

Build Type

Pull #308

github

Committed by

web-flow

Commit Message

Merge 73a1640e9 into 416357998

Pull Request Pull Request #308: Add tests for PciSegment

Run Details

0 of 59 new or added lines in 5 files covered. (0.0%)

1163 existing lines in 25 files now uncovered.

1578 of 7811 relevant lines covered (20.2%)

19.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/alioth/src/virtio/dev/net/tap.rs

// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::fmt::Debug;
use std::fs::{File, OpenOptions};
use std::io::{ErrorKind, IoSlice};
use std::mem::MaybeUninit;
use std::num::NonZeroU16;
use std::os::fd::{AsFd, AsRawFd};
use std::os::unix::prelude::OpenOptionsExt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::mpsc::Receiver;
use std::thread::JoinHandle;

use io_uring::cqueue::Entry as Cqe;
use io_uring::opcode;
use io_uring::types::Fd;
use libc::{IFF_MULTI_QUEUE, IFF_NO_PI, IFF_TAP, IFF_VNET_HDR, O_NONBLOCK};
use mio::event::Event;
use mio::unix::SourceFd;
use mio::{Interest, Registry, Token};
use serde::Deserialize;
use serde_aco::Help;
use zerocopy::{FromBytes, IntoBytes};

use crate::hv::IoeventFd;
use crate::mem::mapped::RamBus;
use crate::sync::notifier::Notifier;
use crate::sys::if_tun::{TunFeature, tun_set_iff, tun_set_offload, tun_set_vnet_hdr_sz};
use crate::virtio::dev::net::mac_addr::MacAddr;
use crate::virtio::dev::net::{
    CtrlAck, CtrlClass, CtrlHdr, CtrlMq, CtrlMqParisSet, NetConfig, NetFeature, VirtioNetHdr,
};
use crate::virtio::dev::{DevParam, DeviceId, Result, Virtio, WakeEvent};
use crate::virtio::queue::{
    DescChain, QueueReg, Status, VirtQueue, copy_from_reader, copy_to_writer,
};
use crate::virtio::worker::WorkerApi;
use crate::virtio::worker::io_uring::{ActiveIoUring, BufferAction, IoUring, VirtioIoUring};
use crate::virtio::worker::mio::{ActiveMio, Mio, VirtioMio};
use crate::virtio::{FEATURE_BUILT_IN, IrqSender, error};

#[derive(Debug)]
pub struct Net {
    name: Arc<str>,
    config: Arc<NetConfig>,
    tap_sockets: Vec<File>,
    feature: NetFeature,
    driver_feature: NetFeature,
    dev_tap: Option<PathBuf>,
    if_name: Option<String>,
    api: WorkerApi,
}

#[derive(Debug, Deserialize, Clone, Help)]
pub struct NetTapParam {
    /// MAC address of the virtual NIC, e.g. 06:3a:76:53:da:3d.
    pub mac: MacAddr,
    /// Maximum transmission unit.
    pub mtu: u16,
    /// Number of pairs of transmit/receive queues. [default: 1]
    #[serde(alias = "qp")]
    pub queue_pairs: Option<NonZeroU16>,
    /// Path to the character device file of a tap interface.
    ///
    /// Required for MacVTap and IPVTap, e.g. /dev/tapX.
    /// Optional for TUN/TAP. [default: /dev/net/tun]
    pub tap: Option<PathBuf>,
    /// Name of a tap interface, e.g. tapX.
    ///
    /// Required for TUN/TAP. Optional for MacVTap and IPVTap.
    #[serde(alias = "if")]
    pub if_name: Option<String>,
    /// System API for asynchronous IO.
    #[serde(default)]
    pub api: WorkerApi,
}

impl DevParam for NetTapParam {
    type Device = Net;

    fn build(self, name: impl Into<Arc<str>>) -> Result<Net> {
        Net::new(self, name)
    }
}

fn new_socket(dev_tap: Option<&Path>, blocking: bool) -> Result<File> {
    let tap_dev = dev_tap.unwrap_or(Path::new("/dev/net/tun"));
    let mut opt = OpenOptions::new();
    opt.read(true).write(true);
    if !blocking {
        opt.custom_flags(O_NONBLOCK);
    }
    let socket = opt.open(tap_dev)?;
    Ok(socket)
}

impl Net {
    pub fn new(param: NetTapParam, name: impl Into<Arc<str>>) -> Result<Self> {
        let mut socket = new_socket(
            param.tap.as_deref(),
            matches!(param.api, WorkerApi::IoUring),
        )?;
        let max_queue_pairs = param.queue_pairs.map(From::from).unwrap_or(1);
        setup_socket(&mut socket, param.if_name.as_deref(), max_queue_pairs > 1)?;
        let mut dev_feat = NetFeature::MAC
            | NetFeature::MTU
            | NetFeature::CSUM
            | NetFeature::HOST_TSO4
            | NetFeature::HOST_TSO6
            | NetFeature::HOST_ECN
            | NetFeature::HOST_UFO
            | NetFeature::HOST_USO
            | NetFeature::CTRL_VQ
            | detect_tap_offload(&socket);
        if max_queue_pairs > 1 {
            dev_feat |= NetFeature::MQ;
        }
        let net = Net {
            name: name.into(),
            config: Arc::new(NetConfig {
                mac: param.mac,
                max_queue_pairs,
                mtu: param.mtu,
                ..Default::default()
            }),
            tap_sockets: vec![socket],
            feature: dev_feat,
            driver_feature: NetFeature::empty(),
            dev_tap: param.tap,
            if_name: param.if_name,
            api: param.api,
        };
        Ok(net)
    }

    fn handle_ctrl_queue(
        &mut self,
        desc: &mut DescChain,
        registry: Option<&Registry>,
    ) -> Result<u32> {
        let Some(header) = desc
            .readable
            .first()
            .and_then(|b| CtrlHdr::read_from_bytes(b).ok())
        else {
            return error::InvalidBuffer.fail();
        };
        let Some(ack_byte) = desc.writable.first_mut().and_then(|v| v.first_mut()) else {
            return error::InvalidBuffer.fail();
        };
        let ack = match header.class {
            CtrlClass::MQ => match CtrlMq(header.command) {
                CtrlMq::VQ_PARIS_SET => {
                    let to_set = |b: &IoSlice| CtrlMqParisSet::read_from_bytes(b).ok();
                    let Some(data) = desc.readable.get(1).and_then(to_set) else {
                        return error::InvalidBuffer.fail();
                    };
                    let pairs = data.virtq_pairs as usize;
                    self.tap_sockets.truncate(pairs);
                    for index in self.tap_sockets.len()..pairs {
                        let mut socket = new_socket(
                            self.dev_tap.as_deref(),
                            matches!(self.api, WorkerApi::IoUring),
                        )?;
                        setup_socket(&mut socket, self.if_name.as_deref(), true)?;
                        enable_tap_offload(&mut socket, self.driver_feature)?;
                        if let Some(r) = registry {
                            r.register(
                                &mut SourceFd(&socket.as_raw_fd()),
                                Token(index),
                                Interest::READABLE | Interest::WRITABLE,
                            )?;
                        }
                        self.tap_sockets.push(socket);
                    }
                    log::info!("{}: using {pairs} pairs of queues", self.name);
                    CtrlAck::OK
                }
                _ => CtrlAck::ERR,
            },
            _ => CtrlAck::ERR,
        };
        *ack_byte = ack.raw();
        Ok(1)
    }
}

impl Virtio for Net {
    type Config = NetConfig;
    type Feature = NetFeature;

    fn id(&self) -> DeviceId {
        DeviceId::Net
    }

    fn name(&self) -> &str {
        &self.name
    }

    fn num_queues(&self) -> u16 {
        let data_queues = self.config.max_queue_pairs << 1;
        if self.feature.contains(NetFeature::CTRL_VQ) {
            data_queues + 1
        } else {
            data_queues
        }
    }

    fn config(&self) -> Arc<NetConfig> {
        self.config.clone()
    }

    fn feature(&self) -> u128 {
        self.feature.bits() | FEATURE_BUILT_IN
    }

    fn spawn_worker<S, E>(
        self,
        event_rx: Receiver<WakeEvent<S, E>>,
        memory: Arc<RamBus>,
        queue_regs: Arc<[QueueReg]>,
    ) -> Result<(JoinHandle<()>, Arc<Notifier>)>
    where
        S: IrqSender,
        E: IoeventFd,
    {
        match self.api {
            WorkerApi::Mio => Mio::spawn_worker(self, event_rx, memory, queue_regs),
            WorkerApi::IoUring => IoUring::spawn_worker(self, event_rx, memory, queue_regs),
        }
    }
}

impl VirtioMio for Net {
    fn reset(&mut self, registry: &Registry) {
        self.tap_sockets.truncate(1);
        let _ = registry.deregister(&mut SourceFd(&self.tap_sockets[0].as_raw_fd()));
    }

    fn activate<'m, Q, S, E>(
        &mut self,
        feature: u128,
        active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        Q: VirtQueue<'m>,
        S: IrqSender,
        E: IoeventFd,
    {
        self.driver_feature = NetFeature::from_bits_retain(feature);
        let socket = &mut self.tap_sockets[0];
        enable_tap_offload(socket, self.driver_feature)?;
        active_mio.poll.registry().register(
            &mut SourceFd(&socket.as_raw_fd()),
            Token(0),
            Interest::READABLE | Interest::WRITABLE,
        )?;
        Ok(())
    }

    fn handle_event<'a, 'm, Q, S, E>(
        &mut self,
        event: &Event,
        active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        Q: VirtQueue<'m>,
        S: IrqSender,
        E: IoeventFd,
    {
        let token = event.token().0;
        let irq_sender = active_mio.irq_sender;
        if event.is_readable() {
            let rx_queue_index = token << 1;
            let Some(Some(queue)) = active_mio.queues.get_mut(rx_queue_index) else {
                log::error!("{}: cannot find rx queue {rx_queue_index}", self.name);
                return Ok(());
            };
            let Some(socket) = self.tap_sockets.get(token) else {
                log::error!("{}: cannot find tap queue {token}", self.name);
                return Ok(());
            };
            queue.handle_desc(rx_queue_index as u16, irq_sender, copy_from_reader(socket))?;
        }
        if event.is_writable() {
            let tx_queue_index = (token << 1) + 1;
            let Some(Some(queue)) = active_mio.queues.get_mut(tx_queue_index) else {
                log::error!("{}: cannot find tx queue {tx_queue_index}", self.name);
                return Ok(());
            };
            let Some(socket) = self.tap_sockets.get(token) else {
                log::error!("{}: cannot find tap queue {token}", self.name);
                return Ok(());
            };
            queue.handle_desc(tx_queue_index as u16, irq_sender, copy_to_writer(socket))?;
        }
        Ok(())
    }

    fn handle_queue<'m, Q, S, E>(
        &mut self,
        index: u16,
        active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        Q: VirtQueue<'m>,
        S: IrqSender,
        E: IoeventFd,
    {
        let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else {
            log::error!("{}: invalid queue index {index}", self.name);
            return Ok(());
        };
        let irq_sender = active_mio.irq_sender;
        let registry = active_mio.poll.registry();
        if index == self.config.max_queue_pairs * 2 {
            return queue.handle_desc(index, irq_sender, |chain| {
                let len = self.handle_ctrl_queue(chain, Some(registry))?;
                Ok(Status::Done { len })
            });
        }
        let Some(socket) = self.tap_sockets.get(index as usize >> 1) else {
            log::error!("{}: invalid tap queue {}", self.name, index >> 1);
            return Ok(());
        };
        if index & 1 == 0 {
            queue.handle_desc(index, irq_sender, copy_from_reader(socket))
        } else {
            queue.handle_desc(index, irq_sender, copy_to_writer(socket))
        }
    }
}

impl VirtioIoUring for Net {
    fn activate<'m, Q, S, E>(
        &mut self,
        feature: u128,
        _ring: &mut ActiveIoUring<'_, '_, 'm, Q, S, E>,
    ) -> Result<()>
    where
        S: IrqSender,
        Q: VirtQueue<'m>,
        E: IoeventFd,
    {
        self.driver_feature = NetFeature::from_bits_retain(feature);
        let socket = &mut self.tap_sockets[0];
        enable_tap_offload(socket, self.driver_feature)?;
        Ok(())
    }

    fn handle_desc(&mut self, q_index: u16, chain: &mut DescChain) -> Result<BufferAction> {
        if q_index == self.config.max_queue_pairs * 2 {
            let len = self.handle_ctrl_queue(chain, None)?;
            return Ok(BufferAction::Written(len));
        }
        let Some(socket) = self.tap_sockets.get(q_index as usize >> 1) else {
            log::error!("{}: invalid tap queue {}", self.name, q_index >> 1);
            return Ok(BufferAction::Written(0));
        };
        let entry = if q_index & 1 == 0 {
            let writable = &chain.writable;
            opcode::Readv::new(
                Fd(socket.as_raw_fd()),
                writable.as_ptr() as *const _,
                writable.len() as _,
            )
            .build()
        } else {
            let readable = &chain.readable;
            opcode::Writev::new(
                Fd(socket.as_raw_fd()),
                readable.as_ptr() as *const _,
                readable.len() as _,
            )
            .build()
        };
        Ok(BufferAction::Sqe(entry))
    }

    fn complete_desc(&mut self, q_index: u16, _chain: &mut DescChain, cqe: &Cqe) -> Result<u32> {
        let ret = cqe.result();
        if ret < 0 {
            let err = std::io::Error::from_raw_os_error(-ret);
            log::error!("{}: failed to send/receive packet: {err}", self.name,);
            return Ok(0);
        }
        if q_index & 1 == 0 {
            Ok(ret as u32)
        } else {
            Ok(0)
        }
    }
}

fn setup_socket(file: &mut File, if_name: Option<&str>, mq: bool) -> Result<()> {
    let mut tap_ifconfig = unsafe { MaybeUninit::<libc::ifreq>::zeroed().assume_init() };

    if let Some(name) = if_name {
        let name_len = std::cmp::min(tap_ifconfig.ifr_name.len() - 1, name.len());
        tap_ifconfig.ifr_name.as_mut_bytes()[0..name_len]
            .copy_from_slice(&name.as_bytes()[0..name_len]);
    }

    let mut flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR;
    if mq {
        flags |= IFF_MULTI_QUEUE;
    }
    tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;

    unsafe { tun_set_iff(file, &tap_ifconfig) }.or_else(|e| {
        if e.kind() == ErrorKind::InvalidInput && !mq {
            flags |= IFF_MULTI_QUEUE;
            tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;
            unsafe { tun_set_iff(file, &tap_ifconfig) }
        } else {
            Err(e)
        }
    })?;

    unsafe { tun_set_vnet_hdr_sz(file, &(size_of::<VirtioNetHdr>() as _)) }?;
    Ok(())
}

fn detect_tap_offload(tap: &impl AsFd) -> NetFeature {
    let mut tap_feature = TunFeature::all();
    let mut dev_feat = NetFeature::GUEST_CSUM
        | NetFeature::GUEST_TSO4
        | NetFeature::GUEST_TSO6
        | NetFeature::GUEST_ECN
        | NetFeature::GUEST_UFO
        | NetFeature::GUEST_USO4
        | NetFeature::GUEST_USO6;
    if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
        return dev_feat;
    }
    tap_feature &= !(TunFeature::USO4 | TunFeature::USO6);
    dev_feat &= !(NetFeature::GUEST_USO4 | NetFeature::GUEST_USO6);
    if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
        return dev_feat;
    }
    tap_feature &= !(TunFeature::UFO);
    dev_feat &= !NetFeature::GUEST_UFO;
    if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
        return dev_feat;
    }
    NetFeature::empty()
}

fn enable_tap_offload(tap: &mut File, feature: NetFeature) -> Result<()> {
    let mut tap_feature = TunFeature::empty();
    if feature.contains(NetFeature::GUEST_CSUM) {
        tap_feature |= TunFeature::CSUM;
    }
    if feature.contains(NetFeature::GUEST_TSO4) {
        tap_feature |= TunFeature::TSO4;
    }
    if feature.contains(NetFeature::GUEST_TSO6) {
        tap_feature |= TunFeature::TSO6;
    }
    if feature.contains(NetFeature::GUEST_ECN) {
        tap_feature |= TunFeature::TSO_ECN;
    }
    if feature.contains(NetFeature::GUEST_UFO) {
        tap_feature |= TunFeature::UFO;
    }
    if feature.contains(NetFeature::GUEST_USO4) {
        tap_feature |= TunFeature::USO4;
    }
    if feature.contains(NetFeature::GUEST_USO6) {
        tap_feature |= TunFeature::USO6;
    }
    unsafe { tun_set_offload(tap, tap_feature.bits()) }?;
    Ok(())
}

1	// Copyright 2024 Google LLC
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// https://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	use std::fmt::Debug;
16	use std::fs::{File, OpenOptions};
17	use std::io::{ErrorKind, IoSlice};
18	use std::mem::MaybeUninit;
19	use std::num::NonZeroU16;
20	use std::os::fd::{AsFd, AsRawFd};
21	use std::os::unix::prelude::OpenOptionsExt;
22	use std::path::{Path, PathBuf};
23	use std::sync::Arc;
24	use std::sync::mpsc::Receiver;
25	use std::thread::JoinHandle;
26
27	use io_uring::cqueue::Entry as Cqe;
28	use io_uring::opcode;
29	use io_uring::types::Fd;
30	use libc::{IFF_MULTI_QUEUE, IFF_NO_PI, IFF_TAP, IFF_VNET_HDR, O_NONBLOCK};
31	use mio::event::Event;
32	use mio::unix::SourceFd;
33	use mio::{Interest, Registry, Token};
34	use serde::Deserialize;
35	use serde_aco::Help;
36	use zerocopy::{FromBytes, IntoBytes};
37
38	use crate::hv::IoeventFd;
39	use crate::mem::mapped::RamBus;
40	use crate::sync::notifier::Notifier;
41	use crate::sys::if_tun::{TunFeature, tun_set_iff, tun_set_offload, tun_set_vnet_hdr_sz};
42	use crate::virtio::dev::net::mac_addr::MacAddr;
43	use crate::virtio::dev::net::{
44	CtrlAck, CtrlClass, CtrlHdr, CtrlMq, CtrlMqParisSet, NetConfig, NetFeature, VirtioNetHdr,
45	};
46	use crate::virtio::dev::{DevParam, DeviceId, Result, Virtio, WakeEvent};
47	use crate::virtio::queue::{
48	DescChain, QueueReg, Status, VirtQueue, copy_from_reader, copy_to_writer,
49	};
50	use crate::virtio::worker::WorkerApi;
51	use crate::virtio::worker::io_uring::{ActiveIoUring, BufferAction, IoUring, VirtioIoUring};
52	use crate::virtio::worker::mio::{ActiveMio, Mio, VirtioMio};
53	use crate::virtio::{FEATURE_BUILT_IN, IrqSender, error};
54
55	#[derive(Debug)]
56	pub struct Net {
57	name: Arc<str>,
58	config: Arc<NetConfig>,
59	tap_sockets: Vec<File>,
60	feature: NetFeature,
61	driver_feature: NetFeature,
62	dev_tap: Option<PathBuf>,
63	if_name: Option<String>,
64	api: WorkerApi,
65	}
66
67	#[derive(Debug, Deserialize, Clone, Help)]
68	pub struct NetTapParam {
69	/// MAC address of the virtual NIC, e.g. 06:3a:76:53:da:3d.
70	pub mac: MacAddr,
71	/// Maximum transmission unit.
72	pub mtu: u16,
73	/// Number of pairs of transmit/receive queues. [default: 1]
74	#[serde(alias = "qp")]
75	pub queue_pairs: Option<NonZeroU16>,
76	/// Path to the character device file of a tap interface.
77	///
78	/// Required for MacVTap and IPVTap, e.g. /dev/tapX.
79	/// Optional for TUN/TAP. [default: /dev/net/tun]
80	pub tap: Option<PathBuf>,
81	/// Name of a tap interface, e.g. tapX.
82	///
83	/// Required for TUN/TAP. Optional for MacVTap and IPVTap.
84	#[serde(alias = "if")]
85	pub if_name: Option<String>,
86	/// System API for asynchronous IO.
87	#[serde(default)]
88	pub api: WorkerApi,
89	}
90
91	impl DevParam for NetTapParam {
92	type Device = Net;
93
94	fn build(self, name: impl Into<Arc<str>>) -> Result<Net> {	×
95	Net::new(self, name)	×
96	}
97	}
98
UNCOV 99	fn new_socket(dev_tap: Option<&Path>, blocking: bool) -> Result<File> {
UNCOV 100	let tap_dev = dev_tap.unwrap_or(Path::new("/dev/net/tun"));
UNCOV 101	let mut opt = OpenOptions::new();
UNCOV 102	opt.read(true).write(true);
UNCOV 103	if !blocking {
UNCOV 104	opt.custom_flags(O_NONBLOCK);
105	}
UNCOV 106	let socket = opt.open(tap_dev)?;
UNCOV 107	Ok(socket)
108	}
109
110	impl Net {
111	pub fn new(param: NetTapParam, name: impl Into<Arc<str>>) -> Result<Self> {	×
112	let mut socket = new_socket(
113	param.tap.as_deref(),	×
114	matches!(param.api, WorkerApi::IoUring),	×
115	)?;
116	let max_queue_pairs = param.queue_pairs.map(From::from).unwrap_or(1);	×
117	setup_socket(&mut socket, param.if_name.as_deref(), max_queue_pairs > 1)?;	×
118	let mut dev_feat = NetFeature::MAC	×
119	\| NetFeature::MTU	×
120	\| NetFeature::CSUM	×
121	\| NetFeature::HOST_TSO4	×
122	\| NetFeature::HOST_TSO6	×
123	\| NetFeature::HOST_ECN	×
124	\| NetFeature::HOST_UFO	×
125	\| NetFeature::HOST_USO	×
126	\| NetFeature::CTRL_VQ	×
127	\| detect_tap_offload(&socket);	×
128	if max_queue_pairs > 1 {	×
129	dev_feat \|= NetFeature::MQ;	×
130	}
131	let net = Net {
132	name: name.into(),	×
133	config: Arc::new(NetConfig {	×
134	mac: param.mac,
135	max_queue_pairs,
136	mtu: param.mtu,
137	..Default::default()
138	}),
139	tap_sockets: vec![socket],	×
140	feature: dev_feat,
141	driver_feature: NetFeature::empty(),	×
142	dev_tap: param.tap,	×
143	if_name: param.if_name,	×
144	api: param.api,	×
145	};
146	Ok(net)	×
147	}
148
UNCOV 149	fn handle_ctrl_queue(
150	&mut self,
151	desc: &mut DescChain,
152	registry: Option<&Registry>,
153	) -> Result<u32> {
UNCOV 154	let Some(header) = desc
UNCOV 155	.readable
156	.first()
UNCOV 157	.and_then(\|b\| CtrlHdr::read_from_bytes(b).ok())
158	else {
UNCOV 159	return error::InvalidBuffer.fail();
160	};
UNCOV 161	let Some(ack_byte) = desc.writable.first_mut().and_then(\|v\| v.first_mut()) else {
UNCOV 162	return error::InvalidBuffer.fail();
163	};
UNCOV 164	let ack = match header.class {
UNCOV 165	CtrlClass::MQ => match CtrlMq(header.command) {
166	CtrlMq::VQ_PARIS_SET => {
UNCOV 167	let to_set = \|b: &IoSlice\| CtrlMqParisSet::read_from_bytes(b).ok();
UNCOV 168	let Some(data) = desc.readable.get(1).and_then(to_set) else {
UNCOV 169	return error::InvalidBuffer.fail();
170	};
UNCOV 171	let pairs = data.virtq_pairs as usize;
UNCOV 172	self.tap_sockets.truncate(pairs);
UNCOV 173	for index in self.tap_sockets.len()..pairs {
174	let mut socket = new_socket(
UNCOV 175	self.dev_tap.as_deref(),
UNCOV 176	matches!(self.api, WorkerApi::IoUring),
177	)?;
UNCOV 178	setup_socket(&mut socket, self.if_name.as_deref(), true)?;
UNCOV 179	enable_tap_offload(&mut socket, self.driver_feature)?;
UNCOV 180	if let Some(r) = registry {
UNCOV 181	r.register(
UNCOV 182	&mut SourceFd(&socket.as_raw_fd()),
UNCOV 183	Token(index),
UNCOV 184	Interest::READABLE \| Interest::WRITABLE,
185	)?;
186	}
UNCOV 187	self.tap_sockets.push(socket);
188	}
UNCOV 189	log::info!("{}: using {pairs} pairs of queues", self.name);
UNCOV 190	CtrlAck::OK
191	}
UNCOV 192	_ => CtrlAck::ERR,
193	},
UNCOV 194	_ => CtrlAck::ERR,
195	};
UNCOV 196	*ack_byte = ack.raw();
UNCOV 197	Ok(1)
198	}
199	}
200
201	impl Virtio for Net {
202	type Config = NetConfig;
203	type Feature = NetFeature;
204
UNCOV 205	fn id(&self) -> DeviceId {
UNCOV 206	DeviceId::Net
207	}
208
UNCOV 209	fn name(&self) -> &str {
UNCOV 210	&self.name
211	}
212
UNCOV 213	fn num_queues(&self) -> u16 {
UNCOV 214	let data_queues = self.config.max_queue_pairs << 1;
UNCOV 215	if self.feature.contains(NetFeature::CTRL_VQ) {
UNCOV 216	data_queues + 1
217	} else {
UNCOV 218	data_queues
219	}
220	}
221
UNCOV 222	fn config(&self) -> Arc<NetConfig> {
UNCOV 223	self.config.clone()
224	}
225
UNCOV 226	fn feature(&self) -> u128 {
UNCOV 227	self.feature.bits() \| FEATURE_BUILT_IN
228	}
229
UNCOV 230	fn spawn_worker<S, E>(
231	self,
232	event_rx: Receiver<WakeEvent<S, E>>,
233	memory: Arc<RamBus>,
234	queue_regs: Arc<[QueueReg]>,
235	) -> Result<(JoinHandle<()>, Arc<Notifier>)>
236	where
237	S: IrqSender,
238	E: IoeventFd,
239	{
240	match self.api {	×
241	WorkerApi::Mio => Mio::spawn_worker(self, event_rx, memory, queue_regs),	×
242	WorkerApi::IoUring => IoUring::spawn_worker(self, event_rx, memory, queue_regs),	×
243	}
244	}
245	}
246
247	impl VirtioMio for Net {
UNCOV 248	fn reset(&mut self, registry: &Registry) {
UNCOV 249	self.tap_sockets.truncate(1);
UNCOV 250	let _ = registry.deregister(&mut SourceFd(&self.tap_sockets[0].as_raw_fd()));
251	}
252
UNCOV 253	fn activate<'m, Q, S, E>(
254	&mut self,
255	feature: u128,
256	active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
257	) -> Result<()>
258	where
259	Q: VirtQueue<'m>,
260	S: IrqSender,
261	E: IoeventFd,
262	{
263	self.driver_feature = NetFeature::from_bits_retain(feature);	×
264	let socket = &mut self.tap_sockets[0];	×
265	enable_tap_offload(socket, self.driver_feature)?;	×
266	active_mio.poll.registry().register(	×
267	&mut SourceFd(&socket.as_raw_fd()),	×
268	Token(0),	×
269	Interest::READABLE \| Interest::WRITABLE,	×
270	)?;
271	Ok(())	×
272	}
273
UNCOV 274	fn handle_event<'a, 'm, Q, S, E>(
275	&mut self,
276	event: &Event,
277	active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
278	) -> Result<()>
279	where
280	Q: VirtQueue<'m>,
281	S: IrqSender,
282	E: IoeventFd,
283	{
284	let token = event.token().0;	×
285	let irq_sender = active_mio.irq_sender;	×
286	if event.is_readable() {	×
287	let rx_queue_index = token << 1;	×
288	let Some(Some(queue)) = active_mio.queues.get_mut(rx_queue_index) else {	×
289	log::error!("{}: cannot find rx queue {rx_queue_index}", self.name);	×
290	return Ok(());	×
291	};
292	let Some(socket) = self.tap_sockets.get(token) else {	×
293	log::error!("{}: cannot find tap queue {token}", self.name);	×
294	return Ok(());	×
295	};
296	queue.handle_desc(rx_queue_index as u16, irq_sender, copy_from_reader(socket))?;	×
297	}
298	if event.is_writable() {	×
299	let tx_queue_index = (token << 1) + 1;	×
300	let Some(Some(queue)) = active_mio.queues.get_mut(tx_queue_index) else {	×
301	log::error!("{}: cannot find tx queue {tx_queue_index}", self.name);	×
302	return Ok(());	×
303	};
304	let Some(socket) = self.tap_sockets.get(token) else {	×
305	log::error!("{}: cannot find tap queue {token}", self.name);	×
306	return Ok(());	×
307	};
308	queue.handle_desc(tx_queue_index as u16, irq_sender, copy_to_writer(socket))?;	×
309	}
310	Ok(())	×
311	}
312
UNCOV 313	fn handle_queue<'m, Q, S, E>(
314	&mut self,
315	index: u16,
316	active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>,
317	) -> Result<()>
318	where
319	Q: VirtQueue<'m>,
320	S: IrqSender,
321	E: IoeventFd,
322	{
323	let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else {	×
324	log::error!("{}: invalid queue index {index}", self.name);	×
325	return Ok(());	×
326	};
327	let irq_sender = active_mio.irq_sender;	×
328	let registry = active_mio.poll.registry();	×
329	if index == self.config.max_queue_pairs * 2 {	×
330	return queue.handle_desc(index, irq_sender, \|chain\| {	×
331	let len = self.handle_ctrl_queue(chain, Some(registry))?;	×
332	Ok(Status::Done { len })	×
333	});
334	}
335	let Some(socket) = self.tap_sockets.get(index as usize >> 1) else {	×
336	log::error!("{}: invalid tap queue {}", self.name, index >> 1);	×
337	return Ok(());	×
338	};
339	if index & 1 == 0 {	×
340	queue.handle_desc(index, irq_sender, copy_from_reader(socket))	×
341	} else {
342	queue.handle_desc(index, irq_sender, copy_to_writer(socket))	×
343	}
344	}
345	}
346
347	impl VirtioIoUring for Net {
UNCOV 348	fn activate<'m, Q, S, E>(
349	&mut self,
350	feature: u128,
351	_ring: &mut ActiveIoUring<'_, '_, 'm, Q, S, E>,
352	) -> Result<()>
353	where
354	S: IrqSender,
355	Q: VirtQueue<'m>,
356	E: IoeventFd,
357	{
358	self.driver_feature = NetFeature::from_bits_retain(feature);	×
359	let socket = &mut self.tap_sockets[0];	×
360	enable_tap_offload(socket, self.driver_feature)?;	×
361	Ok(())	×
362	}
363
UNCOV 364	fn handle_desc(&mut self, q_index: u16, chain: &mut DescChain) -> Result<BufferAction> {
UNCOV 365	if q_index == self.config.max_queue_pairs * 2 {
UNCOV 366	let len = self.handle_ctrl_queue(chain, None)?;
UNCOV 367	return Ok(BufferAction::Written(len));
368	}
UNCOV 369	let Some(socket) = self.tap_sockets.get(q_index as usize >> 1) else {
UNCOV 370	log::error!("{}: invalid tap queue {}", self.name, q_index >> 1);
UNCOV 371	return Ok(BufferAction::Written(0));
372	};
UNCOV 373	let entry = if q_index & 1 == 0 {
UNCOV 374	let writable = &chain.writable;
375	opcode::Readv::new(
UNCOV 376	Fd(socket.as_raw_fd()),
UNCOV 377	writable.as_ptr() as *const _,
UNCOV 378	writable.len() as _,
379	)
380	.build()
381	} else {
UNCOV 382	let readable = &chain.readable;
383	opcode::Writev::new(
UNCOV 384	Fd(socket.as_raw_fd()),
UNCOV 385	readable.as_ptr() as *const _,
UNCOV 386	readable.len() as _,
387	)
388	.build()
389	};
UNCOV 390	Ok(BufferAction::Sqe(entry))
391	}
392
UNCOV 393	fn complete_desc(&mut self, q_index: u16, _chain: &mut DescChain, cqe: &Cqe) -> Result<u32> {
UNCOV 394	let ret = cqe.result();
UNCOV 395	if ret < 0 {
UNCOV 396	let err = std::io::Error::from_raw_os_error(-ret);
UNCOV 397	log::error!("{}: failed to send/receive packet: {err}", self.name,);
UNCOV 398	return Ok(0);
399	}
UNCOV 400	if q_index & 1 == 0 {
UNCOV 401	Ok(ret as u32)
402	} else {
UNCOV 403	Ok(0)
404	}
405	}
406	}
407
UNCOV 408	fn setup_socket(file: &mut File, if_name: Option<&str>, mq: bool) -> Result<()> {
UNCOV 409	let mut tap_ifconfig = unsafe { MaybeUninit::<libc::ifreq>::zeroed().assume_init() };
410
UNCOV 411	if let Some(name) = if_name {
UNCOV 412	let name_len = std::cmp::min(tap_ifconfig.ifr_name.len() - 1, name.len());
UNCOV 413	tap_ifconfig.ifr_name.as_mut_bytes()[0..name_len]
UNCOV 414	.copy_from_slice(&name.as_bytes()[0..name_len]);
415	}
416
UNCOV 417	let mut flags = IFF_TAP \| IFF_NO_PI \| IFF_VNET_HDR;
UNCOV 418	if mq {
UNCOV 419	flags \|= IFF_MULTI_QUEUE;
420	}
UNCOV 421	tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;
422
UNCOV 423	unsafe { tun_set_iff(file, &tap_ifconfig) }.or_else(\|e\| {
UNCOV 424	if e.kind() == ErrorKind::InvalidInput && !mq {
UNCOV 425	flags \|= IFF_MULTI_QUEUE;
UNCOV 426	tap_ifconfig.ifr_ifru.ifru_flags = flags as i16;
UNCOV 427	unsafe { tun_set_iff(file, &tap_ifconfig) }
428	} else {
UNCOV 429	Err(e)
430	}
431	})?;
432
UNCOV 433	unsafe { tun_set_vnet_hdr_sz(file, &(size_of::<VirtioNetHdr>() as _)) }?;
UNCOV 434	Ok(())
435	}
436
UNCOV 437	fn detect_tap_offload(tap: &impl AsFd) -> NetFeature {
UNCOV 438	let mut tap_feature = TunFeature::all();
UNCOV 439	let mut dev_feat = NetFeature::GUEST_CSUM
UNCOV 440	\| NetFeature::GUEST_TSO4
UNCOV 441	\| NetFeature::GUEST_TSO6
UNCOV 442	\| NetFeature::GUEST_ECN
UNCOV 443	\| NetFeature::GUEST_UFO
UNCOV 444	\| NetFeature::GUEST_USO4
UNCOV 445	\| NetFeature::GUEST_USO6;
UNCOV 446	if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
UNCOV 447	return dev_feat;
448	}
UNCOV 449	tap_feature &= !(TunFeature::USO4 \| TunFeature::USO6);
UNCOV 450	dev_feat &= !(NetFeature::GUEST_USO4 \| NetFeature::GUEST_USO6);
UNCOV 451	if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
UNCOV 452	return dev_feat;
453	}
UNCOV 454	tap_feature &= !(TunFeature::UFO);
UNCOV 455	dev_feat &= !NetFeature::GUEST_UFO;
UNCOV 456	if unsafe { tun_set_offload(tap, tap_feature.bits()) }.is_ok() {
UNCOV 457	return dev_feat;
458	}
UNCOV 459	NetFeature::empty()
460	}
461
UNCOV 462	fn enable_tap_offload(tap: &mut File, feature: NetFeature) -> Result<()> {
UNCOV 463	let mut tap_feature = TunFeature::empty();
UNCOV 464	if feature.contains(NetFeature::GUEST_CSUM) {
UNCOV 465	tap_feature \|= TunFeature::CSUM;
466	}
UNCOV 467	if feature.contains(NetFeature::GUEST_TSO4) {
UNCOV 468	tap_feature \|= TunFeature::TSO4;
469	}
UNCOV 470	if feature.contains(NetFeature::GUEST_TSO6) {
UNCOV 471	tap_feature \|= TunFeature::TSO6;
472	}
UNCOV 473	if feature.contains(NetFeature::GUEST_ECN) {
UNCOV 474	tap_feature \|= TunFeature::TSO_ECN;
475	}
UNCOV 476	if feature.contains(NetFeature::GUEST_UFO) {
UNCOV 477	tap_feature \|= TunFeature::UFO;
478	}
UNCOV 479	if feature.contains(NetFeature::GUEST_USO4) {
UNCOV 480	tap_feature \|= TunFeature::USO4;
481	}
UNCOV 482	if feature.contains(NetFeature::GUEST_USO6) {
UNCOV 483	tap_feature \|= TunFeature::USO6;
484	}
UNCOV 485	unsafe { tun_set_offload(tap, tap_feature.bits()) }?;
UNCOV 486	Ok(())
487	}

google / alioth / 18638570188

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous