17385983897

Committed 01 Sep 2025 07:41PM UTC coverage: 18.009% (-0.1%) from 18.149%

Build # 17385983897

Build Type

Pull #281

github

Committed by

web-flow

Commit Message

Merge 782c57b11 into 6ec9a6d6b

Pull Request Pull Request #281: Port to Apple Hypervisor framework

Run Details

0 of 154 new or added lines in 11 files covered. (0.0%)

1166 existing lines in 29 files now uncovered.

1362 of 7563 relevant lines covered (18.01%)

18.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/alioth/src/mem/mem.rs

// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

pub mod addressable;
pub mod emulated;
pub mod mapped;

use std::any::type_name;
use std::fmt::Debug;
use std::sync::Arc;

use parking_lot::{Mutex, RwLock};
use serde::Deserialize;
use serde_aco::Help;
use snafu::Snafu;

use crate::errors::{DebugTrace, trace_error};
use crate::hv::{MemMapOption, VmEntry, VmMemory};

use self::addressable::{Addressable, SlotBackend};
use self::emulated::{Action, MmioBus, MmioRange};
use self::mapped::{ArcMemPages, Ram, RamBus};

#[trace_error]
#[derive(Snafu, DebugTrace)]
#[snafu(module, visibility(pub), context(suffix(false)))]
pub enum Error {
    #[snafu(display("Hypervisor internal error"), context(false))]
    HvError { source: Box<crate::hv::Error> },
    #[snafu(display("Cannot add a zero-sized slot"))]
    ZeroSizedSlot,
    #[snafu(display("(addr={addr:#x}, size={size:#x}) exceeds the address limit"))]
    ExceedsLimit { addr: u64, size: u64 },
    #[snafu(display("{new_item:#x?} overlaps with {exist_item:#x?}"))]
    Overlap {
        new_item: [u64; 2],
        exist_item: [u64; 2],
    },
    #[snafu(display("{addr:#x} is not mapped"))]
    NotMapped { addr: u64 },
    #[snafu(display("Sum of backend range sizes {sum:#x} exceeds the region total size"))]
    BackendTooBig { sum: u64, size: u64 },
    #[snafu(display("address {addr:#x} is not {align}-byte aligned"))]
    NotAligned { addr: u64, align: usize },
    #[snafu(display(
        "Guest address {addr:#x} (size = {:#x}) is not backed by continuous host memory"
    ))]
    NotContinuous { addr: u64, size: u64 },
    #[snafu(display("Error from OS"), context(false))]
    System { error: std::io::Error },
    #[snafu(display("Failed to write data to destination"))]
    Write { error: std::io::Error },
    #[snafu(display("Failed to read data from source"))]
    Read { error: std::io::Error },
    #[snafu(display("Failed to do MMIO"))]
    Mmio {
        source: Box<dyn DebugTrace + Send + Sync + 'static>,
    },
    #[snafu(display("Failed to change memory layout"))]
    ChangeLayout {
        source: Box<dyn DebugTrace + Send + Sync + 'static>,
    },
}

pub type Result<T, E = Error> = std::result::Result<T, E>;

fn default_memory_size() -> u64 {
    1 << 30
}

#[derive(Debug, Deserialize, Default, Help)]
pub struct MemConfig {
    /// Total guest memory size in bytes. [default: 1G]
    #[serde(default = "default_memory_size")]
    pub size: u64,
    /// Host backend [default: anon]
    #[serde(default)]
    pub backend: MemBackend,
    /// mmap() guest memory with MAP_SHARED or MAP_PRIVATE.
    /// [default: false]
    #[serde(default)]
    pub shared: bool,
    /// Enable transparent hugepage. [default: false]
    #[cfg(target_os = "linux")]
    #[serde(default, alias = "thp")]
    pub transparent_hugepage: bool,
}

#[derive(Debug, Deserialize, Default, Help)]
pub enum MemBackend {
    /// Anonymous memory by MAP_ANONYMOUS.
    #[default]
    #[serde(alias = "anon")]
    Anonymous,
    /// Anonymous file by memfd_create(). Always uses MAP_SHARED.
    #[cfg(target_os = "linux")]
    #[serde(alias = "memfd")]
    Memfd,
}

impl MemConfig {
    pub fn has_shared_fd(&self) -> bool {
        match &self.backend {
            #[cfg(target_os = "linux")]
            MemBackend::Memfd => true,
            MemBackend::Anonymous => false,
        }
    }
}

#[derive(Debug)]
pub enum MemRange {
    Ram(ArcMemPages),
    DevMem(ArcMemPages),
    Emulated(MmioRange),
    Span(u64),
}

impl MemRange {
    pub fn size(&self) -> u64 {
        match self {
            MemRange::Ram(pages) | MemRange::DevMem(pages) => pages.size(),
            MemRange::Emulated(range) => range.size(),
            MemRange::Span(size) => *size,
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MemRegionType {
    Hidden,
    Ram,
    Reserved,
    Acpi,
    Pmem,
}

#[derive(Debug, Clone, Copy)]
pub struct MemRegionEntry {
    pub size: u64,
    pub type_: MemRegionType,
}

pub trait MemRegionCallback: Debug + Send + Sync + 'static {
    fn mapped(&self, addr: u64) -> Result<()>;
    fn unmapped(&self) -> Result<()> {
        log::debug!("{} unmapped", type_name::<Self>());
        Ok(())
    }
}

#[derive(Debug)]
pub struct MemRegion {
    pub ranges: Vec<MemRange>,
    pub entries: Vec<MemRegionEntry>,
    pub callbacks: Mutex<Vec<Box<dyn MemRegionCallback>>>,
}

impl MemRegion {
    pub fn size(&self) -> u64 {
        self.entries.iter().fold(0, |accu, e| accu + e.size)
    }

    pub fn with_ram(pages: ArcMemPages, type_: MemRegionType) -> MemRegion {
        let size = pages.size();
        MemRegion {
            ranges: vec![MemRange::Ram(pages)],
            entries: vec![MemRegionEntry { type_, size }],
            callbacks: Mutex::new(vec![]),
        }
    }

    pub fn with_dev_mem(pages: ArcMemPages, type_: MemRegionType) -> MemRegion {
        let size = pages.size();
        MemRegion {
            ranges: vec![MemRange::DevMem(pages)],
            entries: vec![MemRegionEntry { type_, size }],
            callbacks: Mutex::new(vec![]),
        }
    }

    pub fn with_emulated(range: MmioRange, type_: MemRegionType) -> MemRegion {
        let size = range.size();
        MemRegion {
            ranges: vec![MemRange::Emulated(range)],
            entries: vec![MemRegionEntry { type_, size }],
            callbacks: Mutex::new(vec![]),
        }
    }

    pub fn validate(&self) -> Result<()> {
        let entries_size = self.size();
        let ranges_size = self.ranges.iter().fold(0, |accu, r| accu + r.size());
        if ranges_size > entries_size {
            return error::BackendTooBig {
                sum: ranges_size,
                size: entries_size,
            }
            .fail();
        }
        Ok(())
    }
}

impl SlotBackend for Arc<MemRegion> {
    fn size(&self) -> u64 {
        MemRegion::size(self.as_ref())
    }
}

#[derive(Debug)]
pub struct IoRegion {
    pub range: MmioRange,
    pub callbacks: Mutex<Vec<Box<dyn MemRegionCallback>>>,
}

impl IoRegion {
    pub fn new(range: MmioRange) -> IoRegion {
        IoRegion {
            range,
            callbacks: Mutex::new(vec![]),
        }
    }
}

impl SlotBackend for Arc<IoRegion> {
    fn size(&self) -> u64 {
        self.range.size()
    }
}

pub trait LayoutChanged: Debug + Send + Sync + 'static {
    fn ram_added(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>;
    fn ram_removed(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>;
}

pub trait LayoutUpdated: Debug + Send + Sync + 'static {
    fn ram_updated(&self, ram: &Ram) -> Result<()>;
}

#[derive(Debug, Default)]
struct LayoutCallbacks {
    changed: Vec<Box<dyn LayoutChanged>>,
    updated: Vec<Box<dyn LayoutUpdated>>,
}

// lock order: region -> callbacks -> bus
#[derive(Debug)]
pub struct Memory {
    regions: Mutex<Addressable<Arc<MemRegion>>>,
    callbacks: Mutex<LayoutCallbacks>,
    ram_bus: Arc<RamBus>,
    mmio_bus: RwLock<MmioBus>,
    vm_memory: Box<dyn VmMemory>,

    io_bus: RwLock<MmioBus>,
    io_regions: Mutex<Addressable<Arc<IoRegion>>>,
}

impl Memory {
    pub fn new(vm_memory: impl VmMemory) -> Self {
        Memory {
            regions: Mutex::new(Addressable::new()),
            callbacks: Mutex::new(LayoutCallbacks::default()),
            ram_bus: Arc::new(RamBus::new()),
            mmio_bus: RwLock::new(MmioBus::new()),
            vm_memory: Box::new(vm_memory),
            io_bus: RwLock::new(MmioBus::new()),
            io_regions: Mutex::new(Addressable::new()),
        }
    }

    pub fn register_change_callback(&self, callback: Box<dyn LayoutChanged>) -> Result<()> {
        let regions = self.regions.lock();
        for (addr, region) in regions.iter() {
            let mut offset = 0;
            for range in &region.ranges {
                let gpa = addr + offset;
                match range {
                    MemRange::Ram(r) => callback.ram_added(gpa, r)?,
                    MemRange::Span(_) | MemRange::Emulated(_) | MemRange::DevMem(_) => {}
                }
                offset += range.size();
            }
        }
        let mut callbacks = self.callbacks.lock();
        callbacks.changed.push(callback);
        Ok(())
    }

    pub fn register_update_callback(&self, callback: Box<dyn LayoutUpdated>) -> Result<()> {
        let _regions = self.regions.lock();
        let mut callbacks = self.callbacks.lock();
        let ram = self.ram_bus.lock_layout();
        callback.ram_updated(&ram)?;
        callbacks.updated.push(callback);
        Ok(())
    }

    pub fn reset(&self) -> Result<()> {
        self.clear()?;
        self.vm_memory.reset()?;
        Ok(())
    }

    pub fn ram_bus(&self) -> Arc<RamBus> {
        self.ram_bus.clone()
    }

    fn map_to_vm(&self, gpa: u64, user_mem: &ArcMemPages) -> Result<(), Error> {
        let mem_options = MemMapOption {
            read: true,
            write: true,
            exec: true,
            log_dirty: false,
        };
        self.vm_memory
            .mem_map(gpa, user_mem.size(), user_mem.addr(), mem_options)?;
        Ok(())
    }

    fn unmap_from_vm(&self, gpa: u64, user_mem: &ArcMemPages) -> Result<(), Error> {
        self.vm_memory.unmap(gpa, user_mem.size())?;
        Ok(())
    }

    pub fn add_region(&self, addr: u64, region: Arc<MemRegion>) -> Result<()> {
        region.validate()?;
        let mut regions = self.regions.lock();
        regions.add(addr, region.clone())?;
        let mut offset = 0;
        let callbacks = self.callbacks.lock();
        let mut ram_updated = false;
        for range in &region.ranges {
            let gpa = addr + offset;
            match range {
                MemRange::Emulated(r) => {
                    let mut mmio_bus = self.mmio_bus.write();
                    mmio_bus.add(gpa, r.clone())?
                }
                MemRange::Ram(r) => {
                    self.map_to_vm(gpa, r)?;
                    for callback in &callbacks.changed {
                        callback.ram_added(gpa, r)?;
                    }
                    self.ram_bus.add(gpa, r.clone())?;
                    ram_updated = true;
                }
                MemRange::DevMem(r) => self.map_to_vm(gpa, r)?,
                MemRange::Span(_) => {}
            }
            offset += range.size();
        }
        if ram_updated {
            let ram = self.ram_bus.lock_layout();
            for update_callback in &callbacks.updated {
                update_callback.ram_updated(&ram)?;
            }
        }
        let region_callbacks = region.callbacks.lock();
        for callback in region_callbacks.iter() {
            callback.mapped(addr)?;
        }
        Ok(())
    }

    fn unmap_region(&self, addr: u64, region: &MemRegion) -> Result<()> {
        let mut offset = 0;
        let callbacks = self.callbacks.lock();
        let mut ram_updated = false;
        for range in &region.ranges {
            let gpa = addr + offset;
            match range {
                MemRange::Emulated(_) => {
                    let mut mmio_bus = self.mmio_bus.write();
                    mmio_bus.remove(gpa)?;
                }
                MemRange::Ram(r) => {
                    self.ram_bus.remove(gpa)?;
                    for callback in &callbacks.changed {
                        callback.ram_removed(gpa, r)?;
                    }
                    self.unmap_from_vm(gpa, r)?;
                    ram_updated = true;
                }
                MemRange::DevMem(r) => self.unmap_from_vm(gpa, r)?,
                MemRange::Span(_) => {}
            };
            offset += range.size();
        }
        if ram_updated {
            let ram = self.ram_bus.lock_layout();
            for callback in &callbacks.updated {
                callback.ram_updated(&ram)?;
            }
        }
        let region_callbacks = region.callbacks.lock();
        for callback in region_callbacks.iter() {
            callback.unmapped()?;
        }
        Ok(())
    }

    pub fn remove_region(&self, addr: u64) -> Result<Arc<MemRegion>> {
        let mut regions = self.regions.lock();
        let region = regions.remove(addr)?;
        self.unmap_region(addr, &region)?;
        Ok(region)
    }

    // TODO can be optimized
    fn clear(&self) -> Result<()> {
        let mut regions = self.regions.lock();
        let regions = regions.drain(..);
        for (addr, region) in regions {
            self.unmap_region(addr, &region)?;
        }
        #[cfg(target_arch = "x86_64")]
        {
            let mut io_regions = self.io_regions.lock();
            let io_regions = io_regions.drain(..);
            for (port, io_region) in io_regions {
                self.unmap_io_region(port as u16, &io_region)?;
            }
        }
        Ok(())
    }

    pub fn mem_region_entries(&self) -> Vec<(u64, MemRegionEntry)> {
        let mut entries = vec![];
        let regions = self.regions.lock();
        for (start, region) in regions.iter() {
            let mut offset = 0;
            for entry in region.entries.iter() {
                entries.push((start + offset, *entry));
                offset += entry.size;
            }
        }
        entries
    }

    pub fn add_io_dev(&self, port: u16, dev: MmioRange) -> Result<()> {
        self.add_io_region(port, Arc::new(IoRegion::new(dev)))
    }

    pub fn add_io_region(&self, port: u16, region: Arc<IoRegion>) -> Result<()> {
        let mut regions = self.io_regions.lock();
        regions.add(port as u64, region.clone())?;
        let mut io_bus = self.io_bus.write();
        io_bus.add(port as u64, region.range.clone())?;
        let callbacks = region.callbacks.lock();
        for callback in callbacks.iter() {
            callback.mapped(port as u64)?;
        }
        Ok(())
    }

    fn unmap_io_region(&self, port: u16, region: &IoRegion) -> Result<()> {
        let mut io_bus = self.io_bus.write();
        io_bus.remove(port as u64)?;
        let callbacks = region.callbacks.lock();
        for callback in callbacks.iter() {
            callback.unmapped()?;
        }
        Ok(())
    }

    pub fn remove_io_region(&self, port: u16) -> Result<Arc<IoRegion>> {
        let mut io_regions = self.io_regions.lock();
        let io_region = io_regions.remove(port as u64)?;
        self.unmap_io_region(port, &io_region)?;
        Ok(io_region)
    }

    pub fn register_encrypted_pages(&self, pages: &ArcMemPages) -> Result<()> {
        self.vm_memory.register_encrypted_range(pages.as_slice())?;
        Ok(())
    }

    pub fn deregister_encrypted_pages(&self, pages: &ArcMemPages) -> Result<()> {
        self.vm_memory
            .deregister_encrypted_range(pages.as_slice())?;
        Ok(())
    }

    pub fn mark_private_memory(&self, gpa: u64, size: u64, private: bool) -> Result<()> {
        let vm_memory = &self.vm_memory;
        let regions = self.regions.lock();
        let end = gpa + size;
        let mut start = gpa;
        'out: while let Some((mut addr, region)) = regions.search_next(start) {
            let next_start = addr + region.size();
            for range in &region.ranges {
                let (MemRange::DevMem(r) | MemRange::Ram(r)) = range else {
                    addr += range.size();
                    continue;
                };
                let range_end = addr + r.size();
                if range_end <= start {
                    addr = range_end;
                    continue;
                }
                let gpa_start = std::cmp::max(addr, start);
                let gpa_end = std::cmp::min(end, range_end);
                if gpa_start >= gpa_end {
                    break 'out;
                }
                vm_memory.mark_private_memory(gpa_start, gpa_end - gpa_start, private)?;
                start = gpa_end;
            }
            if next_start >= end {
                break;
            }
            start = next_start;
        }
        Ok(())
    }
}

impl Drop for Memory {
    fn drop(&mut self) {
        if let Err(e) = self.clear() {
            log::info!("dropping memory: {e}")
        }
    }
}

impl Memory {
    fn handle_action(&self, action: Action, none: VmEntry) -> Result<VmEntry> {
        match action {
            Action::None => Ok(none),
            Action::Shutdown => Ok(VmEntry::Shutdown),
            Action::Reset => Ok(VmEntry::Reboot),
            Action::ChangeLayout { callback } => {
                callback.change(self)?;
                Ok(none)
            }
        }
    }

    pub fn handle_mmio(&self, gpa: u64, write: Option<u64>, size: u8) -> Result<VmEntry> {
        let mmio_bus = self.mmio_bus.read();
        if let Some(val) = write {
            let action = mmio_bus.write(gpa, size, val)?;
            drop(mmio_bus);
            self.handle_action(action, VmEntry::None)
        } else {
            let data = mmio_bus.read(gpa, size)?;
            Ok(VmEntry::Mmio { data })
        }
    }

    pub fn handle_io(&self, port: u16, write: Option<u32>, size: u8) -> Result<VmEntry> {
        let io_bus = self.io_bus.read();
        if let Some(val) = write {
            let action = io_bus.write(port as u64, size, val as u64)?;
            drop(io_bus);
            self.handle_action(action, VmEntry::Io { data: None })
        } else {
            let data = io_bus.read(port as u64, size)? as u32;
            Ok(VmEntry::Io { data: Some(data) })
        }
    }
}

1	// Copyright 2024 Google LLC
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// https://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	pub mod addressable;
16	pub mod emulated;
17	pub mod mapped;
18
19	use std::any::type_name;
20	use std::fmt::Debug;
21	use std::sync::Arc;
22
23	use parking_lot::{Mutex, RwLock};
24	use serde::Deserialize;
25	use serde_aco::Help;
26	use snafu::Snafu;
27
28	use crate::errors::{DebugTrace, trace_error};
29	use crate::hv::{MemMapOption, VmEntry, VmMemory};
30
31	use self::addressable::{Addressable, SlotBackend};
32	use self::emulated::{Action, MmioBus, MmioRange};
33	use self::mapped::{ArcMemPages, Ram, RamBus};
34
35	#[trace_error]
36	#[derive(Snafu, DebugTrace)]
37	#[snafu(module, visibility(pub), context(suffix(false)))]
38	pub enum Error {
39	#[snafu(display("Hypervisor internal error"), context(false))]
40	HvError { source: Box<crate::hv::Error> },
41	#[snafu(display("Cannot add a zero-sized slot"))]
42	ZeroSizedSlot,
43	#[snafu(display("(addr={addr:#x}, size={size:#x}) exceeds the address limit"))]
44	ExceedsLimit { addr: u64, size: u64 },
45	#[snafu(display("{new_item:#x?} overlaps with {exist_item:#x?}"))]
46	Overlap {
47	new_item: [u64; 2],
48	exist_item: [u64; 2],
49	},
50	#[snafu(display("{addr:#x} is not mapped"))]
51	NotMapped { addr: u64 },
52	#[snafu(display("Sum of backend range sizes {sum:#x} exceeds the region total size"))]
53	BackendTooBig { sum: u64, size: u64 },
54	#[snafu(display("address {addr:#x} is not {align}-byte aligned"))]
55	NotAligned { addr: u64, align: usize },
56	#[snafu(display(
57	"Guest address {addr:#x} (size = {:#x}) is not backed by continuous host memory"
58	))]
59	NotContinuous { addr: u64, size: u64 },
60	#[snafu(display("Error from OS"), context(false))]
61	System { error: std::io::Error },
62	#[snafu(display("Failed to write data to destination"))]
63	Write { error: std::io::Error },
64	#[snafu(display("Failed to read data from source"))]
65	Read { error: std::io::Error },
66	#[snafu(display("Failed to do MMIO"))]
67	Mmio {
68	source: Box<dyn DebugTrace + Send + Sync + 'static>,
69	},
70	#[snafu(display("Failed to change memory layout"))]
71	ChangeLayout {
72	source: Box<dyn DebugTrace + Send + Sync + 'static>,
73	},
74	}
75
76	pub type Result<T, E = Error> = std::result::Result<T, E>;
77
78	fn default_memory_size() -> u64 {
79	1 << 30
80	}
81
82	#[derive(Debug, Deserialize, Default, Help)]
83	pub struct MemConfig {
84	/// Total guest memory size in bytes. [default: 1G]
85	#[serde(default = "default_memory_size")]
86	pub size: u64,
87	/// Host backend [default: anon]
88	#[serde(default)]
89	pub backend: MemBackend,
90	/// mmap() guest memory with MAP_SHARED or MAP_PRIVATE.
91	/// [default: false]
92	#[serde(default)]
93	pub shared: bool,
94	/// Enable transparent hugepage. [default: false]
95	#[cfg(target_os = "linux")]
96	#[serde(default, alias = "thp")]
97	pub transparent_hugepage: bool,
98	}
99
100	#[derive(Debug, Deserialize, Default, Help)]
101	pub enum MemBackend {
102	/// Anonymous memory by MAP_ANONYMOUS.
103	#[default]
104	#[serde(alias = "anon")]
105	Anonymous,
106	/// Anonymous file by memfd_create(). Always uses MAP_SHARED.
107	#[cfg(target_os = "linux")]
108	#[serde(alias = "memfd")]
109	Memfd,
110	}
111
112	impl MemConfig {
113	pub fn has_shared_fd(&self) -> bool {	×
114	match &self.backend {	×
115	#[cfg(target_os = "linux")]
UNCOV 116	MemBackend::Memfd => true,
117	MemBackend::Anonymous => false,
118	}
119	}
120	}
121
122	#[derive(Debug)]
123	pub enum MemRange {
124	Ram(ArcMemPages),
125	DevMem(ArcMemPages),
126	Emulated(MmioRange),
127	Span(u64),
128	}
129
130	impl MemRange {
131	pub fn size(&self) -> u64 {	×
132	match self {	×
133	MemRange::Ram(pages) \| MemRange::DevMem(pages) => pages.size(),	×
134	MemRange::Emulated(range) => range.size(),	×
135	MemRange::Span(size) => *size,	×
136	}
137	}
138	}
139
140	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141	pub enum MemRegionType {
142	Hidden,
143	Ram,
144	Reserved,
145	Acpi,
146	Pmem,
147	}
148
149	#[derive(Debug, Clone, Copy)]
150	pub struct MemRegionEntry {
151	pub size: u64,
152	pub type_: MemRegionType,
153	}
154
155	pub trait MemRegionCallback: Debug + Send + Sync + 'static {
156	fn mapped(&self, addr: u64) -> Result<()>;
157	fn unmapped(&self) -> Result<()> {	×
158	log::debug!("{} unmapped", type_name::<Self>());	×
159	Ok(())	×
160	}
161	}
162
163	#[derive(Debug)]
164	pub struct MemRegion {
165	pub ranges: Vec<MemRange>,
166	pub entries: Vec<MemRegionEntry>,
167	pub callbacks: Mutex<Vec<Box<dyn MemRegionCallback>>>,
168	}
169
170	impl MemRegion {
171	pub fn size(&self) -> u64 {	×
172	self.entries.iter().fold(0, \|accu, e\| accu + e.size)	×
173	}
174
175	pub fn with_ram(pages: ArcMemPages, type_: MemRegionType) -> MemRegion {	×
176	let size = pages.size();	×
177	MemRegion {
178	ranges: vec![MemRange::Ram(pages)],	×
179	entries: vec![MemRegionEntry { type_, size }],	×
180	callbacks: Mutex::new(vec![]),	×
181	}
182	}
183
184	pub fn with_dev_mem(pages: ArcMemPages, type_: MemRegionType) -> MemRegion {	×
185	let size = pages.size();	×
186	MemRegion {
187	ranges: vec![MemRange::DevMem(pages)],	×
188	entries: vec![MemRegionEntry { type_, size }],	×
189	callbacks: Mutex::new(vec![]),	×
190	}
191	}
192
193	pub fn with_emulated(range: MmioRange, type_: MemRegionType) -> MemRegion {	×
194	let size = range.size();	×
195	MemRegion {
196	ranges: vec![MemRange::Emulated(range)],	×
197	entries: vec![MemRegionEntry { type_, size }],	×
198	callbacks: Mutex::new(vec![]),	×
199	}
200	}
201
202	pub fn validate(&self) -> Result<()> {	×
203	let entries_size = self.size();	×
204	let ranges_size = self.ranges.iter().fold(0, \|accu, r\| accu + r.size());	×
205	if ranges_size > entries_size {	×
206	return error::BackendTooBig {
207	sum: ranges_size,
208	size: entries_size,
209	}
210	.fail();	×
211	}
212	Ok(())	×
213	}
214	}
215
216	impl SlotBackend for Arc<MemRegion> {
217	fn size(&self) -> u64 {	×
218	MemRegion::size(self.as_ref())	×
219	}
220	}
221
222	#[derive(Debug)]
223	pub struct IoRegion {
224	pub range: MmioRange,
225	pub callbacks: Mutex<Vec<Box<dyn MemRegionCallback>>>,
226	}
227
228	impl IoRegion {
229	pub fn new(range: MmioRange) -> IoRegion {	×
230	IoRegion {
231	range,
232	callbacks: Mutex::new(vec![]),	×
233	}
234	}
235	}
236
237	impl SlotBackend for Arc<IoRegion> {
238	fn size(&self) -> u64 {	×
239	self.range.size()	×
240	}
241	}
242
243	pub trait LayoutChanged: Debug + Send + Sync + 'static {
244	fn ram_added(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>;
245	fn ram_removed(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>;
246	}
247
248	pub trait LayoutUpdated: Debug + Send + Sync + 'static {
249	fn ram_updated(&self, ram: &Ram) -> Result<()>;
250	}
251
252	#[derive(Debug, Default)]
253	struct LayoutCallbacks {
254	changed: Vec<Box<dyn LayoutChanged>>,
255	updated: Vec<Box<dyn LayoutUpdated>>,
256	}
257
258	// lock order: region -> callbacks -> bus
259	#[derive(Debug)]
260	pub struct Memory {
261	regions: Mutex<Addressable<Arc<MemRegion>>>,
262	callbacks: Mutex<LayoutCallbacks>,
263	ram_bus: Arc<RamBus>,
264	mmio_bus: RwLock<MmioBus>,
265	vm_memory: Box<dyn VmMemory>,
266
267	io_bus: RwLock<MmioBus>,
268	io_regions: Mutex<Addressable<Arc<IoRegion>>>,
269	}
270
271	impl Memory {
272	pub fn new(vm_memory: impl VmMemory) -> Self {	×
273	Memory {
274	regions: Mutex::new(Addressable::new()),	×
275	callbacks: Mutex::new(LayoutCallbacks::default()),	×
276	ram_bus: Arc::new(RamBus::new()),	×
277	mmio_bus: RwLock::new(MmioBus::new()),	×
278	vm_memory: Box::new(vm_memory),	×
279	io_bus: RwLock::new(MmioBus::new()),	×
280	io_regions: Mutex::new(Addressable::new()),	×
281	}
282	}
283
284	pub fn register_change_callback(&self, callback: Box<dyn LayoutChanged>) -> Result<()> {	×
285	let regions = self.regions.lock();	×
286	for (addr, region) in regions.iter() {	×
287	let mut offset = 0;	×
288	for range in &region.ranges {	×
289	let gpa = addr + offset;	×
290	match range {	×
291	MemRange::Ram(r) => callback.ram_added(gpa, r)?,	×
292	MemRange::Span(_) \| MemRange::Emulated(_) \| MemRange::DevMem(_) => {}
293	}
294	offset += range.size();	×
295	}
296	}
297	let mut callbacks = self.callbacks.lock();	×
298	callbacks.changed.push(callback);	×
299	Ok(())	×
300	}
301
302	pub fn register_update_callback(&self, callback: Box<dyn LayoutUpdated>) -> Result<()> {	×
303	let _regions = self.regions.lock();	×
304	let mut callbacks = self.callbacks.lock();	×
305	let ram = self.ram_bus.lock_layout();	×
306	callback.ram_updated(&ram)?;	×
307	callbacks.updated.push(callback);	×
308	Ok(())	×
309	}
310
311	pub fn reset(&self) -> Result<()> {	×
312	self.clear()?;	×
313	self.vm_memory.reset()?;	×
314	Ok(())	×
315	}
316
317	pub fn ram_bus(&self) -> Arc<RamBus> {	×
318	self.ram_bus.clone()	×
319	}
320
321	fn map_to_vm(&self, gpa: u64, user_mem: &ArcMemPages) -> Result<(), Error> {	×
322	let mem_options = MemMapOption {
323	read: true,
324	write: true,
325	exec: true,
326	log_dirty: false,
327	};
328	self.vm_memory	×
329	.mem_map(gpa, user_mem.size(), user_mem.addr(), mem_options)?;	×
330	Ok(())	×
331	}
332
333	fn unmap_from_vm(&self, gpa: u64, user_mem: &ArcMemPages) -> Result<(), Error> {	×
334	self.vm_memory.unmap(gpa, user_mem.size())?;	×
335	Ok(())	×
336	}
337
338	pub fn add_region(&self, addr: u64, region: Arc<MemRegion>) -> Result<()> {	×
339	region.validate()?;	×
340	let mut regions = self.regions.lock();	×
341	regions.add(addr, region.clone())?;	×
342	let mut offset = 0;	×
343	let callbacks = self.callbacks.lock();	×
344	let mut ram_updated = false;	×
345	for range in &region.ranges {	×
346	let gpa = addr + offset;	×
347	match range {	×
348	MemRange::Emulated(r) => {	×
349	let mut mmio_bus = self.mmio_bus.write();	×
350	mmio_bus.add(gpa, r.clone())?	×
351	}
352	MemRange::Ram(r) => {	×
353	self.map_to_vm(gpa, r)?;	×
354	for callback in &callbacks.changed {	×
355	callback.ram_added(gpa, r)?;	×
356	}
357	self.ram_bus.add(gpa, r.clone())?;	×
358	ram_updated = true;	×
359	}
360	MemRange::DevMem(r) => self.map_to_vm(gpa, r)?,	×
361	MemRange::Span(_) => {}
362	}
363	offset += range.size();	×
364	}
365	if ram_updated {	×
366	let ram = self.ram_bus.lock_layout();	×
367	for update_callback in &callbacks.updated {	×
368	update_callback.ram_updated(&ram)?;	×
369	}
370	}
371	let region_callbacks = region.callbacks.lock();	×
372	for callback in region_callbacks.iter() {	×
373	callback.mapped(addr)?;	×
374	}
375	Ok(())	×
376	}
377
378	fn unmap_region(&self, addr: u64, region: &MemRegion) -> Result<()> {	×
379	let mut offset = 0;	×
380	let callbacks = self.callbacks.lock();	×
381	let mut ram_updated = false;	×
382	for range in &region.ranges {	×
383	let gpa = addr + offset;	×
384	match range {	×
385	MemRange::Emulated(_) => {
386	let mut mmio_bus = self.mmio_bus.write();	×
387	mmio_bus.remove(gpa)?;	×
388	}
389	MemRange::Ram(r) => {	×
390	self.ram_bus.remove(gpa)?;	×
391	for callback in &callbacks.changed {	×
392	callback.ram_removed(gpa, r)?;	×
393	}
394	self.unmap_from_vm(gpa, r)?;	×
395	ram_updated = true;	×
396	}
397	MemRange::DevMem(r) => self.unmap_from_vm(gpa, r)?,	×
398	MemRange::Span(_) => {}
399	};
400	offset += range.size();	×
401	}
402	if ram_updated {	×
403	let ram = self.ram_bus.lock_layout();	×
404	for callback in &callbacks.updated {	×
405	callback.ram_updated(&ram)?;	×
406	}
407	}
408	let region_callbacks = region.callbacks.lock();	×
409	for callback in region_callbacks.iter() {	×
410	callback.unmapped()?;	×
411	}
412	Ok(())	×
413	}
414
415	pub fn remove_region(&self, addr: u64) -> Result<Arc<MemRegion>> {	×
416	let mut regions = self.regions.lock();	×
417	let region = regions.remove(addr)?;	×
418	self.unmap_region(addr, &region)?;	×
419	Ok(region)	×
420	}
421
422	// TODO can be optimized
423	fn clear(&self) -> Result<()> {	×
424	let mut regions = self.regions.lock();	×
425	let regions = regions.drain(..);	×
426	for (addr, region) in regions {	×
427	self.unmap_region(addr, &region)?;	×
428	}
429	#[cfg(target_arch = "x86_64")]
430	{
431	let mut io_regions = self.io_regions.lock();
432	let io_regions = io_regions.drain(..);
433	for (port, io_region) in io_regions {
434	self.unmap_io_region(port as u16, &io_region)?;
435	}
436	}
437	Ok(())	×
438	}
439
440	pub fn mem_region_entries(&self) -> Vec<(u64, MemRegionEntry)> {	×
441	let mut entries = vec![];	×
442	let regions = self.regions.lock();	×
443	for (start, region) in regions.iter() {	×
444	let mut offset = 0;	×
445	for entry in region.entries.iter() {	×
446	entries.push((start + offset, *entry));	×
447	offset += entry.size;	×
448	}
449	}
450	entries	×
451	}
452
453	pub fn add_io_dev(&self, port: u16, dev: MmioRange) -> Result<()> {	×
454	self.add_io_region(port, Arc::new(IoRegion::new(dev)))	×
455	}
456
457	pub fn add_io_region(&self, port: u16, region: Arc<IoRegion>) -> Result<()> {	×
458	let mut regions = self.io_regions.lock();	×
459	regions.add(port as u64, region.clone())?;	×
460	let mut io_bus = self.io_bus.write();	×
461	io_bus.add(port as u64, region.range.clone())?;	×
462	let callbacks = region.callbacks.lock();	×
463	for callback in callbacks.iter() {	×
464	callback.mapped(port as u64)?;	×
465	}
466	Ok(())	×
467	}
468
469	fn unmap_io_region(&self, port: u16, region: &IoRegion) -> Result<()> {	×
470	let mut io_bus = self.io_bus.write();	×
471	io_bus.remove(port as u64)?;	×
472	let callbacks = region.callbacks.lock();	×
473	for callback in callbacks.iter() {	×
474	callback.unmapped()?;	×
475	}
476	Ok(())	×
477	}
478
479	pub fn remove_io_region(&self, port: u16) -> Result<Arc<IoRegion>> {	×
480	let mut io_regions = self.io_regions.lock();	×
481	let io_region = io_regions.remove(port as u64)?;	×
482	self.unmap_io_region(port, &io_region)?;	×
483	Ok(io_region)	×
484	}
485
486	pub fn register_encrypted_pages(&self, pages: &ArcMemPages) -> Result<()> {	×
487	self.vm_memory.register_encrypted_range(pages.as_slice())?;	×
488	Ok(())	×
489	}
490
491	pub fn deregister_encrypted_pages(&self, pages: &ArcMemPages) -> Result<()> {	×
492	self.vm_memory	×
493	.deregister_encrypted_range(pages.as_slice())?;	×
494	Ok(())	×
495	}
496
497	pub fn mark_private_memory(&self, gpa: u64, size: u64, private: bool) -> Result<()> {	×
498	let vm_memory = &self.vm_memory;	×
499	let regions = self.regions.lock();	×
500	let end = gpa + size;	×
501	let mut start = gpa;	×
502	'out: while let Some((mut addr, region)) = regions.search_next(start) {	×
503	let next_start = addr + region.size();	×
504	for range in &region.ranges {	×
505	let (MemRange::DevMem(r) \| MemRange::Ram(r)) = range else {	×
506	addr += range.size();	×
507	continue;	×
508	};
509	let range_end = addr + r.size();	×
510	if range_end <= start {	×
511	addr = range_end;	×
512	continue;
513	}
514	let gpa_start = std::cmp::max(addr, start);	×
515	let gpa_end = std::cmp::min(end, range_end);	×
516	if gpa_start >= gpa_end {	×
517	break 'out;
518	}
519	vm_memory.mark_private_memory(gpa_start, gpa_end - gpa_start, private)?;	×
520	start = gpa_end;	×
521	}
522	if next_start >= end {	×
523	break;
524	}
525	start = next_start;	×
526	}
527	Ok(())	×
528	}
529	}
530
531	impl Drop for Memory {
532	fn drop(&mut self) {	×
533	if let Err(e) = self.clear() {	×
534	log::info!("dropping memory: {e}")	×
535	}
536	}
537	}
538
539	impl Memory {
540	fn handle_action(&self, action: Action, none: VmEntry) -> Result<VmEntry> {	×
541	match action {	×
542	Action::None => Ok(none),	×
543	Action::Shutdown => Ok(VmEntry::Shutdown),	×
544	Action::Reset => Ok(VmEntry::Reboot),	×
545	Action::ChangeLayout { callback } => {	×
546	callback.change(self)?;	×
547	Ok(none)	×
548	}
549	}
550	}
551
552	pub fn handle_mmio(&self, gpa: u64, write: Option<u64>, size: u8) -> Result<VmEntry> {	×
553	let mmio_bus = self.mmio_bus.read();	×
554	if let Some(val) = write {	×
555	let action = mmio_bus.write(gpa, size, val)?;	×
556	drop(mmio_bus);	×
557	self.handle_action(action, VmEntry::None)	×
558	} else {
559	let data = mmio_bus.read(gpa, size)?;	×
560	Ok(VmEntry::Mmio { data })	×
561	}
562	}
563
564	pub fn handle_io(&self, port: u16, write: Option<u32>, size: u8) -> Result<VmEntry> {	×
565	let io_bus = self.io_bus.read();	×
566	if let Some(val) = write {	×
567	let action = io_bus.write(port as u64, size, val as u64)?;	×
568	drop(io_bus);	×
569	self.handle_action(action, VmEntry::Io { data: None })	×
570	} else {
571	let data = io_bus.read(port as u64, size)? as u32;	×
572	Ok(VmEntry::Io { data: Some(data) })	×
573	}
574	}
575	}

google / alioth / 17385983897

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous