4382580286

Build Type

push

github

Committed by GitHub

Commit Message

feat: Separate cache operation log environment and index environments (#1199)

Run Details

1370 of 1370 new or added lines in 33 files covered. (100.0%)

28671 of 41023 relevant lines covered (69.89%)

51121.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.23

/dozer-storage/src/lmdb_database/lmdb_val.rs

use dozer_types::{
    borrow::{Borrow, Cow},
    types::{IndexDefinition, Record, SchemaWithIndex},
};

use crate::errors::StorageError;

pub enum Encoded<'a> {
    U8([u8; 1]),
    U8x4([u8; 4]),
    U8x8([u8; 8]),
    U8x16([u8; 16]),
    U8x21([u8; 21]),
    Vec(Vec<u8>),
    Borrowed(&'a [u8]),
}

impl<'a> AsRef<[u8]> for Encoded<'a> {
    fn as_ref(&self) -> &[u8] {
        match self {
            Self::U8(v) => v.as_slice(),
            Self::U8x4(v) => v.as_slice(),
            Self::U8x8(v) => v.as_slice(),
            Self::U8x16(v) => v.as_slice(),
            Self::U8x21(v) => v.as_slice(),
            Self::Vec(v) => v.as_slice(),
            Self::Borrowed(v) => v,
        }
    }
}

pub trait Encode<'a> {
    fn encode(self) -> Result<Encoded<'a>, StorageError>;
}

pub trait BorrowEncode: 'static + for<'a> Borrow<Borrowed<'a> = Self::Encode<'a>> {
    type Encode<'a>: Encode<'a>;
}

pub trait Decode: Borrow {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError>;
}

/// A trait for types that can be used in LMDB.
///
/// # Safety
///
/// - `decode` must match the implementation of `encode`.
pub unsafe trait LmdbVal: BorrowEncode + Decode {}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum LmdbKeyType {
    U32,
    #[cfg(target_pointer_width = "64")]
    U64,
    FixedSizeOtherThanU32OrUsize,
    VariableSize,
}

/// A trait for types that can be used as keys in LMDB.
///
/// # Safety
///
/// - `TYPE` must match the implementation of `encode`.
///
/// # Note
///
/// The implementation for `u32` and `u64` has a caveat: The values are encoded in big-endian but compared in native-endian.
pub unsafe trait LmdbKey: LmdbVal {
    const TYPE: LmdbKeyType;
}

impl<'a> Encode<'a> for &'a u8 {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        Ok(Encoded::U8([*self]))
    }
}

impl BorrowEncode for u8 {
    type Encode<'a> = &'a u8;
}

impl Decode for u8 {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        Ok(Cow::Owned(bytes[0]))
    }
}

unsafe impl LmdbKey for u8 {
    const TYPE: LmdbKeyType = LmdbKeyType::FixedSizeOtherThanU32OrUsize;
}

unsafe impl LmdbVal for u8 {}

impl<'a> Encode<'a> for &'a u32 {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        Ok(Encoded::U8x4(self.to_be_bytes()))
    }
}

impl BorrowEncode for u32 {
    type Encode<'a> = &'a u32;
}

impl Decode for u32 {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        Ok(Cow::Owned(u32::from_be_bytes(bytes.try_into().unwrap())))
    }
}

unsafe impl LmdbKey for u32 {
    const TYPE: LmdbKeyType = LmdbKeyType::U32;
}

unsafe impl LmdbVal for u32 {}

impl<'a> Encode<'a> for &'a u64 {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        Ok(Encoded::U8x8(self.to_be_bytes()))
    }
}

impl BorrowEncode for u64 {
    type Encode<'a> = &'a u64;
}

impl Decode for u64 {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        Ok(Cow::Owned(u64::from_be_bytes(bytes.try_into().unwrap())))
    }
}

unsafe impl LmdbKey for u64 {
    #[cfg(target_pointer_width = "64")]
    const TYPE: LmdbKeyType = LmdbKeyType::U64;
    #[cfg(not(target_pointer_width = "64"))]
    const TYPE: LmdbKeyType = LmdbKeyType::FixedSizeOtherThanU32OrUsize;
}

unsafe impl LmdbVal for u64 {}

impl<'a> Encode<'a> for &'a [u8] {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        Ok(Encoded::Borrowed(self))
    }
}

impl BorrowEncode for Vec<u8> {
    type Encode<'a> = &'a [u8];
}

impl Decode for Vec<u8> {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        Ok(Cow::Borrowed(bytes))
    }
}

unsafe impl LmdbKey for Vec<u8> {
    const TYPE: LmdbKeyType = LmdbKeyType::VariableSize;
}

unsafe impl LmdbVal for Vec<u8> {}

impl<'a> Encode<'a> for &'a str {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        Ok(Encoded::Borrowed(self.as_bytes()))
    }
}

impl BorrowEncode for String {
    type Encode<'a> = &'a str;
}

impl Decode for String {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        Ok(Cow::Borrowed(std::str::from_utf8(bytes).unwrap()))
    }
}

unsafe impl LmdbVal for String {}

unsafe impl LmdbKey for String {
    const TYPE: LmdbKeyType = LmdbKeyType::VariableSize;
}

impl<'a> Encode<'a> for &'a Record {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        dozer_types::bincode::serialize(self)
            .map(Encoded::Vec)
            .map_err(|e| StorageError::SerializationError {
                typ: "Record",
                reason: Box::new(e),
            })
    }
}

impl BorrowEncode for Record {
    type Encode<'a> = &'a Record;
}

impl Decode for Record {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        dozer_types::bincode::deserialize(bytes)
            .map(Cow::Owned)
            .map_err(|e| StorageError::DeserializationError {
                typ: "Record",
                reason: Box::new(e),
            })
    }
}

unsafe impl LmdbKey for Record {
    const TYPE: LmdbKeyType = LmdbKeyType::VariableSize;
}

unsafe impl LmdbVal for Record {}

impl<'a> Encode<'a> for &'a IndexDefinition {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        dozer_types::bincode::serialize(self)
            .map(Encoded::Vec)
            .map_err(|e| StorageError::SerializationError {
                typ: "IndexDefinition",
                reason: Box::new(e),
            })
    }
}

impl BorrowEncode for IndexDefinition {
    type Encode<'a> = &'a IndexDefinition;
}

impl Decode for IndexDefinition {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        dozer_types::bincode::deserialize(bytes)
            .map(Cow::Owned)
            .map_err(|e| StorageError::DeserializationError {
                typ: "IndexDefinition",
                reason: Box::new(e),
            })
    }
}

unsafe impl LmdbVal for IndexDefinition {}

impl<'a> Encode<'a> for &'a SchemaWithIndex {
    fn encode(self) -> Result<Encoded<'a>, StorageError> {
        dozer_types::bincode::serialize(self)
            .map(Encoded::Vec)
            .map_err(|e| StorageError::SerializationError {
                typ: "SchemaWithIndex",
                reason: Box::new(e),
            })
    }
}

impl BorrowEncode for SchemaWithIndex {
    type Encode<'a> = &'a SchemaWithIndex;
}

impl Decode for SchemaWithIndex {
    fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {
        dozer_types::bincode::deserialize(bytes)
            .map(Cow::Owned)
            .map_err(|e| StorageError::DeserializationError {
                typ: "SchemaWithIndex",
                reason: Box::new(e),
            })
    }
}

unsafe impl LmdbVal for SchemaWithIndex {}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_lmdb_key_types() {
        assert_eq!(u8::TYPE, LmdbKeyType::FixedSizeOtherThanU32OrUsize);
        assert_eq!(u32::TYPE, LmdbKeyType::U32);
        assert_eq!(u64::TYPE, LmdbKeyType::U64);
        assert_eq!(Vec::<u8>::TYPE, LmdbKeyType::VariableSize);
    }
}

1	use dozer_types::{
2	borrow::{Borrow, Cow},
3	types::{IndexDefinition, Record, SchemaWithIndex},
4	};
5
6	use crate::errors::StorageError;
7
8	pub enum Encoded<'a> {
9	U8([u8; 1]),
10	U8x4([u8; 4]),
11	U8x8([u8; 8]),
12	U8x16([u8; 16]),
13	U8x21([u8; 21]),
14	Vec(Vec<u8>),
15	Borrowed(&'a [u8]),
16	}
17
18	impl<'a> AsRef<[u8]> for Encoded<'a> {
19	fn as_ref(&self) -> &[u8] {	11,662,160✔
20	match self {	11,662,160✔
21	Self::U8(v) => v.as_slice(),	966,560✔
22	Self::U8x4(v) => v.as_slice(),	8✔
23	Self::U8x8(v) => v.as_slice(),	10,246,117✔
24	Self::U8x16(v) => v.as_slice(),	×
25	Self::U8x21(v) => v.as_slice(),	32,827✔
26	Self::Vec(v) => v.as_slice(),	36,482✔
27	Self::Borrowed(v) => v,	380,166✔
28	}
29	}	11,662,160✔
30	}
31
32	pub trait Encode<'a> {
33	fn encode(self) -> Result<Encoded<'a>, StorageError>;
34	}
35
36	pub trait BorrowEncode: 'static + for<'a> Borrow<Borrowed<'a> = Self::Encode<'a>> {
37	type Encode<'a>: Encode<'a>;
38	}
39
40	pub trait Decode: Borrow {	×
41	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError>;
42	}
43
44	/// A trait for types that can be used in LMDB.	×
45	///
46	/// # Safety
47	///
48	/// - `decode` must match the implementation of `encode`.
49	pub unsafe trait LmdbVal: BorrowEncode + Decode {}
50
51	#[derive(Debug, Clone, Copy, PartialEq)]	4✔
52	pub enum LmdbKeyType {
53	U32,
54	#[cfg(target_pointer_width = "64")]
55	U64,
56	FixedSizeOtherThanU32OrUsize,
57	VariableSize,
58	}
59
60	/// A trait for types that can be used as keys in LMDB.
61	///
62	/// # Safety
63	///
64	/// - `TYPE` must match the implementation of `encode`.
65	///
66	/// # Note
67	///
68	/// The implementation for `u32` and `u64` has a caveat: The values are encoded in big-endian but compared in native-endian.
69	pub unsafe trait LmdbKey: LmdbVal {
70	const TYPE: LmdbKeyType;
71	}	×
72		×
73	impl<'a> Encode<'a> for &'a u8 {	×
74	fn encode(self) -> Result<Encoded<'a>, StorageError> {	977,389✔
75	Ok(Encoded::U8([*self]))	977,389✔
76	}	977,389✔
77	}	×
78		×
79	impl BorrowEncode for u8 {	×
80	type Encode<'a> = &'a u8;	×
81	}	×
82
83	impl Decode for u8 {
84	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	×
85	Ok(Cow::Owned(bytes[0]))	×
86	}	×
87	}	×
88		×
89	unsafe impl LmdbKey for u8 {	×
90	const TYPE: LmdbKeyType = LmdbKeyType::FixedSizeOtherThanU32OrUsize;	×
91	}	×
92
93	unsafe impl LmdbVal for u8 {}	×
94		×
95	impl<'a> Encode<'a> for &'a u32 {	×
96	fn encode(self) -> Result<Encoded<'a>, StorageError> {	8✔
97	Ok(Encoded::U8x4(self.to_be_bytes()))	8✔
98	}	8✔
99	}
100
101	impl BorrowEncode for u32 {	×
102	type Encode<'a> = &'a u32;	×
103	}	×
104		×
105	impl Decode for u32 {	×
106	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	3✔
107	Ok(Cow::Owned(u32::from_be_bytes(bytes.try_into().unwrap())))	3✔
108	}	3✔
109	}	×
110		×
111	unsafe impl LmdbKey for u32 {	×
112	const TYPE: LmdbKeyType = LmdbKeyType::U32;	×
113	}	×
114
115	unsafe impl LmdbVal for u32 {}
116
117	impl<'a> Encode<'a> for &'a u64 {
118	fn encode(self) -> Result<Encoded<'a>, StorageError> {	10,245,811✔
119	Ok(Encoded::U8x8(self.to_be_bytes()))	10,245,811✔
120	}	10,245,811✔
121	}
122		×
123	impl BorrowEncode for u64 {	×
124	type Encode<'a> = &'a u64;	×
125	}	×
126
127	impl Decode for u64 {
128	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	7,465,911✔
129	Ok(Cow::Owned(u64::from_be_bytes(bytes.try_into().unwrap())))	7,465,911✔
130	}	7,465,911✔
131	}
132
133	unsafe impl LmdbKey for u64 {	×
134	#[cfg(target_pointer_width = "64")]	×
135	const TYPE: LmdbKeyType = LmdbKeyType::U64;	×
136	#[cfg(not(target_pointer_width = "64"))]
137	const TYPE: LmdbKeyType = LmdbKeyType::FixedSizeOtherThanU32OrUsize;
138	}	×
139		×
140	unsafe impl LmdbVal for u64 {}	×
141
142	impl<'a> Encode<'a> for &'a [u8] {
143	fn encode(self) -> Result<Encoded<'a>, StorageError> {	380,557✔
144	Ok(Encoded::Borrowed(self))	380,557✔
145	}	380,557✔
146	}	×
147
148	impl BorrowEncode for Vec<u8> {	×
149	type Encode<'a> = &'a [u8];	×
150	}	×
151
152	impl Decode for Vec<u8> {
153	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	12,798,401✔
154	Ok(Cow::Borrowed(bytes))	12,798,401✔
155	}	12,798,401✔
156	}	×
157		×
158	unsafe impl LmdbKey for Vec<u8> {	×
159	const TYPE: LmdbKeyType = LmdbKeyType::VariableSize;	×
160	}	×
161		×
162	unsafe impl LmdbVal for Vec<u8> {}
163
164	impl<'a> Encode<'a> for &'a str {
165	fn encode(self) -> Result<Encoded<'a>, StorageError> {	×
166	Ok(Encoded::Borrowed(self.as_bytes()))	×
167	}	×
168	}	×
169		×
170	impl BorrowEncode for String {	×
171	type Encode<'a> = &'a str;	×
172	}	×
173
174	impl Decode for String {
175	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	×
176	Ok(Cow::Borrowed(std::str::from_utf8(bytes).unwrap()))	×
177	}	×
178	}
179
180	unsafe impl LmdbVal for String {}	×
181		×
182	unsafe impl LmdbKey for String {	×
183	const TYPE: LmdbKeyType = LmdbKeyType::VariableSize;	×
184	}	×
185		×
186	impl<'a> Encode<'a> for &'a Record {	×
187	fn encode(self) -> Result<Encoded<'a>, StorageError> {	×
188	dozer_types::bincode::serialize(self)	×
189	.map(Encoded::Vec)	×
190	.map_err(\|e\| StorageError::SerializationError {	×
191	typ: "Record",	×
192	reason: Box::new(e),	×
193	})	×
194	}	×
195	}	×
196		×
197	impl BorrowEncode for Record {	×
198	type Encode<'a> = &'a Record;	×
199	}
200
201	impl Decode for Record {
202	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	×
203	dozer_types::bincode::deserialize(bytes)	×
204	.map(Cow::Owned)	×
205	.map_err(\|e\| StorageError::DeserializationError {	×
206	typ: "Record",	×
207	reason: Box::new(e),	×
208	})	×
209	}	×
210	}	×
211		×
212	unsafe impl LmdbKey for Record {	×
213	const TYPE: LmdbKeyType = LmdbKeyType::VariableSize;
214	}
215
216	unsafe impl LmdbVal for Record {}
217
218	impl<'a> Encode<'a> for &'a IndexDefinition {	×
219	fn encode(self) -> Result<Encoded<'a>, StorageError> {	1,251✔
220	dozer_types::bincode::serialize(self)	1,251✔
221	.map(Encoded::Vec)	1,251✔
222	.map_err(\|e\| StorageError::SerializationError {	1,251✔
223	typ: "IndexDefinition",	×
224	reason: Box::new(e),	×
225	})	1,251✔
226	}	1,251✔
227	}	×
228		×
229	impl BorrowEncode for IndexDefinition {	×
230	type Encode<'a> = &'a IndexDefinition;
231	}
232
233	impl Decode for IndexDefinition {
234	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	1,071✔
235	dozer_types::bincode::deserialize(bytes)	1,071✔
236	.map(Cow::Owned)	1,071✔
237	.map_err(\|e\| StorageError::DeserializationError {	1,071✔
238	typ: "IndexDefinition",	×
239	reason: Box::new(e),	×
240	})	1,071✔
241	}	1,071✔
242	}	×
243		×
244	unsafe impl LmdbVal for IndexDefinition {}	×
245		×
246	impl<'a> Encode<'a> for &'a SchemaWithIndex {
247	fn encode(self) -> Result<Encoded<'a>, StorageError> {	333✔
248	dozer_types::bincode::serialize(self)	333✔
249	.map(Encoded::Vec)	333✔
250	.map_err(\|e\| StorageError::SerializationError {	333✔
251	typ: "SchemaWithIndex",	×
252	reason: Box::new(e),	×
253	})	333✔
254	}	333✔
255	}	×
256		×
257	impl BorrowEncode for SchemaWithIndex {	×
258	type Encode<'a> = &'a SchemaWithIndex;	×
259	}	×
260
261	impl Decode for SchemaWithIndex {
262	fn decode(bytes: &[u8]) -> Result<Cow<Self>, StorageError> {	288✔
263	dozer_types::bincode::deserialize(bytes)	288✔
264	.map(Cow::Owned)	288✔
265	.map_err(\|e\| StorageError::DeserializationError {	288✔
266	typ: "SchemaWithIndex",	×
267	reason: Box::new(e),	×
268	})	288✔
269	}	288✔
270	}
271
272	unsafe impl LmdbVal for SchemaWithIndex {}
273
274	#[cfg(test)]
275	mod tests {
276	use super::*;
277
278	#[test]	1✔
279	fn test_lmdb_key_types() {	1✔
280	assert_eq!(u8::TYPE, LmdbKeyType::FixedSizeOtherThanU32OrUsize);	1✔
281	assert_eq!(u32::TYPE, LmdbKeyType::U32);	1✔
282	assert_eq!(u64::TYPE, LmdbKeyType::U64);	1✔
283	assert_eq!(Vec::<u8>::TYPE, LmdbKeyType::VariableSize);	1✔
284	}	1✔
285	}

getdozer / dozer / 4382580286

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous