16356360776

Committed 17 Jul 2025 09:27PM UTC coverage: 80.84% (+0.1%) from 80.704%

Build # 16356360776

Build Type

Pull #3810

github

Committed by

web-flow

Commit Message

Merge e2e18c977 into ac7014e8c

Pull Request Pull Request #3810: WIP feat: add ViewLayout for variable length views

Run Details

437 of 625 new or added lines in 17 files covered. (69.92%)

564 existing lines in 14 files now uncovered.

42340 of 52375 relevant lines covered (80.84%)

156114.92 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.39

/vortex-scalar/src/binary.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::fmt::{Display, Formatter};
use std::sync::Arc;

use itertools::Itertools;
use vortex_buffer::ByteBuffer;
use vortex_dtype::{DType, Nullability};
use vortex_error::{VortexError, VortexExpect as _, VortexResult, vortex_bail, vortex_err};

use crate::{InnerScalarValue, Scalar, ScalarValue};

/// A scalar value representing binary data.
///
/// This type provides a view into a binary scalar value, which can be either
/// a valid byte buffer or null.
#[derive(Debug, Hash)]
pub struct BinaryScalar<'a> {
    dtype: &'a DType,
    value: Option<Arc<ByteBuffer>>,
}

impl Display for BinaryScalar<'_> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        match &self.value {
            None => write!(f, "null"),
            Some(v) => write!(
                f,
                "\"{}\"",
                v.as_slice().iter().map(|b| format!("{b:x}")).format(" ")
            ),
        }
    }
}

impl PartialEq for BinaryScalar<'_> {
    fn eq(&self, other: &Self) -> bool {
        self.dtype.eq_ignore_nullability(other.dtype) && self.value == other.value
    }
}

impl Eq for BinaryScalar<'_> {}

impl PartialOrd for BinaryScalar<'_> {
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.value.cmp(&other.value))
    }
}

impl Ord for BinaryScalar<'_> {
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        self.value.cmp(&other.value)
    }
}

impl<'a> BinaryScalar<'a> {
    /// Creates a binary scalar from a data type and scalar value.
    ///
    /// # Errors
    ///
    /// Returns an error if the data type is not a binary type.
    pub fn from_scalar_value(dtype: &'a DType, value: ScalarValue) -> VortexResult<Self> {
        if !matches!(dtype, DType::Binary(..)) {
            vortex_bail!("Can only construct binary scalar from binary dtype, found {dtype}")
        }
        Ok(Self {
            dtype,
            value: value.as_buffer()?,
        })
    }

    /// Returns the data type of this binary scalar.
    #[inline]
    pub fn dtype(&self) -> &'a DType {
        self.dtype
    }

    /// Returns the binary value as a byte buffer, or None if null.
    pub fn value(&self) -> Option<ByteBuffer> {
        self.value.as_ref().map(|v| v.as_ref().clone())
    }

    /// Constructs a value at most `max_length` in size that's greater than this value.
    ///
    /// Returns None if constructing a greater value would overflow.
    pub fn upper_bound(self, max_length: usize) -> Option<Self> {
        if let Some(value) = self.value {
            if value.len() > max_length {
                let sliced = value.slice(0..max_length);
                drop(value);
                let mut sliced_mut = sliced.into_mut();
                for b in sliced_mut.iter_mut().rev() {
                    let (incr, overflow) = b.overflowing_add(1);
                    *b = incr;
                    if !overflow {
                        return Some(Self {
                            dtype: self.dtype,
                            value: Some(Arc::new(sliced_mut.freeze())),
                        });
                    }
                }
                None
            } else {
                Some(Self {
                    dtype: self.dtype,
                    value: Some(value),
                })
            }
        } else {
            Some(self)
        }
    }

    /// Construct a value at most `max_length` in size that's less than ourselves.
    pub fn lower_bound(self, max_length: usize) -> Self {
        if let Some(value) = self.value {
            if value.len() > max_length {
                Self {
                    dtype: self.dtype,
                    value: Some(Arc::new(value.slice(0..max_length))),
                }
            } else {
                Self {
                    dtype: self.dtype,
                    value: Some(value),
                }
            }
        } else {
            self
        }
    }

    pub(crate) fn cast(&self, dtype: &DType) -> VortexResult<Scalar> {
        if !matches!(dtype, DType::Binary(..)) {
            vortex_bail!("Can't cast binary to {}", dtype)
        }
        Ok(Scalar::new(
            dtype.clone(),
            ScalarValue(InnerScalarValue::Buffer(
                self.value
                    .as_ref()
                    .vortex_expect("nullness handled in Scalar::cast")
                    .clone(),
            )),
        ))
    }

    /// Length of the scalar value or None if value is null
    pub fn len(&self) -> Option<usize> {
        self.value.as_ref().map(|v| v.len())
    }

    /// Returns whether its value is non-null and empty, otherwise `None`.
    pub fn is_empty(&self) -> Option<bool> {
        self.value.as_ref().map(|v| v.is_empty())
    }

    /// Extract value as a ScalarValue
    pub fn into_value(self) -> ScalarValue {
        ScalarValue(
            self.value
                .map(InnerScalarValue::Buffer)
                .unwrap_or_else(|| InnerScalarValue::Null),
        )
    }
}

impl Scalar {
    /// Creates a new binary scalar from a byte buffer.
    pub fn binary(buffer: impl Into<Arc<ByteBuffer>>, nullability: Nullability) -> Self {
        Self {
            dtype: DType::Binary(nullability),
            value: ScalarValue(InnerScalarValue::Buffer(buffer.into())),
        }
    }
}

impl<'a> TryFrom<&'a Scalar> for BinaryScalar<'a> {
    type Error = VortexError;

    fn try_from(value: &'a Scalar) -> Result<Self, Self::Error> {
        if !matches!(value.dtype(), DType::Binary(_)) {
            vortex_bail!("Expected binary scalar, found {}", value.dtype())
        }
        Ok(Self {
            dtype: value.dtype(),
            value: value.value.as_buffer()?,
        })
    }
}

impl<'a> TryFrom<&'a Scalar> for ByteBuffer {
    type Error = VortexError;

    fn try_from(scalar: &'a Scalar) -> VortexResult<Self> {
        let binary = scalar
            .as_binary_opt()
            .ok_or_else(|| vortex_err!("Cannot extract buffer from non-buffer scalar"))?;

        binary
            .value()
            .ok_or_else(|| vortex_err!("Cannot extract present value from null scalar"))
    }
}

impl<'a> TryFrom<&'a Scalar> for Option<ByteBuffer> {
    type Error = VortexError;

    fn try_from(scalar: &'a Scalar) -> VortexResult<Self> {
        Ok(scalar
            .as_binary_opt()
            .ok_or_else(|| vortex_err!("Cannot extract buffer from non-buffer scalar"))?
            .value())
    }
}

impl TryFrom<Scalar> for ByteBuffer {
    type Error = VortexError;

    fn try_from(scalar: Scalar) -> VortexResult<Self> {
        Self::try_from(&scalar)
    }
}

impl TryFrom<Scalar> for Option<ByteBuffer> {
    type Error = VortexError;

    fn try_from(scalar: Scalar) -> VortexResult<Self> {
        Self::try_from(&scalar)
    }
}

impl From<&[u8]> for Scalar {
    fn from(value: &[u8]) -> Self {
        Scalar::from(ByteBuffer::from(value.to_vec()))
    }
}

impl From<ByteBuffer> for Scalar {
    fn from(value: ByteBuffer) -> Self {
        Self {
            dtype: DType::Binary(Nullability::NonNullable),
            value: ScalarValue(InnerScalarValue::Buffer(Arc::new(value))),
        }
    }
}

impl From<Arc<ByteBuffer>> for Scalar {
    fn from(value: Arc<ByteBuffer>) -> Self {
        Self {
            dtype: DType::Binary(Nullability::NonNullable),
            value: ScalarValue(InnerScalarValue::Buffer(value)),
        }
    }
}

#[cfg(test)]
mod tests {
    use vortex_buffer::buffer;
    use vortex_dtype::Nullability;
    use vortex_error::{VortexExpect, VortexUnwrap};

    use crate::{BinaryScalar, Scalar};

    #[test]
    fn lower_bound() {
        let binary = Scalar::binary(buffer![0u8, 5, 47, 33, 129], Nullability::NonNullable);
        let expected = Scalar::binary(buffer![0u8, 5], Nullability::NonNullable);
        assert_eq!(
            BinaryScalar::try_from(&binary)
                .vortex_unwrap()
                .lower_bound(2),
            BinaryScalar::try_from(&expected).vortex_unwrap()
        );
    }

    #[test]
    fn upper_bound() {
        let binary = Scalar::binary(buffer![0u8, 5, 255, 234, 23], Nullability::NonNullable);
        let expected = Scalar::binary(buffer![0u8, 6, 0], Nullability::NonNullable);
        assert_eq!(
            BinaryScalar::try_from(&binary)
                .vortex_unwrap()
                .upper_bound(3)
                .vortex_expect("must have upper bound"),
            BinaryScalar::try_from(&expected).vortex_unwrap()
        );
    }

    #[test]
    fn upper_bound_overflow() {
        let binary = Scalar::binary(buffer![255u8, 255, 255], Nullability::NonNullable);
        assert!(
            BinaryScalar::try_from(&binary)
                .vortex_unwrap()
                .upper_bound(2)
                .is_none()
        );
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::fmt::{Display, Formatter};
5	use std::sync::Arc;
6
7	use itertools::Itertools;
8	use vortex_buffer::ByteBuffer;
9	use vortex_dtype::{DType, Nullability};
10	use vortex_error::{VortexError, VortexExpect as _, VortexResult, vortex_bail, vortex_err};
11
12	use crate::{InnerScalarValue, Scalar, ScalarValue};
13
14	/// A scalar value representing binary data.
15	///
16	/// This type provides a view into a binary scalar value, which can be either
17	/// a valid byte buffer or null.
18	#[derive(Debug, Hash)]
19	pub struct BinaryScalar<'a> {
20	dtype: &'a DType,
21	value: Option<Arc<ByteBuffer>>,
22	}
23
24	impl Display for BinaryScalar<'_> {
25	fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {	2✔
26	match &self.value {	2✔
27	None => write!(f, "null"),	1✔
28	Some(v) => write!(	1✔
29	f,	1✔
30	"\"{}\"",	1✔
31	v.as_slice().iter().map(\|b\| format!("{b:x}")).format(" ")	12✔
32	),
33	}
34	}	2✔
35	}
36
37	impl PartialEq for BinaryScalar<'_> {
38	fn eq(&self, other: &Self) -> bool {	4✔
39	self.dtype.eq_ignore_nullability(other.dtype) && self.value == other.value	4✔
40	}	4✔
41	}
42
43	impl Eq for BinaryScalar<'_> {}
44
45	impl PartialOrd for BinaryScalar<'_> {
46	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {	74✔
47	Some(self.value.cmp(&other.value))	74✔
48	}	74✔
49	}
50
51	impl Ord for BinaryScalar<'_> {
UNCOV 52	fn cmp(&self, other: &Self) -> std::cmp::Ordering {	×
UNCOV 53	self.value.cmp(&other.value)	×
UNCOV 54	}	×
55	}
56
57	impl<'a> BinaryScalar<'a> {
58	/// Creates a binary scalar from a data type and scalar value.
59	///
60	/// # Errors
61	///
62	/// Returns an error if the data type is not a binary type.
63	pub fn from_scalar_value(dtype: &'a DType, value: ScalarValue) -> VortexResult<Self> {	148✔
64	if !matches!(dtype, DType::Binary(..)) {	148✔
65	vortex_bail!("Can only construct binary scalar from binary dtype, found {dtype}")	×
66	}	148✔
67	Ok(Self {
68	dtype,	148✔
69	value: value.as_buffer()?,	148✔
70	})
71	}	148✔
72
73	/// Returns the data type of this binary scalar.
74	#[inline]
UNCOV 75	pub fn dtype(&self) -> &'a DType {	×
UNCOV 76	self.dtype	×
UNCOV 77	}	×
78
79	/// Returns the binary value as a byte buffer, or None if null.
80	pub fn value(&self) -> Option<ByteBuffer> {	1,441✔
81	self.value.as_ref().map(\|v\| v.as_ref().clone())	1,441✔
82	}	1,441✔
83
84	/// Constructs a value at most `max_length` in size that's greater than this value.
85	///
86	/// Returns None if constructing a greater value would overflow.
87	pub fn upper_bound(self, max_length: usize) -> Option<Self> {	39✔
88	if let Some(value) = self.value {	39✔
89	if value.len() > max_length {	39✔
90	let sliced = value.slice(0..max_length);	39✔
91	drop(value);	39✔
92	let mut sliced_mut = sliced.into_mut();	39✔
93	for b in sliced_mut.iter_mut().rev() {	41✔
94	let (incr, overflow) = b.overflowing_add(1);	41✔
95	*b = incr;	41✔
96	if !overflow {	41✔
97	return Some(Self {	38✔
98	dtype: self.dtype,	38✔
99	value: Some(Arc::new(sliced_mut.freeze())),	38✔
100	});	38✔
101	}	3✔
102	}
103	None	1✔
104	} else {
UNCOV 105	Some(Self {	×
UNCOV 106	dtype: self.dtype,	×
UNCOV 107	value: Some(value),	×
UNCOV 108	})	×
109	}
110	} else {
UNCOV 111	Some(self)	×
112	}
113	}	39✔
114
115	/// Construct a value at most `max_length` in size that's less than ourselves.
116	pub fn lower_bound(self, max_length: usize) -> Self {	38✔
117	if let Some(value) = self.value {	38✔
118	if value.len() > max_length {	38✔
119	Self {	38✔
120	dtype: self.dtype,	38✔
121	value: Some(Arc::new(value.slice(0..max_length))),	38✔
122	}	38✔
123	} else {
124	Self {	×
125	dtype: self.dtype,	×
126	value: Some(value),	×
127	}	×
128	}
129	} else {
130	self	×
131	}
132	}	38✔
133
134	pub(crate) fn cast(&self, dtype: &DType) -> VortexResult<Scalar> {	×
135	if !matches!(dtype, DType::Binary(..)) {	×
136	vortex_bail!("Can't cast binary to {}", dtype)	×
UNCOV 137	}	×
UNCOV 138	Ok(Scalar::new(	×
UNCOV 139	dtype.clone(),	×
UNCOV 140	ScalarValue(InnerScalarValue::Buffer(	×
UNCOV 141	self.value	×
UNCOV 142	.as_ref()	×
UNCOV 143	.vortex_expect("nullness handled in Scalar::cast")	×
UNCOV 144	.clone(),	×
UNCOV 145	)),	×
UNCOV 146	))	×
UNCOV 147	}	×
148
149	/// Length of the scalar value or None if value is null
150	pub fn len(&self) -> Option<usize> {	148✔
151	self.value.as_ref().map(\|v\| v.len())	148✔
152	}	148✔
153
154	/// Returns whether its value is non-null and empty, otherwise `None`.
155	pub fn is_empty(&self) -> Option<bool> {	244✔
156	self.value.as_ref().map(\|v\| v.is_empty())	244✔
157	}	244✔
158
159	/// Extract value as a ScalarValue
160	pub fn into_value(self) -> ScalarValue {	148✔
161	ScalarValue(
162	self.value	148✔
163	.map(InnerScalarValue::Buffer)	148✔
164	.unwrap_or_else(\|\| InnerScalarValue::Null),	148✔
165	)
166	}	148✔
167	}
168
169	impl Scalar {
170	/// Creates a new binary scalar from a byte buffer.
171	pub fn binary(buffer: impl Into<Arc<ByteBuffer>>, nullability: Nullability) -> Self {	1,093✔
172	Self {	1,093✔
173	dtype: DType::Binary(nullability),	1,093✔
174	value: ScalarValue(InnerScalarValue::Buffer(buffer.into())),	1,093✔
175	}	1,093✔
176	}	1,093✔
177	}
178
179	impl<'a> TryFrom<&'a Scalar> for BinaryScalar<'a> {
180	type Error = VortexError;
181
182	fn try_from(value: &'a Scalar) -> Result<Self, Self::Error> {	1,844✔
183	if !matches!(value.dtype(), DType::Binary(_)) {	1,844✔
184	vortex_bail!("Expected binary scalar, found {}", value.dtype())	×
185	}	1,844✔
186	Ok(Self {
187	dtype: value.dtype(),	1,844✔
188	value: value.value.as_buffer()?,	1,844✔
189	})
190	}	1,844✔
191	}
192
193	impl<'a> TryFrom<&'a Scalar> for ByteBuffer {
194	type Error = VortexError;
195
UNCOV 196	fn try_from(scalar: &'a Scalar) -> VortexResult<Self> {	×
UNCOV 197	let binary = scalar	×
198	.as_binary_opt()	×
199	.ok_or_else(\|\| vortex_err!("Cannot extract buffer from non-buffer scalar"))?;	×
200
201	binary	×
202	.value()	×
203	.ok_or_else(\|\| vortex_err!("Cannot extract present value from null scalar"))	×
UNCOV 204	}	×
205	}
206
207	impl<'a> TryFrom<&'a Scalar> for Option<ByteBuffer> {
208	type Error = VortexError;
209
210	fn try_from(scalar: &'a Scalar) -> VortexResult<Self> {	×
211	Ok(scalar	×
UNCOV 212	.as_binary_opt()	×
UNCOV 213	.ok_or_else(\|\| vortex_err!("Cannot extract buffer from non-buffer scalar"))?	×
UNCOV 214	.value())	×
UNCOV 215	}	×
216	}
217
218	impl TryFrom<Scalar> for ByteBuffer {
219	type Error = VortexError;
220
UNCOV 221	fn try_from(scalar: Scalar) -> VortexResult<Self> {	×
UNCOV 222	Self::try_from(&scalar)	×
223	}	×
224	}
225
226	impl TryFrom<Scalar> for Option<ByteBuffer> {
227	type Error = VortexError;
228
229	fn try_from(scalar: Scalar) -> VortexResult<Self> {	×
230	Self::try_from(&scalar)	×
231	}	×
232	}
233
234	impl From<&[u8]> for Scalar {
UNCOV 235	fn from(value: &[u8]) -> Self {	×
UNCOV 236	Scalar::from(ByteBuffer::from(value.to_vec()))	×
UNCOV 237	}	×
238	}
239
240	impl From<ByteBuffer> for Scalar {
241	fn from(value: ByteBuffer) -> Self {	×
242	Self {	×
243	dtype: DType::Binary(Nullability::NonNullable),	×
UNCOV 244	value: ScalarValue(InnerScalarValue::Buffer(Arc::new(value))),	×
UNCOV 245	}	×
UNCOV 246	}	×
247	}
248
249	impl From<Arc<ByteBuffer>> for Scalar {
UNCOV 250	fn from(value: Arc<ByteBuffer>) -> Self {	×
UNCOV 251	Self {	×
UNCOV 252	dtype: DType::Binary(Nullability::NonNullable),	×
UNCOV 253	value: ScalarValue(InnerScalarValue::Buffer(value)),	×
UNCOV 254	}	×
UNCOV 255	}	×
256	}
257
258	#[cfg(test)]
259	mod tests {
260	use vortex_buffer::buffer;
261	use vortex_dtype::Nullability;
262	use vortex_error::{VortexExpect, VortexUnwrap};
263
264	use crate::{BinaryScalar, Scalar};
265
266	#[test]
267	fn lower_bound() {	1✔
268	let binary = Scalar::binary(buffer![0u8, 5, 47, 33, 129], Nullability::NonNullable);	1✔
269	let expected = Scalar::binary(buffer![0u8, 5], Nullability::NonNullable);	1✔
270	assert_eq!(	1✔
271	BinaryScalar::try_from(&binary)	1✔
272	.vortex_unwrap()	1✔
273	.lower_bound(2),	1✔
274	BinaryScalar::try_from(&expected).vortex_unwrap()	1✔
275	);
276	}	1✔
277
278	#[test]
279	fn upper_bound() {	1✔
280	let binary = Scalar::binary(buffer![0u8, 5, 255, 234, 23], Nullability::NonNullable);	1✔
281	let expected = Scalar::binary(buffer![0u8, 6, 0], Nullability::NonNullable);	1✔
282	assert_eq!(	1✔
283	BinaryScalar::try_from(&binary)	1✔
284	.vortex_unwrap()	1✔
285	.upper_bound(3)	1✔
286	.vortex_expect("must have upper bound"),	1✔
287	BinaryScalar::try_from(&expected).vortex_unwrap()	1✔
288	);
289	}	1✔
290
291	#[test]
292	fn upper_bound_overflow() {	1✔
293	let binary = Scalar::binary(buffer![255u8, 255, 255], Nullability::NonNullable);	1✔
294	assert!(	1✔
295	BinaryScalar::try_from(&binary)	1✔
296	.vortex_unwrap()	1✔
297	.upper_bound(2)	1✔
298	.is_none()	1✔
299	);
300	}	1✔
301	}

vortex-data / vortex / 16356360776

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous