16578751259

Committed 28 Jul 2025 07:46PM UTC coverage: 81.858% (+0.8%) from 81.087%

Build # 16578751259

Build Type

Pull #3992

github

Committed by

web-flow

Commit Message

Merge 0fe171459 into 1ae560509

Pull Request Pull Request #3992: feat: teach SparseArray to canonicalize lists

Run Details

222 of 241 new or added lines in 1 file covered. (92.12%)

495 existing lines in 40 files now uncovered.

43529 of 53176 relevant lines covered (81.86%)

169875.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.41

/vortex-array/src/builders/decimal.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::any::Any;

use vortex_buffer::{Buffer, BufferMut};
use vortex_dtype::{DType, DecimalDType, Nullability};
use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_panic};
use vortex_mask::Mask;
use vortex_scalar::{BigCast, NativeDecimalType, i256, match_each_decimal_value_type};

use crate::arrays::DecimalArray;
use crate::builders::ArrayBuilder;
use crate::builders::lazy_validity_builder::LazyNullBufferBuilder;
use crate::{Array, ArrayRef, IntoArray, ToCanonical};

/// Wrapper around the typed builder.
///
/// We want to be able to downcast a `Box<dyn ArrayBuilder>` to a [`DecimalBuilder`] and we generally
/// don't have enough type information to get the `T` at the call site, so we instead use this
/// to hold values and can push values into the correct buffer type generically.
enum DecimalBuffer {
    I8(BufferMut<i8>),
    I16(BufferMut<i16>),
    I32(BufferMut<i32>),
    I64(BufferMut<i64>),
    I128(BufferMut<i128>),
    I256(BufferMut<i256>),
}

impl Default for DecimalBuffer {
    fn default() -> Self {
        Self::I8(BufferMut::<i8>::empty())
    }
}

macro_rules! impl_from_buffer {
    ($typ:ty, $variant:ident) => {
        impl From<BufferMut<$typ>> for DecimalBuffer {
            fn from(buffer: BufferMut<$typ>) -> Self {
                Self::$variant(buffer)
            }
        }
    };
}
impl_from_buffer!(i8, I8);
impl_from_buffer!(i16, I16);
impl_from_buffer!(i32, I32);
impl_from_buffer!(i64, I64);
impl_from_buffer!(i128, I128);
impl_from_buffer!(i256, I256);

macro_rules! delegate_fn {
    ($self:expr, | $tname:ident, $buffer:ident | $body:block) => {{
        #[allow(unused)]
        match $self {
            DecimalBuffer::I8(buffer) => {
                type $tname = i8;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I16(buffer) => {
                type $tname = i16;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I32(buffer) => {
                type $tname = i32;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I64(buffer) => {
                type $tname = i64;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I128(buffer) => {
                type $tname = i128;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I256(buffer) => {
                type $tname = i256;
                let $buffer = buffer;
                $body
            }
        }
    }};
}

impl DecimalBuffer {
    fn push<V: NativeDecimalType>(&mut self, value: V) {
        delegate_fn!(self, |T, buffer| {
            buffer.push(<T as BigCast>::from(value).vortex_expect("decimal conversion failure"))
        });
    }

    fn push_n<V: NativeDecimalType>(&mut self, value: V, n: usize) {
        delegate_fn!(self, |T, buffer| {
            buffer.push_n(
                <T as BigCast>::from(value).vortex_expect("decimal conversion failure"),
                n,
            )
        });
    }

    fn reserve(&mut self, additional: usize) {
        delegate_fn!(self, |T, buffer| { buffer.reserve(additional) })
    }

    fn capacity(&self) -> usize {
        delegate_fn!(self, |T, buffer| { buffer.capacity() })
    }

    fn len(&self) -> usize {
        delegate_fn!(self, |T, buffer| { buffer.len() })
    }

    pub fn extend<I, V: NativeDecimalType>(&mut self, iter: I)
    where
        I: Iterator<Item = V>,
    {
        delegate_fn!(self, |T, buffer| {
            buffer.extend(
                iter.map(|x| <T as BigCast>::from(x).vortex_expect("decimal conversion failure")),
            )
        })
    }
}

/// An [`ArrayBuilder`] for `Decimal` typed arrays.
///
/// The output will be a new [`DecimalArray`] holding values of `T`. Any value that is
/// a valid [decimal type][NativeDecimalType] can be appended to the builder and it will be
/// immediately coerced into the target type.
pub struct DecimalBuilder {
    values: DecimalBuffer,
    nulls: LazyNullBufferBuilder,
    dtype: DType,
}

const DEFAULT_BUILDER_CAPACITY: usize = 1024;

impl DecimalBuilder {
    pub fn new<T: NativeDecimalType>(precision: u8, scale: i8, nullability: Nullability) -> Self {
        Self::with_capacity::<T>(
            DEFAULT_BUILDER_CAPACITY,
            DecimalDType::new(precision, scale),
            nullability,
        )
    }

    pub fn with_capacity<T: NativeDecimalType>(
        capacity: usize,
        decimal: DecimalDType,
        nullability: Nullability,
    ) -> Self {
        Self {
            values: match_each_decimal_value_type!(T::VALUES_TYPE, |D| {
                DecimalBuffer::from(BufferMut::<D>::with_capacity(capacity))
            }),
            nulls: LazyNullBufferBuilder::new(capacity),
            dtype: DType::Decimal(decimal, nullability),
        }
    }
}

impl DecimalBuilder {
    fn extend_with_validity_mask(&mut self, validity_mask: Mask) {
        self.nulls.append_validity_mask(validity_mask);
    }

    /// Extend the values buffer from another buffer of type V where V can be coerced
    /// to the builder type.
    fn extend_from_buffer<V: NativeDecimalType>(&mut self, values: &Buffer<V>) {
        self.values.extend(values.iter().copied());
    }
}

impl DecimalBuilder {
    pub fn append_value<V: NativeDecimalType>(&mut self, value: V) {
        self.values.push(value);
        self.nulls.append(true);
    }

    pub fn append_option<V: NativeDecimalType>(&mut self, value: Option<V>) {
        match value {
            Some(value) => {
                self.values.push(value);
                self.nulls.append(true);
            }
            None => self.append_null(),
        }
    }

    /// Append a `Mask` to the null buffer.
    pub fn append_mask(&mut self, mask: Mask) {
        self.nulls.append_validity_mask(mask);
    }
}

impl DecimalBuilder {
    pub fn finish_into_decimal(&mut self) -> DecimalArray {
        let validity = self.nulls.finish_with_nullability(self.dtype.nullability());

        let DType::Decimal(decimal_dtype, _) = self.dtype else {
            vortex_panic!("DecimalBuilder must have Decimal DType");
        };

        delegate_fn!(std::mem::take(&mut self.values), |T, values| {
            DecimalArray::new::<T>(values.freeze(), decimal_dtype, validity)
        })
    }
}

impl ArrayBuilder for DecimalBuilder {
    fn as_any(&self) -> &dyn Any {
        self
    }

    fn as_any_mut(&mut self) -> &mut dyn Any {
        self
    }

    fn dtype(&self) -> &DType {
        &self.dtype
    }

    fn len(&self) -> usize {
        self.values.len()
    }

    fn append_zeros(&mut self, n: usize) {
        self.values.push_n(0, n);
        self.nulls.append_n_non_nulls(n);
    }

    fn append_nulls(&mut self, n: usize) {
        self.values.push_n(0, n);
        self.nulls.append_n_nulls(n);
    }

    fn extend_from_array(&mut self, array: &dyn Array) -> VortexResult<()> {
        let array = array.to_decimal()?;

        let DType::Decimal(decimal_dtype, _) = self.dtype else {
            vortex_panic!("DecimalBuilder must have Decimal DType");
        };

        if array.decimal_dtype() != decimal_dtype {
            vortex_bail!(
                "Cannot extend from array with different decimal type: {:?} != {:?}",
                array.decimal_dtype(),
                decimal_dtype
            );
        }

        match_each_decimal_value_type!(array.values_type(), |D| {
            self.extend_from_buffer(&array.buffer::<D>())
        });

        self.extend_with_validity_mask(array.validity_mask()?);

        Ok(())
    }

    fn ensure_capacity(&mut self, capacity: usize) {
        if capacity > self.values.capacity() {
            self.values.reserve(capacity - self.values.len());
            self.nulls.ensure_capacity(capacity);
        }
    }

    fn set_validity(&mut self, validity: Mask) {
        self.nulls = LazyNullBufferBuilder::new(validity.len());
        self.nulls.append_validity_mask(validity);
    }

    fn finish(&mut self) -> ArrayRef {
        self.finish_into_decimal().into_array()
    }
}

#[cfg(test)]
mod tests {
    use crate::builders::{ArrayBuilder, DecimalBuilder};

    #[test]
    fn test_mixed_extend() {
        let mut i8s = DecimalBuilder::new::<i8>(2, 1, false.into());
        i8s.append_value(10);
        i8s.append_value(11);
        i8s.append_value(12);
        let i8s = i8s.finish();

        let mut i128s = DecimalBuilder::new::<i128>(2, 1, false.into());
        i128s.extend_from_array(&i8s).unwrap();
        let i128s = i128s.finish_into_decimal();
        assert_eq!(i128s.buffer::<i128>().as_slice(), &[10, 11, 12]);
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::any::Any;
5
6	use vortex_buffer::{Buffer, BufferMut};
7	use vortex_dtype::{DType, DecimalDType, Nullability};
8	use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_panic};
9	use vortex_mask::Mask;
10	use vortex_scalar::{BigCast, NativeDecimalType, i256, match_each_decimal_value_type};
11
12	use crate::arrays::DecimalArray;
13	use crate::builders::ArrayBuilder;
14	use crate::builders::lazy_validity_builder::LazyNullBufferBuilder;
15	use crate::{Array, ArrayRef, IntoArray, ToCanonical};
16
17	/// Wrapper around the typed builder.
18	///
19	/// We want to be able to downcast a `Box<dyn ArrayBuilder>` to a [`DecimalBuilder`] and we generally
20	/// don't have enough type information to get the `T` at the call site, so we instead use this
21	/// to hold values and can push values into the correct buffer type generically.
22	enum DecimalBuffer {
23	I8(BufferMut<i8>),
24	I16(BufferMut<i16>),
25	I32(BufferMut<i32>),
26	I64(BufferMut<i64>),
27	I128(BufferMut<i128>),
28	I256(BufferMut<i256>),
29	}
30
31	impl Default for DecimalBuffer {
32	fn default() -> Self {	971✔
33	Self::I8(BufferMut::<i8>::empty())	971✔
34	}	971✔
35	}
36
37	macro_rules! impl_from_buffer {
38	($typ:ty, $variant:ident) => {
39	impl From<BufferMut<$typ>> for DecimalBuffer {
40	fn from(buffer: BufferMut<$typ>) -> Self {	971✔
41	Self::$variant(buffer)	971✔
42	}	971✔
43	}
44	};
45	}
46	impl_from_buffer!(i8, I8);
47	impl_from_buffer!(i16, I16);
48	impl_from_buffer!(i32, I32);
49	impl_from_buffer!(i64, I64);
50	impl_from_buffer!(i128, I128);
51	impl_from_buffer!(i256, I256);
52
53	macro_rules! delegate_fn {
54	($self:expr, \| $tname:ident, $buffer:ident \| $body:block) => {{
55	#[allow(unused)]
56	match $self {
57	DecimalBuffer::I8(buffer) => {
58	type $tname = i8;
59	let $buffer = buffer;
60	$body
61	}
62	DecimalBuffer::I16(buffer) => {
63	type $tname = i16;
64	let $buffer = buffer;
65	$body
66	}
67	DecimalBuffer::I32(buffer) => {
68	type $tname = i32;
69	let $buffer = buffer;
70	$body
71	}
72	DecimalBuffer::I64(buffer) => {
73	type $tname = i64;
74	let $buffer = buffer;
75	$body
76	}
77	DecimalBuffer::I128(buffer) => {
78	type $tname = i128;
79	let $buffer = buffer;
80	$body
81	}
82	DecimalBuffer::I256(buffer) => {
83	type $tname = i256;
84	let $buffer = buffer;
85	$body
86	}
87	}
88	}};
89	}
90
91	impl DecimalBuffer {
92	fn push<V: NativeDecimalType>(&mut self, value: V) {	2,525✔
93	delegate_fn!(self, \|T, buffer\| {	2,525✔
94	buffer.push(<T as BigCast>::from(value).vortex_expect("decimal conversion failure"))	157✔
95	});
96	}	2,525✔
97
UNCOV 98	fn push_n<V: NativeDecimalType>(&mut self, value: V, n: usize) {	×
99	delegate_fn!(self, \|T, buffer\| {	×
100	buffer.push_n(	×
101	<T as BigCast>::from(value).vortex_expect("decimal conversion failure"),	×
102	n,	×
103	)
104	});
UNCOV 105	}	×
106
UNCOV 107	fn reserve(&mut self, additional: usize) {	×
108	delegate_fn!(self, \|T, buffer\| { buffer.reserve(additional) })	×
109	}	×
110
UNCOV 111	fn capacity(&self) -> usize {	×
112	delegate_fn!(self, \|T, buffer\| { buffer.capacity() })	×
113	}	×
114
115	fn len(&self) -> usize {	1,508✔
116	delegate_fn!(self, \|T, buffer\| { buffer.len() })	1,508✔
117	}	1,508✔
118
119	pub fn extend<I, V: NativeDecimalType>(&mut self, iter: I)	661✔
120	where	661✔
121	I: Iterator<Item = V>,	661✔
122	{
123	delegate_fn!(self, \|T, buffer\| {	661✔
UNCOV 124	buffer.extend(	×
125	iter.map(\|x\| <T as BigCast>::from(x).vortex_expect("decimal conversion failure")),	5,334,579✔
126	)
127	})
128	}	661✔
129	}
130
131	/// An [`ArrayBuilder`] for `Decimal` typed arrays.
132	///
133	/// The output will be a new [`DecimalArray`] holding values of `T`. Any value that is
134	/// a valid [decimal type][NativeDecimalType] can be appended to the builder and it will be
135	/// immediately coerced into the target type.
136	pub struct DecimalBuilder {
137	values: DecimalBuffer,
138	nulls: LazyNullBufferBuilder,
139	dtype: DType,
140	}
141
142	const DEFAULT_BUILDER_CAPACITY: usize = 1024;
143
144	impl DecimalBuilder {
145	pub fn new<T: NativeDecimalType>(precision: u8, scale: i8, nullability: Nullability) -> Self {	14✔
146	Self::with_capacity::<T>(	14✔
147	DEFAULT_BUILDER_CAPACITY,
148	DecimalDType::new(precision, scale),	14✔
149	nullability,	14✔
150	)
151	}	14✔
152
153	pub fn with_capacity<T: NativeDecimalType>(	971✔
154	capacity: usize,	971✔
155	decimal: DecimalDType,	971✔
156	nullability: Nullability,	971✔
157	) -> Self {	971✔
158	Self {
159	values: match_each_decimal_value_type!(T::VALUES_TYPE, \|D\| {	971✔
160	DecimalBuffer::from(BufferMut::<D>::with_capacity(capacity))	151✔
161	}),
162	nulls: LazyNullBufferBuilder::new(capacity),	971✔
163	dtype: DType::Decimal(decimal, nullability),	971✔
164	}
165	}	971✔
166	}
167
168	impl DecimalBuilder {
169	fn extend_with_validity_mask(&mut self, validity_mask: Mask) {	661✔
170	self.nulls.append_validity_mask(validity_mask);	661✔
171	}	661✔
172
173	/// Extend the values buffer from another buffer of type V where V can be coerced
174	/// to the builder type.
175	fn extend_from_buffer<V: NativeDecimalType>(&mut self, values: &Buffer<V>) {	661✔
176	self.values.extend(values.iter().copied());	661✔
177	}	661✔
178	}
179
180	impl DecimalBuilder {
181	pub fn append_value<V: NativeDecimalType>(&mut self, value: V) {	2,525✔
182	self.values.push(value);	2,525✔
183	self.nulls.append(true);	2,525✔
184	}	2,525✔
185
UNCOV 186	pub fn append_option<V: NativeDecimalType>(&mut self, value: Option<V>) {	×
187	match value {	×
188	Some(value) => {	×
189	self.values.push(value);	×
190	self.nulls.append(true);	×
191	}	×
192	None => self.append_null(),	×
193	}
UNCOV 194	}	×
195
196	/// Append a `Mask` to the null buffer.
UNCOV 197	pub fn append_mask(&mut self, mask: Mask) {	×
198	self.nulls.append_validity_mask(mask);	×
199	}	×
200	}
201
202	impl DecimalBuilder {
203	pub fn finish_into_decimal(&mut self) -> DecimalArray {	971✔
204	let validity = self.nulls.finish_with_nullability(self.dtype.nullability());	971✔
205
206	let DType::Decimal(decimal_dtype, _) = self.dtype else {	971✔
UNCOV 207	vortex_panic!("DecimalBuilder must have Decimal DType");	×
208	};
209
210	delegate_fn!(std::mem::take(&mut self.values), \|T, values\| {	971✔
211	DecimalArray::new::<T>(values.freeze(), decimal_dtype, validity)	151✔
212	})
213	}	971✔
214	}
215
216	impl ArrayBuilder for DecimalBuilder {
UNCOV 217	fn as_any(&self) -> &dyn Any {	×
218	self	×
UNCOV 219	}	×
220
221	fn as_any_mut(&mut self) -> &mut dyn Any {	2,116✔
222	self	2,116✔
223	}	2,116✔
224
225	fn dtype(&self) -> &DType {	4,986✔
226	&self.dtype	4,986✔
227	}	4,986✔
228
229	fn len(&self) -> usize {	1,508✔
230	self.values.len()	1,508✔
231	}	1,508✔
232
UNCOV 233	fn append_zeros(&mut self, n: usize) {	×
UNCOV 234	self.values.push_n(0, n);	×
UNCOV 235	self.nulls.append_n_non_nulls(n);	×
UNCOV 236	}	×
237
UNCOV 238	fn append_nulls(&mut self, n: usize) {	×
UNCOV 239	self.values.push_n(0, n);	×
UNCOV 240	self.nulls.append_n_nulls(n);	×
241	}	×
242
243	fn extend_from_array(&mut self, array: &dyn Array) -> VortexResult<()> {	661✔
244	let array = array.to_decimal()?;	661✔
245
246	let DType::Decimal(decimal_dtype, _) = self.dtype else {	661✔
UNCOV 247	vortex_panic!("DecimalBuilder must have Decimal DType");	×
248	};
249
250	if array.decimal_dtype() != decimal_dtype {	661✔
UNCOV 251	vortex_bail!(	×
UNCOV 252	"Cannot extend from array with different decimal type: {:?} != {:?}",	×
UNCOV 253	array.decimal_dtype(),	×
254	decimal_dtype
255	);
256	}	661✔
257
258	match_each_decimal_value_type!(array.values_type(), \|D\| {	661✔
259	self.extend_from_buffer(&array.buffer::<D>())	1✔
260	});
261
262	self.extend_with_validity_mask(array.validity_mask()?);	661✔
263
264	Ok(())	661✔
265	}	661✔
266
UNCOV 267	fn ensure_capacity(&mut self, capacity: usize) {	×
UNCOV 268	if capacity > self.values.capacity() {	×
UNCOV 269	self.values.reserve(capacity - self.values.len());	×
UNCOV 270	self.nulls.ensure_capacity(capacity);	×
271	}	×
UNCOV 272	}	×
273
UNCOV 274	fn set_validity(&mut self, validity: Mask) {	×
275	self.nulls = LazyNullBufferBuilder::new(validity.len());	×
276	self.nulls.append_validity_mask(validity);	×
277	}	×
278
279	fn finish(&mut self) -> ArrayRef {	933✔
280	self.finish_into_decimal().into_array()	933✔
281	}	933✔
282	}
283
284	#[cfg(test)]
285	mod tests {
286	use crate::builders::{ArrayBuilder, DecimalBuilder};
287
288	#[test]
289	fn test_mixed_extend() {	1✔
290	let mut i8s = DecimalBuilder::new::<i8>(2, 1, false.into());	1✔
291	i8s.append_value(10);	1✔
292	i8s.append_value(11);	1✔
293	i8s.append_value(12);	1✔
294	let i8s = i8s.finish();	1✔
295
296	let mut i128s = DecimalBuilder::new::<i128>(2, 1, false.into());	1✔
297	i128s.extend_from_array(&i8s).unwrap();	1✔
298	let i128s = i128s.finish_into_decimal();	1✔
299	assert_eq!(i128s.buffer::<i128>().as_slice(), &[10, 11, 12]);	1✔
300	}	1✔
301	}

vortex-data / vortex / 16578751259

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous