16992684502

Committed 15 Aug 2025 02:56PM UTC coverage: 87.875% (+0.2%) from 87.72%

Build # 16992684502

Build Type

Pull #2456

github

Committed by

web-flow

Commit Message

Merge 2d540e578 into 4a23f65b3

Pull Request Pull Request #2456: feat: basic BoolBuffer / BoolBufferMut

Run Details

1275 of 1428 new or added lines in 110 files covered. (89.29%)

334 existing lines in 31 files now uncovered.

57169 of 65057 relevant lines covered (87.88%)

658056.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.41

/vortex-array/src/builders/decimal.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::any::Any;

use vortex_buffer::{Buffer, BufferMut};
use vortex_dtype::{DType, DecimalDType, Nullability};
use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_panic};
use vortex_mask::Mask;
use vortex_scalar::{BigCast, NativeDecimalType, i256, match_each_decimal_value_type};

use crate::arrays::DecimalArray;
use crate::builders::ArrayBuilder;
use crate::builders::lazy_validity_builder::LazyBitBufferBuilder;
use crate::{Array, ArrayRef, IntoArray, ToCanonical};

/// Wrapper around the typed builder.
///
/// We want to be able to downcast a `Box<dyn ArrayBuilder>` to a [`DecimalBuilder`] and we generally
/// don't have enough type information to get the `T` at the call site, so we instead use this
/// to hold values and can push values into the correct buffer type generically.
enum DecimalBuffer {
    I8(BufferMut<i8>),
    I16(BufferMut<i16>),
    I32(BufferMut<i32>),
    I64(BufferMut<i64>),
    I128(BufferMut<i128>),
    I256(BufferMut<i256>),
}

impl Default for DecimalBuffer {
    fn default() -> Self {
        Self::I8(BufferMut::<i8>::empty())
    }
}

macro_rules! impl_from_buffer {
    ($typ:ty, $variant:ident) => {
        impl From<BufferMut<$typ>> for DecimalBuffer {
            fn from(buffer: BufferMut<$typ>) -> Self {
                Self::$variant(buffer)
            }
        }
    };
}
impl_from_buffer!(i8, I8);
impl_from_buffer!(i16, I16);
impl_from_buffer!(i32, I32);
impl_from_buffer!(i64, I64);
impl_from_buffer!(i128, I128);
impl_from_buffer!(i256, I256);

macro_rules! delegate_fn {
    ($self:expr, | $tname:ident, $buffer:ident | $body:block) => {{
        #[allow(unused)]
        match $self {
            DecimalBuffer::I8(buffer) => {
                type $tname = i8;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I16(buffer) => {
                type $tname = i16;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I32(buffer) => {
                type $tname = i32;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I64(buffer) => {
                type $tname = i64;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I128(buffer) => {
                type $tname = i128;
                let $buffer = buffer;
                $body
            }
            DecimalBuffer::I256(buffer) => {
                type $tname = i256;
                let $buffer = buffer;
                $body
            }
        }
    }};
}

impl DecimalBuffer {
    fn push<V: NativeDecimalType>(&mut self, value: V) {
        delegate_fn!(self, |T, buffer| {
            buffer.push(<T as BigCast>::from(value).vortex_expect("decimal conversion failure"))
        });
    }

    fn push_n<V: NativeDecimalType>(&mut self, value: V, n: usize) {
        delegate_fn!(self, |T, buffer| {
            buffer.push_n(
                <T as BigCast>::from(value).vortex_expect("decimal conversion failure"),
                n,
            )
        });
    }

    fn reserve(&mut self, additional: usize) {
        delegate_fn!(self, |T, buffer| { buffer.reserve(additional) })
    }

    fn capacity(&self) -> usize {
        delegate_fn!(self, |T, buffer| { buffer.capacity() })
    }

    fn len(&self) -> usize {
        delegate_fn!(self, |T, buffer| { buffer.len() })
    }

    pub fn extend<I, V: NativeDecimalType>(&mut self, iter: I)
    where
        I: Iterator<Item = V>,
    {
        delegate_fn!(self, |T, buffer| {
            buffer.extend(
                iter.map(|x| <T as BigCast>::from(x).vortex_expect("decimal conversion failure")),
            )
        })
    }
}

/// An [`ArrayBuilder`] for `Decimal` typed arrays.
///
/// The output will be a new [`DecimalArray`] holding values of `T`. Any value that is
/// a valid [decimal type][NativeDecimalType] can be appended to the builder and it will be
/// immediately coerced into the target type.
pub struct DecimalBuilder {
    values: DecimalBuffer,
    nulls: LazyBitBufferBuilder,
    dtype: DType,
}

const DEFAULT_BUILDER_CAPACITY: usize = 1024;

impl DecimalBuilder {
    pub fn new<T: NativeDecimalType>(precision: u8, scale: i8, nullability: Nullability) -> Self {
        Self::with_capacity::<T>(
            DEFAULT_BUILDER_CAPACITY,
            DecimalDType::new(precision, scale),
            nullability,
        )
    }

    pub fn with_capacity<T: NativeDecimalType>(
        capacity: usize,
        decimal: DecimalDType,
        nullability: Nullability,
    ) -> Self {
        Self {
            values: match_each_decimal_value_type!(T::VALUES_TYPE, |D| {
                DecimalBuffer::from(BufferMut::<D>::with_capacity(capacity))
            }),
            nulls: LazyBitBufferBuilder::new(capacity),
            dtype: DType::Decimal(decimal, nullability),
        }
    }
}

impl DecimalBuilder {
    fn extend_with_validity_mask(&mut self, validity_mask: Mask) {
        self.nulls.append_validity_mask(validity_mask);
    }

    /// Extend the values buffer from another buffer of type V where V can be coerced
    /// to the builder type.
    fn extend_from_buffer<V: NativeDecimalType>(&mut self, values: &Buffer<V>) {
        self.values.extend(values.iter().copied());
    }
}

impl DecimalBuilder {
    pub fn append_value<V: NativeDecimalType>(&mut self, value: V) {
        self.values.push(value);
        self.nulls.append(true);
    }

    pub fn append_option<V: NativeDecimalType>(&mut self, value: Option<V>) {
        match value {
            Some(value) => {
                self.values.push(value);
                self.nulls.append(true);
            }
            None => self.append_null(),
        }
    }

    /// Append a `Mask` to the null buffer.
    pub fn append_mask(&mut self, mask: Mask) {
        self.nulls.append_validity_mask(mask);
    }
}

impl DecimalBuilder {
    pub fn finish_into_decimal(&mut self) -> DecimalArray {
        let validity = self.nulls.finish_with_nullability(self.dtype.nullability());

        let DType::Decimal(decimal_dtype, _) = self.dtype else {
            vortex_panic!("DecimalBuilder must have Decimal DType");
        };

        delegate_fn!(std::mem::take(&mut self.values), |T, values| {
            DecimalArray::new::<T>(values.freeze(), decimal_dtype, validity)
        })
    }
}

impl ArrayBuilder for DecimalBuilder {
    fn as_any(&self) -> &dyn Any {
        self
    }

    fn as_any_mut(&mut self) -> &mut dyn Any {
        self
    }

    fn dtype(&self) -> &DType {
        &self.dtype
    }

    fn len(&self) -> usize {
        self.values.len()
    }

    fn append_zeros(&mut self, n: usize) {
        self.values.push_n(0, n);
        self.nulls.append_n_non_nulls(n);
    }

    fn append_nulls(&mut self, n: usize) {
        self.values.push_n(0, n);
        self.nulls.append_n_nulls(n);
    }

    fn extend_from_array(&mut self, array: &dyn Array) -> VortexResult<()> {
        let array = array.to_decimal()?;

        let DType::Decimal(decimal_dtype, _) = self.dtype else {
            vortex_panic!("DecimalBuilder must have Decimal DType");
        };

        if array.decimal_dtype() != decimal_dtype {
            vortex_bail!(
                "Cannot extend from array with different decimal type: {:?} != {:?}",
                array.decimal_dtype(),
                decimal_dtype
            );
        }

        match_each_decimal_value_type!(array.values_type(), |D| {
            self.extend_from_buffer(&array.buffer::<D>())
        });

        self.extend_with_validity_mask(array.validity_mask()?);

        Ok(())
    }

    fn ensure_capacity(&mut self, capacity: usize) {
        if capacity > self.values.capacity() {
            self.values.reserve(capacity - self.values.len());
            self.nulls.ensure_capacity(capacity);
        }
    }

    fn set_validity(&mut self, validity: Mask) {
        self.nulls = LazyBitBufferBuilder::new(validity.len());
        self.nulls.append_validity_mask(validity);
    }

    fn finish(&mut self) -> ArrayRef {
        self.finish_into_decimal().into_array()
    }
}

#[cfg(test)]
mod tests {
    use crate::builders::{ArrayBuilder, DecimalBuilder};

    #[test]
    fn test_mixed_extend() {
        let mut i8s = DecimalBuilder::new::<i8>(2, 1, false.into());
        i8s.append_value(10);
        i8s.append_value(11);
        i8s.append_value(12);
        let i8s = i8s.finish();

        let mut i128s = DecimalBuilder::new::<i128>(2, 1, false.into());
        i128s.extend_from_array(&i8s).unwrap();
        let i128s = i128s.finish_into_decimal();
        assert_eq!(i128s.buffer::<i128>().as_slice(), &[10, 11, 12]);
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::any::Any;
5
6	use vortex_buffer::{Buffer, BufferMut};
7	use vortex_dtype::{DType, DecimalDType, Nullability};
8	use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_panic};
9	use vortex_mask::Mask;
10	use vortex_scalar::{BigCast, NativeDecimalType, i256, match_each_decimal_value_type};
11
12	use crate::arrays::DecimalArray;
13	use crate::builders::ArrayBuilder;
14	use crate::builders::lazy_validity_builder::LazyBitBufferBuilder;
15	use crate::{Array, ArrayRef, IntoArray, ToCanonical};
16
17	/// Wrapper around the typed builder.
18	///
19	/// We want to be able to downcast a `Box<dyn ArrayBuilder>` to a [`DecimalBuilder`] and we generally
20	/// don't have enough type information to get the `T` at the call site, so we instead use this
21	/// to hold values and can push values into the correct buffer type generically.
22	enum DecimalBuffer {
23	I8(BufferMut<i8>),
24	I16(BufferMut<i16>),
25	I32(BufferMut<i32>),
26	I64(BufferMut<i64>),
27	I128(BufferMut<i128>),
28	I256(BufferMut<i256>),
29	}
30
31	impl Default for DecimalBuffer {
32	fn default() -> Self {	2,602✔
33	Self::I8(BufferMut::<i8>::empty())	2,602✔
34	}	2,602✔
35	}
36
37	macro_rules! impl_from_buffer {
38	($typ:ty, $variant:ident) => {
39	impl From<BufferMut<$typ>> for DecimalBuffer {
40	fn from(buffer: BufferMut<$typ>) -> Self {	2,602✔
41	Self::$variant(buffer)	2,602✔
42	}	2,602✔
43	}
44	};
45	}
46	impl_from_buffer!(i8, I8);
47	impl_from_buffer!(i16, I16);
48	impl_from_buffer!(i32, I32);
49	impl_from_buffer!(i64, I64);
50	impl_from_buffer!(i128, I128);
51	impl_from_buffer!(i256, I256);
52
53	macro_rules! delegate_fn {
54	($self:expr, \| $tname:ident, $buffer:ident \| $body:block) => {{
55	#[allow(unused)]
56	match $self {
57	DecimalBuffer::I8(buffer) => {
58	type $tname = i8;
59	let $buffer = buffer;
60	$body
61	}
62	DecimalBuffer::I16(buffer) => {
63	type $tname = i16;
64	let $buffer = buffer;
65	$body
66	}
67	DecimalBuffer::I32(buffer) => {
68	type $tname = i32;
69	let $buffer = buffer;
70	$body
71	}
72	DecimalBuffer::I64(buffer) => {
73	type $tname = i64;
74	let $buffer = buffer;
75	$body
76	}
77	DecimalBuffer::I128(buffer) => {
78	type $tname = i128;
79	let $buffer = buffer;
80	$body
81	}
82	DecimalBuffer::I256(buffer) => {
83	type $tname = i256;
84	let $buffer = buffer;
85	$body
86	}
87	}
88	}};
89	}
90
91	impl DecimalBuffer {
92	fn push<V: NativeDecimalType>(&mut self, value: V) {	3,273✔
93	delegate_fn!(self, \|T, buffer\| {	3,273✔
94	buffer.push(<T as BigCast>::from(value).vortex_expect("decimal conversion failure"))	165✔
95	});
96	}	3,273✔
97
98	fn push_n<V: NativeDecimalType>(&mut self, value: V, n: usize) {	×
99	delegate_fn!(self, \|T, buffer\| {	×
100	buffer.push_n(	×
101	<T as BigCast>::from(value).vortex_expect("decimal conversion failure"),	×
102	n,	×
103	)
104	});
105	}	×
106
107	fn reserve(&mut self, additional: usize) {	×
108	delegate_fn!(self, \|T, buffer\| { buffer.reserve(additional) })	×
109	}	×
110
111	fn capacity(&self) -> usize {	×
112	delegate_fn!(self, \|T, buffer\| { buffer.capacity() })	×
113	}	×
114
115	fn len(&self) -> usize {	2,262✔
116	delegate_fn!(self, \|T, buffer\| { buffer.len() })	2,262✔
117	}	2,262✔
118
119	pub fn extend<I, V: NativeDecimalType>(&mut self, iter: I)	991✔
120	where	991✔
121	I: Iterator<Item = V>,	991✔
122	{
123	delegate_fn!(self, \|T, buffer\| {	991✔
124	buffer.extend(	×
125	iter.map(\|x\| <T as BigCast>::from(x).vortex_expect("decimal conversion failure")),	8,001,867✔
126	)
127	})
128	}	991✔
129	}
130
131	/// An [`ArrayBuilder`] for `Decimal` typed arrays.
132	///
133	/// The output will be a new [`DecimalArray`] holding values of `T`. Any value that is
134	/// a valid [decimal type][NativeDecimalType] can be appended to the builder and it will be
135	/// immediately coerced into the target type.
136	pub struct DecimalBuilder {
137	values: DecimalBuffer,
138	nulls: LazyBitBufferBuilder,
139	dtype: DType,
140	}
141
142	const DEFAULT_BUILDER_CAPACITY: usize = 1024;
143
144	impl DecimalBuilder {
145	pub fn new<T: NativeDecimalType>(precision: u8, scale: i8, nullability: Nullability) -> Self {	14✔
146	Self::with_capacity::<T>(	14✔
147	DEFAULT_BUILDER_CAPACITY,
148	DecimalDType::new(precision, scale),	14✔
149	nullability,	14✔
150	)
151	}	14✔
152
153	pub fn with_capacity<T: NativeDecimalType>(	2,602✔
154	capacity: usize,	2,602✔
155	decimal: DecimalDType,	2,602✔
156	nullability: Nullability,	2,602✔
157	) -> Self {	2,602✔
158	Self {
159	values: match_each_decimal_value_type!(T::VALUES_TYPE, \|D\| {	2,602✔
160	DecimalBuffer::from(BufferMut::<D>::with_capacity(capacity))	159✔
161	}),
162	nulls: LazyBitBufferBuilder::new(capacity),	2,602✔
163	dtype: DType::Decimal(decimal, nullability),	2,602✔
164	}
165	}	2,602✔
166	}
167
168	impl DecimalBuilder {
169	fn extend_with_validity_mask(&mut self, validity_mask: Mask) {	991✔
170	self.nulls.append_validity_mask(validity_mask);	991✔
171	}	991✔
172
173	/// Extend the values buffer from another buffer of type V where V can be coerced
174	/// to the builder type.
175	fn extend_from_buffer<V: NativeDecimalType>(&mut self, values: &Buffer<V>) {	991✔
176	self.values.extend(values.iter().copied());	991✔
177	}	991✔
178	}
179
180	impl DecimalBuilder {
181	pub fn append_value<V: NativeDecimalType>(&mut self, value: V) {	3,273✔
182	self.values.push(value);	3,273✔
183	self.nulls.append(true);	3,273✔
184	}	3,273✔
185
186	pub fn append_option<V: NativeDecimalType>(&mut self, value: Option<V>) {	×
187	match value {	×
188	Some(value) => {	×
189	self.values.push(value);	×
190	self.nulls.append(true);	×
191	}	×
192	None => self.append_null(),	×
193	}
194	}	×
195
196	/// Append a `Mask` to the null buffer.
197	pub fn append_mask(&mut self, mask: Mask) {	×
198	self.nulls.append_validity_mask(mask);	×
199	}	×
200	}
201
202	impl DecimalBuilder {
203	pub fn finish_into_decimal(&mut self) -> DecimalArray {	2,602✔
204	let validity = self.nulls.finish_with_nullability(self.dtype.nullability());	2,602✔
205
206	let DType::Decimal(decimal_dtype, _) = self.dtype else {	2,602✔
207	vortex_panic!("DecimalBuilder must have Decimal DType");	×
208	};
209
210	delegate_fn!(std::mem::take(&mut self.values), \|T, values\| {	2,602✔
211	DecimalArray::new::<T>(values.freeze(), decimal_dtype, validity)	159✔
212	})
213	}	2,602✔
214	}
215
216	impl ArrayBuilder for DecimalBuilder {
217	fn as_any(&self) -> &dyn Any {	×
218	self	×
219	}	×
220
221	fn as_any_mut(&mut self) -> &mut dyn Any {	2,844✔
222	self	2,844✔
223	}	2,844✔
224
225	fn dtype(&self) -> &DType {	3,975✔
226	&self.dtype	3,975✔
227	}	3,975✔
228
229	fn len(&self) -> usize {	2,262✔
230	self.values.len()	2,262✔
231	}	2,262✔
232
233	fn append_zeros(&mut self, n: usize) {	×
234	self.values.push_n(0, n);	×
235	self.nulls.append_n_non_nulls(n);	×
236	}	×
237
238	fn append_nulls(&mut self, n: usize) {	×
239	self.values.push_n(0, n);	×
240	self.nulls.append_n_nulls(n);	×
241	}	×
242
243	fn extend_from_array(&mut self, array: &dyn Array) -> VortexResult<()> {	991✔
244	let array = array.to_decimal()?;	991✔
245
246	let DType::Decimal(decimal_dtype, _) = self.dtype else {	991✔
247	vortex_panic!("DecimalBuilder must have Decimal DType");	×
248	};
249
250	if array.decimal_dtype() != decimal_dtype {	991✔
251	vortex_bail!(	×
252	"Cannot extend from array with different decimal type: {:?} != {:?}",	×
253	array.decimal_dtype(),	×
254	decimal_dtype
255	);
256	}	991✔
257
258	match_each_decimal_value_type!(array.values_type(), \|D\| {	991✔
259	self.extend_from_buffer(&array.buffer::<D>())	1✔
260	});
261
262	self.extend_with_validity_mask(array.validity_mask()?);	991✔
263
264	Ok(())	991✔
265	}	991✔
266
267	fn ensure_capacity(&mut self, capacity: usize) {	×
268	if capacity > self.values.capacity() {	×
269	self.values.reserve(capacity - self.values.len());	×
270	self.nulls.ensure_capacity(capacity);	×
271	}	×
272	}	×
273
274	fn set_validity(&mut self, validity: Mask) {	×
NEW 275	self.nulls = LazyBitBufferBuilder::new(validity.len());	×
276	self.nulls.append_validity_mask(validity);	×
277	}	×
278
279	fn finish(&mut self) -> ArrayRef {	2,562✔
280	self.finish_into_decimal().into_array()	2,562✔
281	}	2,562✔
282	}
283
284	#[cfg(test)]
285	mod tests {
286	use crate::builders::{ArrayBuilder, DecimalBuilder};
287
288	#[test]
289	fn test_mixed_extend() {	1✔
290	let mut i8s = DecimalBuilder::new::<i8>(2, 1, false.into());	1✔
291	i8s.append_value(10);	1✔
292	i8s.append_value(11);	1✔
293	i8s.append_value(12);	1✔
294	let i8s = i8s.finish();	1✔
295
296	let mut i128s = DecimalBuilder::new::<i128>(2, 1, false.into());	1✔
297	i128s.extend_from_array(&i8s).unwrap();	1✔
298	let i128s = i128s.finish_into_decimal();	1✔
299	assert_eq!(i128s.buffer::<i128>().as_slice(), &[10, 11, 12]);	1✔
300	}	1✔
301	}

vortex-data / vortex / 16992684502

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous