16832413242

Committed 08 Aug 2025 02:04PM UTC coverage: 84.935% (+0.06%) from 84.877%

Build # 16832413242

Build Type

Pull #4161

github

Committed by

web-flow

Commit Message

Merge 04e9b0a07 into c88d9ada1

Pull Request Pull Request #4161: feat(python): Add Arrow FFI streaming support to write API

Run Details

50657 of 59642 relevant lines covered (84.94%)

568241.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.03

/vortex-array/src/builders/mod.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Builders for Vortex arrays.
//!
//! Every logical type in Vortex has a canonical (uncompressed) in-memory encoding. This module
//! provides pre-allocated builders to construct new canonical arrays.
//!
//! ## Example:
//!
//! ```
//! use vortex_array::builders::{builder_with_capacity, ArrayBuilderExt};
//! use vortex_dtype::{DType, Nullability};
//!
//! // Create a new builder for string data.
//! let mut builder = builder_with_capacity(&DType::Utf8(Nullability::NonNullable), 4);
//!
//! builder.append_scalar(&"a".into()).unwrap();
//! builder.append_scalar(&"b".into()).unwrap();
//! builder.append_scalar(&"c".into()).unwrap();
//! builder.append_scalar(&"d".into()).unwrap();
//!
//! let strings = builder.finish();
//!
//! assert_eq!(strings.scalar_at(0).unwrap(), "a".into());
//! assert_eq!(strings.scalar_at(1).unwrap(), "b".into());
//! assert_eq!(strings.scalar_at(2).unwrap(), "c".into());
//! assert_eq!(strings.scalar_at(3).unwrap(), "d".into());
//! ```

mod bool;
mod decimal;
mod extension;
mod lazy_validity_builder;
mod list;
mod null;
mod primitive;
mod struct_;
mod varbinview;

use std::any::Any;

pub use bool::*;
pub use decimal::*;
pub use extension::*;
pub use list::*;
pub use null::*;
pub use primitive::*;
pub use struct_::*;
pub use varbinview::*;
use vortex_dtype::{DType, match_each_native_ptype};
use vortex_error::{VortexResult, vortex_bail, vortex_err};
use vortex_mask::Mask;
use vortex_scalar::{
    BinaryScalar, BoolScalar, DecimalValue, ExtScalar, ListScalar, PrimitiveScalar, Scalar,
    ScalarValue, StructScalar, Utf8Scalar, match_each_decimal_value, match_each_decimal_value_type,
};

use crate::arrays::smallest_storage_type;
use crate::{Array, ArrayRef};

pub trait ArrayBuilder: Send {
    fn as_any(&self) -> &dyn Any;

    fn as_any_mut(&mut self) -> &mut dyn Any;

    fn dtype(&self) -> &DType;

    fn len(&self) -> usize;

    fn is_empty(&self) -> bool {
        self.len() == 0
    }

    /// Append a "zero" value to the array.
    fn append_zero(&mut self) {
        self.append_zeros(1)
    }

    /// Appends n "zero" values to the array.
    fn append_zeros(&mut self, n: usize);

    /// Append a "null" value to the array.
    fn append_null(&mut self) {
        self.append_nulls(1)
    }

    /// Appends n "null" values to the array.
    fn append_nulls(&mut self, n: usize);

    /// Extends the array with the provided array, canonicalizing if necessary.
    fn extend_from_array(&mut self, array: &dyn Array) -> VortexResult<()>;

    /// Ensure that the builder can hold at least `capacity` number of items
    fn ensure_capacity(&mut self, capacity: usize);

    /// Override builders validity with the one provided
    fn set_validity(&mut self, validity: Mask);

    /// Constructs an Array from the builder components.
    ///
    /// # Panics
    ///
    /// This function may panic if the builder's methods are called with invalid arguments. If only
    /// the methods on this interface are used, the builder should not panic. However, specific
    /// builders have interfaces that may be misused. For example, if the number of values in a
    /// [PrimitiveBuilder]'s [vortex_buffer::BufferMut] does not match the number of validity bits,
    /// the PrimitiveBuilder's [Self::finish] will panic.
    fn finish(&mut self) -> ArrayRef;
}

/// Construct a new canonical builder for the given [`DType`].
///
///
/// # Example
///
/// ```
/// use vortex_array::builders::{builder_with_capacity, ArrayBuilderExt};
/// use vortex_dtype::{DType, Nullability};
///
/// // Create a new builder for string data.
/// let mut builder = builder_with_capacity(&DType::Utf8(Nullability::NonNullable), 4);
///
/// builder.append_scalar(&"a".into()).unwrap();
/// builder.append_scalar(&"b".into()).unwrap();
/// builder.append_scalar(&"c".into()).unwrap();
/// builder.append_scalar(&"d".into()).unwrap();
///
/// let strings = builder.finish();
///
/// assert_eq!(strings.scalar_at(0).unwrap(), "a".into());
/// assert_eq!(strings.scalar_at(1).unwrap(), "b".into());
/// assert_eq!(strings.scalar_at(2).unwrap(), "c".into());
/// assert_eq!(strings.scalar_at(3).unwrap(), "d".into());
/// ```
pub fn builder_with_capacity(dtype: &DType, capacity: usize) -> Box<dyn ArrayBuilder> {
    match dtype {
        DType::Null => Box::new(NullBuilder::new()),
        DType::Bool(n) => Box::new(BoolBuilder::with_capacity(*n, capacity)),
        DType::Primitive(ptype, n) => {
            match_each_native_ptype!(ptype, |P| {
                Box::new(PrimitiveBuilder::<P>::with_capacity(*n, capacity))
            })
        }
        DType::Decimal(decimal_type, n) => {
            match_each_decimal_value_type!(smallest_storage_type(decimal_type), |D| {
                Box::new(DecimalBuilder::with_capacity::<D>(
                    capacity,
                    *decimal_type,
                    *n,
                ))
            })
        }
        DType::Utf8(n) => Box::new(VarBinViewBuilder::with_capacity(DType::Utf8(*n), capacity)),
        DType::Binary(n) => Box::new(VarBinViewBuilder::with_capacity(
            DType::Binary(*n),
            capacity,
        )),
        DType::Struct(struct_dtype, n) => Box::new(StructBuilder::with_capacity(
            struct_dtype.clone(),
            *n,
            capacity,
        )),
        DType::List(dtype, n) => Box::new(ListBuilder::<u64>::with_capacity(
            dtype.clone(),
            *n,
            capacity,
        )),
        DType::Extension(ext_dtype) => {
            Box::new(ExtensionBuilder::with_capacity(ext_dtype.clone(), capacity))
        }
    }
}

pub trait ArrayBuilderExt: ArrayBuilder {
    /// A generic function to append a scalar value to the builder.
    fn append_scalar_value(&mut self, value: ScalarValue) -> VortexResult<()> {
        if value.is_null() {
            self.append_null();
            Ok(())
        } else {
            self.append_scalar(&Scalar::new(self.dtype().clone(), value))
        }
    }

    /// A generic function to append a scalar to the builder.
    fn append_scalar(&mut self, scalar: &Scalar) -> VortexResult<()> {
        if scalar.dtype() != self.dtype() {
            vortex_bail!(
                "Builder has dtype {:?}, scalar has {:?}",
                self.dtype(),
                scalar.dtype()
            )
        }
        match scalar.dtype() {
            DType::Null => self
                .as_any_mut()
                .downcast_mut::<NullBuilder>()
                .ok_or_else(|| vortex_err!("Cannot append null scalar to non-null builder"))?
                .append_null(),
            DType::Bool(_) => self
                .as_any_mut()
                .downcast_mut::<BoolBuilder>()
                .ok_or_else(|| vortex_err!("Cannot append bool scalar to non-bool builder"))?
                .append_option(BoolScalar::try_from(scalar)?.value()),
            DType::Primitive(ptype, ..) => {
                match_each_native_ptype!(ptype, |P| {
                    self.as_any_mut()
                        .downcast_mut::<PrimitiveBuilder<P>>()
                        .ok_or_else(|| {
                            vortex_err!("Cannot append primitive scalar to non-primitive builder")
                        })?
                        .append_option(PrimitiveScalar::try_from(scalar)?.typed_value::<P>())
                })
            }
            DType::Decimal(..) => {
                let builder = self
                    .as_any_mut()
                    .downcast_mut::<DecimalBuilder>()
                    .ok_or_else(|| {
                        vortex_err!("Cannot append decimal scalar to non-decimal builder")
                    })?;
                match scalar.as_decimal().decimal_value() {
                    None => builder.append_null(),
                    Some(v) => match_each_decimal_value!(v, |dec_val| {
                        builder.append_value(dec_val);
                    }),
                }
            }
            DType::Utf8(_) => self
                .as_any_mut()
                .downcast_mut::<VarBinViewBuilder>()
                .ok_or_else(|| vortex_err!("Cannot append utf8 scalar to non-utf8 builder"))?
                .append_option(Utf8Scalar::try_from(scalar)?.value()),
            DType::Binary(_) => self
                .as_any_mut()
                .downcast_mut::<VarBinViewBuilder>()
                .ok_or_else(|| vortex_err!("Cannot append binary scalar to non-binary builder"))?
                .append_option(BinaryScalar::try_from(scalar)?.value()),
            DType::Struct(..) => self
                .as_any_mut()
                .downcast_mut::<StructBuilder>()
                .ok_or_else(|| vortex_err!("Cannot append struct scalar to non-struct builder"))?
                .append_value(StructScalar::try_from(scalar)?)?,
            DType::List(..) => self
                .as_any_mut()
                .downcast_mut::<ListBuilder<u64>>()
                .ok_or_else(|| vortex_err!("Cannot append list scalar to non-list builder"))?
                .append_value(ListScalar::try_from(scalar)?)?,
            DType::Extension(..) => self
                .as_any_mut()
                .downcast_mut::<ExtensionBuilder>()
                .ok_or_else(|| {
                    vortex_err!("Cannot append extension scalar to non-extension builder")
                })?
                .append_value(ExtScalar::try_from(scalar)?)?,
        }
        Ok(())
    }
}

impl<T: ?Sized + ArrayBuilder> ArrayBuilderExt for T {}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	//! Builders for Vortex arrays.
5	//!
6	//! Every logical type in Vortex has a canonical (uncompressed) in-memory encoding. This module
7	//! provides pre-allocated builders to construct new canonical arrays.
8	//!
9	//! ## Example:
10	//!
11	//! ```
12	//! use vortex_array::builders::{builder_with_capacity, ArrayBuilderExt};
13	//! use vortex_dtype::{DType, Nullability};
14	//!
15	//! // Create a new builder for string data.
16	//! let mut builder = builder_with_capacity(&DType::Utf8(Nullability::NonNullable), 4);
17	//!
18	//! builder.append_scalar(&"a".into()).unwrap();
19	//! builder.append_scalar(&"b".into()).unwrap();
20	//! builder.append_scalar(&"c".into()).unwrap();
21	//! builder.append_scalar(&"d".into()).unwrap();
22	//!
23	//! let strings = builder.finish();
24	//!
25	//! assert_eq!(strings.scalar_at(0).unwrap(), "a".into());
26	//! assert_eq!(strings.scalar_at(1).unwrap(), "b".into());
27	//! assert_eq!(strings.scalar_at(2).unwrap(), "c".into());
28	//! assert_eq!(strings.scalar_at(3).unwrap(), "d".into());
29	//! ```
30
31	mod bool;
32	mod decimal;
33	mod extension;
34	mod lazy_validity_builder;
35	mod list;
36	mod null;
37	mod primitive;
38	mod struct_;
39	mod varbinview;
40
41	use std::any::Any;
42
43	pub use bool::*;
44	pub use decimal::*;
45	pub use extension::*;
46	pub use list::*;
47	pub use null::*;
48	pub use primitive::*;
49	pub use struct_::*;
50	pub use varbinview::*;
51	use vortex_dtype::{DType, match_each_native_ptype};
52	use vortex_error::{VortexResult, vortex_bail, vortex_err};
53	use vortex_mask::Mask;
54	use vortex_scalar::{
55	BinaryScalar, BoolScalar, DecimalValue, ExtScalar, ListScalar, PrimitiveScalar, Scalar,
56	ScalarValue, StructScalar, Utf8Scalar, match_each_decimal_value, match_each_decimal_value_type,
57	};
58
59	use crate::arrays::smallest_storage_type;
60	use crate::{Array, ArrayRef};
61
62	pub trait ArrayBuilder: Send {
63	fn as_any(&self) -> &dyn Any;
64
65	fn as_any_mut(&mut self) -> &mut dyn Any;
66
67	fn dtype(&self) -> &DType;
68
69	fn len(&self) -> usize;
70
71	fn is_empty(&self) -> bool {	×
72	self.len() == 0	×
73	}	×
74
75	/// Append a "zero" value to the array.
76	fn append_zero(&mut self) {	465✔
77	self.append_zeros(1)	465✔
78	}	465✔
79
80	/// Appends n "zero" values to the array.
81	fn append_zeros(&mut self, n: usize);
82
83	/// Append a "null" value to the array.
84	fn append_null(&mut self) {	2,551✔
85	self.append_nulls(1)	2,551✔
86	}	2,551✔
87
88	/// Appends n "null" values to the array.
89	fn append_nulls(&mut self, n: usize);
90
91	/// Extends the array with the provided array, canonicalizing if necessary.
92	fn extend_from_array(&mut self, array: &dyn Array) -> VortexResult<()>;
93
94	/// Ensure that the builder can hold at least `capacity` number of items
95	fn ensure_capacity(&mut self, capacity: usize);
96
97	/// Override builders validity with the one provided
98	fn set_validity(&mut self, validity: Mask);
99
100	/// Constructs an Array from the builder components.
101	///
102	/// # Panics
103	///
104	/// This function may panic if the builder's methods are called with invalid arguments. If only
105	/// the methods on this interface are used, the builder should not panic. However, specific
106	/// builders have interfaces that may be misused. For example, if the number of values in a
107	/// [PrimitiveBuilder]'s [vortex_buffer::BufferMut] does not match the number of validity bits,
108	/// the PrimitiveBuilder's [Self::finish] will panic.
109	fn finish(&mut self) -> ArrayRef;
110	}
111
112	/// Construct a new canonical builder for the given [`DType`].
113	///
114	///
115	/// # Example
116	///
117	/// ```
118	/// use vortex_array::builders::{builder_with_capacity, ArrayBuilderExt};
119	/// use vortex_dtype::{DType, Nullability};
120	///
121	/// // Create a new builder for string data.
122	/// let mut builder = builder_with_capacity(&DType::Utf8(Nullability::NonNullable), 4);
123	///
124	/// builder.append_scalar(&"a".into()).unwrap();
125	/// builder.append_scalar(&"b".into()).unwrap();
126	/// builder.append_scalar(&"c".into()).unwrap();
127	/// builder.append_scalar(&"d".into()).unwrap();
128	///
129	/// let strings = builder.finish();
130	///
131	/// assert_eq!(strings.scalar_at(0).unwrap(), "a".into());
132	/// assert_eq!(strings.scalar_at(1).unwrap(), "b".into());
133	/// assert_eq!(strings.scalar_at(2).unwrap(), "c".into());
134	/// assert_eq!(strings.scalar_at(3).unwrap(), "d".into());
135	/// ```
136	pub fn builder_with_capacity(dtype: &DType, capacity: usize) -> Box<dyn ArrayBuilder> {	53,144✔
137	match dtype {	53,144✔
138	DType::Null => Box::new(NullBuilder::new()),	32✔
139	DType::Bool(n) => Box::new(BoolBuilder::with_capacity(*n, capacity)),	2,048✔
140	DType::Primitive(ptype, n) => {	39,223✔
141	match_each_native_ptype!(ptype, \|P\| {	39,223✔
142	Box::new(PrimitiveBuilder::<P>::with_capacity(*n, capacity))	987✔
143	})
144	}
145	DType::Decimal(decimal_type, n) => {	2,570✔
146	match_each_decimal_value_type!(smallest_storage_type(decimal_type), \|D\| {	2,570✔
147	Box::new(DecimalBuilder::with_capacity::<D>(	156✔
148	capacity,	156✔
149	*decimal_type,	156✔
150	*n,	156✔
151	))	156✔
152	})
153	}
154	DType::Utf8(n) => Box::new(VarBinViewBuilder::with_capacity(DType::Utf8(*n), capacity)),	7,039✔
155	DType::Binary(n) => Box::new(VarBinViewBuilder::with_capacity(	130✔
156	DType::Binary(*n),	130✔
157	capacity,	130✔
158	)),	130✔
159	DType::Struct(struct_dtype, n) => Box::new(StructBuilder::with_capacity(	318✔
160	struct_dtype.clone(),	318✔
161	*n,	318✔
162	capacity,	318✔
163	)),	318✔
164	DType::List(dtype, n) => Box::new(ListBuilder::<u64>::with_capacity(	196✔
165	dtype.clone(),	196✔
166	*n,	196✔
167	capacity,	196✔
168	)),	196✔
169	DType::Extension(ext_dtype) => {	1,588✔
170	Box::new(ExtensionBuilder::with_capacity(ext_dtype.clone(), capacity))	1,588✔
171	}
172	}
173	}	53,144✔
174
175	pub trait ArrayBuilderExt: ArrayBuilder {
176	/// A generic function to append a scalar value to the builder.
177	fn append_scalar_value(&mut self, value: ScalarValue) -> VortexResult<()> {	×
178	if value.is_null() {	×
179	self.append_null();	×
180	Ok(())	×
181	} else {
182	self.append_scalar(&Scalar::new(self.dtype().clone(), value))	×
183	}
184	}	×
185
186	/// A generic function to append a scalar to the builder.
187	fn append_scalar(&mut self, scalar: &Scalar) -> VortexResult<()> {	50,679✔
188	if scalar.dtype() != self.dtype() {	50,679✔
189	vortex_bail!(	×
190	"Builder has dtype {:?}, scalar has {:?}",	×
191	self.dtype(),	×
192	scalar.dtype()	×
193	)
194	}	50,679✔
195	match scalar.dtype() {	50,679✔
196	DType::Null => self	×
197	.as_any_mut()	×
198	.downcast_mut::<NullBuilder>()	×
199	.ok_or_else(\|\| vortex_err!("Cannot append null scalar to non-null builder"))?	×
200	.append_null(),	×
201	DType::Bool(_) => self	312✔
202	.as_any_mut()	312✔
203	.downcast_mut::<BoolBuilder>()	312✔
204	.ok_or_else(\|\| vortex_err!("Cannot append bool scalar to non-bool builder"))?	312✔
205	.append_option(BoolScalar::try_from(scalar)?.value()),	312✔
206	DType::Primitive(ptype, ..) => {	38,972✔
207	match_each_native_ptype!(ptype, \|P\| {	38,972✔
208	self.as_any_mut()	11✔
209	.downcast_mut::<PrimitiveBuilder<P>>()	11✔
210	.ok_or_else(\|\| {	11✔
211	vortex_err!("Cannot append primitive scalar to non-primitive builder")	×
212	})?	×
213	.append_option(PrimitiveScalar::try_from(scalar)?.typed_value::<P>())	11✔
214	})
215	}
216	DType::Decimal(..) => {
217	let builder = self	2,844✔
218	.as_any_mut()	2,844✔
219	.downcast_mut::<DecimalBuilder>()	2,844✔
220	.ok_or_else(\|\| {	2,844✔
221	vortex_err!("Cannot append decimal scalar to non-decimal builder")	×
222	})?;	×
223	match scalar.as_decimal().decimal_value() {	2,844✔
224	None => builder.append_null(),	×
225	Some(v) => match_each_decimal_value!(v, \|dec_val\| {	2,844✔
226	builder.append_value(dec_val);	156✔
227	}),	156✔
228	}
229	}
230	DType::Utf8(_) => self	6,907✔
231	.as_any_mut()	6,907✔
232	.downcast_mut::<VarBinViewBuilder>()	6,907✔
233	.ok_or_else(\|\| vortex_err!("Cannot append utf8 scalar to non-utf8 builder"))?	6,907✔
234	.append_option(Utf8Scalar::try_from(scalar)?.value()),	6,907✔
235	DType::Binary(_) => self	156✔
236	.as_any_mut()	156✔
237	.downcast_mut::<VarBinViewBuilder>()	156✔
238	.ok_or_else(\|\| vortex_err!("Cannot append binary scalar to non-binary builder"))?	156✔
239	.append_option(BinaryScalar::try_from(scalar)?.value()),	156✔
240	DType::Struct(..) => self	×
241	.as_any_mut()	×
242	.downcast_mut::<StructBuilder>()	×
243	.ok_or_else(\|\| vortex_err!("Cannot append struct scalar to non-struct builder"))?	×
244	.append_value(StructScalar::try_from(scalar)?)?,	×
245	DType::List(..) => self	×
246	.as_any_mut()	×
247	.downcast_mut::<ListBuilder<u64>>()	×
248	.ok_or_else(\|\| vortex_err!("Cannot append list scalar to non-list builder"))?	×
249	.append_value(ListScalar::try_from(scalar)?)?,	×
250	DType::Extension(..) => self	1,488✔
251	.as_any_mut()	1,488✔
252	.downcast_mut::<ExtensionBuilder>()	1,488✔
253	.ok_or_else(\|\| {	1,488✔
254	vortex_err!("Cannot append extension scalar to non-extension builder")	×
255	})?	×
256	.append_value(ExtScalar::try_from(scalar)?)?,	1,488✔
257	}
258	Ok(())	50,679✔
259	}	50,679✔
260	}
261
262	impl<T: ?Sized + ArrayBuilder> ArrayBuilderExt for T {}

vortex-data / vortex / 16832413242

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous