16331938722

Committed 16 Jul 2025 10:49PM UTC coverage: 80.702% (-0.9%) from 81.557%

Build # 16331938722

Build Type

push

github

Committed by

web-flow

Commit Message

feat: build with stable rust (#3881)

Run Details

120 of 173 new or added lines in 28 files covered. (69.36%)

174 existing lines in 102 files now uncovered.

41861 of 51871 relevant lines covered (80.7%)

157487.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/vortex-array/src/arrays/arbitrary.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::iter;
use std::sync::Arc;

use arbitrary::{Arbitrary, Result, Unstructured};
use arrow_buffer::BooleanBuffer;
use builders::ListBuilder;
use vortex_buffer::Buffer;
use vortex_dtype::{DType, NativePType, Nullability, PType};
use vortex_error::{VortexExpect, VortexUnwrap};
use vortex_scalar::arbitrary::{random_decimal, random_scalar};
use vortex_scalar::{Scalar, match_each_decimal_value_type};

use super::{
    BoolArray, ChunkedArray, NullArray, OffsetPType, PrimitiveArray, StructArray,
    smallest_storage_type,
};
use crate::arrays::{VarBinArray, VarBinViewArray};
use crate::builders::{ArrayBuilder, ArrayBuilderExt, DecimalBuilder};
use crate::validity::Validity;
use crate::{Array, ArrayRef, IntoArray, ToCanonical, builders};

/// A wrapper type to implement `Arbitrary` for `ArrayRef`.
#[derive(Clone, Debug)]
pub struct ArbitraryArray(pub ArrayRef);

impl<'a> Arbitrary<'a> for ArbitraryArray {
    fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self> {
        let dtype = u.arbitrary()?;
        random_array(u, &dtype, None).map(ArbitraryArray)
    }
}

fn split_number_into_parts(n: usize, parts: usize) -> Vec<usize> {
    let reminder = n % parts;
    let division = (n - reminder) / parts;
    iter::repeat_n(division, parts - reminder)
        .chain(iter::repeat_n(division + 1, reminder))
        .collect()
}

fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Result<ArrayRef> {
    let num_chunks = u.int_in_range(1..=3)?;
    let chunk_lens = len.map(|l| split_number_into_parts(l, num_chunks));
    let mut chunks = (0..num_chunks)
        .map(|i| {
            let chunk_len = chunk_lens.as_ref().map(|c| c[i]);
            match dtype {
                DType::Null => Ok(NullArray::new(
                    chunk_len
                        .map(Ok)
                        .unwrap_or_else(|| u.int_in_range(0..=100))?,
                )
                .into_array()),
                DType::Bool(n) => random_bool(u, *n, chunk_len),
                DType::Primitive(ptype, n) => match ptype {
                    PType::U8 => random_primitive::<u8>(u, *n, chunk_len),
                    PType::U16 => random_primitive::<u16>(u, *n, chunk_len),
                    PType::U32 => random_primitive::<u32>(u, *n, chunk_len),
                    PType::U64 => random_primitive::<u64>(u, *n, chunk_len),
                    PType::I8 => random_primitive::<i8>(u, *n, chunk_len),
                    PType::I16 => random_primitive::<i16>(u, *n, chunk_len),
                    PType::I32 => random_primitive::<i32>(u, *n, chunk_len),
                    PType::I64 => random_primitive::<i64>(u, *n, chunk_len),
                    PType::F16 => Ok(random_primitive::<u16>(u, *n, chunk_len)?
                        .to_primitive()
                        .vortex_unwrap()
                        .reinterpret_cast(PType::F16)
                        .into_array()),
                    PType::F32 => random_primitive::<f32>(u, *n, chunk_len),
                    PType::F64 => random_primitive::<f64>(u, *n, chunk_len),
                },
                DType::Decimal(decimal, n) => {
                    let elem_len = chunk_len.unwrap_or(u.int_in_range(0..=20)?);
                    match_each_decimal_value_type!(smallest_storage_type(decimal), |DVT| {
                        let mut builder =
                            DecimalBuilder::new::<DVT>(decimal.precision(), decimal.scale(), *n);
                        for _i in 0..elem_len {
                            builder
                                .append_scalar_value(random_decimal(u, decimal)?)
                                .vortex_unwrap();
                        }
                        Ok(builder.finish())
                    })
                }
                DType::Utf8(n) => random_string(u, *n, chunk_len),
                DType::Binary(n) => random_bytes(u, *n, chunk_len),
                DType::Struct(sdt, n) => {
                    let first_array = sdt
                        .fields()
                        .next()
                        .map(|d| random_array(u, &d, chunk_len))
                        .transpose()?;
                    let resolved_len = first_array
                        .as_ref()
                        .map(|a| a.len())
                        .or(chunk_len)
                        .map(Ok)
                        .unwrap_or_else(|| u.int_in_range(0..=100))?;
                    let children = first_array
                        .into_iter()
                        .map(Ok)
                        .chain(
                            sdt.fields()
                                .skip(1)
                                .map(|d| random_array(u, &d, Some(resolved_len))),
                        )
                        .collect::<Result<Vec<_>>>()?;
                    Ok(StructArray::try_new(
                        sdt.names().clone(),
                        children,
                        resolved_len,
                        random_validity(u, *n, resolved_len)?,
                    )
                    .vortex_unwrap()
                    .into_array())
                }
                DType::List(ldt, n) => random_list(u, ldt, *n, chunk_len),
                DType::Extension(..) => {
                    todo!("Extension arrays are not implemented")
                }
            }
        })
        .collect::<Result<Vec<_>>>()?;

    if chunks.len() == 1 {
        Ok(chunks.remove(0))
    } else {
        let dtype = chunks[0].dtype().clone();
        Ok(ChunkedArray::try_new(chunks, dtype)
            .vortex_unwrap()
            .into_array())
    }
}

fn random_list(
    u: &mut Unstructured,
    ldt: &Arc<DType>,
    n: Nullability,
    chunk_len: Option<usize>,
) -> Result<ArrayRef> {
    match u.int_in_range(0..=5)? {
        0 => random_list_offset::<i16>(u, ldt, n, chunk_len),
        1 => random_list_offset::<i32>(u, ldt, n, chunk_len),
        2 => random_list_offset::<i64>(u, ldt, n, chunk_len),
        3 => random_list_offset::<u16>(u, ldt, n, chunk_len),
        4 => random_list_offset::<u32>(u, ldt, n, chunk_len),
        5 => random_list_offset::<u64>(u, ldt, n, chunk_len),
        _ => unreachable!("int_in_range returns a value in the above range"),
    }
}

fn random_list_offset<O: OffsetPType>(
    u: &mut Unstructured,
    ldt: &Arc<DType>,
    n: Nullability,
    chunk_len: Option<usize>,
) -> Result<ArrayRef> {
    let list_len = chunk_len.unwrap_or(u.int_in_range(0..=20)?);
    let mut builder = ListBuilder::<O>::with_capacity(ldt.clone(), n, 10);
    for _ in 0..list_len {
        if n == Nullability::Nullable && u.arbitrary::<bool>()? {
            builder.append_null();
        } else {
            builder
                .append_value(random_list_scalar(u, ldt, n)?.as_list())
                .vortex_expect("can append value");
        }
    }
    Ok(builder.finish())
}

fn random_list_scalar(
    u: &mut Unstructured,
    elem_dtype: &Arc<DType>,
    n: Nullability,
) -> Result<Scalar> {
    let elem_len = u.int_in_range(0..=20)?;
    let elems = (0..elem_len)
        .map(|_| random_scalar(u, elem_dtype))
        .collect::<Result<Vec<_>>>()?;
    Ok(Scalar::list(elem_dtype.clone(), elems, n))
}

fn random_string(
    u: &mut Unstructured,
    nullability: Nullability,
    len: Option<usize>,
) -> Result<ArrayRef> {
    match nullability {
        Nullability::NonNullable => {
            let v = arbitrary_vec_of_len::<String>(u, len)?;
            Ok(match u.int_in_range(0..=1)? {
                0 => VarBinArray::from_vec(v, DType::Utf8(Nullability::NonNullable)).into_array(),
                1 => VarBinViewArray::from_iter_str(v).into_array(),
                _ => unreachable!(),
            })
        }
        Nullability::Nullable => {
            let v = arbitrary_vec_of_len::<Option<String>>(u, len)?;
            Ok(match u.int_in_range(0..=1)? {
                0 => VarBinArray::from_iter(v, DType::Utf8(Nullability::Nullable)).into_array(),
                1 => VarBinViewArray::from_iter_nullable_str(v).into_array(),
                _ => unreachable!(),
            })
        }
    }
}

fn random_bytes(
    u: &mut Unstructured,
    nullability: Nullability,
    len: Option<usize>,
) -> Result<ArrayRef> {
    match nullability {
        Nullability::NonNullable => {
            let v = arbitrary_vec_of_len::<Vec<u8>>(u, len)?;
            Ok(match u.int_in_range(0..=1)? {
                0 => VarBinArray::from_vec(v, DType::Binary(Nullability::NonNullable)).into_array(),
                1 => VarBinViewArray::from_iter_bin(v).into_array(),
                _ => unreachable!(),
            })
        }
        Nullability::Nullable => {
            let v = arbitrary_vec_of_len::<Option<Vec<u8>>>(u, len)?;
            Ok(match u.int_in_range(0..=1)? {
                0 => VarBinArray::from_iter(v, DType::Binary(Nullability::Nullable)).into_array(),
                1 => VarBinViewArray::from_iter_nullable_bin(v).into_array(),
                _ => unreachable!(),
            })
        }
    }
}

fn random_primitive<'a, T: Arbitrary<'a> + NativePType>(
    u: &mut Unstructured<'a>,
    nullability: Nullability,
    len: Option<usize>,
) -> Result<ArrayRef> {
    let v = arbitrary_vec_of_len::<T>(u, len)?;
    let validity = random_validity(u, nullability, v.len())?;
    Ok(PrimitiveArray::new(Buffer::copy_from(v), validity).into_array())
}

fn random_bool(
    u: &mut Unstructured,
    nullability: Nullability,
    len: Option<usize>,
) -> Result<ArrayRef> {
    let v = arbitrary_vec_of_len(u, len)?;
    let validity = random_validity(u, nullability, v.len())?;
    Ok(BoolArray::new(BooleanBuffer::from(v), validity).into_array())
}

fn random_validity(u: &mut Unstructured, nullability: Nullability, len: usize) -> Result<Validity> {
    match nullability {
        Nullability::NonNullable => Ok(Validity::NonNullable),
        Nullability::Nullable => Ok(match u.int_in_range(0..=2)? {
            0 => Validity::AllValid,
            1 => Validity::AllInvalid,
            2 => Validity::from_iter(arbitrary_vec_of_len::<bool>(u, Some(len))?),
            _ => unreachable!(),
        }),
    }
}

fn arbitrary_vec_of_len<'a, T: Arbitrary<'a>>(
    u: &mut Unstructured<'a>,
    len: Option<usize>,
) -> Result<Vec<T>> {
    len.map(|l| (0..l).map(|_| T::arbitrary(u)).collect::<Result<Vec<_>>>())
        .unwrap_or_else(|| Vec::<T>::arbitrary(u))
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::iter;
5	use std::sync::Arc;
6
7	use arbitrary::{Arbitrary, Result, Unstructured};
8	use arrow_buffer::BooleanBuffer;
9	use builders::ListBuilder;
10	use vortex_buffer::Buffer;
11	use vortex_dtype::{DType, NativePType, Nullability, PType};
12	use vortex_error::{VortexExpect, VortexUnwrap};
13	use vortex_scalar::arbitrary::{random_decimal, random_scalar};
14	use vortex_scalar::{Scalar, match_each_decimal_value_type};
15
16	use super::{
17	BoolArray, ChunkedArray, NullArray, OffsetPType, PrimitiveArray, StructArray,
18	smallest_storage_type,
19	};
20	use crate::arrays::{VarBinArray, VarBinViewArray};
21	use crate::builders::{ArrayBuilder, ArrayBuilderExt, DecimalBuilder};
22	use crate::validity::Validity;
23	use crate::{Array, ArrayRef, IntoArray, ToCanonical, builders};
24
25	/// A wrapper type to implement `Arbitrary` for `ArrayRef`.
26	#[derive(Clone, Debug)]
27	pub struct ArbitraryArray(pub ArrayRef);
28
29	impl<'a> Arbitrary<'a> for ArbitraryArray {
30	fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self> {	×
31	let dtype = u.arbitrary()?;	×
32	random_array(u, &dtype, None).map(ArbitraryArray)	×
33	}	×
34	}
35
36	fn split_number_into_parts(n: usize, parts: usize) -> Vec<usize> {	×
37	let reminder = n % parts;	×
38	let division = (n - reminder) / parts;	×
39	iter::repeat_n(division, parts - reminder)	×
40	.chain(iter::repeat_n(division + 1, reminder))	×
41	.collect()	×
42	}	×
43
44	fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Result<ArrayRef> {	×
45	let num_chunks = u.int_in_range(1..=3)?;	×
46	let chunk_lens = len.map(\|l\| split_number_into_parts(l, num_chunks));	×
47	let mut chunks = (0..num_chunks)	×
48	.map(\|i\| {	×
49	let chunk_len = chunk_lens.as_ref().map(\|c\| c[i]);	×
50	match dtype {	×
UNCOV 51	DType::Null => Ok(NullArray::new(	×
52	chunk_len	×
53	.map(Ok)	×
54	.unwrap_or_else(\|\| u.int_in_range(0..=100))?,	×
55	)
56	.into_array()),	×
57	DType::Bool(n) => random_bool(u, *n, chunk_len),	×
58	DType::Primitive(ptype, n) => match ptype {	×
59	PType::U8 => random_primitive::<u8>(u, *n, chunk_len),	×
60	PType::U16 => random_primitive::<u16>(u, *n, chunk_len),	×
61	PType::U32 => random_primitive::<u32>(u, *n, chunk_len),	×
62	PType::U64 => random_primitive::<u64>(u, *n, chunk_len),	×
63	PType::I8 => random_primitive::<i8>(u, *n, chunk_len),	×
64	PType::I16 => random_primitive::<i16>(u, *n, chunk_len),	×
65	PType::I32 => random_primitive::<i32>(u, *n, chunk_len),	×
66	PType::I64 => random_primitive::<i64>(u, *n, chunk_len),	×
67	PType::F16 => Ok(random_primitive::<u16>(u, *n, chunk_len)?	×
68	.to_primitive()	×
69	.vortex_unwrap()	×
70	.reinterpret_cast(PType::F16)	×
71	.into_array()),	×
72	PType::F32 => random_primitive::<f32>(u, *n, chunk_len),	×
73	PType::F64 => random_primitive::<f64>(u, *n, chunk_len),	×
74	},
75	DType::Decimal(decimal, n) => {	×
76	let elem_len = chunk_len.unwrap_or(u.int_in_range(0..=20)?);	×
77	match_each_decimal_value_type!(smallest_storage_type(decimal), \|DVT\| {	×
78	let mut builder =	×
79	DecimalBuilder::new::<DVT>(decimal.precision(), decimal.scale(), *n);	×
80	for _i in 0..elem_len {	×
81	builder	×
82	.append_scalar_value(random_decimal(u, decimal)?)	×
83	.vortex_unwrap();	×
84	}
85	Ok(builder.finish())	×
86	})
87	}
88	DType::Utf8(n) => random_string(u, *n, chunk_len),	×
89	DType::Binary(n) => random_bytes(u, *n, chunk_len),	×
90	DType::Struct(sdt, n) => {	×
91	let first_array = sdt	×
92	.fields()	×
93	.next()	×
94	.map(\|d\| random_array(u, &d, chunk_len))	×
95	.transpose()?;	×
96	let resolved_len = first_array	×
97	.as_ref()	×
98	.map(\|a\| a.len())	×
99	.or(chunk_len)	×
100	.map(Ok)	×
101	.unwrap_or_else(\|\| u.int_in_range(0..=100))?;	×
102	let children = first_array	×
103	.into_iter()	×
104	.map(Ok)	×
105	.chain(	×
106	sdt.fields()	×
107	.skip(1)	×
108	.map(\|d\| random_array(u, &d, Some(resolved_len))),	×
109	)
110	.collect::<Result<Vec<_>>>()?;	×
UNCOV 111	Ok(StructArray::try_new(	×
112	sdt.names().clone(),	×
113	children,	×
114	resolved_len,	×
115	random_validity(u, *n, resolved_len)?,	×
116	)
117	.vortex_unwrap()	×
118	.into_array())	×
119	}
120	DType::List(ldt, n) => random_list(u, ldt, *n, chunk_len),	×
121	DType::Extension(..) => {
122	todo!("Extension arrays are not implemented")	×
123	}
124	}
125	})	×
126	.collect::<Result<Vec<_>>>()?;	×
127
128	if chunks.len() == 1 {	×
129	Ok(chunks.remove(0))	×
130	} else {
131	let dtype = chunks[0].dtype().clone();	×
132	Ok(ChunkedArray::try_new(chunks, dtype)	×
133	.vortex_unwrap()	×
134	.into_array())	×
135	}
136	}	×
137
138	fn random_list(	×
139	u: &mut Unstructured,	×
140	ldt: &Arc<DType>,	×
141	n: Nullability,	×
142	chunk_len: Option<usize>,	×
143	) -> Result<ArrayRef> {	×
144	match u.int_in_range(0..=5)? {	×
145	0 => random_list_offset::<i16>(u, ldt, n, chunk_len),	×
146	1 => random_list_offset::<i32>(u, ldt, n, chunk_len),	×
147	2 => random_list_offset::<i64>(u, ldt, n, chunk_len),	×
148	3 => random_list_offset::<u16>(u, ldt, n, chunk_len),	×
149	4 => random_list_offset::<u32>(u, ldt, n, chunk_len),	×
150	5 => random_list_offset::<u64>(u, ldt, n, chunk_len),	×
151	_ => unreachable!("int_in_range returns a value in the above range"),	×
152	}
153	}	×
154
155	fn random_list_offset<O: OffsetPType>(	×
156	u: &mut Unstructured,	×
157	ldt: &Arc<DType>,	×
158	n: Nullability,	×
159	chunk_len: Option<usize>,	×
160	) -> Result<ArrayRef> {	×
161	let list_len = chunk_len.unwrap_or(u.int_in_range(0..=20)?);	×
162	let mut builder = ListBuilder::<O>::with_capacity(ldt.clone(), n, 10);	×
163	for _ in 0..list_len {	×
164	if n == Nullability::Nullable && u.arbitrary::<bool>()? {	×
165	builder.append_null();	×
166	} else {	×
167	builder	×
168	.append_value(random_list_scalar(u, ldt, n)?.as_list())	×
169	.vortex_expect("can append value");	×
170	}
171	}
172	Ok(builder.finish())	×
173	}	×
174
175	fn random_list_scalar(	×
176	u: &mut Unstructured,	×
177	elem_dtype: &Arc<DType>,	×
178	n: Nullability,	×
179	) -> Result<Scalar> {	×
180	let elem_len = u.int_in_range(0..=20)?;	×
181	let elems = (0..elem_len)	×
182	.map(\|_\| random_scalar(u, elem_dtype))	×
183	.collect::<Result<Vec<_>>>()?;	×
184	Ok(Scalar::list(elem_dtype.clone(), elems, n))	×
185	}	×
186
187	fn random_string(	×
188	u: &mut Unstructured,	×
189	nullability: Nullability,	×
190	len: Option<usize>,	×
191	) -> Result<ArrayRef> {	×
192	match nullability {	×
193	Nullability::NonNullable => {
194	let v = arbitrary_vec_of_len::<String>(u, len)?;	×
195	Ok(match u.int_in_range(0..=1)? {	×
196	0 => VarBinArray::from_vec(v, DType::Utf8(Nullability::NonNullable)).into_array(),	×
197	1 => VarBinViewArray::from_iter_str(v).into_array(),	×
198	_ => unreachable!(),	×
199	})
200	}
201	Nullability::Nullable => {
202	let v = arbitrary_vec_of_len::<Option<String>>(u, len)?;	×
203	Ok(match u.int_in_range(0..=1)? {	×
204	0 => VarBinArray::from_iter(v, DType::Utf8(Nullability::Nullable)).into_array(),	×
205	1 => VarBinViewArray::from_iter_nullable_str(v).into_array(),	×
206	_ => unreachable!(),	×
207	})
208	}
209	}
210	}	×
211
212	fn random_bytes(	×
213	u: &mut Unstructured,	×
214	nullability: Nullability,	×
215	len: Option<usize>,	×
216	) -> Result<ArrayRef> {	×
217	match nullability {	×
218	Nullability::NonNullable => {
219	let v = arbitrary_vec_of_len::<Vec<u8>>(u, len)?;	×
220	Ok(match u.int_in_range(0..=1)? {	×
221	0 => VarBinArray::from_vec(v, DType::Binary(Nullability::NonNullable)).into_array(),	×
222	1 => VarBinViewArray::from_iter_bin(v).into_array(),	×
223	_ => unreachable!(),	×
224	})
225	}
226	Nullability::Nullable => {
227	let v = arbitrary_vec_of_len::<Option<Vec<u8>>>(u, len)?;	×
228	Ok(match u.int_in_range(0..=1)? {	×
229	0 => VarBinArray::from_iter(v, DType::Binary(Nullability::Nullable)).into_array(),	×
230	1 => VarBinViewArray::from_iter_nullable_bin(v).into_array(),	×
231	_ => unreachable!(),	×
232	})
233	}
234	}
235	}	×
236
237	fn random_primitive<'a, T: Arbitrary<'a> + NativePType>(	×
238	u: &mut Unstructured<'a>,	×
239	nullability: Nullability,	×
240	len: Option<usize>,	×
241	) -> Result<ArrayRef> {	×
242	let v = arbitrary_vec_of_len::<T>(u, len)?;	×
243	let validity = random_validity(u, nullability, v.len())?;	×
244	Ok(PrimitiveArray::new(Buffer::copy_from(v), validity).into_array())	×
245	}	×
246
247	fn random_bool(	×
248	u: &mut Unstructured,	×
249	nullability: Nullability,	×
250	len: Option<usize>,	×
251	) -> Result<ArrayRef> {	×
252	let v = arbitrary_vec_of_len(u, len)?;	×
253	let validity = random_validity(u, nullability, v.len())?;	×
254	Ok(BoolArray::new(BooleanBuffer::from(v), validity).into_array())	×
255	}	×
256
257	fn random_validity(u: &mut Unstructured, nullability: Nullability, len: usize) -> Result<Validity> {	×
258	match nullability {	×
259	Nullability::NonNullable => Ok(Validity::NonNullable),	×
260	Nullability::Nullable => Ok(match u.int_in_range(0..=2)? {	×
261	0 => Validity::AllValid,	×
262	1 => Validity::AllInvalid,	×
263	2 => Validity::from_iter(arbitrary_vec_of_len::<bool>(u, Some(len))?),	×
264	_ => unreachable!(),	×
265	}),
266	}
267	}	×
268
269	fn arbitrary_vec_of_len<'a, T: Arbitrary<'a>>(	×
270	u: &mut Unstructured<'a>,	×
271	len: Option<usize>,	×
272	) -> Result<Vec<T>> {	×
273	len.map(\|l\| (0..l).map(\|_\| T::arbitrary(u)).collect::<Result<Vec<_>>>())	×
274	.unwrap_or_else(\|\| Vec::<T>::arbitrary(u))	×
275	}	×

vortex-data / vortex / 16331938722

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous