16324116701

Committed 16 Jul 2025 03:46PM UTC coverage: 80.69% (-0.9%) from 81.557%

Build # 16324116701

Build Type

Pull #3881

github

Committed by

web-flow

Commit Message

Merge c08ea4033 into ced09d9a8

Pull Request Pull Request #3881: feat: build with stable rust

Run Details

118 of 171 new or added lines in 27 files covered. (69.01%)

174 existing lines in 102 files now uncovered.

41861 of 51879 relevant lines covered (80.69%)

157334.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.76

/vortex-array/src/arrays/chunked/decode.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_buffer::BufferMut;
use vortex_dtype::{DType, Nullability, PType, StructFields};
use vortex_error::{VortexExpect, VortexResult, vortex_err};

use super::ChunkedArray;
use crate::arrays::{ChunkedVTable, ListArray, PrimitiveArray, StructArray};
use crate::builders::{ArrayBuilder, builder_with_capacity};
use crate::compute::cast;
use crate::validity::Validity;
use crate::vtable::CanonicalVTable;
use crate::{Array as _, ArrayRef, Canonical, IntoArray, ToCanonical};

impl CanonicalVTable<ChunkedVTable> for ChunkedVTable {
    fn canonicalize(array: &ChunkedArray) -> VortexResult<Canonical> {
        if array.nchunks() == 0 {
            return Ok(Canonical::empty(array.dtype()));
        }
        if array.nchunks() == 1 {
            return array.chunks()[0].to_canonical();
        }
        match array.dtype() {
            DType::Struct(struct_dtype, _) => {
                let struct_array = swizzle_struct_chunks(
                    array.chunks(),
                    Validity::copy_from_array(array.as_ref())?,
                    struct_dtype,
                )?;
                Ok(Canonical::Struct(struct_array))
            }
            DType::List(elem, _) => Ok(Canonical::List(pack_lists(
                array.chunks(),
                Validity::copy_from_array(array.as_ref())?,
                elem,
            )?)),
            _ => {
                let mut builder = builder_with_capacity(array.dtype(), array.len());
                array.append_to_builder(builder.as_mut())?;
                builder.finish().to_canonical()
            }
        }
    }

    fn append_to_builder(array: &ChunkedArray, builder: &mut dyn ArrayBuilder) -> VortexResult<()> {
        for chunk in array.chunks() {
            chunk.append_to_builder(builder)?;
        }
        Ok(())
    }
}

/// Swizzle the pointers within a ChunkedArray of StructArrays to instead be a single
/// StructArray, where the Array for each Field is a ChunkedArray.
fn swizzle_struct_chunks(
    chunks: &[ArrayRef],
    validity: Validity,
    struct_dtype: &StructFields,
) -> VortexResult<StructArray> {
    let len = chunks.iter().map(|chunk| chunk.len()).sum();
    let mut field_arrays = Vec::new();

    for (field_idx, field_dtype) in struct_dtype.fields().enumerate() {
        let field_chunks = chunks
            .iter()
            .map(|c| {
                c.to_struct()
                    .vortex_expect("Chunk was not a StructArray")
                    .fields()
                    .get(field_idx)
                    .vortex_expect("Invalid field index")
                    .to_array()
            })
            .collect::<Vec<_>>();
        let field_array = ChunkedArray::try_new(field_chunks, field_dtype.clone())?;
        field_arrays.push(field_array.into_array());
    }

    StructArray::try_new_with_dtype(field_arrays, struct_dtype.clone(), len, validity)
}

fn pack_lists(
    chunks: &[ArrayRef],
    validity: Validity,
    elem_dtype: &DType,
) -> VortexResult<ListArray> {
    let len: usize = chunks.iter().map(|c| c.len()).sum();
    let mut offsets = BufferMut::<i64>::with_capacity(len + 1);
    offsets.push(0);
    let mut elements = Vec::new();

    for chunk in chunks {
        let chunk = chunk.to_list()?;
        // TODO: handle i32 offsets if they fit.
        let offsets_arr = cast(
            chunk.offsets(),
            &DType::Primitive(PType::I64, Nullability::NonNullable),
        )?
        .to_primitive()?;

        let first_offset_value: usize = usize::try_from(&offsets_arr.scalar_at(0)?)?;
        let last_offset_value: usize =
            usize::try_from(&offsets_arr.scalar_at(offsets_arr.len() - 1)?)?;
        elements.push(
            chunk
                .elements()
                .slice(first_offset_value, last_offset_value)?,
        );

        let adjustment_from_previous = *offsets
            .last()
            .ok_or_else(|| vortex_err!("List offsets must have at least one element"))?;
        offsets.extend_trusted(
            offsets_arr
                .as_slice::<i64>()
                .iter()
                .skip(1)
                .map(|off| off + adjustment_from_previous - first_offset_value as i64),
        );
    }
    let chunked_elements = ChunkedArray::try_new(elements, elem_dtype.clone())?.into_array();
    let offsets = PrimitiveArray::new(offsets.freeze(), Validity::NonNullable);

    ListArray::try_new(chunked_elements, offsets.into_array(), validity)
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;

    use vortex_dtype::DType::{List, Primitive};
    use vortex_dtype::Nullability::NonNullable;
    use vortex_dtype::PType::I32;

    use crate::accessor::ArrayAccessor;
    use crate::arrays::{ChunkedArray, ListArray, PrimitiveArray, StructArray, VarBinViewArray};
    use crate::validity::Validity;
    use crate::{IntoArray, ToCanonical};

    #[test]
    pub fn pack_nested_structs() {
        let struct_array = StructArray::try_new(
            vec!["a".into()].into(),
            vec![VarBinViewArray::from_iter_str(["foo", "bar", "baz", "quak"]).into_array()],
            4,
            Validity::NonNullable,
        )
        .unwrap();
        let dtype = struct_array.dtype().clone();
        let chunked = ChunkedArray::try_new(
            vec![
                ChunkedArray::try_new(vec![struct_array.to_array()], dtype.clone())
                    .unwrap()
                    .into_array(),
            ],
            dtype,
        )
        .unwrap()
        .into_array();
        let canonical_struct = chunked.to_struct().unwrap();
        let canonical_varbin = canonical_struct.fields()[0].to_varbinview().unwrap();
        let original_varbin = struct_array.fields()[0].to_varbinview().unwrap();
        let orig_values = original_varbin
            .with_iterator(|it| it.map(|a| a.map(|v| v.to_vec())).collect::<Vec<_>>())
            .unwrap();
        let canon_values = canonical_varbin
            .with_iterator(|it| it.map(|a| a.map(|v| v.to_vec())).collect::<Vec<_>>())
            .unwrap();
        assert_eq!(orig_values, canon_values);
    }

    #[test]
    pub fn pack_nested_lists() {
        let l1 = ListArray::try_new(
            PrimitiveArray::from_iter([1, 2, 3, 4]).into_array(),
            PrimitiveArray::from_iter([0, 3]).into_array(),
            Validity::NonNullable,
        )
        .unwrap();

        let l2 = ListArray::try_new(
            PrimitiveArray::from_iter([5, 6]).into_array(),
            PrimitiveArray::from_iter([0, 2]).into_array(),
            Validity::NonNullable,
        )
        .unwrap();

        let chunked_list = ChunkedArray::try_new(
            vec![l1.clone().into_array(), l2.clone().into_array()],
            List(Arc::new(Primitive(I32, NonNullable)), NonNullable),
        );

        let canon_values = chunked_list.unwrap().to_list().unwrap();

        assert_eq!(l1.scalar_at(0).unwrap(), canon_values.scalar_at(0).unwrap());
        assert_eq!(l2.scalar_at(0).unwrap(), canon_values.scalar_at(1).unwrap());
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use vortex_buffer::BufferMut;
5	use vortex_dtype::{DType, Nullability, PType, StructFields};
6	use vortex_error::{VortexExpect, VortexResult, vortex_err};
7
8	use super::ChunkedArray;
9	use crate::arrays::{ChunkedVTable, ListArray, PrimitiveArray, StructArray};
10	use crate::builders::{ArrayBuilder, builder_with_capacity};
11	use crate::compute::cast;
12	use crate::validity::Validity;
13	use crate::vtable::CanonicalVTable;
14	use crate::{Array as _, ArrayRef, Canonical, IntoArray, ToCanonical};
15
16	impl CanonicalVTable<ChunkedVTable> for ChunkedVTable {
17	fn canonicalize(array: &ChunkedArray) -> VortexResult<Canonical> {	11,071✔
18	if array.nchunks() == 0 {	11,071✔
19	return Ok(Canonical::empty(array.dtype()));	72✔
20	}	10,999✔
21	if array.nchunks() == 1 {	10,999✔
22	return array.chunks()[0].to_canonical();	7,348✔
23	}	3,651✔
24	match array.dtype() {	3,651✔
25	DType::Struct(struct_dtype, _) => {	×
26	let struct_array = swizzle_struct_chunks(	×
27	array.chunks(),	×
28	Validity::copy_from_array(array.as_ref())?,	×
29	struct_dtype,	×
30	)?;	×
31	Ok(Canonical::Struct(struct_array))	×
32	}
33	DType::List(elem, _) => Ok(Canonical::List(pack_lists(	38✔
34	array.chunks(),	38✔
35	Validity::copy_from_array(array.as_ref())?,	38✔
36	elem,	38✔
37	)?)),	×
38	_ => {
39	let mut builder = builder_with_capacity(array.dtype(), array.len());	3,613✔
40	array.append_to_builder(builder.as_mut())?;	3,613✔
41	builder.finish().to_canonical()	3,613✔
42	}
43	}
44	}	11,071✔
45
46	fn append_to_builder(array: &ChunkedArray, builder: &mut dyn ArrayBuilder) -> VortexResult<()> {	3,723✔
47	for chunk in array.chunks() {	40,335✔
48	chunk.append_to_builder(builder)?;	40,335✔
49	}
50	Ok(())	3,723✔
51	}	3,723✔
52	}
53
54	/// Swizzle the pointers within a ChunkedArray of StructArrays to instead be a single
55	/// StructArray, where the Array for each Field is a ChunkedArray.
56	fn swizzle_struct_chunks(	×
57	chunks: &[ArrayRef],	×
58	validity: Validity,	×
59	struct_dtype: &StructFields,	×
60	) -> VortexResult<StructArray> {	×
61	let len = chunks.iter().map(\|chunk\| chunk.len()).sum();	×
62	let mut field_arrays = Vec::new();	×
63
64	for (field_idx, field_dtype) in struct_dtype.fields().enumerate() {	×
65	let field_chunks = chunks	×
66	.iter()	×
67	.map(\|c\| {	×
68	c.to_struct()	×
69	.vortex_expect("Chunk was not a StructArray")	×
70	.fields()	×
71	.get(field_idx)	×
72	.vortex_expect("Invalid field index")	×
73	.to_array()	×
74	})	×
75	.collect::<Vec<_>>();	×
76	let field_array = ChunkedArray::try_new(field_chunks, field_dtype.clone())?;	×
77	field_arrays.push(field_array.into_array());	×
78	}
79
80	StructArray::try_new_with_dtype(field_arrays, struct_dtype.clone(), len, validity)	×
81	}	×
82
83	fn pack_lists(	38✔
84	chunks: &[ArrayRef],	38✔
85	validity: Validity,	38✔
86	elem_dtype: &DType,	38✔
87	) -> VortexResult<ListArray> {	38✔
88	let len: usize = chunks.iter().map(\|c\| c.len()).sum();	76✔
89	let mut offsets = BufferMut::<i64>::with_capacity(len + 1);	38✔
90	offsets.push(0);	38✔
91	let mut elements = Vec::new();	38✔
92
93	for chunk in chunks {	114✔
94	let chunk = chunk.to_list()?;	76✔
95	// TODO: handle i32 offsets if they fit.
96	let offsets_arr = cast(	76✔
97	chunk.offsets(),	76✔
98	&DType::Primitive(PType::I64, Nullability::NonNullable),	76✔
UNCOV 99	)?	×
100	.to_primitive()?;	76✔
101
102	let first_offset_value: usize = usize::try_from(&offsets_arr.scalar_at(0)?)?;	76✔
103	let last_offset_value: usize =	76✔
104	usize::try_from(&offsets_arr.scalar_at(offsets_arr.len() - 1)?)?;	76✔
105	elements.push(	76✔
106	chunk	76✔
107	.elements()	76✔
108	.slice(first_offset_value, last_offset_value)?,	76✔
109	);
110
111	let adjustment_from_previous = *offsets	76✔
112	.last()	76✔
113	.ok_or_else(\|\| vortex_err!("List offsets must have at least one element"))?;	76✔
114	offsets.extend_trusted(	76✔
115	offsets_arr	76✔
116	.as_slice::<i64>()	76✔
117	.iter()	76✔
118	.skip(1)	76✔
119	.map(\|off\| off + adjustment_from_previous - first_offset_value as i64),	292✔
120	);
121	}
122	let chunked_elements = ChunkedArray::try_new(elements, elem_dtype.clone())?.into_array();	38✔
123	let offsets = PrimitiveArray::new(offsets.freeze(), Validity::NonNullable);	38✔
124
125	ListArray::try_new(chunked_elements, offsets.into_array(), validity)	38✔
126	}	38✔
127
128	#[cfg(test)]
129	mod tests {
130	use std::sync::Arc;
131
132	use vortex_dtype::DType::{List, Primitive};
133	use vortex_dtype::Nullability::NonNullable;
134	use vortex_dtype::PType::I32;
135
136	use crate::accessor::ArrayAccessor;
137	use crate::arrays::{ChunkedArray, ListArray, PrimitiveArray, StructArray, VarBinViewArray};
138	use crate::validity::Validity;
139	use crate::{IntoArray, ToCanonical};
140
141	#[test]
142	pub fn pack_nested_structs() {	1✔
143	let struct_array = StructArray::try_new(	1✔
144	vec!["a".into()].into(),	1✔
145	vec![VarBinViewArray::from_iter_str(["foo", "bar", "baz", "quak"]).into_array()],	1✔
146	4,
147	Validity::NonNullable,	1✔
148	)
149	.unwrap();	1✔
150	let dtype = struct_array.dtype().clone();	1✔
151	let chunked = ChunkedArray::try_new(	1✔
152	vec![	1✔
153	ChunkedArray::try_new(vec![struct_array.to_array()], dtype.clone())	1✔
154	.unwrap()	1✔
155	.into_array(),	1✔
156	],
157	dtype,	1✔
158	)
159	.unwrap()	1✔
160	.into_array();	1✔
161	let canonical_struct = chunked.to_struct().unwrap();	1✔
162	let canonical_varbin = canonical_struct.fields()[0].to_varbinview().unwrap();	1✔
163	let original_varbin = struct_array.fields()[0].to_varbinview().unwrap();	1✔
164	let orig_values = original_varbin	1✔
165	.with_iterator(\|it\| it.map(\|a\| a.map(\|v\| v.to_vec())).collect::<Vec<_>>())	4✔
166	.unwrap();	1✔
167	let canon_values = canonical_varbin	1✔
168	.with_iterator(\|it\| it.map(\|a\| a.map(\|v\| v.to_vec())).collect::<Vec<_>>())	4✔
169	.unwrap();	1✔
170	assert_eq!(orig_values, canon_values);	1✔
171	}	1✔
172
173	#[test]
174	pub fn pack_nested_lists() {	1✔
175	let l1 = ListArray::try_new(	1✔
176	PrimitiveArray::from_iter([1, 2, 3, 4]).into_array(),	1✔
177	PrimitiveArray::from_iter([0, 3]).into_array(),	1✔
178	Validity::NonNullable,	1✔
179	)
180	.unwrap();	1✔
181
182	let l2 = ListArray::try_new(	1✔
183	PrimitiveArray::from_iter([5, 6]).into_array(),	1✔
184	PrimitiveArray::from_iter([0, 2]).into_array(),	1✔
185	Validity::NonNullable,	1✔
186	)
187	.unwrap();	1✔
188
189	let chunked_list = ChunkedArray::try_new(	1✔
190	vec![l1.clone().into_array(), l2.clone().into_array()],	1✔
191	List(Arc::new(Primitive(I32, NonNullable)), NonNullable),	1✔
192	);
193
194	let canon_values = chunked_list.unwrap().to_list().unwrap();	1✔
195
196	assert_eq!(l1.scalar_at(0).unwrap(), canon_values.scalar_at(0).unwrap());	1✔
197	assert_eq!(l2.scalar_at(0).unwrap(), canon_values.scalar_at(1).unwrap());	1✔
198	}	1✔
199	}

vortex-data / vortex / 16324116701

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous