16578751259

Committed 28 Jul 2025 07:46PM UTC coverage: 81.858% (+0.8%) from 81.087%

Build # 16578751259

Build Type

Pull #3992

github

Committed by

web-flow

Commit Message

Merge 0fe171459 into 1ae560509

Pull Request Pull Request #3992: feat: teach SparseArray to canonicalize lists

Run Details

222 of 241 new or added lines in 1 file covered. (92.12%)

495 existing lines in 40 files now uncovered.

43529 of 53176 relevant lines covered (81.86%)

169875.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.83

/vortex-array/src/canonical.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Encodings that enable zero-copy sharing of data with Arrow.

use vortex_dtype::DType;
use vortex_error::{VortexExpect, VortexResult, vortex_bail};

use crate::arrays::{
    BoolArray, DecimalArray, ExtensionArray, ListArray, NullArray, PrimitiveArray, StructArray,
    VarBinViewArray,
};
use crate::builders::builder_with_capacity;
use crate::{Array, ArrayRef, IntoArray};

/// An enum capturing the default uncompressed encodings for each [Vortex type][DType].
///
/// Any array can be decoded into canonical form via the [`to_canonical`][Array::to_canonical]
/// trait method. This is the simplest encoding for a type, and will not be compressed but may
/// contain compressed child arrays.
///
/// Canonical form is useful for doing type-specific compute where you need to know that all
/// elements are laid out decompressed and contiguous in memory.
///
/// # Laziness
///
/// Canonical form is not recursive, so while a `StructArray` is the canonical format for any
/// `Struct` type, individual column child arrays may still be compressed. This allows
/// compute over Vortex arrays to push decoding as late as possible, and ideally many child arrays
/// never need to be decoded into canonical form at all depending on the compute.
///
/// # Arrow interoperability
///
/// All of the Vortex canonical encodings have an equivalent Arrow encoding that can be built
/// zero-copy, and the corresponding Arrow array types can also be built directly.
///
/// The full list of canonical types and their equivalent Arrow array types are:
///
/// * `NullArray`: [`arrow_array::NullArray`]
/// * `BoolArray`: [`arrow_array::BooleanArray`]
/// * `PrimitiveArray`: [`arrow_array::PrimitiveArray`]
/// * `DecimalArray`: [`arrow_array::Decimal128Array`] and [`arrow_array::Decimal256Array`]
/// * `StructArray`: [`arrow_array::StructArray`]
/// * `ListArray`: [`arrow_array::ListArray`]
/// * `VarBinViewArray`: [`arrow_array::GenericByteViewArray`]
///
/// Vortex uses a logical type system, unlike Arrow which uses physical encodings for its types.
/// As an example, there are at least six valid physical encodings for a `Utf8` array. This can
/// create ambiguity.
/// Thus, if you receive an Arrow array, compress it using Vortex, and then
/// decompress it later to pass to a compute kernel, there are multiple suitable Arrow array
/// variants to hold the data.
///
/// To disambiguate, we choose a canonical physical encoding for every Vortex [`DType`], which
/// will correspond to an arrow-rs [`arrow_schema::DataType`].
///
/// # Views support
///
/// Binary and String views, also known as "German strings" are a better encoding format for
/// nearly all use-cases. Variable-length binary views are part of the Apache Arrow spec, and are
/// fully supported by the Datafusion query engine. We use them as our canonical string encoding
/// for all `Utf8` and `Binary` typed arrays in Vortex. They provide considerably faster filter
/// execution than the core `StringArray` and `BinaryArray` types, at the expense of potentially
/// needing [garbage collection][arrow_array::GenericByteViewArray::gc] to clear unreferenced items
/// from memory.
#[derive(Debug, Clone)]
pub enum Canonical {
    Null(NullArray),
    Bool(BoolArray),
    Primitive(PrimitiveArray),
    Decimal(DecimalArray),
    Struct(StructArray),
    // TODO(joe): maybe this should be a ListView, however this will be annoying in spiral
    List(ListArray),
    VarBinView(VarBinViewArray),
    Extension(ExtensionArray),
}

impl Canonical {
    /// Create an empty canonical array of the given dtype.
    pub fn empty(dtype: &DType) -> Canonical {
        builder_with_capacity(dtype, 0)
            .finish()
            .to_canonical()
            .vortex_expect("cannot fail to convert an empty array to canonical")
    }
}

impl Canonical {
    /// Performs a (potentially expensive) compaction operation on the array before it is complete.
    ///
    /// This is mostly relevant for the variable-length types such as Utf8, Binary or List where
    /// they can accumulate wasted space after slicing and taking operations.
    ///
    /// This operation is very expensive and can result in things like allocations, full-scans
    /// and copy operations.
    pub fn compact(&self) -> VortexResult<Canonical> {
        match self {
            Canonical::VarBinView(array) => Ok(Canonical::VarBinView(array.compact_buffers()?)),
            Canonical::List(array) => Ok(Canonical::List(array.reset_offsets()?)),
            other => Ok(other.clone()),
        }
    }
}

// Unwrap canonical type back down to specialized type.
impl Canonical {
    pub fn into_null(self) -> VortexResult<NullArray> {
        match self {
            Canonical::Null(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap NullArray from {:?}", &self),
        }
    }

    pub fn into_bool(self) -> VortexResult<BoolArray> {
        match self {
            Canonical::Bool(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap BoolArray from {:?}", &self),
        }
    }

    pub fn into_primitive(self) -> VortexResult<PrimitiveArray> {
        match self {
            Canonical::Primitive(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap PrimitiveArray from {:?}", &self),
        }
    }

    pub fn into_decimal(self) -> VortexResult<DecimalArray> {
        match self {
            Canonical::Decimal(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap DecimalArray from {:?}", &self),
        }
    }

    pub fn into_struct(self) -> VortexResult<StructArray> {
        match self {
            Canonical::Struct(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap StructArray from {:?}", &self),
        }
    }

    pub fn into_list(self) -> VortexResult<ListArray> {
        match self {
            Canonical::List(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap ListArray from {:?}", &self),
        }
    }

    pub fn into_varbinview(self) -> VortexResult<VarBinViewArray> {
        match self {
            Canonical::VarBinView(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap VarBinViewArray from {:?}", &self),
        }
    }

    pub fn into_extension(self) -> VortexResult<ExtensionArray> {
        match self {
            Canonical::Extension(a) => Ok(a),
            _ => vortex_bail!("Cannot unwrap ExtensionArray from {:?}", &self),
        }
    }
}

impl AsRef<dyn Array> for Canonical {
    fn as_ref(&self) -> &(dyn Array + 'static) {
        match &self {
            Canonical::Null(a) => a.as_ref(),
            Canonical::Bool(a) => a.as_ref(),
            Canonical::Primitive(a) => a.as_ref(),
            Canonical::Decimal(a) => a.as_ref(),
            Canonical::Struct(a) => a.as_ref(),
            Canonical::List(a) => a.as_ref(),
            Canonical::VarBinView(a) => a.as_ref(),
            Canonical::Extension(a) => a.as_ref(),
        }
    }
}

impl IntoArray for Canonical {
    fn into_array(self) -> ArrayRef {
        match self {
            Canonical::Null(a) => a.into_array(),
            Canonical::Bool(a) => a.into_array(),
            Canonical::Primitive(a) => a.into_array(),
            Canonical::Decimal(a) => a.into_array(),
            Canonical::Struct(a) => a.into_array(),
            Canonical::List(a) => a.into_array(),
            Canonical::VarBinView(a) => a.into_array(),
            Canonical::Extension(a) => a.into_array(),
        }
    }
}

/// Trait for types that can be converted from an owned type into an owned array variant.
///
/// # Canonicalization
///
/// This trait has a blanket implementation for all types implementing [ToCanonical].
pub trait ToCanonical {
    /// Canonicalize into a [`NullArray`] if the target is [`Null`][DType::Null] typed.
    fn to_null(&self) -> VortexResult<NullArray>;

    /// Canonicalize into a [`BoolArray`] if the target is [`Bool`][DType::Bool] typed.
    fn to_bool(&self) -> VortexResult<BoolArray>;

    /// Canonicalize into a [`PrimitiveArray`] if the target is [`Primitive`][DType::Primitive]
    /// typed.
    fn to_primitive(&self) -> VortexResult<PrimitiveArray>;

    /// Canonicalize into a [`DecimalArray`] if the target is [`Decimal`][DType::Decimal]
    /// typed.
    fn to_decimal(&self) -> VortexResult<DecimalArray>;

    /// Canonicalize into a [`StructArray`] if the target is [`Struct`][DType::Struct] typed.
    fn to_struct(&self) -> VortexResult<StructArray>;

    /// Canonicalize into a [`ListArray`] if the target is [`List`][DType::List] typed.
    fn to_list(&self) -> VortexResult<ListArray>;

    /// Canonicalize into a [`VarBinViewArray`] if the target is [`Utf8`][DType::Utf8]
    /// or [`Binary`][DType::Binary] typed.
    fn to_varbinview(&self) -> VortexResult<VarBinViewArray>;

    /// Canonicalize into an [`ExtensionArray`] if the array is [`Extension`][DType::Extension]
    /// typed.
    fn to_extension(&self) -> VortexResult<ExtensionArray>;
}

// Blanket impl for all Array encodings.
impl<A: Array + ?Sized> ToCanonical for A {
    fn to_null(&self) -> VortexResult<NullArray> {
        self.to_canonical()?.into_null()
    }

    fn to_bool(&self) -> VortexResult<BoolArray> {
        self.to_canonical()?.into_bool()
    }

    fn to_primitive(&self) -> VortexResult<PrimitiveArray> {
        self.to_canonical()?.into_primitive()
    }

    fn to_decimal(&self) -> VortexResult<DecimalArray> {
        self.to_canonical()?.into_decimal()
    }

    fn to_struct(&self) -> VortexResult<StructArray> {
        self.to_canonical()?.into_struct()
    }

    fn to_list(&self) -> VortexResult<ListArray> {
        self.to_canonical()?.into_list()
    }

    fn to_varbinview(&self) -> VortexResult<VarBinViewArray> {
        self.to_canonical()?.into_varbinview()
    }

    fn to_extension(&self) -> VortexResult<ExtensionArray> {
        self.to_canonical()?.into_extension()
    }
}

impl From<Canonical> for ArrayRef {
    fn from(value: Canonical) -> Self {
        match value {
            Canonical::Null(a) => a.into_array(),
            Canonical::Bool(a) => a.into_array(),
            Canonical::Primitive(a) => a.into_array(),
            Canonical::Decimal(a) => a.into_array(),
            Canonical::Struct(a) => a.into_array(),
            Canonical::List(a) => a.into_array(),
            Canonical::VarBinView(a) => a.into_array(),
            Canonical::Extension(a) => a.into_array(),
        }
    }
}

#[cfg(test)]
mod test {
    use std::sync::Arc;

    use arrow_array::cast::AsArray;
    use arrow_array::types::{Int32Type, Int64Type, UInt64Type};
    use arrow_array::{
        Array as ArrowArray, ArrayRef as ArrowArrayRef, ListArray as ArrowListArray,
        PrimitiveArray as ArrowPrimitiveArray, StringArray, StringViewArray,
        StructArray as ArrowStructArray,
    };
    use arrow_buffer::{NullBufferBuilder, OffsetBuffer};
    use arrow_schema::{DataType, Field};
    use vortex_buffer::buffer;

    use crate::arrays::{ConstantArray, StructArray};
    use crate::arrow::{FromArrowArray, IntoArrowArray};
    use crate::{ArrayRef, IntoArray};

    #[test]
    fn test_canonicalize_nested_struct() {
        // Create a struct array with multiple internal components.
        let nested_struct_array = StructArray::from_fields(&[
            ("a", buffer![1u64].into_array()),
            (
                "b",
                StructArray::from_fields(&[(
                    "inner_a",
                    // The nested struct contains a ConstantArray representing the primitive array
                    //   [100i64]
                    // ConstantArray is not a canonical type, so converting `into_arrow()` should
                    // map this to the nearest canonical type (PrimitiveArray).
                    ConstantArray::new(100i64, 1).into_array(),
                )])
                .unwrap()
                .into_array(),
            ),
        ])
        .unwrap();

        let arrow_struct = nested_struct_array
            .into_array()
            .into_arrow_preferred()
            .unwrap()
            .as_any()
            .downcast_ref::<ArrowStructArray>()
            .cloned()
            .unwrap();

        assert!(
            arrow_struct
                .column(0)
                .as_any()
                .downcast_ref::<ArrowPrimitiveArray<UInt64Type>>()
                .is_some()
        );

        let inner_struct = arrow_struct
            .column(1)
            .clone()
            .as_any()
            .downcast_ref::<ArrowStructArray>()
            .cloned()
            .unwrap();

        let inner_a = inner_struct
            .column(0)
            .as_any()
            .downcast_ref::<ArrowPrimitiveArray<Int64Type>>();
        assert!(inner_a.is_some());

        assert_eq!(
            inner_a.cloned().unwrap(),
            ArrowPrimitiveArray::from_iter([100i64]),
        );
    }

    #[test]
    fn roundtrip_struct() {
        let mut nulls = NullBufferBuilder::new(6);
        nulls.append_n_non_nulls(4);
        nulls.append_null();
        nulls.append_non_null();
        let names = Arc::new(StringViewArray::from_iter(vec![
            Some("Joseph"),
            None,
            Some("Angela"),
            Some("Mikhail"),
            None,
            None,
        ]));
        let ages = Arc::new(ArrowPrimitiveArray::<Int32Type>::from(vec![
            Some(25),
            Some(31),
            None,
            Some(57),
            None,
            None,
        ]));

        let arrow_struct = ArrowStructArray::new(
            vec![
                Arc::new(Field::new("name", DataType::Utf8View, true)),
                Arc::new(Field::new("age", DataType::Int32, true)),
            ]
            .into(),
            vec![names, ages],
            nulls.finish(),
        );

        let vortex_struct = ArrayRef::from_arrow(&arrow_struct, true);

        assert_eq!(
            &arrow_struct,
            vortex_struct.into_arrow_preferred().unwrap().as_struct()
        );
    }

    #[test]
    fn roundtrip_list() {
        let names = Arc::new(StringArray::from_iter(vec![
            Some("Joseph"),
            Some("Angela"),
            Some("Mikhail"),
        ]));

        let arrow_list = ArrowListArray::new(
            Arc::new(Field::new_list_field(DataType::Utf8, true)),
            OffsetBuffer::from_lengths(vec![0, 2, 1]),
            names,
            None,
        );
        let list_data_type = arrow_list.data_type();

        let vortex_list = ArrayRef::from_arrow(&arrow_list, true);

        let rt_arrow_list = vortex_list.into_arrow(list_data_type).unwrap();

        assert_eq!(
            (Arc::new(arrow_list.clone()) as ArrowArrayRef).as_ref(),
            rt_arrow_list.as_ref()
        );
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	//! Encodings that enable zero-copy sharing of data with Arrow.
5
6	use vortex_dtype::DType;
7	use vortex_error::{VortexExpect, VortexResult, vortex_bail};
8
9	use crate::arrays::{
10	BoolArray, DecimalArray, ExtensionArray, ListArray, NullArray, PrimitiveArray, StructArray,
11	VarBinViewArray,
12	};
13	use crate::builders::builder_with_capacity;
14	use crate::{Array, ArrayRef, IntoArray};
15
16	/// An enum capturing the default uncompressed encodings for each [Vortex type][DType].
17	///
18	/// Any array can be decoded into canonical form via the [`to_canonical`][Array::to_canonical]
19	/// trait method. This is the simplest encoding for a type, and will not be compressed but may
20	/// contain compressed child arrays.
21	///
22	/// Canonical form is useful for doing type-specific compute where you need to know that all
23	/// elements are laid out decompressed and contiguous in memory.
24	///
25	/// # Laziness
26	///
27	/// Canonical form is not recursive, so while a `StructArray` is the canonical format for any
28	/// `Struct` type, individual column child arrays may still be compressed. This allows
29	/// compute over Vortex arrays to push decoding as late as possible, and ideally many child arrays
30	/// never need to be decoded into canonical form at all depending on the compute.
31	///
32	/// # Arrow interoperability
33	///
34	/// All of the Vortex canonical encodings have an equivalent Arrow encoding that can be built
35	/// zero-copy, and the corresponding Arrow array types can also be built directly.
36	///
37	/// The full list of canonical types and their equivalent Arrow array types are:
38	///
39	/// * `NullArray`: [`arrow_array::NullArray`]
40	/// * `BoolArray`: [`arrow_array::BooleanArray`]
41	/// * `PrimitiveArray`: [`arrow_array::PrimitiveArray`]
42	/// * `DecimalArray`: [`arrow_array::Decimal128Array`] and [`arrow_array::Decimal256Array`]
43	/// * `StructArray`: [`arrow_array::StructArray`]
44	/// * `ListArray`: [`arrow_array::ListArray`]
45	/// * `VarBinViewArray`: [`arrow_array::GenericByteViewArray`]
46	///
47	/// Vortex uses a logical type system, unlike Arrow which uses physical encodings for its types.
48	/// As an example, there are at least six valid physical encodings for a `Utf8` array. This can
49	/// create ambiguity.
50	/// Thus, if you receive an Arrow array, compress it using Vortex, and then
51	/// decompress it later to pass to a compute kernel, there are multiple suitable Arrow array
52	/// variants to hold the data.
53	///
54	/// To disambiguate, we choose a canonical physical encoding for every Vortex [`DType`], which
55	/// will correspond to an arrow-rs [`arrow_schema::DataType`].
56	///
57	/// # Views support
58	///
59	/// Binary and String views, also known as "German strings" are a better encoding format for
60	/// nearly all use-cases. Variable-length binary views are part of the Apache Arrow spec, and are
61	/// fully supported by the Datafusion query engine. We use them as our canonical string encoding
62	/// for all `Utf8` and `Binary` typed arrays in Vortex. They provide considerably faster filter
63	/// execution than the core `StringArray` and `BinaryArray` types, at the expense of potentially
64	/// needing [garbage collection][arrow_array::GenericByteViewArray::gc] to clear unreferenced items
65	/// from memory.
66	#[derive(Debug, Clone)]
67	pub enum Canonical {
68	Null(NullArray),
69	Bool(BoolArray),
70	Primitive(PrimitiveArray),
71	Decimal(DecimalArray),
72	Struct(StructArray),
73	// TODO(joe): maybe this should be a ListView, however this will be annoying in spiral
74	List(ListArray),
75	VarBinView(VarBinViewArray),
76	Extension(ExtensionArray),
77	}
78
79	impl Canonical {
80	/// Create an empty canonical array of the given dtype.
81	pub fn empty(dtype: &DType) -> Canonical {	600✔
82	builder_with_capacity(dtype, 0)	600✔
83	.finish()	600✔
84	.to_canonical()	600✔
85	.vortex_expect("cannot fail to convert an empty array to canonical")	600✔
86	}	600✔
87	}
88
89	impl Canonical {
90	/// Performs a (potentially expensive) compaction operation on the array before it is complete.
91	///
92	/// This is mostly relevant for the variable-length types such as Utf8, Binary or List where
93	/// they can accumulate wasted space after slicing and taking operations.
94	///
95	/// This operation is very expensive and can result in things like allocations, full-scans
96	/// and copy operations.
97	pub fn compact(&self) -> VortexResult<Canonical> {	20,970✔
98	match self {	20,970✔
99	Canonical::VarBinView(array) => Ok(Canonical::VarBinView(array.compact_buffers()?)),	3,504✔
100	Canonical::List(array) => Ok(Canonical::List(array.reset_offsets()?)),	37✔
101	other => Ok(other.clone()),	17,429✔
102	}
103	}	20,970✔
104	}
105
106	// Unwrap canonical type back down to specialized type.
107	impl Canonical {
108	pub fn into_null(self) -> VortexResult<NullArray> {	3✔
109	match self {	3✔
110	Canonical::Null(a) => Ok(a),	3✔
UNCOV 111	_ => vortex_bail!("Cannot unwrap NullArray from {:?}", &self),	×
112	}
113	}	3✔
114
115	pub fn into_bool(self) -> VortexResult<BoolArray> {	45,660✔
116	match self {	45,660✔
117	Canonical::Bool(a) => Ok(a),	45,660✔
UNCOV 118	_ => vortex_bail!("Cannot unwrap BoolArray from {:?}", &self),	×
119	}
120	}	45,660✔
121
122	pub fn into_primitive(self) -> VortexResult<PrimitiveArray> {	127,935✔
123	match self {	127,935✔
124	Canonical::Primitive(a) => Ok(a),	127,935✔
UNCOV 125	_ => vortex_bail!("Cannot unwrap PrimitiveArray from {:?}", &self),	×
126	}
127	}	127,935✔
128
129	pub fn into_decimal(self) -> VortexResult<DecimalArray> {	735✔
130	match self {	735✔
131	Canonical::Decimal(a) => Ok(a),	735✔
UNCOV 132	_ => vortex_bail!("Cannot unwrap DecimalArray from {:?}", &self),	×
133	}
134	}	735✔
135
136	pub fn into_struct(self) -> VortexResult<StructArray> {	25,828✔
137	match self {	25,828✔
138	Canonical::Struct(a) => Ok(a),	25,828✔
UNCOV 139	_ => vortex_bail!("Cannot unwrap StructArray from {:?}", &self),	×
140	}
141	}	25,828✔
142
143	pub fn into_list(self) -> VortexResult<ListArray> {	920✔
144	match self {	920✔
145	Canonical::List(a) => Ok(a),	920✔
UNCOV 146	_ => vortex_bail!("Cannot unwrap ListArray from {:?}", &self),	×
147	}
148	}	920✔
149
150	pub fn into_varbinview(self) -> VortexResult<VarBinViewArray> {	14,597✔
151	match self {	14,597✔
152	Canonical::VarBinView(a) => Ok(a),	14,597✔
UNCOV 153	_ => vortex_bail!("Cannot unwrap VarBinViewArray from {:?}", &self),	×
154	}
155	}	14,597✔
156
157	pub fn into_extension(self) -> VortexResult<ExtensionArray> {	×
158	match self {	×
159	Canonical::Extension(a) => Ok(a),	×
UNCOV 160	_ => vortex_bail!("Cannot unwrap ExtensionArray from {:?}", &self),	×
161	}
UNCOV 162	}	×
163	}
164
165	impl AsRef<dyn Array> for Canonical {
166	fn as_ref(&self) -> &(dyn Array + 'static) {	996,099✔
167	match &self {	996,099✔
168	Canonical::Null(a) => a.as_ref(),	9✔
169	Canonical::Bool(a) => a.as_ref(),	168,325✔
170	Canonical::Primitive(a) => a.as_ref(),	595,301✔
171	Canonical::Decimal(a) => a.as_ref(),	26,209✔
172	Canonical::Struct(a) => a.as_ref(),	99,948✔
173	Canonical::List(a) => a.as_ref(),	5,269✔
174	Canonical::VarBinView(a) => a.as_ref(),	93,652✔
175	Canonical::Extension(a) => a.as_ref(),	7,386✔
176	}
177	}	996,099✔
178	}
179
180	impl IntoArray for Canonical {
181	fn into_array(self) -> ArrayRef {	17,786✔
182	match self {	17,786✔
UNCOV 183	Canonical::Null(a) => a.into_array(),	×
184	Canonical::Bool(a) => a.into_array(),	371✔
185	Canonical::Primitive(a) => a.into_array(),	9,205✔
186	Canonical::Decimal(a) => a.into_array(),	2,186✔
187	Canonical::Struct(a) => a.into_array(),	186✔
188	Canonical::List(a) => a.into_array(),	305✔
189	Canonical::VarBinView(a) => a.into_array(),	4,535✔
190	Canonical::Extension(a) => a.into_array(),	998✔
191	}
192	}	17,786✔
193	}
194
195	/// Trait for types that can be converted from an owned type into an owned array variant.
196	///
197	/// # Canonicalization
198	///
199	/// This trait has a blanket implementation for all types implementing [ToCanonical].
200	pub trait ToCanonical {
201	/// Canonicalize into a [`NullArray`] if the target is [`Null`][DType::Null] typed.
202	fn to_null(&self) -> VortexResult<NullArray>;
203
204	/// Canonicalize into a [`BoolArray`] if the target is [`Bool`][DType::Bool] typed.
205	fn to_bool(&self) -> VortexResult<BoolArray>;
206
207	/// Canonicalize into a [`PrimitiveArray`] if the target is [`Primitive`][DType::Primitive]
208	/// typed.
209	fn to_primitive(&self) -> VortexResult<PrimitiveArray>;
210
211	/// Canonicalize into a [`DecimalArray`] if the target is [`Decimal`][DType::Decimal]
212	/// typed.
213	fn to_decimal(&self) -> VortexResult<DecimalArray>;
214
215	/// Canonicalize into a [`StructArray`] if the target is [`Struct`][DType::Struct] typed.
216	fn to_struct(&self) -> VortexResult<StructArray>;
217
218	/// Canonicalize into a [`ListArray`] if the target is [`List`][DType::List] typed.
219	fn to_list(&self) -> VortexResult<ListArray>;
220
221	/// Canonicalize into a [`VarBinViewArray`] if the target is [`Utf8`][DType::Utf8]
222	/// or [`Binary`][DType::Binary] typed.
223	fn to_varbinview(&self) -> VortexResult<VarBinViewArray>;
224
225	/// Canonicalize into an [`ExtensionArray`] if the array is [`Extension`][DType::Extension]
226	/// typed.
227	fn to_extension(&self) -> VortexResult<ExtensionArray>;
228	}
229
230	// Blanket impl for all Array encodings.
231	impl<A: Array + ?Sized> ToCanonical for A {
232	fn to_null(&self) -> VortexResult<NullArray> {	3✔
233	self.to_canonical()?.into_null()	3✔
234	}	3✔
235
236	fn to_bool(&self) -> VortexResult<BoolArray> {	44,880✔
237	self.to_canonical()?.into_bool()	44,880✔
238	}	44,880✔
239
240	fn to_primitive(&self) -> VortexResult<PrimitiveArray> {	125,378✔
241	self.to_canonical()?.into_primitive()	125,378✔
242	}	125,378✔
243
244	fn to_decimal(&self) -> VortexResult<DecimalArray> {	735✔
245	self.to_canonical()?.into_decimal()	735✔
246	}	735✔
247
248	fn to_struct(&self) -> VortexResult<StructArray> {	25,717✔
249	self.to_canonical()?.into_struct()	25,717✔
250	}	25,717✔
251
252	fn to_list(&self) -> VortexResult<ListArray> {	912✔
253	self.to_canonical()?.into_list()	912✔
254	}	912✔
255
256	fn to_varbinview(&self) -> VortexResult<VarBinViewArray> {	14,596✔
257	self.to_canonical()?.into_varbinview()	14,596✔
258	}	14,596✔
259
260	fn to_extension(&self) -> VortexResult<ExtensionArray> {	×
261	self.to_canonical()?.into_extension()	×
UNCOV 262	}	×
263	}
264
265	impl From<Canonical> for ArrayRef {
266	fn from(value: Canonical) -> Self {	×
267	match value {	×
268	Canonical::Null(a) => a.into_array(),	×
269	Canonical::Bool(a) => a.into_array(),	×
270	Canonical::Primitive(a) => a.into_array(),	×
271	Canonical::Decimal(a) => a.into_array(),	×
272	Canonical::Struct(a) => a.into_array(),	×
273	Canonical::List(a) => a.into_array(),	×
274	Canonical::VarBinView(a) => a.into_array(),	×
UNCOV 275	Canonical::Extension(a) => a.into_array(),	×
276	}
UNCOV 277	}	×
278	}
279
280	#[cfg(test)]
281	mod test {
282	use std::sync::Arc;
283
284	use arrow_array::cast::AsArray;
285	use arrow_array::types::{Int32Type, Int64Type, UInt64Type};
286	use arrow_array::{
287	Array as ArrowArray, ArrayRef as ArrowArrayRef, ListArray as ArrowListArray,
288	PrimitiveArray as ArrowPrimitiveArray, StringArray, StringViewArray,
289	StructArray as ArrowStructArray,
290	};
291	use arrow_buffer::{NullBufferBuilder, OffsetBuffer};
292	use arrow_schema::{DataType, Field};
293	use vortex_buffer::buffer;
294
295	use crate::arrays::{ConstantArray, StructArray};
296	use crate::arrow::{FromArrowArray, IntoArrowArray};
297	use crate::{ArrayRef, IntoArray};
298
299	#[test]
300	fn test_canonicalize_nested_struct() {	1✔
301	// Create a struct array with multiple internal components.
302	let nested_struct_array = StructArray::from_fields(&[	1✔
303	("a", buffer![1u64].into_array()),	1✔
304	(	1✔
305	"b",	1✔
306	StructArray::from_fields(&[(	1✔
307	"inner_a",	1✔
308	// The nested struct contains a ConstantArray representing the primitive array	1✔
309	// [100i64]	1✔
310	// ConstantArray is not a canonical type, so converting `into_arrow()` should	1✔
311	// map this to the nearest canonical type (PrimitiveArray).	1✔
312	ConstantArray::new(100i64, 1).into_array(),	1✔
313	)])	1✔
314	.unwrap()	1✔
315	.into_array(),	1✔
316	),	1✔
317	])	1✔
318	.unwrap();	1✔
319
320	let arrow_struct = nested_struct_array	1✔
321	.into_array()	1✔
322	.into_arrow_preferred()	1✔
323	.unwrap()	1✔
324	.as_any()	1✔
325	.downcast_ref::<ArrowStructArray>()	1✔
326	.cloned()	1✔
327	.unwrap();	1✔
328
329	assert!(	1✔
330	arrow_struct	1✔
331	.column(0)	1✔
332	.as_any()	1✔
333	.downcast_ref::<ArrowPrimitiveArray<UInt64Type>>()	1✔
334	.is_some()	1✔
335	);
336
337	let inner_struct = arrow_struct	1✔
338	.column(1)	1✔
339	.clone()	1✔
340	.as_any()	1✔
341	.downcast_ref::<ArrowStructArray>()	1✔
342	.cloned()	1✔
343	.unwrap();	1✔
344
345	let inner_a = inner_struct	1✔
346	.column(0)	1✔
347	.as_any()	1✔
348	.downcast_ref::<ArrowPrimitiveArray<Int64Type>>();	1✔
349	assert!(inner_a.is_some());	1✔
350
351	assert_eq!(	1✔
352	inner_a.cloned().unwrap(),	1✔
353	ArrowPrimitiveArray::from_iter([100i64]),	1✔
354	);
355	}	1✔
356
357	#[test]
358	fn roundtrip_struct() {	1✔
359	let mut nulls = NullBufferBuilder::new(6);	1✔
360	nulls.append_n_non_nulls(4);	1✔
361	nulls.append_null();	1✔
362	nulls.append_non_null();	1✔
363	let names = Arc::new(StringViewArray::from_iter(vec![	1✔
364	Some("Joseph"),	1✔
365	None,	1✔
366	Some("Angela"),	1✔
367	Some("Mikhail"),	1✔
368	None,	1✔
369	None,	1✔
370	]));
371	let ages = Arc::new(ArrowPrimitiveArray::<Int32Type>::from(vec![	1✔
372	Some(25),	1✔
373	Some(31),	1✔
374	None,	1✔
375	Some(57),	1✔
376	None,	1✔
377	None,	1✔
378	]));
379
380	let arrow_struct = ArrowStructArray::new(	1✔
381	vec![	1✔
382	Arc::new(Field::new("name", DataType::Utf8View, true)),	1✔
383	Arc::new(Field::new("age", DataType::Int32, true)),	1✔
384	]
385	.into(),	1✔
386	vec![names, ages],	1✔
387	nulls.finish(),	1✔
388	);
389
390	let vortex_struct = ArrayRef::from_arrow(&arrow_struct, true);	1✔
391
392	assert_eq!(	1✔
393	&arrow_struct,	1✔
394	vortex_struct.into_arrow_preferred().unwrap().as_struct()	1✔
395	);
396	}	1✔
397
398	#[test]
399	fn roundtrip_list() {	1✔
400	let names = Arc::new(StringArray::from_iter(vec![	1✔
401	Some("Joseph"),	1✔
402	Some("Angela"),	1✔
403	Some("Mikhail"),	1✔
404	]));
405
406	let arrow_list = ArrowListArray::new(	1✔
407	Arc::new(Field::new_list_field(DataType::Utf8, true)),	1✔
408	OffsetBuffer::from_lengths(vec![0, 2, 1]),	1✔
409	names,	1✔
410	None,	1✔
411	);
412	let list_data_type = arrow_list.data_type();	1✔
413
414	let vortex_list = ArrayRef::from_arrow(&arrow_list, true);	1✔
415
416	let rt_arrow_list = vortex_list.into_arrow(list_data_type).unwrap();	1✔
417
418	assert_eq!(	1✔
419	(Arc::new(arrow_list.clone()) as ArrowArrayRef).as_ref(),	1✔
420	rt_arrow_list.as_ref()	1✔
421	);
422	}	1✔
423	}

vortex-data / vortex / 16578751259

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous