17075133033

Committed 19 Aug 2025 04:01PM UTC coverage: 87.949% (+0.09%) from 87.856%

Build # 17075133033

Build Type

push

github

Committed by

web-flow

Commit Message

feat: ArrayOperations infallible, eager validation + new_unchecked (#4177)

ArrayOperations currently return VortexResult<>, but they really should
just be infallible. A failed array op is generally indicative of
programmer or encoding error. There's really nothing interesting we can
do to handle an out-of-bounds slice() or scalar_at.

There's a lot that falls out of this, like fixing a bunch of tests,
tweaking our scalar value casting to return Option instead of Result,
etc.

---------

Signed-off-by: Andrew Duffy <andrew@a10y.dev>

Run Details

1744 of 1985 new or added lines in 195 files covered. (87.86%)

36 existing lines in 27 files now uncovered.

56745 of 64520 relevant lines covered (87.95%)

624082.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

29.31

/vortex-array/src/arrow/array.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::fmt::Debug;

use arrow_array::ArrayRef as ArrowArrayRef;
use vortex_dtype::arrow::FromArrowType;
use vortex_dtype::{DType, Nullability};
use vortex_error::{VortexResult, vortex_panic};
use vortex_mask::Mask;
use vortex_scalar::Scalar;

use crate::arrow::FromArrowArray;
use crate::stats::{ArrayStats, StatsSetRef};
use crate::vtable::{
    ArrayVTable, CanonicalVTable, NotSupported, OperationsVTable, VTable, ValidityVTable,
    VisitorVTable,
};
use crate::{
    Array, ArrayBufferVisitor, ArrayChildVisitor, ArrayRef, Canonical, EncodingId, EncodingRef,
    IntoArray, vtable,
};

vtable!(Arrow);

impl VTable for ArrowVTable {
    type Array = ArrowArray;
    type Encoding = ArrowEncoding;
    type ArrayVTable = Self;
    type CanonicalVTable = Self;
    type OperationsVTable = Self;
    type ValidityVTable = Self;
    type VisitorVTable = Self;
    type ComputeVTable = NotSupported;
    type EncodeVTable = NotSupported;
    type SerdeVTable = NotSupported;

    fn id(_encoding: &Self::Encoding) -> EncodingId {
        EncodingId::new_ref("vortex.arrow")
    }

    fn encoding(_array: &Self::Array) -> EncodingRef {
        EncodingRef::new_ref(ArrowEncoding.as_ref())
    }
}

/// A Vortex array that wraps an in-memory Arrow array.
// TODO(ngates): consider having each Arrow encoding be a separate encoding ID.
#[derive(Clone, Debug)]
pub struct ArrowEncoding;

#[derive(Clone, Debug)]
pub struct ArrowArray {
    inner: ArrowArrayRef,
    dtype: DType,
    stats_set: ArrayStats,
}

impl ArrowArray {
    pub fn new(arrow_array: ArrowArrayRef, nullability: Nullability) -> Self {
        let dtype = DType::from_arrow((arrow_array.data_type(), nullability));
        Self {
            inner: arrow_array,
            dtype,
            stats_set: Default::default(),
        }
    }

    pub fn inner(&self) -> &ArrowArrayRef {
        &self.inner
    }
}

impl ArrayVTable<ArrowVTable> for ArrowVTable {
    fn len(array: &ArrowArray) -> usize {
        array.inner.len()
    }

    fn dtype(array: &ArrowArray) -> &DType {
        &array.dtype
    }

    fn stats(array: &ArrowArray) -> StatsSetRef<'_> {
        array.stats_set.to_ref(array.as_ref())
    }
}

impl CanonicalVTable<ArrowVTable> for ArrowVTable {
    fn canonicalize(array: &ArrowArray) -> VortexResult<Canonical> {
        ArrayRef::from_arrow(array.inner.as_ref(), array.dtype.is_nullable()).to_canonical()
    }
}

impl OperationsVTable<ArrowVTable> for ArrowVTable {
    fn slice(array: &ArrowArray, start: usize, stop: usize) -> ArrayRef {
        let inner = array.inner.slice(start, stop - start);
        let new_array = ArrowArray {
            inner,
            dtype: array.dtype.clone(),
            stats_set: Default::default(),
        };
        new_array.into_array()
    }

    fn scalar_at(_array: &ArrowArray, _index: usize) -> Scalar {
        vortex_panic!("Not supported")
    }
}

impl ValidityVTable<ArrowVTable> for ArrowVTable {
    fn is_valid(array: &ArrowArray, index: usize) -> VortexResult<bool> {
        Ok(array.inner.is_valid(index))
    }

    fn all_valid(array: &ArrowArray) -> VortexResult<bool> {
        Ok(array.inner.logical_null_count() == 0)
    }

    fn all_invalid(array: &ArrowArray) -> VortexResult<bool> {
        Ok(array.inner.logical_null_count() == array.inner.len())
    }

    fn validity_mask(array: &ArrowArray) -> VortexResult<Mask> {
        Ok(array
            .inner
            .logical_nulls()
            .map(|null_buffer| Mask::from_buffer(null_buffer.inner().clone()))
            .unwrap_or_else(|| Mask::new_true(array.inner.len())))
    }
}

impl VisitorVTable<ArrowVTable> for ArrowVTable {
    fn visit_buffers(_array: &ArrowArray, _visitor: &mut dyn ArrayBufferVisitor) {}

    fn visit_children(_array: &ArrowArray, _visitor: &mut dyn ArrayChildVisitor) {}
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::fmt::Debug;
5
6	use arrow_array::ArrayRef as ArrowArrayRef;
7	use vortex_dtype::arrow::FromArrowType;
8	use vortex_dtype::{DType, Nullability};
9	use vortex_error::{VortexResult, vortex_panic};
10	use vortex_mask::Mask;
11	use vortex_scalar::Scalar;
12
13	use crate::arrow::FromArrowArray;
14	use crate::stats::{ArrayStats, StatsSetRef};
15	use crate::vtable::{
16	ArrayVTable, CanonicalVTable, NotSupported, OperationsVTable, VTable, ValidityVTable,
17	VisitorVTable,
18	};
19	use crate::{
20	Array, ArrayBufferVisitor, ArrayChildVisitor, ArrayRef, Canonical, EncodingId, EncodingRef,
21	IntoArray, vtable,
22	};
23
24	vtable!(Arrow);
25
26	impl VTable for ArrowVTable {
27	type Array = ArrowArray;
28	type Encoding = ArrowEncoding;
29	type ArrayVTable = Self;
30	type CanonicalVTable = Self;
31	type OperationsVTable = Self;
32	type ValidityVTable = Self;
33	type VisitorVTable = Self;
34	type ComputeVTable = NotSupported;
35	type EncodeVTable = NotSupported;
36	type SerdeVTable = NotSupported;
37
38	fn id(_encoding: &Self::Encoding) -> EncodingId {	×
39	EncodingId::new_ref("vortex.arrow")	×
40	}	×
41
42	fn encoding(_array: &Self::Array) -> EncodingRef {	×
43	EncodingRef::new_ref(ArrowEncoding.as_ref())	×
44	}	×
45	}
46
47	/// A Vortex array that wraps an in-memory Arrow array.
48	// TODO(ngates): consider having each Arrow encoding be a separate encoding ID.
49	#[derive(Clone, Debug)]
50	pub struct ArrowEncoding;
51
52	#[derive(Clone, Debug)]
53	pub struct ArrowArray {
54	inner: ArrowArrayRef,
55	dtype: DType,
56	stats_set: ArrayStats,
57	}
58
59	impl ArrowArray {
60	pub fn new(arrow_array: ArrowArrayRef, nullability: Nullability) -> Self {	524,205✔
61	let dtype = DType::from_arrow((arrow_array.data_type(), nullability));	524,205✔
62	Self {	524,205✔
63	inner: arrow_array,	524,205✔
64	dtype,	524,205✔
65	stats_set: Default::default(),	524,205✔
66	}	524,205✔
67	}	524,205✔
68
69	pub fn inner(&self) -> &ArrowArrayRef {	524,205✔
70	&self.inner	524,205✔
71	}	524,205✔
72	}
73
74	impl ArrayVTable<ArrowVTable> for ArrowVTable {
75	fn len(array: &ArrowArray) -> usize {	524,205✔
76	array.inner.len()	524,205✔
77	}	524,205✔
78
79	fn dtype(array: &ArrowArray) -> &DType {	524,205✔
80	&array.dtype	524,205✔
81	}	524,205✔
82
83	fn stats(array: &ArrowArray) -> StatsSetRef<'_> {	×
84	array.stats_set.to_ref(array.as_ref())	×
85	}	×
86	}
87
88	impl CanonicalVTable<ArrowVTable> for ArrowVTable {
89	fn canonicalize(array: &ArrowArray) -> VortexResult<Canonical> {	×
90	ArrayRef::from_arrow(array.inner.as_ref(), array.dtype.is_nullable()).to_canonical()	×
91	}	×
92	}
93
94	impl OperationsVTable<ArrowVTable> for ArrowVTable {
NEW 95	fn slice(array: &ArrowArray, start: usize, stop: usize) -> ArrayRef {	×
96	let inner = array.inner.slice(start, stop - start);	×
97	let new_array = ArrowArray {	×
98	inner,	×
99	dtype: array.dtype.clone(),	×
100	stats_set: Default::default(),	×
101	};	×
NEW 102	new_array.into_array()	×
103	}	×
104
NEW 105	fn scalar_at(_array: &ArrowArray, _index: usize) -> Scalar {	×
NEW 106	vortex_panic!("Not supported")	×
107	}
108	}
109
110	impl ValidityVTable<ArrowVTable> for ArrowVTable {
111	fn is_valid(array: &ArrowArray, index: usize) -> VortexResult<bool> {	×
112	Ok(array.inner.is_valid(index))	×
113	}	×
114
115	fn all_valid(array: &ArrowArray) -> VortexResult<bool> {	×
116	Ok(array.inner.logical_null_count() == 0)	×
117	}	×
118
119	fn all_invalid(array: &ArrowArray) -> VortexResult<bool> {	×
120	Ok(array.inner.logical_null_count() == array.inner.len())	×
121	}	×
122
123	fn validity_mask(array: &ArrowArray) -> VortexResult<Mask> {	×
124	Ok(array	×
125	.inner	×
126	.logical_nulls()	×
127	.map(\|null_buffer\| Mask::from_buffer(null_buffer.inner().clone()))	×
128	.unwrap_or_else(\|\| Mask::new_true(array.inner.len())))	×
129	}	×
130	}
131
132	impl VisitorVTable<ArrowVTable> for ArrowVTable {
133	fn visit_buffers(_array: &ArrowArray, _visitor: &mut dyn ArrayBufferVisitor) {}	×
134
135	fn visit_children(_array: &ArrowArray, _visitor: &mut dyn ArrayChildVisitor) {}	×
136	}

vortex-data / vortex / 17075133033

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous