16262933935

Committed 14 Jul 2025 09:17AM UTC coverage: 81.432% (+0.3%) from 81.15%

Build # 16262933935

Build Type

Pull #3844

github

Committed by

web-flow

Commit Message

Merge a49b86c01 into 382a2c489

Pull Request Pull Request #3844: feat[compressor]: wire up sequence array into the compressor

Run Details

155 of 159 new or added lines in 8 files covered. (97.48%)

217 existing lines in 24 files now uncovered.

46015 of 56507 relevant lines covered (81.43%)

146742.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.41

/vortex-array/src/arrays/decimal/mod.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

mod compute;
mod ops;
mod patch;
mod serde;

use arrow_buffer::BooleanBufferBuilder;
use vortex_buffer::{Buffer, BufferMut, ByteBuffer};
use vortex_dtype::{DType, DecimalDType};
use vortex_error::{VortexResult, vortex_panic};
use vortex_scalar::{DecimalValueType, NativeDecimalType};

use crate::builders::ArrayBuilder;
use crate::stats::{ArrayStats, StatsSetRef};
use crate::validity::Validity;
use crate::vtable::{
    ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityHelper,
    ValidityVTableFromValidityHelper, VisitorVTable,
};
use crate::{ArrayBufferVisitor, ArrayChildVisitor, Canonical, EncodingId, EncodingRef, vtable};

vtable!(Decimal);

impl VTable for DecimalVTable {
    type Array = DecimalArray;
    type Encoding = DecimalEncoding;

    type ArrayVTable = Self;
    type CanonicalVTable = Self;
    type OperationsVTable = Self;
    type ValidityVTable = ValidityVTableFromValidityHelper;
    type VisitorVTable = Self;
    type ComputeVTable = NotSupported;
    type EncodeVTable = NotSupported;
    type SerdeVTable = Self;

    fn id(_encoding: &Self::Encoding) -> EncodingId {
        EncodingId::new_ref("vortex.decimal")
    }

    fn encoding(_array: &Self::Array) -> EncodingRef {
        EncodingRef::new_ref(DecimalEncoding.as_ref())
    }
}

#[derive(Clone, Debug)]
pub struct DecimalEncoding;

/// Maps a decimal precision into the smallest type that can represent it.
pub fn smallest_storage_type(decimal_dtype: &DecimalDType) -> DecimalValueType {
    match decimal_dtype.precision() {
        1..=2 => DecimalValueType::I8,
        3..=4 => DecimalValueType::I16,
        5..=9 => DecimalValueType::I32,
        10..=18 => DecimalValueType::I64,
        19..=38 => DecimalValueType::I128,
        39..=76 => DecimalValueType::I256,
        0 => unreachable!("precision must be greater than 0"),
        p => unreachable!("precision larger than 76 is invalid found precision {p}"),
    }
}

/// True if `value_type` can represent every value of the type `dtype`.
pub fn compatible_storage_type(value_type: DecimalValueType, dtype: DecimalDType) -> bool {
    value_type >= smallest_storage_type(&dtype)
}

/// Array for decimal-typed real numbers
#[derive(Clone, Debug)]
pub struct DecimalArray {
    dtype: DType,
    values: ByteBuffer,
    values_type: DecimalValueType,
    validity: Validity,
    stats_set: ArrayStats,
}

impl DecimalArray {
    /// Creates a new [`DecimalArray`] from a [`Buffer`] and [`Validity`], without checking
    /// any invariants.
    ///
    /// # Panics
    ///
    /// Panics if the validity length is not compatible with the buffer length.
    pub fn new<T: NativeDecimalType>(
        buffer: Buffer<T>,
        decimal_dtype: DecimalDType,
        validity: Validity,
    ) -> Self {
        if let Some(len) = validity.maybe_len() {
            if buffer.len() != len {
                vortex_panic!(
                    "Buffer and validity length mismatch: buffer={}, validity={}",
                    buffer.len(),
                    len,
                );
            }
        }

        Self {
            dtype: DType::Decimal(decimal_dtype, validity.nullability()),
            values: buffer.into_byte_buffer(),
            values_type: T::VALUES_TYPE,
            validity,
            stats_set: ArrayStats::default(),
        }
    }

    /// Returns the underlying [`ByteBuffer`] of the array.
    pub fn byte_buffer(&self) -> ByteBuffer {
        self.values.clone()
    }

    pub fn buffer<T: NativeDecimalType>(&self) -> Buffer<T> {
        if self.values_type != T::VALUES_TYPE {
            vortex_panic!(
                "Cannot extract Buffer<{:?}> for DecimalArray with values_type {:?}",
                T::VALUES_TYPE,
                self.values_type,
            );
        }
        Buffer::<T>::from_byte_buffer(self.values.clone())
    }

    /// Returns the decimal type information
    pub fn decimal_dtype(&self) -> DecimalDType {
        match &self.dtype {
            DType::Decimal(decimal_dtype, _) => *decimal_dtype,
            _ => vortex_panic!("Expected Decimal dtype, got {:?}", self.dtype),
        }
    }

    pub fn values_type(&self) -> DecimalValueType {
        self.values_type
    }

    pub fn precision(&self) -> u8 {
        self.decimal_dtype().precision()
    }

    pub fn scale(&self) -> i8 {
        self.decimal_dtype().scale()
    }

    pub fn from_option_iter<T: NativeDecimalType, I: IntoIterator<Item = Option<T>>>(
        iter: I,
        decimal_dtype: DecimalDType,
    ) -> Self {
        let iter = iter.into_iter();
        let mut values = BufferMut::with_capacity(iter.size_hint().0);
        let mut validity = BooleanBufferBuilder::new(values.capacity());

        for i in iter {
            match i {
                None => {
                    validity.append(false);
                    values.push(T::default());
                }
                Some(e) => {
                    validity.append(true);
                    values.push(e);
                }
            }
        }
        Self::new(
            values.freeze(),
            decimal_dtype,
            Validity::from(validity.finish()),
        )
    }
}

impl ArrayVTable<DecimalVTable> for DecimalVTable {
    fn len(array: &DecimalArray) -> usize {
        let divisor = match array.values_type {
            DecimalValueType::I8 => 1,
            DecimalValueType::I16 => 2,
            DecimalValueType::I32 => 4,
            DecimalValueType::I64 => 8,
            DecimalValueType::I128 => 16,
            DecimalValueType::I256 => 32,
            ty => vortex_panic!("unknown decimal value type {:?}", ty),
        };
        array.values.len() / divisor
    }

    fn dtype(array: &DecimalArray) -> &DType {
        &array.dtype
    }

    fn stats(array: &DecimalArray) -> StatsSetRef<'_> {
        array.stats_set.to_ref(array.as_ref())
    }
}

impl VisitorVTable<DecimalVTable> for DecimalVTable {
    fn visit_buffers(array: &DecimalArray, visitor: &mut dyn ArrayBufferVisitor) {
        visitor.visit_buffer(&array.values);
    }

    fn visit_children(array: &DecimalArray, visitor: &mut dyn ArrayChildVisitor) {
        visitor.visit_validity(array.validity(), array.len())
    }
}

impl CanonicalVTable<DecimalVTable> for DecimalVTable {
    fn canonicalize(array: &DecimalArray) -> VortexResult<Canonical> {
        Ok(Canonical::Decimal(array.clone()))
    }

    fn append_to_builder(array: &DecimalArray, builder: &mut dyn ArrayBuilder) -> VortexResult<()> {
        builder.extend_from_array(array.as_ref())
    }
}

impl ValidityHelper for DecimalArray {
    fn validity(&self) -> &Validity {
        &self.validity
    }
}

#[cfg(test)]
mod test {
    use arrow_array::Decimal128Array;

    #[test]
    fn test_decimal() {
        // They pass it b/c the DType carries the information. No other way to carry a
        // dtype except via the array.
        let value = Decimal128Array::new_null(100);
        let numeric = value.value(10);
        assert_eq!(numeric, 0i128);
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	mod compute;
5	mod ops;
6	mod patch;
7	mod serde;
8
9	use arrow_buffer::BooleanBufferBuilder;
10	use vortex_buffer::{Buffer, BufferMut, ByteBuffer};
11	use vortex_dtype::{DType, DecimalDType};
12	use vortex_error::{VortexResult, vortex_panic};
13	use vortex_scalar::{DecimalValueType, NativeDecimalType};
14
15	use crate::builders::ArrayBuilder;
16	use crate::stats::{ArrayStats, StatsSetRef};
17	use crate::validity::Validity;
18	use crate::vtable::{
19	ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityHelper,
20	ValidityVTableFromValidityHelper, VisitorVTable,
21	};
22	use crate::{ArrayBufferVisitor, ArrayChildVisitor, Canonical, EncodingId, EncodingRef, vtable};
23
24	vtable!(Decimal);
25
26	impl VTable for DecimalVTable {
27	type Array = DecimalArray;
28	type Encoding = DecimalEncoding;
29
30	type ArrayVTable = Self;
31	type CanonicalVTable = Self;
32	type OperationsVTable = Self;
33	type ValidityVTable = ValidityVTableFromValidityHelper;
34	type VisitorVTable = Self;
35	type ComputeVTable = NotSupported;
36	type EncodeVTable = NotSupported;
37	type SerdeVTable = Self;
38
39	fn id(_encoding: &Self::Encoding) -> EncodingId {	40,684✔
40	EncodingId::new_ref("vortex.decimal")	40,684✔
41	}	40,684✔
42
43	fn encoding(_array: &Self::Array) -> EncodingRef {	5,319✔
44	EncodingRef::new_ref(DecimalEncoding.as_ref())	5,319✔
45	}	5,319✔
46	}
47
48	#[derive(Clone, Debug)]
49	pub struct DecimalEncoding;
50
51	/// Maps a decimal precision into the smallest type that can represent it.
52	pub fn smallest_storage_type(decimal_dtype: &DecimalDType) -> DecimalValueType {	958✔
53	match decimal_dtype.precision() {	958✔
54	1..=2 => DecimalValueType::I8,	958✔
55	3..=4 => DecimalValueType::I16,	814✔
56	5..=9 => DecimalValueType::I32,	598✔
57	10..=18 => DecimalValueType::I64,	454✔
58	19..=38 => DecimalValueType::I128,	144✔
59	39..=76 => DecimalValueType::I256,	×
60	0 => unreachable!("precision must be greater than 0"),	×
UNCOV 61	p => unreachable!("precision larger than 76 is invalid found precision {p}"),	×
62	}
63	}	958✔
64
65	/// True if `value_type` can represent every value of the type `dtype`.
66	pub fn compatible_storage_type(value_type: DecimalValueType, dtype: DecimalDType) -> bool {	36✔
67	value_type >= smallest_storage_type(&dtype)	36✔
68	}	36✔
69
70	/// Array for decimal-typed real numbers
71	#[derive(Clone, Debug)]
72	pub struct DecimalArray {
73	dtype: DType,
74	values: ByteBuffer,
75	values_type: DecimalValueType,
76	validity: Validity,
77	stats_set: ArrayStats,
78	}
79
80	impl DecimalArray {
81	/// Creates a new [`DecimalArray`] from a [`Buffer`] and [`Validity`], without checking
82	/// any invariants.
83	///
84	/// # Panics
85	///
86	/// Panics if the validity length is not compatible with the buffer length.
87	pub fn new<T: NativeDecimalType>(	4,106✔
88	buffer: Buffer<T>,	4,106✔
89	decimal_dtype: DecimalDType,	4,106✔
90	validity: Validity,	4,106✔
91	) -> Self {	4,106✔
92	if let Some(len) = validity.maybe_len() {	4,106✔
93	if buffer.len() != len {	436✔
UNCOV 94	vortex_panic!(	×
UNCOV 95	"Buffer and validity length mismatch: buffer={}, validity={}",	×
UNCOV 96	buffer.len(),	×
UNCOV 97	len,	×
UNCOV 98	);	×
99	}	436✔
100	}	3,670✔
101
102	Self {	4,106✔
103	dtype: DType::Decimal(decimal_dtype, validity.nullability()),	4,106✔
104	values: buffer.into_byte_buffer(),	4,106✔
105	values_type: T::VALUES_TYPE,	4,106✔
106	validity,	4,106✔
107	stats_set: ArrayStats::default(),	4,106✔
108	}	4,106✔
109	}	4,106✔
110
111	/// Returns the underlying [`ByteBuffer`] of the array.
112	pub fn byte_buffer(&self) -> ByteBuffer {	1✔
113	self.values.clone()	1✔
114	}	1✔
115
116	pub fn buffer<T: NativeDecimalType>(&self) -> Buffer<T> {	6,613✔
117	if self.values_type != T::VALUES_TYPE {	6,613✔
UNCOV 118	vortex_panic!(	×
UNCOV 119	"Cannot extract Buffer<{:?}> for DecimalArray with values_type {:?}",	×
UNCOV 120	T::VALUES_TYPE,	×
121	self.values_type,	×
UNCOV 122	);	×
123	}	6,613✔
124	Buffer::<T>::from_byte_buffer(self.values.clone())	6,613✔
125	}	6,613✔
126
127	/// Returns the decimal type information
128	pub fn decimal_dtype(&self) -> DecimalDType {	8,233✔
129	match &self.dtype {	8,233✔
130	DType::Decimal(decimal_dtype, _) => *decimal_dtype,	8,233✔
131	_ => vortex_panic!("Expected Decimal dtype, got {:?}", self.dtype),	×
132	}
133	}	8,233✔
134
135	pub fn values_type(&self) -> DecimalValueType {	6,238✔
136	self.values_type	6,238✔
137	}	6,238✔
138
139	pub fn precision(&self) -> u8 {	×
140	self.decimal_dtype().precision()	×
141	}	×
142
143	pub fn scale(&self) -> i8 {	×
UNCOV 144	self.decimal_dtype().scale()	×
145	}	×
146
147	pub fn from_option_iter<T: NativeDecimalType, I: IntoIterator<Item = Option<T>>>(	×
148	iter: I,	×
149	decimal_dtype: DecimalDType,	×
150	) -> Self {	×
151	let iter = iter.into_iter();	×
152	let mut values = BufferMut::with_capacity(iter.size_hint().0);	×
153	let mut validity = BooleanBufferBuilder::new(values.capacity());	×
154
UNCOV 155	for i in iter {	×
UNCOV 156	match i {	×
157	None => {	×
158	validity.append(false);	×
159	values.push(T::default());	×
160	}	×
161	Some(e) => {	×
162	validity.append(true);	×
UNCOV 163	values.push(e);	×
UNCOV 164	}	×
165	}
166	}
UNCOV 167	Self::new(	×
UNCOV 168	values.freeze(),	×
UNCOV 169	decimal_dtype,	×
UNCOV 170	Validity::from(validity.finish()),	×
UNCOV 171	)	×
UNCOV 172	}	×
173	}
174
175	impl ArrayVTable<DecimalVTable> for DecimalVTable {
176	fn len(array: &DecimalArray) -> usize {	49,712✔
177	let divisor = match array.values_type {	49,712✔
178	DecimalValueType::I8 => 1,	5,834✔
179	DecimalValueType::I16 => 2,	5,026✔
180	DecimalValueType::I32 => 4,	6,166✔
181	DecimalValueType::I64 => 8,	4,828✔
182	DecimalValueType::I128 => 16,	27,848✔
183	DecimalValueType::I256 => 32,	10✔
UNCOV 184	ty => vortex_panic!("unknown decimal value type {:?}", ty),	×
185	};
186	array.values.len() / divisor	49,712✔
187	}	49,712✔
188
189	fn dtype(array: &DecimalArray) -> &DType {	38,857✔
190	&array.dtype	38,857✔
191	}	38,857✔
192
193	fn stats(array: &DecimalArray) -> StatsSetRef<'_> {	31,914✔
194	array.stats_set.to_ref(array.as_ref())	31,914✔
195	}	31,914✔
196	}
197
198	impl VisitorVTable<DecimalVTable> for DecimalVTable {
199	fn visit_buffers(array: &DecimalArray, visitor: &mut dyn ArrayBufferVisitor) {	3,842✔
200	visitor.visit_buffer(&array.values);	3,842✔
201	}	3,842✔
202
203	fn visit_children(array: &DecimalArray, visitor: &mut dyn ArrayChildVisitor) {	3,915✔
204	visitor.visit_validity(array.validity(), array.len())	3,915✔
205	}	3,915✔
206	}
207
208	impl CanonicalVTable<DecimalVTable> for DecimalVTable {
209	fn canonicalize(array: &DecimalArray) -> VortexResult<Canonical> {	5,122✔
210	Ok(Canonical::Decimal(array.clone()))	5,122✔
211	}	5,122✔
212
213	fn append_to_builder(array: &DecimalArray, builder: &mut dyn ArrayBuilder) -> VortexResult<()> {	660✔
214	builder.extend_from_array(array.as_ref())	660✔
215	}	660✔
216	}
217
218	impl ValidityHelper for DecimalArray {
219	fn validity(&self) -> &Validity {	12,035✔
220	&self.validity	12,035✔
221	}	12,035✔
222	}
223
224	#[cfg(test)]
225	mod test {
226	use arrow_array::Decimal128Array;
227
228	#[test]
229	fn test_decimal() {	1✔
230	// They pass it b/c the DType carries the information. No other way to carry a	1✔
231	// dtype except via the array.	1✔
232	let value = Decimal128Array::new_null(100);	1✔
233	let numeric = value.value(10);	1✔
234	assert_eq!(numeric, 0i128);	1✔
235	}	1✔
236	}

vortex-data / vortex / 16262933935

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous