16935267080

Committed 13 Aug 2025 11:00AM UTC coverage: 24.312% (-63.3%) from 87.658%

Build # 16935267080

Build Type

Pull #4226

github

Committed by

web-flow

Commit Message

Merge 81b48c7fb into baa6ea202

Pull Request Pull Request #4226: Support converting TimestampTZ to and from duckdb

Run Details

0 of 2 new or added lines in 1 file covered. (0.0%)

20666 existing lines in 469 files now uncovered.

8726 of 35892 relevant lines covered (24.31%)

147.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/vortex-btrblocks/src/float/dictionary.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Float-specific dictionary encoding implementation.

use vortex_array::IntoArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::validity::Validity;
use vortex_array::vtable::ValidityHelper;
use vortex_buffer::Buffer;
use vortex_dict::DictArray;
use vortex_dtype::half::f16;
use vortex_error::VortexResult;

use crate::float::stats::{ErasedDistinctValues, FloatStats};

macro_rules! typed_encode {
    ($stats:ident, $typed:ident, $validity:ident, $typ:ty) => {{
        let values: Buffer<$typ> = $typed.values.iter().map(|x| x.0).collect();

        let max_code = values.len();
        let codes = if max_code <= u8::MAX as usize {
            let buf =
                <DictEncoder as Encode<$typ, u8>>::encode(&values, $stats.src.as_slice::<$typ>());
            PrimitiveArray::new(buf, $validity.clone()).into_array()
        } else if max_code <= u16::MAX as usize {
            let buf =
                <DictEncoder as Encode<$typ, u16>>::encode(&values, $stats.src.as_slice::<$typ>());
            PrimitiveArray::new(buf, $validity.clone()).into_array()
        } else {
            let buf =
                <DictEncoder as Encode<$typ, u32>>::encode(&values, $stats.src.as_slice::<$typ>());
            PrimitiveArray::new(buf, $validity.clone()).into_array()
        };

        let values_validity = match $validity {
            Validity::NonNullable => Validity::NonNullable,
            _ => Validity::AllValid,
        };
        let values = PrimitiveArray::new(values, values_validity).into_array();

        DictArray::try_new(codes, values)
    }};
}

pub fn dictionary_encode(stats: &FloatStats) -> VortexResult<DictArray> {
    let validity = stats.src.validity();
    match &stats.distinct_values {
        ErasedDistinctValues::F16(typed) => typed_encode!(stats, typed, validity, f16),
        ErasedDistinctValues::F32(typed) => typed_encode!(stats, typed, validity, f32),
        ErasedDistinctValues::F64(typed) => typed_encode!(stats, typed, validity, f64),
    }
}

struct DictEncoder;

trait Encode<T, I> {
    /// Using the distinct value set, turn the values into a set of codes.
    fn encode(distinct: &[T], values: &[T]) -> Buffer<I>;
}

macro_rules! impl_encode {
    ($typ:ty, $utyp:ty) => { impl_encode!($typ, $utyp, u8, u16, u32); };
    ($typ:ty, $utyp:ty, $($ityp:ty),+) => {
        $(
        impl Encode<$typ, $ityp> for DictEncoder {
            #[allow(clippy::cast_possible_truncation)]
            fn encode(distinct: &[$typ], values: &[$typ]) -> Buffer<$ityp> {
                let mut codes =
                    vortex_utils::aliases::hash_map::HashMap::<$utyp, $ityp>::with_capacity(
                        distinct.len(),
                    );
                for (code, &value) in distinct.iter().enumerate() {
                    codes.insert(value.to_bits(), code as $ityp);
                }

                let mut output = vortex_buffer::BufferMut::with_capacity(values.len());
                for value in values {
                    // Any code lookups which fail are for nulls, so their value
                    // does not matter.
                    output.push(codes.get(&value.to_bits()).copied().unwrap_or_default());
                }

                return output.freeze();
            }
        }
        )*
    };
}

impl_encode!(f16, u16);
impl_encode!(f32, u32);
impl_encode!(f64, u64);

#[cfg(test)]
mod tests {
    use vortex_array::arrays::{BoolArray, PrimitiveArray};
    use vortex_array::validity::Validity;
    use vortex_array::{Array, IntoArray, ToCanonical};
    use vortex_buffer::buffer;

    use crate::CompressorStats;
    use crate::float::dictionary::dictionary_encode;
    use crate::float::stats::FloatStats;

    #[test]
    fn test_float_dict_encode() {
        // Create an array that has some nulls
        let values = buffer![1f32, 2f32, 2f32, 0f32, 1f32];
        let validity =
            Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array());
        let array = PrimitiveArray::new(values, validity);

        let stats = FloatStats::generate(&array);
        let dict_array = dictionary_encode(&stats).unwrap();
        assert_eq!(dict_array.values().len(), 2);
        assert_eq!(dict_array.codes().len(), 5);

        let undict = dict_array.to_primitive().unwrap();

        // We just use code zero but it doesn't really matter.
        // We can just shove a whole validity buffer in there instead.
        assert_eq!(undict.as_slice::<f32>(), &[1f32, 2f32, 2f32, 1f32, 1f32]);
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	//! Float-specific dictionary encoding implementation.
5
6	use vortex_array::IntoArray;
7	use vortex_array::arrays::PrimitiveArray;
8	use vortex_array::validity::Validity;
9	use vortex_array::vtable::ValidityHelper;
10	use vortex_buffer::Buffer;
11	use vortex_dict::DictArray;
12	use vortex_dtype::half::f16;
13	use vortex_error::VortexResult;
14
15	use crate::float::stats::{ErasedDistinctValues, FloatStats};
16
17	macro_rules! typed_encode {
18	($stats:ident, $typed:ident, $validity:ident, $typ:ty) => {{
19	let values: Buffer<$typ> = $typed.values.iter().map(\|x\| x.0).collect();
20
21	let max_code = values.len();
22	let codes = if max_code <= u8::MAX as usize {
23	let buf =
24	<DictEncoder as Encode<$typ, u8>>::encode(&values, $stats.src.as_slice::<$typ>());
25	PrimitiveArray::new(buf, $validity.clone()).into_array()
26	} else if max_code <= u16::MAX as usize {
27	let buf =
28	<DictEncoder as Encode<$typ, u16>>::encode(&values, $stats.src.as_slice::<$typ>());
29	PrimitiveArray::new(buf, $validity.clone()).into_array()
30	} else {
31	let buf =
32	<DictEncoder as Encode<$typ, u32>>::encode(&values, $stats.src.as_slice::<$typ>());
33	PrimitiveArray::new(buf, $validity.clone()).into_array()
34	};
35
36	let values_validity = match $validity {
37	Validity::NonNullable => Validity::NonNullable,
38	_ => Validity::AllValid,
39	};
40	let values = PrimitiveArray::new(values, values_validity).into_array();
41
42	DictArray::try_new(codes, values)
43	}};
44	}
45
UNCOV 46	pub fn dictionary_encode(stats: &FloatStats) -> VortexResult<DictArray> {	×
UNCOV 47	let validity = stats.src.validity();	×
UNCOV 48	match &stats.distinct_values {	×
49	ErasedDistinctValues::F16(typed) => typed_encode!(stats, typed, validity, f16),	×
UNCOV 50	ErasedDistinctValues::F32(typed) => typed_encode!(stats, typed, validity, f32),	×
51	ErasedDistinctValues::F64(typed) => typed_encode!(stats, typed, validity, f64),	×
52	}
UNCOV 53	}	×
54
55	struct DictEncoder;
56
57	trait Encode<T, I> {
58	/// Using the distinct value set, turn the values into a set of codes.
59	fn encode(distinct: &[T], values: &[T]) -> Buffer<I>;
60	}
61
62	macro_rules! impl_encode {
63	($typ:ty, $utyp:ty) => { impl_encode!($typ, $utyp, u8, u16, u32); };
64	($typ:ty, $utyp:ty, $($ityp:ty),+) => {
65	$(
66	impl Encode<$typ, $ityp> for DictEncoder {
67	#[allow(clippy::cast_possible_truncation)]
UNCOV 68	fn encode(distinct: &[$typ], values: &[$typ]) -> Buffer<$ityp> {	×
UNCOV 69	let mut codes =	×
UNCOV 70	vortex_utils::aliases::hash_map::HashMap::<$utyp, $ityp>::with_capacity(	×
UNCOV 71	distinct.len(),	×
72	);
UNCOV 73	for (code, &value) in distinct.iter().enumerate() {	×
UNCOV 74	codes.insert(value.to_bits(), code as $ityp);	×
UNCOV 75	}	×
76
UNCOV 77	let mut output = vortex_buffer::BufferMut::with_capacity(values.len());	×
UNCOV 78	for value in values {	×
UNCOV 79	// Any code lookups which fail are for nulls, so their value	×
UNCOV 80	// does not matter.	×
UNCOV 81	output.push(codes.get(&value.to_bits()).copied().unwrap_or_default());	×
UNCOV 82	}	×
83
UNCOV 84	return output.freeze();	×
UNCOV 85	}	×
86	}
87	)*
88	};
89	}
90
91	impl_encode!(f16, u16);
92	impl_encode!(f32, u32);
93	impl_encode!(f64, u64);
94
95	#[cfg(test)]
96	mod tests {
97	use vortex_array::arrays::{BoolArray, PrimitiveArray};
98	use vortex_array::validity::Validity;
99	use vortex_array::{Array, IntoArray, ToCanonical};
100	use vortex_buffer::buffer;
101
102	use crate::CompressorStats;
103	use crate::float::dictionary::dictionary_encode;
104	use crate::float::stats::FloatStats;
105
106	#[test]
107	fn test_float_dict_encode() {
108	// Create an array that has some nulls
109	let values = buffer![1f32, 2f32, 2f32, 0f32, 1f32];
110	let validity =
111	Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array());
112	let array = PrimitiveArray::new(values, validity);
113
114	let stats = FloatStats::generate(&array);
115	let dict_array = dictionary_encode(&stats).unwrap();
116	assert_eq!(dict_array.values().len(), 2);
117	assert_eq!(dict_array.codes().len(), 5);
118
119	let undict = dict_array.to_primitive().unwrap();
120
121	// We just use code zero but it doesn't really matter.
122	// We can just shove a whole validity buffer in there instead.
123	assert_eq!(undict.as_slice::<f32>(), &[1f32, 2f32, 2f32, 1f32, 1f32]);
124	}
125	}

vortex-data / vortex / 16935267080

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous