16728097825

Committed 04 Aug 2025 04:00PM UTC coverage: 48.355% (-35.1%) from 83.429%

Build # 16728097825

Build Type

Pull #4108

github

Committed by

web-flow

Commit Message

Merge 1b2d27fd8 into 649ba9576

Pull Request Pull Request #4108: perf[vortex-array]: use all_valid instead of `invalid_count() == 0`

Coverage Stats

1 of 1 new or added line in 1 file covered. (100.0%)

11596 existing lines in 378 files now uncovered.

18635 of 38538 relevant lines covered (48.35%)

151786.4 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/vortex-btrblocks/src/float.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

mod dictionary;
mod stats;

use vortex_alp::{ALPArray, ALPEncoding, ALPVTable, RDEncoder};
use vortex_array::arrays::{ConstantArray, PrimitiveVTable};
use vortex_array::{ArrayRef, IntoArray, ToCanonical};
use vortex_dict::DictArray;
use vortex_dtype::PType;
use vortex_error::{VortexExpect, VortexResult, vortex_panic};

use self::stats::FloatStats;
use crate::float::dictionary::dictionary_encode;
use crate::integer::{IntCompressor, IntegerStats};
use crate::patches::compress_patches;
use crate::{
    Compressor, CompressorStats, GenerateStatsOptions, Scheme,
    estimate_compression_ratio_with_sampling, integer,
};

pub trait FloatScheme: Scheme<StatsType = FloatStats, CodeType = FloatCode> {}

impl<T> FloatScheme for T where T: Scheme<StatsType = FloatStats, CodeType = FloatCode> {}

pub struct FloatCompressor;

impl Compressor for FloatCompressor {
    type ArrayVTable = PrimitiveVTable;
    type SchemeType = dyn FloatScheme;
    type StatsType = FloatStats;

    fn schemes() -> &'static [&'static Self::SchemeType] {
        &[
            &UncompressedScheme,
            &ConstantScheme,
            &ALPScheme,
            &ALPRDScheme,
            &DictScheme,
        ]
    }

    fn default_scheme() -> &'static Self::SchemeType {
        &UncompressedScheme
    }

    fn dict_scheme_code() -> FloatCode {
        DICT_SCHEME
    }
}

const UNCOMPRESSED_SCHEME: FloatCode = FloatCode(0);
const CONSTANT_SCHEME: FloatCode = FloatCode(1);
const ALP_SCHEME: FloatCode = FloatCode(2);
const ALPRD_SCHEME: FloatCode = FloatCode(3);
const DICT_SCHEME: FloatCode = FloatCode(4);
const RUNEND_SCHEME: FloatCode = FloatCode(5);

#[derive(Debug, Copy, Clone)]
struct UncompressedScheme;

#[derive(Debug, Copy, Clone)]
struct ConstantScheme;

#[derive(Debug, Copy, Clone)]
struct ALPScheme;

#[derive(Debug, Copy, Clone)]
struct ALPRDScheme;

#[derive(Debug, Copy, Clone)]
struct DictScheme;

impl Scheme for UncompressedScheme {
    type StatsType = FloatStats;
    type CodeType = FloatCode;

    fn code(&self) -> FloatCode {
        UNCOMPRESSED_SCHEME
    }

    fn expected_compression_ratio(
        &self,
        _stats: &Self::StatsType,
        _is_sample: bool,
        _allowed_cascading: usize,
        _excludes: &[FloatCode],
    ) -> VortexResult<f64> {
        Ok(1.0)
    }

    fn compress(
        &self,
        stats: &Self::StatsType,
        _is_sample: bool,
        _allowed_cascading: usize,
        _excludes: &[FloatCode],
    ) -> VortexResult<ArrayRef> {
        Ok(stats.source().to_array())
    }
}

impl Scheme for ConstantScheme {
    type StatsType = FloatStats;
    type CodeType = FloatCode;

    fn code(&self) -> FloatCode {
        CONSTANT_SCHEME
    }

    fn expected_compression_ratio(
        &self,
        stats: &Self::StatsType,
        is_sample: bool,
        _allowed_cascading: usize,
        _excludes: &[FloatCode],
    ) -> VortexResult<f64> {
        // Never select Constant when sampling
        if is_sample {
            return Ok(0.0);
        }

        // Can only have 1 distinct value
        if stats.distinct_values_count > 1 {
            return Ok(0.0);
        }

        // Cannot have mix of nulls and non-nulls
        if stats.null_count > 0 && stats.value_count > 0 {
            return Ok(0.0);
        }

        Ok(stats.value_count as f64)
    }

    fn compress(
        &self,
        stats: &Self::StatsType,
        _is_sample: bool,
        _allowed_cascading: usize,
        _excludes: &[FloatCode],
    ) -> VortexResult<ArrayRef> {
        let scalar = stats
            .source()
            .as_constant()
            .vortex_expect("must be constant");

        Ok(ConstantArray::new(scalar, stats.source().len()).into_array())
    }
}

#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub struct FloatCode(u8);

impl Scheme for ALPScheme {
    type StatsType = FloatStats;
    type CodeType = FloatCode;

    fn code(&self) -> FloatCode {
        ALP_SCHEME
    }

    fn expected_compression_ratio(
        &self,
        stats: &Self::StatsType,
        is_sample: bool,
        allowed_cascading: usize,
        excludes: &[FloatCode],
    ) -> VortexResult<f64> {
        // We don't support ALP for f16
        if stats.source().ptype() == PType::F16 {
            return Ok(0.0);
        }

        if allowed_cascading == 0 {
            // ALP does not compress on its own, we need to be able to cascade it with
            // an integer compressor.
            return Ok(0.0);
        }

        estimate_compression_ratio_with_sampling(
            self,
            stats,
            is_sample,
            allowed_cascading,
            excludes,
        )
    }

    fn compress(
        &self,
        stats: &FloatStats,
        is_sample: bool,
        allowed_cascading: usize,
        excludes: &[FloatCode],
    ) -> VortexResult<ArrayRef> {
        let alp_encoded = ALPEncoding
            .encode(&stats.source().to_canonical()?, None)?
            .vortex_expect("Input is a supported floating point array");
        let alp = alp_encoded.as_::<ALPVTable>();
        let alp_ints = alp.encoded().to_primitive()?;

        // Compress the ALP ints.
        // Patches are not compressed. They should be infrequent, and if they are not then we want
        // to keep them linear for easy indexing.
        let mut int_excludes = Vec::new();
        if excludes.contains(&DICT_SCHEME) {
            int_excludes.push(integer::DictScheme.code());
        }
        if excludes.contains(&RUNEND_SCHEME) {
            int_excludes.push(integer::RunEndScheme.code());
        }

        let compressed_alp_ints =
            IntCompressor::compress(&alp_ints, is_sample, allowed_cascading - 1, &int_excludes)?;

        let patches = alp.patches().map(compress_patches).transpose()?;

        Ok(ALPArray::try_new(compressed_alp_ints, alp.exponents(), patches)?.into_array())
    }
}

impl Scheme for ALPRDScheme {
    type StatsType = FloatStats;
    type CodeType = FloatCode;

    fn code(&self) -> FloatCode {
        ALPRD_SCHEME
    }

    fn expected_compression_ratio(
        &self,
        stats: &Self::StatsType,
        is_sample: bool,
        allowed_cascading: usize,
        excludes: &[FloatCode],
    ) -> VortexResult<f64> {
        if stats.source().ptype() == PType::F16 {
            return Ok(0.0);
        }

        estimate_compression_ratio_with_sampling(
            self,
            stats,
            is_sample,
            allowed_cascading,
            excludes,
        )
    }

    fn compress(
        &self,
        stats: &Self::StatsType,
        _is_sample: bool,
        _allowed_cascading: usize,
        _excludes: &[FloatCode],
    ) -> VortexResult<ArrayRef> {
        let encoder = match stats.source().ptype() {
            PType::F32 => RDEncoder::new(stats.source().as_slice::<f32>()),
            PType::F64 => RDEncoder::new(stats.source().as_slice::<f64>()),
            ptype => vortex_panic!("cannot ALPRD compress ptype {ptype}"),
        };

        let mut alp_rd = encoder.encode(stats.source());

        let patches = alp_rd
            .left_parts_patches()
            .map(compress_patches)
            .transpose()?;
        alp_rd.replace_left_parts_patches(patches);

        Ok(alp_rd.into_array())
    }
}

impl Scheme for DictScheme {
    type StatsType = FloatStats;
    type CodeType = FloatCode;

    fn code(&self) -> FloatCode {
        DICT_SCHEME
    }

    fn expected_compression_ratio(
        &self,
        stats: &Self::StatsType,
        is_sample: bool,
        allowed_cascading: usize,
        excludes: &[FloatCode],
    ) -> VortexResult<f64> {
        if stats.value_count == 0 {
            return Ok(0.0);
        }

        // If the array is high cardinality (>50% unique values) skip.
        if stats.distinct_values_count > stats.value_count / 2 {
            return Ok(0.0);
        }

        // Take a sample and run compression on the sample to determine before/after size.
        estimate_compression_ratio_with_sampling(
            self,
            stats,
            is_sample,
            allowed_cascading,
            excludes,
        )
    }

    fn compress(
        &self,
        stats: &Self::StatsType,
        is_sample: bool,
        allowed_cascading: usize,
        _excludes: &[FloatCode],
    ) -> VortexResult<ArrayRef> {
        let dict_array = dictionary_encode(stats)?;

        // Only compress the codes.
        let codes_stats = IntegerStats::generate_opts(
            &dict_array.codes().to_primitive()?,
            GenerateStatsOptions {
                count_distinct_values: false,
            },
        );
        let codes_scheme = IntCompressor::choose_scheme(
            &codes_stats,
            is_sample,
            allowed_cascading - 1,
            &[integer::DictScheme.code(), integer::SequenceScheme.code()],
        )?;
        let compressed_codes = codes_scheme.compress(
            &codes_stats,
            is_sample,
            allowed_cascading - 1,
            &[integer::DictScheme.code()],
        )?;

        let compressed_values = FloatCompressor::compress(
            &dict_array.values().to_primitive()?,
            is_sample,
            allowed_cascading - 1,
            &[DICT_SCHEME],
        )?;

        Ok(DictArray::try_new(compressed_codes, compressed_values)?.into_array())
    }
}

#[cfg(test)]
mod tests {
    use vortex_array::arrays::PrimitiveArray;
    use vortex_array::validity::Validity;
    use vortex_array::{Array, IntoArray, ToCanonical};
    use vortex_buffer::{Buffer, buffer_mut};

    use crate::float::FloatCompressor;
    use crate::{Compressor, MAX_CASCADE};

    #[test]
    fn test_empty() {
        // Make sure empty array compression does not fail
        let result = FloatCompressor::compress(
            &PrimitiveArray::new(Buffer::<f32>::empty(), Validity::NonNullable),
            false,
            3,
            &[],
        )
        .unwrap();

        assert!(result.is_empty());
    }

    #[test]
    fn test_compress() {
        let mut values = buffer_mut![1.0f32; 1024];
        // Sprinkle some other values in.
        for i in 0..1024 {
            // Insert 2.0 at all odd positions.
            // This should force dictionary encoding and exclude run-end due to the
            // average run length being 1.
            values[i] = (i % 50) as f32;
        }

        let floats = values.into_array().to_primitive().unwrap();
        let compressed = FloatCompressor::compress(&floats, false, MAX_CASCADE, &[]).unwrap();
        println!("compressed: {}", compressed.display_tree())
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	mod dictionary;
5	mod stats;
6
7	use vortex_alp::{ALPArray, ALPEncoding, ALPVTable, RDEncoder};
8	use vortex_array::arrays::{ConstantArray, PrimitiveVTable};
9	use vortex_array::{ArrayRef, IntoArray, ToCanonical};
10	use vortex_dict::DictArray;
11	use vortex_dtype::PType;
12	use vortex_error::{VortexExpect, VortexResult, vortex_panic};
13
14	use self::stats::FloatStats;
15	use crate::float::dictionary::dictionary_encode;
16	use crate::integer::{IntCompressor, IntegerStats};
17	use crate::patches::compress_patches;
18	use crate::{
19	Compressor, CompressorStats, GenerateStatsOptions, Scheme,
20	estimate_compression_ratio_with_sampling, integer,
21	};
22
23	pub trait FloatScheme: Scheme<StatsType = FloatStats, CodeType = FloatCode> {}
24
25	impl<T> FloatScheme for T where T: Scheme<StatsType = FloatStats, CodeType = FloatCode> {}
26
27	pub struct FloatCompressor;
28
29	impl Compressor for FloatCompressor {
30	type ArrayVTable = PrimitiveVTable;
31	type SchemeType = dyn FloatScheme;
32	type StatsType = FloatStats;
33
UNCOV 34	fn schemes() -> &'static [&'static Self::SchemeType] {	×
UNCOV 35	&[	×
UNCOV 36	&UncompressedScheme,	×
UNCOV 37	&ConstantScheme,	×
UNCOV 38	&ALPScheme,	×
UNCOV 39	&ALPRDScheme,	×
UNCOV 40	&DictScheme,	×
UNCOV 41	]	×
UNCOV 42	}	×
43
UNCOV 44	fn default_scheme() -> &'static Self::SchemeType {	×
UNCOV 45	&UncompressedScheme	×
UNCOV 46	}	×
47
UNCOV 48	fn dict_scheme_code() -> FloatCode {	×
UNCOV 49	DICT_SCHEME	×
UNCOV 50	}	×
51	}
52
53	const UNCOMPRESSED_SCHEME: FloatCode = FloatCode(0);
54	const CONSTANT_SCHEME: FloatCode = FloatCode(1);
55	const ALP_SCHEME: FloatCode = FloatCode(2);
56	const ALPRD_SCHEME: FloatCode = FloatCode(3);
57	const DICT_SCHEME: FloatCode = FloatCode(4);
58	const RUNEND_SCHEME: FloatCode = FloatCode(5);
59
60	#[derive(Debug, Copy, Clone)]
61	struct UncompressedScheme;
62
63	#[derive(Debug, Copy, Clone)]
64	struct ConstantScheme;
65
66	#[derive(Debug, Copy, Clone)]
67	struct ALPScheme;
68
69	#[derive(Debug, Copy, Clone)]
70	struct ALPRDScheme;
71
72	#[derive(Debug, Copy, Clone)]
73	struct DictScheme;
74
75	impl Scheme for UncompressedScheme {
76	type StatsType = FloatStats;
77	type CodeType = FloatCode;
78
UNCOV 79	fn code(&self) -> FloatCode {	×
UNCOV 80	UNCOMPRESSED_SCHEME	×
UNCOV 81	}	×
82
UNCOV 83	fn expected_compression_ratio(	×
UNCOV 84	&self,	×
UNCOV 85	_stats: &Self::StatsType,	×
UNCOV 86	_is_sample: bool,	×
UNCOV 87	_allowed_cascading: usize,	×
UNCOV 88	_excludes: &[FloatCode],	×
UNCOV 89	) -> VortexResult<f64> {	×
UNCOV 90	Ok(1.0)	×
UNCOV 91	}	×
92
UNCOV 93	fn compress(	×
UNCOV 94	&self,	×
UNCOV 95	stats: &Self::StatsType,	×
UNCOV 96	_is_sample: bool,	×
UNCOV 97	_allowed_cascading: usize,	×
UNCOV 98	_excludes: &[FloatCode],	×
UNCOV 99	) -> VortexResult<ArrayRef> {	×
UNCOV 100	Ok(stats.source().to_array())	×
UNCOV 101	}	×
102	}
103
104	impl Scheme for ConstantScheme {
105	type StatsType = FloatStats;
106	type CodeType = FloatCode;
107
UNCOV 108	fn code(&self) -> FloatCode {	×
UNCOV 109	CONSTANT_SCHEME	×
UNCOV 110	}	×
111
UNCOV 112	fn expected_compression_ratio(	×
UNCOV 113	&self,	×
UNCOV 114	stats: &Self::StatsType,	×
UNCOV 115	is_sample: bool,	×
UNCOV 116	_allowed_cascading: usize,	×
UNCOV 117	_excludes: &[FloatCode],	×
UNCOV 118	) -> VortexResult<f64> {	×
119	// Never select Constant when sampling
UNCOV 120	if is_sample {	×
UNCOV 121	return Ok(0.0);	×
UNCOV 122	}	×
123
124	// Can only have 1 distinct value
UNCOV 125	if stats.distinct_values_count > 1 {	×
UNCOV 126	return Ok(0.0);	×
UNCOV 127	}	×
128
129	// Cannot have mix of nulls and non-nulls
UNCOV 130	if stats.null_count > 0 && stats.value_count > 0 {	×
131	return Ok(0.0);	×
UNCOV 132	}	×
133
UNCOV 134	Ok(stats.value_count as f64)	×
UNCOV 135	}	×
136
137	fn compress(	×
138	&self,	×
139	stats: &Self::StatsType,	×
140	_is_sample: bool,	×
141	_allowed_cascading: usize,	×
142	_excludes: &[FloatCode],	×
143	) -> VortexResult<ArrayRef> {	×
144	let scalar = stats	×
145	.source()	×
146	.as_constant()	×
147	.vortex_expect("must be constant");	×
148
149	Ok(ConstantArray::new(scalar, stats.source().len()).into_array())	×
150	}	×
151	}
152
153	#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
154	pub struct FloatCode(u8);
155
156	impl Scheme for ALPScheme {
157	type StatsType = FloatStats;
158	type CodeType = FloatCode;
159
UNCOV 160	fn code(&self) -> FloatCode {	×
UNCOV 161	ALP_SCHEME	×
UNCOV 162	}	×
163
UNCOV 164	fn expected_compression_ratio(	×
UNCOV 165	&self,	×
UNCOV 166	stats: &Self::StatsType,	×
UNCOV 167	is_sample: bool,	×
UNCOV 168	allowed_cascading: usize,	×
UNCOV 169	excludes: &[FloatCode],	×
UNCOV 170	) -> VortexResult<f64> {	×
171	// We don't support ALP for f16
UNCOV 172	if stats.source().ptype() == PType::F16 {	×
UNCOV 173	return Ok(0.0);	×
UNCOV 174	}	×
175
UNCOV 176	if allowed_cascading == 0 {	×
177	// ALP does not compress on its own, we need to be able to cascade it with
178	// an integer compressor.
179	return Ok(0.0);	×
UNCOV 180	}	×
181
UNCOV 182	estimate_compression_ratio_with_sampling(	×
UNCOV 183	self,	×
UNCOV 184	stats,	×
UNCOV 185	is_sample,	×
UNCOV 186	allowed_cascading,	×
UNCOV 187	excludes,	×
188	)
UNCOV 189	}	×
190
UNCOV 191	fn compress(	×
UNCOV 192	&self,	×
UNCOV 193	stats: &FloatStats,	×
UNCOV 194	is_sample: bool,	×
UNCOV 195	allowed_cascading: usize,	×
UNCOV 196	excludes: &[FloatCode],	×
UNCOV 197	) -> VortexResult<ArrayRef> {	×
UNCOV 198	let alp_encoded = ALPEncoding	×
UNCOV 199	.encode(&stats.source().to_canonical()?, None)?	×
UNCOV 200	.vortex_expect("Input is a supported floating point array");	×
UNCOV 201	let alp = alp_encoded.as_::<ALPVTable>();	×
UNCOV 202	let alp_ints = alp.encoded().to_primitive()?;	×
203
204	// Compress the ALP ints.
205	// Patches are not compressed. They should be infrequent, and if they are not then we want
206	// to keep them linear for easy indexing.
UNCOV 207	let mut int_excludes = Vec::new();	×
UNCOV 208	if excludes.contains(&DICT_SCHEME) {	×
UNCOV 209	int_excludes.push(integer::DictScheme.code());	×
UNCOV 210	}	×
UNCOV 211	if excludes.contains(&RUNEND_SCHEME) {	×
212	int_excludes.push(integer::RunEndScheme.code());	×
UNCOV 213	}	×
214
UNCOV 215	let compressed_alp_ints =	×
UNCOV 216	IntCompressor::compress(&alp_ints, is_sample, allowed_cascading - 1, &int_excludes)?;	×
217
UNCOV 218	let patches = alp.patches().map(compress_patches).transpose()?;	×
219
UNCOV 220	Ok(ALPArray::try_new(compressed_alp_ints, alp.exponents(), patches)?.into_array())	×
UNCOV 221	}	×
222	}
223
224	impl Scheme for ALPRDScheme {
225	type StatsType = FloatStats;
226	type CodeType = FloatCode;
227
UNCOV 228	fn code(&self) -> FloatCode {	×
UNCOV 229	ALPRD_SCHEME	×
UNCOV 230	}	×
231
UNCOV 232	fn expected_compression_ratio(	×
UNCOV 233	&self,	×
UNCOV 234	stats: &Self::StatsType,	×
UNCOV 235	is_sample: bool,	×
UNCOV 236	allowed_cascading: usize,	×
UNCOV 237	excludes: &[FloatCode],	×
UNCOV 238	) -> VortexResult<f64> {	×
UNCOV 239	if stats.source().ptype() == PType::F16 {	×
UNCOV 240	return Ok(0.0);	×
UNCOV 241	}	×
242
UNCOV 243	estimate_compression_ratio_with_sampling(	×
UNCOV 244	self,	×
UNCOV 245	stats,	×
UNCOV 246	is_sample,	×
UNCOV 247	allowed_cascading,	×
UNCOV 248	excludes,	×
249	)
UNCOV 250	}	×
251
UNCOV 252	fn compress(	×
UNCOV 253	&self,	×
UNCOV 254	stats: &Self::StatsType,	×
UNCOV 255	_is_sample: bool,	×
UNCOV 256	_allowed_cascading: usize,	×
UNCOV 257	_excludes: &[FloatCode],	×
UNCOV 258	) -> VortexResult<ArrayRef> {	×
UNCOV 259	let encoder = match stats.source().ptype() {	×
UNCOV 260	PType::F32 => RDEncoder::new(stats.source().as_slice::<f32>()),	×
UNCOV 261	PType::F64 => RDEncoder::new(stats.source().as_slice::<f64>()),	×
262	ptype => vortex_panic!("cannot ALPRD compress ptype {ptype}"),	×
263	};
264
UNCOV 265	let mut alp_rd = encoder.encode(stats.source());	×
266
UNCOV 267	let patches = alp_rd	×
UNCOV 268	.left_parts_patches()	×
UNCOV 269	.map(compress_patches)	×
UNCOV 270	.transpose()?;	×
UNCOV 271	alp_rd.replace_left_parts_patches(patches);	×
272
UNCOV 273	Ok(alp_rd.into_array())	×
UNCOV 274	}	×
275	}
276
277	impl Scheme for DictScheme {
278	type StatsType = FloatStats;
279	type CodeType = FloatCode;
280
UNCOV 281	fn code(&self) -> FloatCode {	×
UNCOV 282	DICT_SCHEME	×
UNCOV 283	}	×
284
UNCOV 285	fn expected_compression_ratio(	×
UNCOV 286	&self,	×
UNCOV 287	stats: &Self::StatsType,	×
UNCOV 288	is_sample: bool,	×
UNCOV 289	allowed_cascading: usize,	×
UNCOV 290	excludes: &[FloatCode],	×
UNCOV 291	) -> VortexResult<f64> {	×
UNCOV 292	if stats.value_count == 0 {	×
UNCOV 293	return Ok(0.0);	×
UNCOV 294	}	×
295
296	// If the array is high cardinality (>50% unique values) skip.
UNCOV 297	if stats.distinct_values_count > stats.value_count / 2 {	×
UNCOV 298	return Ok(0.0);	×
UNCOV 299	}	×
300
301	// Take a sample and run compression on the sample to determine before/after size.
UNCOV 302	estimate_compression_ratio_with_sampling(	×
UNCOV 303	self,	×
UNCOV 304	stats,	×
UNCOV 305	is_sample,	×
UNCOV 306	allowed_cascading,	×
UNCOV 307	excludes,	×
308	)
UNCOV 309	}	×
310
UNCOV 311	fn compress(	×
UNCOV 312	&self,	×
UNCOV 313	stats: &Self::StatsType,	×
UNCOV 314	is_sample: bool,	×
UNCOV 315	allowed_cascading: usize,	×
UNCOV 316	_excludes: &[FloatCode],	×
UNCOV 317	) -> VortexResult<ArrayRef> {	×
UNCOV 318	let dict_array = dictionary_encode(stats)?;	×
319
320	// Only compress the codes.
UNCOV 321	let codes_stats = IntegerStats::generate_opts(	×
UNCOV 322	&dict_array.codes().to_primitive()?,	×
UNCOV 323	GenerateStatsOptions {	×
UNCOV 324	count_distinct_values: false,	×
UNCOV 325	},	×
326	);
UNCOV 327	let codes_scheme = IntCompressor::choose_scheme(	×
UNCOV 328	&codes_stats,	×
UNCOV 329	is_sample,	×
UNCOV 330	allowed_cascading - 1,	×
UNCOV 331	&[integer::DictScheme.code(), integer::SequenceScheme.code()],	×
332	)?;	×
UNCOV 333	let compressed_codes = codes_scheme.compress(	×
UNCOV 334	&codes_stats,	×
UNCOV 335	is_sample,	×
UNCOV 336	allowed_cascading - 1,	×
UNCOV 337	&[integer::DictScheme.code()],	×
338	)?;	×
339
UNCOV 340	let compressed_values = FloatCompressor::compress(	×
UNCOV 341	&dict_array.values().to_primitive()?,	×
UNCOV 342	is_sample,	×
UNCOV 343	allowed_cascading - 1,	×
UNCOV 344	&[DICT_SCHEME],	×
345	)?;	×
346
UNCOV 347	Ok(DictArray::try_new(compressed_codes, compressed_values)?.into_array())	×
UNCOV 348	}	×
349	}
350
351	#[cfg(test)]
352	mod tests {
353	use vortex_array::arrays::PrimitiveArray;
354	use vortex_array::validity::Validity;
355	use vortex_array::{Array, IntoArray, ToCanonical};
356	use vortex_buffer::{Buffer, buffer_mut};
357
358	use crate::float::FloatCompressor;
359	use crate::{Compressor, MAX_CASCADE};
360
361	#[test]
362	fn test_empty() {
363	// Make sure empty array compression does not fail
364	let result = FloatCompressor::compress(
365	&PrimitiveArray::new(Buffer::<f32>::empty(), Validity::NonNullable),
366	false,
367	3,
368	&[],
369	)
370	.unwrap();
371
372	assert!(result.is_empty());
373	}
374
375	#[test]
376	fn test_compress() {
377	let mut values = buffer_mut![1.0f32; 1024];
378	// Sprinkle some other values in.
379	for i in 0..1024 {
380	// Insert 2.0 at all odd positions.
381	// This should force dictionary encoding and exclude run-end due to the
382	// average run length being 1.
383	values[i] = (i % 50) as f32;
384	}
385
386	let floats = values.into_array().to_primitive().unwrap();
387	let compressed = FloatCompressor::compress(&floats, false, MAX_CASCADE, &[]).unwrap();
388	println!("compressed: {}", compressed.display_tree())
389	}
390	}

vortex-data / vortex / 16728097825

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous