16620437478

Committed 30 Jul 2025 10:46AM UTC coverage: 82.754% (+0.06%) from 82.696%

Build # 16620437478

Build Type

push

github

Committed by

web-flow

Commit Message

varbinview zip kernel (#4054)

Uses the buffer deduplicating builder to construct the zipped array.
This guards against the pathological case where we are zipping two
varbinview arrays with a mask that has lots of contiguous slices. Each
`builder.extend_from_array(input.slice(..))` would duplicate the entire
buffers of `input`, and each slice in the mask would add the same
buffers to the result array over and over again.

---------

Signed-off-by: Onur Satici <onur@spiraldb.com>

Coverage Stats

79 of 82 new or added lines in 2 files covered. (96.34%)

2 existing lines in 1 file now uncovered.

45322 of 54767 relevant lines covered (82.75%)

184684.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.0

/vortex-array/src/arrays/varbinview/compute/zip.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_error::VortexResult;

use crate::arrays::{VarBinViewArray, VarBinViewVTable};
use crate::builders::VarBinViewBuilder;
use crate::compute::{ZipKernel, ZipKernelAdapter, zip_impl_with_builder, zip_return_dtype};
use crate::{Array, ArrayRef, register_kernel};

impl ZipKernel for VarBinViewVTable {
    fn zip(
        &self,
        if_true: &VarBinViewArray,
        if_false: &dyn Array,
        mask: &vortex_mask::Mask,
    ) -> VortexResult<Option<ArrayRef>> {
        let Some(if_false) = if_false.as_opt::<VarBinViewVTable>() else {
            return Ok(None);
        };
        Ok(Some(zip_impl_with_builder(
            if_true.as_ref(),
            if_false.as_ref(),
            mask,
            Box::new(VarBinViewBuilder::with_buffer_deduplication(
                zip_return_dtype(if_true.as_ref(), if_false.as_ref()),
                if_true.len(),
            )),
        )?))
    }
}

register_kernel!(ZipKernelAdapter(VarBinViewVTable).lift());

#[cfg(test)]
mod tests {
    use arrow_array::cast::AsArray;
    use arrow_select::zip::zip as arrow_zip;
    use vortex_dtype::{DType, Nullability};
    use vortex_mask::Mask;

    use crate::IntoArray;
    use crate::arrays::VarBinViewVTable;
    use crate::arrow::IntoArrowArray;
    use crate::builders::{ArrayBuilder as _, VarBinViewBuilder};
    use crate::compute::zip;

    #[test]
    fn test_varbinview_zip() {
        let if_true = {
            let mut builder =
                VarBinViewBuilder::with_capacity(DType::Utf8(Nullability::NonNullable), 10);
            for _ in 0..100 {
                builder.append_value("Hello");
                builder.append_value("Hello this is a long string that won't be inlined.");
            }
            builder.finish()
        };

        let if_false = {
            let mut builder =
                VarBinViewBuilder::with_capacity(DType::Utf8(Nullability::NonNullable), 10);
            for _ in 0..100 {
                builder.append_value("Hello2");
                builder.append_value("Hello2 this is a long string that won't be inlined.");
            }
            builder.finish()
        };

        // [1,2,4,5,7,8,..]
        let mask = Mask::from_indices(200, (0..100).filter(|i| i % 3 != 0).collect());

        let zipped = zip(&if_true, &if_false, &mask).unwrap();
        let zipped = zipped.as_opt::<VarBinViewVTable>().unwrap();
        assert_eq!(zipped.nbuffers(), 2);

        // assert the result is the same as arrow
        let expected = arrow_zip(
            mask.into_array()
                .into_arrow_preferred()
                .unwrap()
                .as_boolean(),
            &if_true.into_arrow_preferred().unwrap(),
            &if_false.into_arrow_preferred().unwrap(),
        )
        .unwrap();

        let actual = zipped.clone().into_array().into_arrow_preferred().unwrap();
        assert_eq!(actual.as_ref(), expected.as_ref());
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use vortex_error::VortexResult;
5
6	use crate::arrays::{VarBinViewArray, VarBinViewVTable};
7	use crate::builders::VarBinViewBuilder;
8	use crate::compute::{ZipKernel, ZipKernelAdapter, zip_impl_with_builder, zip_return_dtype};
9	use crate::{Array, ArrayRef, register_kernel};
10
11	impl ZipKernel for VarBinViewVTable {
12	fn zip(	1✔
13	&self,	1✔
14	if_true: &VarBinViewArray,	1✔
15	if_false: &dyn Array,	1✔
16	mask: &vortex_mask::Mask,	1✔
17	) -> VortexResult<Option<ArrayRef>> {	1✔
18	let Some(if_false) = if_false.as_opt::<VarBinViewVTable>() else {	1✔
NEW 19	return Ok(None);	×
20	};
21	Ok(Some(zip_impl_with_builder(	1✔
22	if_true.as_ref(),	1✔
23	if_false.as_ref(),	1✔
24	mask,	1✔
25	Box::new(VarBinViewBuilder::with_buffer_deduplication(	1✔
26	zip_return_dtype(if_true.as_ref(), if_false.as_ref()),	1✔
27	if_true.len(),	1✔
28	)),	1✔
NEW 29	)?))	×
30	}	1✔
31	}
32
33	register_kernel!(ZipKernelAdapter(VarBinViewVTable).lift());
34
35	#[cfg(test)]
36	mod tests {
37	use arrow_array::cast::AsArray;
38	use arrow_select::zip::zip as arrow_zip;
39	use vortex_dtype::{DType, Nullability};
40	use vortex_mask::Mask;
41
42	use crate::IntoArray;
43	use crate::arrays::VarBinViewVTable;
44	use crate::arrow::IntoArrowArray;
45	use crate::builders::{ArrayBuilder as _, VarBinViewBuilder};
46	use crate::compute::zip;
47
48	#[test]
49	fn test_varbinview_zip() {	1✔
50	let if_true = {	1✔
51	let mut builder =	1✔
52	VarBinViewBuilder::with_capacity(DType::Utf8(Nullability::NonNullable), 10);	1✔
53	for _ in 0..100 {	101✔
54	builder.append_value("Hello");	100✔
55	builder.append_value("Hello this is a long string that won't be inlined.");	100✔
56	}	100✔
57	builder.finish()	1✔
58	};
59
60	let if_false = {	1✔
61	let mut builder =	1✔
62	VarBinViewBuilder::with_capacity(DType::Utf8(Nullability::NonNullable), 10);	1✔
63	for _ in 0..100 {	101✔
64	builder.append_value("Hello2");	100✔
65	builder.append_value("Hello2 this is a long string that won't be inlined.");	100✔
66	}	100✔
67	builder.finish()	1✔
68	};
69
70	// [1,2,4,5,7,8,..]
71	let mask = Mask::from_indices(200, (0..100).filter(\|i\| i % 3 != 0).collect());	100✔
72
73	let zipped = zip(&if_true, &if_false, &mask).unwrap();	1✔
74	let zipped = zipped.as_opt::<VarBinViewVTable>().unwrap();	1✔
75	assert_eq!(zipped.nbuffers(), 2);	1✔
76
77	// assert the result is the same as arrow
78	let expected = arrow_zip(	1✔
79	mask.into_array()	1✔
80	.into_arrow_preferred()	1✔
81	.unwrap()	1✔
82	.as_boolean(),	1✔
83	&if_true.into_arrow_preferred().unwrap(),	1✔
84	&if_false.into_arrow_preferred().unwrap(),	1✔
85	)
86	.unwrap();	1✔
87
88	let actual = zipped.clone().into_array().into_arrow_preferred().unwrap();	1✔
89	assert_eq!(actual.as_ref(), expected.as_ref());	1✔
90	}	1✔
91	}

vortex-data / vortex / 16620437478

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous