16387130526

Committed 19 Jul 2025 09:05AM UTC coverage: 81.512% (-0.008%) from 81.52%

Build # 16387130526

Build Type

push

github

Committed by

web-flow

Commit Message

feat: duckdb workstealing (#3927)

Signed-off-by: Alexander Droste <alexander.droste@protonmail.com>

Run Details

16 of 17 new or added lines in 1 file covered. (94.12%)

185 existing lines in 8 files now uncovered.

42000 of 51526 relevant lines covered (81.51%)

171508.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.17

/vortex-array/src/arrays/varbinview/mod.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::fmt::{Debug, Formatter};
use std::ops::Range;

use static_assertions::{assert_eq_align, assert_eq_size};
use vortex_buffer::{Alignment, Buffer, ByteBuffer};
use vortex_dtype::{DType, Nullability};
use vortex_error::{VortexResult, VortexUnwrap, vortex_bail, vortex_panic};

use crate::builders::{ArrayBuilder, VarBinViewBuilder};
use crate::stats::{ArrayStats, StatsSetRef};
use crate::validity::Validity;
use crate::vtable::{
    ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityHelper,
    ValidityVTableFromValidityHelper,
};
use crate::{Canonical, EncodingId, EncodingRef, vtable};

mod accessor;
mod compact;
mod compute;
mod ops;
mod serde;

pub use compact::*;

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(C, align(8))]
pub struct Inlined {
    size: u32,
    data: [u8; BinaryView::MAX_INLINED_SIZE],
}

impl Inlined {
    fn new<const N: usize>(value: &[u8]) -> Self {
        let mut inlined = Self {
            size: N.try_into().vortex_unwrap(),
            data: [0u8; BinaryView::MAX_INLINED_SIZE],
        };
        inlined.data[..N].copy_from_slice(&value[..N]);
        inlined
    }

    #[inline]
    pub fn value(&self) -> &[u8] {
        &self.data[0..(self.size as usize)]
    }
}

#[derive(Clone, Copy, Debug)]
#[repr(C, align(8))]
pub struct Ref {
    size: u32,
    prefix: [u8; 4],
    buffer_index: u32,
    offset: u32,
}

impl Ref {
    pub fn new(size: u32, prefix: [u8; 4], buffer_index: u32, offset: u32) -> Self {
        Self {
            size,
            prefix,
            buffer_index,
            offset,
        }
    }

    #[inline]
    pub fn buffer_index(&self) -> u32 {
        self.buffer_index
    }

    #[inline]
    pub fn offset(&self) -> u32 {
        self.offset
    }

    #[inline]
    pub fn prefix(&self) -> &[u8; 4] {
        &self.prefix
    }

    #[inline]
    pub fn to_range(&self) -> Range<usize> {
        self.offset as usize..(self.offset + self.size) as usize
    }
}

#[derive(Clone, Copy)]
#[repr(C, align(16))]
pub union BinaryView {
    // Numeric representation. This is logically `u128`, but we split it into the high and low
    // bits to preserve the alignment.
    le_bytes: [u8; 16],

    // Inlined representation: strings <= 12 bytes
    inlined: Inlined,

    // Reference type: strings > 12 bytes.
    _ref: Ref,
}

assert_eq_size!(BinaryView, [u8; 16]);
assert_eq_size!(Inlined, [u8; 16]);
assert_eq_size!(Ref, [u8; 16]);
assert_eq_align!(BinaryView, u128);

impl BinaryView {
    pub const MAX_INLINED_SIZE: usize = 12;

    /// Create a view from a value, block and offset
    ///
    /// Depending on the length of the provided value either a new inlined
    /// or a reference view will be constructed.
    ///
    /// Adapted from arrow-rs <https://github.com/apache/arrow-rs/blob/f4fde769ab6e1a9b75f890b7f8b47bc22800830b/arrow-array/src/builder/generic_bytes_view_builder.rs#L524>
    /// Explicitly enumerating inlined view produces code that avoids calling generic `ptr::copy_non_interleave` that's slower than explicit stores
    #[inline(never)]
    pub fn make_view(value: &[u8], block: u32, offset: u32) -> Self {
        match value.len() {
            0 => Self {
                inlined: Inlined::new::<0>(value),
            },
            1 => Self {
                inlined: Inlined::new::<1>(value),
            },
            2 => Self {
                inlined: Inlined::new::<2>(value),
            },
            3 => Self {
                inlined: Inlined::new::<3>(value),
            },
            4 => Self {
                inlined: Inlined::new::<4>(value),
            },
            5 => Self {
                inlined: Inlined::new::<5>(value),
            },
            6 => Self {
                inlined: Inlined::new::<6>(value),
            },
            7 => Self {
                inlined: Inlined::new::<7>(value),
            },
            8 => Self {
                inlined: Inlined::new::<8>(value),
            },
            9 => Self {
                inlined: Inlined::new::<9>(value),
            },
            10 => Self {
                inlined: Inlined::new::<10>(value),
            },
            11 => Self {
                inlined: Inlined::new::<11>(value),
            },
            12 => Self {
                inlined: Inlined::new::<12>(value),
            },
            _ => Self {
                _ref: Ref::new(
                    u32::try_from(value.len()).vortex_unwrap(),
                    value[0..4].try_into().vortex_unwrap(),
                    block,
                    offset,
                ),
            },
        }
    }

    /// Create a new empty view
    #[inline]
    pub fn empty_view() -> Self {
        Self::new_inlined(&[])
    }

    /// Create a new inlined binary view
    #[inline]
    pub fn new_inlined(value: &[u8]) -> Self {
        assert!(
            value.len() <= Self::MAX_INLINED_SIZE,
            "expected inlined value to be <= 12 bytes, was {}",
            value.len()
        );

        Self::make_view(value, 0, 0)
    }

    #[inline]
    pub fn len(&self) -> u32 {
        unsafe { self.inlined.size }
    }

    #[inline]
    pub fn is_empty(&self) -> bool {
        self.len() > 0
    }

    #[inline]
    #[allow(clippy::cast_possible_truncation)]
    pub fn is_inlined(&self) -> bool {
        self.len() <= (Self::MAX_INLINED_SIZE as u32)
    }

    pub fn as_inlined(&self) -> &Inlined {
        unsafe { &self.inlined }
    }

    pub fn as_view(&self) -> &Ref {
        unsafe { &self._ref }
    }

    pub fn as_u128(&self) -> u128 {
        // SAFETY: binary view always safe to read as u128 LE bytes
        unsafe { u128::from_le_bytes(self.le_bytes) }
    }

    /// Shifts the buffer reference by the view by a given offset, useful when merging many
    /// varbinview arrays into one.
    #[inline(always)]
    pub fn offset_view(self, offset: u32) -> Self {
        if self.is_inlined() {
            self
        } else {
            // Referencing views must have their buffer_index adjusted with new offsets
            let view_ref = self.as_view();
            Self {
                _ref: Ref::new(
                    self.len(),
                    *view_ref.prefix(),
                    offset + view_ref.buffer_index(),
                    view_ref.offset(),
                ),
            }
        }
    }
}

impl From<u128> for BinaryView {
    fn from(value: u128) -> Self {
        BinaryView {
            le_bytes: value.to_le_bytes(),
        }
    }
}

impl Debug for BinaryView {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        let mut s = f.debug_struct("BinaryView");
        if self.is_inlined() {
            s.field("inline", &"i".to_string());
        } else {
            s.field("ref", &"r".to_string());
        }
        s.finish()
    }
}

vtable!(VarBinView);

impl VTable for VarBinViewVTable {
    type Array = VarBinViewArray;
    type Encoding = VarBinViewEncoding;

    type ArrayVTable = Self;
    type CanonicalVTable = Self;
    type OperationsVTable = Self;
    type ValidityVTable = ValidityVTableFromValidityHelper;
    type VisitorVTable = Self;
    type ComputeVTable = NotSupported;
    type EncodeVTable = NotSupported;
    type SerdeVTable = Self;

    fn id(_encoding: &Self::Encoding) -> EncodingId {
        EncodingId::new_ref("vortex.varbinview")
    }

    fn encoding(_array: &Self::Array) -> EncodingRef {
        EncodingRef::new_ref(VarBinViewEncoding.as_ref())
    }
}

#[derive(Clone, Debug)]
pub struct VarBinViewArray {
    dtype: DType,
    buffers: Vec<ByteBuffer>,
    views: Buffer<BinaryView>,
    validity: Validity,
    stats_set: ArrayStats,
}

#[derive(Clone, Debug)]
pub struct VarBinViewEncoding;

impl VarBinViewArray {
    pub fn try_new(
        views: Buffer<BinaryView>,
        buffers: Vec<ByteBuffer>,
        dtype: DType,
        validity: Validity,
    ) -> VortexResult<Self> {
        if views.alignment() != Alignment::of::<BinaryView>() {
            vortex_bail!("Views must be aligned to a 128 bits");
        }

        if !matches!(dtype, DType::Binary(_) | DType::Utf8(_)) {
            vortex_bail!(MismatchedTypes: "utf8 or binary", dtype);
        }

        if dtype.is_nullable() == (validity == Validity::NonNullable) {
            vortex_bail!("incorrect validity {:?}", validity);
        }

        Ok(Self {
            dtype,
            buffers,
            views,
            validity,
            stats_set: Default::default(),
        })
    }

    /// Number of raw string data buffers held by this array.
    pub fn nbuffers(&self) -> usize {
        self.buffers.len()
    }

    /// Access to the primitive views buffer.
    ///
    /// Variable-sized binary view buffer contain a "view" child array, with 16-byte entries that
    /// contain either a pointer into one of the array's owned `buffer`s OR an inlined copy of
    /// the string (if the string has 12 bytes or fewer).
    #[inline]
    pub fn views(&self) -> &Buffer<BinaryView> {
        &self.views
    }

    /// Access value bytes at a given index
    ///
    /// Will return a bytebuffer pointing to the underlying data without performing a copy
    #[inline]
    pub fn bytes_at(&self, index: usize) -> ByteBuffer {
        let views = self.views();
        let view = &views[index];
        // Expect this to be the common case: strings > 12 bytes.
        if !view.is_inlined() {
            let view_ref = view.as_view();
            self.buffer(view_ref.buffer_index() as usize)
                .slice(view_ref.to_range())
        } else {
            // Return access to the range of bytes around it.
            views
                .clone()
                .into_byte_buffer()
                .slice_ref(view.as_inlined().value())
        }
    }

    /// Access one of the backing data buffers.
    ///
    /// # Panics
    ///
    /// This method panics if the provided index is out of bounds for the set of buffers provided
    /// at construction time.
    #[inline]
    pub fn buffer(&self, idx: usize) -> &ByteBuffer {
        if idx >= self.nbuffers() {
            vortex_panic!(
                "{idx} buffer index out of bounds, there are {} buffers",
                self.nbuffers()
            );
        }
        &self.buffers[idx]
    }

    /// Iterate over the underlying raw data buffers, not including the views buffer.
    #[inline]
    pub fn buffers(&self) -> &[ByteBuffer] {
        &self.buffers
    }

    /// Accumulate an iterable set of values into our type here.
    #[allow(clippy::same_name_method)]
    pub fn from_iter<T: AsRef<[u8]>, I: IntoIterator<Item = Option<T>>>(
        iter: I,
        dtype: DType,
    ) -> Self {
        let iter = iter.into_iter();
        let mut builder = VarBinViewBuilder::with_capacity(dtype, iter.size_hint().0);

        for item in iter {
            match item {
                None => builder.append_null(),
                Some(v) => builder.append_value(v),
            }
        }

        builder.finish_into_varbinview()
    }

    pub fn from_iter_str<T: AsRef<str>, I: IntoIterator<Item = T>>(iter: I) -> Self {
        let iter = iter.into_iter();
        let mut builder = VarBinViewBuilder::with_capacity(
            DType::Utf8(Nullability::NonNullable),
            iter.size_hint().0,
        );

        for item in iter {
            builder.append_value(item.as_ref());
        }

        builder.finish_into_varbinview()
    }

    pub fn from_iter_nullable_str<T: AsRef<str>, I: IntoIterator<Item = Option<T>>>(
        iter: I,
    ) -> Self {
        let iter = iter.into_iter();
        let mut builder = VarBinViewBuilder::with_capacity(
            DType::Utf8(Nullability::Nullable),
            iter.size_hint().0,
        );

        for item in iter {
            match item {
                None => builder.append_null(),
                Some(v) => builder.append_value(v.as_ref()),
            }
        }

        builder.finish_into_varbinview()
    }

    pub fn from_iter_bin<T: AsRef<[u8]>, I: IntoIterator<Item = T>>(iter: I) -> Self {
        let iter = iter.into_iter();
        let mut builder = VarBinViewBuilder::with_capacity(
            DType::Binary(Nullability::NonNullable),
            iter.size_hint().0,
        );

        for item in iter {
            builder.append_value(item.as_ref());
        }

        builder.finish_into_varbinview()
    }

    pub fn from_iter_nullable_bin<T: AsRef<[u8]>, I: IntoIterator<Item = Option<T>>>(
        iter: I,
    ) -> Self {
        let iter = iter.into_iter();
        let mut builder = VarBinViewBuilder::with_capacity(
            DType::Binary(Nullability::Nullable),
            iter.size_hint().0,
        );

        for item in iter {
            match item {
                None => builder.append_null(),
                Some(v) => builder.append_value(v.as_ref()),
            }
        }

        builder.finish_into_varbinview()
    }
}

impl ArrayVTable<VarBinViewVTable> for VarBinViewVTable {
    fn len(array: &VarBinViewArray) -> usize {
        array.views.len()
    }

    fn dtype(array: &VarBinViewArray) -> &DType {
        &array.dtype
    }

    fn stats(array: &VarBinViewArray) -> StatsSetRef<'_> {
        array.stats_set.to_ref(array.as_ref())
    }
}

impl ValidityHelper for VarBinViewArray {
    fn validity(&self) -> &Validity {
        &self.validity
    }
}

impl CanonicalVTable<VarBinViewVTable> for VarBinViewVTable {
    fn canonicalize(array: &VarBinViewArray) -> VortexResult<Canonical> {
        Ok(Canonical::VarBinView(array.clone()))
    }

    fn append_to_builder(
        array: &VarBinViewArray,
        builder: &mut dyn ArrayBuilder,
    ) -> VortexResult<()> {
        builder.extend_from_array(array.as_ref())
    }
}

impl<'a> FromIterator<Option<&'a [u8]>> for VarBinViewArray {
    fn from_iter<T: IntoIterator<Item = Option<&'a [u8]>>>(iter: T) -> Self {
        Self::from_iter_nullable_bin(iter)
    }
}

impl FromIterator<Option<Vec<u8>>> for VarBinViewArray {
    fn from_iter<T: IntoIterator<Item = Option<Vec<u8>>>>(iter: T) -> Self {
        Self::from_iter_nullable_bin(iter)
    }
}

impl FromIterator<Option<String>> for VarBinViewArray {
    fn from_iter<T: IntoIterator<Item = Option<String>>>(iter: T) -> Self {
        Self::from_iter_nullable_str(iter)
    }
}

impl<'a> FromIterator<Option<&'a str>> for VarBinViewArray {
    fn from_iter<T: IntoIterator<Item = Option<&'a str>>>(iter: T) -> Self {
        Self::from_iter_nullable_str(iter)
    }
}

#[cfg(test)]
mod test {
    use vortex_scalar::Scalar;

    use crate::arrays::varbinview::{BinaryView, VarBinViewArray};
    use crate::{Array, Canonical, IntoArray};

    #[test]
    pub fn varbin_view() {
        let binary_arr =
            VarBinViewArray::from_iter_str(["hello world", "hello world this is a long string"]);
        assert_eq!(binary_arr.len(), 2);
        assert_eq!(
            binary_arr.scalar_at(0).unwrap(),
            Scalar::from("hello world")
        );
        assert_eq!(
            binary_arr.scalar_at(1).unwrap(),
            Scalar::from("hello world this is a long string")
        );
    }

    #[test]
    pub fn slice_array() {
        let binary_arr =
            VarBinViewArray::from_iter_str(["hello world", "hello world this is a long string"])
                .slice(1, 2)
                .unwrap();
        assert_eq!(
            binary_arr.scalar_at(0).unwrap(),
            Scalar::from("hello world this is a long string")
        );
    }

    #[test]
    pub fn flatten_array() {
        let binary_arr = VarBinViewArray::from_iter_str(["string1", "string2"]);

        let flattened = binary_arr.to_canonical().unwrap();
        assert!(matches!(flattened, Canonical::VarBinView(_)));

        let var_bin = flattened.into_varbinview().unwrap().into_array();
        assert_eq!(var_bin.scalar_at(0).unwrap(), Scalar::from("string1"));
        assert_eq!(var_bin.scalar_at(1).unwrap(), Scalar::from("string2"));
    }

    #[test]
    pub fn binary_view_size_and_alignment() {
        assert_eq!(size_of::<BinaryView>(), 16);
        assert_eq!(align_of::<BinaryView>(), 16);
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::fmt::{Debug, Formatter};
5	use std::ops::Range;
6
7	use static_assertions::{assert_eq_align, assert_eq_size};
8	use vortex_buffer::{Alignment, Buffer, ByteBuffer};
9	use vortex_dtype::{DType, Nullability};
10	use vortex_error::{VortexResult, VortexUnwrap, vortex_bail, vortex_panic};
11
12	use crate::builders::{ArrayBuilder, VarBinViewBuilder};
13	use crate::stats::{ArrayStats, StatsSetRef};
14	use crate::validity::Validity;
15	use crate::vtable::{
16	ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityHelper,
17	ValidityVTableFromValidityHelper,
18	};
19	use crate::{Canonical, EncodingId, EncodingRef, vtable};
20
21	mod accessor;
22	mod compact;
23	mod compute;
24	mod ops;
25	mod serde;
26
27	pub use compact::*;
28
29	#[derive(Clone, Copy, Debug, PartialEq, Eq)]
30	#[repr(C, align(8))]
31	pub struct Inlined {
32	size: u32,
33	data: [u8; BinaryView::MAX_INLINED_SIZE],
34	}
35
36	impl Inlined {
37	fn new<const N: usize>(value: &[u8]) -> Self {	909,978✔
38	let mut inlined = Self {	909,978✔
39	size: N.try_into().vortex_unwrap(),	909,978✔
40	data: [0u8; BinaryView::MAX_INLINED_SIZE],	909,978✔
41	};	909,978✔
42	inlined.data[..N].copy_from_slice(&value[..N]);	909,978✔
43	inlined	909,978✔
44	}	909,978✔
45
46	#[inline]
47	pub fn value(&self) -> &[u8] {	20,528,131✔
48	&self.data[0..(self.size as usize)]	20,528,131✔
49	}	20,528,131✔
50	}
51
52	#[derive(Clone, Copy, Debug)]
53	#[repr(C, align(8))]
54	pub struct Ref {
55	size: u32,
56	prefix: [u8; 4],
57	buffer_index: u32,
58	offset: u32,
59	}
60
61	impl Ref {
62	pub fn new(size: u32, prefix: [u8; 4], buffer_index: u32, offset: u32) -> Self {	4,175,147✔
63	Self {	4,175,147✔
64	size,	4,175,147✔
65	prefix,	4,175,147✔
66	buffer_index,	4,175,147✔
67	offset,	4,175,147✔
68	}	4,175,147✔
69	}	4,175,147✔
70
71	#[inline]
72	pub fn buffer_index(&self) -> u32 {	14,928,118✔
73	self.buffer_index	14,928,118✔
74	}	14,928,118✔
75
76	#[inline]
77	pub fn offset(&self) -> u32 {	861,650✔
78	self.offset	861,650✔
79	}	861,650✔
80
81	#[inline]
82	pub fn prefix(&self) -> &[u8; 4] {	861,650✔
83	&self.prefix	861,650✔
84	}	861,650✔
85
86	#[inline]
87	pub fn to_range(&self) -> Range<usize> {	15,758,804✔
88	self.offset as usize..(self.offset + self.size) as usize	15,758,804✔
89	}	15,758,804✔
90	}
91
92	#[derive(Clone, Copy)]
93	#[repr(C, align(16))]
94	pub union BinaryView {
95	// Numeric representation. This is logically `u128`, but we split it into the high and low
96	// bits to preserve the alignment.
97	le_bytes: [u8; 16],
98
99	// Inlined representation: strings <= 12 bytes
100	inlined: Inlined,
101
102	// Reference type: strings > 12 bytes.
103	_ref: Ref,
104	}
105
106	assert_eq_size!(BinaryView, [u8; 16]);
107	assert_eq_size!(Inlined, [u8; 16]);
108	assert_eq_size!(Ref, [u8; 16]);
109	assert_eq_align!(BinaryView, u128);
110
111	impl BinaryView {
112	pub const MAX_INLINED_SIZE: usize = 12;
113
114	/// Create a view from a value, block and offset
115	///
116	/// Depending on the length of the provided value either a new inlined
117	/// or a reference view will be constructed.
118	///
119	/// Adapted from arrow-rs <https://github.com/apache/arrow-rs/blob/f4fde769ab6e1a9b75f890b7f8b47bc22800830b/arrow-array/src/builder/generic_bytes_view_builder.rs#L524>
120	/// Explicitly enumerating inlined view produces code that avoids calling generic `ptr::copy_non_interleave` that's slower than explicit stores
121	#[inline(never)]
122	pub fn make_view(value: &[u8], block: u32, offset: u32) -> Self {	4,751,121✔
123	match value.len() {	4,751,121✔
124	0 => Self {	667,273✔
125	inlined: Inlined::new::<0>(value),	667,273✔
126	},	667,273✔
127	1 => Self {	2,937✔
128	inlined: Inlined::new::<1>(value),	2,937✔
129	},	2,937✔
130	2 => Self {	3,257✔
131	inlined: Inlined::new::<2>(value),	3,257✔
132	},	3,257✔
133	3 => Self {	9,402✔
134	inlined: Inlined::new::<3>(value),	9,402✔
135	},	9,402✔
136	4 => Self {	4,224✔
137	inlined: Inlined::new::<4>(value),	4,224✔
138	},	4,224✔
139	5 => Self {	14,834✔
140	inlined: Inlined::new::<5>(value),	14,834✔
141	},	14,834✔
142	6 => Self {	14,625✔
143	inlined: Inlined::new::<6>(value),	14,625✔
144	},	14,625✔
145	7 => Self {	12,649✔
146	inlined: Inlined::new::<7>(value),	12,649✔
147	},	12,649✔
148	8 => Self {	14,777✔
149	inlined: Inlined::new::<8>(value),	14,777✔
150	},	14,777✔
151	9 => Self {	10,804✔
152	inlined: Inlined::new::<9>(value),	10,804✔
153	},	10,804✔
154	10 => Self {	53,846✔
155	inlined: Inlined::new::<10>(value),	53,846✔
156	},	53,846✔
157	11 => Self {	52,770✔
158	inlined: Inlined::new::<11>(value),	52,770✔
159	},	52,770✔
160	12 => Self {	48,580✔
161	inlined: Inlined::new::<12>(value),	48,580✔
162	},	48,580✔
163	_ => Self {	3,841,143✔
164	_ref: Ref::new(	3,841,143✔
165	u32::try_from(value.len()).vortex_unwrap(),	3,841,143✔
166	value[0..4].try_into().vortex_unwrap(),	3,841,143✔
167	block,	3,841,143✔
168	offset,	3,841,143✔
169	),	3,841,143✔
170	},	3,841,143✔
171	}
172	}	4,751,121✔
173
174	/// Create a new empty view
175	#[inline]
176	pub fn empty_view() -> Self {	1,680✔
177	Self::new_inlined(&[])	1,680✔
178	}	1,680✔
179
180	/// Create a new inlined binary view
181	#[inline]
182	pub fn new_inlined(value: &[u8]) -> Self {	1,680✔
183	assert!(	1,680✔
184	value.len() <= Self::MAX_INLINED_SIZE,	1,680✔
UNCOV 185	"expected inlined value to be <= 12 bytes, was {}",	×
UNCOV 186	value.len()	×
187	);
188
189	Self::make_view(value, 0, 0)	1,680✔
190	}	1,680✔
191
192	#[inline]
193	pub fn len(&self) -> u32 {	41,167,821✔
194	unsafe { self.inlined.size }	41,167,821✔
195	}	41,167,821✔
196
197	#[inline]
UNCOV 198	pub fn is_empty(&self) -> bool {	×
UNCOV 199	self.len() > 0	×
UNCOV 200	}	×
201
202	#[inline]
203	#[allow(clippy::cast_possible_truncation)]
204	pub fn is_inlined(&self) -> bool {	40,228,555✔
205	self.len() <= (Self::MAX_INLINED_SIZE as u32)	40,228,555✔
206	}	40,228,555✔
207
208	pub fn as_inlined(&self) -> &Inlined {	20,614,385✔
209	unsafe { &self.inlined }	20,614,385✔
210	}	20,614,385✔
211
212	pub fn as_view(&self) -> &Ref {	28,697,251✔
213	unsafe { &self._ref }	28,697,251✔
214	}	28,697,251✔
215
216	pub fn as_u128(&self) -> u128 {	2,950,582✔
217	// SAFETY: binary view always safe to read as u128 LE bytes
218	unsafe { u128::from_le_bytes(self.le_bytes) }	2,950,582✔
219	}	2,950,582✔
220
221	/// Shifts the buffer reference by the view by a given offset, useful when merging many
222	/// varbinview arrays into one.
223	#[inline(always)]
224	pub fn offset_view(self, offset: u32) -> Self {	392,933✔
225	if self.is_inlined() {	392,933✔
226	self	58,929✔
227	} else {
228	// Referencing views must have their buffer_index adjusted with new offsets
229	let view_ref = self.as_view();	334,004✔
230	Self {	334,004✔
231	_ref: Ref::new(	334,004✔
232	self.len(),	334,004✔
233	*view_ref.prefix(),	334,004✔
234	offset + view_ref.buffer_index(),	334,004✔
235	view_ref.offset(),	334,004✔
236	),	334,004✔
237	}	334,004✔
238	}
239	}	392,933✔
240	}
241
242	impl From<u128> for BinaryView {
243	fn from(value: u128) -> Self {	×
244	BinaryView {	×
UNCOV 245	le_bytes: value.to_le_bytes(),	×
UNCOV 246	}	×
UNCOV 247	}	×
248	}
249
250	impl Debug for BinaryView {
251	fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {	×
252	let mut s = f.debug_struct("BinaryView");	×
253	if self.is_inlined() {	×
254	s.field("inline", &"i".to_string());	×
255	} else {	×
256	s.field("ref", &"r".to_string());	×
UNCOV 257	}	×
UNCOV 258	s.finish()	×
UNCOV 259	}	×
260	}
261
262	vtable!(VarBinView);
263
264	impl VTable for VarBinViewVTable {
265	type Array = VarBinViewArray;
266	type Encoding = VarBinViewEncoding;
267
268	type ArrayVTable = Self;
269	type CanonicalVTable = Self;
270	type OperationsVTable = Self;
271	type ValidityVTable = ValidityVTableFromValidityHelper;
272	type VisitorVTable = Self;
273	type ComputeVTable = NotSupported;
274	type EncodeVTable = NotSupported;
275	type SerdeVTable = Self;
276
277	fn id(_encoding: &Self::Encoding) -> EncodingId {	72,984✔
278	EncodingId::new_ref("vortex.varbinview")	72,984✔
279	}	72,984✔
280
281	fn encoding(_array: &Self::Array) -> EncodingRef {	16,222✔
282	EncodingRef::new_ref(VarBinViewEncoding.as_ref())	16,222✔
283	}	16,222✔
284	}
285
286	#[derive(Clone, Debug)]
287	pub struct VarBinViewArray {
288	dtype: DType,
289	buffers: Vec<ByteBuffer>,
290	views: Buffer<BinaryView>,
291	validity: Validity,
292	stats_set: ArrayStats,
293	}
294
295	#[derive(Clone, Debug)]
296	pub struct VarBinViewEncoding;
297
298	impl VarBinViewArray {
299	pub fn try_new(	19,377✔
300	views: Buffer<BinaryView>,	19,377✔
301	buffers: Vec<ByteBuffer>,	19,377✔
302	dtype: DType,	19,377✔
303	validity: Validity,	19,377✔
304	) -> VortexResult<Self> {	19,377✔
305	if views.alignment() != Alignment::of::<BinaryView>() {	19,377✔
UNCOV 306	vortex_bail!("Views must be aligned to a 128 bits");	×
307	}	19,377✔
308
309	if !matches!(dtype, DType::Binary(_) \| DType::Utf8(_)) {	19,377✔
UNCOV 310	vortex_bail!(MismatchedTypes: "utf8 or binary", dtype);	×
311	}	19,377✔
312
313	if dtype.is_nullable() == (validity == Validity::NonNullable) {	19,377✔
UNCOV 314	vortex_bail!("incorrect validity {:?}", validity);	×
315	}	19,377✔
316
317	Ok(Self {	19,377✔
318	dtype,	19,377✔
319	buffers,	19,377✔
320	views,	19,377✔
321	validity,	19,377✔
322	stats_set: Default::default(),	19,377✔
323	})	19,377✔
324	}	19,377✔
325
326	/// Number of raw string data buffers held by this array.
327	pub fn nbuffers(&self) -> usize {	2,248,891✔
328	self.buffers.len()	2,248,891✔
329	}	2,248,891✔
330
331	/// Access to the primitive views buffer.
332	///
333	/// Variable-sized binary view buffer contain a "view" child array, with 16-byte entries that
334	/// contain either a pointer into one of the array's owned `buffer`s OR an inlined copy of
335	/// the string (if the string has 12 bytes or fewer).
336	#[inline]
337	pub fn views(&self) -> &Buffer<BinaryView> {	2,361,210✔
338	&self.views	2,361,210✔
339	}	2,361,210✔
340
341	/// Access value bytes at a given index
342	///
343	/// Will return a bytebuffer pointing to the underlying data without performing a copy
344	#[inline]
345	pub fn bytes_at(&self, index: usize) -> ByteBuffer {	2,298,602✔
346	let views = self.views();	2,298,602✔
347	let view = &views[index];	2,298,602✔
348	// Expect this to be the common case: strings > 12 bytes.
349	if !view.is_inlined() {	2,298,602✔
350	let view_ref = view.as_view();	2,133,047✔
351	self.buffer(view_ref.buffer_index() as usize)	2,133,047✔
352	.slice(view_ref.to_range())	2,133,047✔
353	} else {
354	// Return access to the range of bytes around it.
355	views	165,555✔
356	.clone()	165,555✔
357	.into_byte_buffer()	165,555✔
358	.slice_ref(view.as_inlined().value())	165,555✔
359	}
360	}	2,298,602✔
361
362	/// Access one of the backing data buffers.
363	///
364	/// # Panics
365	///
366	/// This method panics if the provided index is out of bounds for the set of buffers provided
367	/// at construction time.
368	#[inline]
369	pub fn buffer(&self, idx: usize) -> &ByteBuffer {	2,221,771✔
370	if idx >= self.nbuffers() {	2,221,771✔
UNCOV 371	vortex_panic!(	×
UNCOV 372	"{idx} buffer index out of bounds, there are {} buffers",	×
UNCOV 373	self.nbuffers()	×
374	);
375	}	2,221,771✔
376	&self.buffers[idx]	2,221,771✔
377	}	2,221,771✔
378
379	/// Iterate over the underlying raw data buffers, not including the views buffer.
380	#[inline]
381	pub fn buffers(&self) -> &[ByteBuffer] {	39,097✔
382	&self.buffers	39,097✔
383	}	39,097✔
384
385	/// Accumulate an iterable set of values into our type here.
386	#[allow(clippy::same_name_method)]
387	pub fn from_iter<T: AsRef<[u8]>, I: IntoIterator<Item = Option<T>>>(	11✔
388	iter: I,	11✔
389	dtype: DType,	11✔
390	) -> Self {	11✔
391	let iter = iter.into_iter();	11✔
392	let mut builder = VarBinViewBuilder::with_capacity(dtype, iter.size_hint().0);	11✔
393
394	for item in iter {	2,093✔
395	match item {	2,082✔
396	None => builder.append_null(),	7✔
397	Some(v) => builder.append_value(v),	2,075✔
398	}
399	}
400
401	builder.finish_into_varbinview()	11✔
402	}	11✔
403
404	pub fn from_iter_str<T: AsRef<str>, I: IntoIterator<Item = T>>(iter: I) -> Self {	13✔
405	let iter = iter.into_iter();	13✔
406	let mut builder = VarBinViewBuilder::with_capacity(	13✔
407	DType::Utf8(Nullability::NonNullable),	13✔
408	iter.size_hint().0,	13✔
409	);
410
411	for item in iter {	56✔
412	builder.append_value(item.as_ref());	43✔
413	}	43✔
414
415	builder.finish_into_varbinview()	13✔
416	}	13✔
417
418	pub fn from_iter_nullable_str<T: AsRef<str>, I: IntoIterator<Item = Option<T>>>(	17✔
419	iter: I,	17✔
420	) -> Self {	17✔
421	let iter = iter.into_iter();	17✔
422	let mut builder = VarBinViewBuilder::with_capacity(	17✔
423	DType::Utf8(Nullability::Nullable),	17✔
424	iter.size_hint().0,	17✔
425	);
426
427	for item in iter {	124✔
428	match item {	107✔
429	None => builder.append_null(),	29✔
430	Some(v) => builder.append_value(v.as_ref()),	78✔
431	}
432	}
433
434	builder.finish_into_varbinview()	17✔
435	}	17✔
436
437	pub fn from_iter_bin<T: AsRef<[u8]>, I: IntoIterator<Item = T>>(iter: I) -> Self {	2✔
438	let iter = iter.into_iter();	2✔
439	let mut builder = VarBinViewBuilder::with_capacity(	2✔
440	DType::Binary(Nullability::NonNullable),	2✔
441	iter.size_hint().0,	2✔
442	);
443
444	for item in iter {	6✔
445	builder.append_value(item.as_ref());	4✔
446	}	4✔
447
448	builder.finish_into_varbinview()	2✔
449	}	2✔
450
451	pub fn from_iter_nullable_bin<T: AsRef<[u8]>, I: IntoIterator<Item = Option<T>>>(	×
452	iter: I,	×
453	) -> Self {	×
454	let iter = iter.into_iter();	×
UNCOV 455	let mut builder = VarBinViewBuilder::with_capacity(	×
UNCOV 456	DType::Binary(Nullability::Nullable),	×
457	iter.size_hint().0,	×
458	);
459
460	for item in iter {	×
UNCOV 461	match item {	×
UNCOV 462	None => builder.append_null(),	×
UNCOV 463	Some(v) => builder.append_value(v.as_ref()),	×
464	}
465	}
466
UNCOV 467	builder.finish_into_varbinview()	×
UNCOV 468	}	×
469	}
470
471	impl ArrayVTable<VarBinViewVTable> for VarBinViewVTable {
472	fn len(array: &VarBinViewArray) -> usize {	2,645,934✔
473	array.views.len()	2,645,934✔
474	}	2,645,934✔
475
476	fn dtype(array: &VarBinViewArray) -> &DType {	188,081✔
477	&array.dtype	188,081✔
478	}	188,081✔
479
480	fn stats(array: &VarBinViewArray) -> StatsSetRef<'_> {	148,753✔
481	array.stats_set.to_ref(array.as_ref())	148,753✔
482	}	148,753✔
483	}
484
485	impl ValidityHelper for VarBinViewArray {
486	fn validity(&self) -> &Validity {	2,500,348✔
487	&self.validity	2,500,348✔
488	}	2,500,348✔
489	}
490
491	impl CanonicalVTable<VarBinViewVTable> for VarBinViewVTable {
492	fn canonicalize(array: &VarBinViewArray) -> VortexResult<Canonical> {	24,737✔
493	Ok(Canonical::VarBinView(array.clone()))	24,737✔
494	}	24,737✔
495
496	fn append_to_builder(	5,283✔
497	array: &VarBinViewArray,	5,283✔
498	builder: &mut dyn ArrayBuilder,	5,283✔
499	) -> VortexResult<()> {	5,283✔
500	builder.extend_from_array(array.as_ref())	5,283✔
501	}	5,283✔
502	}
503
504	impl<'a> FromIterator<Option<&'a [u8]>> for VarBinViewArray {
UNCOV 505	fn from_iter<T: IntoIterator<Item = Option<&'a [u8]>>>(iter: T) -> Self {	×
UNCOV 506	Self::from_iter_nullable_bin(iter)	×
UNCOV 507	}	×
508	}
509
510	impl FromIterator<Option<Vec<u8>>> for VarBinViewArray {
UNCOV 511	fn from_iter<T: IntoIterator<Item = Option<Vec<u8>>>>(iter: T) -> Self {	×
UNCOV 512	Self::from_iter_nullable_bin(iter)	×
UNCOV 513	}	×
514	}
515
516	impl FromIterator<Option<String>> for VarBinViewArray {
UNCOV 517	fn from_iter<T: IntoIterator<Item = Option<String>>>(iter: T) -> Self {	×
UNCOV 518	Self::from_iter_nullable_str(iter)	×
UNCOV 519	}	×
520	}
521
522	impl<'a> FromIterator<Option<&'a str>> for VarBinViewArray {
523	fn from_iter<T: IntoIterator<Item = Option<&'a str>>>(iter: T) -> Self {	6✔
524	Self::from_iter_nullable_str(iter)	6✔
525	}	6✔
526	}
527
528	#[cfg(test)]
529	mod test {
530	use vortex_scalar::Scalar;
531
532	use crate::arrays::varbinview::{BinaryView, VarBinViewArray};
533	use crate::{Array, Canonical, IntoArray};
534
535	#[test]
536	pub fn varbin_view() {	1✔
537	let binary_arr =	1✔
538	VarBinViewArray::from_iter_str(["hello world", "hello world this is a long string"]);	1✔
539	assert_eq!(binary_arr.len(), 2);	1✔
540	assert_eq!(	1✔
541	binary_arr.scalar_at(0).unwrap(),	1✔
542	Scalar::from("hello world")	1✔
543	);
544	assert_eq!(	1✔
545	binary_arr.scalar_at(1).unwrap(),	1✔
546	Scalar::from("hello world this is a long string")	1✔
547	);
548	}	1✔
549
550	#[test]
551	pub fn slice_array() {	1✔
552	let binary_arr =	1✔
553	VarBinViewArray::from_iter_str(["hello world", "hello world this is a long string"])	1✔
554	.slice(1, 2)	1✔
555	.unwrap();	1✔
556	assert_eq!(	1✔
557	binary_arr.scalar_at(0).unwrap(),	1✔
558	Scalar::from("hello world this is a long string")	1✔
559	);
560	}	1✔
561
562	#[test]
563	pub fn flatten_array() {	1✔
564	let binary_arr = VarBinViewArray::from_iter_str(["string1", "string2"]);	1✔
565
566	let flattened = binary_arr.to_canonical().unwrap();	1✔
567	assert!(matches!(flattened, Canonical::VarBinView(_)));	1✔
568
569	let var_bin = flattened.into_varbinview().unwrap().into_array();	1✔
570	assert_eq!(var_bin.scalar_at(0).unwrap(), Scalar::from("string1"));	1✔
571	assert_eq!(var_bin.scalar_at(1).unwrap(), Scalar::from("string2"));	1✔
572	}	1✔
573
574	#[test]
575	pub fn binary_view_size_and_alignment() {	1✔
576	assert_eq!(size_of::<BinaryView>(), 16);	1✔
577	assert_eq!(align_of::<BinaryView>(), 16);	1✔
578	}	1✔
579	}

vortex-data / vortex / 16387130526

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous