16578751259

Committed 28 Jul 2025 07:46PM UTC coverage: 81.858% (+0.8%) from 81.087%

Build # 16578751259

Build Type

Pull #3992

github

Committed by

web-flow

Commit Message

Merge 0fe171459 into 1ae560509

Pull Request Pull Request #3992: feat: teach SparseArray to canonicalize lists

Run Details

222 of 241 new or added lines in 1 file covered. (92.12%)

495 existing lines in 40 files now uncovered.

43529 of 53176 relevant lines covered (81.86%)

169875.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.75

/vortex-array/src/builders/lazy_validity_builder.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use arrow_buffer::{BooleanBuffer, BooleanBufferBuilder, NullBuffer};
use vortex_dtype::Nullability;
use vortex_dtype::Nullability::{NonNullable, Nullable};
use vortex_error::{VortexExpect, vortex_panic};
use vortex_mask::Mask;

use crate::validity::Validity;

/// This is borrowed from arrow's null buffer builder, however we expose a `append_buffer`
/// method to append a boolean buffer directly.
pub struct LazyNullBufferBuilder {
    inner: Option<BooleanBufferBuilder>,
    len: usize,
    capacity: usize,
}

impl LazyNullBufferBuilder {
    /// Creates a new empty builder.
    /// `capacity` is the number of bits in the null buffer.
    pub fn new(capacity: usize) -> Self {
        Self {
            inner: None,
            len: 0,
            capacity,
        }
    }

    #[inline]
    pub fn append_n_non_nulls(&mut self, n: usize) {
        if let Some(buf) = self.inner.as_mut() {
            buf.append_n(n, true)
        } else {
            self.len += n;
        }
    }

    #[inline]
    pub fn append_non_null(&mut self) {
        if let Some(buf) = self.inner.as_mut() {
            buf.append(true)
        } else {
            self.len += 1;
        }
    }

    #[inline]
    pub fn append_n_nulls(&mut self, n: usize) {
        self.materialize_if_needed();
        self.inner
            .as_mut()
            .vortex_expect("cannot append null to non-nullable builder")
            .append_n(n, false);
    }

    #[inline]
    pub fn append_null(&mut self) {
        self.materialize_if_needed();
        self.inner
            .as_mut()
            .vortex_expect("cannot append null to non-nullable builder")
            .append(false);
    }

    #[inline]
    pub fn append(&mut self, not_null: bool) {
        if not_null {
            self.append_non_null()
        } else {
            self.append_null()
        }
    }

    #[inline]
    pub fn append_buffer(&mut self, bool_buffer: &BooleanBuffer) {
        self.materialize_if_needed();
        self.inner
            .as_mut()
            .vortex_expect("buffer just materialized")
            .append_buffer(bool_buffer);
    }

    pub fn append_validity_mask(&mut self, validity_mask: Mask) {
        match validity_mask {
            Mask::AllTrue(len) => self.append_n_non_nulls(len),
            Mask::AllFalse(len) => self.append_n_nulls(len),
            Mask::Values(is_valid) => self.append_buffer(is_valid.boolean_buffer()),
        }
    }

    pub fn set_bit(&mut self, index: usize, v: bool) {
        self.materialize_if_needed();
        self.inner
            .as_mut()
            .vortex_expect("buffer just materialized")
            .set_bit(index, v);
    }

    pub fn len(&self) -> usize {
        // self.len is the length of the builder if the inner buffer is not materialized
        self.inner.as_ref().map(|i| i.len()).unwrap_or(self.len)
    }

    pub fn truncate(&mut self, len: usize) {
        if let Some(b) = self.inner.as_mut() {
            b.truncate(len)
        }
        self.len = len;
    }

    pub fn reserve(&mut self, n: usize) {
        self.materialize_if_needed();
        self.inner
            .as_mut()
            .vortex_expect("buffer just materialized")
            .reserve(n);
    }

    fn finish(&mut self) -> Option<NullBuffer> {
        self.len = 0;
        Some(NullBuffer::new(self.inner.take()?.finish()))
    }

    pub fn finish_with_nullability(&mut self, nullability: Nullability) -> Validity {
        let nulls = self.finish();

        match (nullability, nulls) {
            (NonNullable, None) => Validity::NonNullable,
            (Nullable, None) => Validity::AllValid,
            (Nullable, Some(arr)) => Validity::from(arr),
            _ => vortex_panic!("Invalid nullability/nulls combination"),
        }
    }

    pub fn ensure_capacity(&mut self, capacity: usize) {
        if self.inner.is_none() {
            self.capacity = capacity;
        } else {
            let inner = self
                .inner
                .as_mut()
                .vortex_expect("buffer just materialized");
            if capacity < inner.capacity() {
                inner.reserve(capacity - inner.len());
            }
        }
    }

    #[inline]
    fn materialize_if_needed(&mut self) {
        if self.inner.is_none() {
            self.materialize()
        }
    }

    // This only happens once per builder
    #[cold]
    #[inline(never)]
    fn materialize(&mut self) {
        if self.inner.is_none() {
            let mut b = BooleanBufferBuilder::new(self.len.max(self.capacity));
            b.append_n(self.len, true);
            self.inner = Some(b);
        }
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use arrow_buffer::{BooleanBuffer, BooleanBufferBuilder, NullBuffer};
5	use vortex_dtype::Nullability;
6	use vortex_dtype::Nullability::{NonNullable, Nullable};
7	use vortex_error::{VortexExpect, vortex_panic};
8	use vortex_mask::Mask;
9
10	use crate::validity::Validity;
11
12	/// This is borrowed from arrow's null buffer builder, however we expose a `append_buffer`
13	/// method to append a boolean buffer directly.
14	pub struct LazyNullBufferBuilder {
15	inner: Option<BooleanBufferBuilder>,
16	len: usize,
17	capacity: usize,
18	}
19
20	impl LazyNullBufferBuilder {
21	/// Creates a new empty builder.
22	/// `capacity` is the number of bits in the null buffer.
23	pub fn new(capacity: usize) -> Self {	38,426✔
24	Self {	38,426✔
25	inner: None,	38,426✔
26	len: 0,	38,426✔
27	capacity,	38,426✔
28	}	38,426✔
29	}	38,426✔
30
31	#[inline]
32	pub fn append_n_non_nulls(&mut self, n: usize) {	55,094✔
33	if let Some(buf) = self.inner.as_mut() {	55,094✔
34	buf.append_n(n, true)	75✔
35	} else {	55,019✔
36	self.len += n;	55,019✔
37	}	55,019✔
38	}	55,094✔
39
40	#[inline]
41	pub fn append_non_null(&mut self) {	2,418,728✔
42	if let Some(buf) = self.inner.as_mut() {	2,418,728✔
43	buf.append(true)	2,093✔
44	} else {	2,416,635✔
45	self.len += 1;	2,416,635✔
46	}	2,416,635✔
47	}	2,418,728✔
48
49	#[inline]
50	pub fn append_n_nulls(&mut self, n: usize) {	2,267✔
51	self.materialize_if_needed();	2,267✔
52	self.inner	2,267✔
53	.as_mut()	2,267✔
54	.vortex_expect("cannot append null to non-nullable builder")	2,267✔
55	.append_n(n, false);	2,267✔
56	}	2,267✔
57
58	#[inline]
59	pub fn append_null(&mut self) {	148✔
60	self.materialize_if_needed();	148✔
61	self.inner	148✔
62	.as_mut()	148✔
63	.vortex_expect("cannot append null to non-nullable builder")	148✔
64	.append(false);	148✔
65	}	148✔
66
67	#[inline]
68	pub fn append(&mut self, not_null: bool) {	36,306✔
69	if not_null {	36,306✔
70	self.append_non_null()	36,306✔
71	} else {
72	self.append_null()	×
73	}
74	}	36,306✔
75
76	#[inline]
77	pub fn append_buffer(&mut self, bool_buffer: &BooleanBuffer) {	2,600✔
78	self.materialize_if_needed();	2,600✔
79	self.inner	2,600✔
80	.as_mut()	2,600✔
81	.vortex_expect("buffer just materialized")	2,600✔
82	.append_buffer(bool_buffer);	2,600✔
83	}	2,600✔
84
85	pub fn append_validity_mask(&mut self, validity_mask: Mask) {	51,251✔
86	match validity_mask {	51,251✔
87	Mask::AllTrue(len) => self.append_n_non_nulls(len),	48,651✔
88	Mask::AllFalse(len) => self.append_n_nulls(len),	×
89	Mask::Values(is_valid) => self.append_buffer(is_valid.boolean_buffer()),	2,600✔
90	}
91	}	51,251✔
92
93	pub fn set_bit(&mut self, index: usize, v: bool) {	×
94	self.materialize_if_needed();	×
95	self.inner	×
96	.as_mut()	×
97	.vortex_expect("buffer just materialized")	×
98	.set_bit(index, v);	×
99	}	×
100
101	pub fn len(&self) -> usize {	27,996✔
102	// self.len is the length of the builder if the inner buffer is not materialized
103	self.inner.as_ref().map(\|i\| i.len()).unwrap_or(self.len)	27,996✔
104	}	27,996✔
105
106	pub fn truncate(&mut self, len: usize) {	×
107	if let Some(b) = self.inner.as_mut() {	×
108	b.truncate(len)	×
109	}	×
110	self.len = len;	×
111	}	×
112
113	pub fn reserve(&mut self, n: usize) {	×
114	self.materialize_if_needed();	×
115	self.inner	×
116	.as_mut()	×
117	.vortex_expect("buffer just materialized")	×
118	.reserve(n);	×
119	}	×
120
121	fn finish(&mut self) -> Option<NullBuffer> {	38,340✔
122	self.len = 0;	38,340✔
123	Some(NullBuffer::new(self.inner.take()?.finish()))	38,340✔
124	}	38,340✔
125
126	pub fn finish_with_nullability(&mut self, nullability: Nullability) -> Validity {	38,340✔
127	let nulls = self.finish();	38,340✔
128
129	match (nullability, nulls) {	38,340✔
130	(NonNullable, None) => Validity::NonNullable,	17,945✔
131	(Nullable, None) => Validity::AllValid,	18,506✔
132	(Nullable, Some(arr)) => Validity::from(arr),	1,889✔
UNCOV 133	_ => vortex_panic!("Invalid nullability/nulls combination"),	×
134	}
135	}	38,340✔
136
137	pub fn ensure_capacity(&mut self, capacity: usize) {	×
138	if self.inner.is_none() {	×
139	self.capacity = capacity;	×
140	} else {	×
141	let inner = self	×
142	.inner	×
143	.as_mut()	×
144	.vortex_expect("buffer just materialized");	×
145	if capacity < inner.capacity() {	×
146	inner.reserve(capacity - inner.len());	×
UNCOV 147	}	×
148	}
UNCOV 149	}	×
150
151	#[inline]
152	fn materialize_if_needed(&mut self) {	5,015✔
153	if self.inner.is_none() {	5,015✔
154	self.materialize()	1,889✔
155	}	3,126✔
156	}	5,015✔
157
158	// This only happens once per builder
159	#[cold]
160	#[inline(never)]
161	fn materialize(&mut self) {	1,889✔
162	if self.inner.is_none() {	1,889✔
163	let mut b = BooleanBufferBuilder::new(self.len.max(self.capacity));	1,889✔
164	b.append_n(self.len, true);	1,889✔
165	self.inner = Some(b);	1,889✔
166	}	1,889✔
167	}	1,889✔
168	}

vortex-data / vortex / 16578751259

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous