17075133033

Committed 19 Aug 2025 04:01PM UTC coverage: 87.949% (+0.09%) from 87.856%

Build # 17075133033

Build Type

push

github

Committed by

web-flow

Commit Message

feat: ArrayOperations infallible, eager validation + new_unchecked (#4177)

ArrayOperations currently return VortexResult<>, but they really should
just be infallible. A failed array op is generally indicative of
programmer or encoding error. There's really nothing interesting we can
do to handle an out-of-bounds slice() or scalar_at.

There's a lot that falls out of this, like fixing a bunch of tests,
tweaking our scalar value casting to return Option instead of Result,
etc.

---------

Signed-off-by: Andrew Duffy <andrew@a10y.dev>

Run Details

1744 of 1985 new or added lines in 195 files covered. (87.86%)

36 existing lines in 27 files now uncovered.

56745 of 64520 relevant lines covered (87.95%)

624082.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.76

/vortex-array/src/compute/conformance/filter.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_dtype::DType;
use vortex_error::VortexUnwrap;
use vortex_mask::Mask;

use crate::arrays::BoolArray;
use crate::compute::filter;
use crate::{Array, IntoArray};

// Standard test array sizes
pub const SMALL_SIZE: usize = 5;
pub const MEDIUM_SIZE: usize = 100;
pub const LARGE_SIZE: usize = 1024;

/// Test filter compute function with various array sizes and patterns.
/// The input array can be of any length.
pub fn test_filter_conformance(array: &dyn Array) {
    let len = array.len();

    // Test with arrays of any size
    if len > 0 {
        test_all_filter(array);
        test_none_filter(array);
        test_selective_filter(array);
        test_single_element_filter(array);
        test_nullable_filter(array);
        test_alternating_pattern_filter(array);
        test_runs_pattern_filter(array);
        test_sparse_true_filter(array);
        test_sparse_false_filter(array);
    }

    // Test random pattern for arrays with at least 4 elements
    if len >= 4 {
        test_random_pattern_filter(array);
    }

    // Test edge cases
    test_empty_array_filter(array.dtype());
    test_mismatched_lengths(array);

    // Test with nullable masks
    if len > 0 {
        test_nullable_mask_input(array);
    }
}

// Helper functions for creating standard patterns
pub fn create_alternating_pattern(len: usize) -> Vec<bool> {
    (0..len).map(|i| i % 2 == 0).collect()
}

pub fn create_sparse_pattern(len: usize, true_ratio: f64) -> Vec<bool> {
    (0..len)
        .map(|i| (i as f64 / len as f64) < true_ratio)
        .collect()
}

pub fn create_runs_pattern(len: usize, run_length: usize) -> Vec<bool> {
    (0..len).map(|i| (i / run_length) % 2 == 0).collect()
}

/// Tests that filtering with an all-true mask returns all elements unchanged
fn test_all_filter(array: &dyn Array) {
    let len = array.len();
    let mask = Mask::new_true(len);
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), len);

    // Verify all elements are preserved
    for i in 0..len {
        assert_eq!(filtered.scalar_at(i), array.scalar_at(i));
    }
}

/// Tests that filtering with an all-false mask returns an empty array with the same dtype
fn test_none_filter(array: &dyn Array) {
    let len = array.len();
    let mask = Mask::new_false(len);
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), 0);
    assert_eq!(filtered.dtype(), array.dtype());
}

fn test_selective_filter(array: &dyn Array) {
    let len = array.len();
    if len < 2 {
        return; // Skip for very small arrays
    }

    // Test alternating pattern
    let mask_values: Vec<bool> = (0..len).map(|i| i % 2 == 0).collect();
    let expected_count = mask_values.iter().filter(|&&v| v).count();
    let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);

    // Verify correct elements are kept
    for (filtered_idx, i) in (0..len).step_by(2).enumerate() {
        assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i));
    }

    // Test first and last only
    if len >= 2 {
        let mut mask_values = vec![false; len];
        mask_values[0] = true;
        mask_values[len - 1] = true;
        let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();
        let filtered = filter(array, &mask).vortex_unwrap();
        assert_eq!(filtered.len(), 2);
        assert_eq!(filtered.scalar_at(0), array.scalar_at(0));
        assert_eq!(filtered.scalar_at(1), array.scalar_at(len - 1));
    }
}

fn test_single_element_filter(array: &dyn Array) {
    let len = array.len();
    if len == 0 {
        return;
    }

    // Test selecting only the first element
    let mut mask_values = vec![false; len];
    mask_values[0] = true;
    let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), 1);
    assert_eq!(filtered.scalar_at(0), array.scalar_at(0));

    // Test selecting only the last element
    if len > 1 {
        let mut mask_values = vec![false; len];
        mask_values[len - 1] = true;
        let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();
        let filtered = filter(array, &mask).vortex_unwrap();
        assert_eq!(filtered.len(), 1);
        assert_eq!(filtered.scalar_at(0), array.scalar_at(len - 1));
    }
}

fn test_nullable_filter(array: &dyn Array) {
    let len = array.len();
    if len < 2 {
        return; // Skip for very small arrays
    }

    // Create a nullable mask where nulls are treated as false
    let bool_values: Vec<bool> = (0..len).map(|i| i % 3 == 0).collect();
    let validity_values: Vec<bool> = (0..len).map(|i| i % 2 == 0).collect();

    let expected_count = bool_values
        .iter()
        .zip(validity_values.iter())
        .filter(|(b, v)| **b && **v)
        .count();

    let bool_array = BoolArray::from_iter(bool_values.clone());
    let validity = crate::validity::Validity::from_iter(validity_values.clone());
    let nullable_mask = BoolArray::new(bool_array.boolean_buffer().clone(), validity);

    let mask = Mask::try_from(&nullable_mask).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);

    // Verify correct elements are kept
    let mut filtered_idx = 0;
    for i in 0..len {
        if bool_values[i] && validity_values[i] {
            assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i));
            filtered_idx += 1;
        }
    }
}

fn test_empty_array_filter(dtype: &DType) {
    use crate::Canonical;

    let empty_array = Canonical::empty(dtype).into_array();
    let empty_mask = Mask::new_false(0);
    let filtered = filter(&empty_array, &empty_mask).vortex_unwrap();
    assert_eq!(filtered.len(), 0);

    let empty_mask = Mask::new_true(0);
    let filtered = filter(&empty_array, &empty_mask).vortex_unwrap();
    assert_eq!(filtered.len(), 0);
}

fn test_mismatched_lengths(array: &dyn Array) {
    let len = array.len();

    // Test mask shorter than array
    if len > 0 {
        let short_mask = Mask::new_true(len - 1);
        let result = filter(array, &short_mask);
        assert!(
            result.is_err(),
            "Filter should fail with mismatched lengths"
        );
    }

    // Test mask longer than array
    let long_mask = Mask::new_true(len + 1);
    let result = filter(array, &long_mask);
    assert!(
        result.is_err(),
        "Filter should fail with mismatched lengths"
    );
}

/// Tests filtering with alternating true/false pattern
fn test_alternating_pattern_filter(array: &dyn Array) {
    let len = array.len();
    let pattern = create_alternating_pattern(len);
    let expected_count = pattern.iter().filter(|&&v| v).count();

    let mask = Mask::try_from(&BoolArray::from_iter(pattern.clone())).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);

    // Verify correct elements are kept
    let mut filtered_idx = 0;
    for (i, &keep) in pattern.iter().enumerate() {
        if keep {
            assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i));
            filtered_idx += 1;
        }
    }
}

/// Tests filtering with runs of true/false values
fn test_runs_pattern_filter(array: &dyn Array) {
    let len = array.len();
    if len < 4 {
        return; // Skip for very small arrays
    }

    let run_length = len.min(3);
    let pattern = create_runs_pattern(len, run_length);
    let expected_count = pattern.iter().filter(|&&v| v).count();

    let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);
}

/// Tests filtering with sparse true values (mostly false)
fn test_sparse_true_filter(array: &dyn Array) {
    let len = array.len();
    if len < 10 {
        return; // Skip for small arrays
    }

    // Only keep about 10% of values
    let pattern = create_sparse_pattern(len, 0.1);
    let expected_count = pattern.iter().filter(|&&v| v).count();

    let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);
}

/// Tests filtering with sparse false values (mostly true)
fn test_sparse_false_filter(array: &dyn Array) {
    let len = array.len();
    if len < 10 {
        return; // Skip for small arrays
    }

    // Keep about 90% of values
    let pattern = create_sparse_pattern(len, 0.9);
    let expected_count = pattern.iter().filter(|&&v| v).count();

    let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);
}

/// Tests filtering with random pattern
fn test_random_pattern_filter(array: &dyn Array) {
    let len = array.len();

    // Create a pseudo-random pattern based on array length
    let pattern: Vec<bool> = (0..len)
        .map(|i| ((i * 37 + 17) % 5) < 3) // Deterministic pseudo-random
        .collect();
    let expected_count = pattern.iter().filter(|&&v| v).count();

    let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);
}

/// Tests filtering with nullable mask (nulls treated as false)
fn test_nullable_mask_input(array: &dyn Array) {
    let len = array.len();
    if len < 3 {
        return; // Skip for very small arrays
    }

    // Create a nullable mask where every third value is null
    let bool_values: Vec<bool> = (0..len).map(|i| i % 2 == 0).collect();
    let validity_values: Vec<bool> = (0..len).map(|i| i % 3 != 0).collect();

    // Only values that are true AND valid should pass the filter
    let expected_count = bool_values
        .iter()
        .zip(validity_values.iter())
        .filter(|(b, v)| **b && **v)
        .count();

    let bool_array = BoolArray::from_iter(bool_values.clone());
    let validity = crate::validity::Validity::from_iter(validity_values.clone());
    let nullable_mask = BoolArray::new(bool_array.boolean_buffer().clone(), validity);

    let mask = Mask::try_from(&nullable_mask).vortex_unwrap();
    let filtered = filter(array, &mask).vortex_unwrap();
    assert_eq!(filtered.len(), expected_count);

    // Verify correct elements are kept
    let mut filtered_idx = 0;
    for i in 0..len {
        if bool_values[i] && validity_values[i] {
            assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i),);
            filtered_idx += 1;
        }
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use vortex_dtype::DType;
5	use vortex_error::VortexUnwrap;
6	use vortex_mask::Mask;
7
8	use crate::arrays::BoolArray;
9	use crate::compute::filter;
10	use crate::{Array, IntoArray};
11
12	// Standard test array sizes
13	pub const SMALL_SIZE: usize = 5;
14	pub const MEDIUM_SIZE: usize = 100;
15	pub const LARGE_SIZE: usize = 1024;
16
17	/// Test filter compute function with various array sizes and patterns.
18	/// The input array can be of any length.
19	pub fn test_filter_conformance(array: &dyn Array) {	1,290✔
20	let len = array.len();	1,290✔
21
22	// Test with arrays of any size
23	if len > 0 {	1,290✔
24	test_all_filter(array);	1,290✔
25	test_none_filter(array);	1,290✔
26	test_selective_filter(array);	1,290✔
27	test_single_element_filter(array);	1,290✔
28	test_nullable_filter(array);	1,290✔
29	test_alternating_pattern_filter(array);	1,290✔
30	test_runs_pattern_filter(array);	1,290✔
31	test_sparse_true_filter(array);	1,290✔
32	test_sparse_false_filter(array);	1,290✔
33	}	1,290✔
34
35	// Test random pattern for arrays with at least 4 elements
36	if len >= 4 {	1,290✔
37	test_random_pattern_filter(array);	1,244✔
38	}	1,244✔
39
40	// Test edge cases
41	test_empty_array_filter(array.dtype());	1,290✔
42	test_mismatched_lengths(array);	1,290✔
43
44	// Test with nullable masks
45	if len > 0 {	1,290✔
46	test_nullable_mask_input(array);	1,290✔
47	}	1,290✔
48	}	1,290✔
49
50	// Helper functions for creating standard patterns
51	pub fn create_alternating_pattern(len: usize) -> Vec<bool> {	1,290✔
52	(0..len).map(\|i\| i % 2 == 0).collect()	9,210✔
53	}	1,290✔
54
55	pub fn create_sparse_pattern(len: usize, true_ratio: f64) -> Vec<bool> {	92✔
56	(0..len)	92✔
57	.map(\|i\| (i as f64 / len as f64) < true_ratio)	6,336✔
58	.collect()	92✔
59	}	92✔
60
61	pub fn create_runs_pattern(len: usize, run_length: usize) -> Vec<bool> {	1,244✔
62	(0..len).map(\|i\| (i / run_length) % 2 == 0).collect()	9,162✔
63	}	1,244✔
64
65	/// Tests that filtering with an all-true mask returns all elements unchanged
66	fn test_all_filter(array: &dyn Array) {	1,290✔
67	let len = array.len();	1,290✔
68	let mask = Mask::new_true(len);	1,290✔
69	let filtered = filter(array, &mask).vortex_unwrap();	1,290✔
70	assert_eq!(filtered.len(), len);	1,290✔
71
72	// Verify all elements are preserved
73	for i in 0..len {	9,210✔
74	assert_eq!(filtered.scalar_at(i), array.scalar_at(i));	9,210✔
75	}
76	}	1,290✔
77
78	/// Tests that filtering with an all-false mask returns an empty array with the same dtype
79	fn test_none_filter(array: &dyn Array) {	1,290✔
80	let len = array.len();	1,290✔
81	let mask = Mask::new_false(len);	1,290✔
82	let filtered = filter(array, &mask).vortex_unwrap();	1,290✔
83	assert_eq!(filtered.len(), 0);	1,290✔
84	assert_eq!(filtered.dtype(), array.dtype());	1,290✔
85	}	1,290✔
86
87	fn test_selective_filter(array: &dyn Array) {	1,290✔
88	let len = array.len();	1,290✔
89	if len < 2 {	1,290✔
90	return; // Skip for very small arrays	44✔
91	}	1,246✔
92
93	// Test alternating pattern
94	let mask_values: Vec<bool> = (0..len).map(\|i\| i % 2 == 0).collect();	9,166✔
95	let expected_count = mask_values.iter().filter(\|&&v\| v).count();	1,246✔
96	let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();	1,246✔
97	let filtered = filter(array, &mask).vortex_unwrap();	1,246✔
98	assert_eq!(filtered.len(), expected_count);	1,246✔
99
100	// Verify correct elements are kept
101	for (filtered_idx, i) in (0..len).step_by(2).enumerate() {	5,181✔
102	assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i));	5,181✔
103	}
104
105	// Test first and last only
106	if len >= 2 {	1,246✔
107	let mut mask_values = vec![false; len];	1,246✔
108	mask_values[0] = true;	1,246✔
109	mask_values[len - 1] = true;	1,246✔
110	let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();	1,246✔
111	let filtered = filter(array, &mask).vortex_unwrap();	1,246✔
112	assert_eq!(filtered.len(), 2);	1,246✔
113	assert_eq!(filtered.scalar_at(0), array.scalar_at(0));	1,246✔
114	assert_eq!(filtered.scalar_at(1), array.scalar_at(len - 1));	1,246✔
UNCOV 115	}	×
116	}	1,290✔
117
118	fn test_single_element_filter(array: &dyn Array) {	1,290✔
119	let len = array.len();	1,290✔
120	if len == 0 {	1,290✔
121	return;	×
122	}	1,290✔
123
124	// Test selecting only the first element
125	let mut mask_values = vec![false; len];	1,290✔
126	mask_values[0] = true;	1,290✔
127	let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();	1,290✔
128	let filtered = filter(array, &mask).vortex_unwrap();	1,290✔
129	assert_eq!(filtered.len(), 1);	1,290✔
130	assert_eq!(filtered.scalar_at(0), array.scalar_at(0));	1,290✔
131
132	// Test selecting only the last element
133	if len > 1 {	1,290✔
134	let mut mask_values = vec![false; len];	1,246✔
135	mask_values[len - 1] = true;	1,246✔
136	let mask = Mask::try_from(&BoolArray::from_iter(mask_values)).vortex_unwrap();	1,246✔
137	let filtered = filter(array, &mask).vortex_unwrap();	1,246✔
138	assert_eq!(filtered.len(), 1);	1,246✔
139	assert_eq!(filtered.scalar_at(0), array.scalar_at(len - 1));	1,246✔
140	}	44✔
141	}	1,290✔
142
143	fn test_nullable_filter(array: &dyn Array) {	1,290✔
144	let len = array.len();	1,290✔
145	if len < 2 {	1,290✔
146	return; // Skip for very small arrays	44✔
147	}	1,246✔
148
149	// Create a nullable mask where nulls are treated as false
150	let bool_values: Vec<bool> = (0..len).map(\|i\| i % 3 == 0).collect();	9,166✔
151	let validity_values: Vec<bool> = (0..len).map(\|i\| i % 2 == 0).collect();	9,166✔
152
153	let expected_count = bool_values	1,246✔
154	.iter()	1,246✔
155	.zip(validity_values.iter())	1,246✔
156	.filter(\|(b, v)\| b && v)	9,166✔
157	.count();	1,246✔
158
159	let bool_array = BoolArray::from_iter(bool_values.clone());	1,246✔
160	let validity = crate::validity::Validity::from_iter(validity_values.clone());	1,246✔
161	let nullable_mask = BoolArray::new(bool_array.boolean_buffer().clone(), validity);	1,246✔
162
163	let mask = Mask::try_from(&nullable_mask).vortex_unwrap();	1,246✔
164	let filtered = filter(array, &mask).vortex_unwrap();	1,246✔
165	assert_eq!(filtered.len(), expected_count);	1,246✔
166
167	// Verify correct elements are kept
168	let mut filtered_idx = 0;	1,246✔
169	for i in 0..len {	9,166✔
170	if bool_values[i] && validity_values[i] {	9,166✔
171	assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i));	1,757✔
172	filtered_idx += 1;	1,757✔
173	}	7,409✔
174	}
175	}	1,290✔
176
177	fn test_empty_array_filter(dtype: &DType) {	1,290✔
178	use crate::Canonical;
179
180	let empty_array = Canonical::empty(dtype).into_array();	1,290✔
181	let empty_mask = Mask::new_false(0);	1,290✔
182	let filtered = filter(&empty_array, &empty_mask).vortex_unwrap();	1,290✔
183	assert_eq!(filtered.len(), 0);	1,290✔
184
185	let empty_mask = Mask::new_true(0);	1,290✔
186	let filtered = filter(&empty_array, &empty_mask).vortex_unwrap();	1,290✔
187	assert_eq!(filtered.len(), 0);	1,290✔
188	}	1,290✔
189
190	fn test_mismatched_lengths(array: &dyn Array) {	1,290✔
191	let len = array.len();	1,290✔
192
193	// Test mask shorter than array
194	if len > 0 {	1,290✔
195	let short_mask = Mask::new_true(len - 1);	1,290✔
196	let result = filter(array, &short_mask);	1,290✔
197	assert!(	1,290✔
198	result.is_err(),	1,290✔
199	"Filter should fail with mismatched lengths"	×
200	);
201	}	×
202
203	// Test mask longer than array
204	let long_mask = Mask::new_true(len + 1);	1,290✔
205	let result = filter(array, &long_mask);	1,290✔
206	assert!(	1,290✔
207	result.is_err(),	1,290✔
208	"Filter should fail with mismatched lengths"	×
209	);
210	}	1,290✔
211
212	/// Tests filtering with alternating true/false pattern
213	fn test_alternating_pattern_filter(array: &dyn Array) {	1,290✔
214	let len = array.len();	1,290✔
215	let pattern = create_alternating_pattern(len);	1,290✔
216	let expected_count = pattern.iter().filter(\|&&v\| v).count();	1,290✔
217
218	let mask = Mask::try_from(&BoolArray::from_iter(pattern.clone())).vortex_unwrap();	1,290✔
219	let filtered = filter(array, &mask).vortex_unwrap();	1,290✔
220	assert_eq!(filtered.len(), expected_count);	1,290✔
221
222	// Verify correct elements are kept
223	let mut filtered_idx = 0;	1,290✔
224	for (i, &keep) in pattern.iter().enumerate() {	9,210✔
225	if keep {	9,210✔
226	assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i));	5,225✔
227	filtered_idx += 1;	5,225✔
228	}	3,985✔
229	}
230	}	1,290✔
231
232	/// Tests filtering with runs of true/false values
233	fn test_runs_pattern_filter(array: &dyn Array) {	1,290✔
234	let len = array.len();	1,290✔
235	if len < 4 {	1,290✔
236	return; // Skip for very small arrays	46✔
237	}	1,244✔
238
239	let run_length = len.min(3);	1,244✔
240	let pattern = create_runs_pattern(len, run_length);	1,244✔
241	let expected_count = pattern.iter().filter(\|&&v\| v).count();	1,244✔
242
243	let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();	1,244✔
244	let filtered = filter(array, &mask).vortex_unwrap();	1,244✔
245	assert_eq!(filtered.len(), expected_count);	1,244✔
246	}	1,290✔
247
248	/// Tests filtering with sparse true values (mostly false)
249	fn test_sparse_true_filter(array: &dyn Array) {	1,290✔
250	let len = array.len();	1,290✔
251	if len < 10 {	1,290✔
252	return; // Skip for small arrays	1,244✔
253	}	46✔
254
255	// Only keep about 10% of values
256	let pattern = create_sparse_pattern(len, 0.1);	46✔
257	let expected_count = pattern.iter().filter(\|&&v\| v).count();	46✔
258
259	let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();	46✔
260	let filtered = filter(array, &mask).vortex_unwrap();	46✔
261	assert_eq!(filtered.len(), expected_count);	46✔
262	}	1,290✔
263
264	/// Tests filtering with sparse false values (mostly true)
265	fn test_sparse_false_filter(array: &dyn Array) {	1,290✔
266	let len = array.len();	1,290✔
267	if len < 10 {	1,290✔
268	return; // Skip for small arrays	1,244✔
269	}	46✔
270
271	// Keep about 90% of values
272	let pattern = create_sparse_pattern(len, 0.9);	46✔
273	let expected_count = pattern.iter().filter(\|&&v\| v).count();	46✔
274
275	let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();	46✔
276	let filtered = filter(array, &mask).vortex_unwrap();	46✔
277	assert_eq!(filtered.len(), expected_count);	46✔
278	}	1,290✔
279
280	/// Tests filtering with random pattern
281	fn test_random_pattern_filter(array: &dyn Array) {	1,244✔
282	let len = array.len();	1,244✔
283
284	// Create a pseudo-random pattern based on array length
285	let pattern: Vec<bool> = (0..len)	1,244✔
286	.map(\|i\| ((i * 37 + 17) % 5) < 3) // Deterministic pseudo-random	9,162✔
287	.collect();	1,244✔
288	let expected_count = pattern.iter().filter(\|&&v\| v).count();	1,244✔
289
290	let mask = Mask::try_from(&BoolArray::from_iter(pattern)).vortex_unwrap();	1,244✔
291	let filtered = filter(array, &mask).vortex_unwrap();	1,244✔
292	assert_eq!(filtered.len(), expected_count);	1,244✔
293	}	1,244✔
294
295	/// Tests filtering with nullable mask (nulls treated as false)
296	fn test_nullable_mask_input(array: &dyn Array) {	1,290✔
297	let len = array.len();	1,290✔
298	if len < 3 {	1,290✔
299	return; // Skip for very small arrays	46✔
300	}	1,244✔
301
302	// Create a nullable mask where every third value is null
303	let bool_values: Vec<bool> = (0..len).map(\|i\| i % 2 == 0).collect();	9,162✔
304	let validity_values: Vec<bool> = (0..len).map(\|i\| i % 3 != 0).collect();	9,162✔
305
306	// Only values that are true AND valid should pass the filter
307	let expected_count = bool_values	1,244✔
308	.iter()	1,244✔
309	.zip(validity_values.iter())	1,244✔
310	.filter(\|(b, v)\| b && v)	9,162✔
311	.count();	1,244✔
312
313	let bool_array = BoolArray::from_iter(bool_values.clone());	1,244✔
314	let validity = crate::validity::Validity::from_iter(validity_values.clone());	1,244✔
315	let nullable_mask = BoolArray::new(bool_array.boolean_buffer().clone(), validity);	1,244✔
316
317	let mask = Mask::try_from(&nullable_mask).vortex_unwrap();	1,244✔
318	let filtered = filter(array, &mask).vortex_unwrap();	1,244✔
319	assert_eq!(filtered.len(), expected_count);	1,244✔
320
321	// Verify correct elements are kept
322	let mut filtered_idx = 0;	1,244✔
323	for i in 0..len {	9,162✔
324	if bool_values[i] && validity_values[i] {	9,162✔
325	assert_eq!(filtered.scalar_at(filtered_idx), array.scalar_at(i),);	3,424✔
326	filtered_idx += 1;	3,424✔
327	}	5,738✔
328	}
329	}	1,290✔

vortex-data / vortex / 17075133033

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous