• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

vortex-data / vortex / 16324116701

16 Jul 2025 03:46PM UTC coverage: 80.69% (-0.9%) from 81.557%
16324116701

Pull #3881

github

web-flow
Merge c08ea4033 into ced09d9a8
Pull Request #3881: feat: build with stable rust

118 of 171 new or added lines in 27 files covered. (69.01%)

174 existing lines in 102 files now uncovered.

41861 of 51879 relevant lines covered (80.69%)

157334.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.01
/vortex-array/src/arrays/list/mod.rs
1
// SPDX-License-Identifier: Apache-2.0
2
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3

4
mod compute;
5
mod serde;
6

7
use std::sync::Arc;
8

9
use itertools::Itertools;
10
use num_traits::{AsPrimitive, PrimInt};
11
use vortex_dtype::{DType, NativePType, match_each_native_ptype};
12
use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_panic};
13
use vortex_scalar::Scalar;
14

15
use crate::arrays::PrimitiveVTable;
16
#[cfg(feature = "test-harness")]
17
use crate::builders::{ArrayBuilder, ListBuilder};
18
use crate::stats::{ArrayStats, StatsSetRef};
19
use crate::validity::Validity;
20
use crate::vtable::{
21
    ArrayVTable, CanonicalVTable, NotSupported, OperationsVTable, VTable, ValidityHelper,
22
    ValidityVTableFromValidityHelper,
23
};
24
use crate::{Array, ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, vtable};
25

26
vtable!(List);
27

28
impl VTable for ListVTable {
29
    type Array = ListArray;
30
    type Encoding = ListEncoding;
31

32
    type ArrayVTable = Self;
33
    type CanonicalVTable = Self;
34
    type OperationsVTable = Self;
35
    type ValidityVTable = ValidityVTableFromValidityHelper;
36
    type VisitorVTable = Self;
37
    type ComputeVTable = NotSupported;
38
    type EncodeVTable = NotSupported;
39
    type SerdeVTable = Self;
40

41
    fn id(_encoding: &Self::Encoding) -> EncodingId {
33,287✔
42
        EncodingId::new_ref("vortex.list")
33,287✔
43
    }
33,287✔
44

45
    fn encoding(_array: &Self::Array) -> EncodingRef {
49✔
46
        EncodingRef::new_ref(ListEncoding.as_ref())
49✔
47
    }
49✔
48
}
49

50
#[derive(Clone, Debug)]
51
pub struct ListArray {
52
    dtype: DType,
53
    elements: ArrayRef,
54
    offsets: ArrayRef,
55
    validity: Validity,
56
    stats_set: ArrayStats,
57
}
58

59
#[derive(Clone, Debug)]
60
pub struct ListEncoding;
61

62
pub trait OffsetPType: NativePType + PrimInt + AsPrimitive<usize> + Into<Scalar> {}
63

64
impl<T> OffsetPType for T where T: NativePType + PrimInt + AsPrimitive<usize> + Into<Scalar> {}
65

66
// A list is valid if the:
67
// - offsets start at a value in elements
68
// - offsets are sorted
69
// - the final offset points to an element in the elements list, pointing to zero
70
//   if elements are empty.
71
// - final_offset >= start_offset
72
// - The size of the validity is the size-1 of the offset array
73

74
impl ListArray {
75
    pub fn try_new(
587✔
76
        elements: ArrayRef,
587✔
77
        offsets: ArrayRef,
587✔
78
        validity: Validity,
587✔
79
    ) -> VortexResult<Self> {
587✔
80
        let nullability = validity.nullability();
587✔
81

82
        if !offsets.dtype().is_int() || offsets.dtype().is_nullable() {
587✔
83
            vortex_bail!(
×
84
                "Expected offsets to be an non-nullable integer type, got {:?}",
×
85
                offsets.dtype()
×
86
            );
87
        }
587✔
88

89
        if offsets.is_empty() {
587✔
90
            vortex_bail!("Offsets must have at least one element, [0] for an empty list");
×
91
        }
587✔
92

93
        Ok(Self {
587✔
94
            dtype: DType::List(Arc::new(elements.dtype().clone()), nullability),
587✔
95
            elements,
587✔
96
            offsets,
587✔
97
            validity,
587✔
98
            stats_set: Default::default(),
587✔
99
        })
587✔
100
    }
587✔
101

102
    // TODO: merge logic with varbin
103
    // TODO(ngates): should return a result if it requires canonicalizing offsets
104
    pub fn offset_at(&self, index: usize) -> usize {
3,430✔
105
        self.offsets()
3,430✔
106
            .as_opt::<PrimitiveVTable>()
3,430✔
107
            .map(|p| match_each_native_ptype!(p.ptype(), |P| { p.as_slice::<P>()[index].as_() }))
3,430✔
108
            .unwrap_or_else(|| {
3,430✔
109
                self.offsets()
×
110
                    .scalar_at(index)
×
111
                    .unwrap_or_else(|err| {
×
112
                        vortex_panic!(err, "Failed to get offset at index: {}", index)
×
113
                    })
114
                    .as_ref()
×
115
                    .try_into()
×
116
                    .vortex_expect("Failed to convert offset to usize")
×
UNCOV
117
            })
×
118
    }
3,430✔
119

120
    // TODO: fetches the elements at index
121
    pub fn elements_at(&self, index: usize) -> VortexResult<ArrayRef> {
906✔
122
        let start = self.offset_at(index);
906✔
123
        let end = self.offset_at(index + 1);
906✔
124
        self.elements().slice(start, end)
906✔
125
    }
906✔
126

127
    // TODO: fetches the offsets of the array ignoring validity
128
    pub fn offsets(&self) -> &ArrayRef {
4,077✔
129
        &self.offsets
4,077✔
130
    }
4,077✔
131

132
    // TODO: fetches the elements of the array ignoring validity
133
    pub fn elements(&self) -> &ArrayRef {
1,578✔
134
        &self.elements
1,578✔
135
    }
1,578✔
136
}
137

138
impl ArrayVTable<ListVTable> for ListVTable {
139
    fn len(array: &ListArray) -> usize {
7,293✔
140
        array.offsets.len().saturating_sub(1)
7,293✔
141
    }
7,293✔
142

143
    fn dtype(array: &ListArray) -> &DType {
8,514✔
144
        &array.dtype
8,514✔
145
    }
8,514✔
146

147
    fn stats(array: &ListArray) -> StatsSetRef<'_> {
3,435✔
148
        array.stats_set.to_ref(array.as_ref())
3,435✔
149
    }
3,435✔
150
}
151

152
impl OperationsVTable<ListVTable> for ListVTable {
153
    fn slice(array: &ListArray, start: usize, stop: usize) -> VortexResult<ArrayRef> {
3✔
154
        Ok(ListArray::try_new(
3✔
155
            array.elements().clone(),
3✔
156
            array.offsets().slice(start, stop + 1)?,
3✔
157
            array.validity().slice(start, stop)?,
3✔
158
        )?
×
159
        .into_array())
3✔
160
    }
3✔
161

162
    fn scalar_at(array: &ListArray, index: usize) -> VortexResult<Scalar> {
901✔
163
        let elem = array.elements_at(index)?;
901✔
164
        let scalars: Vec<Scalar> = (0..elem.len()).map(|i| elem.scalar_at(i)).try_collect()?;
2,395✔
165

166
        Ok(Scalar::list(
901✔
167
            Arc::new(elem.dtype().clone()),
901✔
168
            scalars,
901✔
169
            array.dtype().nullability(),
901✔
170
        ))
901✔
171
    }
901✔
172
}
173

174
impl CanonicalVTable<ListVTable> for ListVTable {
175
    fn canonicalize(array: &ListArray) -> VortexResult<Canonical> {
481✔
176
        Ok(Canonical::List(array.clone()))
481✔
177
    }
481✔
178
}
179

180
impl ValidityHelper for ListArray {
181
    fn validity(&self) -> &Validity {
2,716✔
182
        &self.validity
2,716✔
183
    }
2,716✔
184
}
185

186
#[cfg(feature = "test-harness")]
187
impl ListArray {
188
    /// This is a convenience method to create a list array from an iterator of iterators.
189
    /// This method is slow however since each element is first converted to a scalar and then
190
    /// appended to the array.
191
    pub fn from_iter_slow<O: OffsetPType, I: IntoIterator>(
7✔
192
        iter: I,
7✔
193
        dtype: Arc<DType>,
7✔
194
    ) -> VortexResult<ArrayRef>
7✔
195
    where
7✔
196
        I::Item: IntoIterator,
7✔
197
        <I::Item as IntoIterator>::Item: Into<Scalar>,
7✔
198
    {
199
        let iter = iter.into_iter();
7✔
200
        let mut builder = ListBuilder::<O>::with_capacity(
7✔
201
            dtype.clone(),
7✔
202
            vortex_dtype::Nullability::NonNullable,
7✔
203
            iter.size_hint().0,
7✔
204
        );
205

206
        for v in iter {
29✔
207
            let elem = Scalar::list(
22✔
208
                dtype.clone(),
22✔
209
                v.into_iter().map(|x| x.into()).collect_vec(),
30✔
210
                dtype.nullability(),
22✔
211
            );
212
            builder.append_value(elem.as_list())?
22✔
213
        }
214
        Ok(builder.finish())
7✔
215
    }
7✔
216

217
    pub fn from_iter_opt_slow<O: OffsetPType, I: IntoIterator<Item = Option<T>>, T>(
16✔
218
        iter: I,
16✔
219
        dtype: Arc<DType>,
16✔
220
    ) -> VortexResult<ArrayRef>
16✔
221
    where
16✔
222
        T: IntoIterator,
16✔
223
        T::Item: Into<Scalar>,
16✔
224
    {
225
        let iter = iter.into_iter();
16✔
226
        let mut builder = ListBuilder::<O>::with_capacity(
16✔
227
            dtype.clone(),
16✔
228
            vortex_dtype::Nullability::Nullable,
16✔
229
            iter.size_hint().0,
16✔
230
        );
231

232
        for v in iter {
88✔
233
            if let Some(v) = v {
72✔
234
                let elem = Scalar::list(
48✔
235
                    dtype.clone(),
48✔
236
                    v.into_iter().map(|x| x.into()).collect_vec(),
120✔
237
                    dtype.nullability(),
48✔
238
                );
239
                builder.append_value(elem.as_list())?
48✔
240
            } else {
241
                builder.append_null()
24✔
242
            }
243
        }
244
        Ok(builder.finish())
16✔
245
    }
16✔
246
}
247

248
#[cfg(test)]
249
mod test {
250
    use std::sync::Arc;
251

252
    use arrow_buffer::BooleanBuffer;
253
    use vortex_dtype::Nullability;
254
    use vortex_dtype::PType::I32;
255
    use vortex_mask::Mask;
256
    use vortex_scalar::Scalar;
257

258
    use crate::arrays::PrimitiveArray;
259
    use crate::arrays::list::ListArray;
260
    use crate::compute::filter;
261
    use crate::validity::Validity;
262
    use crate::{Array, IntoArray};
263

264
    #[test]
265
    fn test_empty_list_array() {
1✔
266
        let elements = PrimitiveArray::empty::<u32>(Nullability::NonNullable);
1✔
267
        let offsets = PrimitiveArray::from_iter([0]);
1✔
268
        let validity = Validity::AllValid;
1✔
269

270
        let list =
1✔
271
            ListArray::try_new(elements.into_array(), offsets.into_array(), validity).unwrap();
1✔
272

273
        assert_eq!(0, list.len());
1✔
274
    }
1✔
275

276
    #[test]
277
    fn test_simple_list_array() {
1✔
278
        let elements = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]);
1✔
279
        let offsets = PrimitiveArray::from_iter([0, 2, 4, 5]);
1✔
280
        let validity = Validity::AllValid;
1✔
281

282
        let list =
1✔
283
            ListArray::try_new(elements.into_array(), offsets.into_array(), validity).unwrap();
1✔
284

285
        assert_eq!(
1✔
286
            Scalar::list(
1✔
287
                Arc::new(I32.into()),
1✔
288
                vec![1.into(), 2.into()],
1✔
289
                Nullability::Nullable
1✔
290
            ),
291
            list.scalar_at(0).unwrap()
1✔
292
        );
293
        assert_eq!(
1✔
294
            Scalar::list(
1✔
295
                Arc::new(I32.into()),
1✔
296
                vec![3.into(), 4.into()],
1✔
297
                Nullability::Nullable
1✔
298
            ),
299
            list.scalar_at(1).unwrap()
1✔
300
        );
301
        assert_eq!(
1✔
302
            Scalar::list(Arc::new(I32.into()), vec![5.into()], Nullability::Nullable),
1✔
303
            list.scalar_at(2).unwrap()
1✔
304
        );
305
    }
1✔
306

307
    #[test]
308
    fn test_simple_list_array_from_iter() {
1✔
309
        let elements = PrimitiveArray::from_iter([1i32, 2, 3]);
1✔
310
        let offsets = PrimitiveArray::from_iter([0, 2, 3]);
1✔
311
        let validity = Validity::NonNullable;
1✔
312

313
        let list =
1✔
314
            ListArray::try_new(elements.into_array(), offsets.into_array(), validity).unwrap();
1✔
315

316
        let list_from_iter =
1✔
317
            ListArray::from_iter_slow::<u32, _>(vec![vec![1i32, 2], vec![3]], Arc::new(I32.into()))
1✔
318
                .unwrap();
1✔
319

320
        assert_eq!(list.len(), list_from_iter.len());
1✔
321
        assert_eq!(
1✔
322
            list.scalar_at(0).unwrap(),
1✔
323
            list_from_iter.scalar_at(0).unwrap()
1✔
324
        );
325
        assert_eq!(
1✔
326
            list.scalar_at(1).unwrap(),
1✔
327
            list_from_iter.scalar_at(1).unwrap()
1✔
328
        );
329
    }
1✔
330

331
    #[test]
332
    fn test_simple_list_filter() {
1✔
333
        let elements = PrimitiveArray::from_option_iter([None, Some(2), Some(3), Some(4), Some(5)]);
1✔
334
        let offsets = PrimitiveArray::from_iter([0, 2, 4, 5]);
1✔
335
        let validity = Validity::AllValid;
1✔
336

337
        let list = ListArray::try_new(elements.into_array(), offsets.into_array(), validity)
1✔
338
            .unwrap()
1✔
339
            .into_array();
1✔
340

341
        let filtered = filter(
1✔
342
            &list,
1✔
343
            &Mask::from(BooleanBuffer::from(vec![false, true, true])),
1✔
344
        );
345

346
        assert!(filtered.is_ok())
1✔
347
    }
1✔
348
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc