• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

vortex-data / vortex / 16204612549

10 Jul 2025 07:50PM UTC coverage: 81.152% (+2.9%) from 78.263%
16204612549

Pull #3825

github

web-flow
Merge d0d2717da into be9c2fd3e
Pull Request #3825: feat: Add optimize ArrayOp with VBView implementation

178 of 211 new or added lines in 4 files covered. (84.36%)

330 existing lines in 34 files now uncovered.

45433 of 55985 relevant lines covered (81.15%)

145951.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.0
/vortex-expr/src/exprs/pack.rs
1
// SPDX-License-Identifier: Apache-2.0
2
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3

4
use std::fmt::Display;
5
use std::hash::Hash;
6

7
use itertools::Itertools as _;
8
use vortex_array::arrays::StructArray;
9
use vortex_array::validity::Validity;
10
use vortex_array::{ArrayRef, DeserializeMetadata, IntoArray, ProstMetadata};
11
use vortex_dtype::{DType, FieldName, FieldNames, Nullability, StructFields};
12
use vortex_error::{VortexExpect as _, VortexResult, vortex_bail, vortex_err};
13
use vortex_proto::expr as pb;
14

15
use crate::{AnalysisExpr, ExprEncodingRef, ExprId, ExprRef, IntoExpr, Scope, VTable, vtable};
16

17
vtable!(Pack);
18

19
/// Pack zero or more expressions into a structure with named fields.
20
///
21
/// # Examples
22
///
23
/// ```
24
/// use vortex_array::{IntoArray, ToCanonical};
25
/// use vortex_buffer::buffer;
26
/// use vortex_expr::{root, PackExpr, Scope, VortexExpr};
27
/// use vortex_scalar::Scalar;
28
/// use vortex_dtype::Nullability;
29
///
30
/// let example = PackExpr::try_new(
31
///     ["x", "x copy", "second x copy"].into(),
32
///     vec![root(), root(), root()],
33
///     Nullability::NonNullable,
34
/// ).unwrap();
35
/// let packed = example.evaluate(&Scope::new(buffer![100, 110, 200].into_array())).unwrap();
36
/// let x_copy = packed
37
///     .to_struct()
38
///     .unwrap()
39
///     .field_by_name("x copy")
40
///     .unwrap()
41
///     .clone();
42
/// assert_eq!(x_copy.scalar_at(0).unwrap(), Scalar::from(100));
43
/// assert_eq!(x_copy.scalar_at(1).unwrap(), Scalar::from(110));
44
/// assert_eq!(x_copy.scalar_at(2).unwrap(), Scalar::from(200));
45
/// ```
46
///
47
#[allow(clippy::derived_hash_with_manual_eq)]
48
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
49
pub struct PackExpr {
50
    names: FieldNames,
51
    values: Vec<ExprRef>,
52
    nullability: Nullability,
53
}
54

55
pub struct PackExprEncoding;
56

57
impl VTable for PackVTable {
58
    type Expr = PackExpr;
59
    type Encoding = PackExprEncoding;
60
    type Metadata = ProstMetadata<pb::PackOpts>;
61

62
    fn id(_encoding: &Self::Encoding) -> ExprId {
123✔
63
        ExprId::new_ref("pack")
123✔
64
    }
123✔
65

UNCOV
66
    fn encoding(_expr: &Self::Expr) -> ExprEncodingRef {
×
UNCOV
67
        ExprEncodingRef::new_ref(PackExprEncoding.as_ref())
×
68
    }
×
69

70
    fn metadata(expr: &Self::Expr) -> Option<Self::Metadata> {
×
UNCOV
71
        Some(ProstMetadata(pb::PackOpts {
×
72
            paths: expr.names.iter().map(|n| n.to_string()).collect(),
×
73
            nullable: expr.nullability.into(),
×
74
        }))
×
75
    }
×
76

77
    fn children(expr: &Self::Expr) -> Vec<&ExprRef> {
152,872✔
78
        expr.values.iter().collect()
152,872✔
79
    }
152,872✔
80

81
    fn with_children(expr: &Self::Expr, children: Vec<ExprRef>) -> VortexResult<Self::Expr> {
5,972✔
82
        PackExpr::try_new(expr.names.clone(), children, expr.nullability)
5,972✔
83
    }
5,972✔
84

UNCOV
85
    fn build(
×
UNCOV
86
        _encoding: &Self::Encoding,
×
87
        metadata: &<Self::Metadata as DeserializeMetadata>::Output,
×
88
        children: Vec<ExprRef>,
×
89
    ) -> VortexResult<Self::Expr> {
×
90
        if children.len() != metadata.paths.len() {
×
91
            vortex_bail!(
×
92
                "Pack expression expects {} children, got {}",
×
93
                metadata.paths.len(),
×
94
                children.len()
×
95
            );
×
96
        }
×
97
        let names: FieldNames = metadata
×
98
            .paths
×
99
            .iter()
×
100
            .map(|name| FieldName::from(name.as_str()))
×
101
            .collect();
×
102
        PackExpr::try_new(names, children, metadata.nullable.into())
×
103
    }
×
104

105
    fn evaluate(expr: &Self::Expr, scope: &Scope) -> VortexResult<ArrayRef> {
7,839✔
106
        let len = scope.len();
7,839✔
107
        let value_arrays = expr
7,839✔
108
            .values
7,839✔
109
            .iter()
7,839✔
110
            .map(|value_expr| value_expr.unchecked_evaluate(scope))
11,295✔
111
            .process_results(|it| it.collect::<Vec<_>>())?;
7,839✔
112
        let validity = match expr.nullability {
7,839✔
113
            Nullability::NonNullable => Validity::NonNullable,
7,838✔
114
            Nullability::Nullable => Validity::AllValid,
1✔
115
        };
116
        Ok(StructArray::try_new(expr.names.clone(), value_arrays, len, validity)?.into_array())
7,839✔
117
    }
7,839✔
118

119
    fn return_dtype(expr: &Self::Expr, scope: &DType) -> VortexResult<DType> {
19,640✔
120
        let value_dtypes = expr
19,640✔
121
            .values
19,640✔
122
            .iter()
19,640✔
123
            .map(|value_expr| value_expr.return_dtype(scope))
24,434✔
124
            .process_results(|it| it.collect())?;
19,640✔
125
        Ok(DType::Struct(
19,640✔
126
            StructFields::new(expr.names.clone(), value_dtypes),
19,640✔
127
            expr.nullability,
19,640✔
128
        ))
19,640✔
129
    }
19,640✔
130
}
131

132
impl PackExpr {
133
    pub fn try_new(
14,468✔
134
        names: FieldNames,
14,468✔
135
        values: Vec<ExprRef>,
14,468✔
136
        nullability: Nullability,
14,468✔
137
    ) -> VortexResult<Self> {
14,468✔
138
        if names.len() != values.len() {
14,468✔
UNCOV
139
            vortex_bail!("length mismatch {} {}", names.len(), values.len());
×
140
        }
14,468✔
141
        Ok(PackExpr {
14,468✔
142
            names,
14,468✔
143
            values,
14,468✔
144
            nullability,
14,468✔
145
        })
14,468✔
146
    }
14,468✔
147

UNCOV
148
    pub fn names(&self) -> &FieldNames {
×
UNCOV
149
        &self.names
×
150
    }
×
151

152
    pub fn field(&self, field_name: &FieldName) -> VortexResult<ExprRef> {
3,263✔
153
        let idx = self
3,263✔
154
            .names
3,263✔
155
            .iter()
3,263✔
156
            .position(|name| name == field_name)
11,926✔
157
            .ok_or_else(|| {
3,263✔
UNCOV
158
                vortex_err!(
×
UNCOV
159
                    "Cannot find field {} in pack fields {:?}",
×
160
                    field_name,
×
161
                    self.names
×
162
                )
×
163
            })?;
3,263✔
164

165
        self.values
3,263✔
166
            .get(idx)
3,263✔
167
            .cloned()
3,263✔
168
            .ok_or_else(|| vortex_err!("field index out of bounds: {}", idx))
3,263✔
169
    }
3,263✔
170

UNCOV
171
    pub fn nullability(&self) -> Nullability {
×
UNCOV
172
        self.nullability
×
173
    }
×
174
}
175

176
pub fn pack(
7,972✔
177
    elements: impl IntoIterator<Item = (impl Into<FieldName>, ExprRef)>,
7,972✔
178
    nullability: Nullability,
7,972✔
179
) -> ExprRef {
7,972✔
180
    let (names, values): (Vec<_>, Vec<_>) = elements
7,972✔
181
        .into_iter()
7,972✔
182
        .map(|(name, value)| (name.into(), value))
12,879✔
183
        .unzip();
7,972✔
184
    PackExpr::try_new(names.into(), values, nullability)
7,972✔
185
        .vortex_expect("pack names and values have the same length")
7,972✔
186
        .into_expr()
7,972✔
187
}
7,972✔
188

189
impl Display for PackExpr {
190
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1✔
191
        write!(
1✔
192
            f,
1✔
193
            "pack({}){}",
1✔
194
            self.names
1✔
195
                .iter()
1✔
196
                .zip(&self.values)
1✔
197
                .format_with(", ", |(name, expr), f| f(&format_args!("{name}: {expr}"))),
2✔
198
            self.nullability
1✔
199
        )
1✔
200
    }
1✔
201
}
202

203
impl AnalysisExpr for PackExpr {}
204

205
#[cfg(test)]
206
mod tests {
207

208
    use vortex_array::arrays::{PrimitiveArray, StructArray};
209
    use vortex_array::validity::Validity;
210
    use vortex_array::vtable::ValidityHelper;
211
    use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
212
    use vortex_buffer::buffer;
213
    use vortex_dtype::{FieldNames, Nullability};
214
    use vortex_error::{VortexResult, vortex_bail};
215

216
    use crate::{IntoExpr, PackExpr, Scope, col};
217

218
    fn test_array() -> ArrayRef {
4✔
219
        StructArray::from_fields(&[
4✔
220
            ("a", buffer![0, 1, 2].into_array()),
4✔
221
            ("b", buffer![4, 5, 6].into_array()),
4✔
222
        ])
4✔
223
        .unwrap()
4✔
224
        .into_array()
4✔
225
    }
4✔
226

227
    fn primitive_field(array: &dyn Array, field_path: &[&str]) -> VortexResult<PrimitiveArray> {
7✔
228
        let mut field_path = field_path.iter();
7✔
229

230
        let Some(field) = field_path.next() else {
7✔
231
            vortex_bail!("empty field path");
232
        };
233

234
        let mut array = array.to_struct()?.field_by_name(field)?.clone();
7✔
235
        for field in field_path {
9✔
236
            array = array.to_struct()?.field_by_name(field)?.clone();
2✔
237
        }
238
        Ok(array.to_primitive().unwrap())
7✔
239
    }
7✔
240

241
    #[test]
242
    pub fn test_empty_pack() {
1✔
243
        let expr =
1✔
244
            PackExpr::try_new(FieldNames::default(), Vec::new(), Nullability::NonNullable).unwrap();
1✔
245

1✔
246
        let test_array = test_array();
1✔
247
        let actual_array = expr.evaluate(&Scope::new(test_array.clone())).unwrap();
1✔
248
        assert_eq!(actual_array.len(), test_array.len());
1✔
249
        assert_eq!(
1✔
250
            actual_array.to_struct().unwrap().struct_fields().nfields(),
1✔
251
            0
1✔
252
        );
1✔
253
    }
1✔
254

255
    #[test]
256
    pub fn test_simple_pack() {
1✔
257
        let expr = PackExpr::try_new(
1✔
258
            ["one", "two", "three"].into(),
1✔
259
            vec![col("a"), col("b"), col("a")],
1✔
260
            Nullability::NonNullable,
1✔
261
        )
1✔
262
        .unwrap();
1✔
263

1✔
264
        let actual_array = expr
1✔
265
            .evaluate(&Scope::new(test_array()))
1✔
266
            .unwrap()
1✔
267
            .to_struct()
1✔
268
            .unwrap();
1✔
269
        let expected_names: FieldNames = ["one", "two", "three"].into();
1✔
270
        assert_eq!(actual_array.names(), &expected_names);
1✔
271
        assert_eq!(actual_array.validity(), &Validity::NonNullable);
1✔
272

273
        assert_eq!(
1✔
274
            primitive_field(actual_array.as_ref(), &["one"])
1✔
275
                .unwrap()
1✔
276
                .as_slice::<i32>(),
1✔
277
            [0, 1, 2]
1✔
278
        );
1✔
279
        assert_eq!(
1✔
280
            primitive_field(actual_array.as_ref(), &["two"])
1✔
281
                .unwrap()
1✔
282
                .as_slice::<i32>(),
1✔
283
            [4, 5, 6]
1✔
284
        );
1✔
285
        assert_eq!(
1✔
286
            primitive_field(actual_array.as_ref(), &["three"])
1✔
287
                .unwrap()
1✔
288
                .as_slice::<i32>(),
1✔
289
            [0, 1, 2]
1✔
290
        );
1✔
291
    }
1✔
292

293
    #[test]
294
    pub fn test_nested_pack() {
1✔
295
        let expr = PackExpr::try_new(
1✔
296
            ["one", "two", "three"].into(),
1✔
297
            vec![
1✔
298
                col("a"),
1✔
299
                PackExpr::try_new(
1✔
300
                    ["two_one", "two_two"].into(),
1✔
301
                    vec![col("b"), col("b")],
1✔
302
                    Nullability::NonNullable,
1✔
303
                )
1✔
304
                .unwrap()
1✔
305
                .into_expr(),
1✔
306
                col("a"),
1✔
307
            ],
1✔
308
            Nullability::NonNullable,
1✔
309
        )
1✔
310
        .unwrap();
1✔
311

1✔
312
        let actual_array = expr
1✔
313
            .evaluate(&Scope::new(test_array()))
1✔
314
            .unwrap()
1✔
315
            .to_struct()
1✔
316
            .unwrap();
1✔
317
        let expected_names = FieldNames::from(["one", "two", "three"]);
1✔
318
        assert_eq!(actual_array.names(), &expected_names);
1✔
319

320
        assert_eq!(
1✔
321
            primitive_field(actual_array.as_ref(), &["one"])
1✔
322
                .unwrap()
1✔
323
                .as_slice::<i32>(),
1✔
324
            [0, 1, 2]
1✔
325
        );
1✔
326
        assert_eq!(
1✔
327
            primitive_field(actual_array.as_ref(), &["two", "two_one"])
1✔
328
                .unwrap()
1✔
329
                .as_slice::<i32>(),
1✔
330
            [4, 5, 6]
1✔
331
        );
1✔
332
        assert_eq!(
1✔
333
            primitive_field(actual_array.as_ref(), &["two", "two_two"])
1✔
334
                .unwrap()
1✔
335
                .as_slice::<i32>(),
1✔
336
            [4, 5, 6]
1✔
337
        );
1✔
338
        assert_eq!(
1✔
339
            primitive_field(actual_array.as_ref(), &["three"])
1✔
340
                .unwrap()
1✔
341
                .as_slice::<i32>(),
1✔
342
            [0, 1, 2]
1✔
343
        );
1✔
344
    }
1✔
345

346
    #[test]
347
    pub fn test_pack_nullable() {
1✔
348
        let expr = PackExpr::try_new(
1✔
349
            ["one", "two", "three"].into(),
1✔
350
            vec![col("a"), col("b"), col("a")],
1✔
351
            Nullability::Nullable,
1✔
352
        )
1✔
353
        .unwrap();
1✔
354

1✔
355
        let actual_array = expr
1✔
356
            .evaluate(&Scope::new(test_array()))
1✔
357
            .unwrap()
1✔
358
            .to_struct()
1✔
359
            .unwrap();
1✔
360
        let expected_names: FieldNames = ["one", "two", "three"].into();
1✔
361
        assert_eq!(actual_array.names(), &expected_names);
1✔
362
        assert_eq!(actual_array.validity(), &Validity::AllValid);
1✔
363
    }
1✔
364
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc