16655250593

Committed 31 Jul 2025 05:01PM UTC coverage: 83.008% (-0.02%) from 83.029%

Build # 16655250593

Build Type

push

github

Committed by

web-flow

Commit Message

fix: Add select-expr deserialization (#4082)

Apparantly I got too excited by fancy pattern matching in #4077 to
remember I also need to deserialize expressions

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Run Details

0 of 17 new or added lines in 1 file covered. (0.0%)

1 existing line in 1 file now uncovered.

46014 of 55433 relevant lines covered (83.01%)

452615.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.49

/vortex-expr/src/exprs/select.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::fmt::Display;

use itertools::Itertools;
use vortex_array::{ArrayRef, DeserializeMetadata, IntoArray, ProstMetadata, ToCanonical};
use vortex_dtype::{DType, FieldNames};
use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_err};
use vortex_proto::expr::select_opts::Opts;
use vortex_proto::expr::{FieldNames as ProtoFieldNames, SelectOpts};

use crate::field::DisplayFieldNames;
use crate::{AnalysisExpr, ExprEncodingRef, ExprId, ExprRef, IntoExpr, Scope, VTable, vtable};

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum SelectField {
    Include(FieldNames),
    Exclude(FieldNames),
}

vtable!(Select);

#[derive(Debug, Clone, Hash)]
#[allow(clippy::derived_hash_with_manual_eq)]
pub struct SelectExpr {
    fields: SelectField,
    child: ExprRef,
}

impl PartialEq for SelectExpr {
    fn eq(&self, other: &Self) -> bool {
        self.fields == other.fields && self.child.eq(&other.child)
    }
}

pub struct SelectExprEncoding;

impl VTable for SelectVTable {
    type Expr = SelectExpr;
    type Encoding = SelectExprEncoding;
    type Metadata = ProstMetadata<SelectOpts>;

    fn id(_encoding: &Self::Encoding) -> ExprId {
        ExprId::new_ref("select")
    }

    fn encoding(_expr: &Self::Expr) -> ExprEncodingRef {
        ExprEncodingRef::new_ref(SelectExprEncoding.as_ref())
    }

    fn metadata(expr: &Self::Expr) -> Option<Self::Metadata> {
        let names = expr
            .fields()
            .fields()
            .iter()
            .map(|f| f.to_string())
            .collect_vec();

        let opts = if expr.fields().is_include() {
            Opts::Include(ProtoFieldNames { names })
        } else {
            Opts::Exclude(ProtoFieldNames { names })
        };

        Some(ProstMetadata(SelectOpts { opts: Some(opts) }))
    }

    fn children(expr: &Self::Expr) -> Vec<&ExprRef> {
        vec![&expr.child]
    }

    fn with_children(expr: &Self::Expr, children: Vec<ExprRef>) -> VortexResult<Self::Expr> {
        Ok(SelectExpr {
            fields: expr.fields.clone(),
            child: children[0].clone(),
        })
    }

    fn build(
        _encoding: &Self::Encoding,
        metadata: &<Self::Metadata as DeserializeMetadata>::Output,
        mut children: Vec<ExprRef>,
    ) -> VortexResult<Self::Expr> {
        if children.len() != 1 {
            vortex_bail!("Select expression must have exactly one child");
        }

        let fields = match metadata.opts.as_ref() {
            Some(opts) => match opts {
                Opts::Include(field_names) => SelectField::Include(FieldNames::from_iter(
                    field_names.names.iter().map(|s| s.as_str()),
                )),
                Opts::Exclude(field_names) => SelectField::Exclude(FieldNames::from_iter(
                    field_names.names.iter().map(|s| s.as_str()),
                )),
            },
            None => {
                vortex_bail!("Select expressions must be provided with fields to select or exclude")
            }
        };

        let child = children
            .drain(..)
            .next()
            .vortex_expect("number of children validated to be one");

        Ok(SelectExpr { fields, child })
    }

    fn evaluate(expr: &Self::Expr, scope: &Scope) -> VortexResult<ArrayRef> {
        let batch = expr.child.unchecked_evaluate(scope)?.to_struct()?;
        Ok(match &expr.fields {
            SelectField::Include(f) => batch.project(f.as_ref()),
            SelectField::Exclude(names) => {
                let included_names = batch
                    .names()
                    .iter()
                    .filter(|&f| !names.as_ref().contains(f))
                    .cloned()
                    .collect::<Vec<_>>();
                batch.project(included_names.as_slice())
            }
        }?
        .into_array())
    }

    fn return_dtype(expr: &Self::Expr, scope: &DType) -> VortexResult<DType> {
        let child_dtype = expr.child.return_dtype(scope)?;
        let child_struct_dtype = child_dtype
            .as_struct()
            .ok_or_else(|| vortex_err!("Select child not a struct dtype"))?;

        let projected = match &expr.fields {
            SelectField::Include(fields) => child_struct_dtype.project(fields.as_ref())?,
            SelectField::Exclude(fields) => child_struct_dtype
                .names()
                .iter()
                .cloned()
                .zip_eq(child_struct_dtype.fields())
                .filter(|(name, _)| !fields.as_ref().contains(name))
                .collect(),
        };

        Ok(DType::Struct(projected, child_dtype.nullability()))
    }
}

pub fn select(fields: impl Into<FieldNames>, child: ExprRef) -> ExprRef {
    SelectExpr::include_expr(fields.into(), child)
}

pub fn select_exclude(fields: impl Into<FieldNames>, child: ExprRef) -> ExprRef {
    SelectExpr::exclude_expr(fields.into(), child)
}

impl SelectExpr {
    pub fn new(fields: SelectField, child: ExprRef) -> Self {
        Self { fields, child }
    }

    pub fn new_expr(fields: SelectField, child: ExprRef) -> ExprRef {
        Self::new(fields, child).into_expr()
    }

    pub fn include_expr(columns: FieldNames, child: ExprRef) -> ExprRef {
        Self::new(SelectField::Include(columns), child).into_expr()
    }

    pub fn exclude_expr(columns: FieldNames, child: ExprRef) -> ExprRef {
        Self::new(SelectField::Exclude(columns), child).into_expr()
    }

    pub fn fields(&self) -> &SelectField {
        &self.fields
    }

    pub fn child(&self) -> &ExprRef {
        &self.child
    }

    pub fn as_include(&self, field_names: &FieldNames) -> VortexResult<ExprRef> {
        Ok(Self::new(
            SelectField::Include(self.fields.as_include_names(field_names)?),
            self.child.clone(),
        )
        .into_expr())
    }
}

impl SelectField {
    pub fn include(columns: FieldNames) -> Self {
        assert_eq!(columns.iter().unique().collect_vec().len(), columns.len());
        Self::Include(columns)
    }

    pub fn exclude(columns: FieldNames) -> Self {
        assert_eq!(columns.iter().unique().collect_vec().len(), columns.len());
        Self::Exclude(columns)
    }

    pub fn is_include(&self) -> bool {
        matches!(self, Self::Include(_))
    }

    pub fn is_exclude(&self) -> bool {
        matches!(self, Self::Exclude(_))
    }

    pub fn fields(&self) -> &FieldNames {
        let (SelectField::Include(fields) | SelectField::Exclude(fields)) = self;

        fields
    }

    pub fn as_include_names(&self, field_names: &FieldNames) -> VortexResult<FieldNames> {
        if self
            .fields()
            .iter()
            .any(|f| !field_names.iter().contains(f))
        {
            vortex_bail!(
                "Field {:?} in select not in field names {:?}",
                self,
                field_names
            );
        }
        match self {
            SelectField::Include(fields) => Ok(fields.clone()),
            SelectField::Exclude(exc_fields) => Ok(field_names
                .iter()
                .filter(|f| exc_fields.iter().contains(f))
                .cloned()
                .collect()),
        }
    }
}

impl Display for SelectField {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            SelectField::Include(fields) => write!(f, "{{{}}}", DisplayFieldNames(fields)),
            SelectField::Exclude(fields) => write!(f, "~{{{}}}", DisplayFieldNames(fields)),
        }
    }
}

impl Display for SelectExpr {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}{}", self.child, self.fields)
    }
}

impl AnalysisExpr for SelectExpr {}

#[cfg(test)]
mod tests {

    use vortex_array::arrays::StructArray;
    use vortex_array::{IntoArray, ToCanonical};
    use vortex_buffer::buffer;
    use vortex_dtype::{DType, FieldName, Nullability};

    use crate::{Scope, root, select, select_exclude, test_harness};

    fn test_array() -> StructArray {
        StructArray::from_fields(&[
            ("a", buffer![0, 1, 2].into_array()),
            ("b", buffer![4, 5, 6].into_array()),
        ])
        .unwrap()
    }

    #[test]
    pub fn include_columns() {
        let st = test_array();
        let select = select(vec![FieldName::from("a")], root());
        let selected = select
            .evaluate(&Scope::new(st.to_array()))
            .unwrap()
            .to_struct()
            .unwrap();
        let selected_names = selected.names().clone();
        assert_eq!(selected_names.as_ref(), &["a".into()]);
    }

    #[test]
    pub fn exclude_columns() {
        let st = test_array();
        let select = select_exclude(vec![FieldName::from("a")], root());
        let selected = select
            .evaluate(&Scope::new(st.to_array()))
            .unwrap()
            .to_struct()
            .unwrap();
        let selected_names = selected.names().clone();
        assert_eq!(selected_names.as_ref(), &["b".into()]);
    }

    #[test]
    fn dtype() {
        let dtype = test_harness::struct_dtype();

        let select_expr = select(vec![FieldName::from("a")], root());
        let expected_dtype = DType::Struct(
            dtype.as_struct().unwrap().project(&["a".into()]).unwrap(),
            Nullability::NonNullable,
        );
        assert_eq!(select_expr.return_dtype(&dtype).unwrap(), expected_dtype);

        let select_expr_exclude = select_exclude(
            vec![
                FieldName::from("col1"),
                FieldName::from("col2"),
                FieldName::from("bool1"),
                FieldName::from("bool2"),
            ],
            root(),
        );
        assert_eq!(
            select_expr_exclude.return_dtype(&dtype).unwrap(),
            expected_dtype
        );

        let select_expr_exclude = select_exclude(
            vec![FieldName::from("col1"), FieldName::from("col2")],
            root(),
        );
        assert_eq!(
            select_expr_exclude.return_dtype(&dtype).unwrap(),
            DType::Struct(
                dtype
                    .as_struct()
                    .unwrap()
                    .project(&["a".into(), "bool1".into(), "bool2".into()])
                    .unwrap(),
                Nullability::NonNullable
            )
        );
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::fmt::Display;
5
6	use itertools::Itertools;
7	use vortex_array::{ArrayRef, DeserializeMetadata, IntoArray, ProstMetadata, ToCanonical};
8	use vortex_dtype::{DType, FieldNames};
9	use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_err};
10	use vortex_proto::expr::select_opts::Opts;
11	use vortex_proto::expr::{FieldNames as ProtoFieldNames, SelectOpts};
12
13	use crate::field::DisplayFieldNames;
14	use crate::{AnalysisExpr, ExprEncodingRef, ExprId, ExprRef, IntoExpr, Scope, VTable, vtable};
15
16	#[derive(Debug, Clone, PartialEq, Eq, Hash)]
17	pub enum SelectField {
18	Include(FieldNames),
19	Exclude(FieldNames),
20	}
21
22	vtable!(Select);
23
24	#[derive(Debug, Clone, Hash)]
25	#[allow(clippy::derived_hash_with_manual_eq)]
26	pub struct SelectExpr {
27	fields: SelectField,
28	child: ExprRef,
29	}
30
31	impl PartialEq for SelectExpr {
32	fn eq(&self, other: &Self) -> bool {	×
33	self.fields == other.fields && self.child.eq(&other.child)	×
34	}	×
35	}
36
37	pub struct SelectExprEncoding;
38
39	impl VTable for SelectVTable {
40	type Expr = SelectExpr;
41	type Encoding = SelectExprEncoding;
42	type Metadata = ProstMetadata<SelectOpts>;
43
44	fn id(_encoding: &Self::Encoding) -> ExprId {	128✔
45	ExprId::new_ref("select")	128✔
46	}	128✔
47
48	fn encoding(_expr: &Self::Expr) -> ExprEncodingRef {	×
49	ExprEncodingRef::new_ref(SelectExprEncoding.as_ref())	×
50	}	×
51
52	fn metadata(expr: &Self::Expr) -> Option<Self::Metadata> {	×
53	let names = expr	×
54	.fields()	×
55	.fields()	×
56	.iter()	×
57	.map(\|f\| f.to_string())	×
58	.collect_vec();	×
59
60	let opts = if expr.fields().is_include() {	×
61	Opts::Include(ProtoFieldNames { names })	×
62	} else {
63	Opts::Exclude(ProtoFieldNames { names })	×
64	};
65
66	Some(ProstMetadata(SelectOpts { opts: Some(opts) }))	×
67	}	×
68
69	fn children(expr: &Self::Expr) -> Vec<&ExprRef> {	3,037✔
70	vec![&expr.child]	3,037✔
71	}	3,037✔
72
73	fn with_children(expr: &Self::Expr, children: Vec<ExprRef>) -> VortexResult<Self::Expr> {	×
74	Ok(SelectExpr {	×
75	fields: expr.fields.clone(),	×
76	child: children[0].clone(),	×
77	})	×
78	}	×
79
80	fn build(	×
81	_encoding: &Self::Encoding,	×
NEW 82	metadata: &<Self::Metadata as DeserializeMetadata>::Output,	×
NEW 83	mut children: Vec<ExprRef>,	×
84	) -> VortexResult<Self::Expr> {	×
NEW 85	if children.len() != 1 {	×
NEW 86	vortex_bail!("Select expression must have exactly one child");	×
NEW 87	}	×
88
NEW 89	let fields = match metadata.opts.as_ref() {	×
NEW 90	Some(opts) => match opts {	×
NEW 91	Opts::Include(field_names) => SelectField::Include(FieldNames::from_iter(	×
NEW 92	field_names.names.iter().map(\|s\| s.as_str()),	×
93	)),
NEW 94	Opts::Exclude(field_names) => SelectField::Exclude(FieldNames::from_iter(	×
NEW 95	field_names.names.iter().map(\|s\| s.as_str()),	×
96	)),
97	},
98	None => {
NEW 99	vortex_bail!("Select expressions must be provided with fields to select or exclude")	×
100	}
101	};
102
NEW 103	let child = children	×
NEW 104	.drain(..)	×
NEW 105	.next()	×
NEW 106	.vortex_expect("number of children validated to be one");	×
107
NEW 108	Ok(SelectExpr { fields, child })	×
UNCOV 109	}	×
110
111	fn evaluate(expr: &Self::Expr, scope: &Scope) -> VortexResult<ArrayRef> {	2✔
112	let batch = expr.child.unchecked_evaluate(scope)?.to_struct()?;	2✔
113	Ok(match &expr.fields {	2✔
114	SelectField::Include(f) => batch.project(f.as_ref()),	1✔
115	SelectField::Exclude(names) => {	1✔
116	let included_names = batch	1✔
117	.names()	1✔
118	.iter()	1✔
119	.filter(\|&f\| !names.as_ref().contains(f))	2✔
120	.cloned()	1✔
121	.collect::<Vec<_>>();	1✔
122	batch.project(included_names.as_slice())	1✔
123	}
124	}?	×
125	.into_array())	2✔
126	}	2✔
127
128	fn return_dtype(expr: &Self::Expr, scope: &DType) -> VortexResult<DType> {	68✔
129	let child_dtype = expr.child.return_dtype(scope)?;	68✔
130	let child_struct_dtype = child_dtype	68✔
131	.as_struct()	68✔
132	.ok_or_else(\|\| vortex_err!("Select child not a struct dtype"))?;	68✔
133
134	let projected = match &expr.fields {	68✔
135	SelectField::Include(fields) => child_struct_dtype.project(fields.as_ref())?,	65✔
136	SelectField::Exclude(fields) => child_struct_dtype	3✔
137	.names()	3✔
138	.iter()	3✔
139	.cloned()	3✔
140	.zip_eq(child_struct_dtype.fields())	3✔
141	.filter(\|(name, _)\| !fields.as_ref().contains(name))	12✔
142	.collect(),	3✔
143	};
144
145	Ok(DType::Struct(projected, child_dtype.nullability()))	68✔
146	}	68✔
147	}
148
149	pub fn select(fields: impl Into<FieldNames>, child: ExprRef) -> ExprRef {	676✔
150	SelectExpr::include_expr(fields.into(), child)	676✔
151	}	676✔
152
153	pub fn select_exclude(fields: impl Into<FieldNames>, child: ExprRef) -> ExprRef {	4✔
154	SelectExpr::exclude_expr(fields.into(), child)	4✔
155	}	4✔
156
157	impl SelectExpr {
158	pub fn new(fields: SelectField, child: ExprRef) -> Self {	1,000✔
159	Self { fields, child }	1,000✔
160	}	1,000✔
161
162	pub fn new_expr(fields: SelectField, child: ExprRef) -> ExprRef {	×
163	Self::new(fields, child).into_expr()	×
164	}	×
165
166	pub fn include_expr(columns: FieldNames, child: ExprRef) -> ExprRef {	996✔
167	Self::new(SelectField::Include(columns), child).into_expr()	996✔
168	}	996✔
169
170	pub fn exclude_expr(columns: FieldNames, child: ExprRef) -> ExprRef {	4✔
171	Self::new(SelectField::Exclude(columns), child).into_expr()	4✔
172	}	4✔
173
174	pub fn fields(&self) -> &SelectField {	1,013✔
175	&self.fields	1,013✔
176	}	1,013✔
177
178	pub fn child(&self) -> &ExprRef {	1,013✔
179	&self.child	1,013✔
180	}	1,013✔
181
182	pub fn as_include(&self, field_names: &FieldNames) -> VortexResult<ExprRef> {	×
183	Ok(Self::new(	×
184	SelectField::Include(self.fields.as_include_names(field_names)?),	×
185	self.child.clone(),	×
186	)
187	.into_expr())	×
188	}	×
189	}
190
191	impl SelectField {
192	pub fn include(columns: FieldNames) -> Self {	×
193	assert_eq!(columns.iter().unique().collect_vec().len(), columns.len());	×
194	Self::Include(columns)	×
195	}	×
196
197	pub fn exclude(columns: FieldNames) -> Self {	×
198	assert_eq!(columns.iter().unique().collect_vec().len(), columns.len());	×
199	Self::Exclude(columns)	×
200	}	×
201
202	pub fn is_include(&self) -> bool {	×
203	matches!(self, Self::Include(_))	×
204	}	×
205
206	pub fn is_exclude(&self) -> bool {	×
207	matches!(self, Self::Exclude(_))	×
208	}	×
209
210	pub fn fields(&self) -> &FieldNames {	1,013✔
211	let (SelectField::Include(fields) \| SelectField::Exclude(fields)) = self;	1,013✔
212
213	fields	1,013✔
214	}	1,013✔
215
216	pub fn as_include_names(&self, field_names: &FieldNames) -> VortexResult<FieldNames> {	1,013✔
217	if self	1,013✔
218	.fields()	1,013✔
219	.iter()	1,013✔
220	.any(\|f\| !field_names.iter().contains(f))	2,248✔
221	{
222	vortex_bail!(	×
223	"Field {:?} in select not in field names {:?}",	×
224	self,
225	field_names
226	);
227	}	1,013✔
228	match self {	1,013✔
229	SelectField::Include(fields) => Ok(fields.clone()),	1,013✔
230	SelectField::Exclude(exc_fields) => Ok(field_names	×
231	.iter()	×
232	.filter(\|f\| exc_fields.iter().contains(f))	×
233	.cloned()	×
234	.collect()),	×
235	}
236	}	1,013✔
237	}
238
239	impl Display for SelectField {
240	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {	3✔
241	match self {	3✔
242	SelectField::Include(fields) => write!(f, "{{{}}}", DisplayFieldNames(fields)),	2✔
243	SelectField::Exclude(fields) => write!(f, "~{{{}}}", DisplayFieldNames(fields)),	1✔
244	}
245	}	3✔
246	}
247
248	impl Display for SelectExpr {
249	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {	3✔
250	write!(f, "{}{}", self.child, self.fields)	3✔
251	}	3✔
252	}
253
254	impl AnalysisExpr for SelectExpr {}
255
256	#[cfg(test)]
257	mod tests {
258
259	use vortex_array::arrays::StructArray;
260	use vortex_array::{IntoArray, ToCanonical};
261	use vortex_buffer::buffer;
262	use vortex_dtype::{DType, FieldName, Nullability};
263
264	use crate::{Scope, root, select, select_exclude, test_harness};
265
266	fn test_array() -> StructArray {	2✔
267	StructArray::from_fields(&[	2✔
268	("a", buffer![0, 1, 2].into_array()),	2✔
269	("b", buffer![4, 5, 6].into_array()),	2✔
270	])	2✔
271	.unwrap()	2✔
272	}	2✔
273
274	#[test]
275	pub fn include_columns() {	1✔
276	let st = test_array();	1✔
277	let select = select(vec![FieldName::from("a")], root());	1✔
278	let selected = select	1✔
279	.evaluate(&Scope::new(st.to_array()))	1✔
280	.unwrap()	1✔
281	.to_struct()	1✔
282	.unwrap();	1✔
283	let selected_names = selected.names().clone();	1✔
284	assert_eq!(selected_names.as_ref(), &["a".into()]);	1✔
285	}	1✔
286
287	#[test]
288	pub fn exclude_columns() {	1✔
289	let st = test_array();	1✔
290	let select = select_exclude(vec![FieldName::from("a")], root());	1✔
291	let selected = select	1✔
292	.evaluate(&Scope::new(st.to_array()))	1✔
293	.unwrap()	1✔
294	.to_struct()	1✔
295	.unwrap();	1✔
296	let selected_names = selected.names().clone();	1✔
297	assert_eq!(selected_names.as_ref(), &["b".into()]);	1✔
298	}	1✔
299
300	#[test]
301	fn dtype() {	1✔
302	let dtype = test_harness::struct_dtype();	1✔
303
304	let select_expr = select(vec![FieldName::from("a")], root());	1✔
305	let expected_dtype = DType::Struct(	1✔
306	dtype.as_struct().unwrap().project(&["a".into()]).unwrap(),	1✔
307	Nullability::NonNullable,	1✔
308	);	1✔
309	assert_eq!(select_expr.return_dtype(&dtype).unwrap(), expected_dtype);	1✔
310
311	let select_expr_exclude = select_exclude(	1✔
312	vec![	1✔
313	FieldName::from("col1"),	1✔
314	FieldName::from("col2"),	1✔
315	FieldName::from("bool1"),	1✔
316	FieldName::from("bool2"),	1✔
317	],
318	root(),	1✔
319	);
320	assert_eq!(	1✔
321	select_expr_exclude.return_dtype(&dtype).unwrap(),	1✔
322	expected_dtype
323	);
324
325	let select_expr_exclude = select_exclude(	1✔
326	vec![FieldName::from("col1"), FieldName::from("col2")],	1✔
327	root(),	1✔
328	);
329	assert_eq!(	1✔
330	select_expr_exclude.return_dtype(&dtype).unwrap(),	1✔
331	DType::Struct(	1✔
332	dtype	1✔
333	.as_struct()	1✔
334	.unwrap()	1✔
335	.project(&["a".into(), "bool1".into(), "bool2".into()])	1✔
336	.unwrap(),	1✔
337	Nullability::NonNullable	1✔
338	)	1✔
339	);
340	}	1✔
341	}

vortex-data / vortex / 16655250593

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous