16139349253

Committed 08 Jul 2025 09:26AM UTC coverage: 78.057% (-0.2%) from 78.253%

Build # 16139349253

Build Type

push

github

Committed by

web-flow

Commit Message

VortexExpr VTables (#3713)

Adds the same vtable machinery as arrays and layouts already use. It
uses the "Encoding" naming scheme from arrays and layouts. I don't
particularly like it, but it's consistent. Open to renames later.

Further, adds an expression registry to the Vortex session that will be
used for deserialization.

Expressions only decide their "options" serialization. So in theory, can
support many container formats, not just proto, provided each expression
can deserialize their own options format.

---------

Signed-off-by: Nicholas Gates <nick@nickgates.com>

Run Details

800 of 1190 new or added lines in 38 files covered. (67.23%)

40 existing lines in 13 files now uncovered.

44100 of 56497 relevant lines covered (78.06%)

54989.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.91

/vortex-expr/src/exprs/var.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::fmt::Display;

use vortex_array::stats::Stat;
use vortex_array::{ArrayRef, DeserializeMetadata, ProstMetadata};
use vortex_dtype::{DType, FieldPath};
use vortex_error::{VortexResult, vortex_bail, vortex_err};
use vortex_proto::expr as pb;

use crate::{
    AccessPath, AnalysisExpr, ExprEncodingRef, ExprId, ExprRef, Identifier, IntoExpr, Scope,
    ScopeDType, StatsCatalog, VTable, vtable,
};

vtable!(Var);

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct VarExpr {
    identifier: Identifier,
}

pub struct VarExprEncoding;

impl VTable for VarVTable {
    type Expr = VarExpr;
    type Encoding = VarExprEncoding;
    type Metadata = ProstMetadata<pb::VarOpts>;

    fn id(_encoding: &Self::Encoding) -> ExprId {
        ExprId::new_ref("var")
    }

    fn encoding(_expr: &Self::Expr) -> ExprEncodingRef {
        ExprEncodingRef::new_ref(VarExprEncoding.as_ref())
    }

    fn metadata(expr: &Self::Expr) -> Option<Self::Metadata> {
        let var = match &expr.identifier {
            Identifier::Identity => "".to_string(),
            Identifier::Other(var) => var.to_string(),
        };
        Some(ProstMetadata(pb::VarOpts { var }))
    }

    fn children(_expr: &Self::Expr) -> Vec<&ExprRef> {
        vec![]
    }

    fn with_children(expr: &Self::Expr, children: Vec<ExprRef>) -> VortexResult<Self::Expr> {
        if !children.is_empty() {
            vortex_bail!("Var expression does not have children, got: {:?}", children);
        }
        Ok(expr.clone())
    }

    fn build(
        _encoding: &Self::Encoding,
        metadata: &<Self::Metadata as DeserializeMetadata>::Output,
        children: Vec<ExprRef>,
    ) -> VortexResult<Self::Expr> {
        if !children.is_empty() {
            vortex_bail!("Var expression does not have children, got: {:?}", children);
        }
        Ok(VarExpr {
            identifier: Identifier::from(metadata.var.as_str()),
        })
    }

    fn evaluate(expr: &Self::Expr, scope: &Scope) -> VortexResult<ArrayRef> {
        scope
            .array(&expr.identifier)
            .cloned()
            .ok_or_else(|| vortex_err!("cannot find '{}' in arrays scope", expr.identifier))
    }

    fn return_dtype(expr: &Self::Expr, scope: &ScopeDType) -> VortexResult<DType> {
        scope
            .dtype(&expr.identifier)
            .cloned()
            .ok_or_else(|| vortex_err!("cannot find '{}' in dtype scope", expr.identifier))
    }
}

/// Used to extract values (Arrays from the Scope).
/// see `Scope`.
impl VarExpr {
    pub fn new(identifier: Identifier) -> Self {
        Self { identifier }
    }

    pub fn var(&self) -> &Identifier {
        &self.identifier
    }
}

impl Display for VarExpr {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "${}", self.identifier)
    }
}

impl AnalysisExpr for VarExpr {
    fn max(&self, catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {
        catalog.stats_ref(&self.field_path()?, Stat::Max)
    }

    fn min(&self, catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {
        catalog.stats_ref(&self.field_path()?, Stat::Min)
    }

    fn field_path(&self) -> Option<AccessPath> {
        Some(AccessPath::new(FieldPath::root(), self.identifier.clone()))
    }
}

pub fn var(ident: impl Into<Identifier>) -> ExprRef {
    VarExpr::new(ident.into()).into_expr()
}

/// Return a global pointer to the identity token.
/// This is the name of the data found in a vortex array or file.
pub fn root() -> ExprRef {
    VarExpr::new(Identifier::Identity).into_expr()
}

pub fn is_root(expr: &ExprRef) -> bool {
    expr.as_opt::<VarVTable>()
        .is_some_and(|v| v.var().is_identity())
}

#[cfg(test)]
mod tests {
    use std::str::FromStr;

    use itertools::Itertools;
    use vortex_array::ToCanonical;
    use vortex_array::arrays::PrimitiveArray;
    use vortex_array::validity::Validity;
    use vortex_buffer::buffer;

    use crate::{Identifier, Scope, eq, var};

    #[test]
    fn test_two_vars() {
        let a1 = PrimitiveArray::new(buffer![5, 4, 3, 2, 1, 0], Validity::AllValid).to_array();
        let a2 = PrimitiveArray::from_iter(1..=6).to_array();

        let expr = eq(var(Identifier::Identity), var("row"));
        let res = expr
            .evaluate(&Scope::new(a1).with_array("row".parse().unwrap(), a2))
            .unwrap();
        let res = res.to_bool().unwrap().boolean_buffer().iter().collect_vec();

        assert_eq!(res, vec![false, false, true, false, false, false])
    }

    #[test]
    fn test_empty_string_ident_not_allowed() {
        assert!(Identifier::from_str("").is_err());
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::fmt::Display;
5
6	use vortex_array::stats::Stat;
7	use vortex_array::{ArrayRef, DeserializeMetadata, ProstMetadata};
8	use vortex_dtype::{DType, FieldPath};
9	use vortex_error::{VortexResult, vortex_bail, vortex_err};
10	use vortex_proto::expr as pb;
11
12	use crate::{
13	AccessPath, AnalysisExpr, ExprEncodingRef, ExprId, ExprRef, Identifier, IntoExpr, Scope,
14	ScopeDType, StatsCatalog, VTable, vtable,
15	};
16
17	vtable!(Var);
18
19	#[derive(Debug, Clone, PartialEq, Eq, Hash)]
20	pub struct VarExpr {
21	identifier: Identifier,
22	}
23
24	pub struct VarExprEncoding;
25
26	impl VTable for VarVTable {
27	type Expr = VarExpr;
28	type Encoding = VarExprEncoding;
29	type Metadata = ProstMetadata<pb::VarOpts>;
30
31	fn id(_encoding: &Self::Encoding) -> ExprId {	109✔
32	ExprId::new_ref("var")	109✔
33	}	109✔
34
35	fn encoding(_expr: &Self::Expr) -> ExprEncodingRef {	3✔
36	ExprEncodingRef::new_ref(VarExprEncoding.as_ref())	3✔
37	}	3✔
38
39	fn metadata(expr: &Self::Expr) -> Option<Self::Metadata> {	3✔
40	let var = match &expr.identifier {	3✔
41	Identifier::Identity => "".to_string(),	3✔
NEW 42	Identifier::Other(var) => var.to_string(),	×
43	};
44	Some(ProstMetadata(pb::VarOpts { var }))	3✔
45	}	3✔
46
47	fn children(_expr: &Self::Expr) -> Vec<&ExprRef> {	60,380✔
48	vec![]	60,380✔
49	}	60,380✔
50
NEW 51	fn with_children(expr: &Self::Expr, children: Vec<ExprRef>) -> VortexResult<Self::Expr> {	×
NEW 52	if !children.is_empty() {	×
NEW 53	vortex_bail!("Var expression does not have children, got: {:?}", children);	×
UNCOV 54	}	×
NEW 55	Ok(expr.clone())	×
UNCOV 56	}	×
57
58	fn build(	3✔
59	_encoding: &Self::Encoding,	3✔
60	metadata: &<Self::Metadata as DeserializeMetadata>::Output,	3✔
61	children: Vec<ExprRef>,	3✔
62	) -> VortexResult<Self::Expr> {	3✔
63	if !children.is_empty() {	3✔
NEW 64	vortex_bail!("Var expression does not have children, got: {:?}", children);	×
65	}	3✔
66	Ok(VarExpr {	3✔
67	identifier: Identifier::from(metadata.var.as_str()),	3✔
68	})	3✔
69	}	3✔
70
71	fn evaluate(expr: &Self::Expr, scope: &Scope) -> VortexResult<ArrayRef> {	2,808✔
72	scope	2,808✔
73	.array(&expr.identifier)	2,808✔
74	.cloned()	2,808✔
75	.ok_or_else(\|\| vortex_err!("cannot find '{}' in arrays scope", expr.identifier))	2,808✔
76	}	2,808✔
77
78	fn return_dtype(expr: &Self::Expr, scope: &ScopeDType) -> VortexResult<DType> {	5,678✔
79	scope	5,678✔
80	.dtype(&expr.identifier)	5,678✔
81	.cloned()	5,678✔
82	.ok_or_else(\|\| vortex_err!("cannot find '{}' in dtype scope", expr.identifier))	5,678✔
83	}	5,678✔
84	}
85
86	/// Used to extract values (Arrays from the Scope).
87	/// see `Scope`.
88	impl VarExpr {
89	pub fn new(identifier: Identifier) -> Self {	7,455✔
90	Self { identifier }	7,455✔
91	}	7,455✔
92
93	pub fn var(&self) -> &Identifier {	5,394✔
94	&self.identifier	5,394✔
95	}	5,394✔
96	}
97
98	impl Display for VarExpr {
99	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {	26✔
100	write!(f, "${}", self.identifier)	26✔
101	}	26✔
102	}
103
104	impl AnalysisExpr for VarExpr {
105	fn max(&self, catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {	253✔
106	catalog.stats_ref(&self.field_path()?, Stat::Max)	253✔
107	}	253✔
108
109	fn min(&self, catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {	127✔
110	catalog.stats_ref(&self.field_path()?, Stat::Min)	127✔
111	}	127✔
112
113	fn field_path(&self) -> Option<AccessPath> {	401✔
114	Some(AccessPath::new(FieldPath::root(), self.identifier.clone()))	401✔
115	}	401✔
116	}
117
118	pub fn var(ident: impl Into<Identifier>) -> ExprRef {	566✔
119	VarExpr::new(ident.into()).into_expr()	566✔
120	}	566✔
121
122	/// Return a global pointer to the identity token.
123	/// This is the name of the data found in a vortex array or file.
124	pub fn root() -> ExprRef {	6,889✔
125	VarExpr::new(Identifier::Identity).into_expr()	6,889✔
126	}	6,889✔
127
128	pub fn is_root(expr: &ExprRef) -> bool {	11,098✔
129	expr.as_opt::<VarVTable>()	11,098✔
130	.is_some_and(\|v\| v.var().is_identity())	11,098✔
131	}	11,098✔
132
133	#[cfg(test)]
134	mod tests {
135	use std::str::FromStr;
136
137	use itertools::Itertools;
138	use vortex_array::ToCanonical;
139	use vortex_array::arrays::PrimitiveArray;
140	use vortex_array::validity::Validity;
141	use vortex_buffer::buffer;
142
143	use crate::{Identifier, Scope, eq, var};
144
145	#[test]
146	fn test_two_vars() {	1✔
147	let a1 = PrimitiveArray::new(buffer![5, 4, 3, 2, 1, 0], Validity::AllValid).to_array();	1✔
148	let a2 = PrimitiveArray::from_iter(1..=6).to_array();	1✔
149		1✔
150	let expr = eq(var(Identifier::Identity), var("row"));	1✔
151	let res = expr	1✔
152	.evaluate(&Scope::new(a1).with_array("row".parse().unwrap(), a2))	1✔
153	.unwrap();	1✔
154	let res = res.to_bool().unwrap().boolean_buffer().iter().collect_vec();	1✔
155		1✔
156	assert_eq!(res, vec![false, false, true, false, false, false])	1✔
157	}	1✔
158
159	#[test]
160	fn test_empty_string_ident_not_allowed() {	1✔
161	assert!(Identifier::from_str("").is_err());	1✔
162	}	1✔
163	}

vortex-data / vortex / 16139349253

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous