16139349253

Committed 08 Jul 2025 09:26AM UTC coverage: 78.057% (-0.2%) from 78.253%

Build # 16139349253

Build Type

push

github

Committed by

web-flow

Commit Message

VortexExpr VTables (#3713)

Adds the same vtable machinery as arrays and layouts already use. It
uses the "Encoding" naming scheme from arrays and layouts. I don't
particularly like it, but it's consistent. Open to renames later.

Further, adds an expression registry to the Vortex session that will be
used for deserialization.

Expressions only decide their "options" serialization. So in theory, can
support many container formats, not just proto, provided each expression
can deserialize their own options format.

---------

Signed-off-by: Nicholas Gates <nick@nickgates.com>

Run Details

800 of 1190 new or added lines in 38 files covered. (67.23%)

40 existing lines in 13 files now uncovered.

44100 of 56497 relevant lines covered (78.06%)

54989.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/vortex-datafusion/src/convert/exprs.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use datafusion::logical_expr::Operator as DFOperator;
use datafusion::physical_expr::{PhysicalExpr, expressions};
use vortex::error::{VortexResult, vortex_bail, vortex_err};
use vortex::expr::{BinaryExpr, ExprRef, IntoExpr, LikeExpr, Operator, get_item, lit, root};
use vortex::scalar::Scalar;

use crate::convert::{FromDataFusion, TryFromDataFusion};

// TODO(joe): Don't return an error when we have an unsupported node, bubble up "TRUE" as in keep
//  for that node, up to any `and` or `or` node.
impl TryFromDataFusion<dyn PhysicalExpr> for ExprRef {
    fn try_from_df(df: &dyn PhysicalExpr) -> VortexResult<Self> {
        if let Some(binary_expr) = df.as_any().downcast_ref::<expressions::BinaryExpr>() {
            let left = ExprRef::try_from_df(binary_expr.left().as_ref())?;
            let right = ExprRef::try_from_df(binary_expr.right().as_ref())?;
            let operator = Operator::try_from_df(binary_expr.op())?;

            return Ok(BinaryExpr::new_expr(left, operator, right));
        }

        if let Some(col_expr) = df.as_any().downcast_ref::<expressions::Column>() {
            return Ok(get_item(col_expr.name().to_owned(), root()));
        }

        if let Some(like) = df.as_any().downcast_ref::<expressions::LikeExpr>() {
            let child = ExprRef::try_from_df(like.expr().as_ref())?;
            let pattern = ExprRef::try_from_df(like.pattern().as_ref())?;
            return Ok(
                LikeExpr::new(child, pattern, like.negated(), like.case_insensitive()).into_expr(),
            );
        }

        if let Some(literal) = df.as_any().downcast_ref::<expressions::Literal>() {
            let value = Scalar::from_df(literal.value());
            return Ok(lit(value));
        }

        vortex_bail!("Couldn't convert DataFusion physical {df} expression to a vortex expression")
    }
}

impl TryFromDataFusion<DFOperator> for Operator {
    fn try_from_df(value: &DFOperator) -> VortexResult<Self> {
        match value {
            DFOperator::Eq => Ok(Operator::Eq),
            DFOperator::NotEq => Ok(Operator::NotEq),
            DFOperator::Lt => Ok(Operator::Lt),
            DFOperator::LtEq => Ok(Operator::Lte),
            DFOperator::Gt => Ok(Operator::Gt),
            DFOperator::GtEq => Ok(Operator::Gte),
            DFOperator::And => Ok(Operator::And),
            DFOperator::Or => Ok(Operator::Or),
            DFOperator::IsDistinctFrom
            | DFOperator::IsNotDistinctFrom
            | DFOperator::RegexMatch
            | DFOperator::RegexIMatch
            | DFOperator::RegexNotMatch
            | DFOperator::RegexNotIMatch
            | DFOperator::LikeMatch
            | DFOperator::ILikeMatch
            | DFOperator::NotLikeMatch
            | DFOperator::NotILikeMatch
            | DFOperator::BitwiseAnd
            | DFOperator::BitwiseOr
            | DFOperator::BitwiseXor
            | DFOperator::BitwiseShiftRight
            | DFOperator::BitwiseShiftLeft
            | DFOperator::StringConcat
            | DFOperator::AtArrow
            | DFOperator::ArrowAt
            | DFOperator::Plus
            | DFOperator::Minus
            | DFOperator::Multiply
            | DFOperator::Divide
            | DFOperator::Modulo
            | DFOperator::Arrow
            | DFOperator::LongArrow
            | DFOperator::HashArrow
            | DFOperator::HashLongArrow
            | DFOperator::AtAt
            | DFOperator::IntegerDivide
            | DFOperator::HashMinus
            | DFOperator::AtQuestion
            | DFOperator::Question
            | DFOperator::QuestionAnd
            | DFOperator::QuestionPipe => {
                Err(vortex_err!("Unsupported datafusion operator {value}"))
            }
        }
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use datafusion::logical_expr::Operator as DFOperator;
5	use datafusion::physical_expr::{PhysicalExpr, expressions};
6	use vortex::error::{VortexResult, vortex_bail, vortex_err};
7	use vortex::expr::{BinaryExpr, ExprRef, IntoExpr, LikeExpr, Operator, get_item, lit, root};
8	use vortex::scalar::Scalar;
9
10	use crate::convert::{FromDataFusion, TryFromDataFusion};
11
12	// TODO(joe): Don't return an error when we have an unsupported node, bubble up "TRUE" as in keep
13	// for that node, up to any `and` or `or` node.
14	impl TryFromDataFusion<dyn PhysicalExpr> for ExprRef {
15	fn try_from_df(df: &dyn PhysicalExpr) -> VortexResult<Self> {	×
16	if let Some(binary_expr) = df.as_any().downcast_ref::<expressions::BinaryExpr>() {	×
17	let left = ExprRef::try_from_df(binary_expr.left().as_ref())?;	×
18	let right = ExprRef::try_from_df(binary_expr.right().as_ref())?;	×
19	let operator = Operator::try_from_df(binary_expr.op())?;	×
20
21	return Ok(BinaryExpr::new_expr(left, operator, right));	×
22	}	×
23
24	if let Some(col_expr) = df.as_any().downcast_ref::<expressions::Column>() {	×
25	return Ok(get_item(col_expr.name().to_owned(), root()));	×
26	}	×
27
28	if let Some(like) = df.as_any().downcast_ref::<expressions::LikeExpr>() {	×
29	let child = ExprRef::try_from_df(like.expr().as_ref())?;	×
30	let pattern = ExprRef::try_from_df(like.pattern().as_ref())?;	×
NEW 31	return Ok(	×
NEW 32	LikeExpr::new(child, pattern, like.negated(), like.case_insensitive()).into_expr(),	×
NEW 33	);	×
UNCOV 34	}	×
35
36	if let Some(literal) = df.as_any().downcast_ref::<expressions::Literal>() {	×
37	let value = Scalar::from_df(literal.value());	×
38	return Ok(lit(value));	×
39	}	×
40		×
41	vortex_bail!("Couldn't convert DataFusion physical {df} expression to a vortex expression")	×
42	}	×
43	}
44
45	impl TryFromDataFusion<DFOperator> for Operator {
46	fn try_from_df(value: &DFOperator) -> VortexResult<Self> {	×
47	match value {	×
48	DFOperator::Eq => Ok(Operator::Eq),	×
49	DFOperator::NotEq => Ok(Operator::NotEq),	×
50	DFOperator::Lt => Ok(Operator::Lt),	×
51	DFOperator::LtEq => Ok(Operator::Lte),	×
52	DFOperator::Gt => Ok(Operator::Gt),	×
53	DFOperator::GtEq => Ok(Operator::Gte),	×
54	DFOperator::And => Ok(Operator::And),	×
55	DFOperator::Or => Ok(Operator::Or),	×
56	DFOperator::IsDistinctFrom
57	\| DFOperator::IsNotDistinctFrom
58	\| DFOperator::RegexMatch
59	\| DFOperator::RegexIMatch
60	\| DFOperator::RegexNotMatch
61	\| DFOperator::RegexNotIMatch
62	\| DFOperator::LikeMatch
63	\| DFOperator::ILikeMatch
64	\| DFOperator::NotLikeMatch
65	\| DFOperator::NotILikeMatch
66	\| DFOperator::BitwiseAnd
67	\| DFOperator::BitwiseOr
68	\| DFOperator::BitwiseXor
69	\| DFOperator::BitwiseShiftRight
70	\| DFOperator::BitwiseShiftLeft
71	\| DFOperator::StringConcat
72	\| DFOperator::AtArrow
73	\| DFOperator::ArrowAt
74	\| DFOperator::Plus
75	\| DFOperator::Minus
76	\| DFOperator::Multiply
77	\| DFOperator::Divide
78	\| DFOperator::Modulo
79	\| DFOperator::Arrow
80	\| DFOperator::LongArrow
81	\| DFOperator::HashArrow
82	\| DFOperator::HashLongArrow
83	\| DFOperator::AtAt
84	\| DFOperator::IntegerDivide
85	\| DFOperator::HashMinus
86	\| DFOperator::AtQuestion
87	\| DFOperator::Question
88	\| DFOperator::QuestionAnd
89	\| DFOperator::QuestionPipe => {
90	Err(vortex_err!("Unsupported datafusion operator {value}"))	×
91	}
92	}
93	}	×
94	}

vortex-data / vortex / 16139349253

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous