16486479294

Committed 24 Jul 2025 02:42AM UTC coverage: 81.018% (-0.05%) from 81.067%

Build # 16486479294

Build Type

push

github

Committed by

web-flow

Commit Message

fix: Pruning expressions check NanCount where appropriate (#3973)

Fixes #3958 

NanCount stat is checked in the pruning evaluation if it is present
(i.e. for float columns and literals)

- [x] Verify fixes fuzz failure
- [x] Add unit tests

Signed-off-by: Andrew Duffy <andrew@a10y.dev>

---------

Signed-off-by: Andrew Duffy <andrew@a10y.dev>
Signed-off-by: Robert Kruszewski <github@robertk.io>
Co-authored-by: Robert Kruszewski <github@robertk.io>

Run Details

106 of 132 new or added lines in 12 files covered. (80.3%)

4 existing lines in 2 files now uncovered.

42012 of 51855 relevant lines covered (81.02%)

173899.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.51

/vortex-expr/src/exprs/literal.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::fmt::Display;

use vortex_array::arrays::ConstantArray;
use vortex_array::{ArrayRef, DeserializeMetadata, IntoArray, ProstMetadata};
use vortex_dtype::{DType, match_each_float_ptype};
use vortex_error::{VortexResult, vortex_bail, vortex_err};
use vortex_proto::expr as pb;
use vortex_scalar::Scalar;

use crate::{
    AnalysisExpr, ExprEncodingRef, ExprId, ExprRef, IntoExpr, Scope, StatsCatalog, VTable, vtable,
};

vtable!(Literal);

#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct LiteralExpr {
    value: Scalar,
}

pub struct LiteralExprEncoding;

impl VTable for LiteralVTable {
    type Expr = LiteralExpr;
    type Encoding = LiteralExprEncoding;
    type Metadata = ProstMetadata<pb::LiteralOpts>;

    fn id(_encoding: &Self::Encoding) -> ExprId {
        ExprId::new_ref("literal")
    }

    fn encoding(_expr: &Self::Expr) -> ExprEncodingRef {
        ExprEncodingRef::new_ref(LiteralExprEncoding.as_ref())
    }

    fn metadata(expr: &Self::Expr) -> Option<Self::Metadata> {
        Some(ProstMetadata(pb::LiteralOpts {
            value: Some((&expr.value).into()),
        }))
    }

    fn children(_expr: &Self::Expr) -> Vec<&ExprRef> {
        vec![]
    }

    fn with_children(expr: &Self::Expr, _children: Vec<ExprRef>) -> VortexResult<Self::Expr> {
        Ok(expr.clone())
    }

    fn build(
        _encoding: &Self::Encoding,
        metadata: &<Self::Metadata as DeserializeMetadata>::Output,
        children: Vec<ExprRef>,
    ) -> VortexResult<Self::Expr> {
        if !children.is_empty() {
            vortex_bail!(
                "Literal expression does not have children, got: {:?}",
                children
            );
        }
        let value: Scalar = metadata
            .value
            .as_ref()
            .ok_or_else(|| vortex_err!("Literal metadata missing value"))?
            .try_into()?;
        Ok(LiteralExpr::new(value))
    }

    fn evaluate(expr: &Self::Expr, scope: &Scope) -> VortexResult<ArrayRef> {
        Ok(ConstantArray::new(expr.value.clone(), scope.len()).into_array())
    }

    fn return_dtype(expr: &Self::Expr, _scope: &DType) -> VortexResult<DType> {
        Ok(expr.value.dtype().clone())
    }
}

impl LiteralExpr {
    pub fn new(value: impl Into<Scalar>) -> Self {
        Self {
            value: value.into(),
        }
    }

    pub fn new_expr(value: impl Into<Scalar>) -> ExprRef {
        Self::new(value).into_expr()
    }

    pub fn value(&self) -> &Scalar {
        &self.value
    }

    pub fn maybe_from(expr: &ExprRef) -> Option<&LiteralExpr> {
        expr.as_opt::<LiteralVTable>()
    }
}

impl Display for LiteralExpr {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.value)
    }
}

impl AnalysisExpr for LiteralExpr {
    fn max(&self, _catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {
        Some(lit(self.value.clone()))
    }

    fn min(&self, _catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {
        Some(lit(self.value.clone()))
    }

    fn nan_count(&self, _catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {
        // The NaNCount for a non-float literal is not defined.
        // For floating point types, the NaNCount is 1 for lit(NaN), and 0 otherwise.
        let value = self.value.as_primitive_opt()?;
        if !value.ptype().is_float() {
            return None;
        }

        match_each_float_ptype!(value.ptype(), |T| {
            match value.typed_value::<T>() {
                None => Some(lit(0u64)),
                Some(value) if value.is_nan() => Some(lit(1u64)),
                _ => Some(lit(0u64)),
            }
        })
    }
}

/// Create a new `Literal` expression from a type that coerces to `Scalar`.
///
///
/// ## Example usage
///
/// ```
/// use vortex_array::arrays::PrimitiveArray;
/// use vortex_dtype::Nullability;
/// use vortex_expr::{lit, LiteralVTable};
/// use vortex_scalar::Scalar;
///
/// let number = lit(34i32);
///
/// let literal = number.as_::<LiteralVTable>();
/// assert_eq!(literal.value(), &Scalar::primitive(34i32, Nullability::NonNullable));
/// ```
pub fn lit(value: impl Into<Scalar>) -> ExprRef {
    LiteralExpr::new(value.into()).into_expr()
}

#[cfg(test)]
mod tests {
    use vortex_dtype::{DType, Nullability, PType, StructFields};
    use vortex_scalar::Scalar;

    use crate::{lit, test_harness};

    #[test]
    fn dtype() {
        let dtype = test_harness::struct_dtype();

        assert_eq!(
            lit(10).return_dtype(&dtype).unwrap(),
            DType::Primitive(PType::I32, Nullability::NonNullable)
        );
        assert_eq!(
            lit(i64::MAX).return_dtype(&dtype).unwrap(),
            DType::Primitive(PType::I64, Nullability::NonNullable)
        );
        assert_eq!(
            lit(true).return_dtype(&dtype).unwrap(),
            DType::Bool(Nullability::NonNullable)
        );
        assert_eq!(
            lit(Scalar::null(DType::Bool(Nullability::Nullable)))
                .return_dtype(&dtype)
                .unwrap(),
            DType::Bool(Nullability::Nullable)
        );

        let sdtype = DType::Struct(
            StructFields::new(
                ["dog", "cat"].into(),
                vec![
                    DType::Primitive(PType::U32, Nullability::NonNullable),
                    DType::Utf8(Nullability::NonNullable),
                ],
            ),
            Nullability::NonNullable,
        );
        assert_eq!(
            lit(Scalar::struct_(
                sdtype.clone(),
                vec![Scalar::from(32_u32), Scalar::from("rufus".to_string())]
            ))
            .return_dtype(&dtype)
            .unwrap(),
            sdtype
        );
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::fmt::Display;
5
6	use vortex_array::arrays::ConstantArray;
7	use vortex_array::{ArrayRef, DeserializeMetadata, IntoArray, ProstMetadata};
8	use vortex_dtype::{DType, match_each_float_ptype};
9	use vortex_error::{VortexResult, vortex_bail, vortex_err};
10	use vortex_proto::expr as pb;
11	use vortex_scalar::Scalar;
12
13	use crate::{
14	AnalysisExpr, ExprEncodingRef, ExprId, ExprRef, IntoExpr, Scope, StatsCatalog, VTable, vtable,
15	};
16
17	vtable!(Literal);
18
19	#[derive(Clone, Debug, PartialEq, Eq, Hash)]
20	pub struct LiteralExpr {
21	value: Scalar,
22	}
23
24	pub struct LiteralExprEncoding;
25
26	impl VTable for LiteralVTable {
27	type Expr = LiteralExpr;
28	type Encoding = LiteralExprEncoding;
29	type Metadata = ProstMetadata<pb::LiteralOpts>;
30
31	fn id(_encoding: &Self::Encoding) -> ExprId {	131✔
32	ExprId::new_ref("literal")	131✔
33	}	131✔
34
35	fn encoding(_expr: &Self::Expr) -> ExprEncodingRef {	3✔
36	ExprEncodingRef::new_ref(LiteralExprEncoding.as_ref())	3✔
37	}	3✔
38
39	fn metadata(expr: &Self::Expr) -> Option<Self::Metadata> {	3✔
40	Some(ProstMetadata(pb::LiteralOpts {	3✔
41	value: Some((&expr.value).into()),	3✔
42	}))	3✔
43	}	3✔
44
45	fn children(_expr: &Self::Expr) -> Vec<&ExprRef> {	39,542✔
46	vec![]	39,542✔
47	}	39,542✔
48
49	fn with_children(expr: &Self::Expr, _children: Vec<ExprRef>) -> VortexResult<Self::Expr> {	×
50	Ok(expr.clone())	×
51	}	×
52
53	fn build(	3✔
54	_encoding: &Self::Encoding,	3✔
55	metadata: &<Self::Metadata as DeserializeMetadata>::Output,	3✔
56	children: Vec<ExprRef>,	3✔
57	) -> VortexResult<Self::Expr> {	3✔
58	if !children.is_empty() {	3✔
59	vortex_bail!(	×
60	"Literal expression does not have children, got: {:?}",	×
61	children
62	);
63	}	3✔
64	let value: Scalar = metadata	3✔
65	.value	3✔
66	.as_ref()	3✔
67	.ok_or_else(\|\| vortex_err!("Literal metadata missing value"))?	3✔
68	.try_into()?;	3✔
69	Ok(LiteralExpr::new(value))	3✔
70	}	3✔
71
72	fn evaluate(expr: &Self::Expr, scope: &Scope) -> VortexResult<ArrayRef> {	6,182✔
73	Ok(ConstantArray::new(expr.value.clone(), scope.len()).into_array())	6,182✔
74	}	6,182✔
75
76	fn return_dtype(expr: &Self::Expr, _scope: &DType) -> VortexResult<DType> {	8,601✔
77	Ok(expr.value.dtype().clone())	8,601✔
78	}	8,601✔
79	}
80
81	impl LiteralExpr {
82	pub fn new(value: impl Into<Scalar>) -> Self {	38,784✔
83	Self {	38,784✔
84	value: value.into(),	38,784✔
85	}	38,784✔
86	}	38,784✔
87
88	pub fn new_expr(value: impl Into<Scalar>) -> ExprRef {	6✔
89	Self::new(value).into_expr()	6✔
90	}	6✔
91
92	pub fn value(&self) -> &Scalar {	676✔
93	&self.value	676✔
94	}	676✔
95
96	pub fn maybe_from(expr: &ExprRef) -> Option<&LiteralExpr> {	741✔
97	expr.as_opt::<LiteralVTable>()	741✔
98	}	741✔
99	}
100
101	impl Display for LiteralExpr {
102	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {	12✔
103	write!(f, "{}", self.value)	12✔
104	}	12✔
105	}
106
107	impl AnalysisExpr for LiteralExpr {
108	fn max(&self, _catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {	688✔
109	Some(lit(self.value.clone()))	688✔
110	}	688✔
111
112	fn min(&self, _catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {	868✔
113	Some(lit(self.value.clone()))	868✔
114	}	868✔
115
116	fn nan_count(&self, _catalog: &mut dyn StatsCatalog) -> Option<ExprRef> {	1,073✔
117	// The NaNCount for a non-float literal is not defined.
118	// For floating point types, the NaNCount is 1 for lit(NaN), and 0 otherwise.
119	let value = self.value.as_primitive_opt()?;	1,073✔
120	if !value.ptype().is_float() {	715✔
121	return None;	671✔
122	}	44✔
123
124	match_each_float_ptype!(value.ptype(), \|T\| {	44✔
NEW 125	match value.typed_value::<T>() {	×
NEW 126	None => Some(lit(0u64)),	×
NEW 127	Some(value) if value.is_nan() => Some(lit(1u64)),	×
NEW 128	_ => Some(lit(0u64)),	×
129	}
130	})
131	}	1,073✔
132	}
133
134	/// Create a new `Literal` expression from a type that coerces to `Scalar`.
135	///
136	///
137	/// ## Example usage
138	///
139	/// ```
140	/// use vortex_array::arrays::PrimitiveArray;
141	/// use vortex_dtype::Nullability;
142	/// use vortex_expr::{lit, LiteralVTable};
143	/// use vortex_scalar::Scalar;
144	///
145	/// let number = lit(34i32);
146	///
147	/// let literal = number.as_::<LiteralVTable>();
148	/// assert_eq!(literal.value(), &Scalar::primitive(34i32, Nullability::NonNullable));
149	/// ```
150	pub fn lit(value: impl Into<Scalar>) -> ExprRef {	37,993✔
151	LiteralExpr::new(value.into()).into_expr()	37,993✔
152	}	37,993✔
153
154	#[cfg(test)]
155	mod tests {
156	use vortex_dtype::{DType, Nullability, PType, StructFields};
157	use vortex_scalar::Scalar;
158
159	use crate::{lit, test_harness};
160
161	#[test]
162	fn dtype() {	1✔
163	let dtype = test_harness::struct_dtype();	1✔
164
165	assert_eq!(	1✔
166	lit(10).return_dtype(&dtype).unwrap(),	1✔
167	DType::Primitive(PType::I32, Nullability::NonNullable)
168	);
169	assert_eq!(	1✔
170	lit(i64::MAX).return_dtype(&dtype).unwrap(),	1✔
171	DType::Primitive(PType::I64, Nullability::NonNullable)
172	);
173	assert_eq!(	1✔
174	lit(true).return_dtype(&dtype).unwrap(),	1✔
175	DType::Bool(Nullability::NonNullable)
176	);
177	assert_eq!(	1✔
178	lit(Scalar::null(DType::Bool(Nullability::Nullable)))	1✔
179	.return_dtype(&dtype)	1✔
180	.unwrap(),	1✔
181	DType::Bool(Nullability::Nullable)
182	);
183
184	let sdtype = DType::Struct(	1✔
185	StructFields::new(	1✔
186	["dog", "cat"].into(),	1✔
187	vec![	1✔
188	DType::Primitive(PType::U32, Nullability::NonNullable),	1✔
189	DType::Utf8(Nullability::NonNullable),	1✔
190	],	1✔
191	),	1✔
192	Nullability::NonNullable,	1✔
193	);	1✔
194	assert_eq!(	1✔
195	lit(Scalar::struct_(	1✔
196	sdtype.clone(),	1✔
197	vec![Scalar::from(32_u32), Scalar::from("rufus".to_string())]	1✔
198	))	1✔
199	.return_dtype(&dtype)	1✔
200	.unwrap(),	1✔
201	sdtype
202	);
203	}	1✔
204	}

vortex-data / vortex / 16486479294

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous