16935267080

Committed 13 Aug 2025 11:00AM UTC coverage: 24.312% (-63.3%) from 87.658%

Build # 16935267080

Build Type

Pull #4226

github

Committed by

web-flow

Commit Message

Merge 81b48c7fb into baa6ea202

Pull Request Pull Request #4226: Support converting TimestampTZ to and from duckdb

Run Details

0 of 2 new or added lines in 1 file covered. (0.0%)

20666 existing lines in 469 files now uncovered.

8726 of 35892 relevant lines covered (24.31%)

147.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.87

/vortex-array/src/compute/is_constant.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::any::Any;
use std::sync::LazyLock;

use arcref::ArcRef;
use vortex_dtype::{DType, Nullability};
use vortex_error::{VortexError, VortexResult, vortex_bail, vortex_err};
use vortex_scalar::Scalar;

use crate::Array;
use crate::arrays::{ConstantVTable, NullVTable};
use crate::compute::{ComputeFn, ComputeFnVTable, InvocationArgs, Kernel, Options, Output};
use crate::stats::{Precision, Stat, StatsProvider};
use crate::vtable::VTable;

static IS_CONSTANT_FN: LazyLock<ComputeFn> = LazyLock::new(|| {
    let compute = ComputeFn::new("is_constant".into(), ArcRef::new_ref(&IsConstant));
    for kernel in inventory::iter::<IsConstantKernelRef> {
        compute.register_kernel(kernel.0.clone());
    }
    compute
});

/// Computes whether an array has constant values. If the array's encoding doesn't implement the
/// relevant VTable, it'll try and canonicalize in order to make a determination.
///
/// An array is constant IFF at least one of the following conditions apply:
/// 1. It has at least one element (**Note** - an empty array isn't constant).
/// 1. It's encoded as a [`crate::arrays::ConstantArray`] or [`crate::arrays::NullArray`]
/// 1. Has an exact statistic attached to it, saying its constant.
/// 1. Is all invalid.
/// 1. Is all valid AND has minimum and maximum statistics that are equal.
///
/// If the array has some null values but is not all null, it'll never be constant.
///
/// Returns `Ok(None)` if we could not determine whether the array is constant, e.g. if
/// canonicalization is disabled and the no kernel exists for the array's encoding.
pub fn is_constant(array: &dyn Array) -> VortexResult<Option<bool>> {
    let opts = IsConstantOpts::default();
    is_constant_opts(array, &opts)
}

/// Computes whether an array has constant values. Configurable by [`IsConstantOpts`].
///
/// Please see [`is_constant`] for a more detailed explanation of its behavior.
pub fn is_constant_opts(array: &dyn Array, options: &IsConstantOpts) -> VortexResult<Option<bool>> {
    let result = IS_CONSTANT_FN
        .invoke(&InvocationArgs {
            inputs: &[array.into()],
            options,
        })?
        .unwrap_scalar()?
        .as_bool()
        .value();

    Ok(result)
}

struct IsConstant;

impl ComputeFnVTable for IsConstant {
    fn invoke(
        &self,
        args: &InvocationArgs,
        kernels: &[ArcRef<dyn Kernel>],
    ) -> VortexResult<Output> {
        let IsConstantArgs { array, options } = IsConstantArgs::try_from(args)?;

        // We try and rely on some easy-to-get stats
        if let Some(Precision::Exact(value)) = array.statistics().get_as::<bool>(Stat::IsConstant) {
            return Ok(Scalar::from(Some(value)).into());
        }

        let value = is_constant_impl(array, options, kernels)?;

        if options.cost == Cost::Canonicalize {
            // When we run linear canonicalize, there we must always return an exact answer.
            assert!(
                value.is_some(),
                "is constant in array {array} canonicalize returned None"
            );
        }

        // Only if we made a determination do we update the stats.
        if let Some(value) = value {
            array
                .statistics()
                .set(Stat::IsConstant, Precision::Exact(value.into()));
        }

        Ok(Scalar::from(value).into())
    }

    fn return_dtype(&self, _args: &InvocationArgs) -> VortexResult<DType> {
        // We always return a nullable boolean where `null` indicates we couldn't determine
        // whether the array is constant.
        Ok(DType::Bool(Nullability::Nullable))
    }

    fn return_len(&self, _args: &InvocationArgs) -> VortexResult<usize> {
        Ok(1)
    }

    fn is_elementwise(&self) -> bool {
        false
    }
}

fn is_constant_impl(
    array: &dyn Array,
    options: &IsConstantOpts,
    kernels: &[ArcRef<dyn Kernel>],
) -> VortexResult<Option<bool>> {
    match array.len() {
        // Our current semantics are that we can always get a value out of a constant array. We might want to change that in the future.
        0 => return Ok(Some(false)),
        // Array of length 1 is always constant.
        1 => return Ok(Some(true)),
        _ => {}
    }

    // Constant and null arrays are always constant
    if array.is::<ConstantVTable>() || array.is::<NullVTable>() {
        return Ok(Some(true));
    }

    let all_invalid = array.all_invalid()?;
    if all_invalid {
        return Ok(Some(true));
    }

    let all_valid = array.all_valid()?;

    // If we have some nulls, array can't be constant
    if !all_valid && !all_invalid {
        return Ok(Some(false));
    }

    // We already know here that the array is all valid, so we check for min/max stats.
    let min = array.statistics().get(Stat::Min);
    let max = array.statistics().get(Stat::Max);

    if let Some((min, max)) = min.zip(max) {
        // min/max are equal and exact and there are no NaNs
        if min.is_exact()
            && min == max
            && (Stat::NaNCount.dtype(array.dtype()).is_none()
                || array.statistics().get_as::<u64>(Stat::NaNCount) == Some(Precision::exact(0u64)))
        {
            return Ok(Some(true));
        }
    }

    assert!(
        all_valid,
        "All values must be valid as an invariant of the VTable."
    );
    let args = InvocationArgs {
        inputs: &[array.into()],
        options,
    };
    for kernel in kernels {
        if let Some(output) = kernel.invoke(&args)? {
            return Ok(output.unwrap_scalar()?.as_bool().value());
        }
    }
    if let Some(output) = array.invoke(&IS_CONSTANT_FN, &args)? {
        return Ok(output.unwrap_scalar()?.as_bool().value());
    }

    log::debug!(
        "No is_constant implementation found for {}",
        array.encoding_id()
    );

    if options.cost == Cost::Canonicalize && !array.is_canonical() {
        let array = array.to_canonical()?;
        let is_constant = is_constant_opts(array.as_ref(), options)?;
        return Ok(is_constant);
    }

    // Otherwise, we cannot determine if the array is constant.
    Ok(None)
}

pub struct IsConstantKernelRef(ArcRef<dyn Kernel>);
inventory::collect!(IsConstantKernelRef);

pub trait IsConstantKernel: VTable {
    /// # Preconditions
    ///
    /// * All values are valid
    /// * array.len() > 1
    ///
    /// Returns `Ok(None)` to signal we couldn't make an exact determination.
    fn is_constant(&self, array: &Self::Array, opts: &IsConstantOpts)
    -> VortexResult<Option<bool>>;
}

#[derive(Debug)]
pub struct IsConstantKernelAdapter<V: VTable>(pub V);

impl<V: VTable + IsConstantKernel> IsConstantKernelAdapter<V> {
    pub const fn lift(&'static self) -> IsConstantKernelRef {
        IsConstantKernelRef(ArcRef::new_ref(self))
    }
}

impl<V: VTable + IsConstantKernel> Kernel for IsConstantKernelAdapter<V> {
    fn invoke(&self, args: &InvocationArgs) -> VortexResult<Option<Output>> {
        let args = IsConstantArgs::try_from(args)?;
        let Some(array) = args.array.as_opt::<V>() else {
            return Ok(None);
        };
        let is_constant = V::is_constant(&self.0, array, args.options)?;
        Ok(Some(Scalar::from(is_constant).into()))
    }
}

struct IsConstantArgs<'a> {
    array: &'a dyn Array,
    options: &'a IsConstantOpts,
}

impl<'a> TryFrom<&InvocationArgs<'a>> for IsConstantArgs<'a> {
    type Error = VortexError;

    fn try_from(value: &InvocationArgs<'a>) -> Result<Self, Self::Error> {
        if value.inputs.len() != 1 {
            vortex_bail!("Expected 1 input, found {}", value.inputs.len());
        }
        let array = value.inputs[0]
            .array()
            .ok_or_else(|| vortex_err!("Expected input 0 to be an array"))?;
        let options = value
            .options
            .as_any()
            .downcast_ref::<IsConstantOpts>()
            .ok_or_else(|| vortex_err!("Expected options to be of type IsConstantOpts"))?;
        Ok(Self { array, options })
    }
}

/// When calling `is_constant` the children are all checked for constantness.
/// This enum decide at each precision/cost level the constant check should run as.
/// The cost increase as we move down the list.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Cost {
    /// Only apply constant time computation to estimate constantness.
    Negligible,
    /// Allow the encoding to do a linear amount of work to determine is constant.
    /// Each encoding should implement short-circuiting make the common case runtime well below
    /// a linear scan.
    Specialized,
    /// Same as linear, but when necessary canonicalize the array and check is constant.
    /// This *must* always return a known answer.
    Canonicalize,
}

/// Configuration for [`is_constant_opts`] operations.
#[derive(Clone, Debug)]
pub struct IsConstantOpts {
    /// What precision cost trade off should be used
    pub cost: Cost,
}

impl Default for IsConstantOpts {
    fn default() -> Self {
        Self {
            cost: Cost::Canonicalize,
        }
    }
}

impl Options for IsConstantOpts {
    fn as_any(&self) -> &dyn Any {
        self
    }
}

impl IsConstantOpts {
    pub fn is_negligible_cost(&self) -> bool {
        self.cost == Cost::Negligible
    }
}

#[cfg(test)]
mod tests {
    use crate::arrays::PrimitiveArray;
    use crate::stats::Stat;

    #[test]
    fn is_constant_min_max_no_nan() {
        let arr = PrimitiveArray::from_iter([0, 1]);
        arr.statistics()
            .compute_all(&[Stat::Min, Stat::Max])
            .unwrap();
        assert!(!arr.is_constant());

        let arr = PrimitiveArray::from_iter([0, 0]);
        arr.statistics()
            .compute_all(&[Stat::Min, Stat::Max])
            .unwrap();
        assert!(arr.is_constant());

        let arr = PrimitiveArray::from_option_iter([Some(0), Some(0)]);
        assert!(arr.is_constant());
    }

    #[test]
    fn is_constant_min_max_with_nan() {
        let arr = PrimitiveArray::from_iter([0.0, 0.0, f32::NAN]);
        arr.statistics()
            .compute_all(&[Stat::Min, Stat::Max])
            .unwrap();
        assert!(!arr.is_constant());

        let arr =
            PrimitiveArray::from_option_iter([Some(f32::NEG_INFINITY), Some(f32::NEG_INFINITY)]);
        arr.statistics()
            .compute_all(&[Stat::Min, Stat::Max])
            .unwrap();
        assert!(arr.is_constant());
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use std::any::Any;
5	use std::sync::LazyLock;
6
7	use arcref::ArcRef;
8	use vortex_dtype::{DType, Nullability};
9	use vortex_error::{VortexError, VortexResult, vortex_bail, vortex_err};
10	use vortex_scalar::Scalar;
11
12	use crate::Array;
13	use crate::arrays::{ConstantVTable, NullVTable};
14	use crate::compute::{ComputeFn, ComputeFnVTable, InvocationArgs, Kernel, Options, Output};
15	use crate::stats::{Precision, Stat, StatsProvider};
16	use crate::vtable::VTable;
17
18	static IS_CONSTANT_FN: LazyLock<ComputeFn> = LazyLock::new(\|\| {	2✔
19	let compute = ComputeFn::new("is_constant".into(), ArcRef::new_ref(&IsConstant));	2✔
20	for kernel in inventory::iter::<IsConstantKernelRef> {	32✔
21	compute.register_kernel(kernel.0.clone());	30✔
22	}	30✔
23	compute	2✔
24	});	2✔
25
26	/// Computes whether an array has constant values. If the array's encoding doesn't implement the
27	/// relevant VTable, it'll try and canonicalize in order to make a determination.
28	///
29	/// An array is constant IFF at least one of the following conditions apply:
30	/// 1. It has at least one element (Note - an empty array isn't constant).
31	/// 1. It's encoded as a [`crate::arrays::ConstantArray`] or [`crate::arrays::NullArray`]
32	/// 1. Has an exact statistic attached to it, saying its constant.
33	/// 1. Is all invalid.
34	/// 1. Is all valid AND has minimum and maximum statistics that are equal.
35	///
36	/// If the array has some null values but is not all null, it'll never be constant.
37	///
38	/// Returns `Ok(None)` if we could not determine whether the array is constant, e.g. if
39	/// canonicalization is disabled and the no kernel exists for the array's encoding.
40	pub fn is_constant(array: &dyn Array) -> VortexResult<Option<bool>> {	12✔
41	let opts = IsConstantOpts::default();	12✔
42	is_constant_opts(array, &opts)	12✔
43	}	12✔
44
45	/// Computes whether an array has constant values. Configurable by [`IsConstantOpts`].
46	///
47	/// Please see [`is_constant`] for a more detailed explanation of its behavior.
48	pub fn is_constant_opts(array: &dyn Array, options: &IsConstantOpts) -> VortexResult<Option<bool>> {	12✔
49	let result = IS_CONSTANT_FN	12✔
50	.invoke(&InvocationArgs {	12✔
51	inputs: &[array.into()],	12✔
52	options,	12✔
53	})?	12✔
54	.unwrap_scalar()?	12✔
55	.as_bool()	12✔
56	.value();	12✔
57
58	Ok(result)	12✔
59	}	12✔
60
61	struct IsConstant;
62
63	impl ComputeFnVTable for IsConstant {
64	fn invoke(	12✔
65	&self,	12✔
66	args: &InvocationArgs,	12✔
67	kernels: &[ArcRef<dyn Kernel>],	12✔
68	) -> VortexResult<Output> {	12✔
69	let IsConstantArgs { array, options } = IsConstantArgs::try_from(args)?;	12✔
70
71	// We try and rely on some easy-to-get stats
72	if let Some(Precision::Exact(value)) = array.statistics().get_as::<bool>(Stat::IsConstant) {	12✔
UNCOV 73	return Ok(Scalar::from(Some(value)).into());	×
74	}	12✔
75
76	let value = is_constant_impl(array, options, kernels)?;	12✔
77
78	if options.cost == Cost::Canonicalize {	12✔
79	// When we run linear canonicalize, there we must always return an exact answer.
80	assert!(	12✔
81	value.is_some(),	12✔
82	"is constant in array {array} canonicalize returned None"	×
83	);
UNCOV 84	}	×
85
86	// Only if we made a determination do we update the stats.
87	if let Some(value) = value {	12✔
88	array	12✔
89	.statistics()	12✔
90	.set(Stat::IsConstant, Precision::Exact(value.into()));	12✔
91	}	12✔
92
93	Ok(Scalar::from(value).into())	12✔
94	}	12✔
95
96	fn return_dtype(&self, _args: &InvocationArgs) -> VortexResult<DType> {	12✔
97	// We always return a nullable boolean where `null` indicates we couldn't determine
98	// whether the array is constant.
99	Ok(DType::Bool(Nullability::Nullable))	12✔
100	}	12✔
101
102	fn return_len(&self, _args: &InvocationArgs) -> VortexResult<usize> {	12✔
103	Ok(1)	12✔
104	}	12✔
105
106	fn is_elementwise(&self) -> bool {	12✔
107	false	12✔
108	}	12✔
109	}
110
111	fn is_constant_impl(	12✔
112	array: &dyn Array,	12✔
113	options: &IsConstantOpts,	12✔
114	kernels: &[ArcRef<dyn Kernel>],	12✔
115	) -> VortexResult<Option<bool>> {	12✔
116	match array.len() {	12✔
117	// Our current semantics are that we can always get a value out of a constant array. We might want to change that in the future.
118	0 => return Ok(Some(false)),	×
119	// Array of length 1 is always constant.
120	1 => return Ok(Some(true)),	4✔
121	_ => {}	8✔
122	}
123
124	// Constant and null arrays are always constant
125	if array.is::<ConstantVTable>() \|\| array.is::<NullVTable>() {	8✔
UNCOV 126	return Ok(Some(true));	×
127	}	8✔
128
129	let all_invalid = array.all_invalid()?;	8✔
130	if all_invalid {	8✔
UNCOV 131	return Ok(Some(true));	×
132	}	8✔
133
134	let all_valid = array.all_valid()?;	8✔
135
136	// If we have some nulls, array can't be constant
137	if !all_valid && !all_invalid {	8✔
UNCOV 138	return Ok(Some(false));	×
139	}	8✔
140
141	// We already know here that the array is all valid, so we check for min/max stats.
142	let min = array.statistics().get(Stat::Min);	8✔
143	let max = array.statistics().get(Stat::Max);	8✔
144
145	if let Some((min, max)) = min.zip(max) {	8✔
146	// min/max are equal and exact and there are no NaNs
UNCOV 147	if min.is_exact()	×
UNCOV 148	&& min == max	×
UNCOV 149	&& (Stat::NaNCount.dtype(array.dtype()).is_none()	×
UNCOV 150	\|\| array.statistics().get_as::<u64>(Stat::NaNCount) == Some(Precision::exact(0u64)))	×
151	{
UNCOV 152	return Ok(Some(true));	×
UNCOV 153	}	×
154	}	8✔
155
156	assert!(	8✔
157	all_valid,	8✔
158	"All values must be valid as an invariant of the VTable."	×
159	);
160	let args = InvocationArgs {	8✔
161	inputs: &[array.into()],	8✔
162	options,	8✔
163	};	8✔
164	for kernel in kernels {	56✔
165	if let Some(output) = kernel.invoke(&args)? {	56✔
166	return Ok(output.unwrap_scalar()?.as_bool().value());	8✔
167	}	48✔
168	}
UNCOV 169	if let Some(output) = array.invoke(&IS_CONSTANT_FN, &args)? {	×
170	return Ok(output.unwrap_scalar()?.as_bool().value());	×
UNCOV 171	}	×
172
UNCOV 173	log::debug!(	×
174	"No is_constant implementation found for {}",	×
175	array.encoding_id()	×
176	);
177
UNCOV 178	if options.cost == Cost::Canonicalize && !array.is_canonical() {	×
UNCOV 179	let array = array.to_canonical()?;	×
UNCOV 180	let is_constant = is_constant_opts(array.as_ref(), options)?;	×
UNCOV 181	return Ok(is_constant);	×
UNCOV 182	}	×
183
184	// Otherwise, we cannot determine if the array is constant.
UNCOV 185	Ok(None)	×
186	}	12✔
187
188	pub struct IsConstantKernelRef(ArcRef<dyn Kernel>);
189	inventory::collect!(IsConstantKernelRef);
190
191	pub trait IsConstantKernel: VTable {
192	/// # Preconditions
193	///
194	/// * All values are valid
195	/// * array.len() > 1
196	///
197	/// Returns `Ok(None)` to signal we couldn't make an exact determination.
198	fn is_constant(&self, array: &Self::Array, opts: &IsConstantOpts)
199	-> VortexResult<Option<bool>>;
200	}
201
202	#[derive(Debug)]
203	pub struct IsConstantKernelAdapter<V: VTable>(pub V);
204
205	impl<V: VTable + IsConstantKernel> IsConstantKernelAdapter<V> {
206	pub const fn lift(&'static self) -> IsConstantKernelRef {	×
207	IsConstantKernelRef(ArcRef::new_ref(self))	×
208	}	×
209	}
210
211	impl<V: VTable + IsConstantKernel> Kernel for IsConstantKernelAdapter<V> {
212	fn invoke(&self, args: &InvocationArgs) -> VortexResult<Option<Output>> {	56✔
213	let args = IsConstantArgs::try_from(args)?;	56✔
214	let Some(array) = args.array.as_opt::<V>() else {	56✔
215	return Ok(None);	48✔
216	};
217	let is_constant = V::is_constant(&self.0, array, args.options)?;	8✔
218	Ok(Some(Scalar::from(is_constant).into()))	8✔
219	}	56✔
220	}
221
222	struct IsConstantArgs<'a> {
223	array: &'a dyn Array,
224	options: &'a IsConstantOpts,
225	}
226
227	impl<'a> TryFrom<&InvocationArgs<'a>> for IsConstantArgs<'a> {
228	type Error = VortexError;
229
230	fn try_from(value: &InvocationArgs<'a>) -> Result<Self, Self::Error> {	68✔
231	if value.inputs.len() != 1 {	68✔
232	vortex_bail!("Expected 1 input, found {}", value.inputs.len());	×
233	}	68✔
234	let array = value.inputs[0]	68✔
235	.array()	68✔
236	.ok_or_else(\|\| vortex_err!("Expected input 0 to be an array"))?;	68✔
237	let options = value	68✔
238	.options	68✔
239	.as_any()	68✔
240	.downcast_ref::<IsConstantOpts>()	68✔
241	.ok_or_else(\|\| vortex_err!("Expected options to be of type IsConstantOpts"))?;	68✔
242	Ok(Self { array, options })	68✔
243	}	68✔
244	}
245
246	/// When calling `is_constant` the children are all checked for constantness.
247	/// This enum decide at each precision/cost level the constant check should run as.
248	/// The cost increase as we move down the list.
249	#[derive(Clone, Copy, Debug, Eq, PartialEq)]
250	pub enum Cost {
251	/// Only apply constant time computation to estimate constantness.
252	Negligible,
253	/// Allow the encoding to do a linear amount of work to determine is constant.
254	/// Each encoding should implement short-circuiting make the common case runtime well below
255	/// a linear scan.
256	Specialized,
257	/// Same as linear, but when necessary canonicalize the array and check is constant.
258	/// This must always return a known answer.
259	Canonicalize,
260	}
261
262	/// Configuration for [`is_constant_opts`] operations.
263	#[derive(Clone, Debug)]
264	pub struct IsConstantOpts {
265	/// What precision cost trade off should be used
266	pub cost: Cost,
267	}
268
269	impl Default for IsConstantOpts {
270	fn default() -> Self {	12✔
271	Self {	12✔
272	cost: Cost::Canonicalize,	12✔
273	}	12✔
274	}	12✔
275	}
276
277	impl Options for IsConstantOpts {
278	fn as_any(&self) -> &dyn Any {	68✔
279	self	68✔
280	}	68✔
281	}
282
283	impl IsConstantOpts {
284	pub fn is_negligible_cost(&self) -> bool {	8✔
285	self.cost == Cost::Negligible	8✔
286	}	8✔
287	}
288
289	#[cfg(test)]
290	mod tests {
291	use crate::arrays::PrimitiveArray;
292	use crate::stats::Stat;
293
294	#[test]
295	fn is_constant_min_max_no_nan() {
296	let arr = PrimitiveArray::from_iter([0, 1]);
297	arr.statistics()
298	.compute_all(&[Stat::Min, Stat::Max])
299	.unwrap();
300	assert!(!arr.is_constant());
301
302	let arr = PrimitiveArray::from_iter([0, 0]);
303	arr.statistics()
304	.compute_all(&[Stat::Min, Stat::Max])
305	.unwrap();
306	assert!(arr.is_constant());
307
308	let arr = PrimitiveArray::from_option_iter([Some(0), Some(0)]);
309	assert!(arr.is_constant());
310	}
311
312	#[test]
313	fn is_constant_min_max_with_nan() {
314	let arr = PrimitiveArray::from_iter([0.0, 0.0, f32::NAN]);
315	arr.statistics()
316	.compute_all(&[Stat::Min, Stat::Max])
317	.unwrap();
318	assert!(!arr.is_constant());
319
320	let arr =
321	PrimitiveArray::from_option_iter([Some(f32::NEG_INFINITY), Some(f32::NEG_INFINITY)]);
322	arr.statistics()
323	.compute_all(&[Stat::Min, Stat::Max])
324	.unwrap();
325	assert!(arr.is_constant());
326	}
327	}

vortex-data / vortex / 16935267080

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous