16902926348

Committed 12 Aug 2025 08:08AM UTC coverage: 86.682% (+0.002%) from 86.68%

Build # 16902926348

Build Type

Pull #4207

github

Committed by

web-flow

Commit Message

Merge c69473d6a into c2d09ae50

Pull Request Pull Request #4207: Use sum compute function

Run Details

14 of 14 new or added lines in 1 file covered. (100.0%)

27 existing lines in 1 file now uncovered.

54354 of 62705 relevant lines covered (86.68%)

540709.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.33

/vortex-array/src/validity.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Array validity and nullability behavior, used by arrays and compute functions.

use std::fmt::Debug;
use std::ops::{BitAnd, Not};

use arrow_buffer::{BooleanBuffer, NullBuffer};
use vortex_dtype::{DType, Nullability};
use vortex_error::{VortexExpect as _, VortexResult, vortex_bail, vortex_err, vortex_panic};
use vortex_mask::{AllOr, Mask, MaskValues};
use vortex_scalar::Scalar;

use crate::arrays::{BoolArray, ConstantArray};
use crate::compute::{fill_null, filter, sum, take};
use crate::patches::Patches;
use crate::{Array, ArrayRef, IntoArray, ToCanonical};

/// Validity information for an array
#[derive(Clone, Debug)]
pub enum Validity {
    /// Items *can't* be null
    NonNullable,
    /// All items are valid
    AllValid,
    /// All items are null
    AllInvalid,
    /// Specified items are null
    Array(ArrayRef),
}

impl Validity {
    /// The [`DType`] of the underlying validity array (if it exists).
    pub const DTYPE: DType = DType::Bool(Nullability::NonNullable);

    pub fn null_count(&self, length: usize) -> VortexResult<usize> {
        match self {
            Self::NonNullable | Self::AllValid => Ok(0),
            Self::AllInvalid => Ok(length),
            Self::Array(a) => {
                let validity_len = a.len();
                if validity_len != length {
                    vortex_bail!(
                        "Validity array length {} doesn't match array length {}",
                        validity_len,
                        length
                    )
                }
                let true_count = sum(a)?
                    .as_primitive()
                    .as_::<usize>()?
                    .ok_or_else(|| vortex_err!("Failed to compute true count"))?;
                Ok(length - true_count)
            }
        }
    }

    /// If Validity is [`Validity::Array`], returns the array, otherwise returns `None`.
    pub fn into_array(self) -> Option<ArrayRef> {
        match self {
            Self::Array(a) => Some(a),
            _ => None,
        }
    }

    /// If Validity is [`Validity::Array`], returns a reference to the array array, otherwise returns `None`.
    pub fn as_array(&self) -> Option<&ArrayRef> {
        match self {
            Self::Array(a) => Some(a),
            _ => None,
        }
    }

    pub fn nullability(&self) -> Nullability {
        match self {
            Self::NonNullable => Nullability::NonNullable,
            _ => Nullability::Nullable,
        }
    }

    /// The union nullability and validity.
    pub fn union_nullability(self, nullability: Nullability) -> Self {
        match nullability {
            Nullability::NonNullable => self,
            Nullability::Nullable => self.into_nullable(),
        }
    }

    pub fn all_valid(&self) -> VortexResult<bool> {
        Ok(match self {
            Validity::NonNullable | Validity::AllValid => true,
            Validity::AllInvalid => false,
            Validity::Array(array) => {
                sum(array)
                    .map(|v| {
                        v.as_primitive()
                            .typed_value::<u64>()
                            .map(|count| count as usize == array.len())
                    })?
                    .ok_or(vortex_err!("Failed to compute sum for validity array"))?
            }
        })
    }

    pub fn all_invalid(&self) -> VortexResult<bool> {
        Ok(match self {
            Validity::NonNullable | Validity::AllValid => false,
            Validity::AllInvalid => true,
            Validity::Array(array) => {
                sum(array)
                    .map(|v| {
                        v.as_primitive()
                            .typed_value::<u64>()
                            .map(|count| count as usize == 0)
                    })?
                    .ok_or(vortex_err!("Failed to compute sum for validity array"))?
            }
        })
    }

    /// Returns whether the `index` item is valid.
    #[inline]
    pub fn is_valid(&self, index: usize) -> VortexResult<bool> {
        Ok(match self {
            Self::NonNullable | Self::AllValid => true,
            Self::AllInvalid => false,
            Self::Array(a) => {
                let scalar = a.scalar_at(index)?;
                scalar
                    .as_bool()
                    .value()
                    .vortex_expect("Validity must be non-nullable")
            }
        })
    }

    #[inline]
    pub fn is_null(&self, index: usize) -> VortexResult<bool> {
        Ok(!self.is_valid(index)?)
    }

    pub fn slice(&self, start: usize, stop: usize) -> VortexResult<Self> {
        match self {
            Self::Array(a) => Ok(Self::Array(a.slice(start, stop)?)),
            _ => Ok(self.clone()),
        }
    }

    pub fn take(&self, indices: &dyn Array) -> VortexResult<Self> {
        match self {
            Self::NonNullable => match indices.validity_mask()?.boolean_buffer() {
                AllOr::All => {
                    if indices.dtype().is_nullable() {
                        Ok(Self::AllValid)
                    } else {
                        Ok(Self::NonNullable)
                    }
                }
                AllOr::None => Ok(Self::AllInvalid),
                AllOr::Some(buf) => Ok(Validity::from(buf.clone())),
            },
            Self::AllValid => match indices.validity_mask()?.boolean_buffer() {
                AllOr::All => Ok(Self::AllValid),
                AllOr::None => Ok(Self::AllInvalid),
                AllOr::Some(buf) => Ok(Validity::from(buf.clone())),
            },
            Self::AllInvalid => Ok(Self::AllInvalid),
            Self::Array(is_valid) => {
                let maybe_is_valid = take(is_valid, indices)?;
                // Null indices invalidate that position.
                let is_valid = fill_null(&maybe_is_valid, &Scalar::from(false))?;
                Ok(Self::Array(is_valid))
            }
        }
    }

    /// Keep only the entries for which the mask is true.
    ///
    /// The result has length equal to the number of true values in mask.
    pub fn filter(&self, mask: &Mask) -> VortexResult<Self> {
        // NOTE(ngates): we take the mask as a reference to avoid the caller cloning unnecessarily
        //  if we happen to be NonNullable, AllValid, or AllInvalid.
        match self {
            v @ (Validity::NonNullable | Validity::AllValid | Validity::AllInvalid) => {
                Ok(v.clone())
            }
            Validity::Array(arr) => Ok(Validity::Array(filter(arr, mask)?)),
        }
    }

    /// Set to false any entries for which the mask is true.
    ///
    /// The result is always nullable. The result has the same length as self.
    pub fn mask(&self, mask: &Mask) -> VortexResult<Self> {
        match mask.boolean_buffer() {
            AllOr::All => Ok(Validity::AllInvalid),
            AllOr::None => Ok(self.clone()),
            AllOr::Some(make_invalid) => Ok(match self {
                Validity::NonNullable | Validity::AllValid => {
                    Validity::Array(BoolArray::from(make_invalid.not()).into_array())
                }
                Validity::AllInvalid => Validity::AllInvalid,
                Validity::Array(is_valid) => {
                    let is_valid = is_valid.to_bool()?;
                    let keep_valid = make_invalid.not();
                    Validity::from(is_valid.boolean_buffer().bitand(&keep_valid))
                }
            }),
        }
    }

    pub fn to_mask(&self, length: usize) -> VortexResult<Mask> {
        Ok(match self {
            Self::NonNullable | Self::AllValid => Mask::AllTrue(length),
            Self::AllInvalid => Mask::AllFalse(length),
            Self::Array(is_valid) => {
                assert_eq!(
                    is_valid.len(),
                    length,
                    "Validity::Array length must equal to_logical's argument: {}, {}.",
                    is_valid.len(),
                    length,
                );
                Mask::try_from(&is_valid.to_bool()?)?
            }
        })
    }

    /// Logically & two Validity values of the same length
    pub fn and(self, rhs: Validity) -> VortexResult<Validity> {
        let validity = match (self, rhs) {
            // Should be pretty clear
            (Validity::NonNullable, Validity::NonNullable) => Validity::NonNullable,
            // Any `AllInvalid` makes the output all invalid values
            (Validity::AllInvalid, _) | (_, Validity::AllInvalid) => Validity::AllInvalid,
            // All truthy values on one side, which makes no effect on an `Array` variant
            (Validity::Array(a), Validity::AllValid)
            | (Validity::Array(a), Validity::NonNullable)
            | (Validity::NonNullable, Validity::Array(a))
            | (Validity::AllValid, Validity::Array(a)) => Validity::Array(a),
            // Both sides are all valid
            (Validity::NonNullable, Validity::AllValid)
            | (Validity::AllValid, Validity::NonNullable)
            | (Validity::AllValid, Validity::AllValid) => Validity::AllValid,
            // Here we actually have to do some work
            (Validity::Array(lhs), Validity::Array(rhs)) => {
                let lhs = lhs.to_bool()?;
                let rhs = rhs.to_bool()?;

                let lhs = lhs.boolean_buffer();
                let rhs = rhs.boolean_buffer();

                Validity::from(lhs.bitand(rhs))
            }
        };

        Ok(validity)
    }

    pub fn patch(
        self,
        len: usize,
        indices_offset: usize,
        indices: &dyn Array,
        patches: &Validity,
    ) -> VortexResult<Self> {
        match (&self, patches) {
            (Validity::NonNullable, Validity::NonNullable) => return Ok(Validity::NonNullable),
            (Validity::NonNullable, _) => {
                vortex_bail!("Can't patch a non-nullable validity with nullable validity")
            }
            (_, Validity::NonNullable) => {
                vortex_bail!("Can't patch a nullable validity with non-nullable validity")
            }
            (Validity::AllValid, Validity::AllValid) => return Ok(Validity::AllValid),
            (Validity::AllInvalid, Validity::AllInvalid) => return Ok(Validity::AllInvalid),
            _ => {}
        };

        let own_nullability = if self == Validity::NonNullable {
            Nullability::NonNullable
        } else {
            Nullability::Nullable
        };

        let source = match self {
            Validity::NonNullable => BoolArray::from(BooleanBuffer::new_set(len)),
            Validity::AllValid => BoolArray::from(BooleanBuffer::new_set(len)),
            Validity::AllInvalid => BoolArray::from(BooleanBuffer::new_unset(len)),
            Validity::Array(a) => a.to_bool()?,
        };

        let patch_values = match patches {
            Validity::NonNullable => BoolArray::from(BooleanBuffer::new_set(indices.len())),
            Validity::AllValid => BoolArray::from(BooleanBuffer::new_set(indices.len())),
            Validity::AllInvalid => BoolArray::from(BooleanBuffer::new_unset(indices.len())),
            Validity::Array(a) => a.to_bool()?,
        };

        let patches = Patches::new(
            len,
            indices_offset,
            indices.to_array(),
            patch_values.into_array(),
        );

        Ok(Self::from_array(
            source.patch(&patches)?.into_array(),
            own_nullability,
        ))
    }

    /// Convert into a nullable variant
    pub fn into_nullable(self) -> Validity {
        match self {
            Self::NonNullable => Self::AllValid,
            _ => self,
        }
    }

    /// Convert into a non-nullable variant
    pub fn into_non_nullable(self) -> Option<Validity> {
        match self {
            Self::NonNullable => Some(Self::NonNullable),
            Self::AllValid => Some(Self::NonNullable),
            Self::AllInvalid => None,
            Self::Array(is_valid) => {
                is_valid
                    .statistics()
                    .compute_min::<bool>()
                    .vortex_expect("validity array must support min")
                    .then(|| {
                        // min true => all true
                        Self::NonNullable
                    })
            }
        }
    }

    /// Convert into a variant compatible with the given nullability, if possible.
    pub fn cast_nullability(self, nullability: Nullability) -> VortexResult<Validity> {
        match nullability {
            Nullability::NonNullable => self.into_non_nullable().ok_or_else(|| {
                vortex_err!("Cannot cast array with invalid values to non-nullable type.")
            }),
            Nullability::Nullable => Ok(self.into_nullable()),
        }
    }

    /// Create Validity by copying the given array's validity.
    pub fn copy_from_array(array: &dyn Array) -> VortexResult<Self> {
        Ok(Validity::from_mask(
            array.validity_mask()?,
            array.dtype().nullability(),
        ))
    }

    /// Create Validity from boolean array with given nullability of the array.
    ///
    /// Note: You want to pass the nullability of parent array and not the nullability of the validity array itself
    ///     as that is always nonnullable
    fn from_array(value: ArrayRef, nullability: Nullability) -> Self {
        if !matches!(value.dtype(), DType::Bool(Nullability::NonNullable)) {
            vortex_panic!("Expected a non-nullable boolean array")
        }
        match nullability {
            Nullability::NonNullable => Self::NonNullable,
            Nullability::Nullable => Self::Array(value),
        }
    }

    /// Returns the length of the validity array, if it exists.
    pub fn maybe_len(&self) -> Option<usize> {
        match self {
            Self::NonNullable | Self::AllValid | Self::AllInvalid => None,
            Self::Array(a) => Some(a.len()),
        }
    }

    pub fn uncompressed_size(&self) -> usize {
        if let Validity::Array(a) = self {
            a.len().div_ceil(8)
        } else {
            0
        }
    }

    pub fn is_array(&self) -> bool {
        matches!(self, Validity::Array(_))
    }
}

impl PartialEq for Validity {
    fn eq(&self, other: &Self) -> bool {
        match (self, other) {
            (Self::NonNullable, Self::NonNullable) => true,
            (Self::AllValid, Self::AllValid) => true,
            (Self::AllInvalid, Self::AllInvalid) => true,
            (Self::Array(a), Self::Array(b)) => {
                let a = a
                    .to_bool()
                    .vortex_expect("Failed to get Validity Array as BoolArray");
                let b = b
                    .to_bool()
                    .vortex_expect("Failed to get Validity Array as BoolArray");
                a.boolean_buffer() == b.boolean_buffer()
            }
            _ => false,
        }
    }
}

impl From<BooleanBuffer> for Validity {
    fn from(value: BooleanBuffer) -> Self {
        if value.count_set_bits() == value.len() {
            Self::AllValid
        } else if value.count_set_bits() == 0 {
            Self::AllInvalid
        } else {
            Self::Array(BoolArray::from(value).into_array())
        }
    }
}

impl From<NullBuffer> for Validity {
    fn from(value: NullBuffer) -> Self {
        value.into_inner().into()
    }
}

impl FromIterator<Mask> for Validity {
    fn from_iter<T: IntoIterator<Item = Mask>>(iter: T) -> Self {
        Validity::from_mask(iter.into_iter().collect(), Nullability::Nullable)
    }
}

impl FromIterator<bool> for Validity {
    fn from_iter<T: IntoIterator<Item = bool>>(iter: T) -> Self {
        Validity::from(BooleanBuffer::from_iter(iter))
    }
}

impl From<Nullability> for Validity {
    fn from(value: Nullability) -> Self {
        match value {
            Nullability::NonNullable => Validity::NonNullable,
            Nullability::Nullable => Validity::AllValid,
        }
    }
}

impl Validity {
    pub fn from_mask(mask: Mask, nullability: Nullability) -> Self {
        assert!(
            nullability == Nullability::Nullable || matches!(mask, Mask::AllTrue(_)),
            "NonNullable validity must be AllValid",
        );
        match mask {
            Mask::AllTrue(_) => match nullability {
                Nullability::NonNullable => Validity::NonNullable,
                Nullability::Nullable => Validity::AllValid,
            },
            Mask::AllFalse(_) => Validity::AllInvalid,
            Mask::Values(values) => Validity::Array(values.into_array()),
        }
    }
}

impl IntoArray for Mask {
    fn into_array(self) -> ArrayRef {
        match self {
            Self::AllTrue(len) => ConstantArray::new(true, len).into_array(),
            Self::AllFalse(len) => ConstantArray::new(false, len).into_array(),
            Self::Values(a) => a.into_array(),
        }
    }
}

impl IntoArray for &MaskValues {
    fn into_array(self) -> ArrayRef {
        BoolArray::new(self.boolean_buffer().clone(), Validity::NonNullable).into_array()
    }
}

#[cfg(test)]
mod tests {
    use rstest::rstest;
    use vortex_buffer::{Buffer, buffer};
    use vortex_dtype::Nullability;
    use vortex_mask::Mask;

    use crate::arrays::{BoolArray, PrimitiveArray};
    use crate::validity::Validity;
    use crate::{ArrayRef, IntoArray};

    #[rstest]
    #[case(Validity::AllValid, 5, &[2, 4], Validity::AllValid, Validity::AllValid)]
    #[case(Validity::AllValid, 5, &[2, 4], Validity::AllInvalid, Validity::Array(BoolArray::from_iter([true, true, false, true, false]).into_array())
    )]
    #[case(Validity::AllValid, 5, &[2, 4], Validity::Array(BoolArray::from_iter([true, false]).into_array()), Validity::Array(BoolArray::from_iter([true, true, true, true, false]).into_array())
    )]
    #[case(Validity::AllInvalid, 5, &[2, 4], Validity::AllValid, Validity::Array(BoolArray::from_iter([false, false, true, false, true]).into_array())
    )]
    #[case(Validity::AllInvalid, 5, &[2, 4], Validity::AllInvalid, Validity::AllInvalid)]
    #[case(Validity::AllInvalid, 5, &[2, 4], Validity::Array(BoolArray::from_iter([true, false]).into_array()), Validity::Array(BoolArray::from_iter([false, false, true, false, false]).into_array())
    )]
    #[case(Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()), 5, &[2, 4], Validity::AllValid, Validity::Array(BoolArray::from_iter([false, true, true, true, true]).into_array())
    )]
    #[case(Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()), 5, &[2, 4], Validity::AllInvalid, Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array())
    )]
    #[case(Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()), 5, &[2, 4], Validity::Array(BoolArray::from_iter([true, false]).into_array()), Validity::Array(BoolArray::from_iter([false, true, true, true, false]).into_array())
    )]
    fn patch_validity(
        #[case] validity: Validity,
        #[case] len: usize,
        #[case] positions: &[u64],
        #[case] patches: Validity,
        #[case] expected: Validity,
    ) {
        let indices =
            PrimitiveArray::new(Buffer::copy_from(positions), Validity::NonNullable).into_array();
        assert_eq!(
            validity.patch(len, 0, &indices, &patches).unwrap(),
            expected
        );
    }

    #[test]
    #[should_panic]
    fn out_of_bounds_patch() {
        Validity::NonNullable
            .patch(2, 0, &buffer![4].into_array(), &Validity::AllInvalid)
            .unwrap();
    }

    #[test]
    #[should_panic]
    fn into_validity_nullable() {
        Validity::from_mask(Mask::AllFalse(10), Nullability::NonNullable);
    }

    #[test]
    #[should_panic]
    fn into_validity_nullable_array() {
        Validity::from_mask(Mask::from_iter(vec![true, false]), Nullability::NonNullable);
    }

    #[rstest]
    #[case(Validity::AllValid, PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(), Validity::from_iter(vec![true, false]))]
    #[case(Validity::AllValid, buffer![0, 1].into_array(), Validity::AllValid)]
    #[case(Validity::AllValid, PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(), Validity::AllInvalid)]
    #[case(Validity::NonNullable, PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(), Validity::from_iter(vec![true, false]))]
    #[case(Validity::NonNullable, buffer![0, 1].into_array(), Validity::NonNullable)]
    #[case(Validity::NonNullable, PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(), Validity::AllInvalid)]
    fn validity_take(
        #[case] validity: Validity,
        #[case] indices: ArrayRef,
        #[case] expected: Validity,
    ) {
        assert_eq!(validity.take(&indices).unwrap(), expected);
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	//! Array validity and nullability behavior, used by arrays and compute functions.
5
6	use std::fmt::Debug;
7	use std::ops::{BitAnd, Not};
8
9	use arrow_buffer::{BooleanBuffer, NullBuffer};
10	use vortex_dtype::{DType, Nullability};
11	use vortex_error::{VortexExpect as _, VortexResult, vortex_bail, vortex_err, vortex_panic};
12	use vortex_mask::{AllOr, Mask, MaskValues};
13	use vortex_scalar::Scalar;
14
15	use crate::arrays::{BoolArray, ConstantArray};
16	use crate::compute::{fill_null, filter, sum, take};
17	use crate::patches::Patches;
18	use crate::{Array, ArrayRef, IntoArray, ToCanonical};
19
20	/// Validity information for an array
21	#[derive(Clone, Debug)]
22	pub enum Validity {
23	/// Items can't be null
24	NonNullable,
25	/// All items are valid
26	AllValid,
27	/// All items are null
28	AllInvalid,
29	/// Specified items are null
30	Array(ArrayRef),
31	}
32
33	impl Validity {
34	/// The [`DType`] of the underlying validity array (if it exists).
35	pub const DTYPE: DType = DType::Bool(Nullability::NonNullable);
36
37	pub fn null_count(&self, length: usize) -> VortexResult<usize> {	×
38	match self {	×
39	Self::NonNullable \| Self::AllValid => Ok(0),	×
40	Self::AllInvalid => Ok(length),	×
41	Self::Array(a) => {	×
42	let validity_len = a.len();	×
43	if validity_len != length {	×
44	vortex_bail!(	×
45	"Validity array length {} doesn't match array length {}",	×
46	validity_len,
47	length
48	)
49	}	×
50	let true_count = sum(a)?	×
51	.as_primitive()	×
52	.as_::<usize>()?	×
53	.ok_or_else(\|\| vortex_err!("Failed to compute true count"))?;	×
54	Ok(length - true_count)	×
55	}
56	}
57	}	×
58
59	/// If Validity is [`Validity::Array`], returns the array, otherwise returns `None`.
60	pub fn into_array(self) -> Option<ArrayRef> {	×
61	match self {	×
62	Self::Array(a) => Some(a),	×
63	_ => None,	×
64	}
65	}	×
66
67	/// If Validity is [`Validity::Array`], returns a reference to the array array, otherwise returns `None`.
68	pub fn as_array(&self) -> Option<&ArrayRef> {	×
69	match self {	×
70	Self::Array(a) => Some(a),	×
71	_ => None,	×
72	}
73	}	×
74
75	pub fn nullability(&self) -> Nullability {	5,088,120✔
76	match self {	5,088,120✔
77	Self::NonNullable => Nullability::NonNullable,	4,103,923✔
78	_ => Nullability::Nullable,	984,197✔
79	}
80	}	5,088,120✔
81
82	/// The union nullability and validity.
83	pub fn union_nullability(self, nullability: Nullability) -> Self {	907✔
84	match nullability {	907✔
85	Nullability::NonNullable => self,	406✔
86	Nullability::Nullable => self.into_nullable(),	501✔
87	}
88	}	907✔
89
90	pub fn all_valid(&self) -> VortexResult<bool> {	19,409,177✔
91	Ok(match self {	19,409,177✔
92	Validity::NonNullable \| Validity::AllValid => true,	19,396,227✔
93	Validity::AllInvalid => false,	46✔
94	Validity::Array(array) => {	12,904✔
95	sum(array)	12,904✔
96	.map(\|v\| {	12,904✔
97	v.as_primitive()	12,904✔
98	.typed_value::<u64>()	12,904✔
99	.map(\|count\| count as usize == array.len())	12,904✔
100	})?	12,904✔
101	.ok_or(vortex_err!("Failed to compute sum for validity array"))?	12,904✔
102	}
103	})
104	}	19,409,177✔
105
106	pub fn all_invalid(&self) -> VortexResult<bool> {	513,435✔
107	Ok(match self {	513,435✔
108	Validity::NonNullable \| Validity::AllValid => false,	481,313✔
109	Validity::AllInvalid => true,	765✔
110	Validity::Array(array) => {	31,357✔
111	sum(array)	31,357✔
112	.map(\|v\| {	31,357✔
113	v.as_primitive()	31,357✔
114	.typed_value::<u64>()	31,357✔
115	.map(\|count\| count as usize == 0)	31,357✔
116	})?	31,357✔
117	.ok_or(vortex_err!("Failed to compute sum for validity array"))?	31,357✔
118	}
119	})
120	}	513,435✔
121
122	/// Returns whether the `index` item is valid.
123	#[inline]
124	pub fn is_valid(&self, index: usize) -> VortexResult<bool> {	73,908,101✔
125	Ok(match self {	73,908,101✔
126	Self::NonNullable \| Self::AllValid => true,	67,524,023✔
127	Self::AllInvalid => false,	173,882✔
128	Self::Array(a) => {	6,210,196✔
129	let scalar = a.scalar_at(index)?;	6,210,196✔
130	scalar	6,210,196✔
131	.as_bool()	6,210,196✔
132	.value()	6,210,196✔
133	.vortex_expect("Validity must be non-nullable")	6,210,196✔
134	}
135	})
136	}	73,908,101✔
137
138	#[inline]
139	pub fn is_null(&self, index: usize) -> VortexResult<bool> {	147,615✔
140	Ok(!self.is_valid(index)?)	147,615✔
141	}	147,615✔
142
143	pub fn slice(&self, start: usize, stop: usize) -> VortexResult<Self> {	3,942,958✔
144	match self {	3,942,958✔
145	Self::Array(a) => Ok(Self::Array(a.slice(start, stop)?)),	9,885✔
146	_ => Ok(self.clone()),	3,933,073✔
147	}
148	}	3,942,958✔
149
150	pub fn take(&self, indices: &dyn Array) -> VortexResult<Self> {	75,925✔
151	match self {	75,925✔
152	Self::NonNullable => match indices.validity_mask()?.boolean_buffer() {	59,587✔
153	AllOr::All => {
154	if indices.dtype().is_nullable() {	52,645✔
155	Ok(Self::AllValid)	277✔
156	} else {
157	Ok(Self::NonNullable)	52,368✔
158	}
159	}
160	AllOr::None => Ok(Self::AllInvalid),	1✔
161	AllOr::Some(buf) => Ok(Validity::from(buf.clone())),	6,941✔
162	},
163	Self::AllValid => match indices.validity_mask()?.boolean_buffer() {	6,698✔
164	AllOr::All => Ok(Self::AllValid),	3,262✔
165	AllOr::None => Ok(Self::AllInvalid),	1✔
166	AllOr::Some(buf) => Ok(Validity::from(buf.clone())),	3,435✔
167	},
168	Self::AllInvalid => Ok(Self::AllInvalid),	123✔
169	Self::Array(is_valid) => {	9,517✔
170	let maybe_is_valid = take(is_valid, indices)?;	9,517✔
171	// Null indices invalidate that position.
172	let is_valid = fill_null(&maybe_is_valid, &Scalar::from(false))?;	9,517✔
173	Ok(Self::Array(is_valid))	9,517✔
174	}
175	}
176	}	75,925✔
177
178	/// Keep only the entries for which the mask is true.
179	///
180	/// The result has length equal to the number of true values in mask.
181	pub fn filter(&self, mask: &Mask) -> VortexResult<Self> {	58,737✔
182	// NOTE(ngates): we take the mask as a reference to avoid the caller cloning unnecessarily
183	// if we happen to be NonNullable, AllValid, or AllInvalid.
184	match self {	58,737✔
185	v @ (Validity::NonNullable \| Validity::AllValid \| Validity::AllInvalid) => {	46,890✔
186	Ok(v.clone())	46,890✔
187	}
188	Validity::Array(arr) => Ok(Validity::Array(filter(arr, mask)?)),	11,847✔
189	}
190	}	58,737✔
191
192	/// Set to false any entries for which the mask is true.
193	///
194	/// The result is always nullable. The result has the same length as self.
195	pub fn mask(&self, mask: &Mask) -> VortexResult<Self> {	11,585✔
196	match mask.boolean_buffer() {	11,585✔
UNCOV 197	AllOr::All => Ok(Validity::AllInvalid),	×
UNCOV 198	AllOr::None => Ok(self.clone()),	×
199	AllOr::Some(make_invalid) => Ok(match self {	11,585✔
200	Validity::NonNullable \| Validity::AllValid => {
201	Validity::Array(BoolArray::from(make_invalid.not()).into_array())	4,748✔
202	}
203	Validity::AllInvalid => Validity::AllInvalid,	55✔
204	Validity::Array(is_valid) => {	6,782✔
205	let is_valid = is_valid.to_bool()?;	6,782✔
206	let keep_valid = make_invalid.not();	6,782✔
207	Validity::from(is_valid.boolean_buffer().bitand(&keep_valid))	6,782✔
208	}
209	}),
210	}
211	}	11,585✔
212
213	pub fn to_mask(&self, length: usize) -> VortexResult<Mask> {	1,463,360✔
214	Ok(match self {	1,463,360✔
215	Self::NonNullable \| Self::AllValid => Mask::AllTrue(length),	1,392,249✔
216	Self::AllInvalid => Mask::AllFalse(length),	499✔
217	Self::Array(is_valid) => {	70,612✔
218	assert_eq!(	70,612✔
219	is_valid.len(),	70,612✔
220	length,
UNCOV 221	"Validity::Array length must equal to_logical's argument: {}, {}.",	×
UNCOV 222	is_valid.len(),	×
223	length,
224	);
225	Mask::try_from(&is_valid.to_bool()?)?	70,612✔
226	}
227	})
228	}	1,463,360✔
229
230	/// Logically & two Validity values of the same length
231	pub fn and(self, rhs: Validity) -> VortexResult<Validity> {	55✔
232	let validity = match (self, rhs) {	55✔
233	// Should be pretty clear
234	(Validity::NonNullable, Validity::NonNullable) => Validity::NonNullable,	55✔
235	// Any `AllInvalid` makes the output all invalid values
236	(Validity::AllInvalid, _) \| (_, Validity::AllInvalid) => Validity::AllInvalid,	×
237	// All truthy values on one side, which makes no effect on an `Array` variant
UNCOV 238	(Validity::Array(a), Validity::AllValid)	×
UNCOV 239	\| (Validity::Array(a), Validity::NonNullable)	×
UNCOV 240	\| (Validity::NonNullable, Validity::Array(a))	×
241	\| (Validity::AllValid, Validity::Array(a)) => Validity::Array(a),	×
242	// Both sides are all valid
243	(Validity::NonNullable, Validity::AllValid)
244	\| (Validity::AllValid, Validity::NonNullable)
245	\| (Validity::AllValid, Validity::AllValid) => Validity::AllValid,	×
246	// Here we actually have to do some work
247	(Validity::Array(lhs), Validity::Array(rhs)) => {	×
248	let lhs = lhs.to_bool()?;	×
UNCOV 249	let rhs = rhs.to_bool()?;	×
250
UNCOV 251	let lhs = lhs.boolean_buffer();	×
UNCOV 252	let rhs = rhs.boolean_buffer();	×
253
UNCOV 254	Validity::from(lhs.bitand(rhs))	×
255	}
256	};
257
258	Ok(validity)	55✔
259	}	55✔
260
261	pub fn patch(	12,051✔
262	self,	12,051✔
263	len: usize,	12,051✔
264	indices_offset: usize,	12,051✔
265	indices: &dyn Array,	12,051✔
266	patches: &Validity,	12,051✔
267	) -> VortexResult<Self> {	12,051✔
268	match (&self, patches) {	12,051✔
269	(Validity::NonNullable, Validity::NonNullable) => return Ok(Validity::NonNullable),	9,264✔
270	(Validity::NonNullable, _) => {
271	vortex_bail!("Can't patch a non-nullable validity with nullable validity")	1✔
272	}
273	(_, Validity::NonNullable) => {
UNCOV 274	vortex_bail!("Can't patch a nullable validity with non-nullable validity")	×
275	}
276	(Validity::AllValid, Validity::AllValid) => return Ok(Validity::AllValid),	1✔
277	(Validity::AllInvalid, Validity::AllInvalid) => return Ok(Validity::AllInvalid),	1✔
278	_ => {}	2,784✔
279	};
280
281	let own_nullability = if self == Validity::NonNullable {	2,784✔
UNCOV 282	Nullability::NonNullable	×
283	} else {
284	Nullability::Nullable	2,784✔
285	};
286
287	let source = match self {	2,784✔
UNCOV 288	Validity::NonNullable => BoolArray::from(BooleanBuffer::new_set(len)),	×
289	Validity::AllValid => BoolArray::from(BooleanBuffer::new_set(len)),	392✔
290	Validity::AllInvalid => BoolArray::from(BooleanBuffer::new_unset(len)),	2,311✔
291	Validity::Array(a) => a.to_bool()?,	81✔
292	};
293
294	let patch_values = match patches {	2,784✔
UNCOV 295	Validity::NonNullable => BoolArray::from(BooleanBuffer::new_set(indices.len())),	×
296	Validity::AllValid => BoolArray::from(BooleanBuffer::new_set(indices.len())),	1,804✔
297	Validity::AllInvalid => BoolArray::from(BooleanBuffer::new_unset(indices.len())),	2✔
298	Validity::Array(a) => a.to_bool()?,	978✔
299	};
300
301	let patches = Patches::new(	2,784✔
302	len,	2,784✔
303	indices_offset,	2,784✔
304	indices.to_array(),	2,784✔
305	patch_values.into_array(),	2,784✔
306	);
307
308	Ok(Self::from_array(	2,784✔
309	source.patch(&patches)?.into_array(),	2,784✔
310	own_nullability,	2,784✔
311	))
312	}	12,051✔
313
314	/// Convert into a nullable variant
315	pub fn into_nullable(self) -> Validity {	19,228✔
316	match self {	19,228✔
317	Self::NonNullable => Self::AllValid,	18,991✔
318	_ => self,	237✔
319	}
320	}	19,228✔
321
322	/// Convert into a non-nullable variant
323	pub fn into_non_nullable(self) -> Option<Validity> {	2,555✔
324	match self {	2,555✔
325	Self::NonNullable => Some(Self::NonNullable),	508✔
326	Self::AllValid => Some(Self::NonNullable),	1,282✔
UNCOV 327	Self::AllInvalid => None,	×
328	Self::Array(is_valid) => {	765✔
329	is_valid	765✔
330	.statistics()	765✔
331	.compute_min::<bool>()	765✔
332	.vortex_expect("validity array must support min")	765✔
333	.then(\|\| {	765✔
334	// min true => all true
335	Self::NonNullable	238✔
336	})	238✔
337	}
338	}
339	}	2,555✔
340
341	/// Convert into a variant compatible with the given nullability, if possible.
342	pub fn cast_nullability(self, nullability: Nullability) -> VortexResult<Validity> {	9,010✔
343	match nullability {	9,010✔
344	Nullability::NonNullable => self.into_non_nullable().ok_or_else(\|\| {	2,555✔
345	vortex_err!("Cannot cast array with invalid values to non-nullable type.")	527✔
346	}),	527✔
347	Nullability::Nullable => Ok(self.into_nullable()),	6,455✔
348	}
349	}	9,010✔
350
351	/// Create Validity by copying the given array's validity.
352	pub fn copy_from_array(array: &dyn Array) -> VortexResult<Self> {	23,477✔
353	Ok(Validity::from_mask(	23,477✔
354	array.validity_mask()?,	23,477✔
355	array.dtype().nullability(),	23,477✔
356	))
357	}	23,477✔
358
359	/// Create Validity from boolean array with given nullability of the array.
360	///
361	/// Note: You want to pass the nullability of parent array and not the nullability of the validity array itself
362	/// as that is always nonnullable
363	fn from_array(value: ArrayRef, nullability: Nullability) -> Self {	2,784✔
364	if !matches!(value.dtype(), DType::Bool(Nullability::NonNullable)) {	2,784✔
UNCOV 365	vortex_panic!("Expected a non-nullable boolean array")	×
366	}	2,784✔
367	match nullability {	2,784✔
UNCOV 368	Nullability::NonNullable => Self::NonNullable,	×
369	Nullability::Nullable => Self::Array(value),	2,784✔
370	}
371	}	2,784✔
372
373	/// Returns the length of the validity array, if it exists.
374	pub fn maybe_len(&self) -> Option<usize> {	6,201,525✔
375	match self {	6,201,525✔
376	Self::NonNullable \| Self::AllValid \| Self::AllInvalid => None,	6,073,306✔
377	Self::Array(a) => Some(a.len()),	128,219✔
378	}
379	}	6,201,525✔
380
381	pub fn uncompressed_size(&self) -> usize {	×
UNCOV 382	if let Validity::Array(a) = self {	×
383	a.len().div_ceil(8)	×
384	} else {
385	0	×
386	}
387	}	×
388
UNCOV 389	pub fn is_array(&self) -> bool {	×
UNCOV 390	matches!(self, Validity::Array(_))	×
UNCOV 391	}	×
392	}
393
394	impl PartialEq for Validity {
395	fn eq(&self, other: &Self) -> bool {	63,718✔
396	match (self, other) {	63,718✔
397	(Self::NonNullable, Self::NonNullable) => true,	36,442✔
398	(Self::AllValid, Self::AllValid) => true,	126✔
399	(Self::AllInvalid, Self::AllInvalid) => true,	123✔
400	(Self::Array(a), Self::Array(b)) => {	526✔
401	let a = a	526✔
402	.to_bool()	526✔
403	.vortex_expect("Failed to get Validity Array as BoolArray");	526✔
404	let b = b	526✔
405	.to_bool()	526✔
406	.vortex_expect("Failed to get Validity Array as BoolArray");	526✔
407	a.boolean_buffer() == b.boolean_buffer()	526✔
408	}
409	_ => false,	26,501✔
410	}
411	}	63,718✔
412	}
413
414	impl From<BooleanBuffer> for Validity {
415	fn from(value: BooleanBuffer) -> Self {	68,268✔
416	if value.count_set_bits() == value.len() {	68,268✔
417	Self::AllValid	374✔
418	} else if value.count_set_bits() == 0 {	67,894✔
419	Self::AllInvalid	2,235✔
420	} else {
421	Self::Array(BoolArray::from(value).into_array())	65,659✔
422	}
423	}	68,268✔
424	}
425
426	impl From<NullBuffer> for Validity {
427	fn from(value: NullBuffer) -> Self {	14,942✔
428	value.into_inner().into()	14,942✔
429	}	14,942✔
430	}
431
432	impl FromIterator<Mask> for Validity {
UNCOV 433	fn from_iter<T: IntoIterator<Item = Mask>>(iter: T) -> Self {	×
UNCOV 434	Validity::from_mask(iter.into_iter().collect(), Nullability::Nullable)	×
UNCOV 435	}	×
436	}
437
438	impl FromIterator<bool> for Validity {
439	fn from_iter<T: IntoIterator<Item = bool>>(iter: T) -> Self {	3,391✔
440	Validity::from(BooleanBuffer::from_iter(iter))	3,391✔
441	}	3,391✔
442	}
443
444	impl From<Nullability> for Validity {
445	fn from(value: Nullability) -> Self {	48,816✔
446	match value {	48,816✔
447	Nullability::NonNullable => Validity::NonNullable,	43,382✔
448	Nullability::Nullable => Validity::AllValid,	5,434✔
449	}
450	}	48,816✔
451	}
452
453	impl Validity {
454	pub fn from_mask(mask: Mask, nullability: Nullability) -> Self {	24,243✔
455	assert!(	24,243✔
456	nullability == Nullability::Nullable \|\| matches!(mask, Mask::AllTrue(_)),	24,243✔
457	"NonNullable validity must be AllValid",	2✔
458	);
459	match mask {	24,241✔
460	Mask::AllTrue(_) => match nullability {	20,018✔
461	Nullability::NonNullable => Validity::NonNullable,	19,740✔
462	Nullability::Nullable => Validity::AllValid,	278✔
463	},
464	Mask::AllFalse(_) => Validity::AllInvalid,	78✔
465	Mask::Values(values) => Validity::Array(values.into_array()),	4,145✔
466	}
467	}	24,241✔
468	}
469
470	impl IntoArray for Mask {
471	fn into_array(self) -> ArrayRef {	40✔
472	match self {	40✔
UNCOV 473	Self::AllTrue(len) => ConstantArray::new(true, len).into_array(),	×
UNCOV 474	Self::AllFalse(len) => ConstantArray::new(false, len).into_array(),	×
475	Self::Values(a) => a.into_array(),	40✔
476	}
477	}	40✔
478	}
479
480	impl IntoArray for &MaskValues {
481	fn into_array(self) -> ArrayRef {	4,224✔
482	BoolArray::new(self.boolean_buffer().clone(), Validity::NonNullable).into_array()	4,224✔
483	}	4,224✔
484	}
485
486	#[cfg(test)]
487	mod tests {
488	use rstest::rstest;
489	use vortex_buffer::{Buffer, buffer};
490	use vortex_dtype::Nullability;
491	use vortex_mask::Mask;
492
493	use crate::arrays::{BoolArray, PrimitiveArray};
494	use crate::validity::Validity;
495	use crate::{ArrayRef, IntoArray};
496
497	#[rstest]
498	#[case(Validity::AllValid, 5, &[2, 4], Validity::AllValid, Validity::AllValid)]
499	#[case(Validity::AllValid, 5, &[2, 4], Validity::AllInvalid, Validity::Array(BoolArray::from_iter([true, true, false, true, false]).into_array())
500	)]
501	#[case(Validity::AllValid, 5, &[2, 4], Validity::Array(BoolArray::from_iter([true, false]).into_array()), Validity::Array(BoolArray::from_iter([true, true, true, true, false]).into_array())
502	)]
503	#[case(Validity::AllInvalid, 5, &[2, 4], Validity::AllValid, Validity::Array(BoolArray::from_iter([false, false, true, false, true]).into_array())
504	)]
505	#[case(Validity::AllInvalid, 5, &[2, 4], Validity::AllInvalid, Validity::AllInvalid)]
506	#[case(Validity::AllInvalid, 5, &[2, 4], Validity::Array(BoolArray::from_iter([true, false]).into_array()), Validity::Array(BoolArray::from_iter([false, false, true, false, false]).into_array())
507	)]
508	#[case(Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()), 5, &[2, 4], Validity::AllValid, Validity::Array(BoolArray::from_iter([false, true, true, true, true]).into_array())
509	)]
510	#[case(Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()), 5, &[2, 4], Validity::AllInvalid, Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array())
511	)]
512	#[case(Validity::Array(BoolArray::from_iter([false, true, false, true, false]).into_array()), 5, &[2, 4], Validity::Array(BoolArray::from_iter([true, false]).into_array()), Validity::Array(BoolArray::from_iter([false, true, true, true, false]).into_array())
513	)]
514	fn patch_validity(
515	#[case] validity: Validity,
516	#[case] len: usize,
517	#[case] positions: &[u64],
518	#[case] patches: Validity,
519	#[case] expected: Validity,
520	) {
521	let indices =
522	PrimitiveArray::new(Buffer::copy_from(positions), Validity::NonNullable).into_array();
523	assert_eq!(
524	validity.patch(len, 0, &indices, &patches).unwrap(),
525	expected
526	);
527	}
528
529	#[test]
530	#[should_panic]
531	fn out_of_bounds_patch() {	1✔
532	Validity::NonNullable	1✔
533	.patch(2, 0, &buffer![4].into_array(), &Validity::AllInvalid)	1✔
534	.unwrap();	1✔
535	}	1✔
536
537	#[test]
538	#[should_panic]
539	fn into_validity_nullable() {	1✔
540	Validity::from_mask(Mask::AllFalse(10), Nullability::NonNullable);	1✔
541	}	1✔
542
543	#[test]
544	#[should_panic]
545	fn into_validity_nullable_array() {	1✔
546	Validity::from_mask(Mask::from_iter(vec![true, false]), Nullability::NonNullable);	1✔
547	}	1✔
548
549	#[rstest]
550	#[case(Validity::AllValid, PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(), Validity::from_iter(vec![true, false]))]
551	#[case(Validity::AllValid, buffer![0, 1].into_array(), Validity::AllValid)]
552	#[case(Validity::AllValid, PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(), Validity::AllInvalid)]
553	#[case(Validity::NonNullable, PrimitiveArray::new(buffer![0, 1], Validity::from_iter(vec![true, false])).into_array(), Validity::from_iter(vec![true, false]))]
554	#[case(Validity::NonNullable, buffer![0, 1].into_array(), Validity::NonNullable)]
555	#[case(Validity::NonNullable, PrimitiveArray::new(buffer![0, 1], Validity::AllInvalid).into_array(), Validity::AllInvalid)]
556	fn validity_take(
557	#[case] validity: Validity,
558	#[case] indices: ArrayRef,
559	#[case] expected: Validity,
560	) {
561	assert_eq!(validity.take(&indices).unwrap(), expected);
562	}
563	}

vortex-data / vortex / 16902926348

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous