16176491672

Committed 09 Jul 2025 05:48PM UTC coverage: 78.1% (+0.01%) from 78.09%

Build # 16176491672

Build Type

push

github

Committed by

web-flow

Commit Message

ci: update target dir for coverage runs (#3805)

The `-C instrument-coverage` flag doesn't play well with ASAN, because
both instruments require hooking into shutdown so one ends up overriding
the other.

Signed-off-by: Andrew Duffy <andrew@a10y.dev>

Run Details

44146 of 56525 relevant lines covered (78.1%)

53416.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.77

/vortex-array/src/arrow/convert.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use arrow_array::array::{
    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray, GenericByteArray,
    NullArray as ArrowNullArray, OffsetSizeTrait, PrimitiveArray as ArrowPrimitiveArray,
    StructArray as ArrowStructArray,
};
use arrow_array::cast::{AsArray, as_null_array};
use arrow_array::types::{
    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,
    Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
    Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
    TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
};
use arrow_array::{
    BinaryViewArray, GenericByteViewArray, GenericListArray, RecordBatch, StringViewArray,
    make_array,
};
use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
use itertools::Itertools;
use vortex_buffer::{Alignment, Buffer, ByteBuffer};
use vortex_dtype::datetime::TimeUnit;
use vortex_dtype::{DType, DecimalDType, NativePType, PType};
use vortex_error::{VortexExpect as _, vortex_panic};
use vortex_scalar::i256;

use crate::arrays::{
    BoolArray, DecimalArray, ListArray, NullArray, PrimitiveArray, StructArray, TemporalArray,
    VarBinArray, VarBinViewArray,
};
use crate::arrow::FromArrowArray;
use crate::validity::Validity;
use crate::{ArrayRef, IntoArray};

impl IntoArray for ArrowBuffer {
    fn into_array(self) -> ArrayRef {
        PrimitiveArray::from_byte_buffer(
            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
            PType::U8,
            Validity::NonNullable,
        )
        .into_array()
    }
}

impl IntoArray for BooleanBuffer {
    fn into_array(self) -> ArrayRef {
        BoolArray::new(self, Validity::NonNullable).into_array()
    }
}

impl<T> IntoArray for ScalarBuffer<T>
where
    T: ArrowNativeType + NativePType,
{
    fn into_array(self) -> ArrayRef {
        PrimitiveArray::new(
            Buffer::<T>::from_arrow_scalar_buffer(self),
            Validity::NonNullable,
        )
        .into_array()
    }
}

impl<O> IntoArray for OffsetBuffer<O>
where
    O: NativePType + OffsetSizeTrait,
{
    fn into_array(self) -> ArrayRef {
        let primitive = PrimitiveArray::new(
            Buffer::from_arrow_scalar_buffer(self.into_inner()),
            Validity::NonNullable,
        );

        primitive.into_array()
    }
}

macro_rules! impl_from_arrow_primitive {
    ($ty:path) => {
        impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
            fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {
                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
                let validity = nulls(value.nulls(), nullable);
                PrimitiveArray::new(buffer, validity).into_array()
            }
        }
    };
}

impl_from_arrow_primitive!(Int8Type);
impl_from_arrow_primitive!(Int16Type);
impl_from_arrow_primitive!(Int32Type);
impl_from_arrow_primitive!(Int64Type);
impl_from_arrow_primitive!(UInt8Type);
impl_from_arrow_primitive!(UInt16Type);
impl_from_arrow_primitive!(UInt32Type);
impl_from_arrow_primitive!(UInt64Type);
impl_from_arrow_primitive!(Float16Type);
impl_from_arrow_primitive!(Float32Type);
impl_from_arrow_primitive!(Float64Type);

impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
        let decimal_type = DecimalDType::new(array.precision(), array.scale());
        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
        let validity = nulls(array.nulls(), nullable);
        DecimalArray::new(buffer, decimal_type, validity).into_array()
    }
}

impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
        let decimal_type = DecimalDType::new(array.precision(), array.scale());
        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
        //  of either type.
        let buffer =
            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
        let validity = nulls(array.nulls(), nullable);
        DecimalArray::new(buffer, decimal_type, validity).into_array()
    }
}

macro_rules! impl_from_arrow_temporal {
    ($ty:path) => {
        impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
            fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {
                temporal_array(value, nullable)
            }
        }
    };
}

// timestamp
impl_from_arrow_temporal!(TimestampSecondType);
impl_from_arrow_temporal!(TimestampMillisecondType);
impl_from_arrow_temporal!(TimestampMicrosecondType);
impl_from_arrow_temporal!(TimestampNanosecondType);

// time
impl_from_arrow_temporal!(Time32SecondType);
impl_from_arrow_temporal!(Time32MillisecondType);
impl_from_arrow_temporal!(Time64MicrosecondType);
impl_from_arrow_temporal!(Time64NanosecondType);

// date
impl_from_arrow_temporal!(Date32Type);
impl_from_arrow_temporal!(Date64Type);

fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
where
    T::Native: NativePType,
{
    let arr = PrimitiveArray::new(
        Buffer::from_arrow_scalar_buffer(value.values().clone()),
        nulls(value.nulls(), nullable),
    )
    .into_array();

    match T::DATA_TYPE {
        DataType::Timestamp(time_unit, tz) => {
            let tz = tz.map(|s| s.to_string());
            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
        }
        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::D).into(),
        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Ms).into(),
        DataType::Duration(_) => unimplemented!(),
        DataType::Interval(_) => unimplemented!(),
        _ => vortex_panic!("Invalid temporal type: {}", T::DATA_TYPE),
    }
}

impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
where
    <T as ByteArrayType>::Offset: NativePType,
{
    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
        let dtype = match T::DATA_TYPE {
            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
            _ => vortex_panic!("Invalid data type for ByteArray: {}", T::DATA_TYPE),
        };
        VarBinArray::try_new(
            value.offsets().clone().into_array(),
            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
            dtype,
            nulls(value.nulls(), nullable),
        )
        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
        .into_array()
    }
}

impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
        let dtype = match T::DATA_TYPE {
            DataType::BinaryView => DType::Binary(nullable.into()),
            DataType::Utf8View => DType::Utf8(nullable.into()),
            _ => vortex_panic!("Invalid data type for ByteViewArray: {}", T::DATA_TYPE),
        };

        let views_buffer = Buffer::from_byte_buffer(
            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
        );

        VarBinViewArray::try_new(
            views_buffer,
            value
                .data_buffers()
                .iter()
                .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
                .collect::<Vec<_>>(),
            dtype,
            nulls(value.nulls(), nullable),
        )
        .vortex_expect("Failed to convert Arrow GenericByteViewArray to Vortex VarBinViewArray")
        .into_array()
    }
}

impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
        BoolArray::new(value.values().clone(), nulls(value.nulls(), nullable)).into_array()
    }
}

/// Strip out the nulls from this array and return a new array without nulls.
fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
    if data.null_count() == 0 {
        // No nulls to remove, return the array as is
        return data;
    }

    let children = match data.data_type() {
        DataType::Struct(fields) => Some(
            fields
                .iter()
                .zip(data.child_data().iter())
                .map(|(field, child_data)| {
                    if field.is_nullable() {
                        child_data.clone()
                    } else {
                        remove_nulls(child_data.clone())
                    }
                })
                .collect_vec(),
        ),
        DataType::List(f)
        | DataType::LargeList(f)
        | DataType::ListView(f)
        | DataType::LargeListView(f)
        | DataType::FixedSizeList(f, _)
            if !f.is_nullable() =>
        {
            // All list types only have one child
            assert_eq!(
                data.child_data().len(),
                1,
                "List types should have one child"
            );
            Some(vec![remove_nulls(data.child_data()[0].clone())])
        }
        _ => None,
    };

    let mut builder = data.into_builder().nulls(None);
    if let Some(children) = children {
        builder = builder.child_data(children);
    }
    builder
        .build()
        .vortex_expect("reconstructing array without nulls")
}

impl FromArrowArray<&ArrowStructArray> for ArrayRef {
    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
        StructArray::try_new(
            value.column_names().iter().copied().collect(),
            value
                .columns()
                .iter()
                .zip(value.fields())
                .map(|(c, field)| {
                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
                    // out here because Vortex is a little more strict.
                    if c.null_count() > 0 && !field.is_nullable() {
                        let stripped = make_array(remove_nulls(c.into_data()));
                        Self::from_arrow(stripped.as_ref(), false)
                    } else {
                        Self::from_arrow(c.as_ref(), field.is_nullable())
                    }
                })
                .collect(),
            value.len(),
            nulls(value.nulls(), nullable),
        )
        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
        .into_array()
    }
}

impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for ArrayRef {
    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
        // Extract the validity of the underlying element array
        let elem_nullable = match value.data_type() {
            DataType::List(field) => field.is_nullable(),
            DataType::LargeList(field) => field.is_nullable(),
            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
        };
        ListArray::try_new(
            Self::from_arrow(value.values().as_ref(), elem_nullable),
            // offsets are always non-nullable
            value.offsets().clone().into_array(),
            nulls(value.nulls(), nullable),
        )
        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
        .into_array()
    }
}

impl FromArrowArray<&ArrowNullArray> for ArrayRef {
    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
        assert!(nullable);
        NullArray::new(value.len()).into_array()
    }
}

fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
    if nullable {
        nulls
            .map(|nulls| {
                if nulls.null_count() == nulls.len() {
                    Validity::AllInvalid
                } else {
                    Validity::from(nulls.inner().clone())
                }
            })
            .unwrap_or_else(|| Validity::AllValid)
    } else {
        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
        Validity::NonNullable
    }
}

impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
        match array.data_type() {
            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
            DataType::BinaryView => Self::from_arrow(
                array
                    .as_any()
                    .downcast_ref::<BinaryViewArray>()
                    .vortex_expect("Expected Arrow BinaryViewArray for DataType::BinaryView"),
                nullable,
            ),
            DataType::Utf8View => Self::from_arrow(
                array
                    .as_any()
                    .downcast_ref::<StringViewArray>()
                    .vortex_expect("Expected Arrow StringViewArray for DataType::Utf8View"),
                nullable,
            ),
            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
            DataType::Timestamp(u, _) => match u {
                ArrowTimeUnit::Second => {
                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
                }
                ArrowTimeUnit::Millisecond => {
                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
                }
                ArrowTimeUnit::Microsecond => {
                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
                }
                ArrowTimeUnit::Nanosecond => {
                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
                }
            },
            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
            DataType::Time32(u) => match u {
                ArrowTimeUnit::Second => {
                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
                }
                ArrowTimeUnit::Millisecond => {
                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
                }
                _ => unreachable!(),
            },
            DataType::Time64(u) => match u {
                ArrowTimeUnit::Microsecond => {
                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
                }
                ArrowTimeUnit::Nanosecond => {
                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
                }
                _ => unreachable!(),
            },
            DataType::Decimal128(..) => {
                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
            }
            DataType::Decimal256(..) => {
                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
            }
            _ => vortex_panic!(
                "Array encoding not implemented for Arrow data type {}",
                array.data_type().clone()
            ),
        }
    }
}

impl FromArrowArray<RecordBatch> for ArrayRef {
    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
    }
}

impl FromArrowArray<&RecordBatch> for ArrayRef {
    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
        Self::from_arrow(array.clone(), nullable)
    }
}

#[cfg(test)]
mod tests {
    use arrow_array::new_null_array;
    use arrow_schema::{DataType, Field, Fields};

    use crate::ArrayRef;
    use crate::arrow::FromArrowArray as _;

    #[test]
    pub fn nullable_may_contain_non_nullable() {
        let null_struct_array_with_non_nullable_field = new_null_array(
            &DataType::Struct(Fields::from(vec![Field::new(
                "non_nullable_inner",
                DataType::Int32,
                false,
            )])),
            1,
        );
        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
    }

    #[test]
    pub fn nullable_may_contain_deeply_nested_non_nullable() {
        let null_struct_array_with_non_nullable_field = new_null_array(
            &DataType::Struct(Fields::from(vec![Field::new(
                "non_nullable_inner",
                DataType::Struct(Fields::from(vec![Field::new(
                    "non_nullable_deeper_inner",
                    DataType::Int32,
                    false,
                )])),
                false,
            )])),
            1,
        );
        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
    }

    #[test]
    #[should_panic]
    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
        let null_struct_array_with_non_nullable_field = new_null_array(
            &DataType::Struct(Fields::from(vec![Field::new(
                "non_nullable_deeper_inner",
                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
                false,
            )])),
            1,
        );

        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	use arrow_array::array::{
5	Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray, GenericByteArray,
6	NullArray as ArrowNullArray, OffsetSizeTrait, PrimitiveArray as ArrowPrimitiveArray,
7	StructArray as ArrowStructArray,
8	};
9	use arrow_array::cast::{AsArray, as_null_array};
10	use arrow_array::types::{
11	ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,
12	Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
13	Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
14	TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
15	TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
16	};
17	use arrow_array::{
18	BinaryViewArray, GenericByteViewArray, GenericListArray, RecordBatch, StringViewArray,
19	make_array,
20	};
21	use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
22	use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
23	use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
24	use itertools::Itertools;
25	use vortex_buffer::{Alignment, Buffer, ByteBuffer};
26	use vortex_dtype::datetime::TimeUnit;
27	use vortex_dtype::{DType, DecimalDType, NativePType, PType};
28	use vortex_error::{VortexExpect as _, vortex_panic};
29	use vortex_scalar::i256;
30
31	use crate::arrays::{
32	BoolArray, DecimalArray, ListArray, NullArray, PrimitiveArray, StructArray, TemporalArray,
33	VarBinArray, VarBinViewArray,
34	};
35	use crate::arrow::FromArrowArray;
36	use crate::validity::Validity;
37	use crate::{ArrayRef, IntoArray};
38
39	impl IntoArray for ArrowBuffer {
40	fn into_array(self) -> ArrayRef {	×
41	PrimitiveArray::from_byte_buffer(	×
42	ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),	×
43	PType::U8,	×
44	Validity::NonNullable,	×
45	)	×
46	.into_array()	×
47	}	×
48	}
49
50	impl IntoArray for BooleanBuffer {
51	fn into_array(self) -> ArrayRef {	×
52	BoolArray::new(self, Validity::NonNullable).into_array()	×
53	}	×
54	}
55
56	impl<T> IntoArray for ScalarBuffer<T>
57	where
58	T: ArrowNativeType + NativePType,
59	{
60	fn into_array(self) -> ArrayRef {	×
61	PrimitiveArray::new(	×
62	Buffer::<T>::from_arrow_scalar_buffer(self),	×
63	Validity::NonNullable,	×
64	)	×
65	.into_array()	×
66	}	×
67	}
68
69	impl<O> IntoArray for OffsetBuffer<O>
70	where
71	O: NativePType + OffsetSizeTrait,
72	{
73	fn into_array(self) -> ArrayRef {	38✔
74	let primitive = PrimitiveArray::new(	38✔
75	Buffer::from_arrow_scalar_buffer(self.into_inner()),	38✔
76	Validity::NonNullable,	38✔
77	);	38✔
78		38✔
79	primitive.into_array()	38✔
80	}	38✔
81	}
82
83	macro_rules! impl_from_arrow_primitive {
84	($ty:path) => {
85	impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
86	fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {	3,712✔
87	let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());	3,712✔
88	let validity = nulls(value.nulls(), nullable);	3,712✔
89	PrimitiveArray::new(buffer, validity).into_array()	3,712✔
90	}	3,712✔
91	}
92	};
93	}
94
95	impl_from_arrow_primitive!(Int8Type);
96	impl_from_arrow_primitive!(Int16Type);
97	impl_from_arrow_primitive!(Int32Type);
98	impl_from_arrow_primitive!(Int64Type);
99	impl_from_arrow_primitive!(UInt8Type);
100	impl_from_arrow_primitive!(UInt16Type);
101	impl_from_arrow_primitive!(UInt32Type);
102	impl_from_arrow_primitive!(UInt64Type);
103	impl_from_arrow_primitive!(Float16Type);
104	impl_from_arrow_primitive!(Float32Type);
105	impl_from_arrow_primitive!(Float64Type);
106
107	impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
108	fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {	×
109	let decimal_type = DecimalDType::new(array.precision(), array.scale());	×
110	let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());	×
111	let validity = nulls(array.nulls(), nullable);	×
112	DecimalArray::new(buffer, decimal_type, validity).into_array()	×
113	}	×
114	}
115
116	impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
117	fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {	×
118	let decimal_type = DecimalDType::new(array.precision(), array.scale());	×
119	let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());	×
120	// SAFETY: Our i256 implementation has the same bit-pattern representation of the	×
121	// arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values	×
122	// of either type.	×
123	let buffer =	×
124	unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };	×
125	let validity = nulls(array.nulls(), nullable);	×
126	DecimalArray::new(buffer, decimal_type, validity).into_array()	×
127	}	×
128	}
129
130	macro_rules! impl_from_arrow_temporal {
131	($ty:path) => {
132	impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
133	fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {	72✔
134	temporal_array(value, nullable)	72✔
135	}	72✔
136	}
137	};
138	}
139
140	// timestamp
141	impl_from_arrow_temporal!(TimestampSecondType);
142	impl_from_arrow_temporal!(TimestampMillisecondType);
143	impl_from_arrow_temporal!(TimestampMicrosecondType);
144	impl_from_arrow_temporal!(TimestampNanosecondType);
145
146	// time
147	impl_from_arrow_temporal!(Time32SecondType);
148	impl_from_arrow_temporal!(Time32MillisecondType);
149	impl_from_arrow_temporal!(Time64MicrosecondType);
150	impl_from_arrow_temporal!(Time64NanosecondType);
151
152	// date
153	impl_from_arrow_temporal!(Date32Type);
154	impl_from_arrow_temporal!(Date64Type);
155
156	fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef	72✔
157	where	72✔
158	T::Native: NativePType,	72✔
159	{	72✔
160	let arr = PrimitiveArray::new(	72✔
161	Buffer::from_arrow_scalar_buffer(value.values().clone()),	72✔
162	nulls(value.nulls(), nullable),	72✔
163	)	72✔
164	.into_array();	72✔
165		72✔
166	match T::DATA_TYPE {	72✔
167	DataType::Timestamp(time_unit, tz) => {	72✔
168	let tz = tz.map(\|s\| s.to_string());	72✔
169	TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()	72✔
170	}
171	DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),	×
172	DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),	×
173	DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::D).into(),	×
174	DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Ms).into(),	×
175	DataType::Duration(_) => unimplemented!(),	×
176	DataType::Interval(_) => unimplemented!(),	×
177	_ => vortex_panic!("Invalid temporal type: {}", T::DATA_TYPE),	×
178	}
179	}	72✔
180
181	impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
182	where
183	<T as ByteArrayType>::Offset: NativePType,
184	{
185	fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {	37✔
186	let dtype = match T::DATA_TYPE {	37✔
187	DataType::Binary \| DataType::LargeBinary => DType::Binary(nullable.into()),	×
188	DataType::Utf8 \| DataType::LargeUtf8 => DType::Utf8(nullable.into()),	37✔
189	_ => vortex_panic!("Invalid data type for ByteArray: {}", T::DATA_TYPE),	×
190	};
191	VarBinArray::try_new(	37✔
192	value.offsets().clone().into_array(),	37✔
193	ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),	37✔
194	dtype,	37✔
195	nulls(value.nulls(), nullable),	37✔
196	)	37✔
197	.vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")	37✔
198	.into_array()	37✔
199	}	37✔
200	}
201
202	impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
203	fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {	1,867✔
204	let dtype = match T::DATA_TYPE {	1,867✔
205	DataType::BinaryView => DType::Binary(nullable.into()),	10✔
206	DataType::Utf8View => DType::Utf8(nullable.into()),	1,857✔
207	_ => vortex_panic!("Invalid data type for ByteViewArray: {}", T::DATA_TYPE),	×
208	};
209
210	let views_buffer = Buffer::from_byte_buffer(	1,867✔
211	Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),	1,867✔
212	);	1,867✔
213		1,867✔
214	VarBinViewArray::try_new(	1,867✔
215	views_buffer,	1,867✔
216	value	1,867✔
217	.data_buffers()	1,867✔
218	.iter()	1,867✔
219	.map(\|b\| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))	1,867✔
220	.collect::<Vec<_>>(),	1,867✔
221	dtype,	1,867✔
222	nulls(value.nulls(), nullable),	1,867✔
223	)	1,867✔
224	.vortex_expect("Failed to convert Arrow GenericByteViewArray to Vortex VarBinViewArray")	1,867✔
225	.into_array()	1,867✔
226	}	1,867✔
227	}
228
229	impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
230	fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {	2,819✔
231	BoolArray::new(value.values().clone(), nulls(value.nulls(), nullable)).into_array()	2,819✔
232	}	2,819✔
233	}
234
235	/// Strip out the nulls from this array and return a new array without nulls.
236	fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {	4✔
237	if data.null_count() == 0 {	4✔
238	// No nulls to remove, return the array as is
239	return data;	×
240	}	4✔
241
242	let children = match data.data_type() {	4✔
243	DataType::Struct(fields) => Some(	1✔
244	fields	1✔
245	.iter()	1✔
246	.zip(data.child_data().iter())	1✔
247	.map(\|(field, child_data)\| {	1✔
248	if field.is_nullable() {	1✔
249	child_data.clone()	×
250	} else {
251	remove_nulls(child_data.clone())	1✔
252	}
253	})	1✔
254	.collect_vec(),	1✔
255	),	1✔
256	DataType::List(f)	×
257	\| DataType::LargeList(f)	×
258	\| DataType::ListView(f)	×
259	\| DataType::LargeListView(f)	×
260	\| DataType::FixedSizeList(f, _)	×
261	if !f.is_nullable() =>	×
262	{
263	// All list types only have one child
264	assert_eq!(	×
265	data.child_data().len(),	×
266	1,
267	"List types should have one child"	×
268	);
269	Some(vec![remove_nulls(data.child_data()[0].clone())])	×
270	}
271	_ => None,	3✔
272	};
273
274	let mut builder = data.into_builder().nulls(None);	4✔
275	if let Some(children) = children {	4✔
276	builder = builder.child_data(children);	1✔
277	}	3✔
278	builder	4✔
279	.build()	4✔
280	.vortex_expect("reconstructing array without nulls")	4✔
281	}	4✔
282
283	impl FromArrowArray<&ArrowStructArray> for ArrayRef {
284	fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {	113✔
285	StructArray::try_new(	113✔
286	value.column_names().iter().copied().collect(),	113✔
287	value	113✔
288	.columns()	113✔
289	.iter()	113✔
290	.zip(value.fields())	113✔
291	.map(\|(c, field)\| {	834✔
292	// Arrow pushes down nulls, even into non-nullable fields. So we strip them	834✔
293	// out here because Vortex is a little more strict.	834✔
294	if c.null_count() > 0 && !field.is_nullable() {	834✔
295	let stripped = make_array(remove_nulls(c.into_data()));	3✔
296	Self::from_arrow(stripped.as_ref(), false)	3✔
297	} else {
298	Self::from_arrow(c.as_ref(), field.is_nullable())	831✔
299	}
300	})	834✔
301	.collect(),	113✔
302	value.len(),	113✔
303	nulls(value.nulls(), nullable),	113✔
304	)	113✔
305	.vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")	113✔
306	.into_array()	113✔
307	}	113✔
308	}
309
310	impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for ArrayRef {
311	fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {	1✔
312	// Extract the validity of the underlying element array
313	let elem_nullable = match value.data_type() {	1✔
314	DataType::List(field) => field.is_nullable(),	1✔
315	DataType::LargeList(field) => field.is_nullable(),	×
316	dt => vortex_panic!("Invalid data type for ListArray: {dt}"),	×
317	};
318	ListArray::try_new(	1✔
319	Self::from_arrow(value.values().as_ref(), elem_nullable),	1✔
320	// offsets are always non-nullable	1✔
321	value.offsets().clone().into_array(),	1✔
322	nulls(value.nulls(), nullable),	1✔
323	)	1✔
324	.vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")	1✔
325	.into_array()	1✔
326	}	1✔
327	}
328
329	impl FromArrowArray<&ArrowNullArray> for ArrayRef {
330	fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {	×
331	assert!(nullable);	×
332	NullArray::new(value.len()).into_array()	×
333	}	×
334	}
335
336	fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {	8,620✔
337	if nullable {	8,620✔
338	nulls	4,441✔
339	.map(\|nulls\| {	4,441✔
340	if nulls.null_count() == nulls.len() {	2,304✔
341	Validity::AllInvalid	5✔
342	} else {
343	Validity::from(nulls.inner().clone())	2,299✔
344	}
345	})	4,441✔
346	.unwrap_or_else(\|\| Validity::AllValid)	4,441✔
347	} else {
348	assert!(nulls.map(\|x\| x.null_count() == 0).unwrap_or(true));	4,179✔
349	Validity::NonNullable	4,179✔
350	}
351	}	8,620✔
352
353	impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
354	fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {	5,381✔
355	match array.data_type() {	5,381✔
356	DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),	×
357	DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),	38✔
358	DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),	294✔
359	DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),	5✔
360	DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),	618✔
361	DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),	2✔
362	DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),	2✔
363	DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),	2,203✔
364	DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),	183✔
365	DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),	2✔
366	DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),	4✔
367	DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),	361✔
368	DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),	1✔
369	DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),	36✔
370	DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),	×
371	DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),	×
372	DataType::BinaryView => Self::from_arrow(	4✔
373	array	4✔
374	.as_any()	4✔
375	.downcast_ref::<BinaryViewArray>()	4✔
376	.vortex_expect("Expected Arrow BinaryViewArray for DataType::BinaryView"),	4✔
377	nullable,	4✔
378	),	4✔
379	DataType::Utf8View => Self::from_arrow(	1,552✔
380	array	1,552✔
381	.as_any()	1,552✔
382	.downcast_ref::<StringViewArray>()	1,552✔
383	.vortex_expect("Expected Arrow StringViewArray for DataType::Utf8View"),	1,552✔
384	nullable,	1,552✔
385	),	1,552✔
386	DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),	4✔
387	DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),	×
388	DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),	×
389	DataType::Null => Self::from_arrow(as_null_array(array), nullable),	×
390	DataType::Timestamp(u, _) => match u {	72✔
391	ArrowTimeUnit::Second => {
392	Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)	×
393	}
394	ArrowTimeUnit::Millisecond => {
395	Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)	×
396	}
397	ArrowTimeUnit::Microsecond => {
398	Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)	72✔
399	}
400	ArrowTimeUnit::Nanosecond => {
401	Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)	×
402	}
403	},
404	DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),	×
405	DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),	×
406	DataType::Time32(u) => match u {	×
407	ArrowTimeUnit::Second => {
408	Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)	×
409	}
410	ArrowTimeUnit::Millisecond => {
411	Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)	×
412	}
413	_ => unreachable!(),	×
414	},
415	DataType::Time64(u) => match u {	×
416	ArrowTimeUnit::Microsecond => {
417	Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)	×
418	}
419	ArrowTimeUnit::Nanosecond => {
420	Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)	×
421	}
422	_ => unreachable!(),	×
423	},
424	DataType::Decimal128(..) => {
425	Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)	×
426	}
427	DataType::Decimal256(..) => {
428	Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)	×
429	}
430	_ => vortex_panic!(	×
431	"Array encoding not implemented for Arrow data type {}",	×
432	array.data_type().clone()	×
433	),	×
434	}
435	}	5,381✔
436	}
437
438	impl FromArrowArray<RecordBatch> for ArrayRef {
439	fn from_arrow(array: RecordBatch, nullable: bool) -> Self {	108✔
440	ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)	108✔
441	}	108✔
442	}
443
444	impl FromArrowArray<&RecordBatch> for ArrayRef {
445	fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {	×
446	Self::from_arrow(array.clone(), nullable)	×
447	}	×
448	}
449
450	#[cfg(test)]
451	mod tests {
452	use arrow_array::new_null_array;
453	use arrow_schema::{DataType, Field, Fields};
454
455	use crate::ArrayRef;
456	use crate::arrow::FromArrowArray as _;
457
458	#[test]
459	pub fn nullable_may_contain_non_nullable() {	1✔
460	let null_struct_array_with_non_nullable_field = new_null_array(	1✔
461	&DataType::Struct(Fields::from(vec![Field::new(	1✔
462	"non_nullable_inner",	1✔
463	DataType::Int32,	1✔
464	false,	1✔
465	)])),	1✔
466	1,	1✔
467	);	1✔
468	ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);	1✔
469	}	1✔
470
471	#[test]
472	pub fn nullable_may_contain_deeply_nested_non_nullable() {	1✔
473	let null_struct_array_with_non_nullable_field = new_null_array(	1✔
474	&DataType::Struct(Fields::from(vec![Field::new(	1✔
475	"non_nullable_inner",	1✔
476	DataType::Struct(Fields::from(vec![Field::new(	1✔
477	"non_nullable_deeper_inner",	1✔
478	DataType::Int32,	1✔
479	false,	1✔
480	)])),	1✔
481	false,	1✔
482	)])),	1✔
483	1,	1✔
484	);	1✔
485	ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);	1✔
486	}	1✔
487
488	#[test]
489	#[should_panic]
490	pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {	1✔
491	let null_struct_array_with_non_nullable_field = new_null_array(	1✔
492	&DataType::Struct(Fields::from(vec![Field::new(	1✔
493	"non_nullable_deeper_inner",	1✔
494	DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),	1✔
495	false,	1✔
496	)])),	1✔
497	1,	1✔
498	);	1✔
499		1✔
500	ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);	1✔
501	}	1✔
502	}

vortex-data / vortex / 16176491672

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous