vortex-data / vortex / 16372702366

coverage: 81.499% (-0.03%) from 81.529%
16372702366

Pull #3899

github

web-flow 
Merge b2016e2e9 into e971e6c7f
Pull Request #3899: Remove async API for scanning

188 of 207 new or added lines in 9 files covered. (90.82%)

10 existing lines in 2 files now uncovered.

42034 of 51576 relevant lines covered (81.5%)

171519.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.67
/vortex-file/src/tests.rs
1 
// SPDX-License-Identifier: Apache-2.0
2 
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3 









4



#![allow(clippy::cast_possible_truncation)]










5



use std::iter;










6



use std::sync::Arc;










7



use vortex_array::iter::ArrayIteratorExt;










8












9



use bytes::Bytes;










10



use futures::executor::block_on;










11



use itertools::Itertools;










12



use vortex_array::accessor::ArrayAccessor;










13



use vortex_array::arrays::{










14



    ChunkedArray, ConstantArray, DecimalArray, ListArray, PrimitiveArray, StructArray, VarBinArray,










15



    VarBinViewArray,










16



};










17



use vortex_array::validity::Validity;










18



use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};










19



use vortex_buffer::{Buffer, ByteBufferMut, buffer};










20



use vortex_dtype::PType::I32;










21



use vortex_dtype::{DType, DecimalDType, Nullability, PType, StructFields};










22



use vortex_error::VortexResult;










23



use vortex_expr::{PackExpr, and, eq, get_item, gt, gt_eq, lit, lt, lt_eq, or, root, select};










24



use vortex_scalar::Scalar;










25



use vortex_scan::ScanBuilder;










26












27



use crate::{V1_FOOTER_FBS_SIZE, VERSION, VortexFile, VortexOpenOptions, VortexWriteOptions};










28












29



#[test]










30



fn test_eof_values() {




1✔





31



    // this test exists as a reminder to think about whether we should increment the version










32



    // when we change the footer










33



    assert_eq!(VERSION, 1);




1✔





34



    assert_eq!(V1_FOOTER_FBS_SIZE, 32);




1✔





35



}




1✔





36












37



#[test]










38



#[cfg_attr(miri, ignore)]











39



fn test_read_simple() {




1✔





40



    let strings = ChunkedArray::from_iter([




1✔





41



        VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(),




1✔





42



        VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(),




1✔





43



    ])




1✔





44



    .into_array();




1✔





45












46



    let numbers = ChunkedArray::from_iter([




1✔





47



        buffer![1u32, 2, 3, 4].into_array(),




1✔





48



        buffer![5u32, 6, 7, 8].into_array(),




1✔





49



    ])




1✔





50



    .into_array();




1✔





51












52



    let st = StructArray::from_fields(&[("strings", strings), ("numbers", numbers)]).unwrap();




1✔






53



    let buf =




1✔






54



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






55



            .unwrap();




1✔





56













57



    let iter = VortexOpenOptions::in_memory()




1✔





58



        .open(buf)




1✔





59



        .unwrap()




1✔





60



        .scan()




1✔





61



        .unwrap()




1✔






62



        .into_array_iter()




1✔





63



        .unwrap();




1✔





64












65



    let mut row_count = 0;




1✔





66













67



    for array in iter {




2✔





68



        let array = array.unwrap();




1✔





69



        row_count += array.len();




1✔





70



    }




1✔





71












72



    assert_eq!(row_count, 8);




1✔





73



}




1✔





74












75



#[test]










76



#[cfg_attr(miri, ignore)]











77



fn test_round_trip_many_types() {




1✔





78



    let strings = VarBinArray::from(vec!["ab", "foo", "bar"]).into_array();




1✔





79












80



    let numbers = buffer![1u32, 2, 3].into_array();




1✔





81












82



    let decimal_2 = DecimalArray::new(




1✔





83



        buffer![100i8, 10i8, 2i8],




1✔





84



        DecimalDType::new(2, 1),




1✔





85



        Validity::from_iter([false, true, false]),




1✔





86



    )










87



    .into_array();




1✔





88












89



    let decimal_4 = DecimalArray::new(




1✔





90



        buffer![100i16, 10i16, 2i16],




1✔





91



        DecimalDType::new(4, 2),




1✔





92



        Validity::from_iter([false, true, false]),




1✔





93



    )










94



    .into_array();




1✔





95












96



    let decimal_9 = DecimalArray::new(




1✔





97



        buffer![100i32, 10i32, 2i32],




1✔





98



        DecimalDType::new(9, 2),




1✔





99



        Validity::from_iter([false, true, false]),




1✔





100



    )










101



    .into_array();




1✔





102












103



    let decimal_17 = DecimalArray::new(




1✔





104



        buffer![100i64, 10i64, 20234i64],




1✔





105



        DecimalDType::new(17, 2),




1✔





106



        Validity::from_iter([false, true, false]),




1✔





107



    )










108



    .into_array();




1✔





109












110



    let decimal_35 = DecimalArray::new(




1✔





111



        buffer![100i128, 139348340i128, 23943942i128],




1✔





112



        DecimalDType::new(35, 2),




1✔





113



        Validity::from_iter([true, false, false]),




1✔





114



    )










115



    .into_array();




1✔





116












117



    let st = StructArray::from_fields(&[




1✔





118



        ("strings", strings),




1✔





119



        ("numbers", numbers),




1✔





120



        ("decimal_2", decimal_2),




1✔





121



        ("decimal_4", decimal_4),




1✔





122



        ("decimal_9", decimal_9),




1✔





123



        ("decimal_17", decimal_17),




1✔





124



        ("decimal_35", decimal_35),




1✔





125



    ])




1✔





126



    .unwrap();




1✔






127



    let buf =




1✔






128



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






129



            .unwrap();




1✔





130













131



    let iter = VortexOpenOptions::in_memory()




1✔





132



        .open(buf)




1✔





133



        .unwrap()




1✔





134



        .scan()




1✔





135



        .unwrap()




1✔






136



        .into_array_iter()




1✔





137



        .unwrap();




1✔





138













139



    let chunks = iter.collect::<Vec<_>>();




1✔





140



    let chunks = chunks




1✔





141



        .into_iter()




1✔





142



        .collect::<VortexResult<Vec<_>>>()




1✔





143



        .unwrap();




1✔





144



    let read = ChunkedArray::try_new(chunks, st.dtype().clone()).unwrap();




1✔





145












146



    assert_eq!(read.len(), 3);




1✔





147



}




1✔





148












149



#[test]










150



#[cfg_attr(miri, ignore)]











151



fn test_read_simple_with_spawn() {




1✔





152



    let strings = ChunkedArray::from_iter([




1✔





153



        VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(),




1✔





154



        VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(),




1✔





155



    ])




1✔





156



    .into_array();




1✔





157












158



    let numbers = ChunkedArray::from_iter([




1✔





159



        buffer![1u32, 2, 3, 4].into_array(),




1✔





160



        buffer![5u32, 6, 7, 8].into_array(),




1✔





161



    ])




1✔





162



    .into_array();




1✔





163












164



    let lists = ChunkedArray::from_iter([




1✔





165



        ListArray::from_iter_slow::<i16, _>(




1✔





166



            vec![vec![11, 12], vec![21, 22], vec![31, 32], vec![41, 42]],




1✔





167



            Arc::new(I32.into()),




1✔





168



        )




1✔





169



        .unwrap(),




1✔





170



        ListArray::from_iter_slow::<i8, _>(




1✔





171



            vec![vec![51, 52], vec![61, 62], vec![71, 72], vec![81, 82]],




1✔





172



            Arc::new(I32.into()),




1✔





173



        )




1✔





174



        .unwrap(),




1✔





175



    ])




1✔





176



    .into_array();




1✔





177












178



    let st =




1✔





179



        StructArray::from_fields(&[("strings", strings), ("numbers", numbers), ("lists", lists)])




1✔





180



            .unwrap();




1✔





181













182



    let buf =




1✔






183



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






184



            .unwrap();




1✔





185












186



    assert!(!buf.is_empty());




1✔





187



}




1✔





188












189



#[test]










190



#[cfg_attr(miri, ignore)]











191



fn test_read_projection() {




1✔





192



    let strings_expected = ["ab", "foo", "bar", "baz", "ab", "foo", "bar", "baz"];




1✔





193



    let strings = ChunkedArray::from_iter([




1✔





194



        VarBinArray::from(strings_expected[..4].to_vec()).into_array(),




1✔





195



        VarBinArray::from(strings_expected[4..].to_vec()).into_array(),




1✔





196



    ])




1✔





197



    .into_array();




1✔





198



    let strings_dtype = strings.dtype().clone();




1✔





199












200



    let numbers_expected = [1u32, 2, 3, 4, 5, 6, 7, 8];




1✔





201



    let numbers = ChunkedArray::from_iter([




1✔





202



        Buffer::copy_from(&numbers_expected[..4]).into_array(),




1✔





203



        Buffer::copy_from(&numbers_expected[4..]).into_array(),




1✔





204



    ])




1✔





205



    .into_array();




1✔





206



    let numbers_dtype = numbers.dtype().clone();




1✔





207












208



    let st = StructArray::from_fields(&[("strings", strings), ("numbers", numbers)]).unwrap();




1✔





209













210



    let buf =




1✔






211



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






212



            .unwrap();




1✔





213













214



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





215



    let array = file




1✔





216



        .scan()




1✔





217



        .unwrap()




1✔





218



        .with_projection(select(["strings"], root()))




1✔






219



        .into_array_iter()




1✔





220



        .unwrap()




1✔





221



        .read_all()




1✔





222



        .unwrap();




1✔





223












224



    assert_eq!(




1✔





225



        array.dtype(),




1✔





226



        &DType::Struct(




1✔






227



            StructFields::new(vec!["strings".into()].into(), vec![strings_dtype]),




1✔





228



            Nullability::NonNullable,




1✔





229



        )




1✔





230



    );










231












232



    let actual = array.to_struct().unwrap().fields()[0]




1✔





233



        .to_varbinview()




1✔





234



        .unwrap()




1✔





235



        .with_iterator(|x| {




1✔





236



            x.map(|x| unsafe { String::from_utf8_unchecked(x.unwrap().to_vec()) })




8✔





237



                .collect::<Vec<_>>()




1✔





238



        })




1✔





239



        .unwrap();




1✔





240



    assert_eq!(actual, strings_expected);




1✔





241












242



    let array = file




1✔





243



        .scan()




1✔





244



        .unwrap()




1✔





245



        .with_projection(select(["numbers"], root()))




1✔






246



        .into_array_iter()




1✔





247



        .unwrap()




1✔





248



        .read_all()




1✔





249



        .unwrap();




1✔





250












251



    assert_eq!(




1✔





252



        array.dtype(),




1✔





253



        &DType::Struct(




1✔






254



            StructFields::new(["numbers"].into(), vec![numbers_dtype]),




1✔





255



            Nullability::NonNullable,




1✔





256



        )




1✔





257



    );










258












259



    let primitive_array = array.to_struct().unwrap().fields()[0]




1✔





260



        .to_primitive()




1✔





261



        .unwrap();




1✔





262



    let actual = primitive_array.as_slice::<u32>();




1✔





263



    assert_eq!(actual, numbers_expected);




1✔





264



}




1✔





265












266



#[test]










267



#[cfg_attr(miri, ignore)]











268



fn unequal_batches() {




1✔





269



    let strings = ChunkedArray::from_iter([




1✔





270



        VarBinArray::from(vec!["ab", "foo", "bar", "bob"]).into_array(),




1✔





271



        VarBinArray::from(vec!["baz", "ab", "foo", "bar", "baz", "alice"]).into_array(),




1✔





272



    ])




1✔





273



    .into_array();




1✔





274












275



    let numbers = ChunkedArray::from_iter([




1✔





276



        buffer![1u32, 2, 3, 4, 5].into_array(),




1✔





277



        buffer![6u32, 7, 8, 9, 10].into_array(),




1✔





278



    ])




1✔





279



    .into_array();




1✔





280












281



    let st = StructArray::from_fields(&[("strings", strings), ("numbers", numbers)]).unwrap();




1✔






282



    let buf =




1✔






283



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






284



            .unwrap();




1✔





285













286



    let iter = VortexOpenOptions::in_memory()




1✔






287



        .open(buf)




1✔






288



        .unwrap()




1✔






289



        .scan()




1✔






290



        .unwrap()




1✔






291



        .into_array_iter()




1✔






292



        .unwrap();




1✔





293












294



    let mut item_count = 0;




1✔





295













296



    for array in iter {




2✔





297



        let array = array.unwrap();




1✔





298



        item_count += array.len();




1✔





299












300



        let numbers = array




1✔





301



            .to_struct()




1✔





302



            .unwrap()




1✔





303



            .field_by_name("numbers")




1✔





304



            .unwrap()




1✔





305



            .to_primitive()




1✔





306



            .unwrap();




1✔





307



        assert_eq!(numbers.ptype(), PType::U32);




1✔





308



    }










309



    assert_eq!(item_count, 10);




1✔





310



}




1✔





311












312



#[test]










313



#[cfg_attr(miri, ignore)]











314



fn write_chunked() {




1✔





315



    let strings = VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array();




1✔





316



    let string_dtype = strings.dtype().clone();




1✔





317



    let strings_chunked = ChunkedArray::try_new(iter::repeat_n(strings, 4).collect(), string_dtype)




1✔





318



        .unwrap()




1✔





319



        .into_array();




1✔





320



    let numbers = buffer![1u32, 2, 3, 4].into_array();




1✔





321



    let numbers_dtype = numbers.dtype().clone();




1✔





322



    let numbers_chunked =




1✔





323



        ChunkedArray::try_new(iter::repeat_n(numbers, 4).collect(), numbers_dtype)




1✔





324



            .unwrap()




1✔





325



            .into_array();




1✔





326



    let st = StructArray::try_new(




1✔





327



        ["strings", "numbers"].into(),




1✔





328



        vec![strings_chunked, numbers_chunked],




1✔





329



        16,










330



        Validity::NonNullable,




1✔





331



    )










332



    .unwrap()




1✔





333



    .into_array();




1✔





334



    let st_dtype = st.dtype().clone();




1✔





335












336



    let chunked_st = ChunkedArray::try_new(iter::repeat_n(st, 3).collect(), st_dtype)




1✔





337



        .unwrap()




1✔





338



        .into_array();




1✔






339



    let buf = block_on(




1✔






340



        VortexWriteOptions::default().write(ByteBufferMut::empty(), chunked_st.to_array_stream()),




1✔





341



    )











342



    .unwrap();




1✔





343













344



    let iter = VortexOpenOptions::in_memory()




1✔






345



        .open(buf)




1✔






346



        .unwrap()




1✔






347



        .scan()




1✔






348



        .unwrap()




1✔






349



        .into_array_iter()




1✔






350



        .unwrap();




1✔





351



    let mut array_len: usize = 0;




1✔






352



    for array in iter {




2✔





353



        array_len += array.unwrap().len();




1✔





354



    }




1✔





355



    assert_eq!(array_len, 48);




1✔





356



}




1✔





357












358



#[test]










359



#[cfg_attr(miri, ignore)]











360



fn test_empty_varbin_array_roundtrip() {




1✔





361



    let empty = VarBinArray::from(Vec::<&str>::new()).into_array();




1✔





362












363



    let st = StructArray::from_fields(&[("a", empty)]).unwrap();




1✔





364













365



    let buf =




1✔






366



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






367



            .unwrap();




1✔





368













369



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





370












371



    let result = file




1✔





372



        .scan()




1✔





373



        .unwrap()




1✔






374



        .into_array_iter()




1✔





375



        .unwrap()




1✔





376



        .read_all()




1✔





377



        .unwrap();




1✔





378












379



    assert_eq!(result.len(), 0);




1✔





380



    assert_eq!(result.dtype(), st.dtype());




1✔





381



}




1✔





382












383



#[test]










384



#[cfg_attr(miri, ignore)]











385



fn filter_string() {




1✔





386



    let names_orig = VarBinArray::from_iter(




1✔





387



        vec![Some("Joseph"), None, Some("Angela"), Some("Mikhail"), None],




1✔





388



        DType::Utf8(Nullability::Nullable),




1✔





389



    )










390



    .into_array();




1✔





391



    let ages_orig =




1✔





392



        PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]).into_array();




1✔





393



    let st = StructArray::try_new(




1✔





394



        ["name", "age"].into(),




1✔





395



        vec![names_orig, ages_orig],




1✔





396



        5,










397



        Validity::NonNullable,




1✔





398



    )










399



    .unwrap()




1✔





400



    .into_array();




1✔






401



    let buf =




1✔






402



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






403



            .unwrap();




1✔





404













405



    let result: Vec<_> = VortexOpenOptions::in_memory()




1✔





406



        .open(buf)




1✔





407



        .unwrap()




1✔





408



        .scan()




1✔





409



        .unwrap()




1✔





410



        .with_filter(eq(get_item("name", root()), lit("Joseph")))




1✔






411



        .into_array_iter()




1✔





412



        .unwrap()




1✔






413



        .try_collect()




1✔





414



        .unwrap();




1✔





415












416



    assert_eq!(result.len(), 1);




1✔





417



    let names = result[0].to_struct().unwrap().fields()[0].clone();




1✔





418



    assert_eq!(




1✔





419



        names




1✔





420



            .to_varbinview()




1✔





421



            .unwrap()




1✔





422



            .with_iterator(|iter| iter




1✔





423



                .flatten()




1✔





424



                .map(|s| unsafe { String::from_utf8_unchecked(s.to_vec()) })




1✔





425



                .collect::<Vec<_>>())




1✔





426



            .unwrap(),




1✔





427



        vec!["Joseph".to_string()]




1✔





428



    );










429



    let ages = result[0].to_struct().unwrap().fields()[1].clone();




1✔





430



    assert_eq!(ages.to_primitive().unwrap().as_slice::<i32>(), vec![25]);




1✔





431



}




1✔





432












433



#[test]










434



#[cfg_attr(miri, ignore)]











435



fn filter_or() {




1✔





436



    let names = VarBinArray::from_iter(




1✔





437



        vec![Some("Joseph"), None, Some("Angela"), Some("Mikhail"), None],




1✔





438



        DType::Utf8(Nullability::Nullable),




1✔





439



    );










440



    let ages = PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]);




1✔





441



    let st = StructArray::try_new(




1✔





442



        ["name", "age"].into(),




1✔





443



        vec![names.into_array(), ages.into_array()],




1✔





444



        5,










445



        Validity::NonNullable,




1✔





446



    )










447



    .unwrap()




1✔





448



    .into_array();




1✔





449













450



    let buf =




1✔






451



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






452



            .unwrap();




1✔





453













454



    let result: Vec<_> = VortexOpenOptions::in_memory()




1✔





455



        .open(buf)




1✔





456



        .unwrap()




1✔





457



        .scan()




1✔





458



        .unwrap()




1✔





459



        .with_filter(or(




1✔





460



            eq(get_item("name", root()), lit("Angela")),




1✔





461



            and(




1✔





462



                gt_eq(get_item("age", root()), lit(20)),




1✔





463



                lt_eq(get_item("age", root()), lit(30)),




1✔





464



            ),




1✔





465



        ))




1✔






466



        .into_array_iter()




1✔





467



        .unwrap()




1✔






468



        .try_collect()




1✔





469



        .unwrap();




1✔





470












471



    assert_eq!(result.len(), 1);




1✔





472



    let names = result[0].to_struct().unwrap().fields()[0].clone();




1✔





473



    assert_eq!(




1✔





474



        names




1✔





475



            .to_varbinview()




1✔





476



            .unwrap()




1✔





477



            .with_iterator(|iter| iter




1✔





478



                .flatten()




1✔





479



                .map(|s| unsafe { String::from_utf8_unchecked(s.to_vec()) })




2✔





480



                .collect::<Vec<_>>())




1✔





481



            .unwrap(),




1✔





482



        vec!["Joseph".to_string(), "Angela".to_string()]




1✔





483



    );










484



    let ages = result[0].to_struct().unwrap().fields()[1].clone();




1✔





485



    assert_eq!(




1✔





486



        ages.to_primitive()




1✔





487



            .unwrap()




1✔





488



            .with_iterator(|iter| iter.map(|x| x.cloned()).collect::<Vec<_>>())




2✔





489



            .unwrap(),




1✔





490



        vec![Some(25), None]




1✔





491



    );










492



}




1✔





493












494



#[test]










495



#[cfg_attr(miri, ignore)]











496



fn filter_and() {




1✔





497



    let names = VarBinArray::from_iter(




1✔





498



        vec![Some("Joseph"), None, Some("Angela"), Some("Mikhail"), None],




1✔





499



        DType::Utf8(Nullability::Nullable),




1✔





500



    );










501



    let ages = PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]);




1✔





502



    let st = StructArray::try_new(




1✔





503



        ["name", "age"].into(),




1✔





504



        vec![names.into_array(), ages.into_array()],




1✔





505



        5,










506



        Validity::NonNullable,




1✔





507



    )










508



    .unwrap()




1✔





509



    .into_array();




1✔





510













511



    let buf =




1✔






512



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






513



            .unwrap();




1✔





514













515



    let result: Vec<_> = VortexOpenOptions::in_memory()




1✔





516



        .open(buf)




1✔





517



        .unwrap()




1✔





518



        .scan()




1✔





519



        .unwrap()




1✔





520



        .with_filter(and(




1✔





521



            gt(get_item("age", root()), lit(21)),




1✔





522



            lt_eq(get_item("age", root()), lit(33)),




1✔





523



        ))




1✔






524



        .into_array_iter()




1✔





525



        .unwrap()




1✔






526



        .try_collect()




1✔





527



        .unwrap();




1✔





528












529



    assert_eq!(result.len(), 1);




1✔





530



    let names = result[0].to_struct().unwrap().fields()[0].clone();




1✔





531



    assert_eq!(




1✔





532



        names




1✔





533



            .to_varbinview()




1✔





534



            .unwrap()




1✔





535



            .with_iterator(|iter| iter




1✔





536



                .map(|s| s.map(|st| unsafe { String::from_utf8_unchecked(st.to_vec()) }))




2✔





537



                .collect::<Vec<_>>())




1✔





538



            .unwrap(),




1✔





539



        vec![Some("Joseph".to_string()), None]




1✔





540



    );










541



    let ages = result[0].to_struct().unwrap().fields()[1].clone();




1✔





542



    assert_eq!(ages.to_primitive().unwrap().as_slice::<i32>(), vec![25, 31]);




1✔





543



}




1✔





544












545



#[test]










546



#[cfg_attr(miri, ignore)]











547



fn test_with_indices_simple() {




1✔





548



    let expected_numbers_split: Vec<Buffer<i16>> = (0..5).map(|_| (0_i16..100).collect()).collect();




5✔





549



    let expected_array = StructArray::from_fields(&[(




1✔





550



        "numbers",




1✔





551



        ChunkedArray::from_iter(




1✔





552



            expected_numbers_split




1✔





553



                .iter()




1✔





554



                .cloned()




1✔





555



                .map(IntoArray::into_array),




1✔





556



        )




1✔





557



        .into_array(),




1✔





558



    )])




1✔





559



    .unwrap();




1✔





560



    let expected_numbers: Vec<i16> = expected_numbers_split.into_iter().flatten().collect();




1✔





561













562



    let buf = block_on(




1✔






563



        VortexWriteOptions::default()




1✔






564



            .write(ByteBufferMut::empty(), expected_array.to_array_stream()),




1✔





565



    )











566



    .unwrap();




1✔





567













568



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





569












570



    // test no indices










571



    let actual_kept_array = file




1✔





572



        .scan()




1✔





573



        .unwrap()




1✔





574



        .with_row_indices(Buffer::<u64>::empty())




1✔






575



        .into_array_iter()




1✔





576



        .unwrap()




1✔





577



        .read_all()




1✔





578



        .unwrap()




1✔





579



        .to_struct()




1✔





580



        .unwrap();




1✔





581












582



    assert_eq!(actual_kept_array.len(), 0);




1✔





583












584



    // test a few indices










585



    let kept_indices = [0_u64, 3, 99, 100, 101, 399, 400, 401, 499];




1✔





586












587



    let actual_kept_array = file




1✔





588



        .scan()




1✔





589



        .unwrap()




1✔





590



        .with_row_indices(Buffer::from_iter(kept_indices))




1✔






591



        .into_array_iter()




1✔





592



        .unwrap()




1✔





593



        .read_all()




1✔





594



        .unwrap()




1✔





595



        .to_struct()




1✔





596



        .unwrap();




1✔





597



    let actual_kept_numbers_array = actual_kept_array.fields()[0].to_primitive().unwrap();




1✔





598












599



    let expected_kept_numbers: Vec<i16> = kept_indices




1✔





600



        .iter()




1✔





601



        .map(|&x| expected_numbers[x as usize])




9✔





602



        .collect();




1✔





603



    let actual_kept_numbers = actual_kept_numbers_array.as_slice::<i16>();




1✔





604












605



    assert_eq!(expected_kept_numbers, actual_kept_numbers);




1✔





606












607



    // test all indices










608



    let actual_array = file




1✔





609



        .scan()




1✔





610



        .unwrap()




1✔





611



        .with_row_indices((0u64..500).collect::<Buffer<_>>())




1✔






612



        .into_array_iter()




1✔





613



        .unwrap()




1✔





614



        .read_all()




1✔





615



        .unwrap()




1✔





616



        .to_struct()




1✔





617



        .unwrap();




1✔





618



    let actual_numbers_array = actual_array.fields()[0].to_primitive().unwrap();




1✔





619



    let actual_numbers = actual_numbers_array.as_slice::<i16>();




1✔





620












621



    assert_eq!(expected_numbers, actual_numbers);




1✔





622



}




1✔





623












624



#[test]










625



#[cfg_attr(miri, ignore)]











626



fn test_with_indices_on_two_columns() {




1✔





627



    let strings_expected = ["ab", "foo", "bar", "baz", "ab", "foo", "bar", "baz"];




1✔





628



    let strings = ChunkedArray::from_iter([




1✔





629



        VarBinArray::from(strings_expected[..4].to_vec()).into_array(),




1✔





630



        VarBinArray::from(strings_expected[4..].to_vec()).into_array(),




1✔





631



    ])




1✔





632



    .into_array();




1✔





633












634



    let numbers_expected = [1u32, 2, 3, 4, 5, 6, 7, 8];




1✔





635



    let numbers = ChunkedArray::from_iter([




1✔





636



        Buffer::copy_from(&numbers_expected[..4]).into_array(),




1✔





637



        Buffer::copy_from(&numbers_expected[4..]).into_array(),




1✔





638



    ])




1✔





639



    .into_array();




1✔





640












641



    let st = StructArray::from_fields(&[("strings", strings), ("numbers", numbers)]).unwrap();




1✔






642



    let buf =




1✔






643



        block_on(VortexWriteOptions::default().write(ByteBufferMut::empty(), st.to_array_stream()))




1✔






644



            .unwrap();




1✔





645













646



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





647












648



    let kept_indices = [0_u64, 3, 7];




1✔





649



    let array = file




1✔





650



        .scan()




1✔





651



        .unwrap()




1✔





652



        .with_row_indices(Buffer::from_iter(kept_indices))




1✔






653



        .into_array_iter()




1✔





654



        .unwrap()




1✔





655



        .read_all()




1✔





656



        .unwrap()




1✔





657



        .to_struct()




1✔





658



        .unwrap()




1✔





659



        .to_struct()




1✔





660



        .unwrap();




1✔





661












662



    let strings_actual = array.fields()[0]




1✔





663



        .to_varbinview()




1✔





664



        .unwrap()




1✔





665



        .with_iterator(|x| {




1✔





666



            x.map(|x| unsafe { String::from_utf8_unchecked(x.unwrap().to_vec()) })




3✔





667



                .collect::<Vec<_>>()




1✔





668



        })




1✔





669



        .unwrap();




1✔





670



    assert_eq!(




1✔





671



        strings_actual,










672



        kept_indices




1✔





673



            .iter()




1✔





674



            .map(|&x| strings_expected[x as usize])




3✔





675



            .collect::<Vec<_>>()




1✔





676



    );










677












678



    let numbers_actual_array = array.fields()[1].to_primitive().unwrap();




1✔





679



    let numbers_actual = numbers_actual_array.as_slice::<u32>();




1✔





680



    assert_eq!(




1✔





681



        numbers_actual,










682



        kept_indices




1✔





683



            .iter()




1✔





684



            .map(|&x| numbers_expected[x as usize])




3✔





685



            .collect::<Vec<u32>>()




1✔





686



    );










687



}




1✔





688












689



#[test]










690



#[cfg_attr(miri, ignore)]











691



fn test_with_indices_and_with_row_filter_simple() {




1✔





692



    let expected_numbers_split: Vec<Buffer<i16>> = (0..5).map(|_| (0_i16..100).collect()).collect();




5✔





693



    let expected_array = StructArray::from_fields(&[(




1✔





694



        "numbers",




1✔





695



        ChunkedArray::from_iter(




1✔





696



            expected_numbers_split




1✔





697



                .iter()




1✔





698



                .cloned()




1✔





699



                .map(IntoArray::into_array),




1✔





700



        )




1✔





701



        .into_array(),




1✔





702



    )])




1✔





703



    .unwrap();




1✔





704



    let expected_numbers: Vec<i16> = expected_numbers_split.into_iter().flatten().collect();




1✔





705













706



    let buf = block_on(




1✔






707



        VortexWriteOptions::default()




1✔






708



            .write(ByteBufferMut::empty(), expected_array.to_array_stream()),




1✔





709



    )











710



    .unwrap();




1✔





711













712



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





713












714



    let actual_kept_array = file




1✔





715



        .scan()




1✔





716



        .unwrap()




1✔





717



        .with_filter(gt(get_item("numbers", root()), lit(50_i16)))




1✔





718



        .with_row_indices(Buffer::empty())




1✔






719



        .into_array_iter()




1✔





720



        .unwrap()




1✔





721



        .read_all()




1✔





722



        .unwrap()




1✔





723



        .to_struct()




1✔





724



        .unwrap();




1✔





725












726



    assert_eq!(actual_kept_array.len(), 0);




1✔





727












728



    // test a few indices










729



    let kept_indices = [0u64, 3, 99, 100, 101, 399, 400, 401, 499];




1✔





730












731



    let actual_kept_array = file




1✔





732



        .scan()




1✔





733



        .unwrap()




1✔





734



        .with_filter(gt(get_item("numbers", root()), lit(50_i16)))




1✔





735



        .with_row_indices(Buffer::from_iter(kept_indices))




1✔






736



        .into_array_iter()




1✔





737



        .unwrap()




1✔





738



        .read_all()




1✔





739



        .unwrap()




1✔





740



        .to_struct()




1✔





741



        .unwrap();




1✔





742












743



    let actual_kept_numbers_array = actual_kept_array.fields()[0].to_primitive().unwrap();




1✔





744












745



    let expected_kept_numbers: Buffer<i16> = kept_indices




1✔





746



        .iter()




1✔





747



        .map(|&x| expected_numbers[x as usize])




9✔





748



        .filter(|&x| x > 50)




9✔





749



        .collect();




1✔





750



    let actual_kept_numbers = actual_kept_numbers_array.as_slice::<i16>();




1✔





751












752



    assert_eq!(expected_kept_numbers.as_slice(), actual_kept_numbers);




1✔





753












754



    // test all indices










755



    let actual_array = file




1✔





756



        .scan()




1✔





757



        .unwrap()




1✔





758



        .with_filter(gt(get_item("numbers", root()), lit(50_i16)))




1✔





759



        .with_row_indices((0..500).collect::<Buffer<_>>())




1✔






760



        .into_array_iter()




1✔





761



        .unwrap()




1✔





762



        .read_all()




1✔





763



        .unwrap()




1✔





764



        .to_struct()




1✔





765



        .unwrap();




1✔





766












767



    let actual_numbers_array = actual_array.fields()[0].to_primitive().unwrap();




1✔





768



    let actual_numbers = actual_numbers_array.as_slice::<i16>();




1✔





769












770



    assert_eq!(




1✔





771



        expected_numbers




1✔





772



            .iter()




1✔





773



            .filter(|&&x| x > 50)




500✔





774



            .cloned()




1✔





775



            .collect::<Vec<_>>(),




1✔





776



        actual_numbers










777



    );










778



}




1✔





779












780



#[test]










781



#[cfg_attr(miri, ignore)]











782



fn filter_string_chunked() {




1✔





783



    let name_chunk1 =




1✔





784



        VarBinViewArray::from_iter_nullable_str([Some("Joseph"), Some("James"), Some("Angela")])




1✔





785



            .into_array();




1✔





786



    let age_chunk1 = PrimitiveArray::from_option_iter([Some(25_i32), Some(31), None]).into_array();




1✔





787



    let name_chunk2 = VarBinViewArray::from_iter_nullable_str([




1✔





788



        Some("Pharrell".to_owned()),




1✔





789



        Some("Khalil".to_owned()),




1✔





790



        Some("Mikhail".to_owned()),




1✔





791



        None,




1✔





792



    ])




1✔





793



    .into_array();




1✔





794



    let age_chunk2 =




1✔





795



        PrimitiveArray::from_option_iter([Some(57_i32), Some(18), None, Some(32)]).into_array();




1✔





796












797



    let chunk1 = StructArray::from_fields(&[("name", name_chunk1), ("age", age_chunk1)])




1✔





798



        .unwrap()




1✔





799



        .into_array();




1✔





800



    let chunk2 = StructArray::from_fields(&[("name", name_chunk2), ("age", age_chunk2)])




1✔





801



        .unwrap()




1✔





802



        .into_array();




1✔





803



    let dtype = chunk1.dtype().clone();




1✔





804












805



    let array = ChunkedArray::try_new(vec![chunk1, chunk2], dtype)




1✔





806



        .unwrap()




1✔





807



        .into_array();




1✔





808













809



    let buf = block_on(




1✔






810



        VortexWriteOptions::default().write(ByteBufferMut::empty(), array.to_array_stream()),




1✔





811



    )











812



    .unwrap();




1✔





813













814



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





815












816



    let actual_array = file




1✔





817



        .scan()




1✔





818



        .unwrap()




1✔





819



        .with_filter(eq(get_item("name", root()), lit("Joseph")))




1✔






820



        .into_array_iter()




1✔





821



        .unwrap()




1✔





822



        .read_all()




1✔





823



        .unwrap()




1✔





824



        .to_struct()




1✔





825



        .unwrap();




1✔





826












827



    assert_eq!(actual_array.len(), 1);




1✔





828



    let names = &actual_array.fields()[0];




1✔





829



    assert_eq!(




1✔





830



        names




1✔





831



            .to_varbinview()




1✔





832



            .unwrap()




1✔





833



            .with_iterator(|iter| iter




1✔





834



                .flatten()




1✔





835



                .map(|s| unsafe { String::from_utf8_unchecked(s.to_vec()) })




1✔





836



                .collect::<Vec<_>>())




1✔





837



            .unwrap(),




1✔





838



        vec!["Joseph".to_string()]




1✔





839



    );










840



    let ages = &actual_array.fields()[1];




1✔





841



    assert_eq!(ages.to_primitive().unwrap().as_slice::<i32>(), vec![25]);




1✔





842



}




1✔





843












844



#[test]










845



#[cfg_attr(miri, ignore)]











846



fn test_pruning_with_or() {




1✔





847



    let letter_chunk1 = VarBinViewArray::from_iter_nullable_str([




1✔





848



        Some("A".to_owned()),




1✔





849



        Some("B".to_owned()),




1✔





850



        Some("D".to_owned()),




1✔





851



    ])




1✔





852



    .into_array();




1✔





853



    let number_chunk1 =




1✔





854



        PrimitiveArray::from_option_iter([Some(25_i32), Some(31), None]).into_array();




1✔





855



    let letter_chunk2 = VarBinViewArray::from_iter_nullable_str([




1✔





856



        Some("G".to_owned()),




1✔





857



        Some("I".to_owned()),




1✔





858



        Some("J".to_owned()),




1✔





859



        None,




1✔





860



    ])




1✔





861



    .into_array();




1✔





862



    let number_chunk2 =




1✔





863



        PrimitiveArray::from_option_iter([Some(4_i32), Some(18), None, Some(21)]).into_array();




1✔





864



    let letter_chunk3 = VarBinViewArray::from_iter_nullable_str([




1✔





865



        Some("L".to_owned()),




1✔





866



        None,




1✔





867



        Some("O".to_owned()),




1✔





868



        Some("P".to_owned()),




1✔





869



    ])




1✔





870



    .into_array();




1✔





871



    let number_chunk3 =




1✔





872



        PrimitiveArray::from_option_iter([Some(10_i32), Some(15), None, Some(22)]).into_array();




1✔





873



    let letter_chunk4 = VarBinViewArray::from_iter_nullable_str([




1✔





874



        Some("X".to_owned()),




1✔





875



        Some("Y".to_owned()),




1✔





876



        Some("Z".to_owned()),




1✔





877



    ])




1✔





878



    .into_array();




1✔





879



    let number_chunk4 =




1✔





880



        PrimitiveArray::from_option_iter([Some(66_i32), Some(77), Some(88)]).into_array();




1✔





881












882



    let chunk1 = StructArray::from_fields(&[("letter", letter_chunk1), ("number", number_chunk1)])




1✔





883



        .unwrap()




1✔





884



        .into_array();




1✔





885



    let chunk2 = StructArray::from_fields(&[("letter", letter_chunk2), ("number", number_chunk2)])




1✔





886



        .unwrap()




1✔





887



        .into_array();




1✔





888



    let chunk3 = StructArray::from_fields(&[("letter", letter_chunk3), ("number", number_chunk3)])




1✔





889



        .unwrap()




1✔





890



        .into_array();




1✔





891



    let chunk4 = StructArray::from_fields(&[("letter", letter_chunk4), ("number", number_chunk4)])




1✔





892



        .unwrap()




1✔





893



        .into_array();




1✔





894



    let dtype = chunk1.dtype().clone();




1✔





895












896



    let array = ChunkedArray::try_new(vec![chunk1, chunk2, chunk3, chunk4], dtype)




1✔





897



        .unwrap()




1✔





898



        .into_array();




1✔





899













900



    let buf = block_on(




1✔






901



        VortexWriteOptions::default().write(ByteBufferMut::empty(), array.to_array_stream()),




1✔





902



    )











903



    .unwrap();




1✔





904













905



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





906












907



    let actual_array = file




1✔





908



        .scan()




1✔





909



        .unwrap()




1✔





910



        .with_filter(or(




1✔





911



            lt_eq(get_item("letter", root()), lit("J")),




1✔





912



            lt(get_item("number", root()), lit(25)),




1✔





913



        ))




1✔






914



        .into_array_iter()




1✔





915



        .unwrap()




1✔





916



        .read_all()




1✔





917



        .unwrap()




1✔





918



        .to_struct()




1✔





919



        .unwrap();




1✔





920












921



    assert_eq!(actual_array.len(), 10);




1✔





922



    let letters = &actual_array.fields()[0];




1✔





923



    assert_eq!(




1✔





924



        letters




1✔





925



            .to_varbinview()




1✔





926



            .unwrap()




1✔





927



            .with_iterator(|iter| iter




1✔





928



                .map(|opt| opt.map(|s| unsafe { String::from_utf8_unchecked(s.to_vec()) }))




10✔





929



                .collect::<Vec<_>>())




1✔





930



            .unwrap(),




1✔





931



        vec![




1✔





932



            Some("A".to_string()),




1✔





933



            Some("B".to_string()),




1✔





934



            Some("D".to_string()),




1✔





935



            Some("G".to_string()),




1✔





936



            Some("I".to_string()),




1✔





937



            Some("J".to_string()),




1✔





938



            None,




1✔





939



            Some("L".to_string()),




1✔





940



            None,




1✔





941



            Some("P".to_string())




1✔





942



        ]










943



    );










944



    let numbers = &actual_array.fields()[1];




1✔





945



    assert_eq!(




1✔





946



        (0..numbers.len())




1✔





947



            .map(|index| -> Option<i32> {




10✔





948



                numbers




10✔





949



                    .scalar_at(index)




10✔





950



                    .unwrap()




10✔





951



                    .as_primitive()




10✔





952



                    .typed_value::<i32>()




10✔





953



            })




10✔





954



            .collect::<Vec<_>>(),




1✔





955



        vec![




1✔





956



            Some(25),




1✔





957



            Some(31),




1✔





958



            None,




1✔





959



            Some(4),




1✔





960



            Some(18),




1✔





961



            None,




1✔





962



            Some(21),




1✔





963



            Some(10),




1✔





964



            Some(15),




1✔





965



            Some(22)




1✔





966



        ]










967



    );










968



}




1✔





969












970



#[test]











971



fn test_repeated_projection() {




1✔





972



    let strings = ChunkedArray::from_iter([




1✔





973



        VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(),




1✔





974



        VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(),




1✔





975



    ])




1✔





976



    .into_array();




1✔





977












978



    let single_column_array = StructArray::from_fields(&[("strings", strings.clone())])




1✔





979



        .unwrap()




1✔





980



        .into_array();




1✔





981












982



    let expected = StructArray::from_fields(&[("strings", strings.clone()), ("strings", strings)])




1✔





983



        .unwrap()




1✔





984



        .into_array();




1✔





985













986



    let buf = block_on(VortexWriteOptions::default().write(




1✔






987



        ByteBufferMut::empty(),




1✔






988



        single_column_array.to_array_stream(),




1✔





989



    ))











990



    .unwrap();




1✔





991













992



    let file = VortexOpenOptions::in_memory().open(buf).unwrap();




1✔





993












994



    let actual = file




1✔





995



        .scan()




1✔





996



        .unwrap()




1✔





997



        .with_projection(select(["strings", "strings"], root()))




1✔






998



        .into_array_iter()




1✔





999



        .unwrap()




1✔





1000



        .read_all()




1✔





1001



        .unwrap()




1✔





1002



        .to_struct()




1✔





1003



        .unwrap();




1✔





1004












1005



    assert_eq!(




1✔





1006



        (0..actual.len())




1✔





1007



            .map(|index| actual.scalar_at(index).unwrap())




8✔





1008



            .collect_vec(),




1✔





1009



        (0..expected.len())




1✔





1010



            .map(|index| expected.scalar_at(index).unwrap())




8✔





1011



            .collect_vec()




1✔





1012



    );










1013



}




1✔





1014













1015



fn chunked_file() -> VortexResult<VortexFile> {




2✔





1016



    let array = ChunkedArray::from_iter([




2✔





1017



        buffer![0, 1, 2].into_array(),




2✔





1018



        buffer![3, 4, 5].into_array(),




2✔





1019



        buffer![6, 7, 8].into_array(),




2✔





1020



    ])




2✔





1021



    .into_array();




2✔





1022













1023



    let buffer: Bytes =




2✔






1024



        block_on(VortexWriteOptions::default().write(vec![], array.to_array_stream()))?.into();




2✔






1025



    VortexOpenOptions::in_memory().open(buffer)




2✔





1026



}




2✔





1027












1028



#[test]











1029



fn basic_file_roundtrip() -> VortexResult<()> {




1✔






1030



    let vxf = chunked_file()?;




1✔






1031



    let result = vxf.scan()?.into_array_iter()?.read_all()?.to_primitive()?;




1✔





1032












1033



    assert_eq!(result.as_slice::<i32>(), &[0, 1, 2, 3, 4, 5, 6, 7, 8]);




1✔





1034












1035



    Ok(())




1✔





1036



}




1✔





1037












1038



#[test]











1039



fn file_excluding_dtype() -> VortexResult<()> {




1✔





1040



    let array = ChunkedArray::from_iter([




1✔





1041



        buffer![0, 1, 2].into_array(),




1✔





1042



        buffer![3, 4, 5].into_array(),




1✔





1043



        buffer![6, 7, 8].into_array(),




1✔





1044



    ])




1✔





1045



    .into_array();




1✔





1046



    let dtype = array.dtype().clone();




1✔





1047













1048



    let buffer: Bytes = block_on(




1✔






1049



        VortexWriteOptions::default()




1✔






1050



            .exclude_dtype()




1✔






1051



            .write(vec![], array.to_array_stream()),




1✔






NEW


1052



    )?




×








1053



    .into();




1✔





1054












1055



    // Fail to open without DType.











1056



    let vxf = VortexOpenOptions::in_memory().open(buffer.clone());




1✔





1057



    assert!(vxf.is_err(), "Opening without DType should fail");




1✔





1058












1059



    let vxf = VortexOpenOptions::in_memory()




1✔





1060



        .with_dtype(dtype.clone())




1✔






1061



        .open(buffer)?;




1✔





1062



    assert_eq!(vxf.dtype(), &dtype);




1✔





1063



    assert_eq!(vxf.row_count(), 9);




1✔





1064












1065



    Ok(())




1✔





1066



}




1✔





1067












1068



#[test]











1069



fn file_take() -> VortexResult<()> {




1✔






1070



    let vxf = chunked_file()?;




1✔





1071



    let result = vxf




1✔





1072



        .scan()?




1✔





1073



        .with_row_indices(buffer![0, 1, 8])




1✔






1074



        .into_array_iter()?




1✔






1075



        .read_all()?




1✔





1076



        .to_primitive()?;




1✔





1077












1078



    assert_eq!(result.as_slice::<i32>(), &[0, 1, 8]);




1✔





1079












1080



    Ok(())




1✔





1081



}




1✔





1082












1083



#[test]










1084



#[should_panic(










1085



    expected = "FileStatsAccumulator temporarily does not support nullable top-level structs"










1086



)]











1087



fn write_nullable_top_level_struct() {




1✔





1088



    let ages = PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]);




1✔





1089












1090



    let array = StructArray::try_new(




1✔





1091



        ["age"].into(),




1✔





1092



        vec![ages.into_array()],




1✔





1093



        5,










1094



        Validity::AllValid,




1✔





1095



    )










1096



    .unwrap()




1✔





1097



    .into_array();




1✔





1098













1099



    block_on(VortexWriteOptions::default().write(vec![], array.to_array_stream())).unwrap();




1✔





1100



}




1✔





1101













1102



fn round_trip(




2✔





1103



    array: &dyn Array,




2✔





1104



    f: impl Fn(ScanBuilder<ArrayRef>) -> VortexResult<ScanBuilder<ArrayRef>>,




2✔





1105



) -> VortexResult<ArrayRef> {




2✔






1106



    let buffer: Bytes =




2✔






1107



        block_on(VortexWriteOptions::default().write(vec![], array.to_array_stream()))?.into();




2✔





1108












1109



    let vxf = VortexOpenOptions::in_memory()




2✔





1110



        .with_dtype(array.dtype().clone())




2✔






1111



        .open(buffer)?;




2✔





1112












1113



    assert_eq!(vxf.dtype(), array.dtype());




2✔





1114



    assert_eq!(vxf.row_count(), array.len() as u64);




2✔





1115













1116



    f(vxf.scan()?)?.into_array_iter()?.read_all()




2✔





1117



}




2✔





1118












1119



#[test]











1120



fn write_nullable_nested_struct() -> VortexResult<()> {




1✔





1121



    let nested_dtype = DType::struct_(




1✔





1122



        [(




1✔





1123



            "nested_field",




1✔





1124



            DType::Primitive(PType::F16, Nullability::Nullable),




1✔





1125



        )],




1✔





1126



        Nullability::Nullable,




1✔





1127



    );










1128












1129



    let struct_ = ConstantArray::new(Scalar::null(nested_dtype.clone()), 3).to_array();




1✔





1130












1131



    let array = StructArray::try_new(




1✔





1132



        ["struct"].into(),




1✔





1133



        vec![struct_.into_array()],




1✔





1134



        3,










1135



        Validity::NonNullable,




1✔





1136



    )?




×







1137



    .into_array();




1✔





1138













1139



    let result = round_trip(&array, Ok)?.to_struct()?;




1✔





1140












1141



    assert_eq!(result.len(), 3);




1✔





1142



    assert_eq!(result.fields().len(), 1);




1✔





1143



    assert!(result.all_valid()?);




1✔





1144












1145



    let nested_struct = result.field_by_name("struct")?.to_struct()?;




1✔





1146



    assert_eq!(nested_struct.dtype(), &nested_dtype);




1✔





1147



    assert_eq!(nested_struct.len(), 3);




1✔





1148



    assert!(nested_struct.all_invalid()?);




1✔





1149












1150



    Ok(())




1✔





1151



}




1✔





1152












1153



#[test]











1154



fn scan_empty_fields() -> VortexResult<()> {




1✔





1155



    let array = (0..10000).collect::<PrimitiveArray>();




1✔





1156












1157



    let result = round_trip(array.as_ref(), |scan| {




1✔





1158



        Ok(scan.with_projection(PackExpr::try_new_expr(




1✔





1159



            Default::default(),




1✔





1160



            vec![],




1✔





1161



            Nullability::Nullable,




1✔





1162



        )?))




×








1163



    })?;




1✔





1164












1165



    assert_eq!(result.len(), array.len());




1✔





1166












1167



    Ok(())




1✔





1168



}




1✔



















    

  • 

    •