3929938005

Build Type

push

github

Committed by GitHub

Commit Message

Merge #713

Run Details

84930 of 96741 relevant lines covered (87.79%)

79640.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.65

/datatypes/src/raster/operations/map_elements.rs

use crate::raster::{Grid, GridOrEmpty, GridOrEmpty2D, GridSize, MaskedGrid, RasterTile2D};
use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};

const MIN_ELEMENTS_PER_THREAD: usize = 16 * 512;

/// This trait models a map operation from a `Grid` of type `In` into a `Grid` of Type `Out`. This is done using a provided function that maps each element to a new value.
///
/// Most useful implementations are on: `Grid`, `MaskedGrid`, `GridOrEmpty` and `RasterTile2D`.
///
/// On `Grid` elements are mapped, e.g., as `|element: In| { element + 1 }` with `F: Fn(In) -> Out`
///
/// On `MaskedGrid` you can map either only valid data (excluding no data), e.g., `|element: In| { element + 1 }` with `F: Fn(In) -> Out` or handling no data as `|element: Option<In>| { element.map(|e| e+ 1) }` with `F: Fn(Option<In>) -> Option<Out>`.
pub trait MapElements<In, Out, F: Fn(In) -> Out> {
    type Output;
    /// Create a new instance from the current one. The `map_fn` transforms all elements to a new value.
    fn map_elements(self, map_fn: F) -> Self::Output;
}

/// This trait is equal to `MapElements` but uses a thread pool to do the operation in parallel.
/// This trait models a map operation from a `Grid` of type `In` into a `Grid` of Type `Out`. This is done using a provided function that maps each element to a new value.
///
/// Most useful implementations are on: `Grid`, `MaskedGrid`, `GridOrEmpty` and `RasterTile2D`.
///
/// On `Grid` elements are mapped, e.g., as `|element: In| { element + 1 }` with `F: Fn(In) -> Out`
///
/// On `MaskedGrid` you can map either only valid data (excluding no data), e.g., `|element: In| { element + 1 }` with `F: Fn(In) -> Out` or handling no data as `|element: Option<In>| { element.map(|e| e+ 1) }` with `F: Fn(Option<In>) -> Option<Out>`.

pub trait MapElementsParallel<In, Out, F: Fn(In) -> Out> {
    type Output;
    /// Create a new instance from the current one. The `map_fn` transforms all elements to a new value. Use a `ThreadPool` for parallel map operations.
    fn map_elements_parallel(self, map_fn: F) -> Self::Output;
}

// Implementation for Grid: F: Fn(In) -> Out.
impl<In, Out, F, G> MapElements<In, Out, F> for Grid<G, In>
where
    In: 'static,
    Out: 'static,
    G: GridSize + Clone,
    F: Fn(In) -> Out,
{
    type Output = Grid<G, Out>;
    fn map_elements(self, map_fn: F) -> Self::Output {
        let shape = self.shape;
        let data = self.data.into_iter().map(map_fn).collect();

        Grid { shape, data }
    }
}

// Implementation for MaskedGrid to enable update of the inner data with F: Fn(T) -> T.
impl<In, Out, F, G> MapElements<In, Out, F> for MaskedGrid<G, In>
where
    In: 'static + Clone,
    Out: 'static + Clone,
    G: GridSize + Clone + PartialEq,
    F: Fn(In) -> Out,
    Grid<G, In>: MapElements<In, Out, F, Output = Grid<G, Out>>,
{
    type Output = MaskedGrid<G, Out>;

    fn map_elements(self, map_fn: F) -> Self::Output {
        MaskedGrid::new(self.inner_grid.map_elements(map_fn), self.validity_mask)
            .expect("Creation failed for prev valid dimensions")
    }
}

// Implementation for MaskedGrid: F: Fn(Option<T>) -> Option<T>.
impl<In, Out, F, G> MapElements<Option<In>, Option<Out>, F> for MaskedGrid<G, In>
where
    In: 'static + Clone,
    Out: 'static + Clone + Default,
    G: GridSize + Clone + PartialEq,
    F: Fn(Option<In>) -> Option<Out>,
{
    type Output = MaskedGrid<G, Out>;

    fn map_elements(self, map_fn: F) -> Self::Output {
        let MaskedGrid {
            inner_grid: data,
            mut validity_mask,
        } = self;

        let mut new_data = Grid::new_filled(data.shape.clone(), Out::default());

        new_data
            .data
            .iter_mut()
            .zip(validity_mask.data.iter_mut())
            .zip(data.data.into_iter())
            .for_each(|((o, m), i)| {
                let in_value = if *m { Some(i) } else { None };

                let new_out_value = map_fn(in_value);
                *m = new_out_value.is_some();

                if let Some(out_value) = new_out_value {
                    *o = out_value;
                }
            });

        MaskedGrid::new(new_data, validity_mask).expect("Masked grid cration should work because `new_data` has the same shape as the original data and thus the dimensions must equal the validity mask.")
    }
}

// Implementation for GridOrEmpty.
// Works with:
//    F: Fn(T) -> T
impl<In, Out, F, G> MapElements<In, Out, F> for GridOrEmpty<G, In>
where
    In: 'static + Copy,
    Out: 'static + Copy,
    G: GridSize + Clone + PartialEq,
    F: Fn(In) -> Out,
    MaskedGrid<G, In>: MapElements<In, Out, F, Output = MaskedGrid<G, Out>>,
{
    type Output = GridOrEmpty<G, Out>;

    fn map_elements(self, map_fn: F) -> Self::Output {
        match self {
            GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements(map_fn)),
            GridOrEmpty::Empty(empty) => GridOrEmpty::Empty(empty.convert_dtype()),
        }
    }
}

// Implementation for GridOrEmpty.
// Works with:
//    F: Fn(Option<T>) -> Option<T>,
impl<In, Out, F, G> MapElements<Option<In>, Option<Out>, F> for GridOrEmpty<G, In>
where
    In: 'static,
    Out: 'static + Clone,
    G: GridSize + Clone + PartialEq,
    F: Fn(Option<In>) -> Option<Out>,
    MaskedGrid<G, In>: MapElements<Option<In>, Option<Out>, F, Output = MaskedGrid<G, Out>>,
{
    type Output = GridOrEmpty<G, Out>;

    fn map_elements(self, map_fn: F) -> Self::Output {
        match self {
            GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements(map_fn)),
            GridOrEmpty::Empty(empty) => {
                // if None maps to a value we can be sure that the whole grid will turn to that value.
                if let Some(fill_value) = map_fn(None as Option<In>) {
                    return GridOrEmpty::Grid(MaskedGrid::new_filled(empty.shape, fill_value));
                }
                GridOrEmpty::Empty(empty.convert_dtype())
            }
        }
    }
}

// Implementation for GridOrEmpty.
// Works with:
//    F: Fn(Option<T>) -> Option<T>,
//    F: Fn(T) -> T
impl<FIn, FOut, TIn, TOut, F> MapElements<FIn, FOut, F> for RasterTile2D<TIn>
where
    FIn: 'static + Copy,
    FOut: 'static + Copy,
    TIn: 'static,
    TOut: 'static,
    F: Fn(FIn) -> FOut,
    GridOrEmpty2D<TIn>: MapElements<FIn, FOut, F, Output = GridOrEmpty2D<TOut>>,
{
    type Output = RasterTile2D<TOut>;

    fn map_elements(self, map_fn: F) -> Self::Output {
        RasterTile2D {
            grid_array: self.grid_array.map_elements(map_fn),
            time: self.time,
            tile_position: self.tile_position,
            global_geo_transform: self.global_geo_transform,
            properties: self.properties,
        }
    }
}

// Implementation for Grid: F: Fn(In) -> Out.
impl<In, Out, F, G> MapElementsParallel<In, Out, F> for Grid<G, In>
where
    In: 'static + Copy + PartialEq + Send + Sync,
    Out: 'static + Copy + Send + Sync,
    G: GridSize + Clone + Send + Sync,
    F: Fn(In) -> Out + Sync + Send,
{
    type Output = Grid<G, Out>;
    fn map_elements_parallel(self, map_fn: F) -> Self::Output {
        let shape = self.shape.clone();
        let num_elements_per_thread = num::integer::div_ceil(
            self.shape.number_of_elements(),
            rayon::current_num_threads(),
        )
        .max(MIN_ELEMENTS_PER_THREAD);

        let mut data = Vec::with_capacity(self.shape.number_of_elements());

        self.data
            .into_par_iter()
            .with_min_len(num_elements_per_thread)
            .map(map_fn)
            .collect_into_vec(data.as_mut());
        Grid { shape, data }
    }
}

// Implementation for MaskedGrid to enable update of the inner data with F: Fn(T) -> T.
impl<In, Out, F, G> MapElementsParallel<In, Out, F> for MaskedGrid<G, In>
where
    In: 'static + Copy + PartialEq + Send + Sync,
    Out: 'static + Copy + Send + Sync,
    G: GridSize + Clone + Send + Sync + PartialEq,
    F: Fn(In) -> Out + Sync + Send,
{
    type Output = MaskedGrid<G, Out>;
    fn map_elements_parallel(self, map_fn: F) -> Self::Output {
        let MaskedGrid {
            inner_grid: data,
            validity_mask,
        } = self;
        let new_data = data.map_elements_parallel(map_fn);
        MaskedGrid::new(new_data, validity_mask)
            .expect("MaskedGrid creation should work because the shape of the grid and the data size are valid and left unchanged.")
    }
}

// Implementation for MaskedGrid: F: Fn(Option<T>) -> Option<T>.
impl<In, Out, F, G> MapElementsParallel<Option<In>, Option<Out>, F> for MaskedGrid<G, In>
where
    In: 'static + Copy + Send + Sync,
    Out: 'static + Copy + Send + Sync + Default,
    G: GridSize + Clone + Send + Sync + PartialEq,
    F: Fn(Option<In>) -> Option<Out> + Sync + Send,
{
    type Output = MaskedGrid<G, Out>;
    fn map_elements_parallel(self, map_fn: F) -> Self::Output {
        let MaskedGrid {
            inner_grid: data,
            validity_mask,
        } = self;

        let shape = data.shape.clone();
        let num_elements_per_thread =
            num::integer::div_ceil(shape.number_of_elements(), rayon::current_num_threads())
                .max(MIN_ELEMENTS_PER_THREAD);

        let mut out_data = Vec::with_capacity(shape.number_of_elements());
        let mut out_validity = Vec::with_capacity(shape.number_of_elements());

        data.data
            .into_par_iter()
            .with_min_len(num_elements_per_thread)
            .zip(
                validity_mask
                    .data
                    .into_par_iter()
                    .with_min_len(num_elements_per_thread),
            )
            .map(|(i, m)| {
                let in_value = if m { Some(i) } else { None };

                if let Some(o) = map_fn(in_value) {
                    (o, m)
                } else {
                    (Out::default(), false)
                }
            })
            .unzip_into_vecs(out_data.as_mut(), out_validity.as_mut());

        MaskedGrid::new(
            Grid::new(shape.clone(), out_data).expect(
                "Grid creation shoud work because the shape of the grid and the data size match",
            ),
            Grid::new(shape, out_validity).expect(
                "Grid creation shoud work because the shape of the grid and the data size match",
            ),
        )
        .expect("Grid creation shoud work because the shape of the data and validity grid match")
    }
}

// Implementation for GridOrEmpty.
// Works with:
//    F: Fn(T) -> T
impl<In, Out, F, G> MapElementsParallel<In, Out, F> for GridOrEmpty<G, In>
where
    In: 'static,
    Out: 'static,
    G: GridSize,
    F: Fn(In) -> Out + Send + Sync,
    MaskedGrid<G, In>: MapElementsParallel<In, Out, F, Output = MaskedGrid<G, Out>>,
{
    type Output = GridOrEmpty<G, Out>;

    fn map_elements_parallel(self, map_fn: F) -> Self::Output {
        match self {
            GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements_parallel(map_fn)),
            GridOrEmpty::Empty(empty) => GridOrEmpty::Empty(empty.convert_dtype()),
        }
    }
}

// Implementation for GridOrEmpty.
// Works with:
//    F: Fn(Option<T>) -> Option<T>,
impl<In, Out, F, G> MapElementsParallel<Option<In>, Option<Out>, F> for GridOrEmpty<G, In>
where
    In: 'static,
    Out: 'static + Clone,
    G: GridSize + PartialEq + Clone,
    F: Fn(Option<In>) -> Option<Out> + Send + Sync,
    MaskedGrid<G, In>: MapElementsParallel<Option<In>, Option<Out>, F, Output = MaskedGrid<G, Out>>,
{
    type Output = GridOrEmpty<G, Out>;

    fn map_elements_parallel(self, map_fn: F) -> Self::Output {
        match self {
            GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements_parallel(map_fn)),
            GridOrEmpty::Empty(empty) => {
                // if None maps to a value we can be sure that the whole grid will turn to that value.
                if let Some(fill_value) = map_fn(None as Option<In>) {
                    return GridOrEmpty::Grid(MaskedGrid::new_filled(empty.shape, fill_value));
                }
                GridOrEmpty::Empty(empty.convert_dtype())
            }
        }
    }
}

// Implementation for GridOrEmpty.
// Works with:
//    F: Fn(Option<T>) -> Option<T>,
//    F: Fn(T) -> T
impl<FIn, FOut, TIn, TOut, F> MapElementsParallel<FIn, FOut, F> for RasterTile2D<TIn>
where
    TIn: 'static,
    TOut: 'static,
    F: Fn(FIn) -> FOut + Send + Sync,
    GridOrEmpty2D<TIn>: MapElementsParallel<FIn, FOut, F, Output = GridOrEmpty2D<TOut>>,
{
    type Output = RasterTile2D<TOut>;

    fn map_elements_parallel(self, map_fn: F) -> Self::Output {
        RasterTile2D {
            grid_array: self.grid_array.map_elements_parallel(map_fn),
            time: self.time,
            tile_position: self.tile_position,
            global_geo_transform: self.global_geo_transform,
            properties: self.properties,
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::{
        primitives::TimeInterval,
        raster::{EmptyGrid2D, GeoTransform, Grid2D},
        util::test::TestDefault,
    };

    use super::*;

    #[test]
    fn map_grid() {
        let dim = [2, 2];
        let data = vec![1, 2, 3, 4];

        let r1 = Grid2D::new(dim.into(), data).unwrap();
        let scaled_r1 = r1.map_elements(|p| p * 2 + 1);

        let expected = [3, 5, 7, 9];
        assert_eq!(scaled_r1.data, expected);
    }

    #[test]
    fn map_grid_or_empty() {
        let dim = [2, 2];
        let data = vec![1, 2, 3, 4];

        let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));
        let scaled_r1 = r1.map_elements(|p: i32| p * 2 + 1);

        let expected = [3, 5, 7, 9];

        match scaled_r1 {
            GridOrEmpty::Grid(g) => {
                assert_eq!(g.inner_grid.data, expected);
            }
            GridOrEmpty::Empty(_) => panic!("Expected grid"),
        }

        let r2 = GridOrEmpty::Empty::<_, u8>(EmptyGrid2D::new(dim.into()));
        let scaled_r2 = r2.map_elements(|p: u8| p - 10);

        match scaled_r2 {
            GridOrEmpty::Grid(_) => {
                panic!("Expected empty grid")
            }
            GridOrEmpty::Empty(e) => {
                assert_eq!(e.shape, dim.into());
            }
        }
    }

    #[test]
    fn map_grid_parallel() {
        let dim = [2, 2];
        let data = vec![7, 7, 8, 8];

        let r1 = Grid2D::new(dim.into(), data).unwrap();
        let scaled_r1 = r1.map_elements_parallel(|p: i32| p * 2 + 1);

        let expected = [15, 15, 17, 17];
        assert_eq!(scaled_r1.data, expected);
    }

    #[test]
    fn map_grid_or_empty_parallel() {
        let dim = [2, 2];
        let data = vec![7, 7, 8, 8];

        let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));
        let scaled_r1 = r1.map_elements_parallel(|p: i32| p * 2 + 1);

        let expected = [15, 15, 17, 17];

        match scaled_r1 {
            GridOrEmpty::Grid(g) => {
                assert_eq!(g.inner_grid.data, expected);
            }
            GridOrEmpty::Empty(_) => panic!("Expected grid"),
        }

        let r2 = GridOrEmpty::Empty::<_, u8>(EmptyGrid2D::new(dim.into()));
        let scaled_r2 = r2.map_elements_parallel(|p: u8| p - 10);

        match scaled_r2 {
            GridOrEmpty::Grid(_) => {
                panic!("Expected empty grid")
            }
            GridOrEmpty::Empty(e) => {
                assert_eq!(e.shape, dim.into());
            }
        }
    }

    #[test]
    fn map_raster_tile_parallel() {
        let dim = [2, 2];
        let data = vec![7, 7, 8, 8];
        let geo = GeoTransform::test_default();

        let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));
        let t1 = RasterTile2D::new(TimeInterval::default(), [0, 0].into(), geo, r1);

        let scaled_r1 = t1.map_elements_parallel(|p: u8| p * 2 + 1);
        let mat_scaled_r1 = scaled_r1.into_materialized_tile();

        let expected = [15, 15, 17, 17];

        assert_eq!(mat_scaled_r1.grid_array.inner_grid.data, expected);
    }

    #[test]
    fn map_raster_tile() {
        let dim = [2, 2];
        let data = vec![7, 7, 8, 8];
        let geo = GeoTransform::test_default();

        let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));
        let t1 = RasterTile2D::new(TimeInterval::default(), [0, 0].into(), geo, r1);

        let scaled_r1 = t1.map_elements(|p| {
            if let Some(p) = p {
                if p == 7 {
                    Some(p * 2 + 1)
                } else {
                    None
                }
            } else {
                None
            }
        });
        let mat_scaled_r1 = scaled_r1.into_materialized_tile();

        let expected = [15, 15, 0, 0];
        let expected_mask = [true, true, false, false];

        assert_eq!(mat_scaled_r1.grid_array.inner_grid.data, expected);
        assert_eq!(mat_scaled_r1.grid_array.validity_mask.data, expected_mask);
    }
}

1	use crate::raster::{Grid, GridOrEmpty, GridOrEmpty2D, GridSize, MaskedGrid, RasterTile2D};
2	use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};
3
4	const MIN_ELEMENTS_PER_THREAD: usize = 16 * 512;
5
6	/// This trait models a map operation from a `Grid` of type `In` into a `Grid` of Type `Out`. This is done using a provided function that maps each element to a new value.
7	///
8	/// Most useful implementations are on: `Grid`, `MaskedGrid`, `GridOrEmpty` and `RasterTile2D`.
9	///
10	/// On `Grid` elements are mapped, e.g., as `\|element: In\| { element + 1 }` with `F: Fn(In) -> Out`
11	///
12	/// On `MaskedGrid` you can map either only valid data (excluding no data), e.g., `\|element: In\| { element + 1 }` with `F: Fn(In) -> Out` or handling no data as `\|element: Option<In>\| { element.map(\|e\| e+ 1) }` with `F: Fn(Option<In>) -> Option<Out>`.
13	pub trait MapElements<In, Out, F: Fn(In) -> Out> {
14	type Output;
15	/// Create a new instance from the current one. The `map_fn` transforms all elements to a new value.
16	fn map_elements(self, map_fn: F) -> Self::Output;
17	}
18
19	/// This trait is equal to `MapElements` but uses a thread pool to do the operation in parallel.
20	/// This trait models a map operation from a `Grid` of type `In` into a `Grid` of Type `Out`. This is done using a provided function that maps each element to a new value.
21	///
22	/// Most useful implementations are on: `Grid`, `MaskedGrid`, `GridOrEmpty` and `RasterTile2D`.
23	///
24	/// On `Grid` elements are mapped, e.g., as `\|element: In\| { element + 1 }` with `F: Fn(In) -> Out`
25	///
26	/// On `MaskedGrid` you can map either only valid data (excluding no data), e.g., `\|element: In\| { element + 1 }` with `F: Fn(In) -> Out` or handling no data as `\|element: Option<In>\| { element.map(\|e\| e+ 1) }` with `F: Fn(Option<In>) -> Option<Out>`.
27
28	pub trait MapElementsParallel<In, Out, F: Fn(In) -> Out> {
29	type Output;
30	/// Create a new instance from the current one. The `map_fn` transforms all elements to a new value. Use a `ThreadPool` for parallel map operations.
31	fn map_elements_parallel(self, map_fn: F) -> Self::Output;
32	}
33
34	// Implementation for Grid: F: Fn(In) -> Out.
35	impl<In, Out, F, G> MapElements<In, Out, F> for Grid<G, In>
36	where
37	In: 'static,
38	Out: 'static,
39	G: GridSize + Clone,
40	F: Fn(In) -> Out,
41	{
42	type Output = Grid<G, Out>;
43	fn map_elements(self, map_fn: F) -> Self::Output {	14✔
44	let shape = self.shape;	14✔
45	let data = self.data.into_iter().map(map_fn).collect();	14✔
46		14✔
47	Grid { shape, data }	14✔
48	}	14✔
49	}
50
51	// Implementation for MaskedGrid to enable update of the inner data with F: Fn(T) -> T.
52	impl<In, Out, F, G> MapElements<In, Out, F> for MaskedGrid<G, In>
53	where
54	In: 'static + Clone,
55	Out: 'static + Clone,
56	G: GridSize + Clone + PartialEq,
57	F: Fn(In) -> Out,
58	Grid<G, In>: MapElements<In, Out, F, Output = Grid<G, Out>>,
59	{
60	type Output = MaskedGrid<G, Out>;
61
62	fn map_elements(self, map_fn: F) -> Self::Output {	1✔
63	MaskedGrid::new(self.inner_grid.map_elements(map_fn), self.validity_mask)	1✔
64	.expect("Creation failed for prev valid dimensions")	1✔
65	}	1✔
66	}
67
68	// Implementation for MaskedGrid: F: Fn(Option<T>) -> Option<T>.
69	impl<In, Out, F, G> MapElements<Option<In>, Option<Out>, F> for MaskedGrid<G, In>
70	where
71	In: 'static + Clone,
72	Out: 'static + Clone + Default,
73	G: GridSize + Clone + PartialEq,
74	F: Fn(Option<In>) -> Option<Out>,
75	{
76	type Output = MaskedGrid<G, Out>;
77
78	fn map_elements(self, map_fn: F) -> Self::Output {	24✔
79	let MaskedGrid {	24✔
80	inner_grid: data,	24✔
81	mut validity_mask,	24✔
82	} = self;	24✔
83		24✔
84	let mut new_data = Grid::new_filled(data.shape.clone(), Out::default());	24✔
85		24✔
86	new_data	24✔
87	.data	24✔
88	.iter_mut()	24✔
89	.zip(validity_mask.data.iter_mut())	24✔
90	.zip(data.data.into_iter())	24✔
91	.for_each(\|((o, m), i)\| {	24✔
92	let in_value = if *m { Some(i) } else { None };	126✔
93
94	let new_out_value = map_fn(in_value);	126✔
95	*m = new_out_value.is_some();	126✔
96
97	if let Some(out_value) = new_out_value {	126✔
98	*o = out_value;	118✔
99	}	118✔
100	});	126✔
101		24✔
102	MaskedGrid::new(new_data, validity_mask).expect("Masked grid cration should work because `new_data` has the same shape as the original data and thus the dimensions must equal the validity mask.")	24✔
103	}	24✔
104	}
105
106	// Implementation for GridOrEmpty.
107	// Works with:
108	// F: Fn(T) -> T
109	impl<In, Out, F, G> MapElements<In, Out, F> for GridOrEmpty<G, In>
110	where
111	In: 'static + Copy,
112	Out: 'static + Copy,
113	G: GridSize + Clone + PartialEq,
114	F: Fn(In) -> Out,
115	MaskedGrid<G, In>: MapElements<In, Out, F, Output = MaskedGrid<G, Out>>,
116	{
117	type Output = GridOrEmpty<G, Out>;
118
119	fn map_elements(self, map_fn: F) -> Self::Output {	2✔
120	match self {	2✔
121	GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements(map_fn)),	1✔
122	GridOrEmpty::Empty(empty) => GridOrEmpty::Empty(empty.convert_dtype()),	1✔
123	}
124	}	2✔
125	}
126
127	// Implementation for GridOrEmpty.
128	// Works with:
129	// F: Fn(Option<T>) -> Option<T>,
130	impl<In, Out, F, G> MapElements<Option<In>, Option<Out>, F> for GridOrEmpty<G, In>
131	where
132	In: 'static,
133	Out: 'static + Clone,
134	G: GridSize + Clone + PartialEq,
135	F: Fn(Option<In>) -> Option<Out>,
136	MaskedGrid<G, In>: MapElements<Option<In>, Option<Out>, F, Output = MaskedGrid<G, Out>>,
137	{
138	type Output = GridOrEmpty<G, Out>;
139
140	fn map_elements(self, map_fn: F) -> Self::Output {	1✔
141	match self {	1✔
142	GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements(map_fn)),	1✔
143	GridOrEmpty::Empty(empty) => {	×
144	// if None maps to a value we can be sure that the whole grid will turn to that value.
145	if let Some(fill_value) = map_fn(None as Option<In>) {	×
146	return GridOrEmpty::Grid(MaskedGrid::new_filled(empty.shape, fill_value));	×
147	}	×
148	GridOrEmpty::Empty(empty.convert_dtype())	×
149	}
150	}
151	}	1✔
152	}
153
154	// Implementation for GridOrEmpty.
155	// Works with:
156	// F: Fn(Option<T>) -> Option<T>,
157	// F: Fn(T) -> T
158	impl<FIn, FOut, TIn, TOut, F> MapElements<FIn, FOut, F> for RasterTile2D<TIn>
159	where
160	FIn: 'static + Copy,
161	FOut: 'static + Copy,
162	TIn: 'static,
163	TOut: 'static,
164	F: Fn(FIn) -> FOut,
165	GridOrEmpty2D<TIn>: MapElements<FIn, FOut, F, Output = GridOrEmpty2D<TOut>>,
166	{
167	type Output = RasterTile2D<TOut>;
168
169	fn map_elements(self, map_fn: F) -> Self::Output {	1✔
170	RasterTile2D {	1✔
171	grid_array: self.grid_array.map_elements(map_fn),	1✔
172	time: self.time,	1✔
173	tile_position: self.tile_position,	1✔
174	global_geo_transform: self.global_geo_transform,	1✔
175	properties: self.properties,	1✔
176	}	1✔
177	}	1✔
178	}
179
180	// Implementation for Grid: F: Fn(In) -> Out.
181	impl<In, Out, F, G> MapElementsParallel<In, Out, F> for Grid<G, In>
182	where
183	In: 'static + Copy + PartialEq + Send + Sync,
184	Out: 'static + Copy + Send + Sync,
185	G: GridSize + Clone + Send + Sync,
186	F: Fn(In) -> Out + Sync + Send,
187	{
188	type Output = Grid<G, Out>;
189	fn map_elements_parallel(self, map_fn: F) -> Self::Output {	3✔
190	let shape = self.shape.clone();	3✔
191	let num_elements_per_thread = num::integer::div_ceil(	3✔
192	self.shape.number_of_elements(),	3✔
193	rayon::current_num_threads(),	3✔
194	)	3✔
195	.max(MIN_ELEMENTS_PER_THREAD);	3✔
196		3✔
197	let mut data = Vec::with_capacity(self.shape.number_of_elements());	3✔
198		3✔
199	self.data	3✔
200	.into_par_iter()	3✔
201	.with_min_len(num_elements_per_thread)	3✔
202	.map(map_fn)	3✔
203	.collect_into_vec(data.as_mut());	3✔
204	Grid { shape, data }	3✔
205	}	3✔
206	}
207
208	// Implementation for MaskedGrid to enable update of the inner data with F: Fn(T) -> T.
209	impl<In, Out, F, G> MapElementsParallel<In, Out, F> for MaskedGrid<G, In>
210	where
211	In: 'static + Copy + PartialEq + Send + Sync,
212	Out: 'static + Copy + Send + Sync,
213	G: GridSize + Clone + Send + Sync + PartialEq,
214	F: Fn(In) -> Out + Sync + Send,
215	{
216	type Output = MaskedGrid<G, Out>;
217	fn map_elements_parallel(self, map_fn: F) -> Self::Output {	2✔
218	let MaskedGrid {	2✔
219	inner_grid: data,	2✔
220	validity_mask,	2✔
221	} = self;	2✔
222	let new_data = data.map_elements_parallel(map_fn);	2✔
223	MaskedGrid::new(new_data, validity_mask)	2✔
224	.expect("MaskedGrid creation should work because the shape of the grid and the data size are valid and left unchanged.")	2✔
225	}	2✔
226	}
227
228	// Implementation for MaskedGrid: F: Fn(Option<T>) -> Option<T>.
229	impl<In, Out, F, G> MapElementsParallel<Option<In>, Option<Out>, F> for MaskedGrid<G, In>
230	where
231	In: 'static + Copy + Send + Sync,
232	Out: 'static + Copy + Send + Sync + Default,
233	G: GridSize + Clone + Send + Sync + PartialEq,
234	F: Fn(Option<In>) -> Option<Out> + Sync + Send,
235	{
236	type Output = MaskedGrid<G, Out>;
237	fn map_elements_parallel(self, map_fn: F) -> Self::Output {	15✔
238	let MaskedGrid {	15✔
239	inner_grid: data,	15✔
240	validity_mask,	15✔
241	} = self;	15✔
242		15✔
243	let shape = data.shape.clone();	15✔
244	let num_elements_per_thread =	15✔
245	num::integer::div_ceil(shape.number_of_elements(), rayon::current_num_threads())	15✔
246	.max(MIN_ELEMENTS_PER_THREAD);	15✔
247		15✔
248	let mut out_data = Vec::with_capacity(shape.number_of_elements());	15✔
249	let mut out_validity = Vec::with_capacity(shape.number_of_elements());	15✔
250		15✔
251	data.data	15✔
252	.into_par_iter()	15✔
253	.with_min_len(num_elements_per_thread)	15✔
254	.zip(	15✔
255	validity_mask	15✔
256	.data	15✔
257	.into_par_iter()	15✔
258	.with_min_len(num_elements_per_thread),	15✔
259	)	15✔
260	.map(\|(i, m)\| {	15✔
261	let in_value = if m { Some(i) } else { None };	90✔
262
263	if let Some(o) = map_fn(in_value) {	90✔
264	(o, m)	83✔
265	} else {
266	(Out::default(), false)	7✔
267	}
268	})	90✔
269	.unzip_into_vecs(out_data.as_mut(), out_validity.as_mut());	15✔
270		15✔
271	MaskedGrid::new(	15✔
272	Grid::new(shape.clone(), out_data).expect(	15✔
273	"Grid creation shoud work because the shape of the grid and the data size match",	15✔
274	),	15✔
275	Grid::new(shape, out_validity).expect(	15✔
276	"Grid creation shoud work because the shape of the grid and the data size match",	15✔
277	),	15✔
278	)	15✔
279	.expect("Grid creation shoud work because the shape of the data and validity grid match")	15✔
280	}	15✔
281	}
282
283	// Implementation for GridOrEmpty.
284	// Works with:
285	// F: Fn(T) -> T
286	impl<In, Out, F, G> MapElementsParallel<In, Out, F> for GridOrEmpty<G, In>
287	where
288	In: 'static,
289	Out: 'static,
290	G: GridSize,
291	F: Fn(In) -> Out + Send + Sync,
292	MaskedGrid<G, In>: MapElementsParallel<In, Out, F, Output = MaskedGrid<G, Out>>,
293	{
294	type Output = GridOrEmpty<G, Out>;
295
296	fn map_elements_parallel(self, map_fn: F) -> Self::Output {	3✔
297	match self {	3✔
298	GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements_parallel(map_fn)),	2✔
299	GridOrEmpty::Empty(empty) => GridOrEmpty::Empty(empty.convert_dtype()),	1✔
300	}
301	}	3✔
302	}
303
304	// Implementation for GridOrEmpty.
305	// Works with:
306	// F: Fn(Option<T>) -> Option<T>,
307	impl<In, Out, F, G> MapElementsParallel<Option<In>, Option<Out>, F> for GridOrEmpty<G, In>
308	where
309	In: 'static,
310	Out: 'static + Clone,
311	G: GridSize + PartialEq + Clone,
312	F: Fn(Option<In>) -> Option<Out> + Send + Sync,
313	MaskedGrid<G, In>: MapElementsParallel<Option<In>, Option<Out>, F, Output = MaskedGrid<G, Out>>,
314	{
315	type Output = GridOrEmpty<G, Out>;
316
317	fn map_elements_parallel(self, map_fn: F) -> Self::Output {	17✔
318	match self {	17✔
319	GridOrEmpty::Grid(grid) => GridOrEmpty::Grid(grid.map_elements_parallel(map_fn)),	15✔
320	GridOrEmpty::Empty(empty) => {	2✔
321	// if None maps to a value we can be sure that the whole grid will turn to that value.
322	if let Some(fill_value) = map_fn(None as Option<In>) {	2✔
323	return GridOrEmpty::Grid(MaskedGrid::new_filled(empty.shape, fill_value));	×
324	}	2✔
325	GridOrEmpty::Empty(empty.convert_dtype())	2✔
326	}
327	}
328	}	17✔
329	}
330
331	// Implementation for GridOrEmpty.
332	// Works with:
333	// F: Fn(Option<T>) -> Option<T>,
334	// F: Fn(T) -> T
335	impl<FIn, FOut, TIn, TOut, F> MapElementsParallel<FIn, FOut, F> for RasterTile2D<TIn>
336	where
337	TIn: 'static,
338	TOut: 'static,
339	F: Fn(FIn) -> FOut + Send + Sync,
340	GridOrEmpty2D<TIn>: MapElementsParallel<FIn, FOut, F, Output = GridOrEmpty2D<TOut>>,
341	{
342	type Output = RasterTile2D<TOut>;
343
344	fn map_elements_parallel(self, map_fn: F) -> Self::Output {	7✔
345	RasterTile2D {	7✔
346	grid_array: self.grid_array.map_elements_parallel(map_fn),	7✔
347	time: self.time,	7✔
348	tile_position: self.tile_position,	7✔
349	global_geo_transform: self.global_geo_transform,	7✔
350	properties: self.properties,	7✔
351	}	7✔
352	}	7✔
353	}
354
355	#[cfg(test)]
356	mod tests {
357	use crate::{
358	primitives::TimeInterval,
359	raster::{EmptyGrid2D, GeoTransform, Grid2D},
360	util::test::TestDefault,
361	};
362
363	use super::*;
364
365	#[test]	1✔
366	fn map_grid() {	1✔
367	let dim = [2, 2];	1✔
368	let data = vec![1, 2, 3, 4];	1✔
369		1✔
370	let r1 = Grid2D::new(dim.into(), data).unwrap();	1✔
371	let scaled_r1 = r1.map_elements(\|p\| p * 2 + 1);	4✔
372		1✔
373	let expected = [3, 5, 7, 9];	1✔
374	assert_eq!(scaled_r1.data, expected);	1✔
375	}	1✔
376
377	#[test]	1✔
378	fn map_grid_or_empty() {	1✔
379	let dim = [2, 2];	1✔
380	let data = vec![1, 2, 3, 4];	1✔
381		1✔
382	let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));	1✔
383	let scaled_r1 = r1.map_elements(\|p: i32\| p * 2 + 1);	4✔
384		1✔
385	let expected = [3, 5, 7, 9];	1✔
386		1✔
387	match scaled_r1 {	1✔
388	GridOrEmpty::Grid(g) => {	1✔
389	assert_eq!(g.inner_grid.data, expected);	1✔
390	}
391	GridOrEmpty::Empty(_) => panic!("Expected grid"),	×
392	}
393
394	let r2 = GridOrEmpty::Empty::<_, u8>(EmptyGrid2D::new(dim.into()));	1✔
395	let scaled_r2 = r2.map_elements(\|p: u8\| p - 10);	1✔
396		1✔
397	match scaled_r2 {	1✔
398	GridOrEmpty::Grid(_) => {
399	panic!("Expected empty grid")	×
400	}
401	GridOrEmpty::Empty(e) => {	1✔
402	assert_eq!(e.shape, dim.into());	1✔
403	}
404	}
405	}	1✔
406
407	#[test]	1✔
408	fn map_grid_parallel() {	1✔
409	let dim = [2, 2];	1✔
410	let data = vec![7, 7, 8, 8];	1✔
411		1✔
412	let r1 = Grid2D::new(dim.into(), data).unwrap();	1✔
413	let scaled_r1 = r1.map_elements_parallel(\|p: i32\| p * 2 + 1);	4✔
414		1✔
415	let expected = [15, 15, 17, 17];	1✔
416	assert_eq!(scaled_r1.data, expected);	1✔
417	}	1✔
418
419	#[test]	1✔
420	fn map_grid_or_empty_parallel() {	1✔
421	let dim = [2, 2];	1✔
422	let data = vec![7, 7, 8, 8];	1✔
423		1✔
424	let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));	1✔
425	let scaled_r1 = r1.map_elements_parallel(\|p: i32\| p * 2 + 1);	4✔
426		1✔
427	let expected = [15, 15, 17, 17];	1✔
428		1✔
429	match scaled_r1 {	1✔
430	GridOrEmpty::Grid(g) => {	1✔
431	assert_eq!(g.inner_grid.data, expected);	1✔
432	}
433	GridOrEmpty::Empty(_) => panic!("Expected grid"),	×
434	}
435
436	let r2 = GridOrEmpty::Empty::<_, u8>(EmptyGrid2D::new(dim.into()));	1✔
437	let scaled_r2 = r2.map_elements_parallel(\|p: u8\| p - 10);	1✔
438		1✔
439	match scaled_r2 {	1✔
440	GridOrEmpty::Grid(_) => {
441	panic!("Expected empty grid")	×
442	}
443	GridOrEmpty::Empty(e) => {	1✔
444	assert_eq!(e.shape, dim.into());	1✔
445	}
446	}
447	}	1✔
448
449	#[test]	1✔
450	fn map_raster_tile_parallel() {	1✔
451	let dim = [2, 2];	1✔
452	let data = vec![7, 7, 8, 8];	1✔
453	let geo = GeoTransform::test_default();	1✔
454		1✔
455	let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));	1✔
456	let t1 = RasterTile2D::new(TimeInterval::default(), [0, 0].into(), geo, r1);	1✔
457		1✔
458	let scaled_r1 = t1.map_elements_parallel(\|p: u8\| p * 2 + 1);	4✔
459	let mat_scaled_r1 = scaled_r1.into_materialized_tile();	1✔
460		1✔
461	let expected = [15, 15, 17, 17];	1✔
462		1✔
463	assert_eq!(mat_scaled_r1.grid_array.inner_grid.data, expected);	1✔
464	}	1✔
465
466	#[test]	1✔
467	fn map_raster_tile() {	1✔
468	let dim = [2, 2];	1✔
469	let data = vec![7, 7, 8, 8];	1✔
470	let geo = GeoTransform::test_default();	1✔
471		1✔
472	let r1 = GridOrEmpty::Grid(MaskedGrid::from(Grid2D::new(dim.into(), data).unwrap()));	1✔
473	let t1 = RasterTile2D::new(TimeInterval::default(), [0, 0].into(), geo, r1);	1✔
474		1✔
475	let scaled_r1 = t1.map_elements(\|p\| {	1✔
476	if let Some(p) = p {	4✔
477	if p == 7 {	4✔
478	Some(p * 2 + 1)	2✔
479	} else {
480	None	2✔
481	}
482	} else {
483	None	×
484	}
485	});	4✔
486	let mat_scaled_r1 = scaled_r1.into_materialized_tile();	1✔
487		1✔
488	let expected = [15, 15, 0, 0];	1✔
489	let expected_mask = [true, true, false, false];	1✔
490		1✔
491	assert_eq!(mat_scaled_r1.grid_array.inner_grid.data, expected);	1✔
492	assert_eq!(mat_scaled_r1.grid_array.validity_mask.data, expected_mask);	1✔
493	}	1✔
494	}

geo-engine / geoengine / 3929938005

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous