9209766736

Committed 23 May 2024 01:55PM UTC coverage: 90.571% (-0.004%) from 90.575%

Build # 9209766736

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #948 from geo-engine/percentiles

Compute Percentile Estimates

Run Details

309 of 348 new or added lines in 4 files covered. (88.79%)

12 existing lines in 9 files now uncovered.

131051 of 144694 relevant lines covered (90.57%)

53422.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.49

/operators/src/processing/temporal_raster_aggregation/aggregators.rs

use crate::util::{statistics::SafePSquareQuantileEstimator, Result};
use geoengine_datatypes::raster::{GridOrEmpty2D, MapIndexedElements, Pixel};
use std::marker::PhantomData;

/// An aggregator that uses input values to produce an inner state that can be used to produce an output aggregate value.
pub trait TemporalRasterPixelAggregator<P: Pixel>: Send + Clone {
    type PixelState: Send + Sync + Copy + Clone + Default;

    /// Tell whether the aggregator ignores incoming no data values
    const IGNORE_NO_DATA: bool;

    /// Initialize the state from the first value
    fn initialize(value: Option<P>) -> Option<Self::PixelState>;

    /// Produce a new state from the current state and new value
    fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState>;

    /// Produce a tile from the state container
    fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>>;
}

pub trait GlobalStateTemporalRasterPixelAggregator<P: Pixel>: Send + Sync + Clone {
    type PixelState: Send + Sync + Copy + Clone + /* required by [`MapIndexElements`] */ Default;

    /// Tell whether the aggregator ignores incoming no data values
    const IGNORE_NO_DATA: bool;

    /// Initialize the state from the first value
    fn initialize(&self, value: Option<P>) -> Option<Self::PixelState>;

    /// Produce a new state from the current state and new value
    fn aggregate(
        &self,
        state: Option<Self::PixelState>,
        value: Option<P>,
    ) -> Option<Self::PixelState>;

    /// Produce a tile from the state container
    fn to_grid(&self, state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>>;
}

/// A method to process to pixel values inside the aggregator.
trait BinaryOperation<P: Pixel>: Send + Clone {
    fn unit(value: P) -> P {
        value
    }

    fn op(state: P, value: P) -> P;
}

#[derive(Clone)]
pub struct OpPixelAggregator<Op> {
    op: PhantomData<Op>,
}

impl<P: Pixel, Op: BinaryOperation<P>> TemporalRasterPixelAggregator<P> for OpPixelAggregator<Op> {
    type PixelState = P;

    const IGNORE_NO_DATA: bool = false;

    fn initialize(value: Option<P>) -> Option<Self::PixelState> {
        value.map(Op::unit)
    }

    fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState> {
        match (state, value) {
            (Some(state), Some(value)) => Some(Op::op(state, value)),
            _ => None,
        }
    }

    fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {
        Ok(state)
    }
}

#[derive(Clone)]
pub struct OpPixelAggregatorIngoringNoData<Op> {
    op: PhantomData<Op>,
}

impl<P: Pixel, Op: BinaryOperation<P>> TemporalRasterPixelAggregator<P>
    for OpPixelAggregatorIngoringNoData<Op>
{
    type PixelState = P;

    const IGNORE_NO_DATA: bool = true;

    fn initialize(value: Option<P>) -> Option<Self::PixelState> {
        value.map(Op::unit)
    }

    fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState> {
        match (state, value) {
            (Some(state), Some(value)) => Some(Op::op(state, value)),
            (Some(state), None) => Some(state),
            (None, Some(value)) => Some(Op::unit(value)),
            _ => None,
        }
    }

    fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {
        Ok(state)
    }
}

#[derive(Clone)]
pub struct Sum;

impl<P: Pixel> BinaryOperation<P> for Sum {
    fn op(state: P, value: P) -> P {
        state.saturating_add(value)
    }
}

pub type SumPixelAggregator = OpPixelAggregator<Sum>;
pub type SumPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Sum>;

#[derive(Clone)]
pub struct Count;

impl<P: Pixel> BinaryOperation<P> for Count {
    fn unit(_value: P) -> P {
        P::one()
    }

    fn op(state: P, _value: P) -> P {
        state.saturating_add(P::one())
    }
}

pub type CountPixelAggregator = OpPixelAggregator<Count>;
pub type CountPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Count>;

#[derive(Clone)]
pub struct Min;

impl<P: Pixel> BinaryOperation<P> for Min {
    fn op(state: P, value: P) -> P {
        if state < value {
            state
        } else {
            value
        }
    }
}

pub type MinPixelAggregator = OpPixelAggregator<Min>;
pub type MinPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Min>;

#[derive(Clone)]
pub struct Max;

impl<P: Pixel> BinaryOperation<P> for Max {
    fn op(state: P, value: P) -> P {
        if state > value {
            state
        } else {
            value
        }
    }
}

pub type MaxPixelAggregator = OpPixelAggregator<Max>;
pub type MaxPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Max>;

#[derive(Clone)]
pub struct First;

impl<P: Pixel> BinaryOperation<P> for First {
    fn op(state: P, _value: P) -> P {
        state
    }
}

pub type FirstPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<First>;

#[derive(Clone)]
pub struct Last;

impl<P: Pixel> BinaryOperation<P> for Last {
    fn op(_state: P, value: P) -> P {
        value
    }
}

pub type LastPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Last>;

#[derive(Clone)]
pub struct MeanPixelAggregator<const IGNORE_NO_DATA: bool>;

impl<P: Pixel, const IGNORE_NO_DATA: bool> TemporalRasterPixelAggregator<P>
    for MeanPixelAggregator<IGNORE_NO_DATA>
{
    type PixelState = (f64, usize);

    const IGNORE_NO_DATA: bool = IGNORE_NO_DATA;

    fn initialize(value: Option<P>) -> Option<Self::PixelState> {
        value.map(|v| (v.as_(), 1))
    }

    fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState> {
        if IGNORE_NO_DATA {
            match (state, value) {
                (Some(state), Some(value)) => Some(mean_of_state_and_value(state, value)),
                (Some(state), None) => Some(state),
                (None, Some(value)) => Self::initialize(Some(value)),
                _ => None,
            }
        } else {
            match (state, value) {
                (Some(state), Some(value)) => Some(mean_of_state_and_value(state, value)),
                _ => None,
            }
        }
    }

    fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {
        Ok(state.map_indexed_elements(|_index: usize, (mean, _count)| P::from_(mean)))
    }
}

fn mean_of_state_and_value<P: Pixel>(
    (mean_value, count): (f64, usize),
    new_value: P,
) -> (f64, usize) {
    let new_value: f64 = new_value.as_();
    let new_count = count + 1;
    let delta: f64 = new_value - mean_value;
    let new_state_value = mean_value + delta / (new_count as f64);
    (new_state_value, new_count)
}

#[derive(Clone)]
pub struct PercentileEstimateAggregator<const IGNORE_NO_DATA: bool> {
    percentile: f64,
}

impl<const IGNORE_NO_DATA: bool> PercentileEstimateAggregator<IGNORE_NO_DATA> {
    pub fn new(percentile: f64) -> Self {
        Self { percentile }
    }
}

impl<P: Pixel, const IGNORE_NO_DATA: bool> GlobalStateTemporalRasterPixelAggregator<P>
    for PercentileEstimateAggregator<IGNORE_NO_DATA>
{
    // we need a default value, so we wrap it again into an `Option`
    type PixelState = Option<SafePSquareQuantileEstimator<P>>;

    const IGNORE_NO_DATA: bool = IGNORE_NO_DATA;

    fn initialize(&self, value: Option<P>) -> Option<Self::PixelState> {
        Some(SafePSquareQuantileEstimator::new(self.percentile, value?).ok())
    }

    fn aggregate(
        &self,
        state: Option<Self::PixelState>,
        value: Option<P>,
    ) -> Option<Self::PixelState> {
        if IGNORE_NO_DATA {
            match (state, value) {
                (Some(Some(mut state)), Some(value)) => {
                    state.update(value);
                    Some(Some(state))
                }
                (Some(state), None) => Some(state),
                (None, Some(value)) => self.initialize(Some(value)),
                _ => None,
            }
        } else {
            match (state, value) {
                (Some(Some(mut state)), Some(value)) => {
                    state.update(value);
                    Some(Some(state))
                }
                _ => None,
            }
        }
    }

    fn to_grid(&self, state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {
        Ok(
            state.map_indexed_elements(|_index: usize, estimator_opt: Option<Self::PixelState>| {
                let estimator = estimator_opt??;
                let value = P::from_(estimator.quantile_estimate());
                Some(value)
            }),
        )
    }
}

1	use crate::util::{statistics::SafePSquareQuantileEstimator, Result};
2	use geoengine_datatypes::raster::{GridOrEmpty2D, MapIndexedElements, Pixel};
3	use std::marker::PhantomData;
4
5	/// An aggregator that uses input values to produce an inner state that can be used to produce an output aggregate value.
6	pub trait TemporalRasterPixelAggregator<P: Pixel>: Send + Clone {
7	type PixelState: Send + Sync + Copy + Clone + Default;
8
9	/// Tell whether the aggregator ignores incoming no data values
10	const IGNORE_NO_DATA: bool;
11
12	/// Initialize the state from the first value
13	fn initialize(value: Option<P>) -> Option<Self::PixelState>;
14
15	/// Produce a new state from the current state and new value
16	fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState>;
17
18	/// Produce a tile from the state container
19	fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>>;
20	}
21
22	pub trait GlobalStateTemporalRasterPixelAggregator<P: Pixel>: Send + Sync + Clone {
23	type PixelState: Send + Sync + Copy + Clone + /* required by [`MapIndexElements`] */ Default;
24
25	/// Tell whether the aggregator ignores incoming no data values
26	const IGNORE_NO_DATA: bool;
27
28	/// Initialize the state from the first value
29	fn initialize(&self, value: Option<P>) -> Option<Self::PixelState>;
30
31	/// Produce a new state from the current state and new value
32	fn aggregate(
33	&self,
34	state: Option<Self::PixelState>,
35	value: Option<P>,
36	) -> Option<Self::PixelState>;
37
38	/// Produce a tile from the state container
39	fn to_grid(&self, state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>>;
40	}
41
42	/// A method to process to pixel values inside the aggregator.
43	trait BinaryOperation<P: Pixel>: Send + Clone {
44	fn unit(value: P) -> P {	230✔
45	value	230✔
46	}	230✔
47
48	fn op(state: P, value: P) -> P;
49	}
50
51	#[derive(Clone)]	×
52	pub struct OpPixelAggregator<Op> {
53	op: PhantomData<Op>,
54	}
55
56	impl<P: Pixel, Op: BinaryOperation<P>> TemporalRasterPixelAggregator<P> for OpPixelAggregator<Op> {
57	type PixelState = P;
58
59	const IGNORE_NO_DATA: bool = false;
60
61	fn initialize(value: Option<P>) -> Option<Self::PixelState> {	204✔
62	value.map(Op::unit)	204✔
63	}	204✔
64
65	fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState> {	192✔
66	match (state, value) {	192✔
67	(Some(state), Some(value)) => Some(Op::op(state, value)),	192✔
68	_ => None,	×
69	}
70	}	192✔
71
72	fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {	37✔
73	Ok(state)	37✔
74	}	37✔
75	}
76
77	#[derive(Clone)]	×
78	pub struct OpPixelAggregatorIngoringNoData<Op> {
79	op: PhantomData<Op>,
80	}
81
82	impl<P: Pixel, Op: BinaryOperation<P>> TemporalRasterPixelAggregator<P>
83	for OpPixelAggregatorIngoringNoData<Op>
84	{
85	type PixelState = P;
86
87	const IGNORE_NO_DATA: bool = true;
88
89	fn initialize(value: Option<P>) -> Option<Self::PixelState> {	72✔
90	value.map(Op::unit)	72✔
91	}	72✔
92
93	fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState> {	48✔
94	match (state, value) {	48✔
95	(Some(state), Some(value)) => Some(Op::op(state, value)),	40✔
96	(Some(state), None) => Some(state),	4✔
97	(None, Some(value)) => Some(Op::unit(value)),	4✔
98	_ => None,	×
99	}
100	}	48✔
101
102	fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {	12✔
103	Ok(state)	12✔
104	}	12✔
105	}
106
107	#[derive(Clone)]	×
108	pub struct Sum;
109
110	impl<P: Pixel> BinaryOperation<P> for Sum {
111	fn op(state: P, value: P) -> P {	100✔
112	state.saturating_add(value)	100✔
113	}	100✔
114	}
115
116	pub type SumPixelAggregator = OpPixelAggregator<Sum>;
117	pub type SumPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Sum>;
118
119	#[derive(Clone)]	×
120	pub struct Count;
121
122	impl<P: Pixel> BinaryOperation<P> for Count {
123	fn unit(_value: P) -> P {	40✔
124	P::one()	40✔
125	}	40✔
126
127	fn op(state: P, _value: P) -> P {	28✔
128	state.saturating_add(P::one())	28✔
129	}	28✔
130	}
131
132	pub type CountPixelAggregator = OpPixelAggregator<Count>;
133	pub type CountPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Count>;
134
135	#[derive(Clone)]	×
136	pub struct Min;
137
138	impl<P: Pixel> BinaryOperation<P> for Min {
139	fn op(state: P, value: P) -> P {	24✔
140	if state < value {	24✔
141	state	12✔
142	} else {
143	value	12✔
144	}
145	}	24✔
146	}
147
148	pub type MinPixelAggregator = OpPixelAggregator<Min>;
149	pub type MinPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Min>;
150
151	#[derive(Clone)]	×
152	pub struct Max;
153
154	impl<P: Pixel> BinaryOperation<P> for Max {
155	fn op(state: P, value: P) -> P {	72✔
156	if state > value {	72✔
157	state	36✔
158	} else {
159	value	36✔
160	}
161	}	72✔
162	}
163
164	pub type MaxPixelAggregator = OpPixelAggregator<Max>;
165	pub type MaxPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Max>;
166
167	#[derive(Clone)]	×
168	pub struct First;
169
170	impl<P: Pixel> BinaryOperation<P> for First {
171	fn op(state: P, _value: P) -> P {	4✔
172	state	4✔
173	}	4✔
174	}
175
176	pub type FirstPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<First>;
177
178	#[derive(Clone)]	×
179	pub struct Last;
180
181	impl<P: Pixel> BinaryOperation<P> for Last {
182	fn op(_state: P, value: P) -> P {	4✔
183	value	4✔
184	}	4✔
185	}
186
187	pub type LastPixelAggregatorIngoringNoData = OpPixelAggregatorIngoringNoData<Last>;
188
189	#[derive(Clone)]	×
190	pub struct MeanPixelAggregator<const IGNORE_NO_DATA: bool>;
191
192	impl<P: Pixel, const IGNORE_NO_DATA: bool> TemporalRasterPixelAggregator<P>
193	for MeanPixelAggregator<IGNORE_NO_DATA>
194	{
195	type PixelState = (f64, usize);
196
197	const IGNORE_NO_DATA: bool = IGNORE_NO_DATA;
198
199	fn initialize(value: Option<P>) -> Option<Self::PixelState> {	19✔
200	value.map(\|v\| (v.as_(), 1))	19✔
201	}	19✔
202
203	fn aggregate(state: Option<Self::PixelState>, value: Option<P>) -> Option<Self::PixelState> {	6✔
204	if IGNORE_NO_DATA {	6✔
205	match (state, value) {	6✔
206	(Some(state), Some(value)) => Some(mean_of_state_and_value(state, value)),	4✔
207	(Some(state), None) => Some(state),	1✔
208	(None, Some(value)) => Self::initialize(Some(value)),	1✔
NEW 209	_ => None,	×
210	}
211	} else {
NEW 212	match (state, value) {	×
NEW 213	(Some(state), Some(value)) => Some(mean_of_state_and_value(state, value)),	×
NEW 214	_ => None,	×
215	}
216	}
217	}	6✔
218
219	fn into_grid(state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {	4✔
220	Ok(state.map_indexed_elements(\|_index: usize, (mean, _count)\| P::from_(mean)))	18✔
221	}	4✔
222	}
223
224	fn mean_of_state_and_value<P: Pixel>(	4✔
225	(mean_value, count): (f64, usize),	4✔
226	new_value: P,	4✔
227	) -> (f64, usize) {	4✔
228	let new_value: f64 = new_value.as_();	4✔
229	let new_count = count + 1;	4✔
230	let delta: f64 = new_value - mean_value;	4✔
231	let new_state_value = mean_value + delta / (new_count as f64);	4✔
232	(new_state_value, new_count)	4✔
233	}	4✔
234
UNCOV 235	#[derive(Clone)]	×
236	pub struct PercentileEstimateAggregator<const IGNORE_NO_DATA: bool> {
237	percentile: f64,
238	}
239
240	impl<const IGNORE_NO_DATA: bool> PercentileEstimateAggregator<IGNORE_NO_DATA> {
241	pub fn new(percentile: f64) -> Self {	1✔
242	Self { percentile }	1✔
243	}	1✔
244	}
245
246	impl<P: Pixel, const IGNORE_NO_DATA: bool> GlobalStateTemporalRasterPixelAggregator<P>
247	for PercentileEstimateAggregator<IGNORE_NO_DATA>
248	{
249	// we need a default value, so we wrap it again into an `Option`
250	type PixelState = Option<SafePSquareQuantileEstimator<P>>;
251
252	const IGNORE_NO_DATA: bool = IGNORE_NO_DATA;
253
254	fn initialize(&self, value: Option<P>) -> Option<Self::PixelState> {	12✔
255	Some(SafePSquareQuantileEstimator::new(self.percentile, value?).ok())	12✔
256	}	12✔
257
258	fn aggregate(	36✔
259	&self,	36✔
260	state: Option<Self::PixelState>,	36✔
261	value: Option<P>,	36✔
262	) -> Option<Self::PixelState> {	36✔
263	if IGNORE_NO_DATA {	36✔
NEW 264	match (state, value) {	×
NEW 265	(Some(Some(mut state)), Some(value)) => {	×
NEW 266	state.update(value);	×
NEW 267	Some(Some(state))	×
268	}
NEW 269	(Some(state), None) => Some(state),	×
NEW 270	(None, Some(value)) => self.initialize(Some(value)),	×
NEW 271	_ => None,	×
272	}
273	} else {
274	match (state, value) {	36✔
275	(Some(Some(mut state)), Some(value)) => {	36✔
276	state.update(value);	36✔
277	Some(Some(state))	36✔
278	}
NEW 279	_ => None,	×
280	}
281	}
282	}	36✔
283
284	fn to_grid(&self, state: GridOrEmpty2D<Self::PixelState>) -> Result<GridOrEmpty2D<P>> {	2✔
285	Ok(	2✔
286	state.map_indexed_elements(\|_index: usize, estimator_opt: Option<Self::PixelState>\| {	12✔
287	let estimator = estimator_opt??;	12✔
288	let value = P::from_(estimator.quantile_estimate());	12✔
289	Some(value)	12✔
290	}),	12✔
291	)	2✔
292	}	2✔
293	}

geo-engine / geoengine / 9209766736

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous