3929938005

Build Type

push

github

Committed by GitHub

Commit Message

Merge #713

Run Details

84930 of 96741 relevant lines covered (87.79%)

79640.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.83

/operators/src/processing/meteosat/mod.rs

use geoengine_datatypes::raster::RasterPropertiesKey;

mod radiance;
mod reflectance;
mod satellite;
mod temperature;

fn new_slope_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "calibration_slope".into(),
    }
}

fn new_offset_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "calibration_offset".into(),
    }
}

fn new_channel_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "channel_number".into(),
    }
}

fn new_satellite_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "satellite_number".into(),
    }
}

#[cfg(test)]
mod test_util {
    use std::str::FromStr;

    use futures::StreamExt;
    use geoengine_datatypes::hashmap;
    use geoengine_datatypes::util::test::TestDefault;
    use num_traits::AsPrimitive;

    use geoengine_datatypes::dataset::{DataId, DatasetId};
    use geoengine_datatypes::primitives::{
        ContinuousMeasurement, DateTime, DateTimeParseFormat, Measurement, RasterQueryRectangle,
        SpatialPartition2D, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval,
        TimeStep,
    };
    use geoengine_datatypes::raster::{
        Grid2D, GridOrEmpty, GridOrEmpty2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties,
        RasterPropertiesEntry, RasterPropertiesEntryType, RasterTile2D, TileInformation,
    };
    use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority};
    use geoengine_datatypes::util::Identifier;

    use crate::engine::{
        MockExecutionContext, MockQueryContext, QueryProcessor, RasterOperator,
        RasterResultDescriptor,
    };
    use crate::mock::{MockRasterSource, MockRasterSourceParams};
    use crate::processing::meteosat::{
        new_channel_key, new_offset_key, new_satellite_key, new_slope_key,
    };
    use crate::source::{
        FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular,
        GdalMetadataMapping, GdalSource, GdalSourceParameters, GdalSourceTimePlaceholder,
        TimeReference,
    };
    use crate::test_data;
    use crate::util::Result;

    pub(crate) async fn process<T>(
        make_op: T,
        query: RasterQueryRectangle,
        ctx: &MockExecutionContext,
    ) -> Result<RasterTile2D<f32>>
    where
        T: FnOnce() -> Box<dyn RasterOperator>,
    {
        // let input = make_raster(props, custom_data, measurement);

        let op = make_op().initialize(ctx).await?;

        let processor = op.query_processor().unwrap().get_f32().unwrap();

        let ctx = MockQueryContext::test_default();
        let result_stream = processor.query(query, &ctx).await.unwrap();
        let mut result: Vec<Result<RasterTile2D<f32>>> = result_stream.collect().await;
        assert_eq!(1, result.len());
        result.pop().unwrap()
    }

    pub(crate) fn create_properties(
        channel: Option<u8>,
        satellite: Option<u8>,
        offset: Option<f64>,
        slope: Option<f64>,
    ) -> RasterProperties {
        let mut props = RasterProperties::default();

        if let Some(v) = channel {
            props
                .properties_map
                .insert(new_channel_key(), RasterPropertiesEntry::Number(v.as_()));
        }

        if let Some(v) = satellite {
            props
                .properties_map
                .insert(new_satellite_key(), RasterPropertiesEntry::Number(v.as_()));
        }

        if let Some(v) = slope {
            props
                .properties_map
                .insert(new_slope_key(), RasterPropertiesEntry::Number(v));
        }

        if let Some(v) = offset {
            props
                .properties_map
                .insert(new_offset_key(), RasterPropertiesEntry::Number(v));
        }
        props
    }

    pub(crate) fn _create_gdal_query() -> RasterQueryRectangle {
        let sr = SpatialResolution::new_unchecked(3_000.403_165_817_261, 3_000.403_165_817_261);
        let ul = (0., 0.).into();
        let lr = (599. * sr.x, -599. * sr.y).into();
        RasterQueryRectangle {
            spatial_bounds: SpatialPartition2D::new_unchecked(ul, lr),
            time_interval: TimeInterval::new_unchecked(
                TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 0, 0)),
                TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 15, 0)),
            ),
            spatial_resolution: sr,
        }
    }

    pub(crate) fn create_mock_query() -> RasterQueryRectangle {
        RasterQueryRectangle {
            spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()),
            time_interval: Default::default(),
            spatial_resolution: SpatialResolution::one(),
        }
    }

    pub(crate) fn create_mock_source<P: Pixel>(
        props: RasterProperties,
        custom_data: Option<GridOrEmpty2D<P>>,
        measurement: Option<Measurement>,
    ) -> MockRasterSource<P> {
        let raster = match custom_data {
            Some(g) => g,
            None => GridOrEmpty::Grid(
                MaskedGrid2D::new(
                    Grid2D::<P>::new(
                        [3, 2].into(),
                        vec![
                            P::from_(1),
                            P::from_(2),
                            P::from_(3),
                            P::from_(4),
                            P::from_(5),
                            P::from_(0),
                        ],
                    )
                    .unwrap(),
                    Grid2D::new([3, 2].into(), vec![true, true, true, true, true, false]).unwrap(),
                )
                .unwrap(),
            ),
        };

        let raster_tile = RasterTile2D::new_with_tile_info_and_properties(
            TimeInterval::default(),
            TileInformation {
                global_tile_position: [-1, 0].into(),
                tile_size_in_pixels: [3, 2].into(),
                global_geo_transform: TestDefault::test_default(),
            },
            raster,
            props,
        );

        MockRasterSource {
            params: MockRasterSourceParams {
                data: vec![raster_tile],
                result_descriptor: RasterResultDescriptor {
                    data_type: RasterDataType::F32,
                    spatial_reference: SpatialReference::epsg_4326().into(),
                    measurement: measurement.unwrap_or_else(|| {
                        Measurement::Continuous(ContinuousMeasurement {
                            measurement: "raw".to_string(),
                            unit: None,
                        })
                    }),
                    time: None,
                    bbox: None,
                    resolution: None,
                },
            },
        }
    }

    pub(crate) fn _create_gdal_src(ctx: &mut MockExecutionContext) -> GdalSource {
        let dataset_id: DataId = DatasetId::new().into();

        let no_data_value = Some(0.);
        let meta = GdalMetaDataRegular {
            data_time: TimeInterval::new_unchecked(
                TimeInstance::from_str("2012-12-12T12:00:00.000Z").unwrap(),
                TimeInstance::from_str("2020-12-12T12:00:00.000Z").unwrap(),
            ),
            step: TimeStep {
                granularity: TimeGranularity::Minutes,
                step: 15,
            },
            time_placeholders: hashmap! {
                "%_START_TIME_%".to_string() => GdalSourceTimePlaceholder {
                    format: DateTimeParseFormat::custom("%Y%m%d_%H%M".to_string()),
                    reference: TimeReference::Start,
                },
            },
            params: GdalDatasetParameters {
                file_path: test_data!("raster/msg/%_START_TIME_%.tif").into(),
                rasterband_channel: 1,
                geo_transform: GdalDatasetGeoTransform {
                    origin_coordinate: (-5_570_248.477_339_745, 5_570_248.477_339_745).into(),
                    x_pixel_size: 3_000.403_165_817_261,
                    y_pixel_size: -3_000.403_165_817_261,
                },
                width: 3712,
                height: 3712,
                file_not_found_handling: FileNotFoundHandling::Error,
                no_data_value,
                properties_mapping: Some(vec![
                    GdalMetadataMapping::identity(
                        new_satellite_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                    GdalMetadataMapping::identity(
                        new_channel_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                    GdalMetadataMapping::identity(
                        new_offset_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                    GdalMetadataMapping::identity(
                        new_slope_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                ]),
                gdal_open_options: None,
                gdal_config_options: None,
                allow_alphaband_as_mask: true,
            },
            result_descriptor: RasterResultDescriptor {
                data_type: RasterDataType::I16,
                spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81)
                    .into(),
                measurement: Measurement::Continuous(ContinuousMeasurement {
                    measurement: "raw".to_string(),
                    unit: None,
                }),
                time: None,
                bbox: None,
                resolution: None,
            },
        };
        ctx.add_meta_data(dataset_id.clone(), Box::new(meta));

        GdalSource {
            params: GdalSourceParameters { data: dataset_id },
        }
    }
}

1	use geoengine_datatypes::raster::RasterPropertiesKey;
2
3	mod radiance;
4	mod reflectance;
5	mod satellite;
6	mod temperature;
7
8	fn new_slope_key() -> RasterPropertiesKey {	34✔
9	RasterPropertiesKey {	34✔
10	domain: Some("msg".into()),	34✔
11	key: "calibration_slope".into(),	34✔
12	}	34✔
13	}	34✔
14
15	fn new_offset_key() -> RasterPropertiesKey {	34✔
16	RasterPropertiesKey {	34✔
17	domain: Some("msg".into()),	34✔
18	key: "calibration_offset".into(),	34✔
19	}	34✔
20	}	34✔
21
22	fn new_channel_key() -> RasterPropertiesKey {	46✔
23	RasterPropertiesKey {	46✔
24	domain: Some("msg".into()),	46✔
25	key: "channel_number".into(),	46✔
26	}	46✔
27	}	46✔
28
29	fn new_satellite_key() -> RasterPropertiesKey {	45✔
30	RasterPropertiesKey {	45✔
31	domain: Some("msg".into()),	45✔
32	key: "satellite_number".into(),	45✔
33	}	45✔
34	}	45✔
35
36	#[cfg(test)]
37	mod test_util {
38	use std::str::FromStr;
39
40	use futures::StreamExt;
41	use geoengine_datatypes::hashmap;
42	use geoengine_datatypes::util::test::TestDefault;
43	use num_traits::AsPrimitive;
44
45	use geoengine_datatypes::dataset::{DataId, DatasetId};
46	use geoengine_datatypes::primitives::{
47	ContinuousMeasurement, DateTime, DateTimeParseFormat, Measurement, RasterQueryRectangle,
48	SpatialPartition2D, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval,
49	TimeStep,
50	};
51	use geoengine_datatypes::raster::{
52	Grid2D, GridOrEmpty, GridOrEmpty2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties,
53	RasterPropertiesEntry, RasterPropertiesEntryType, RasterTile2D, TileInformation,
54	};
55	use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority};
56	use geoengine_datatypes::util::Identifier;
57
58	use crate::engine::{
59	MockExecutionContext, MockQueryContext, QueryProcessor, RasterOperator,
60	RasterResultDescriptor,
61	};
62	use crate::mock::{MockRasterSource, MockRasterSourceParams};
63	use crate::processing::meteosat::{
64	new_channel_key, new_offset_key, new_satellite_key, new_slope_key,
65	};
66	use crate::source::{
67	FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular,
68	GdalMetadataMapping, GdalSource, GdalSourceParameters, GdalSourceTimePlaceholder,
69	TimeReference,
70	};
71	use crate::test_data;
72	use crate::util::Result;
73
74	pub(crate) async fn process<T>(	34✔
75	make_op: T,	34✔
76	query: RasterQueryRectangle,	34✔
77	ctx: &MockExecutionContext,	34✔
78	) -> Result<RasterTile2D<f32>>	34✔
79	where	34✔
80	T: FnOnce() -> Box<dyn RasterOperator>,	34✔
81	{	34✔
82	// let input = make_raster(props, custom_data, measurement);
83
84	let op = make_op().initialize(ctx).await?;	34✔
85
86	let processor = op.query_processor().unwrap().get_f32().unwrap();	25✔
87		25✔
88	let ctx = MockQueryContext::test_default();	25✔
89	let result_stream = processor.query(query, &ctx).await.unwrap();	25✔
90	let mut result: Vec<Result<RasterTile2D<f32>>> = result_stream.collect().await;	25✔
91	assert_eq!(1, result.len());	25✔
92	result.pop().unwrap()	25✔
93	}	34✔
94
95	pub(crate) fn create_properties(	34✔
96	channel: Option<u8>,	34✔
97	satellite: Option<u8>,	34✔
98	offset: Option<f64>,	34✔
99	slope: Option<f64>,	34✔
100	) -> RasterProperties {	34✔
101	let mut props = RasterProperties::default();	34✔
102
103	if let Some(v) = channel {	34✔
104	props	25✔
105	.properties_map	25✔
106	.insert(new_channel_key(), RasterPropertiesEntry::Number(v.as_()));	25✔
107	}	25✔
108
109	if let Some(v) = satellite {	34✔
110	props	24✔
111	.properties_map	24✔
112	.insert(new_satellite_key(), RasterPropertiesEntry::Number(v.as_()));	24✔
113	}	24✔
114
115	if let Some(v) = slope {	34✔
116	props	19✔
117	.properties_map	19✔
118	.insert(new_slope_key(), RasterPropertiesEntry::Number(v));	19✔
119	}	19✔
120
121	if let Some(v) = offset {	34✔
122	props	19✔
123	.properties_map	19✔
124	.insert(new_offset_key(), RasterPropertiesEntry::Number(v));	19✔
125	}	19✔
126	props	34✔
127	}	34✔
128
129	pub(crate) fn _create_gdal_query() -> RasterQueryRectangle {	×
130	let sr = SpatialResolution::new_unchecked(3_000.403_165_817_261, 3_000.403_165_817_261);	×
131	let ul = (0., 0.).into();	×
132	let lr = (599. * sr.x, -599. * sr.y).into();	×
133	RasterQueryRectangle {	×
134	spatial_bounds: SpatialPartition2D::new_unchecked(ul, lr),	×
135	time_interval: TimeInterval::new_unchecked(	×
136	TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 0, 0)),	×
137	TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 15, 0)),	×
138	),	×
139	spatial_resolution: sr,	×
140	}	×
141	}	×
142
143	pub(crate) fn create_mock_query() -> RasterQueryRectangle {	34✔
144	RasterQueryRectangle {	34✔
145	spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()),	34✔
146	time_interval: Default::default(),	34✔
147	spatial_resolution: SpatialResolution::one(),	34✔
148	}	34✔
149	}	34✔
150
151	pub(crate) fn create_mock_source<P: Pixel>(	34✔
152	props: RasterProperties,	34✔
153	custom_data: Option<GridOrEmpty2D<P>>,	34✔
154	measurement: Option<Measurement>,	34✔
155	) -> MockRasterSource<P> {	34✔
156	let raster = match custom_data {	34✔
157	Some(g) => g,	4✔
158	None => GridOrEmpty::Grid(	30✔
159	MaskedGrid2D::new(	30✔
160	Grid2D::<P>::new(	30✔
161	[3, 2].into(),	30✔
162	vec![	30✔
163	P::from_(1),	30✔
164	P::from_(2),	30✔
165	P::from_(3),	30✔
166	P::from_(4),	30✔
167	P::from_(5),	30✔
168	P::from_(0),	30✔
169	],	30✔
170	)	30✔
171	.unwrap(),	30✔
172	Grid2D::new([3, 2].into(), vec![true, true, true, true, true, false]).unwrap(),	30✔
173	)	30✔
174	.unwrap(),	30✔
175	),	30✔
176	};
177
178	let raster_tile = RasterTile2D::new_with_tile_info_and_properties(	34✔
179	TimeInterval::default(),	34✔
180	TileInformation {	34✔
181	global_tile_position: [-1, 0].into(),	34✔
182	tile_size_in_pixels: [3, 2].into(),	34✔
183	global_geo_transform: TestDefault::test_default(),	34✔
184	},	34✔
185	raster,	34✔
186	props,	34✔
187	);	34✔
188		34✔
189	MockRasterSource {	34✔
190	params: MockRasterSourceParams {	34✔
191	data: vec![raster_tile],	34✔
192	result_descriptor: RasterResultDescriptor {	34✔
193	data_type: RasterDataType::F32,	34✔
194	spatial_reference: SpatialReference::epsg_4326().into(),	34✔
195	measurement: measurement.unwrap_or_else(\|\| {	34✔
196	Measurement::Continuous(ContinuousMeasurement {	15✔
197	measurement: "raw".to_string(),	15✔
198	unit: None,	15✔
199	})	15✔
200	}),	34✔
201	time: None,	34✔
202	bbox: None,	34✔
203	resolution: None,	34✔
204	},	34✔
205	},	34✔
206	}	34✔
207	}	34✔
208
209	pub(crate) fn _create_gdal_src(ctx: &mut MockExecutionContext) -> GdalSource {	×
210	let dataset_id: DataId = DatasetId::new().into();	×
211		×
212	let no_data_value = Some(0.);	×
213	let meta = GdalMetaDataRegular {	×
214	data_time: TimeInterval::new_unchecked(	×
215	TimeInstance::from_str("2012-12-12T12:00:00.000Z").unwrap(),	×
216	TimeInstance::from_str("2020-12-12T12:00:00.000Z").unwrap(),	×
217	),	×
218	step: TimeStep {	×
219	granularity: TimeGranularity::Minutes,	×
220	step: 15,	×
221	},	×
222	time_placeholders: hashmap! {	×
223	"%_START_TIME_%".to_string() => GdalSourceTimePlaceholder {	×
224	format: DateTimeParseFormat::custom("%Y%m%d_%H%M".to_string()),	×
225	reference: TimeReference::Start,	×
226	},	×
227	},	×
228	params: GdalDatasetParameters {	×
229	file_path: test_data!("raster/msg/%_START_TIME_%.tif").into(),	×
230	rasterband_channel: 1,	×
231	geo_transform: GdalDatasetGeoTransform {	×
232	origin_coordinate: (-5_570_248.477_339_745, 5_570_248.477_339_745).into(),	×
233	x_pixel_size: 3_000.403_165_817_261,	×
234	y_pixel_size: -3_000.403_165_817_261,	×
235	},	×
236	width: 3712,	×
237	height: 3712,	×
238	file_not_found_handling: FileNotFoundHandling::Error,	×
239	no_data_value,	×
240	properties_mapping: Some(vec![	×
241	GdalMetadataMapping::identity(	×
242	new_satellite_key(),	×
243	RasterPropertiesEntryType::Number,	×
244	),	×
245	GdalMetadataMapping::identity(	×
246	new_channel_key(),	×
247	RasterPropertiesEntryType::Number,	×
248	),	×
249	GdalMetadataMapping::identity(	×
250	new_offset_key(),	×
251	RasterPropertiesEntryType::Number,	×
252	),	×
253	GdalMetadataMapping::identity(	×
254	new_slope_key(),	×
255	RasterPropertiesEntryType::Number,	×
256	),	×
257	]),	×
258	gdal_open_options: None,	×
259	gdal_config_options: None,	×
260	allow_alphaband_as_mask: true,	×
261	},	×
262	result_descriptor: RasterResultDescriptor {	×
263	data_type: RasterDataType::I16,	×
264	spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81)	×
265	.into(),	×
266	measurement: Measurement::Continuous(ContinuousMeasurement {	×
267	measurement: "raw".to_string(),	×
268	unit: None,	×
269	}),	×
270	time: None,	×
271	bbox: None,	×
272	resolution: None,	×
273	},	×
274	};	×
275	ctx.add_meta_data(dataset_id.clone(), Box::new(meta));	×
276		×
277	GdalSource {	×
278	params: GdalSourceParameters { data: dataset_id },	×
279	}	×
280	}	×
281	}

geo-engine / geoengine / 3929938005

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous