7006568925

Committed 27 Nov 2023 02:07PM UTC coverage: 89.651% (+0.2%) from 89.498%

Build # 7006568925

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #888 from geo-engine/raster_stacks

raster stacking

Run Details

4032 of 4274 new or added lines in 107 files covered. (94.34%)

12 existing lines in 8 files now uncovered.

113020 of 126066 relevant lines covered (89.65%)

59901.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.85

/operators/src/processing/meteosat/mod.rs

use geoengine_datatypes::raster::RasterPropertiesKey;

mod radiance;
mod reflectance;
mod satellite;
mod temperature;

fn new_slope_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "calibration_slope".into(),
    }
}

fn new_offset_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "calibration_offset".into(),
    }
}

fn new_channel_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "channel_number".into(),
    }
}

fn new_satellite_key() -> RasterPropertiesKey {
    RasterPropertiesKey {
        domain: Some("msg".into()),
        key: "satellite_number".into(),
    }
}

#[cfg(test)]
mod test_util {
    use std::str::FromStr;

    use futures::StreamExt;
    use geoengine_datatypes::hashmap;
    use geoengine_datatypes::primitives::{BandSelection, CacheHint, CacheTtlSeconds};
    use geoengine_datatypes::util::test::TestDefault;
    use num_traits::AsPrimitive;

    use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData};
    use geoengine_datatypes::primitives::{
        ContinuousMeasurement, DateTime, DateTimeParseFormat, Measurement, RasterQueryRectangle,
        SpatialPartition2D, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval,
        TimeStep,
    };
    use geoengine_datatypes::raster::{
        Grid2D, GridOrEmpty, GridOrEmpty2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties,
        RasterPropertiesEntry, RasterPropertiesEntryType, RasterTile2D, TileInformation,
    };
    use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority};
    use geoengine_datatypes::util::Identifier;

    use crate::engine::{
        MockExecutionContext, MockQueryContext, QueryProcessor, RasterBandDescriptor,
        RasterBandDescriptors, RasterOperator, RasterResultDescriptor, WorkflowOperatorPath,
    };
    use crate::mock::{MockRasterSource, MockRasterSourceParams};
    use crate::processing::meteosat::{
        new_channel_key, new_offset_key, new_satellite_key, new_slope_key,
    };
    use crate::source::{
        FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular,
        GdalMetadataMapping, GdalSource, GdalSourceParameters, GdalSourceTimePlaceholder,
        TimeReference,
    };
    use crate::test_data;
    use crate::util::Result;

    pub(crate) async fn process<T>(
        make_op: T,
        query: RasterQueryRectangle,
        ctx: &MockExecutionContext,
    ) -> Result<RasterTile2D<f32>>
    where
        T: FnOnce() -> Box<dyn RasterOperator>,
    {
        // let input = make_raster(props, custom_data, measurement);

        let op = make_op()
            .initialize(WorkflowOperatorPath::initialize_root(), ctx)
            .await?;

        let processor = op.query_processor().unwrap().get_f32().unwrap();

        let ctx = MockQueryContext::test_default();
        let result_stream = processor.query(query, &ctx).await.unwrap();
        let mut result: Vec<Result<RasterTile2D<f32>>> = result_stream.collect().await;
        assert_eq!(1, result.len());
        result.pop().unwrap()
    }

    pub(crate) fn create_properties(
        channel: Option<u8>,
        satellite: Option<u8>,
        offset: Option<f64>,
        slope: Option<f64>,
    ) -> RasterProperties {
        let mut props = RasterProperties::default();

        if let Some(v) = channel {
            props.insert_property(new_channel_key(), RasterPropertiesEntry::Number(v.as_()));
        }

        if let Some(v) = satellite {
            props.insert_property(new_satellite_key(), RasterPropertiesEntry::Number(v.as_()));
        }

        if let Some(v) = slope {
            props.insert_property(new_slope_key(), RasterPropertiesEntry::Number(v));
        }

        if let Some(v) = offset {
            props.insert_property(new_offset_key(), RasterPropertiesEntry::Number(v));
        }
        props
    }

    pub(crate) fn _create_gdal_query() -> RasterQueryRectangle {
        let sr = SpatialResolution::new_unchecked(3_000.403_165_817_261, 3_000.403_165_817_261);
        let ul = (0., 0.).into();
        let lr = (599. * sr.x, -599. * sr.y).into();
        RasterQueryRectangle {
            spatial_bounds: SpatialPartition2D::new_unchecked(ul, lr),
            time_interval: TimeInterval::new_unchecked(
                TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 0, 0)),
                TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 15, 0)),
            ),
            spatial_resolution: sr,
            attributes: BandSelection::first(),
        }
    }

    pub(crate) fn create_mock_query() -> RasterQueryRectangle {
        RasterQueryRectangle {
            spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()),
            time_interval: Default::default(),
            spatial_resolution: SpatialResolution::one(),
            attributes: BandSelection::first(),
        }
    }

    pub(crate) fn create_mock_source<P: Pixel>(
        props: RasterProperties,
        custom_data: Option<GridOrEmpty2D<P>>,
        measurement: Option<Measurement>,
    ) -> MockRasterSource<P> {
        let raster = match custom_data {
            Some(g) => g,
            None => GridOrEmpty::Grid(
                MaskedGrid2D::new(
                    Grid2D::<P>::new(
                        [3, 2].into(),
                        vec![
                            P::from_(1),
                            P::from_(2),
                            P::from_(3),
                            P::from_(4),
                            P::from_(5),
                            P::from_(0),
                        ],
                    )
                    .unwrap(),
                    Grid2D::new([3, 2].into(), vec![true, true, true, true, true, false]).unwrap(),
                )
                .unwrap(),
            ),
        };

        let raster_tile = RasterTile2D::new_with_tile_info_and_properties(
            TimeInterval::default(),
            TileInformation {
                global_tile_position: [-1, 0].into(),
                tile_size_in_pixels: [3, 2].into(),
                global_geo_transform: TestDefault::test_default(),
            },
            0,
            raster,
            props,
            CacheHint::default(),
        );

        MockRasterSource {
            params: MockRasterSourceParams {
                data: vec![raster_tile],
                result_descriptor: RasterResultDescriptor {
                    data_type: RasterDataType::F32,
                    spatial_reference: SpatialReference::epsg_4326().into(),
                    time: None,
                    bbox: None,
                    resolution: None,
                    bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new(
                        "band".into(),
                        measurement.unwrap_or_else(|| {
                            Measurement::Continuous(ContinuousMeasurement {
                                measurement: "raw".to_string(),
                                unit: None,
                            })
                        }),
                    )])
                    .unwrap(),
                },
            },
        }
    }

    pub(crate) fn _create_gdal_src(ctx: &mut MockExecutionContext) -> GdalSource {
        let dataset_id: DataId = DatasetId::new().into();
        let dataset_name = NamedData::with_system_name("gdal-ds");

        let no_data_value = Some(0.);
        let meta = GdalMetaDataRegular {
            data_time: TimeInterval::new_unchecked(
                TimeInstance::from_str("2012-12-12T12:00:00.000Z").unwrap(),
                TimeInstance::from_str("2020-12-12T12:00:00.000Z").unwrap(),
            ),
            step: TimeStep {
                granularity: TimeGranularity::Minutes,
                step: 15,
            },
            time_placeholders: hashmap! {
                "%_START_TIME_%".to_string() => GdalSourceTimePlaceholder {
                    format: DateTimeParseFormat::custom("%Y%m%d_%H%M".to_string()),
                    reference: TimeReference::Start,
                },
            },
            params: GdalDatasetParameters {
                file_path: test_data!("raster/msg/%_START_TIME_%.tif").into(),
                rasterband_channel: 1,
                geo_transform: GdalDatasetGeoTransform {
                    origin_coordinate: (-5_570_248.477_339_745, 5_570_248.477_339_745).into(),
                    x_pixel_size: 3_000.403_165_817_261,
                    y_pixel_size: -3_000.403_165_817_261,
                },
                width: 3712,
                height: 3712,
                file_not_found_handling: FileNotFoundHandling::Error,
                no_data_value,
                properties_mapping: Some(vec![
                    GdalMetadataMapping::identity(
                        new_satellite_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                    GdalMetadataMapping::identity(
                        new_channel_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                    GdalMetadataMapping::identity(
                        new_offset_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                    GdalMetadataMapping::identity(
                        new_slope_key(),
                        RasterPropertiesEntryType::Number,
                    ),
                ]),
                gdal_open_options: None,
                gdal_config_options: None,
                allow_alphaband_as_mask: true,
                retry: None,
            },
            result_descriptor: RasterResultDescriptor {
                data_type: RasterDataType::I16,
                spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81)
                    .into(),
                time: None,
                bbox: None,
                resolution: None,
                bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new(
                    "band".into(),
                    Measurement::Continuous(ContinuousMeasurement {
                        measurement: "raw".to_string(),
                        unit: None,
                    }),
                )])
                .unwrap(),
            },
            cache_ttl: CacheTtlSeconds::default(),
        };
        ctx.add_meta_data(dataset_id, dataset_name.clone(), Box::new(meta));

        GdalSource {
            params: GdalSourceParameters { data: dataset_name },
        }
    }
}

1	use geoengine_datatypes::raster::RasterPropertiesKey;
2
3	mod radiance;
4	mod reflectance;
5	mod satellite;
6	mod temperature;
7
8	fn new_slope_key() -> RasterPropertiesKey {	34✔
9	RasterPropertiesKey {	34✔
10	domain: Some("msg".into()),	34✔
11	key: "calibration_slope".into(),	34✔
12	}	34✔
13	}	34✔
14
15	fn new_offset_key() -> RasterPropertiesKey {	34✔
16	RasterPropertiesKey {	34✔
17	domain: Some("msg".into()),	34✔
18	key: "calibration_offset".into(),	34✔
19	}	34✔
20	}	34✔
21
22	fn new_channel_key() -> RasterPropertiesKey {	46✔
23	RasterPropertiesKey {	46✔
24	domain: Some("msg".into()),	46✔
25	key: "channel_number".into(),	46✔
26	}	46✔
27	}	46✔
28
29	fn new_satellite_key() -> RasterPropertiesKey {	45✔
30	RasterPropertiesKey {	45✔
31	domain: Some("msg".into()),	45✔
32	key: "satellite_number".into(),	45✔
33	}	45✔
34	}	45✔
35
36	#[cfg(test)]
37	mod test_util {
38	use std::str::FromStr;
39
40	use futures::StreamExt;
41	use geoengine_datatypes::hashmap;
42	use geoengine_datatypes::primitives::{BandSelection, CacheHint, CacheTtlSeconds};
43	use geoengine_datatypes::util::test::TestDefault;
44	use num_traits::AsPrimitive;
45
46	use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData};
47	use geoengine_datatypes::primitives::{
48	ContinuousMeasurement, DateTime, DateTimeParseFormat, Measurement, RasterQueryRectangle,
49	SpatialPartition2D, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval,
50	TimeStep,
51	};
52	use geoengine_datatypes::raster::{
53	Grid2D, GridOrEmpty, GridOrEmpty2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties,
54	RasterPropertiesEntry, RasterPropertiesEntryType, RasterTile2D, TileInformation,
55	};
56	use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority};
57	use geoengine_datatypes::util::Identifier;
58
59	use crate::engine::{
60	MockExecutionContext, MockQueryContext, QueryProcessor, RasterBandDescriptor,
61	RasterBandDescriptors, RasterOperator, RasterResultDescriptor, WorkflowOperatorPath,
62	};
63	use crate::mock::{MockRasterSource, MockRasterSourceParams};
64	use crate::processing::meteosat::{
65	new_channel_key, new_offset_key, new_satellite_key, new_slope_key,
66	};
67	use crate::source::{
68	FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular,
69	GdalMetadataMapping, GdalSource, GdalSourceParameters, GdalSourceTimePlaceholder,
70	TimeReference,
71	};
72	use crate::test_data;
73	use crate::util::Result;
74
75	pub(crate) async fn process<T>(	34✔
76	make_op: T,	34✔
77	query: RasterQueryRectangle,	34✔
78	ctx: &MockExecutionContext,	34✔
79	) -> Result<RasterTile2D<f32>>	34✔
80	where	34✔
81	T: FnOnce() -> Box<dyn RasterOperator>,	34✔
82	{	34✔
83	// let input = make_raster(props, custom_data, measurement);
84
85	let op = make_op()	34✔
86	.initialize(WorkflowOperatorPath::initialize_root(), ctx)	34✔
87	.await?;	9✔
88
89	let processor = op.query_processor().unwrap().get_f32().unwrap();	25✔
90		25✔
91	let ctx = MockQueryContext::test_default();	25✔
92	let result_stream = processor.query(query, &ctx).await.unwrap();	25✔
93	let mut result: Vec<Result<RasterTile2D<f32>>> = result_stream.collect().await;	25✔
94	assert_eq!(1, result.len());	25✔
95	result.pop().unwrap()	25✔
96	}	34✔
97
98	pub(crate) fn create_properties(	34✔
99	channel: Option<u8>,	34✔
100	satellite: Option<u8>,	34✔
101	offset: Option<f64>,	34✔
102	slope: Option<f64>,	34✔
103	) -> RasterProperties {	34✔
104	let mut props = RasterProperties::default();	34✔
105
106	if let Some(v) = channel {	34✔
107	props.insert_property(new_channel_key(), RasterPropertiesEntry::Number(v.as_()));	25✔
108	}	25✔
109
110	if let Some(v) = satellite {	34✔
111	props.insert_property(new_satellite_key(), RasterPropertiesEntry::Number(v.as_()));	24✔
112	}	24✔
113
114	if let Some(v) = slope {	34✔
115	props.insert_property(new_slope_key(), RasterPropertiesEntry::Number(v));	19✔
116	}	19✔
117
118	if let Some(v) = offset {	34✔
119	props.insert_property(new_offset_key(), RasterPropertiesEntry::Number(v));	19✔
120	}	19✔
121	props	34✔
122	}	34✔
123
124	pub(crate) fn _create_gdal_query() -> RasterQueryRectangle {	×
125	let sr = SpatialResolution::new_unchecked(3_000.403_165_817_261, 3_000.403_165_817_261);	×
126	let ul = (0., 0.).into();	×
127	let lr = (599. * sr.x, -599. * sr.y).into();	×
128	RasterQueryRectangle {	×
129	spatial_bounds: SpatialPartition2D::new_unchecked(ul, lr),	×
130	time_interval: TimeInterval::new_unchecked(	×
131	TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 0, 0)),	×
132	TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 15, 0)),	×
133	),	×
134	spatial_resolution: sr,	×
NEW 135	attributes: BandSelection::first(),	×
136	}	×
137	}	×
138
139	pub(crate) fn create_mock_query() -> RasterQueryRectangle {	34✔
140	RasterQueryRectangle {	34✔
141	spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()),	34✔
142	time_interval: Default::default(),	34✔
143	spatial_resolution: SpatialResolution::one(),	34✔
144	attributes: BandSelection::first(),	34✔
145	}	34✔
146	}	34✔
147
148	pub(crate) fn create_mock_source<P: Pixel>(	34✔
149	props: RasterProperties,	34✔
150	custom_data: Option<GridOrEmpty2D<P>>,	34✔
151	measurement: Option<Measurement>,	34✔
152	) -> MockRasterSource<P> {	34✔
153	let raster = match custom_data {	34✔
154	Some(g) => g,	4✔
155	None => GridOrEmpty::Grid(	30✔
156	MaskedGrid2D::new(	30✔
157	Grid2D::<P>::new(	30✔
158	[3, 2].into(),	30✔
159	vec![	30✔
160	P::from_(1),	30✔
161	P::from_(2),	30✔
162	P::from_(3),	30✔
163	P::from_(4),	30✔
164	P::from_(5),	30✔
165	P::from_(0),	30✔
166	],	30✔
167	)	30✔
168	.unwrap(),	30✔
169	Grid2D::new([3, 2].into(), vec![true, true, true, true, true, false]).unwrap(),	30✔
170	)	30✔
171	.unwrap(),	30✔
172	),	30✔
173	};
174
175	let raster_tile = RasterTile2D::new_with_tile_info_and_properties(	34✔
176	TimeInterval::default(),	34✔
177	TileInformation {	34✔
178	global_tile_position: [-1, 0].into(),	34✔
179	tile_size_in_pixels: [3, 2].into(),	34✔
180	global_geo_transform: TestDefault::test_default(),	34✔
181	},	34✔
182	0,	34✔
183	raster,	34✔
184	props,	34✔
185	CacheHint::default(),	34✔
186	);	34✔
187		34✔
188	MockRasterSource {	34✔
189	params: MockRasterSourceParams {	34✔
190	data: vec![raster_tile],	34✔
191	result_descriptor: RasterResultDescriptor {	34✔
192	data_type: RasterDataType::F32,	34✔
193	spatial_reference: SpatialReference::epsg_4326().into(),	34✔
194	time: None,	34✔
195	bbox: None,	34✔
196	resolution: None,	34✔
197	bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new(	34✔
198	"band".into(),	34✔
199	measurement.unwrap_or_else(\|\| {	34✔
200	Measurement::Continuous(ContinuousMeasurement {	15✔
201	measurement: "raw".to_string(),	15✔
202	unit: None,	15✔
203	})	15✔
204	}),	34✔
205	)])	34✔
206	.unwrap(),	34✔
207	},	34✔
208	},	34✔
209	}	34✔
210	}	34✔
211
212	pub(crate) fn _create_gdal_src(ctx: &mut MockExecutionContext) -> GdalSource {	×
213	let dataset_id: DataId = DatasetId::new().into();	×
214	let dataset_name = NamedData::with_system_name("gdal-ds");	×
215		×
216	let no_data_value = Some(0.);	×
217	let meta = GdalMetaDataRegular {	×
218	data_time: TimeInterval::new_unchecked(	×
219	TimeInstance::from_str("2012-12-12T12:00:00.000Z").unwrap(),	×
220	TimeInstance::from_str("2020-12-12T12:00:00.000Z").unwrap(),	×
221	),	×
222	step: TimeStep {	×
223	granularity: TimeGranularity::Minutes,	×
224	step: 15,	×
225	},	×
226	time_placeholders: hashmap! {	×
227	"%_START_TIME_%".to_string() => GdalSourceTimePlaceholder {	×
228	format: DateTimeParseFormat::custom("%Y%m%d_%H%M".to_string()),	×
229	reference: TimeReference::Start,	×
230	},	×
231	},	×
232	params: GdalDatasetParameters {	×
233	file_path: test_data!("raster/msg/%_START_TIME_%.tif").into(),	×
234	rasterband_channel: 1,	×
235	geo_transform: GdalDatasetGeoTransform {	×
236	origin_coordinate: (-5_570_248.477_339_745, 5_570_248.477_339_745).into(),	×
237	x_pixel_size: 3_000.403_165_817_261,	×
238	y_pixel_size: -3_000.403_165_817_261,	×
239	},	×
240	width: 3712,	×
241	height: 3712,	×
242	file_not_found_handling: FileNotFoundHandling::Error,	×
243	no_data_value,	×
244	properties_mapping: Some(vec![	×
245	GdalMetadataMapping::identity(	×
246	new_satellite_key(),	×
247	RasterPropertiesEntryType::Number,	×
248	),	×
249	GdalMetadataMapping::identity(	×
250	new_channel_key(),	×
251	RasterPropertiesEntryType::Number,	×
252	),	×
253	GdalMetadataMapping::identity(	×
254	new_offset_key(),	×
255	RasterPropertiesEntryType::Number,	×
256	),	×
257	GdalMetadataMapping::identity(	×
258	new_slope_key(),	×
259	RasterPropertiesEntryType::Number,	×
260	),	×
261	]),	×
262	gdal_open_options: None,	×
263	gdal_config_options: None,	×
264	allow_alphaband_as_mask: true,	×
265	retry: None,	×
266	},	×
267	result_descriptor: RasterResultDescriptor {	×
268	data_type: RasterDataType::I16,	×
269	spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81)	×
270	.into(),	×
271	time: None,	×
272	bbox: None,	×
273	resolution: None,	×
NEW 274	bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new(	×
NEW 275	"band".into(),	×
NEW 276	Measurement::Continuous(ContinuousMeasurement {	×
NEW 277	measurement: "raw".to_string(),	×
NEW 278	unit: None,	×
NEW 279	}),	×
NEW 280	)])	×
NEW 281	.unwrap(),	×
282	},	×
283	cache_ttl: CacheTtlSeconds::default(),	×
284	};	×
285	ctx.add_meta_data(dataset_id, dataset_name.clone(), Box::new(meta));	×
286		×
287	GdalSource {	×
288	params: GdalSourceParameters { data: dataset_name },	×
289	}	×
290	}	×
291	}

geo-engine / geoengine / 7006568925

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous