• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

geo-engine / geoengine / 19240886216

10 Nov 2025 05:48PM UTC coverage: 88.94%. First build
19240886216

Pull #1083

github

web-flow
Merge eef5160d4 into 113de40ca
Pull Request #1083: feat: new gdal source workflow optimization

6215 of 7057 new or added lines in 71 files covered. (88.07%)

116335 of 130802 relevant lines covered (88.94%)

496353.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.38
/operators/src/optimization/mod.rs
1
use geoengine_datatypes::{error::ErrorSource, primitives::SpatialResolution};
2
use snafu::{Snafu, ensure};
3

4
use crate::engine::InitializedRasterOperator;
5

6
#[derive(Debug, Snafu)]
7
#[snafu(visibility(pub(crate)))]
8
#[snafu(context(suffix(false)))] // disables default `Snafu` suffix
9
pub enum OptimizationError {
10
    TargetResolutionMustNotBeSmallerThanSourceResolution {
11
        source_resolution: SpatialResolution,
12
        target_resolution: SpatialResolution,
13
    },
14
    TargetResolutionMustBeDivisibleBySourceResolution {
15
        source_resolution: SpatialResolution,
16
        target_resolution: SpatialResolution,
17
    },
18
    SourcesMustNotUseOverviews {
19
        data: String,
20
        oveview_level: u32,
21
    },
22
    OptimizationNotYetImplementedForOperator {
23
        operator: String,
24
    },
25
    ProjectionOptimizationFailed {
26
        source: Box<dyn ErrorSource>,
27
    },
28
}
29

30
// TODO: move `optimize` method from `RasterOperator` to `OptimizableOperator`
31
pub trait OptimizableOperator: InitializedRasterOperator {
32
    // TODO: automatically call before `optimize` or make it part of the `optimize` method
33
    fn ensure_resolution_is_compatible_for_optimization(
40✔
34
        &self,
40✔
35
        target_resolution: SpatialResolution,
40✔
36
    ) -> Result<(), OptimizationError> {
40✔
37
        let original_resolution = self.result_descriptor().spatial_grid.spatial_resolution();
40✔
38
        ensure!(
40✔
39
            target_resolution >= original_resolution,
40✔
NEW
40
            TargetResolutionMustNotBeSmallerThanSourceResolution {
×
NEW
41
                source_resolution: original_resolution,
×
NEW
42
                target_resolution,
×
NEW
43
            }
×
44
        );
45

46
        ensure!(
40✔
47
            target_resolution.x % original_resolution.x == 0.
40✔
48
                && target_resolution.y % original_resolution.y == 0.,
40✔
NEW
49
            TargetResolutionMustBeDivisibleBySourceResolution {
×
NEW
50
                source_resolution: original_resolution,
×
NEW
51
                target_resolution,
×
NEW
52
            }
×
53
        );
54

55
        Ok(())
40✔
56
    }
40✔
57
}
58

59
impl<T> OptimizableOperator for T where T: InitializedRasterOperator {}
60

61
#[cfg(test)]
62
mod tests {
63
    use geoengine_datatypes::{
64
        collections::VectorDataType,
65
        dataset::{DataId, DatasetId, NamedData},
66
        primitives::{BoundingBox2D, CacheTtlSeconds, VectorQueryRectangle},
67
        raster::{RasterDataType, RenameBands},
68
        spatial_reference::{SpatialReference, SpatialReferenceAuthority},
69
        test_data,
70
        util::{Identifier, test::TestDefault},
71
    };
72

73
    use crate::{
74
        engine::{
75
            MockExecutionContext, MultipleRasterSources, PlotOperator, RasterOperator,
76
            SingleRasterOrVectorSource, SingleRasterSource, SingleVectorMultipleRasterSources,
77
            SingleVectorSource, StaticMetaData, VectorOperator, VectorResultDescriptor,
78
            WorkflowOperatorPath,
79
        },
80
        plot::{Histogram, HistogramBounds, HistogramBuckets, HistogramParams},
81
        processing::{
82
            ColumnNames, ColumnRangeFilter, ColumnRangeFilterParams, DeriveOutRasterSpecsSource,
83
            Downsampling, DownsamplingMethod, DownsamplingParams, DownsamplingResolution,
84
            Expression, ExpressionParams, FeatureAggregationMethod, Interpolation,
85
            InterpolationMethod, InterpolationParams, InterpolationResolution, RasterStacker,
86
            RasterStackerParams, RasterTypeConversion, RasterTypeConversionParams,
87
            RasterVectorJoin, RasterVectorJoinParams, Rasterization, RasterizationParams,
88
            Reprojection, ReprojectionParams, TemporalAggregationMethod,
89
        },
90
        source::{
91
            GdalSource, GdalSourceParameters, OgrSource, OgrSourceDataset,
92
            OgrSourceDatasetTimeType, OgrSourceErrorSpec, OgrSourceParameters,
93
        },
94
        util::{
95
            gdal::{add_ndvi_dataset, add_ndvi_downscaled_3x_dataset},
96
            input::RasterOrVectorOperator,
97
        },
98
    };
99

100
    use super::*;
101

102
    #[tokio::test]
103
    async fn it_optimizes_gdal_source() {
1✔
104
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
105

106
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
107

108
        let gdal = GdalSource {
1✔
109
            params: GdalSourceParameters {
1✔
110
                data: ndvi_id.clone(),
1✔
111
                overview_level: None,
1✔
112
            },
1✔
113
        }
1✔
114
        .boxed();
1✔
115

116
        let gdal_initialized = gdal
1✔
117
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
118
            .await
1✔
119
            .unwrap();
1✔
120

121
        assert_eq!(
1✔
122
            gdal_initialized
1✔
123
                .result_descriptor()
1✔
124
                .spatial_grid
1✔
125
                .spatial_resolution(),
1✔
126
            SpatialResolution::new(0.1, 0.1).unwrap()
1✔
127
        );
128

129
        let gdal_optimized = gdal_initialized
1✔
130
            .optimize(SpatialResolution::new(0.8, 0.8).unwrap())
1✔
131
            .unwrap();
1✔
132

133
        let json = serde_json::to_value(&gdal_optimized).unwrap();
1✔
134

135
        // TODO: add test where downsampling is injected because there are is no overview available with required level
136

137
        // assert_eq!(
138
        //     json,
139
        //     serde_json::json!({
140
        //         "params": {
141
        //             "outputOriginReference": null,
142
        //             "outputResolution": {
143
        //                 "type": "resolution",
144
        //                 "x":0.8,
145
        //                 "y": 0.8
146
        //             },
147
        //             "samplingMethod": "nearestNeighbor"
148
        //         },
149
        //         "sources": {
150
        //             "raster": {
151
        //                 "params": {
152
        //                     "data": "ndvi",
153
        //                     "overviewLevel": 3
154
        //                 },
155
        //                 "type": "GdalSource"
156
        //             }
157
        //         },
158
        //         "type": "Downsampling"
159
        //     })
160
        // );
161

162
        assert_eq!(
1✔
163
            json,
164
            serde_json::json!({
1✔
165
                    "params": {
1✔
166
                        "data": "ndvi",
1✔
167
                        "overviewLevel": 8
1✔
168
                    },
169
                    "type": "GdalSource"
1✔
170
                }
171
            )
172
        );
173

174
        let gdal_optimized_initialized = gdal_optimized
1✔
175
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
176
            .await
1✔
177
            .unwrap();
1✔
178

179
        assert_eq!(
1✔
180
            gdal_optimized_initialized
1✔
181
                .result_descriptor()
1✔
182
                .spatial_grid
1✔
183
                .spatial_resolution(),
1✔
184
            SpatialResolution::new(0.8, 0.8).unwrap()
1✔
185
        );
1✔
186
    }
1✔
187

188
    #[tokio::test]
189
    async fn it_optimizes_expression() {
1✔
190
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
191

192
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
193

194
        let gdal = GdalSource {
1✔
195
            params: GdalSourceParameters {
1✔
196
                data: ndvi_id.clone(),
1✔
197
                overview_level: None,
1✔
198
            },
1✔
199
        }
1✔
200
        .boxed();
1✔
201

202
        let expression = Expression {
1✔
203
            params: ExpressionParams {
1✔
204
                expression: "A + 1".to_string(),
1✔
205
                output_type: RasterDataType::U8,
1✔
206
                output_band: None,
1✔
207
                map_no_data: false,
1✔
208
            },
1✔
209
            sources: SingleRasterSource { raster: gdal },
1✔
210
        }
1✔
211
        .boxed();
1✔
212

213
        let expression_initialized = expression
1✔
214
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
215
            .await
1✔
216
            .unwrap();
1✔
217

218
        assert_eq!(
1✔
219
            expression_initialized
1✔
220
                .result_descriptor()
1✔
221
                .spatial_grid
1✔
222
                .spatial_resolution(),
1✔
223
            SpatialResolution::new(0.1, 0.1).unwrap()
1✔
224
        );
225

226
        let expression_optimized = expression_initialized
1✔
227
            .optimize(SpatialResolution::new(0.8, 0.8).unwrap())
1✔
228
            .unwrap();
1✔
229

230
        let json = serde_json::to_value(&expression_optimized).unwrap();
1✔
231

232
        // assert_eq!(
233
        //     json,
234
        //     serde_json::json!({
235
        //         "params": {
236
        //             "expression": "A + 1",
237
        //             "mapNoData": false,
238
        //             "outputBand": {
239
        //                 "measurement": {
240
        //                     "type": "unitless"
241
        //                 },
242
        //                 "name": "expression"
243
        //             },
244
        //             "outputType": "U8"
245
        //         },
246
        //         "sources": {
247
        //             "raster": {
248
        //                 "params": {
249
        //                     "outputOriginReference": null,
250
        //                     "outputResolution": {
251
        //                         "type": "resolution",
252
        //                         "x": 0.8,
253
        //                         "y": 0.8
254
        //                     },
255
        //                     "samplingMethod": "nearestNeighbor"
256
        //                 },
257
        //                 "sources": {
258
        //                     "raster": {
259
        //                         "params": {
260
        //                             "data": "ndvi",
261
        //                             "overviewLevel": 3
262
        //                         },
263
        //                         "type": "GdalSource"
264
        //                     }
265
        //                 },
266
        //                 "type": "Downsampling"
267
        //             }
268
        //         },
269
        //         "type": "Expression"
270
        //     })
271
        // );
272

273
        assert_eq!(
1✔
274
            json,
275
            serde_json::json!({
1✔
276
                "params": {
1✔
277
                    "expression": "A + 1",
1✔
278
                    "mapNoData": false,
1✔
279
                    "outputBand": {
1✔
280
                        "measurement": {
1✔
281
                            "type": "unitless"
1✔
282
                        },
283
                        "name": "expression"
1✔
284
                    },
285
                    "outputType": "U8"
1✔
286
                },
287
                "sources": {
1✔
288
                    "raster": {
1✔
289
                        "params": {
1✔
290
                            "data": "ndvi",
1✔
291
                            "overviewLevel": 8
1✔
292
                        },
293
                        "type": "GdalSource"
1✔
294
                    }
295
                },
296
                "type": "Expression"
1✔
297
            })
298
        );
299

300
        let expression_optimized_initialized = expression_optimized
1✔
301
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
302
            .await
1✔
303
            .unwrap();
1✔
304

305
        assert_eq!(
1✔
306
            expression_optimized_initialized
1✔
307
                .result_descriptor()
1✔
308
                .spatial_grid
1✔
309
                .spatial_resolution(),
1✔
310
            SpatialResolution::new(0.8, 0.8).unwrap()
1✔
311
        );
1✔
312
    }
1✔
313

314
    #[tokio::test]
315
    async fn it_removes_upsampling() {
1✔
316
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
317

318
        let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx);
1✔
319

320
        let workflow = Interpolation {
1✔
321
            params: InterpolationParams {
1✔
322
                interpolation: InterpolationMethod::NearestNeighbor,
1✔
323
                output_resolution: InterpolationResolution::Resolution(
1✔
324
                    SpatialResolution::new_unchecked(0.15, 0.15),
1✔
325
                ),
1✔
326
                output_origin_reference: None,
1✔
327
            },
1✔
328
            sources: SingleRasterSource {
1✔
329
                raster: GdalSource {
1✔
330
                    params: GdalSourceParameters {
1✔
331
                        data: ndvi_downscaled_3x_id.clone(),
1✔
332
                        overview_level: None,
1✔
333
                    },
1✔
334
                }
1✔
335
                .boxed(),
1✔
336
            },
1✔
337
        }
1✔
338
        .boxed();
1✔
339

340
        let workflow_initialized = workflow
1✔
341
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
342
            .await
1✔
343
            .unwrap();
1✔
344

345
        assert_eq!(
1✔
346
            workflow_initialized
1✔
347
                .result_descriptor()
1✔
348
                .spatial_grid
1✔
349
                .spatial_resolution(),
1✔
350
            SpatialResolution::new(0.15, 0.15).unwrap()
1✔
351
        );
352

353
        let workflow_optimized = workflow_initialized
1✔
354
            .optimize(SpatialResolution::new(0.3, 0.3).unwrap())
1✔
355
            .unwrap();
1✔
356

357
        let json = serde_json::to_value(&workflow_optimized).unwrap();
1✔
358

359
        assert_eq!(
1✔
360
            json,
361
            serde_json::json!({
1✔
362
                    "params": {
1✔
363
                        "data": "ndvi_downscaled_3x",
1✔
364
                        "overviewLevel": 0
1✔
365
                    },
366
                    "type": "GdalSource"
1✔
367
                }
368
            )
369
        );
370

371
        let gdal_optimized_initialized = workflow_optimized
1✔
372
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
373
            .await
1✔
374
            .unwrap();
1✔
375

376
        assert_eq!(
1✔
377
            gdal_optimized_initialized
1✔
378
                .result_descriptor()
1✔
379
                .spatial_grid
1✔
380
                .spatial_resolution(),
1✔
381
            SpatialResolution::new(0.3, 0.3).unwrap()
1✔
382
        );
1✔
383
    }
1✔
384

385
    #[tokio::test]
386
    #[allow(clippy::too_many_lines)]
387
    async fn it_reduces_interpolation_resolution() {
1✔
388
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
389

390
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
391
        let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx);
1✔
392

393
        let workflow = RasterStacker {
1✔
394
            params: RasterStackerParams {
1✔
395
                rename_bands: RenameBands::Default,
1✔
396
            },
1✔
397
            sources: MultipleRasterSources {
1✔
398
                rasters: vec![
1✔
399
                    GdalSource {
1✔
400
                        params: GdalSourceParameters {
1✔
401
                            data: ndvi_id.clone(),
1✔
402
                            overview_level: None,
1✔
403
                        },
1✔
404
                    }
1✔
405
                    .boxed(),
1✔
406
                    Interpolation {
1✔
407
                        params: InterpolationParams {
1✔
408
                            interpolation: InterpolationMethod::NearestNeighbor,
1✔
409
                            output_resolution: InterpolationResolution::Resolution(
1✔
410
                                SpatialResolution::new_unchecked(0.1, 0.1),
1✔
411
                            ),
1✔
412
                            output_origin_reference: None,
1✔
413
                        },
1✔
414
                        sources: SingleRasterSource {
1✔
415
                            raster: GdalSource {
1✔
416
                                params: GdalSourceParameters {
1✔
417
                                    data: ndvi_downscaled_3x_id.clone(),
1✔
418
                                    overview_level: None,
1✔
419
                                },
1✔
420
                            }
1✔
421
                            .boxed(),
1✔
422
                        },
1✔
423
                    }
1✔
424
                    .boxed(),
1✔
425
                ],
1✔
426
            },
1✔
427
        }
1✔
428
        .boxed();
1✔
429

430
        let workflow_initialized = workflow
1✔
431
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
432
            .await
1✔
433
            .unwrap();
1✔
434

435
        assert_eq!(
1✔
436
            workflow_initialized
1✔
437
                .result_descriptor()
1✔
438
                .spatial_grid
1✔
439
                .spatial_resolution(),
1✔
440
            SpatialResolution::new(0.1, 0.1).unwrap()
1✔
441
        );
442

443
        let workflow_optimized = workflow_initialized
1✔
444
            .optimize(SpatialResolution::new(0.2, 0.2).unwrap())
1✔
445
            .unwrap();
1✔
446

447
        let json = serde_json::to_value(&workflow_optimized).unwrap();
1✔
448

449
        assert_eq!(
1✔
450
            json,
451
            serde_json::json!({
1✔
452
                "params": {
1✔
453
                    "renameBands": {
1✔
454
                        "type": "default"
1✔
455
                    }
456
                },
457
                "sources": {
1✔
458
                    "rasters": [
1✔
459
                        {
460
                            "params": {
1✔
461
                                "data": "ndvi",
1✔
462
                                "overviewLevel": 2
1✔
463
                            },
464
                            "type": "GdalSource"
1✔
465
                        },
466
                        {
467
                            "params": {
1✔
468
                                "interpolation": "nearestNeighbor",
1✔
469
                                "outputOriginReference": null,
1✔
470
                                "outputResolution": {
1✔
471
                                    "type": "resolution",
1✔
472
                                    "x": 0.2,
1✔
473
                                    "y": 0.2
1✔
474
                                }
475
                            },
476
                            "sources": {
1✔
477
                                "raster": {
1✔
478
                                    "params": {
1✔
479
                                        "data": "ndvi_downscaled_3x",
1✔
480
                                        "overviewLevel": 0
1✔
481
                                    },
482
                                    "type": "GdalSource"
1✔
483
                                }
484
                            },
485
                            "type": "Interpolation"
1✔
486
                        }
487
                    ]
488
                },
489
                "type": "RasterStacker"
1✔
490
            })
491
        );
492

493
        let gdal_optimized_initialized = workflow_optimized
1✔
494
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
495
            .await
1✔
496
            .unwrap();
1✔
497

498
        assert_eq!(
1✔
499
            gdal_optimized_initialized
1✔
500
                .result_descriptor()
1✔
501
                .spatial_grid
1✔
502
                .spatial_resolution(),
1✔
503
            SpatialResolution::new(0.2, 0.2).unwrap()
1✔
504
        );
1✔
505
    }
1✔
506

507
    #[tokio::test]
508
    #[allow(clippy::too_many_lines)]
509
    async fn it_replaces_upsampling_with_downsampling() {
1✔
510
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
511

512
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
513
        let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx);
1✔
514

515
        let workflow = RasterStacker {
1✔
516
            params: RasterStackerParams {
1✔
517
                rename_bands: RenameBands::Default,
1✔
518
            },
1✔
519
            sources: MultipleRasterSources {
1✔
520
                rasters: vec![
1✔
521
                    GdalSource {
1✔
522
                        params: GdalSourceParameters {
1✔
523
                            data: ndvi_id.clone(),
1✔
524
                            overview_level: None,
1✔
525
                        },
1✔
526
                    }
1✔
527
                    .boxed(),
1✔
528
                    Interpolation {
1✔
529
                        params: InterpolationParams {
1✔
530
                            interpolation: InterpolationMethod::NearestNeighbor,
1✔
531
                            output_resolution: InterpolationResolution::Resolution(
1✔
532
                                SpatialResolution::new_unchecked(0.1, 0.1),
1✔
533
                            ),
1✔
534
                            output_origin_reference: None,
1✔
535
                        },
1✔
536
                        sources: SingleRasterSource {
1✔
537
                            raster: GdalSource {
1✔
538
                                params: GdalSourceParameters {
1✔
539
                                    data: ndvi_downscaled_3x_id.clone(),
1✔
540
                                    overview_level: None,
1✔
541
                                },
1✔
542
                            }
1✔
543
                            .boxed(),
1✔
544
                        },
1✔
545
                    }
1✔
546
                    .boxed(),
1✔
547
                ],
1✔
548
            },
1✔
549
        }
1✔
550
        .boxed();
1✔
551

552
        let workflow_initialized = workflow
1✔
553
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
554
            .await
1✔
555
            .unwrap();
1✔
556

557
        assert_eq!(
1✔
558
            workflow_initialized
1✔
559
                .result_descriptor()
1✔
560
                .spatial_grid
1✔
561
                .spatial_resolution(),
1✔
562
            SpatialResolution::new(0.1, 0.1).unwrap()
1✔
563
        );
564

565
        let workflow_optimized = workflow_initialized
1✔
566
            .optimize(SpatialResolution::new(0.8, 0.8).unwrap())
1✔
567
            .unwrap();
1✔
568

569
        let json = serde_json::to_value(&workflow_optimized).unwrap();
1✔
570

571
        assert_eq!(
1✔
572
            json,
573
            serde_json::json!({
1✔
574
                "params": {
1✔
575
                    "renameBands": {
1✔
576
                        "type": "default"
1✔
577
                    }
578
                },
579
                "sources": {
1✔
580
                    "rasters": [
1✔
581
                        {
582
                            "params": {
1✔
583
                                "data": "ndvi",
1✔
584
                                "overviewLevel": 8
1✔
585
                            },
586
                            "type": "GdalSource"
1✔
587
                        },
588
                        {
589
                            "params": {
1✔
590
                                "outputOriginReference": null,
1✔
591
                                "outputResolution": {
1✔
592
                                    "type": "resolution",
1✔
593
                                    "x": 0.8,
1✔
594
                                    "y": 0.8
1✔
595
                                },
596
                                "samplingMethod": "nearestNeighbor"
1✔
597
                            },
598
                            "sources": {
1✔
599
                                "raster": {
1✔
600
                                    "params": {
1✔
601
                                        "data": "ndvi_downscaled_3x",
1✔
602
                                        "overviewLevel": 2
1✔
603
                                    },
604
                                    "type": "GdalSource"
1✔
605
                                }
606
                            },
607
                            "type": "Downsampling"
1✔
608
                        }
609
                    ]
610
                },
611
                "type": "RasterStacker"
1✔
612
            })
613
        );
614

615
        let gdal_optimized_initialized = workflow_optimized
1✔
616
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
617
            .await
1✔
618
            .unwrap();
1✔
619

620
        assert_eq!(
1✔
621
            gdal_optimized_initialized
1✔
622
                .result_descriptor()
1✔
623
                .spatial_grid
1✔
624
                .spatial_resolution(),
1✔
625
            SpatialResolution::new(0.8, 0.8).unwrap()
1✔
626
        );
1✔
627
    }
1✔
628

629
    #[tokio::test]
630
    #[allow(clippy::too_many_lines)]
631
    async fn it_optimizes_downsampling() {
1✔
632
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
633

634
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
635
        let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx);
1✔
636

637
        let workflow = RasterStacker {
1✔
638
            params: RasterStackerParams {
1✔
639
                rename_bands: RenameBands::Default,
1✔
640
            },
1✔
641
            sources: MultipleRasterSources {
1✔
642
                rasters: vec![
1✔
643
                    GdalSource {
1✔
644
                        params: GdalSourceParameters {
1✔
645
                            data: ndvi_downscaled_3x_id.clone(),
1✔
646
                            overview_level: None,
1✔
647
                        },
1✔
648
                    }
1✔
649
                    .boxed(),
1✔
650
                    Downsampling {
1✔
651
                        params: DownsamplingParams {
1✔
652
                            sampling_method: DownsamplingMethod::NearestNeighbor,
1✔
653
                            output_resolution: DownsamplingResolution::Resolution(
1✔
654
                                SpatialResolution::new_unchecked(0.3, 0.3),
1✔
655
                            ),
1✔
656
                            output_origin_reference: None,
1✔
657
                        },
1✔
658
                        sources: SingleRasterSource {
1✔
659
                            raster: GdalSource {
1✔
660
                                params: GdalSourceParameters {
1✔
661
                                    data: ndvi_id.clone(),
1✔
662
                                    overview_level: None,
1✔
663
                                },
1✔
664
                            }
1✔
665
                            .boxed(),
1✔
666
                        },
1✔
667
                    }
1✔
668
                    .boxed(),
1✔
669
                ],
1✔
670
            },
1✔
671
        }
1✔
672
        .boxed();
1✔
673

674
        let workflow_initialized = workflow
1✔
675
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
676
            .await
1✔
677
            .unwrap();
1✔
678

679
        assert_eq!(
1✔
680
            workflow_initialized
1✔
681
                .result_descriptor()
1✔
682
                .spatial_grid
1✔
683
                .spatial_resolution(),
1✔
684
            SpatialResolution::new(0.3, 0.3).unwrap()
1✔
685
        );
686

687
        let workflow_optimized = workflow_initialized
1✔
688
            .optimize(SpatialResolution::new(0.6, 0.6).unwrap())
1✔
689
            .unwrap();
1✔
690

691
        let json = serde_json::to_value(&workflow_optimized).unwrap();
1✔
692

693
        assert_eq!(
1✔
694
            json,
695
            serde_json::json!({
1✔
696
                "params": {
1✔
697
                    "renameBands": {
1✔
698
                        "type": "default"
1✔
699
                    }
700
                },
701
                "sources": {
1✔
702
                    "rasters": [
1✔
703
                        {
704
                            "params": {
1✔
705
                                "data": "ndvi_downscaled_3x",
1✔
706
                                "overviewLevel": 2
1✔
707
                            },
708
                            "type": "GdalSource"
1✔
709
                        },
710
                        {
711
                            "params": {
1✔
712
                                "outputOriginReference": null,
1✔
713
                                "outputResolution": {
1✔
714
                                    "type": "resolution",
1✔
715
                                    "x": 0.6,
1✔
716
                                    "y": 0.6
1✔
717
                                },
718
                                "samplingMethod": "nearestNeighbor"
1✔
719
                            },
720
                            "sources": {
1✔
721
                                "raster": {
1✔
722
                                    "params": {
1✔
723
                                        "data": "ndvi",
1✔
724
                                        "overviewLevel": 4
1✔
725
                                    },
726
                                    "type": "GdalSource"
1✔
727
                                }
728
                            },
729
                            "type": "Downsampling"
1✔
730
                        }
731
                    ]
732
                },
733
                "type": "RasterStacker"
1✔
734
            })
735
        );
736

737
        let gdal_optimized_initialized = workflow_optimized
1✔
738
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
739
            .await
1✔
740
            .unwrap();
1✔
741

742
        assert_eq!(
1✔
743
            gdal_optimized_initialized
1✔
744
                .result_descriptor()
1✔
745
                .spatial_grid
1✔
746
                .spatial_resolution(),
1✔
747
            SpatialResolution::new(0.6, 0.6).unwrap()
1✔
748
        );
1✔
749
    }
1✔
750

751
    #[tokio::test]
752
    #[allow(clippy::too_many_lines)]
753
    async fn it_optimizes_reprojection() {
1✔
754
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
755

756
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
757

758
        let workflow: Box<dyn RasterOperator> = Box::new(Reprojection {
1✔
759
            params: ReprojectionParams {
1✔
760
                target_spatial_reference: SpatialReference::new(
1✔
761
                    SpatialReferenceAuthority::Epsg,
1✔
762
                    3857,
1✔
763
                ),
1✔
764
                derive_out_spec: DeriveOutRasterSpecsSource::default(),
1✔
765
            },
1✔
766
            sources: SingleRasterOrVectorSource {
1✔
767
                source: RasterOrVectorOperator::Raster(
1✔
768
                    GdalSource {
1✔
769
                        params: GdalSourceParameters {
1✔
770
                            data: ndvi_id.clone(),
1✔
771
                            overview_level: None,
1✔
772
                        },
1✔
773
                    }
1✔
774
                    .boxed(),
1✔
775
                ),
1✔
776
            },
1✔
777
        });
1✔
778

779
        let workflow_initialized = workflow
1✔
780
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
781
            .await
1✔
782
            .unwrap();
1✔
783

784
        let expected_resolution = 14_255.015_508_816_849;
1✔
785

786
        assert_eq!(
1✔
787
            workflow_initialized
1✔
788
                .result_descriptor()
1✔
789
                .spatial_grid
1✔
790
                .spatial_resolution(),
1✔
791
            SpatialResolution::new(expected_resolution, expected_resolution).unwrap()
1✔
792
        );
793

794
        let optimize_resolution = expected_resolution * 2.0;
1✔
795

796
        let workflow_optimized = workflow_initialized
1✔
797
            .optimize(SpatialResolution::new(optimize_resolution, optimize_resolution).unwrap())
1✔
798
            .unwrap();
1✔
799

800
        let json = serde_json::to_value(&workflow_optimized).unwrap();
1✔
801

802
        // require an interpolation in addition to the reprojection
803
        // TODO: should we instead load the source in higher resolution and downsample?
804
        //       or: we could make an exception that the reprojection may not have to produce the exact resolution, but how would we choose the correct resolution for the top level operator then?
805
        assert_eq!(
1✔
806
            json,
807
            serde_json::json!({
1✔
808
                "params": {
1✔
809
                    "interpolation": "nearestNeighbor",
1✔
810
                    "outputOriginReference": null,
1✔
811
                    "outputResolution": {
1✔
812
                        "type": "resolution",
1✔
813
                        "x": 28_510.031_017_633_697,
1✔
814
                        "y": 28_510.031_017_633_697
1✔
815
                    }
816
                },
817
                "sources": {
1✔
818
                    "raster": {
1✔
819
                        "params": {
1✔
820
                            "deriveOutSpec": "projectionBounds",
1✔
821
                            "targetSpatialReference": "EPSG:3857"
1✔
822
                        },
823
                        "sources": {
1✔
824
                            "source": {
1✔
825
                                "params": {
1✔
826
                                    "data": "ndvi",
1✔
827
                                    "overviewLevel": 2
1✔
828
                                },
829
                                "type": "GdalSource"
1✔
830
                            }
831
                        },
832
                        "type": "Reprojection"
1✔
833
                    }
834
                },
835
                "type": "Interpolation"
1✔
836
            })
837
        );
838

839
        let gdal_optimized_initialized = workflow_optimized
1✔
840
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
841
            .await
1✔
842
            .unwrap();
1✔
843

844
        assert_eq!(
1✔
845
            gdal_optimized_initialized
1✔
846
                .result_descriptor()
1✔
847
                .spatial_grid
1✔
848
                .spatial_resolution(),
1✔
849
            SpatialResolution::new(optimize_resolution, optimize_resolution).unwrap()
1✔
850
        );
851

852
        // check that the reprojection was necessary
853
        let workflow: Box<dyn RasterOperator> = Box::new(Reprojection {
1✔
854
            params: ReprojectionParams {
1✔
855
                target_spatial_reference: SpatialReference::new(
1✔
856
                    SpatialReferenceAuthority::Epsg,
1✔
857
                    3857,
1✔
858
                ),
1✔
859
                derive_out_spec: DeriveOutRasterSpecsSource::default(),
1✔
860
            },
1✔
861
            sources: SingleRasterOrVectorSource {
1✔
862
                source: RasterOrVectorOperator::Raster(
1✔
863
                    GdalSource {
1✔
864
                        params: GdalSourceParameters {
1✔
865
                            data: ndvi_id.clone(),
1✔
866
                            overview_level: Some(2),
1✔
867
                        },
1✔
868
                    }
1✔
869
                    .boxed(),
1✔
870
                ),
1✔
871
            },
1✔
872
        });
1✔
873

874
        let workflow_initialized = workflow
1✔
875
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
876
            .await
1✔
877
            .unwrap();
1✔
878

879
        assert!(
1✔
880
            workflow_initialized
1✔
881
                .result_descriptor()
1✔
882
                .spatial_grid
1✔
883
                .spatial_resolution()
1✔
884
                > SpatialResolution::new(optimize_resolution, optimize_resolution).unwrap()
1✔
885
        );
1✔
886
    }
1✔
887

888
    #[tokio::test]
889
    #[allow(clippy::too_many_lines)]
890
    async fn it_optimizes_vector_reprojection() {
1✔
891
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
892

893
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
894
        let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx);
1✔
895

896
        let id: DataId = DatasetId::new().into();
1✔
897
        let name = NamedData::with_system_name("ne_10m_ports");
1✔
898

899
        exe_ctx.add_meta_data::<OgrSourceDataset, VectorResultDescriptor, VectorQueryRectangle>(
1✔
900
            id.clone(),
1✔
901
            name.clone(),
1✔
902
            Box::new(StaticMetaData {
1✔
903
                loading_info: OgrSourceDataset {
1✔
904
                    file_name: test_data!(
1✔
905
                        "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg"
1✔
906
                    )
1✔
907
                    .into(),
1✔
908
                    layer_name: "ne_10m_ports".to_string(),
1✔
909
                    data_type: Some(VectorDataType::MultiPoint),
1✔
910
                    time: OgrSourceDatasetTimeType::None,
1✔
911
                    default_geometry: None,
1✔
912
                    columns: None,
1✔
913
                    force_ogr_time_filter: false,
1✔
914
                    force_ogr_spatial_filter: false,
1✔
915
                    on_error: OgrSourceErrorSpec::Ignore,
1✔
916
                    sql_query: None,
1✔
917
                    attribute_query: None,
1✔
918
                    cache_ttl: CacheTtlSeconds::default(),
1✔
919
                },
1✔
920
                result_descriptor: VectorResultDescriptor {
1✔
921
                    data_type: VectorDataType::MultiPoint,
1✔
922
                    spatial_reference: SpatialReference::epsg_4326().into(),
1✔
923
                    columns: Default::default(),
1✔
924
                    time: None,
1✔
925
                    bbox: Some(BoundingBox2D::new_unchecked(
1✔
926
                        [-171.757_95, -54.809_444].into(),
1✔
927
                        [179.309_364, 78.226_111].into(),
1✔
928
                    )),
1✔
929
                },
1✔
930
                phantom: Default::default(),
1✔
931
            }),
1✔
932
        );
933

934
        let ogr_source = OgrSource {
1✔
935
            params: OgrSourceParameters {
1✔
936
                data: name,
1✔
937
                attribute_projection: None,
1✔
938
                attribute_filters: None,
1✔
939
            },
1✔
940
        }
1✔
941
        .boxed();
1✔
942

943
        let gdal_source = GdalSource {
1✔
944
            params: GdalSourceParameters {
1✔
945
                data: ndvi_id.clone(),
1✔
946
                overview_level: None,
1✔
947
            },
1✔
948
        }
1✔
949
        .boxed();
1✔
950

951
        let gdal_source2 = GdalSource {
1✔
952
            params: GdalSourceParameters {
1✔
953
                data: ndvi_downscaled_3x_id.clone(),
1✔
954
                overview_level: None,
1✔
955
            },
1✔
956
        }
1✔
957
        .boxed();
1✔
958

959
        let raster_vector_join = RasterVectorJoin {
1✔
960
            params: RasterVectorJoinParams {
1✔
961
                names: ColumnNames::Default,
1✔
962
                feature_aggregation: FeatureAggregationMethod::First,
1✔
963
                feature_aggregation_ignore_no_data: false,
1✔
964
                temporal_aggregation: TemporalAggregationMethod::None,
1✔
965
                temporal_aggregation_ignore_no_data: false,
1✔
966
            },
1✔
967
            sources: SingleVectorMultipleRasterSources {
1✔
968
                vector: ogr_source,
1✔
969
                rasters: vec![gdal_source, gdal_source2],
1✔
970
            },
1✔
971
        }
1✔
972
        .boxed();
1✔
973

974
        let vector_reprojection = VectorOperator::boxed(Reprojection {
1✔
975
            params: ReprojectionParams {
1✔
976
                target_spatial_reference: SpatialReference::new(
1✔
977
                    SpatialReferenceAuthority::Epsg,
1✔
978
                    3857,
1✔
979
                ),
1✔
980
                derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds,
1✔
981
            },
1✔
982
            sources: SingleRasterOrVectorSource {
1✔
983
                source: raster_vector_join.into(),
1✔
984
            },
1✔
985
        });
1✔
986

987
        let vector_reprojection_initialized = vector_reprojection
1✔
988
            .clone()
1✔
989
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
990
            .await
1✔
991
            .unwrap();
1✔
992

993
        let vector_reprojection_optimized = vector_reprojection_initialized
1✔
994
            .optimize(SpatialResolution::new(0.8, 0.8).unwrap())
1✔
995
            .unwrap();
1✔
996

997
        let json = serde_json::to_value(&vector_reprojection_optimized).unwrap();
1✔
998

999
        assert_eq!(
1✔
1000
            json,
1001
            serde_json::json!({
1✔
1002
              "params": {
1✔
1003
                "deriveOutSpec": "projectionBounds",
1✔
1004
                "targetSpatialReference": "EPSG:3857"
1✔
1005
              },
1006
              "sources": {
1✔
1007
                "source": {
1✔
1008
                  "params": {
1✔
1009
                    "featureAggregation": "first",
1✔
1010
                    "featureAggregationIgnoreNoData": false,
1✔
1011
                    "names": { "type": "default" },
1✔
1012
                    "temporalAggregation": "none",
1✔
1013
                    "temporalAggregationIgnoreNoData": false
1✔
1014
                  },
1015
                  "sources": {
1✔
1016
                    "rasters": [
1✔
1017
                      {
1018
                        "params": { "data": "ndvi", "overviewLevel": 8 },
1✔
1019
                        "type": "GdalSource"
1✔
1020
                      },
1021
                      {
1022
                        "params": { "data": "ndvi_downscaled_3x", "overviewLevel": 2 },
1✔
1023
                        "type": "GdalSource"
1✔
1024
                      }
1025
                    ],
1026
                    "vector": {
1✔
1027
                      "params": {
1✔
1028
                        "attributeFilters": null,
1✔
1029
                        "attributeProjection": null,
1✔
1030
                        "data": "ne_10m_ports"
1✔
1031
                      },
1032
                      "type": "OgrSource"
1✔
1033
                    }
1034
                  },
1035
                  "type": "RasterVectorJoin"
1✔
1036
                }
1037
              },
1038
              "type": "Reprojection"
1✔
1039
            })
1040
        );
1041

1042
        assert!(
1✔
1043
            vector_reprojection_optimized
1✔
1044
                .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1045
                .await
1✔
1046
                .is_ok()
1✔
1047
        );
1048

1049
        // check case where output is finer than input
1050
        let vector_reprojection_optimized = vector_reprojection_initialized
1✔
1051
            .optimize(SpatialResolution::new(0.05, 0.05).unwrap())
1✔
1052
            .unwrap();
1✔
1053

1054
        let json = serde_json::to_value(&vector_reprojection_optimized).unwrap();
1✔
1055

1056
        assert_eq!(
1✔
1057
            json,
1✔
1058
            serde_json::json!({
1✔
1059
              "params": {
1✔
1060
                "deriveOutSpec": "projectionBounds",
1✔
1061
                "targetSpatialReference": "EPSG:3857"
1✔
1062
              },
1✔
1063
              "sources": {
1✔
1064
                "source": {
1✔
1065
                  "params": {
1✔
1066
                    "featureAggregation": "first",
1✔
1067
                    "featureAggregationIgnoreNoData": false,
1✔
1068
                    "names": { "type": "default" },
1✔
1069
                    "temporalAggregation": "none",
1✔
1070
                    "temporalAggregationIgnoreNoData": false
1✔
1071
                  },
1✔
1072
                  "sources": {
1✔
1073
                    "rasters": [
1✔
1074
                      {
1✔
1075
                        "params": { "data": "ndvi", "overviewLevel": 0 },
1✔
1076
                        "type": "GdalSource"
1✔
1077
                      },
1✔
1078
                      {
1✔
1079
                        "params": { "data": "ndvi_downscaled_3x", "overviewLevel": 0 },
1✔
1080
                        "type": "GdalSource"
1✔
1081
                      }
1✔
1082
                    ],
1✔
1083
                    "vector": {
1✔
1084
                      "params": {
1✔
1085
                        "attributeFilters": null,
1✔
1086
                        "attributeProjection": null,
1✔
1087
                        "data": "ne_10m_ports"
1✔
1088
                      },
1✔
1089
                      "type": "OgrSource"
1✔
1090
                    }
1✔
1091
                  },
1✔
1092
                  "type": "RasterVectorJoin"
1✔
1093
                }
1✔
1094
              },
1✔
1095
              "type": "Reprojection"
1✔
1096
            })
1✔
1097
        );
1✔
1098
    }
1✔
1099

1100
    #[tokio::test]
1101
    #[allow(clippy::too_many_lines)]
1102
    async fn it_optimizes_rasterization() {
1✔
1103
        let id: DataId = DatasetId::new().into();
1✔
1104
        let name = NamedData::with_system_name("ne_10m_ports");
1✔
1105
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
1106
        exe_ctx.add_meta_data::<OgrSourceDataset, VectorResultDescriptor, VectorQueryRectangle>(
1✔
1107
            id.clone(),
1✔
1108
            name.clone(),
1✔
1109
            Box::new(StaticMetaData {
1✔
1110
                loading_info: OgrSourceDataset {
1✔
1111
                    file_name: test_data!(
1✔
1112
                        "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg"
1✔
1113
                    )
1✔
1114
                    .into(),
1✔
1115
                    layer_name: "ne_10m_ports".to_string(),
1✔
1116
                    data_type: Some(VectorDataType::MultiPoint),
1✔
1117
                    time: OgrSourceDatasetTimeType::None,
1✔
1118
                    default_geometry: None,
1✔
1119
                    columns: None,
1✔
1120
                    force_ogr_time_filter: false,
1✔
1121
                    force_ogr_spatial_filter: false,
1✔
1122
                    on_error: OgrSourceErrorSpec::Ignore,
1✔
1123
                    sql_query: None,
1✔
1124
                    attribute_query: None,
1✔
1125
                    cache_ttl: CacheTtlSeconds::default(),
1✔
1126
                },
1✔
1127
                result_descriptor: VectorResultDescriptor {
1✔
1128
                    data_type: VectorDataType::MultiPoint,
1✔
1129
                    spatial_reference: SpatialReference::epsg_4326().into(),
1✔
1130
                    columns: Default::default(),
1✔
1131
                    time: None,
1✔
1132
                    bbox: Some(BoundingBox2D::new_unchecked(
1✔
1133
                        [-171.757_95, -54.809_444].into(),
1✔
1134
                        [179.309_364, 78.226_111].into(),
1✔
1135
                    )),
1✔
1136
                },
1✔
1137
                phantom: Default::default(),
1✔
1138
            }),
1✔
1139
        );
1140

1141
        let source = OgrSource {
1✔
1142
            params: OgrSourceParameters {
1✔
1143
                data: name,
1✔
1144
                attribute_projection: None,
1✔
1145
                attribute_filters: None,
1✔
1146
            },
1✔
1147
        }
1✔
1148
        .boxed();
1✔
1149

1150
        let rasterization = Rasterization {
1✔
1151
            params: RasterizationParams {
1✔
1152
                spatial_resolution: SpatialResolution::new_unchecked(0.1, 0.1),
1✔
1153
                origin_coordinate: (0., 0.).into(),
1✔
1154
                density_params: None,
1✔
1155
            },
1✔
1156
            sources: SingleVectorSource { vector: source },
1✔
1157
        }
1✔
1158
        .boxed();
1✔
1159

1160
        let rasterization_initialized = rasterization
1✔
1161
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1162
            .await
1✔
1163
            .unwrap();
1✔
1164

1165
        assert_eq!(
1✔
1166
            rasterization_initialized
1✔
1167
                .result_descriptor()
1✔
1168
                .spatial_grid
1✔
1169
                .spatial_resolution(),
1✔
1170
            SpatialResolution::new(0.1, 0.1).unwrap()
1✔
1171
        );
1172

1173
        let rasterization_optimized = rasterization_initialized
1✔
1174
            .optimize(SpatialResolution::new(0.8, 0.8).unwrap())
1✔
1175
            .unwrap();
1✔
1176

1177
        let json = serde_json::to_value(&rasterization_optimized).unwrap();
1✔
1178

1179
        assert_eq!(
1✔
1180
            json,
1181
            serde_json::json!({
1✔
1182
              "params": {
1✔
1183
                "densityParams": null,
1✔
1184
                "originCoordinate": {
1✔
1185
                  "x": 0.0,
1✔
1186
                  "y": 0.0
1✔
1187
                },
1188
                "spatialResolution": {
1✔
1189
                  "x": 0.8,
1✔
1190
                  "y": 0.8
1✔
1191
                }
1192
              },
1193
              "sources": {
1✔
1194
                "vector": {
1✔
1195
                  "params": {
1✔
1196
                    "attributeFilters": null,
1✔
1197
                    "attributeProjection": null,
1✔
1198
                    "data": "ne_10m_ports"
1✔
1199
                  },
1200
                  "type": "OgrSource"
1✔
1201
                }
1202
              },
1203
              "type": "Rasterization"
1✔
1204
            })
1205
        );
1206

1207
        let expression_optimized_initialized = rasterization_optimized
1✔
1208
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1209
            .await
1✔
1210
            .unwrap();
1✔
1211

1212
        assert_eq!(
1✔
1213
            expression_optimized_initialized
1✔
1214
                .result_descriptor()
1✔
1215
                .spatial_grid
1✔
1216
                .spatial_resolution(),
1✔
1217
            SpatialResolution::new(0.8, 0.8).unwrap()
1✔
1218
        );
1✔
1219
    }
1✔
1220

1221
    #[tokio::test]
1222
    #[allow(clippy::too_many_lines)]
1223
    async fn it_optimizes_raster_vector_join() {
1✔
1224
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
1225

1226
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
1227
        let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx);
1✔
1228

1229
        let id: DataId = DatasetId::new().into();
1✔
1230
        let name = NamedData::with_system_name("ne_10m_ports");
1✔
1231

1232
        exe_ctx.add_meta_data::<OgrSourceDataset, VectorResultDescriptor, VectorQueryRectangle>(
1✔
1233
            id.clone(),
1✔
1234
            name.clone(),
1✔
1235
            Box::new(StaticMetaData {
1✔
1236
                loading_info: OgrSourceDataset {
1✔
1237
                    file_name: test_data!(
1✔
1238
                        "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg"
1✔
1239
                    )
1✔
1240
                    .into(),
1✔
1241
                    layer_name: "ne_10m_ports".to_string(),
1✔
1242
                    data_type: Some(VectorDataType::MultiPoint),
1✔
1243
                    time: OgrSourceDatasetTimeType::None,
1✔
1244
                    default_geometry: None,
1✔
1245
                    columns: None,
1✔
1246
                    force_ogr_time_filter: false,
1✔
1247
                    force_ogr_spatial_filter: false,
1✔
1248
                    on_error: OgrSourceErrorSpec::Ignore,
1✔
1249
                    sql_query: None,
1✔
1250
                    attribute_query: None,
1✔
1251
                    cache_ttl: CacheTtlSeconds::default(),
1✔
1252
                },
1✔
1253
                result_descriptor: VectorResultDescriptor {
1✔
1254
                    data_type: VectorDataType::MultiPoint,
1✔
1255
                    spatial_reference: SpatialReference::epsg_4326().into(),
1✔
1256
                    columns: Default::default(),
1✔
1257
                    time: None,
1✔
1258
                    bbox: Some(BoundingBox2D::new_unchecked(
1✔
1259
                        [-171.757_95, -54.809_444].into(),
1✔
1260
                        [179.309_364, 78.226_111].into(),
1✔
1261
                    )),
1✔
1262
                },
1✔
1263
                phantom: Default::default(),
1✔
1264
            }),
1✔
1265
        );
1266

1267
        let ogr_source = OgrSource {
1✔
1268
            params: OgrSourceParameters {
1✔
1269
                data: name,
1✔
1270
                attribute_projection: None,
1✔
1271
                attribute_filters: None,
1✔
1272
            },
1✔
1273
        }
1✔
1274
        .boxed();
1✔
1275

1276
        let gdal_source = GdalSource {
1✔
1277
            params: GdalSourceParameters {
1✔
1278
                data: ndvi_id.clone(),
1✔
1279
                overview_level: None,
1✔
1280
            },
1✔
1281
        }
1✔
1282
        .boxed();
1✔
1283

1284
        let gdal_source2 = GdalSource {
1✔
1285
            params: GdalSourceParameters {
1✔
1286
                data: ndvi_downscaled_3x_id.clone(),
1✔
1287
                overview_level: None,
1✔
1288
            },
1✔
1289
        }
1✔
1290
        .boxed();
1✔
1291

1292
        let raster_vector_join = RasterVectorJoin {
1✔
1293
            params: RasterVectorJoinParams {
1✔
1294
                names: ColumnNames::Default,
1✔
1295
                feature_aggregation: FeatureAggregationMethod::First,
1✔
1296
                feature_aggregation_ignore_no_data: false,
1✔
1297
                temporal_aggregation: TemporalAggregationMethod::None,
1✔
1298
                temporal_aggregation_ignore_no_data: false,
1✔
1299
            },
1✔
1300
            sources: SingleVectorMultipleRasterSources {
1✔
1301
                vector: ogr_source,
1✔
1302
                rasters: vec![gdal_source, gdal_source2],
1✔
1303
            },
1✔
1304
        }
1✔
1305
        .boxed();
1✔
1306

1307
        let raster_vector_join_initialized = raster_vector_join
1✔
1308
            .clone()
1✔
1309
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1310
            .await
1✔
1311
            .unwrap();
1✔
1312

1313
        let raster_vector_join_optimized = raster_vector_join_initialized
1✔
1314
            .optimize(SpatialResolution::new(0.8, 0.8).unwrap())
1✔
1315
            .unwrap();
1✔
1316

1317
        let json = serde_json::to_value(&raster_vector_join_optimized).unwrap();
1✔
1318

1319
        assert_eq!(
1✔
1320
            json,
1321
            serde_json::json!({
1✔
1322
                "params": {
1✔
1323
                    "featureAggregation": "first",
1✔
1324
                    "featureAggregationIgnoreNoData": false,
1✔
1325
                    "names": {
1✔
1326
                        "type": "default"
1✔
1327
                    },
1328
                    "temporalAggregation": "none",
1✔
1329
                    "temporalAggregationIgnoreNoData": false
1✔
1330
                },
1331
                "sources": {
1✔
1332
                    "rasters": [
1✔
1333
                        {
1334
                            "params": {
1✔
1335
                                "data": "ndvi",
1✔
1336
                                "overviewLevel": 8
1✔
1337
                            },
1338
                            "type": "GdalSource"
1✔
1339
                        },
1340
                        {
1341
                            "params": {
1✔
1342
                                "data": "ndvi_downscaled_3x",
1✔
1343
                                "overviewLevel": 2
1✔
1344
                            },
1345
                            "type": "GdalSource"
1✔
1346
                        }
1347
                    ],
1348
                    "vector": {
1✔
1349
                        "params": {
1✔
1350
                            "attributeFilters": null,
1✔
1351
                            "attributeProjection": null,
1✔
1352
                            "data": "ne_10m_ports"
1✔
1353
                        },
1354
                        "type": "OgrSource"
1✔
1355
                    }
1356
                },
1357
                "type": "RasterVectorJoin"
1✔
1358
            })
1359
        );
1360

1361
        assert!(
1✔
1362
            raster_vector_join_optimized
1✔
1363
                .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1364
                .await
1✔
1365
                .is_ok()
1✔
1366
        );
1367

1368
        // check case where output is finer than input
1369
        let raster_vector_join_optimized = raster_vector_join_initialized
1✔
1370
            .optimize(SpatialResolution::new(0.05, 0.05).unwrap())
1✔
1371
            .unwrap();
1✔
1372

1373
        let json = serde_json::to_value(&raster_vector_join_optimized).unwrap();
1✔
1374

1375
        assert_eq!(
1✔
1376
            json,
1✔
1377
            serde_json::json!({
1✔
1378
                "params": {
1✔
1379
                    "featureAggregation": "first",
1✔
1380
                    "featureAggregationIgnoreNoData": false,
1✔
1381
                    "names": {
1✔
1382
                        "type": "default"
1✔
1383
                    },
1✔
1384
                    "temporalAggregation": "none",
1✔
1385
                    "temporalAggregationIgnoreNoData": false
1✔
1386
                },
1✔
1387
                "sources": {
1✔
1388
                    "rasters": [
1✔
1389
                        {
1✔
1390
                            "params": {
1✔
1391
                                "data": "ndvi",
1✔
1392
                                "overviewLevel": 0
1✔
1393
                            },
1✔
1394
                            "type": "GdalSource"
1✔
1395
                        },
1✔
1396
                        {
1✔
1397
                            "params": {
1✔
1398
                                "data": "ndvi_downscaled_3x",
1✔
1399
                                "overviewLevel": 0
1✔
1400
                            },
1✔
1401
                            "type": "GdalSource"
1✔
1402
                        }
1✔
1403
                    ],
1✔
1404
                    "vector": {
1✔
1405
                        "params": {
1✔
1406
                            "attributeFilters": null,
1✔
1407
                            "attributeProjection": null,
1✔
1408
                            "data": "ne_10m_ports"
1✔
1409
                        },
1✔
1410
                        "type": "OgrSource"
1✔
1411
                    }
1✔
1412
                },
1✔
1413
                "type": "RasterVectorJoin"
1✔
1414
            })
1✔
1415
        );
1✔
1416
    }
1✔
1417

1418
    #[tokio::test]
1419
    #[allow(clippy::too_many_lines)]
1420
    async fn it_optimizes_complex_workflow() {
1✔
1421
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
1422

1423
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
1424
        let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx);
1✔
1425

1426
        let id: DataId = DatasetId::new().into();
1✔
1427
        let ports_name = NamedData::with_system_name("ne_10m_ports");
1✔
1428

1429
        exe_ctx.add_meta_data::<OgrSourceDataset, VectorResultDescriptor, VectorQueryRectangle>(
1✔
1430
            id.clone(),
1✔
1431
            ports_name.clone(),
1✔
1432
            Box::new(StaticMetaData {
1✔
1433
                loading_info: OgrSourceDataset {
1✔
1434
                    file_name: test_data!(
1✔
1435
                        "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg"
1✔
1436
                    )
1✔
1437
                    .into(),
1✔
1438
                    layer_name: "ne_10m_ports".to_string(),
1✔
1439
                    data_type: Some(VectorDataType::MultiPoint),
1✔
1440
                    time: OgrSourceDatasetTimeType::None,
1✔
1441
                    default_geometry: None,
1✔
1442
                    columns: None,
1✔
1443
                    force_ogr_time_filter: false,
1✔
1444
                    force_ogr_spatial_filter: false,
1✔
1445
                    on_error: OgrSourceErrorSpec::Ignore,
1✔
1446
                    sql_query: None,
1✔
1447
                    attribute_query: None,
1✔
1448
                    cache_ttl: CacheTtlSeconds::default(),
1✔
1449
                },
1✔
1450
                result_descriptor: VectorResultDescriptor {
1✔
1451
                    data_type: VectorDataType::MultiPoint,
1✔
1452
                    spatial_reference: SpatialReference::epsg_4326().into(),
1✔
1453
                    columns: Default::default(),
1✔
1454
                    time: None,
1✔
1455
                    bbox: Some(BoundingBox2D::new_unchecked(
1✔
1456
                        [-171.757_95, -54.809_444].into(),
1✔
1457
                        [179.309_364, 78.226_111].into(),
1✔
1458
                    )),
1✔
1459
                },
1✔
1460
                phantom: Default::default(),
1✔
1461
            }),
1✔
1462
        );
1463

1464
        let workflow = Expression {
1✔
1465
            params: ExpressionParams {
1✔
1466
                expression: "A + B".to_string(),
1✔
1467
                output_type: RasterDataType::F64,
1✔
1468
                output_band: None,
1✔
1469
                map_no_data: false,
1✔
1470
            },
1✔
1471
            sources: SingleRasterSource {
1✔
1472
                raster: RasterStacker {
1✔
1473
                    params: RasterStackerParams {
1✔
1474
                        rename_bands: RenameBands::Default,
1✔
1475
                    },
1✔
1476
                    sources: MultipleRasterSources {
1✔
1477
                        rasters: vec![
1✔
1478
                            RasterTypeConversion {
1✔
1479
                                params: RasterTypeConversionParams {
1✔
1480
                                    output_data_type: RasterDataType::F64,
1✔
1481
                                },
1✔
1482
                                sources: SingleRasterSource {
1✔
1483
                                    raster: GdalSource {
1✔
1484
                                        params: GdalSourceParameters {
1✔
1485
                                            data: ndvi_downscaled_3x_id.clone(),
1✔
1486
                                            overview_level: None,
1✔
1487
                                        },
1✔
1488
                                    }
1✔
1489
                                    .boxed(),
1✔
1490
                                },
1✔
1491
                            }
1✔
1492
                            .boxed(),
1✔
1493
                            Rasterization {
1✔
1494
                                params: RasterizationParams {
1✔
1495
                                    spatial_resolution: SpatialResolution::new_unchecked(0.3, 0.3),
1✔
1496
                                    origin_coordinate: (0., 0.).into(),
1✔
1497
                                    density_params: None,
1✔
1498
                                },
1✔
1499
                                sources: SingleVectorSource {
1✔
1500
                                    vector: ColumnRangeFilter {
1✔
1501
                                        params: ColumnRangeFilterParams {
1✔
1502
                                            column: "natlscale".to_string(),
1✔
1503
                                            ranges: vec![(1..=2).into()],
1✔
1504
                                            keep_nulls: false,
1✔
1505
                                        },
1✔
1506
                                        sources: SingleVectorSource {
1✔
1507
                                            vector: RasterVectorJoin {
1✔
1508
                                                params: RasterVectorJoinParams {
1✔
1509
                                                    names: ColumnNames::Default,
1✔
1510
                                                    feature_aggregation:
1✔
1511
                                                        FeatureAggregationMethod::First,
1✔
1512
                                                    feature_aggregation_ignore_no_data: false,
1✔
1513
                                                    temporal_aggregation:
1✔
1514
                                                        TemporalAggregationMethod::None,
1✔
1515
                                                    temporal_aggregation_ignore_no_data: false,
1✔
1516
                                                },
1✔
1517
                                                sources: SingleVectorMultipleRasterSources {
1✔
1518
                                                    rasters: vec![Downsampling {
1✔
1519
                                                params: DownsamplingParams {
1✔
1520
                                                    sampling_method:
1✔
1521
                                                        DownsamplingMethod::NearestNeighbor,
1✔
1522
                                                    output_resolution:
1✔
1523
                                                        DownsamplingResolution::Resolution(
1✔
1524
                                                            SpatialResolution::new_unchecked(
1✔
1525
                                                                0.3, 0.3,
1✔
1526
                                                            ),
1✔
1527
                                                        ),
1✔
1528
                                                    output_origin_reference: None,
1✔
1529
                                                },
1✔
1530
                                                sources: SingleRasterSource {
1✔
1531
                                                    raster: GdalSource {
1✔
1532
                                                        params: GdalSourceParameters {
1✔
1533
                                                            data: ndvi_id.clone(),
1✔
1534
                                                            overview_level: None,
1✔
1535
                                                        },
1✔
1536
                                                    }
1✔
1537
                                                    .boxed(),
1✔
1538
                                                },
1✔
1539
                                            }
1✔
1540
                                            .boxed()],
1✔
1541
                                                    vector: OgrSource {
1✔
1542
                                                        params: OgrSourceParameters {
1✔
1543
                                                            data: ports_name,
1✔
1544
                                                            attribute_projection: None,
1✔
1545
                                                            attribute_filters: None,
1✔
1546
                                                        },
1✔
1547
                                                    }
1✔
1548
                                                    .boxed(),
1✔
1549
                                                },
1✔
1550
                                            }
1✔
1551
                                            .boxed(),
1✔
1552
                                        },
1✔
1553
                                    }
1✔
1554
                                    .boxed(),
1✔
1555
                                },
1✔
1556
                            }
1✔
1557
                            .boxed(),
1✔
1558
                        ],
1✔
1559
                    },
1✔
1560
                }
1✔
1561
                .boxed(),
1✔
1562
            },
1✔
1563
        }
1✔
1564
        .boxed();
1✔
1565

1566
        let workflow_initialized = workflow
1✔
1567
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1568
            .await
1✔
1569
            .unwrap();
1✔
1570

1571
        assert_eq!(
1✔
1572
            workflow_initialized
1✔
1573
                .result_descriptor()
1✔
1574
                .spatial_grid
1✔
1575
                .spatial_resolution(),
1✔
1576
            SpatialResolution::new(0.3, 0.3).unwrap()
1✔
1577
        );
1578

1579
        let workflow_optimized = workflow_initialized
1✔
1580
            .optimize(SpatialResolution::new(0.6, 0.6).unwrap())
1✔
1581
            .unwrap();
1✔
1582

1583
        let json = serde_json::to_value(&workflow_optimized).unwrap();
1✔
1584

1585
        assert_eq!(
1✔
1586
            json,
1587
            serde_json::json!({
1✔
1588
                "params": {
1✔
1589
                    "expression": "A + B",
1✔
1590
                    "mapNoData": false,
1✔
1591
                    "outputBand": {
1✔
1592
                        "measurement": {
1✔
1593
                            "type": "unitless"
1✔
1594
                        },
1595
                        "name": "expression"
1✔
1596
                    },
1597
                    "outputType": "F64"
1✔
1598
                },
1599
                "sources": {
1✔
1600
                    "raster": {
1✔
1601
                        "params": {
1✔
1602
                            "renameBands": {
1✔
1603
                                "type": "default"
1✔
1604
                            }
1605
                        },
1606
                        "sources": {
1✔
1607
                            "rasters": [
1✔
1608
                                {
1609
                                    "params": {
1✔
1610
                                        "outputDataType": "F64"
1✔
1611
                                    },
1612
                                    "sources": {
1✔
1613
                                        "raster": {
1✔
1614
                                            "params": {
1✔
1615
                                                "data": "ndvi_downscaled_3x",
1✔
1616
                                                "overviewLevel": 2
1✔
1617
                                            },
1618
                                            "type": "GdalSource"
1✔
1619
                                        }
1620
                                    },
1621
                                    "type": "RasterTypeConversion"
1✔
1622
                                },
1623
                                {
1624
                                    "params": {
1✔
1625
                                        "densityParams": null,
1✔
1626
                                        "originCoordinate": {
1✔
1627
                                            "x": 0.0,
1✔
1628
                                            "y": 0.0
1✔
1629
                                        },
1630
                                        "spatialResolution": {
1✔
1631
                                            "x": 0.6,
1✔
1632
                                            "y": 0.6
1✔
1633
                                        }
1634
                                    },
1635
                                    "sources": {
1✔
1636
                                        "vector": {
1✔
1637
                                            "params": {
1✔
1638
                                                "column": "natlscale",
1✔
1639
                                                "keepNulls": false,
1✔
1640
                                                "ranges": [
1✔
1641
                                                    [
1642
                                                        1,
1643
                                                        2
1644
                                                    ]
1645
                                                ]
1646
                                            },
1647
                                            "sources": {
1✔
1648
                                                "vector": {
1✔
1649
                                                    "params": {
1✔
1650
                                                        "featureAggregation": "first",
1✔
1651
                                                        "featureAggregationIgnoreNoData": false,
1✔
1652
                                                        "names": {
1✔
1653
                                                            "type": "default"
1✔
1654
                                                        },
1655
                                                        "temporalAggregation": "none",
1✔
1656
                                                        "temporalAggregationIgnoreNoData": false
1✔
1657
                                                    },
1658
                                                    "sources": {
1✔
1659
                                                        "rasters": [
1✔
1660
                                                            {
1661
                                                                "params": {
1✔
1662
                                                                    "outputOriginReference": null,
1✔
1663
                                                                    "outputResolution": {
1✔
1664
                                                                        "type": "resolution",
1✔
1665
                                                                        "x": 0.6,
1✔
1666
                                                                        "y": 0.6
1✔
1667
                                                                    },
1668
                                                                    "samplingMethod": "nearestNeighbor"
1✔
1669
                                                                },
1670
                                                                "sources": {
1✔
1671
                                                                    "raster": {
1✔
1672
                                                                        "params": {
1✔
1673
                                                                            "data": "ndvi",
1✔
1674
                                                                            "overviewLevel": 4
1✔
1675
                                                                        },
1676
                                                                        "type": "GdalSource"
1✔
1677
                                                                    }
1678
                                                                },
1679
                                                                "type": "Downsampling"
1✔
1680
                                                            }
1681
                                                        ],
1682
                                                        "vector": {
1✔
1683
                                                            "params": {
1✔
1684
                                                                "attributeFilters": null,
1✔
1685
                                                                "attributeProjection": null,
1✔
1686
                                                                "data": "ne_10m_ports"
1✔
1687
                                                            },
1688
                                                            "type": "OgrSource"
1✔
1689
                                                        }
1690
                                                    },
1691
                                                    "type": "RasterVectorJoin"
1✔
1692
                                                }
1693
                                            },
1694
                                            "type": "ColumnRangeFilter"
1✔
1695
                                        }
1696
                                    },
1697
                                    "type": "Rasterization"
1✔
1698
                                }
1699
                            ]
1700
                        },
1701
                        "type": "RasterStacker"
1✔
1702
                    }
1703
                },
1704
                "type": "Expression"
1✔
1705
            })
1706
        );
1707

1708
        let gdal_optimized_initialized = workflow_optimized
1✔
1709
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1710
            .await
1✔
1711
            .unwrap();
1✔
1712

1713
        assert_eq!(
1✔
1714
            gdal_optimized_initialized
1✔
1715
                .result_descriptor()
1✔
1716
                .spatial_grid
1✔
1717
                .spatial_resolution(),
1✔
1718
            SpatialResolution::new(0.6, 0.6).unwrap()
1✔
1719
        );
1✔
1720
    }
1✔
1721

1722
    #[tokio::test]
1723
    async fn it_optimizes_plot() {
1✔
1724
        let mut exe_ctx = MockExecutionContext::test_default();
1✔
1725

1726
        let ndvi_id = add_ndvi_dataset(&mut exe_ctx);
1✔
1727

1728
        let gdal = GdalSource {
1✔
1729
            params: GdalSourceParameters {
1✔
1730
                data: ndvi_id.clone(),
1✔
1731
                overview_level: None,
1✔
1732
            },
1✔
1733
        }
1✔
1734
        .boxed();
1✔
1735

1736
        let expression = Histogram {
1✔
1737
            params: HistogramParams {
1✔
1738
                attribute_name: "ndvi".to_string(),
1✔
1739
                bounds: HistogramBounds::Values {
1✔
1740
                    min: 0.0,
1✔
1741
                    max: 255.0,
1✔
1742
                },
1✔
1743
                buckets: HistogramBuckets::Number { value: 8 },
1✔
1744
                interactive: false,
1✔
1745
            },
1✔
1746
            sources: SingleRasterOrVectorSource {
1✔
1747
                source: RasterOrVectorOperator::Raster(gdal),
1✔
1748
            },
1✔
1749
        }
1✔
1750
        .boxed();
1✔
1751

1752
        let expression_initialized = expression
1✔
1753
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1754
            .await
1✔
1755
            .unwrap();
1✔
1756

1757
        let expression_optimized = expression_initialized
1✔
1758
            .optimize(SpatialResolution::new(0.8, 0.8).unwrap())
1✔
1759
            .unwrap();
1✔
1760

1761
        let json = serde_json::to_value(&expression_optimized).unwrap();
1✔
1762

1763
        assert_eq!(
1✔
1764
            json,
1765
            serde_json::json!({
1✔
1766
                "params": {
1✔
1767
                    "attributeName": "ndvi",
1✔
1768
                    "bounds": {
1✔
1769
                        "max": 255.0,
1✔
1770
                        "min": 0.0
1✔
1771
                    },
1772
                    "buckets": {
1✔
1773
                        "type": "number",
1✔
1774
                        "value": 8
1✔
1775
                    },
1776
                    "interactive": false
1✔
1777
                },
1778
                "sources": {
1✔
1779
                    "source": {
1✔
1780
                        "params": {
1✔
1781
                            "data": "ndvi",
1✔
1782
                            "overviewLevel": 8
1✔
1783
                        },
1784
                        "type": "GdalSource"
1✔
1785
                    }
1786
                },
1787
                "type": "Histogram"
1✔
1788
            })
1789
        );
1790

1791
        expression_optimized
1✔
1792
            .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx)
1✔
1793
            .await
1✔
1794
            .unwrap();
1✔
1795
    }
1✔
1796
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc