16410182728

Committed 21 Jul 2025 06:33AM UTC coverage: 88.748% (+0.01%) from 88.738%

Build # 16410182728

Build Type

push

github

Committed by

web-flow

Commit Message

feat(operators): skip empty tiles and merge masks in onnx; remove trace/debug in release mode (#1061)

* wip

* remove more trace and debug calls on release builds

* rename tracing removed macros

* add mask merge and tile skip params to onnx operator

* fmt

* rust 1.88

* clippy auto-fixes

* manual clippy fixes

* update deps

* cargo update

* update onnx

* cargo fmt

* update sqlfluff

* update lock

* adapt to main changes

* adapt array4 usage in onnnx

* add docs

* replace log with tracing

* rework nodata handling

* rework nodata handling #2

* set features and rename variable

* move ml model no data handling to model metadata

* refactor ml model api types

* delegate MlModelName serde to impl type

* move migration sql into own file

* tagged or untagged ???

* lint sql

* cargo.toml

* use next()

* event_anabled

* don't serialize operator type

* remove todo

* debug tracing

* openapi camel case

---------

Co-authored-by: Christian Beilschmidt <christian.beilschmidt@geoengine.de>

Run Details

419 of 578 new or added lines in 44 files covered. (72.49%)

15 existing lines in 7 files now uncovered.

111437 of 125565 relevant lines covered (88.75%)

80307.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.97

/operators/src/machine_learning/metadata.rs

use std::path::PathBuf;

use geoengine_datatypes::{machine_learning::MlTensorShape3D, raster::RasterDataType};
use postgres_types::{FromSql, ToSql};

/// Strategies to handle no-data in model inputs.
/// - `EncodedNoData`: If inputs have empty (no-data) pixels, pixels are mapped to a `no_data_value`. This is usefull if the model can handle missing data.
/// - `SkipIfNoData`: If any input pixel is empty (no-data), the output is also empty (no-data). This is usefull if the model can't handle missing data.
#[derive(PartialEq, Debug, Copy, Clone)]
pub enum MlModelInputNoDataHandling {
    EncodedNoData { no_data_value: f32 },
    SkipIfNoData,
}

impl MlModelInputNoDataHandling {
    pub fn no_data_value_encoding(self) -> Option<f32> {
        match self {
            MlModelInputNoDataHandling::EncodedNoData { no_data_value } => Some(no_data_value),
            MlModelInputNoDataHandling::SkipIfNoData => None,
        }
    }
}

/// Strategies to handle no-data in model outputs.
/// - `EncodedNoData`: If the model outputs a `no_data_value` pixel with the `no_data_value` are masked and are ignored by other operators.
/// - `NanIsNoData`: If the model produces NaN values, they are masked as as no data.
#[derive(PartialEq, Debug, Copy, Clone)]
pub enum MlModelOutputNoDataHandling {
    EncodedNoData { no_data_value: f32 },
    NanIsNoData,
}

impl MlModelOutputNoDataHandling {
    pub fn no_data_value_encoding(self) -> Option<f32> {
        match self {
            MlModelOutputNoDataHandling::EncodedNoData { no_data_value } => Some(no_data_value),
            MlModelOutputNoDataHandling::NanIsNoData => None,
        }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct MlModelLoadingInfo {
    pub storage_path: PathBuf,
    pub metadata: MlModelMetadata,
}

impl MlModelLoadingInfo {
    pub fn model_path_owned(&self) -> PathBuf {
        self.storage_path.clone()
    }
}

// For now we assume all models are pixel-wise, i.e., they take a single pixel with multiple bands as input and produce a single output value.
// To support different inputs, we would need a more sophisticated logic to produce the inputs for the model.
#[derive(Debug, Clone, PartialEq, ToSql, FromSql)]
pub struct MlModelMetadata {
    pub input_type: RasterDataType,
    pub output_type: RasterDataType,
    pub input_shape: MlTensorShape3D,
    pub output_shape: MlTensorShape3D, // TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
    pub input_no_data_handling: MlModelInputNoDataHandling,
    pub output_no_data_handling: MlModelOutputNoDataHandling,
}

impl MlModelMetadata {
    pub fn num_input_bands(&self) -> u32 {
        self.input_shape.bands
    }

    pub fn num_output_bands(&self) -> u32 {
        self.output_shape.bands
    }

    pub fn input_is_single_pixel(&self) -> bool {
        self.input_shape.x == 1 && self.input_shape.y == 1
    }

    pub fn output_is_single_pixel(&self) -> bool {
        self.output_shape.x == 1 && self.output_shape.y == 1
    }

    pub fn output_is_single_attribute(&self) -> bool {
        self.num_output_bands() == 1
    }
}

1	use std::path::PathBuf;
2
3	use geoengine_datatypes::{machine_learning::MlTensorShape3D, raster::RasterDataType};
4	use postgres_types::{FromSql, ToSql};
5
6	/// Strategies to handle no-data in model inputs.
7	/// - `EncodedNoData`: If inputs have empty (no-data) pixels, pixels are mapped to a `no_data_value`. This is usefull if the model can handle missing data.
8	/// - `SkipIfNoData`: If any input pixel is empty (no-data), the output is also empty (no-data). This is usefull if the model can't handle missing data.
9	#[derive(PartialEq, Debug, Copy, Clone)]
10	pub enum MlModelInputNoDataHandling {
11	EncodedNoData { no_data_value: f32 },
12	SkipIfNoData,
13	}
14
15	impl MlModelInputNoDataHandling {
16	pub fn no_data_value_encoding(self) -> Option<f32> {	3✔
17	match self {	3✔
NEW 18	MlModelInputNoDataHandling::EncodedNoData { no_data_value } => Some(no_data_value),	×
19	MlModelInputNoDataHandling::SkipIfNoData => None,	3✔
20	}
21	}	3✔
22	}
23
24	/// Strategies to handle no-data in model outputs.
25	/// - `EncodedNoData`: If the model outputs a `no_data_value` pixel with the `no_data_value` are masked and are ignored by other operators.
26	/// - `NanIsNoData`: If the model produces NaN values, they are masked as as no data.
27	#[derive(PartialEq, Debug, Copy, Clone)]
28	pub enum MlModelOutputNoDataHandling {
29	EncodedNoData { no_data_value: f32 },
30	NanIsNoData,
31	}
32
33	impl MlModelOutputNoDataHandling {
34	pub fn no_data_value_encoding(self) -> Option<f32> {	3✔
35	match self {	3✔
NEW 36	MlModelOutputNoDataHandling::EncodedNoData { no_data_value } => Some(no_data_value),	×
37	MlModelOutputNoDataHandling::NanIsNoData => None,	3✔
38	}
39	}	3✔
40	}
41
42	#[derive(Debug, Clone, PartialEq)]
43	pub struct MlModelLoadingInfo {
44	pub storage_path: PathBuf,
45	pub metadata: MlModelMetadata,
46	}
47
48	impl MlModelLoadingInfo {
NEW 49	pub fn model_path_owned(&self) -> PathBuf {	×
NEW 50	self.storage_path.clone()	×
NEW 51	}	×
52	}
53
54	// For now we assume all models are pixel-wise, i.e., they take a single pixel with multiple bands as input and produce a single output value.
55	// To support different inputs, we would need a more sophisticated logic to produce the inputs for the model.
NEW 56	#[derive(Debug, Clone, PartialEq, ToSql, FromSql)]	×
57	pub struct MlModelMetadata {
58	pub input_type: RasterDataType,
59	pub output_type: RasterDataType,
60	pub input_shape: MlTensorShape3D,
61	pub output_shape: MlTensorShape3D, // TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
62	pub input_no_data_handling: MlModelInputNoDataHandling,
63	pub output_no_data_handling: MlModelOutputNoDataHandling,
64	}
65
66	impl MlModelMetadata {
NEW 67	pub fn num_input_bands(&self) -> u32 {	×
NEW 68	self.input_shape.bands	×
NEW 69	}	×
70
71	pub fn num_output_bands(&self) -> u32 {	4✔
72	self.output_shape.bands	4✔
73	}	4✔
74
75	pub fn input_is_single_pixel(&self) -> bool {	13✔
76	self.input_shape.x == 1 && self.input_shape.y == 1	13✔
77	}	13✔
78
79	pub fn output_is_single_pixel(&self) -> bool {	4✔
80	self.output_shape.x == 1 && self.output_shape.y == 1	4✔
81	}	4✔
82
83	pub fn output_is_single_attribute(&self) -> bool {	4✔
84	self.num_output_bands() == 1	4✔
85	}	4✔
86	}

geo-engine / geoengine / 16410182728

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous