13074893880

Committed 31 Jan 2025 02:38PM UTC coverage: 90.074% (+0.05%) from 90.027%

Build # 13074893880

Build Type

Pull #1007

github

Committed by

web-flow

Commit Message

Merge 1b97fc4d1 into 847320154

Pull Request Pull Request #1007: Add ml permissions

Run Details

401 of 440 new or added lines in 9 files covered. (91.14%)

1 existing line in 1 file now uncovered.

125968 of 139849 relevant lines covered (90.07%)

57556.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

6.25

/services/src/machine_learning/mod.rs

use crate::{
    api::model::datatypes::RasterDataType,
    config::{get_config_element, MachineLearning},
    datasets::upload::{UploadId, UploadRootPath},
    identifier,
    util::path_with_base_path,
};
use async_trait::async_trait;
use error::{error::CouldNotFindMlModelFileMachineLearningError, MachineLearningError};
use name::MlModelName;
use postgres_types::{FromSql, ToSql};
use serde::{Deserialize, Serialize};
use snafu::ResultExt;
use std::borrow::Cow;
use utoipa::{IntoParams, ToSchema};
use validator::{Validate, ValidationError};

pub mod error;
pub mod name;
mod postgres;

identifier!(MlModelId);

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModelIdAndName {
    pub id: MlModelId,
    pub name: MlModelName,
}

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModel {
    pub name: MlModelName,
    pub display_name: String,
    pub description: String,
    pub upload: UploadId,
    pub metadata: MlModelMetadata,
}

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModelMetadata {
    pub file_name: String,
    pub input_type: RasterDataType,
    pub num_input_bands: u32, // number of features per sample (bands per pixel)
    pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
                                     // TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
}

impl MlModel {
    pub fn metadata_for_operator(
        &self,
    ) -> Result<geoengine_datatypes::machine_learning::MlModelMetadata, MachineLearningError> {
        Ok(geoengine_datatypes::machine_learning::MlModelMetadata {
            file_path: path_with_base_path(
                &self
                    .upload
                    .root_path()
                    .context(CouldNotFindMlModelFileMachineLearningError)?,
                self.metadata.file_name.as_ref(),
            )
            .context(CouldNotFindMlModelFileMachineLearningError)?,
            input_type: self.metadata.input_type.into(),
            num_input_bands: self.metadata.num_input_bands,
            output_type: self.metadata.output_type.into(),
        })
    }
}

#[derive(Debug, Deserialize, Serialize, Clone, IntoParams, Validate)]
#[into_params(parameter_in = Query)]
pub struct MlModelListOptions {
    #[param(example = 0)]
    pub offset: u32,
    #[param(example = 2)]
    #[validate(custom(function = "validate_list_limit"))]
    pub limit: u32,
}

fn validate_list_limit(value: u32) -> Result<(), ValidationError> {
    let limit = get_config_element::<MachineLearning>()
        .expect("should exist because it is defined in the default config")
        .list_limit;
    if value <= limit {
        return Ok(());
    }

    let mut err = ValidationError::new("limit (too large)");
    err.add_param::<u32>(Cow::Borrowed("max limit"), &limit);
    Err(err)
}

#[async_trait]
pub trait MlModelDb {
    async fn list_models(
        &self,
        options: &MlModelListOptions,
    ) -> Result<Vec<MlModel>, MachineLearningError>;

    async fn load_model(&self, name: &MlModelName) -> Result<MlModel, MachineLearningError>;

    async fn load_model_metadata(
        &self,
        name: &MlModelName,
    ) -> Result<MlModelMetadata, MachineLearningError>;

    async fn add_model(&self, model: MlModel) -> Result<MlModelIdAndName, MachineLearningError>;

    async fn resolve_model_name_to_id(
        &self,
        name: &MlModelName,
    ) -> Result<Option<MlModelId>, MachineLearningError>;
}

1	use crate::{
2	api::model::datatypes::RasterDataType,
3	config::{get_config_element, MachineLearning},
4	datasets::upload::{UploadId, UploadRootPath},
5	identifier,
6	util::path_with_base_path,
7	};
8	use async_trait::async_trait;
9	use error::{error::CouldNotFindMlModelFileMachineLearningError, MachineLearningError};
10	use name::MlModelName;
11	use postgres_types::{FromSql, ToSql};
12	use serde::{Deserialize, Serialize};
13	use snafu::ResultExt;
14	use std::borrow::Cow;
15	use utoipa::{IntoParams, ToSchema};
16	use validator::{Validate, ValidationError};
17
18	pub mod error;
19	pub mod name;
20	mod postgres;
21
22	identifier!(MlModelId);
23
NEW 24	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	×
25	#[serde(rename_all = "camelCase")]
26	pub struct MlModelIdAndName {
27	pub id: MlModelId,
28	pub name: MlModelName,
29	}
30
31	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	2✔
32	#[serde(rename_all = "camelCase")]
33	pub struct MlModel {
34	pub name: MlModelName,
35	pub display_name: String,
36	pub description: String,
37	pub upload: UploadId,
38	pub metadata: MlModelMetadata,
39	}
40
41	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	2✔
42	#[serde(rename_all = "camelCase")]
43	pub struct MlModelMetadata {
44	pub file_name: String,
45	pub input_type: RasterDataType,
46	pub num_input_bands: u32, // number of features per sample (bands per pixel)
47	pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
48	// TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
49	}
50
51	impl MlModel {
52	pub fn metadata_for_operator(	×
53	&self,	×
54	) -> Result<geoengine_datatypes::machine_learning::MlModelMetadata, MachineLearningError> {	×
55	Ok(geoengine_datatypes::machine_learning::MlModelMetadata {	×
56	file_path: path_with_base_path(	×
57	&self	×
58	.upload	×
59	.root_path()	×
60	.context(CouldNotFindMlModelFileMachineLearningError)?,	×
61	self.metadata.file_name.as_ref(),	×
62	)	×
63	.context(CouldNotFindMlModelFileMachineLearningError)?,	×
64	input_type: self.metadata.input_type.into(),	×
65	num_input_bands: self.metadata.num_input_bands,	×
66	output_type: self.metadata.output_type.into(),	×
67	})
68	}	×
69	}
70
71	#[derive(Debug, Deserialize, Serialize, Clone, IntoParams, Validate)]	×
72	#[into_params(parameter_in = Query)]
73	pub struct MlModelListOptions {
74	#[param(example = 0)]
75	pub offset: u32,
76	#[param(example = 2)]
77	#[validate(custom(function = "validate_list_limit"))]
78	pub limit: u32,
79	}
80
81	fn validate_list_limit(value: u32) -> Result<(), ValidationError> {	×
82	let limit = get_config_element::<MachineLearning>()	×
83	.expect("should exist because it is defined in the default config")	×
84	.list_limit;	×
85	if value <= limit {	×
86	return Ok(());	×
87	}	×
88		×
89	let mut err = ValidationError::new("limit (too large)");	×
90	err.add_param::<u32>(Cow::Borrowed("max limit"), &limit);	×
91	Err(err)	×
92	}	×
93
94	#[async_trait]
95	pub trait MlModelDb {
96	async fn list_models(
97	&self,
98	options: &MlModelListOptions,
99	) -> Result<Vec<MlModel>, MachineLearningError>;
100
101	async fn load_model(&self, name: &MlModelName) -> Result<MlModel, MachineLearningError>;
102
103	async fn load_model_metadata(
104	&self,
105	name: &MlModelName,
106	) -> Result<MlModelMetadata, MachineLearningError>;
107
108	async fn add_model(&self, model: MlModel) -> Result<MlModelIdAndName, MachineLearningError>;
109
110	async fn resolve_model_name_to_id(
111	&self,
112	name: &MlModelName,
113	) -> Result<Option<MlModelId>, MachineLearningError>;
114	}

geo-engine / geoengine / 13074893880

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous