13030171108

Committed 29 Jan 2025 11:18AM UTC coverage: 90.104% (+0.01%) from 90.094%

Build # 13030171108

Build Type

Pull #1007

github

Committed by

web-flow

Commit Message

Merge 2e70fa7a6 into b4abb5d41

Pull Request Pull Request #1007: Add ml permissions

Run Details

192 of 248 new or added lines in 9 files covered. (77.42%)

2 existing lines in 2 files now uncovered.

125846 of 139667 relevant lines covered (90.1%)

57633.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

6.25

/services/src/machine_learning/mod.rs

use crate::{
    api::model::datatypes::RasterDataType,
    config::{get_config_element, MachineLearning},
    datasets::upload::{UploadId, UploadRootPath},
    identifier,
    util::path_with_base_path,
};
use async_trait::async_trait;
use error::{error::CouldNotFindMlModelFileMachineLearningError, MachineLearningError};
use name::MlModelName;
use postgres_types::{FromSql, ToSql};
use serde::{Deserialize, Serialize};
use snafu::ResultExt;
use std::borrow::Cow;
use utoipa::{IntoParams, ToSchema};
use validator::{Validate, ValidationError};

pub mod error;
pub mod name;
mod postgres;

identifier!(MlModelId);

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModelIdAndName {
    pub id: MlModelId,
    pub name: MlModelName,
}

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModel {
    pub name: MlModelName,
    pub display_name: String,
    pub description: String,
    pub upload: UploadId,
    pub metadata: MlModelMetadata,
}

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModelMetadata {
    pub file_name: String,
    pub input_type: RasterDataType,
    pub num_input_bands: u32, // number of features per sample (bands per pixel)
    pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
                                     // TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
}

impl MlModel {
    pub fn metadata_for_operator(
        &self,
    ) -> Result<geoengine_datatypes::machine_learning::MlModelMetadata, MachineLearningError> {
        Ok(geoengine_datatypes::machine_learning::MlModelMetadata {
            file_path: path_with_base_path(
                &self
                    .upload
                    .root_path()
                    .context(CouldNotFindMlModelFileMachineLearningError)?,
                self.metadata.file_name.as_ref(),
            )
            .context(CouldNotFindMlModelFileMachineLearningError)?,
            input_type: self.metadata.input_type.into(),
            num_input_bands: self.metadata.num_input_bands,
            output_type: self.metadata.output_type.into(),
        })
    }
}

#[derive(Debug, Deserialize, Serialize, Clone, IntoParams, Validate)]
#[into_params(parameter_in = Query)]
pub struct MlModelListOptions {
    #[param(example = 0)]
    pub offset: u32,
    #[param(example = 2)]
    #[validate(custom(function = "validate_list_limit"))]
    pub limit: u32,
}

fn validate_list_limit(value: u32) -> Result<(), ValidationError> {
    let limit = get_config_element::<MachineLearning>()
        .expect("should exist because it is defined in the default config")
        .list_limit;
    if value <= limit {
        return Ok(());
    }

    let mut err = ValidationError::new("limit (too large)");
    err.add_param::<u32>(Cow::Borrowed("max limit"), &limit);
    Err(err)
}

#[async_trait]
pub trait MlModelDb {
    async fn list_models(
        &self,
        options: &MlModelListOptions,
    ) -> Result<Vec<MlModel>, MachineLearningError>;

    async fn load_model(&self, name: &MlModelName) -> Result<MlModel, MachineLearningError>;

    async fn load_model_metadata(
        &self,
        name: &MlModelName,
    ) -> Result<MlModelMetadata, MachineLearningError>;

    async fn add_model(&self, model: MlModel) -> Result<MlModelIdAndName, MachineLearningError>;

    async fn resolve_model_name_to_id(
        &self,
        name: &MlModelName,
    ) -> Result<Option<MlModelId>, MachineLearningError>;
}

1	use crate::{
2	api::model::datatypes::RasterDataType,
3	config::{get_config_element, MachineLearning},
4	datasets::upload::{UploadId, UploadRootPath},
5	identifier,
6	util::path_with_base_path,
7	};
8	use async_trait::async_trait;
9	use error::{error::CouldNotFindMlModelFileMachineLearningError, MachineLearningError};
10	use name::MlModelName;
11	use postgres_types::{FromSql, ToSql};
12	use serde::{Deserialize, Serialize};
13	use snafu::ResultExt;
14	use std::borrow::Cow;
15	use utoipa::{IntoParams, ToSchema};
16	use validator::{Validate, ValidationError};
17
18	pub mod error;
19	pub mod name;
20	mod postgres;
21
22	identifier!(MlModelId);
23
NEW 24	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	×
25	#[serde(rename_all = "camelCase")]
26	pub struct MlModelIdAndName {
27	pub id: MlModelId,
28	pub name: MlModelName,
29	}
30
31	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	2✔
32	#[serde(rename_all = "camelCase")]
33	pub struct MlModel {
34	pub name: MlModelName,
35	pub display_name: String,
36	pub description: String,
37	pub upload: UploadId,
38	pub metadata: MlModelMetadata,
39	}
40
41	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	12✔
42	#[serde(rename_all = "camelCase")]
43	pub struct MlModelMetadata {
44	pub file_name: String,
45	pub input_type: RasterDataType,
46	pub num_input_bands: u32, // number of features per sample (bands per pixel)
47	pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
48	// TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
49	}
50
51	impl MlModel {
52	pub fn metadata_for_operator(	×
53	&self,	×
54	) -> Result<geoengine_datatypes::machine_learning::MlModelMetadata, MachineLearningError> {	×
55	Ok(geoengine_datatypes::machine_learning::MlModelMetadata {	×
56	file_path: path_with_base_path(	×
57	&self	×
58	.upload	×
59	.root_path()	×
60	.context(CouldNotFindMlModelFileMachineLearningError)?,	×
61	self.metadata.file_name.as_ref(),	×
62	)	×
63	.context(CouldNotFindMlModelFileMachineLearningError)?,	×
64	input_type: self.metadata.input_type.into(),	×
65	num_input_bands: self.metadata.num_input_bands,	×
66	output_type: self.metadata.output_type.into(),	×
67	})
68	}	×
69	}
70
71	#[derive(Debug, Deserialize, Serialize, Clone, IntoParams, Validate)]	×
72	#[into_params(parameter_in = Query)]
73	pub struct MlModelListOptions {
74	#[param(example = 0)]
75	pub offset: u32,
76	#[param(example = 2)]
77	#[validate(custom(function = "validate_list_limit"))]
78	pub limit: u32,
79	}
80
81	fn validate_list_limit(value: u32) -> Result<(), ValidationError> {	×
82	let limit = get_config_element::<MachineLearning>()	×
83	.expect("should exist because it is defined in the default config")	×
84	.list_limit;	×
85	if value <= limit {	×
86	return Ok(());	×
87	}	×
88		×
89	let mut err = ValidationError::new("limit (too large)");	×
90	err.add_param::<u32>(Cow::Borrowed("max limit"), &limit);	×
91	Err(err)	×
92	}	×
93
94	#[async_trait]
95	pub trait MlModelDb {
96	async fn list_models(
97	&self,
98	options: &MlModelListOptions,
99	) -> Result<Vec<MlModel>, MachineLearningError>;
100
101	async fn load_model(&self, name: &MlModelName) -> Result<MlModel, MachineLearningError>;
102
103	async fn load_model_metadata(
104	&self,
105	name: &MlModelName,
106	) -> Result<MlModelMetadata, MachineLearningError>;
107
108	async fn add_model(&self, model: MlModel) -> Result<MlModelIdAndName, MachineLearningError>;
109
110	async fn resolve_model_name_to_id(
111	&self,
112	name: &MlModelName,
113	) -> Result<Option<MlModelId>, MachineLearningError>;
114	}

geo-engine / geoengine / 13030171108

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous