12754240180

Committed 13 Jan 2025 07:05PM UTC coverage: 90.575% (-0.001%) from 90.576%

Build # 12754240180

Build Type

Pull #1007

github

Committed by

web-flow

Commit Message

Merge 8743ef34d into 8239dda75

Pull Request Pull Request #1007: Add ml permissions

Run Details

11 of 15 new or added lines in 5 files covered. (73.33%)

1 existing line in 1 file now uncovered.

134016 of 147962 relevant lines covered (90.57%)

54410.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

4.55

/services/src/machine_learning/mod.rs

use std::borrow::Cow;

use async_trait::async_trait;
use error::{error::CouldNotFindMlModelFileMachineLearningError, MachineLearningError};
use name::MlModelName;
use postgres_types::{FromSql, ToSql};
use serde::{Deserialize, Serialize};
use snafu::ResultExt;
use tokio_postgres::{
    tls::{MakeTlsConnect, TlsConnect},
    Socket,
};
use utoipa::{IntoParams, ToSchema};
use validator::{Validate, ValidationError};

use crate::{
    api::model::datatypes::RasterDataType,
    contexts::PostgresDb,
    datasets::upload::{UploadId, UploadRootPath},
    identifier,
    util::{
        config::{get_config_element, MachineLearning},
        path_with_base_path,
    },
};

pub mod error;
pub mod name;

identifier!(MlModelId);

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModelIdAndName {
    pub id: MlModelId,
    pub name: MlModelName,
}

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModel {
    pub name: MlModelName,
    pub display_name: String,
    pub description: String,
    pub upload: UploadId,
    pub metadata: MlModelMetadata,
}

#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]
#[serde(rename_all = "camelCase")]
pub struct MlModelMetadata {
    pub file_name: String,
    pub input_type: RasterDataType,
    pub num_input_bands: u32, // number of features per sample (bands per pixel)
    pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
                                     // TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
}

impl MlModel {
    pub fn metadata_for_operator(
        &self,
    ) -> Result<geoengine_datatypes::machine_learning::MlModelMetadata, MachineLearningError> {
        Ok(geoengine_datatypes::machine_learning::MlModelMetadata {
            file_path: path_with_base_path(
                &self
                    .upload
                    .root_path()
                    .context(CouldNotFindMlModelFileMachineLearningError)?,
                self.metadata.file_name.as_ref(),
            )
            .context(CouldNotFindMlModelFileMachineLearningError)?,
            input_type: self.metadata.input_type.into(),
            num_input_bands: self.metadata.num_input_bands,
            output_type: self.metadata.output_type.into(),
        })
    }
}

#[derive(Debug, Deserialize, Serialize, Clone, IntoParams, Validate)]
#[into_params(parameter_in = Query)]
pub struct MlModelListOptions {
    #[param(example = 0)]
    pub offset: u32,
    #[param(example = 2)]
    #[validate(custom(function = "validate_list_limit"))]
    pub limit: u32,
}

fn validate_list_limit(value: u32) -> Result<(), ValidationError> {
    let limit = get_config_element::<MachineLearning>()
        .expect("should exist because it is defined in the default config")
        .list_limit;
    if value <= limit {
        return Ok(());
    }

    let mut err = ValidationError::new("limit (too large)");
    err.add_param::<u32>(Cow::Borrowed("max limit"), &limit);
    Err(err)
}

#[async_trait]
pub trait MlModelDb {
    async fn list_models(
        &self,
        options: &MlModelListOptions,
    ) -> Result<Vec<MlModel>, MachineLearningError>;

    async fn load_model(&self, name: &MlModelName) -> Result<MlModel, MachineLearningError>;

    async fn load_model_metadata(
        &self,
        name: &MlModelName,
    ) -> Result<MlModelMetadata, MachineLearningError>;

    async fn add_model(&self, model: MlModel) -> Result<MlModelIdAndName, MachineLearningError>;
}

#[async_trait]
impl<Tls> MlModelDb for PostgresDb<Tls>
where
    Tls: MakeTlsConnect<Socket> + Clone + Send + Sync + 'static + std::fmt::Debug,
    <Tls as MakeTlsConnect<Socket>>::Stream: Send + Sync,
    <Tls as MakeTlsConnect<Socket>>::TlsConnect: Send,
    <<Tls as MakeTlsConnect<Socket>>::TlsConnect as TlsConnect<Socket>>::Future: Send,
{
    async fn list_models(
        &self,
        _options: &MlModelListOptions,
    ) -> Result<Vec<MlModel>, MachineLearningError> {
        unimplemented!()
    }

    async fn load_model(&self, _name: &MlModelName) -> Result<MlModel, MachineLearningError> {
        unimplemented!()
    }

    async fn load_model_metadata(
        &self,
        _name: &MlModelName,
    ) -> Result<MlModelMetadata, MachineLearningError> {
        unimplemented!()
    }

    async fn add_model(&self, _model: MlModel) -> Result<MlModelIdAndName, MachineLearningError> {
        unimplemented!()
    }
}

1	use std::borrow::Cow;
2
3	use async_trait::async_trait;
4	use error::{error::CouldNotFindMlModelFileMachineLearningError, MachineLearningError};
5	use name::MlModelName;
6	use postgres_types::{FromSql, ToSql};
7	use serde::{Deserialize, Serialize};
8	use snafu::ResultExt;
9	use tokio_postgres::{
10	tls::{MakeTlsConnect, TlsConnect},
11	Socket,
12	};
13	use utoipa::{IntoParams, ToSchema};
14	use validator::{Validate, ValidationError};
15
16	use crate::{
17	api::model::datatypes::RasterDataType,
18	contexts::PostgresDb,
19	datasets::upload::{UploadId, UploadRootPath},
20	identifier,
21	util::{
22	config::{get_config_element, MachineLearning},
23	path_with_base_path,
24	},
25	};
26
27	pub mod error;
28	pub mod name;
29
30	identifier!(MlModelId);
31
NEW 32	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	×
33	#[serde(rename_all = "camelCase")]
34	pub struct MlModelIdAndName {
35	pub id: MlModelId,
36	pub name: MlModelName,
37	}
38
39	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	2✔
40	#[serde(rename_all = "camelCase")]
41	pub struct MlModel {
42	pub name: MlModelName,
43	pub display_name: String,
44	pub description: String,
45	pub upload: UploadId,
46	pub metadata: MlModelMetadata,
47	}
48
49	#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, FromSql, ToSql)]	12✔
50	#[serde(rename_all = "camelCase")]
51	pub struct MlModelMetadata {
52	pub file_name: String,
53	pub input_type: RasterDataType,
54	pub num_input_bands: u32, // number of features per sample (bands per pixel)
55	pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
56	// TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
57	}
58
59	impl MlModel {
60	pub fn metadata_for_operator(	×
61	&self,	×
62	) -> Result<geoengine_datatypes::machine_learning::MlModelMetadata, MachineLearningError> {	×
63	Ok(geoengine_datatypes::machine_learning::MlModelMetadata {	×
64	file_path: path_with_base_path(	×
65	&self	×
66	.upload	×
67	.root_path()	×
68	.context(CouldNotFindMlModelFileMachineLearningError)?,	×
69	self.metadata.file_name.as_ref(),	×
70	)	×
71	.context(CouldNotFindMlModelFileMachineLearningError)?,	×
72	input_type: self.metadata.input_type.into(),	×
73	num_input_bands: self.metadata.num_input_bands,	×
74	output_type: self.metadata.output_type.into(),	×
75	})
76	}	×
77	}
78
79	#[derive(Debug, Deserialize, Serialize, Clone, IntoParams, Validate)]	×
80	#[into_params(parameter_in = Query)]
81	pub struct MlModelListOptions {
82	#[param(example = 0)]
83	pub offset: u32,
84	#[param(example = 2)]
85	#[validate(custom(function = "validate_list_limit"))]
86	pub limit: u32,
87	}
88
89	fn validate_list_limit(value: u32) -> Result<(), ValidationError> {	×
90	let limit = get_config_element::<MachineLearning>()	×
91	.expect("should exist because it is defined in the default config")	×
92	.list_limit;	×
93	if value <= limit {	×
94	return Ok(());	×
95	}	×
96		×
97	let mut err = ValidationError::new("limit (too large)");	×
98	err.add_param::<u32>(Cow::Borrowed("max limit"), &limit);	×
99	Err(err)	×
100	}	×
101
102	#[async_trait]
103	pub trait MlModelDb {
104	async fn list_models(
105	&self,
106	options: &MlModelListOptions,
107	) -> Result<Vec<MlModel>, MachineLearningError>;
108
109	async fn load_model(&self, name: &MlModelName) -> Result<MlModel, MachineLearningError>;
110
111	async fn load_model_metadata(
112	&self,
113	name: &MlModelName,
114	) -> Result<MlModelMetadata, MachineLearningError>;
115
116	async fn add_model(&self, model: MlModel) -> Result<MlModelIdAndName, MachineLearningError>;
117	}
118
119	#[async_trait]
120	impl<Tls> MlModelDb for PostgresDb<Tls>
121	where
122	Tls: MakeTlsConnect<Socket> + Clone + Send + Sync + 'static + std::fmt::Debug,
123	<Tls as MakeTlsConnect<Socket>>::Stream: Send + Sync,
124	<Tls as MakeTlsConnect<Socket>>::TlsConnect: Send,
125	<<Tls as MakeTlsConnect<Socket>>::TlsConnect as TlsConnect<Socket>>::Future: Send,
126	{
127	async fn list_models(
128	&self,
129	_options: &MlModelListOptions,
130	) -> Result<Vec<MlModel>, MachineLearningError> {	×
131	unimplemented!()	×
132	}	×
133
134	async fn load_model(&self, _name: &MlModelName) -> Result<MlModel, MachineLearningError> {	×
135	unimplemented!()	×
136	}	×
137
138	async fn load_model_metadata(
139	&self,
140	_name: &MlModelName,
141	) -> Result<MlModelMetadata, MachineLearningError> {	×
142	unimplemented!()	×
143	}	×
144
NEW 145	async fn add_model(&self, _model: MlModel) -> Result<MlModelIdAndName, MachineLearningError> {	×
146	unimplemented!()	×
147	}	×
148	}

geo-engine / geoengine / 12754240180

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous