13075333516

Committed 31 Jan 2025 03:04PM UTC coverage: 90.073% (+0.05%) from 90.027%

Build # 13075333516

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #1007 from geo-engine/add_ml_permissions

Add ml permissions

Run Details

401 of 440 new or added lines in 9 files covered. (91.14%)

3 existing lines in 3 files now uncovered.

125966 of 139849 relevant lines covered (90.07%)

57556.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.0

/datatypes/src/machine_learning.rs

use crate::{
    dataset::{is_invalid_name_char, SYSTEM_NAMESPACE},
    raster::RasterDataType,
};
use serde::{de::Visitor, Deserialize, Serialize};
use snafu::Snafu;
use std::path::PathBuf;
use std::str::FromStr;
use strum::IntoStaticStr;

const NAME_DELIMITER: char = ':';

#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub struct MlModelName {
    pub namespace: Option<String>,
    pub name: String,
}

#[derive(Snafu, IntoStaticStr, Debug)]
#[snafu(visibility(pub(crate)))]
#[snafu(context(suffix(false)))] // disables default `Snafu` suffix
pub enum MlModelNameError {
    #[snafu(display("MlModelName is empty"))]
    IsEmpty,
    #[snafu(display("invalid character '{invalid_char}' in named model"))]
    InvalidCharacter { invalid_char: String },
    #[snafu(display("ml model name must consist of at most two parts"))]
    TooManyParts,
}

impl MlModelName {
    /// Canonicalize a name that reflects the system namespace and model.
    fn canonicalize<S: Into<String> + PartialEq<&'static str>>(
        name: S,
        system_name: &'static str,
    ) -> Option<String> {
        if name == system_name {
            None
        } else {
            Some(name.into())
        }
    }

    pub fn new<S: Into<String>>(namespace: Option<String>, name: S) -> Self {
        Self {
            namespace,
            name: name.into(),
        }
    }
}

impl Serialize for MlModelName {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer,
    {
        let d = NAME_DELIMITER;
        let serialized = match (&self.namespace, &self.name) {
            (None, name) => name.to_string(),
            (Some(namespace), name) => {
                format!("{namespace}{d}{name}")
            }
        };

        serializer.serialize_str(&serialized)
    }
}

impl<'de> Deserialize<'de> for MlModelName {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: serde::Deserializer<'de>,
    {
        deserializer.deserialize_str(MlModelNameDeserializeVisitor)
    }
}

impl FromStr for MlModelName {
    type Err = MlModelNameError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let mut strings = [None, None];
        let mut split = s.split(NAME_DELIMITER);

        for (buffer, part) in strings.iter_mut().zip(&mut split) {
            if part.is_empty() {
                return Err(MlModelNameError::IsEmpty);
            }

            if let Some(c) = part.matches(is_invalid_name_char).next() {
                return Err(MlModelNameError::InvalidCharacter {
                    invalid_char: c.to_string(),
                });
            }

            *buffer = Some(part.to_string());
        }

        if split.next().is_some() {
            return Err(MlModelNameError::TooManyParts);
        }

        match strings {
            [Some(namespace), Some(name)] => Ok(MlModelName {
                namespace: MlModelName::canonicalize(namespace, SYSTEM_NAMESPACE),
                name,
            }),
            [Some(name), None] => Ok(MlModelName {
                namespace: None,
                name,
            }),
            _ => Err(MlModelNameError::IsEmpty),
        }
    }
}

struct MlModelNameDeserializeVisitor;

impl Visitor<'_> for MlModelNameDeserializeVisitor {
    type Value = MlModelName;

    /// always keep in sync with [`is_allowed_name_char`]
    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(
            formatter,
            "a string consisting of a namespace and name name, separated by a colon, only using alphanumeric characters, underscores & dashes"
        )
    }

    fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
    where
        E: serde::de::Error,
    {
        MlModelName::from_str(s).map_err(|e| E::custom(e.to_string()))
    }
}

// For now we assume all models are pixel-wise, i.e., they take a single pixel with multiple bands as input and produce a single output value.
// To support different inputs, we would need a more sophisticated logic to produce the inputs for the model.
#[derive(Debug, Clone, Hash, Eq, PartialEq, Deserialize, Serialize)]
pub struct MlModelMetadata {
    pub file_path: PathBuf,
    pub input_type: RasterDataType,
    pub num_input_bands: u32, // number of features per sample (bands per pixel)
    pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
                                     // TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ml_model_name_from_str() {
        const ML_MODEL_NAME: &str = "myModelName";
        let mln = MlModelName::from_str(ML_MODEL_NAME).unwrap();
        assert_eq!(mln.name, ML_MODEL_NAME);
        assert!(mln.namespace.is_none());
    }

    #[test]
    fn ml_model_name_from_str_prefixed() {
        const ML_MODEL_NAME: &str = "d5328854-6190-4af9-ad69-4e74b0961ac9:myModelName";
        let mln = MlModelName::from_str(ML_MODEL_NAME).unwrap();
        assert_eq!(mln.name, "myModelName".to_string());
        assert_eq!(
            mln.namespace,
            Some("d5328854-6190-4af9-ad69-4e74b0961ac9".to_string())
        );
    }

    #[test]
    fn ml_model_name_from_str_system() {
        const ML_MODEL_NAME: &str = "_:myModelName";
        let mln = MlModelName::from_str(ML_MODEL_NAME).unwrap();
        assert_eq!(mln.name, "myModelName".to_string());
        assert!(mln.namespace.is_none());
    }
}

1	use crate::{
2	dataset::{is_invalid_name_char, SYSTEM_NAMESPACE},
3	raster::RasterDataType,
4	};
5	use serde::{de::Visitor, Deserialize, Serialize};
6	use snafu::Snafu;
7	use std::path::PathBuf;
8	use std::str::FromStr;
9	use strum::IntoStaticStr;
10
11	const NAME_DELIMITER: char = ':';
12
13	#[derive(Debug, Clone, Hash, Eq, PartialEq)]
14	pub struct MlModelName {
15	pub namespace: Option<String>,
16	pub name: String,
17	}
18
NEW 19	#[derive(Snafu, IntoStaticStr, Debug)]	×
20	#[snafu(visibility(pub(crate)))]
21	#[snafu(context(suffix(false)))] // disables default `Snafu` suffix
22	pub enum MlModelNameError {
23	#[snafu(display("MlModelName is empty"))]
24	IsEmpty,
25	#[snafu(display("invalid character '{invalid_char}' in named model"))]
26	InvalidCharacter { invalid_char: String },
27	#[snafu(display("ml model name must consist of at most two parts"))]
28	TooManyParts,
29	}
30
31	impl MlModelName {
32	/// Canonicalize a name that reflects the system namespace and model.
33	fn canonicalize<S: Into<String> + PartialEq<&'static str>>(	2✔
34	name: S,	2✔
35	system_name: &'static str,	2✔
36	) -> Option<String> {	2✔
37	if name == system_name {	2✔
38	None	1✔
39	} else {
40	Some(name.into())	1✔
41	}
42	}	2✔
43
44	pub fn new<S: Into<String>>(namespace: Option<String>, name: S) -> Self {	2✔
45	Self {	2✔
46	namespace,	2✔
47	name: name.into(),	2✔
48	}	2✔
49	}	2✔
50	}
51
52	impl Serialize for MlModelName {
53	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>	3✔
54	where	3✔
55	S: serde::Serializer,	3✔
56	{	3✔
57	let d = NAME_DELIMITER;	3✔
58	let serialized = match (&self.namespace, &self.name) {	3✔
59	(None, name) => name.to_string(),	3✔
60	(Some(namespace), name) => {	×
61	format!("{namespace}{d}{name}")	×
62	}
63	};
64
65	serializer.serialize_str(&serialized)	3✔
66	}	3✔
67	}
68
69	impl<'de> Deserialize<'de> for MlModelName {
70	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>	×
71	where	×
72	D: serde::Deserializer<'de>,	×
73	{	×
74	deserializer.deserialize_str(MlModelNameDeserializeVisitor)	×
75	}	×
76	}
77
78	impl FromStr for MlModelName {
79	type Err = MlModelNameError;
80
81	fn from_str(s: &str) -> Result<Self, Self::Err> {	5✔
82	let mut strings = [None, None];	5✔
83	let mut split = s.split(NAME_DELIMITER);	5✔
84
85	for (buffer, part) in strings.iter_mut().zip(&mut split) {	7✔
86	if part.is_empty() {	7✔
NEW 87	return Err(MlModelNameError::IsEmpty);	×
88	}	7✔
89
90	if let Some(c) = part.matches(is_invalid_name_char).next() {	7✔
NEW 91	return Err(MlModelNameError::InvalidCharacter {	×
NEW 92	invalid_char: c.to_string(),	×
NEW 93	});	×
94	}	7✔
95		7✔
96	*buffer = Some(part.to_string());	7✔
97	}
98
99	if split.next().is_some() {	5✔
NEW 100	return Err(MlModelNameError::TooManyParts);	×
101	}	5✔
102
103	match strings {	5✔
104	[Some(namespace), Some(name)] => Ok(MlModelName {	2✔
105	namespace: MlModelName::canonicalize(namespace, SYSTEM_NAMESPACE),	2✔
106	name,	2✔
107	}),	2✔
108	[Some(name), None] => Ok(MlModelName {	3✔
109	namespace: None,	3✔
110	name,	3✔
111	}),	3✔
NEW 112	_ => Err(MlModelNameError::IsEmpty),	×
113	}
114	}	5✔
115	}
116
117	struct MlModelNameDeserializeVisitor;
118
119	impl Visitor<'_> for MlModelNameDeserializeVisitor {
120	type Value = MlModelName;
121
122	/// always keep in sync with [`is_allowed_name_char`]
NEW 123	fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {	×
NEW 124	write!(	×
NEW 125	formatter,	×
NEW 126	"a string consisting of a namespace and name name, separated by a colon, only using alphanumeric characters, underscores & dashes"	×
NEW 127	)	×
NEW 128	}	×
129
NEW 130	fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>	×
NEW 131	where	×
NEW 132	E: serde::de::Error,	×
NEW 133	{	×
NEW 134	MlModelName::from_str(s).map_err(\|e\| E::custom(e.to_string()))	×
NEW 135	}	×
136	}
137
138	// For now we assume all models are pixel-wise, i.e., they take a single pixel with multiple bands as input and produce a single output value.
139	// To support different inputs, we would need a more sophisticated logic to produce the inputs for the model.
140	#[derive(Debug, Clone, Hash, Eq, PartialEq, Deserialize, Serialize)]
141	pub struct MlModelMetadata {
142	pub file_path: PathBuf,
143	pub input_type: RasterDataType,
144	pub num_input_bands: u32, // number of features per sample (bands per pixel)
145	pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
146	// TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
147	}
148
149	#[cfg(test)]
150	mod tests {
151	use super::*;
152
153	#[test]
154	fn ml_model_name_from_str() {	1✔
155	const ML_MODEL_NAME: &str = "myModelName";
156	let mln = MlModelName::from_str(ML_MODEL_NAME).unwrap();	1✔
157	assert_eq!(mln.name, ML_MODEL_NAME);	1✔
158	assert!(mln.namespace.is_none());	1✔
159	}	1✔
160
161	#[test]
162	fn ml_model_name_from_str_prefixed() {	1✔
163	const ML_MODEL_NAME: &str = "d5328854-6190-4af9-ad69-4e74b0961ac9:myModelName";
164	let mln = MlModelName::from_str(ML_MODEL_NAME).unwrap();	1✔
165	assert_eq!(mln.name, "myModelName".to_string());	1✔
166	assert_eq!(	1✔
167	mln.namespace,	1✔
168	Some("d5328854-6190-4af9-ad69-4e74b0961ac9".to_string())	1✔
169	);	1✔
170	}	1✔
171
172	#[test]
173	fn ml_model_name_from_str_system() {	1✔
174	const ML_MODEL_NAME: &str = "_:myModelName";
175	let mln = MlModelName::from_str(ML_MODEL_NAME).unwrap();	1✔
176	assert_eq!(mln.name, "myModelName".to_string());	1✔
177	assert!(mln.namespace.is_none());	1✔
178	}	1✔
179	}

geo-engine / geoengine / 13075333516

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous