10699003826

Committed 04 Sep 2024 09:11AM UTC coverage: 91.003% (-0.1%) from 91.122%

Build # 10699003826

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #977 from geo-engine/model_db

db for ml models (wip)

Run Details

460 of 843 new or added lines in 25 files covered. (54.57%)

21 existing lines in 9 files now uncovered.

133663 of 146878 relevant lines covered (91.0%)

52516.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

13.43

/datatypes/src/machine_learning.rs

use std::path::PathBuf;

use serde::{de::Visitor, Deserialize, Serialize};

use crate::{
    dataset::{is_invalid_name_char, SYSTEM_NAMESPACE},
    raster::RasterDataType,
};

const NAME_DELIMITER: char = ':';

#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub struct MlModelName {
    pub namespace: Option<String>,
    pub name: String,
}

impl MlModelName {
    /// Canonicalize a name that reflects the system namespace and model.
    fn canonicalize<S: Into<String> + PartialEq<&'static str>>(
        name: S,
        system_name: &'static str,
    ) -> Option<String> {
        if name == system_name {
            None
        } else {
            Some(name.into())
        }
    }

    pub fn new<S: Into<String>>(namespace: Option<String>, name: S) -> Self {
        Self {
            namespace,
            name: name.into(),
        }
    }
}

impl Serialize for MlModelName {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer,
    {
        let d = NAME_DELIMITER;
        let serialized = match (&self.namespace, &self.name) {
            (None, name) => name.to_string(),
            (Some(namespace), name) => {
                format!("{namespace}{d}{name}")
            }
        };

        serializer.serialize_str(&serialized)
    }
}

impl<'de> Deserialize<'de> for MlModelName {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: serde::Deserializer<'de>,
    {
        deserializer.deserialize_str(MlModelNameDeserializeVisitor)
    }
}

struct MlModelNameDeserializeVisitor;

impl<'de> Visitor<'de> for MlModelNameDeserializeVisitor {
    type Value = MlModelName;

    /// always keep in sync with [`is_allowed_name_char`]
    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(
            formatter,
            "a string consisting of a namespace and name name, separated by a colon, only using alphanumeric characters, underscores & dashes"
        )
    }

    fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
    where
        E: serde::de::Error,
    {
        let mut strings = [None, None];
        let mut split = s.split(NAME_DELIMITER);

        for (buffer, part) in strings.iter_mut().zip(&mut split) {
            if part.is_empty() {
                return Err(E::custom("empty part in named data"));
            }

            if let Some(c) = part.matches(is_invalid_name_char).next() {
                return Err(E::custom(format!("invalid character '{c}' in named model")));
            }

            *buffer = Some(part.to_string());
        }

        if split.next().is_some() {
            return Err(E::custom("named model must consist of at most two parts"));
        }

        match strings {
            [Some(namespace), Some(name)] => Ok(MlModelName {
                namespace: MlModelName::canonicalize(namespace, SYSTEM_NAMESPACE),
                name,
            }),
            [Some(name), None] => Ok(MlModelName {
                namespace: None,
                name,
            }),
            _ => Err(E::custom("empty named data")),
        }
    }
}

// For now we assume all models are pixel-wise, i.e., they take a single pixel with multiple bands as input and produce a single output value.
// To support different inputs, we would need a more sophisticated logic to produce the inputs for the model.
#[derive(Debug, Clone, Hash, Eq, PartialEq, Deserialize, Serialize)]
pub struct MlModelMetadata {
    pub file_path: PathBuf,
    pub input_type: RasterDataType,
    pub num_input_bands: u32, // number of features per sample (bands per pixel)
    pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
                                     // TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
}

1	use std::path::PathBuf;
2
3	use serde::{de::Visitor, Deserialize, Serialize};
4
5	use crate::{
6	dataset::{is_invalid_name_char, SYSTEM_NAMESPACE},
7	raster::RasterDataType,
8	};
9
10	const NAME_DELIMITER: char = ':';
11
12	#[derive(Debug, Clone, Hash, Eq, PartialEq)]
13	pub struct MlModelName {
14	pub namespace: Option<String>,
15	pub name: String,
16	}
17
18	impl MlModelName {
19	/// Canonicalize a name that reflects the system namespace and model.
NEW 20	fn canonicalize<S: Into<String> + PartialEq<&'static str>>(	×
NEW 21	name: S,	×
NEW 22	system_name: &'static str,	×
NEW 23	) -> Option<String> {	×
NEW 24	if name == system_name {	×
NEW 25	None	×
26	} else {
NEW 27	Some(name.into())	×
28	}
NEW 29	}	×
30
NEW 31	pub fn new<S: Into<String>>(namespace: Option<String>, name: S) -> Self {	×
NEW 32	Self {	×
NEW 33	namespace,	×
NEW 34	name: name.into(),	×
NEW 35	}	×
NEW 36	}	×
37	}
38
39	impl Serialize for MlModelName {
40	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>	2✔
41	where	2✔
42	S: serde::Serializer,	2✔
43	{	2✔
44	let d = NAME_DELIMITER;	2✔
45	let serialized = match (&self.namespace, &self.name) {	2✔
46	(None, name) => name.to_string(),	2✔
NEW 47	(Some(namespace), name) => {	×
NEW 48	format!("{namespace}{d}{name}")	×
49	}
50	};
51
52	serializer.serialize_str(&serialized)	2✔
53	}	2✔
54	}
55
56	impl<'de> Deserialize<'de> for MlModelName {
NEW 57	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>	×
NEW 58	where	×
NEW 59	D: serde::Deserializer<'de>,	×
NEW 60	{	×
NEW 61	deserializer.deserialize_str(MlModelNameDeserializeVisitor)	×
NEW 62	}	×
63	}
64
65	struct MlModelNameDeserializeVisitor;
66
67	impl<'de> Visitor<'de> for MlModelNameDeserializeVisitor {
68	type Value = MlModelName;
69
70	/// always keep in sync with [`is_allowed_name_char`]
NEW 71	fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {	×
NEW 72	write!(	×
NEW 73	formatter,	×
NEW 74	"a string consisting of a namespace and name name, separated by a colon, only using alphanumeric characters, underscores & dashes"	×
NEW 75	)	×
NEW 76	}	×
77
NEW 78	fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>	×
NEW 79	where	×
NEW 80	E: serde::de::Error,	×
NEW 81	{	×
NEW 82	let mut strings = [None, None];	×
NEW 83	let mut split = s.split(NAME_DELIMITER);	×
84
NEW 85	for (buffer, part) in strings.iter_mut().zip(&mut split) {	×
NEW 86	if part.is_empty() {	×
NEW 87	return Err(E::custom("empty part in named data"));	×
NEW 88	}	×
89
NEW 90	if let Some(c) = part.matches(is_invalid_name_char).next() {	×
NEW 91	return Err(E::custom(format!("invalid character '{c}' in named model")));	×
NEW 92	}	×
NEW 93		×
NEW 94	*buffer = Some(part.to_string());	×
95	}
96
NEW 97	if split.next().is_some() {	×
NEW 98	return Err(E::custom("named model must consist of at most two parts"));	×
NEW 99	}	×
100
NEW 101	match strings {	×
NEW 102	[Some(namespace), Some(name)] => Ok(MlModelName {	×
NEW 103	namespace: MlModelName::canonicalize(namespace, SYSTEM_NAMESPACE),	×
NEW 104	name,	×
NEW 105	}),	×
NEW 106	[Some(name), None] => Ok(MlModelName {	×
NEW 107	namespace: None,	×
NEW 108	name,	×
NEW 109	}),	×
NEW 110	_ => Err(E::custom("empty named data")),	×
111	}
NEW 112	}	×
113	}
114
115	// For now we assume all models are pixel-wise, i.e., they take a single pixel with multiple bands as input and produce a single output value.
116	// To support different inputs, we would need a more sophisticated logic to produce the inputs for the model.
NEW 117	#[derive(Debug, Clone, Hash, Eq, PartialEq, Deserialize, Serialize)]	×
118	pub struct MlModelMetadata {
119	pub file_path: PathBuf,
120	pub input_type: RasterDataType,
121	pub num_input_bands: u32, // number of features per sample (bands per pixel)
122	pub output_type: RasterDataType, // TODO: support multiple outputs, e.g. one band for the probability of prediction
123	// TODO: output measurement, e.g. classification or regression, label names for classification. This would have to be provided by the model creator along the model file as it cannot be extracted from the model file(?)
124	}

geo-engine / geoengine / 10699003826

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous