6224433969

Committed 18 Sep 2023 03:07PM UTC coverage: 89.873% (-0.01%) from 89.887%

Build # 6224433969

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #846 from geo-engine/ml_training

Ml training

Run Details

2102 of 2102 new or added lines in 22 files covered. (100.0%)

109530 of 121872 relevant lines covered (89.87%)

59591.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/services/src/pro/machine_learning/ml_error.rs

use snafu::Snafu;

use xgboost_rs::XGBError;

use geoengine_datatypes::{error::ErrorSource, pro::MlModelId};

#[derive(Debug, Snafu)]
#[snafu(visibility(pub(crate)), context(suffix(false)), module(error))]
pub enum MachineLearningError {
    #[snafu(display("XG Booster instance could not complete training process.",))]
    BoosterTrainingError {
        source: XGBError,
    },

    #[snafu(display("The XGBoost library could not complete the operation successfully.",))]
    LibraryError {
        source: XGBError,
    },

    #[snafu(display("Couldn't parse the model file contents.",))]
    ModelFileParsingError {
        source: std::io::Error,
    },

    #[snafu(display("Could not write model to buffer.",))]
    ModelStorageError {
        source: XGBError,
    },

    #[snafu(display("Couldn't create a booster instance from the content of the model file.",))]
    LoadBoosterFromModelError {
        source: XGBError,
    },

    #[snafu(display("Couldn't generate a xgboost dmatrix from the given data.",))]
    CreateDMatrixError {
        source: XGBError,
    },

    #[snafu(display("Couldn't set the label data for the given dmatrix.",))]
    DMatrixSetLabelsError {
        source: XGBError,
    },

    #[snafu(display("Couldn't calculate predictions from the given data.",))]
    PredictionError {
        source: XGBError,
    },

    #[snafu(display("Couldn't get a base tile.",))]
    BaseTileError,

    #[snafu(display("No input data error. At least one raster is required.",))]
    NoInputData,

    #[snafu(display("There was an error with the creation of a new grid.",))]
    DataTypesError {
        source: geoengine_datatypes::error::Error,
    },

    #[snafu(display("There was an error with the joining of tokio tasks.",))]
    TokioJoinError {
        source: tokio::task::JoinError,
    },

    #[snafu(display(
        "There must be the same number of feature names and feature workflows provided. Got {} feature workflows and {} feature names instead.",
        n_feature_workflows,
        feature_names
    ))]
    WrongNumberOfFeatureNamesProvided {
        n_feature_workflows: usize,
        feature_names: usize,
    },

    #[snafu(display(
        "There must be the same number of label names and label workflows provided. Got {} label workflows and {} label names instead.",
        n_label_workflows,
        label_names
    ))]
    WrongNumberOfLabelNamesProvided {
        n_label_workflows: usize,
        label_names: usize,
    },

    MachineLearningMustHaveAtLeastOneFeatureAndOneLabel,
    MachineLearningFeatureDataNotAvailable,
    #[snafu(display("The aggregator at index {} could not be referenced", index))]
    CouldNotGetMlAggregatorRef {
        index: usize,
    },
    #[snafu(display("The feature name at index {} could not be found", index))]
    CouldNotGetMlFeatureName {
        index: usize,
    },
    CouldNotGetMlLabelKeyName,
    CouldNotGetMlModelPath,
    CouldNotGetMlModelUUID,

    CouldNotGetMlModelConfigPath {
        source: Box<dyn ErrorSource>,
    },

    // XGB Config Builder Errors
    // TODO: refactor these errors into a separate module?
    #[snafu(display("The number of distinct classes in the label data must be set!"))]
    UndefinedNumberOfClasses,

    #[snafu(display("Unknown or unsupported booster type: {}", s))]
    UnknownBoosterType {
        s: String,
    },

    #[snafu(display("Unknown or unsupported tree method: {}", s))]
    UnknownTreeMethod {
        s: String,
    },

    #[snafu(display("Unknown or unsupported process type: {}", s))]
    UnknownProcessType {
        s: String,
    },

    #[snafu(display("Unknown or unsupported objective function type: {}", s))]
    UnknownObjective {
        s: String,
    },

    #[snafu(display("Unknown or unsupported grow policy: {}", s))]
    UnknownGrowPolicy {
        s: String,
    },

    #[snafu(display("Unknown or unsupported predictor type: {}", s))]
    UnknownPredictorType {
        s: String,
    },

    #[snafu(display("Invalid refresh leaf input: {}. Only 0 | 1 is allowed as input", s))]
    InvalidRefreshLeafInput {
        s: String,
    },

    #[snafu(display("Unknown or unsupported config parameter for xgboost: {}", s))]
    UnknownXGBConfigParameter {
        s: String,
    },

    #[snafu(display(
        "Invalid 'eta' input for xgboost: {}. 'eta' should be between 0 and 1 (inclusive).",
        s
    ))]
    InvalidEtaInput {
        s: String,
    },

    #[snafu(display("Invalid 'max_leaves' input for xgboost: {}. 'max_leaves' should be a non-negative integer.", s))]
    InvalidMaxLeavesInput {
        s: String,
    },

    #[snafu(display("Invalid 'min_child_weight' input for xgboost: {}. 'min_child_weight' should be a non-negative value.", s))]
    InvalidMinChildWeightInput {
        s: String,
    },

    #[snafu(display("Invalid 'subsample' input for xgboost: {}. 'subsample' should be between 0 and 1 (inclusive).", s))]
    InvalidSubsampleInput {
        s: String,
    },

    #[snafu(display("Invalid 'scale_pos_weight' input for xgboost: {}. 'scale_pos_weight' should be a non-negative value.", s))]
    InvalidScalePosWeightInput {
        s: String,
    },

    #[snafu(display("Invalid 'base_score' input for xgboost: {}.", s))]
    InvalidBaseScoreInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'gamma' input for xgboost: {}. 'gamma' should be a non-negative value.",
        s
    ))]
    InvalidGammaInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'seed' input for xgboost: {}. 'seed' should be a non-negative integer.",
        s
    ))]
    InvalidSeedInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'num_class' input for xgboost: {}. 'num_class' should be a positive integer.",
        s
    ))]
    InvalidNumClassInput {
        s: String,
    },
    #[snafu(display(
        "Invalid 'silent' input for xgboost: {}. 'silent' should be either '0' or '1'.",
        s
    ))]
    InvalidSilentInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'max_depth' input for xgboost: {}. 'max_depth' should be a non-negative integer.",
        s
    ))]
    InvalidMaxDepthInput {
        s: String,
    },

    #[snafu(display("Invalid 'colsample_bylevel' input for xgboost: {}. 'colsample_bylevel' should be between 0 and 1 (inclusive).", s))]
    InvalidColsampleBylevelInput {
        s: String,
    },

    #[snafu(display("Invalid 'colsample_bytree' input for xgboost: {}. 'colsample_bytree' should be between 0 and 1 (inclusive).", s))]
    InvalidColsampleBytreeInput {
        s: String,
    },

    #[snafu(display("Invalid 'colsample_bynode' input for xgboost: {}. 'colsample_bynode' should be between 0 and 1 (inclusive).", s))]
    InvalidColsampleBynodeInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'sketch_eps' input for xgboost: {}. 'sketch_eps' should be a non-negative value.",
        s
    ))]
    InvalidSketchEpsInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'max_bin' input for xgboost: {}. 'max_bin' should be a positive integer.",
        s
    ))]
    InvalidMaxBinInput {
        s: String,
    },

    #[snafu(display("Invalid 'max_delta_step' input for xgboost: {}. 'max_delta_step' should be a non-negative value.", s))]
    InvalidMaxDeltaStepInput {
        s: String,
    },

    #[snafu(display("Invalid 'num_parallel_tree' input for xgboost: {}. 'num_parallel_tree' should be a positive integer.", s))]
    InvalidNumParallelTreeInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'lambda' input for xgboost: {}. 'lambda' should be a non-negative value.",
        s
    ))]
    InvalidLambdaInput {
        s: String,
    },

    #[snafu(display(
        "Invalid 'alpha' input for xgboost: {}. 'alpha' should be a non-negative value.",
        s
    ))]
    InvalidAlphaInput {
        s: String,
    },
    #[snafu(display(
        "Invalid 'verbosity' input for xgboost: {}. 'verbosity' should be either true or false.",
        s
    ))]
    InvalidDebugLevelInput {
        s: String,
    },

    #[snafu(display("Invalid 'colsample_bylevel' input for xgboost: {}. 'colsample_bylevel' should be between 0 and 1 (inclusive).", s))]
    InvalidColsampleByLevelInput {
        s: String,
    },

    #[snafu(display("Invalid 'colsample_bytree' input for xgboost: {}. 'colsample_bytree' should be between 0 and 1 (inclusive).", s))]
    InvalidColsampleByTreeInput {
        s: String,
    },

    #[snafu(display("Invalid 'colsample_bynode' input for xgboost: {}. 'colsample_bynode' should be between 0 and 1 (inclusive).", s))]
    InvalidColsampleByNodeInput {
        s: String,
    },
    #[snafu(display("Invalid booster parameters: {}", s))]
    InvalidBoosterParameters {
        s: String,
    },
    #[snafu(display("Invalid learning task parameters: {}", s))]
    InvalidLearningTaskParameters {
        s: String,
    },

    #[snafu(display("Invalid tree booster parameters: {}", s))]
    InvalidTreeBoosterParameters {
        s: String,
    },

    #[snafu(display("The model with id {} was not found in postgres", model_id))]
    UnknownModelIdInPostgres {
        model_id: MlModelId,
    },
}

impl From<std::io::Error> for MachineLearningError {
    fn from(source: std::io::Error) -> Self {
        Self::ModelFileParsingError { source }
    }
}

impl From<XGBError> for MachineLearningError {
    fn from(source: XGBError) -> Self {
        Self::LibraryError { source }
    }
}

1	use snafu::Snafu;
2
3	use xgboost_rs::XGBError;
4
5	use geoengine_datatypes::{error::ErrorSource, pro::MlModelId};
6
7	#[derive(Debug, Snafu)]	×
8	#[snafu(visibility(pub(crate)), context(suffix(false)), module(error))]
9	pub enum MachineLearningError {
10	#[snafu(display("XG Booster instance could not complete training process.",))]
11	BoosterTrainingError {
12	source: XGBError,
13	},
14
15	#[snafu(display("The XGBoost library could not complete the operation successfully.",))]
16	LibraryError {
17	source: XGBError,
18	},
19
20	#[snafu(display("Couldn't parse the model file contents.",))]
21	ModelFileParsingError {
22	source: std::io::Error,
23	},
24
25	#[snafu(display("Could not write model to buffer.",))]
26	ModelStorageError {
27	source: XGBError,
28	},
29
30	#[snafu(display("Couldn't create a booster instance from the content of the model file.",))]
31	LoadBoosterFromModelError {
32	source: XGBError,
33	},
34
35	#[snafu(display("Couldn't generate a xgboost dmatrix from the given data.",))]
36	CreateDMatrixError {
37	source: XGBError,
38	},
39
40	#[snafu(display("Couldn't set the label data for the given dmatrix.",))]
41	DMatrixSetLabelsError {
42	source: XGBError,
43	},
44
45	#[snafu(display("Couldn't calculate predictions from the given data.",))]
46	PredictionError {
47	source: XGBError,
48	},
49
50	#[snafu(display("Couldn't get a base tile.",))]
51	BaseTileError,
52
53	#[snafu(display("No input data error. At least one raster is required.",))]
54	NoInputData,
55
56	#[snafu(display("There was an error with the creation of a new grid.",))]
57	DataTypesError {
58	source: geoengine_datatypes::error::Error,
59	},
60
61	#[snafu(display("There was an error with the joining of tokio tasks.",))]
62	TokioJoinError {
63	source: tokio::task::JoinError,
64	},
65
66	#[snafu(display(
67	"There must be the same number of feature names and feature workflows provided. Got {} feature workflows and {} feature names instead.",
68	n_feature_workflows,
69	feature_names
70	))]
71	WrongNumberOfFeatureNamesProvided {
72	n_feature_workflows: usize,
73	feature_names: usize,
74	},
75
76	#[snafu(display(
77	"There must be the same number of label names and label workflows provided. Got {} label workflows and {} label names instead.",
78	n_label_workflows,
79	label_names
80	))]
81	WrongNumberOfLabelNamesProvided {
82	n_label_workflows: usize,
83	label_names: usize,
84	},
85
86	MachineLearningMustHaveAtLeastOneFeatureAndOneLabel,
87	MachineLearningFeatureDataNotAvailable,
88	#[snafu(display("The aggregator at index {} could not be referenced", index))]
89	CouldNotGetMlAggregatorRef {
90	index: usize,
91	},
92	#[snafu(display("The feature name at index {} could not be found", index))]
93	CouldNotGetMlFeatureName {
94	index: usize,
95	},
96	CouldNotGetMlLabelKeyName,
97	CouldNotGetMlModelPath,
98	CouldNotGetMlModelUUID,
99
100	CouldNotGetMlModelConfigPath {
101	source: Box<dyn ErrorSource>,
102	},
103
104	// XGB Config Builder Errors
105	// TODO: refactor these errors into a separate module?
106	#[snafu(display("The number of distinct classes in the label data must be set!"))]
107	UndefinedNumberOfClasses,
108
109	#[snafu(display("Unknown or unsupported booster type: {}", s))]
110	UnknownBoosterType {
111	s: String,
112	},
113
114	#[snafu(display("Unknown or unsupported tree method: {}", s))]
115	UnknownTreeMethod {
116	s: String,
117	},
118
119	#[snafu(display("Unknown or unsupported process type: {}", s))]
120	UnknownProcessType {
121	s: String,
122	},
123
124	#[snafu(display("Unknown or unsupported objective function type: {}", s))]
125	UnknownObjective {
126	s: String,
127	},
128
129	#[snafu(display("Unknown or unsupported grow policy: {}", s))]
130	UnknownGrowPolicy {
131	s: String,
132	},
133
134	#[snafu(display("Unknown or unsupported predictor type: {}", s))]
135	UnknownPredictorType {
136	s: String,
137	},
138
139	#[snafu(display("Invalid refresh leaf input: {}. Only 0 \| 1 is allowed as input", s))]
140	InvalidRefreshLeafInput {
141	s: String,
142	},
143
144	#[snafu(display("Unknown or unsupported config parameter for xgboost: {}", s))]
145	UnknownXGBConfigParameter {
146	s: String,
147	},
148
149	#[snafu(display(
150	"Invalid 'eta' input for xgboost: {}. 'eta' should be between 0 and 1 (inclusive).",
151	s
152	))]
153	InvalidEtaInput {
154	s: String,
155	},
156
157	#[snafu(display("Invalid 'max_leaves' input for xgboost: {}. 'max_leaves' should be a non-negative integer.", s))]
158	InvalidMaxLeavesInput {
159	s: String,
160	},
161
162	#[snafu(display("Invalid 'min_child_weight' input for xgboost: {}. 'min_child_weight' should be a non-negative value.", s))]
163	InvalidMinChildWeightInput {
164	s: String,
165	},
166
167	#[snafu(display("Invalid 'subsample' input for xgboost: {}. 'subsample' should be between 0 and 1 (inclusive).", s))]
168	InvalidSubsampleInput {
169	s: String,
170	},
171
172	#[snafu(display("Invalid 'scale_pos_weight' input for xgboost: {}. 'scale_pos_weight' should be a non-negative value.", s))]
173	InvalidScalePosWeightInput {
174	s: String,
175	},
176
177	#[snafu(display("Invalid 'base_score' input for xgboost: {}.", s))]
178	InvalidBaseScoreInput {
179	s: String,
180	},
181
182	#[snafu(display(
183	"Invalid 'gamma' input for xgboost: {}. 'gamma' should be a non-negative value.",
184	s
185	))]
186	InvalidGammaInput {
187	s: String,
188	},
189
190	#[snafu(display(
191	"Invalid 'seed' input for xgboost: {}. 'seed' should be a non-negative integer.",
192	s
193	))]
194	InvalidSeedInput {
195	s: String,
196	},
197
198	#[snafu(display(
199	"Invalid 'num_class' input for xgboost: {}. 'num_class' should be a positive integer.",
200	s
201	))]
202	InvalidNumClassInput {
203	s: String,
204	},
205	#[snafu(display(
206	"Invalid 'silent' input for xgboost: {}. 'silent' should be either '0' or '1'.",
207	s
208	))]
209	InvalidSilentInput {
210	s: String,
211	},
212
213	#[snafu(display(
214	"Invalid 'max_depth' input for xgboost: {}. 'max_depth' should be a non-negative integer.",
215	s
216	))]
217	InvalidMaxDepthInput {
218	s: String,
219	},
220
221	#[snafu(display("Invalid 'colsample_bylevel' input for xgboost: {}. 'colsample_bylevel' should be between 0 and 1 (inclusive).", s))]
222	InvalidColsampleBylevelInput {
223	s: String,
224	},
225
226	#[snafu(display("Invalid 'colsample_bytree' input for xgboost: {}. 'colsample_bytree' should be between 0 and 1 (inclusive).", s))]
227	InvalidColsampleBytreeInput {
228	s: String,
229	},
230
231	#[snafu(display("Invalid 'colsample_bynode' input for xgboost: {}. 'colsample_bynode' should be between 0 and 1 (inclusive).", s))]
232	InvalidColsampleBynodeInput {
233	s: String,
234	},
235
236	#[snafu(display(
237	"Invalid 'sketch_eps' input for xgboost: {}. 'sketch_eps' should be a non-negative value.",
238	s
239	))]
240	InvalidSketchEpsInput {
241	s: String,
242	},
243
244	#[snafu(display(
245	"Invalid 'max_bin' input for xgboost: {}. 'max_bin' should be a positive integer.",
246	s
247	))]
248	InvalidMaxBinInput {
249	s: String,
250	},
251
252	#[snafu(display("Invalid 'max_delta_step' input for xgboost: {}. 'max_delta_step' should be a non-negative value.", s))]
253	InvalidMaxDeltaStepInput {
254	s: String,
255	},
256
257	#[snafu(display("Invalid 'num_parallel_tree' input for xgboost: {}. 'num_parallel_tree' should be a positive integer.", s))]
258	InvalidNumParallelTreeInput {
259	s: String,
260	},
261
262	#[snafu(display(
263	"Invalid 'lambda' input for xgboost: {}. 'lambda' should be a non-negative value.",
264	s
265	))]
266	InvalidLambdaInput {
267	s: String,
268	},
269
270	#[snafu(display(
271	"Invalid 'alpha' input for xgboost: {}. 'alpha' should be a non-negative value.",
272	s
273	))]
274	InvalidAlphaInput {
275	s: String,
276	},
277	#[snafu(display(
278	"Invalid 'verbosity' input for xgboost: {}. 'verbosity' should be either true or false.",
279	s
280	))]
281	InvalidDebugLevelInput {
282	s: String,
283	},
284
285	#[snafu(display("Invalid 'colsample_bylevel' input for xgboost: {}. 'colsample_bylevel' should be between 0 and 1 (inclusive).", s))]
286	InvalidColsampleByLevelInput {
287	s: String,
288	},
289
290	#[snafu(display("Invalid 'colsample_bytree' input for xgboost: {}. 'colsample_bytree' should be between 0 and 1 (inclusive).", s))]
291	InvalidColsampleByTreeInput {
292	s: String,
293	},
294
295	#[snafu(display("Invalid 'colsample_bynode' input for xgboost: {}. 'colsample_bynode' should be between 0 and 1 (inclusive).", s))]
296	InvalidColsampleByNodeInput {
297	s: String,
298	},
299	#[snafu(display("Invalid booster parameters: {}", s))]
300	InvalidBoosterParameters {
301	s: String,
302	},
303	#[snafu(display("Invalid learning task parameters: {}", s))]
304	InvalidLearningTaskParameters {
305	s: String,
306	},
307
308	#[snafu(display("Invalid tree booster parameters: {}", s))]
309	InvalidTreeBoosterParameters {
310	s: String,
311	},
312
313	#[snafu(display("The model with id {} was not found in postgres", model_id))]
314	UnknownModelIdInPostgres {
315	model_id: MlModelId,
316	},
317	}
318
319	impl From<std::io::Error> for MachineLearningError {
320	fn from(source: std::io::Error) -> Self {	×
321	Self::ModelFileParsingError { source }	×
322	}	×
323	}
324
325	impl From<XGBError> for MachineLearningError {
326	fn from(source: XGBError) -> Self {	×
327	Self::LibraryError { source }	×
328	}	×
329	}

geo-engine / geoengine / 6224433969

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous