4020902227

Build # 4020902227

Build Type

github

Committed by GitHub

Commit Message

Merge 57279c6b6 into a12da35a5

Pull Request Pull Request #743: Chore clippy fix

Run Details

165 of 165 new or added lines in 60 files covered. (100.0%)

23638 of 35485 relevant lines covered (66.61%)

38417.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.47

/dozer-api/src/generator/protoc/generator.rs

use crate::{errors::GenerationError, PipelineDetails};
use dozer_cache::cache::Cache;
use dozer_types::log::error;
use dozer_types::models::api_security::ApiSecurity;
use dozer_types::models::flags::Flags;
use dozer_types::serde::{self, Deserialize, Serialize};
use dozer_types::types::FieldType;
use handlebars::Handlebars;
use inflector::Inflector;
use prost_reflect::DescriptorPool;
use std::fmt::Write;
use std::path::{Path, PathBuf};

use super::utils::{create_descriptor_set, get_proto_descriptor};

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(crate = "self::serde")]
pub struct ProtoMetadata {
    import_libs: Vec<String>,
    messages: Vec<RPCMessage>,
    package_name: String,
    lower_name: String,
    plural_pascal_name: String,
    pascal_name: String,
    enable_token: bool,
    enable_on_event: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(crate = "self::serde")]
pub struct RPCMessage {
    pub(crate) name: String,
    pub(crate) props: Vec<String>,
}

pub struct ProtoResponse {
    pub descriptor: DescriptorPool,
    pub descriptor_bytes: Vec<u8>,
}

pub struct ProtoGenerator<'a> {
    handlebars: Handlebars<'a>,
    schema: dozer_types::types::Schema,
    schema_name: String,
    folder_path: &'a Path,
    security: &'a Option<ApiSecurity>,
    flags: &'a Option<Flags>,
}

fn safe_name(name: &str) -> String {
    if name.contains('-') {
        error!("Name of the endpoint should not contains `-`.");
    }
    name.replace(|c: char| !c.is_ascii_alphanumeric(), "_")
}
impl<'a> ProtoGenerator<'a> {
    pub fn new(
        pipeline_details: PipelineDetails,
        folder_path: &'a Path,
        security: &'a Option<ApiSecurity>,
        flags: &'a Option<Flags>,
    ) -> Result<Self, GenerationError> {
        let cache = pipeline_details.cache_endpoint.cache.clone();
        let schema_name = safe_name(&pipeline_details.cache_endpoint.endpoint.name);
        let schema = cache
            .get_schema_and_indexes_by_name(&schema_name)
            .unwrap()
            .0;

        let mut generator = Self {
            handlebars: Handlebars::new(),
            schema,
            schema_name,
            folder_path,
            security,
            flags,
        };
        generator.register_template()?;
        Ok(generator)
    }

    fn register_template(&mut self) -> Result<(), GenerationError> {
        let main_template = include_str!("template/proto.tmpl");
        self.handlebars
            .register_template_string("main", main_template)
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;
        Ok(())
    }

    fn resource_message(&self) -> RPCMessage {
        let props_message: Vec<String> = self
            .schema
            .fields
            .iter()
            .enumerate()
            .map(|(idx, field)| -> String {
                let mut result = "".to_owned();
                if field.nullable {
                    result.push_str("optional ");
                }
                let proto_type = convert_dozer_type_to_proto_type(field.typ.to_owned()).unwrap();
                let _ = writeln!(
                    result,
                    "{} {} = {}; ",
                    proto_type,
                    safe_name(&field.name),
                    idx + 1
                );
                result
            })
            .collect();

        RPCMessage {
            name: self.schema_name.to_pascal_case().to_singular(),
            props: props_message,
        }
    }

    pub fn libs_by_type(&self) -> Result<Vec<String>, GenerationError> {
        let type_need_import_libs = ["google.protobuf.Timestamp"];
        let mut libs_import: Vec<String> = self
            .schema
            .fields
            .iter()
            .map(|field| convert_dozer_type_to_proto_type(field.to_owned().typ).unwrap())
            .filter(|proto_type| -> bool {
                type_need_import_libs.contains(&proto_type.to_owned().as_str())
            })
            .map(|proto_type| match proto_type.as_str() {
                "google.protobuf.Timestamp" => "google/protobuf/timestamp.proto".to_owned(),
                _ => "".to_owned(),
            })
            .collect();
        libs_import.push("types.proto".to_owned());
        libs_import.sort();
        libs_import.dedup();
        Ok(libs_import)
    }

    pub fn get_metadata(&self) -> Result<ProtoMetadata, GenerationError> {
        let package_name = format!("dozer.generated.{}", self.schema_name);

        let messages = vec![self.resource_message()];

        let import_libs: Vec<String> = self.libs_by_type()?;
        let metadata = ProtoMetadata {
            package_name,
            messages,
            import_libs,
            lower_name: self.schema_name.to_lowercase(),
            plural_pascal_name: self.schema_name.to_pascal_case().to_plural(),
            pascal_name: self.schema_name.to_pascal_case().to_singular(),
            enable_token: self.security.is_some(),
            enable_on_event: self.flags.to_owned().unwrap_or_default().push_events,
        };
        Ok(metadata)
    }

    pub fn _generate_proto(&self) -> Result<(String, PathBuf), GenerationError> {
        if !Path::new(&self.folder_path).exists() {
            return Err(GenerationError::DirPathNotExist);
        }

        let metadata = self.get_metadata()?;

        let types_proto = include_str!("../../../protos/types.proto");

        let resource_proto = self
            .handlebars
            .render("main", &metadata)
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        // Copy types proto file
        let mut types_file = std::fs::File::create(self.folder_path.join("types.proto"))
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        let resource_path = self
            .folder_path
            .join(format!("{}.proto", self.schema_name.to_lowercase()));
        let mut resource_file = std::fs::File::create(resource_path.clone())
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        std::io::Write::write_all(&mut types_file, types_proto.as_bytes())
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        std::io::Write::write_all(&mut resource_file, resource_proto.as_bytes())
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        Ok((resource_proto, resource_path))
    }

    pub fn copy_common(folder_path: &Path) -> Result<(), GenerationError> {
        let common_proto = include_str!("../../../protos/api.proto");
        let mut common_file = std::fs::File::create(folder_path.join("common.proto"))
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        std::io::Write::write_all(&mut common_file, common_proto.as_bytes())
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        Ok(())
    }

    pub fn generate(
        folder_path: &Path,
        details: PipelineDetails,
        security: &Option<ApiSecurity>,
        flags: &Option<Flags>,
    ) -> Result<(), GenerationError> {
        let generator = ProtoGenerator::new(details, folder_path, security, flags)?;
        generator._generate_proto()?;
        Ok(())
    }

    pub fn generate_descriptor(
        folder_path: &Path,
        resources: Vec<String>,
    ) -> Result<ProtoResponse, GenerationError> {
        let descriptor_path = create_descriptor_set(folder_path, &resources)
            .map_err(|e| GenerationError::InternalError(Box::new(e)))?;

        let (descriptor_bytes, descriptor) = get_proto_descriptor(&descriptor_path)?;

        Ok(ProtoResponse {
            descriptor,
            descriptor_bytes,
        })
    }

    pub fn read(folder_path: &Path) -> Result<ProtoResponse, GenerationError> {
        let descriptor_path = folder_path.join("file_descriptor_set.bin");
        let (descriptor_bytes, descriptor) = get_proto_descriptor(&descriptor_path)?;

        Ok(ProtoResponse {
            descriptor,
            descriptor_bytes,
        })
    }
}

fn convert_dozer_type_to_proto_type(field_type: FieldType) -> Result<String, GenerationError> {
    match field_type {
        FieldType::UInt => Ok("uint64".to_owned()),
        FieldType::Int => Ok("int64".to_owned()),
        FieldType::Float => Ok("double".to_owned()),
        FieldType::Boolean => Ok("bool".to_owned()),
        FieldType::String => Ok("string".to_owned()),
        FieldType::Decimal => Ok("double".to_owned()),
        FieldType::Timestamp => Ok("google.protobuf.Timestamp".to_owned()),
        FieldType::Date => Ok("string".to_owned()),
        FieldType::Bson => Ok("google.protobuf.Any".to_owned()),
        _ => Err(GenerationError::DozerToProtoTypeNotSupported(format!(
            "{field_type:?}"
        ))),
    }
}

1	use crate::{errors::GenerationError, PipelineDetails};
2	use dozer_cache::cache::Cache;
3	use dozer_types::log::error;
4	use dozer_types::models::api_security::ApiSecurity;
5	use dozer_types::models::flags::Flags;
6	use dozer_types::serde::{self, Deserialize, Serialize};
7	use dozer_types::types::FieldType;
8	use handlebars::Handlebars;
9	use inflector::Inflector;
10	use prost_reflect::DescriptorPool;
11	use std::fmt::Write;
12	use std::path::{Path, PathBuf};
13
14	use super::utils::{create_descriptor_set, get_proto_descriptor};
15
16	#[derive(Debug, Clone, Serialize, Deserialize)]	3✔
17	#[serde(crate = "self::serde")]
18	pub struct ProtoMetadata {
19	import_libs: Vec<String>,
20	messages: Vec<RPCMessage>,
21	package_name: String,
22	lower_name: String,
23	plural_pascal_name: String,
24	pascal_name: String,
25	enable_token: bool,
26	enable_on_event: bool,
27	}
28
29	#[derive(Debug, Clone, Serialize, Deserialize)]	3✔
30	#[serde(crate = "self::serde")]
31	pub struct RPCMessage {
32	pub(crate) name: String,
33	pub(crate) props: Vec<String>,
34	}
35
36	pub struct ProtoResponse {
37	pub descriptor: DescriptorPool,
38	pub descriptor_bytes: Vec<u8>,
39	}
40
41	pub struct ProtoGenerator<'a> {
42	handlebars: Handlebars<'a>,
43	schema: dozer_types::types::Schema,
44	schema_name: String,
45	folder_path: &'a Path,
46	security: &'a Option<ApiSecurity>,
47	flags: &'a Option<Flags>,
48	}
49
50	fn safe_name(name: &str) -> String {	18✔
51	if name.contains('-') {	18✔
52	error!("Name of the endpoint should not contains `-`.");	×
53	}	18✔
54	name.replace(\|c: char\| !c.is_ascii_alphanumeric(), "_")	168✔
55	}	18✔
56	impl<'a> ProtoGenerator<'a> {
57	pub fn new(	3✔
58	pipeline_details: PipelineDetails,	3✔
59	folder_path: &'a Path,	3✔
60	security: &'a Option<ApiSecurity>,	3✔
61	flags: &'a Option<Flags>,	3✔
62	) -> Result<Self, GenerationError> {	3✔
63	let cache = pipeline_details.cache_endpoint.cache.clone();	3✔
64	let schema_name = safe_name(&pipeline_details.cache_endpoint.endpoint.name);	3✔
65	let schema = cache	3✔
66	.get_schema_and_indexes_by_name(&schema_name)	3✔
67	.unwrap()	3✔
68	.0;	3✔
69		3✔
70	let mut generator = Self {	3✔
71	handlebars: Handlebars::new(),	3✔
72	schema,	3✔
73	schema_name,	3✔
74	folder_path,	3✔
75	security,	3✔
76	flags,	3✔
77	};	3✔
78	generator.register_template()?;	3✔
79	Ok(generator)	3✔
80	}	3✔
81
82	fn register_template(&mut self) -> Result<(), GenerationError> {	3✔
83	let main_template = include_str!("template/proto.tmpl");	3✔
84	self.handlebars	3✔
85	.register_template_string("main", main_template)	3✔
86	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	3✔
87	Ok(())	3✔
88	}	3✔
89
90	fn resource_message(&self) -> RPCMessage {	3✔
91	let props_message: Vec<String> = self	3✔
92	.schema	3✔
93	.fields	3✔
94	.iter()	3✔
95	.enumerate()	3✔
96	.map(\|(idx, field)\| -> String {	15✔
97	let mut result = "".to_owned();	15✔
98	if field.nullable {	15✔
99	result.push_str("optional ");	12✔
100	}	12✔
101	let proto_type = convert_dozer_type_to_proto_type(field.typ.to_owned()).unwrap();	15✔
102	let _ = writeln!(	15✔
103	result,	15✔
104	"{} {} = {}; ",	15✔
105	proto_type,	15✔
106	safe_name(&field.name),	15✔
107	idx + 1	15✔
108	);	15✔
109	result	15✔
110	})	15✔
111	.collect();	3✔
112		3✔
113	RPCMessage {	3✔
114	name: self.schema_name.to_pascal_case().to_singular(),	3✔
115	props: props_message,	3✔
116	}	3✔
117	}	3✔
118
119	pub fn libs_by_type(&self) -> Result<Vec<String>, GenerationError> {	3✔
120	let type_need_import_libs = ["google.protobuf.Timestamp"];	3✔
121	let mut libs_import: Vec<String> = self	3✔
122	.schema	3✔
123	.fields	3✔
124	.iter()	3✔
125	.map(\|field\| convert_dozer_type_to_proto_type(field.to_owned().typ).unwrap())	15✔
126	.filter(\|proto_type\| -> bool {	15✔
127	type_need_import_libs.contains(&proto_type.to_owned().as_str())	15✔
128	})	15✔
129	.map(\|proto_type\| match proto_type.as_str() {	3✔
130	"google.protobuf.Timestamp" => "google/protobuf/timestamp.proto".to_owned(),	3✔
131	_ => "".to_owned(),	×
132	})	3✔
133	.collect();	3✔
134	libs_import.push("types.proto".to_owned());	3✔
135	libs_import.sort();	3✔
136	libs_import.dedup();	3✔
137	Ok(libs_import)	3✔
138	}	3✔
139
140	pub fn get_metadata(&self) -> Result<ProtoMetadata, GenerationError> {	3✔
141	let package_name = format!("dozer.generated.{}", self.schema_name);	3✔
142		3✔
143	let messages = vec![self.resource_message()];	3✔
144
145	let import_libs: Vec<String> = self.libs_by_type()?;	3✔
146	let metadata = ProtoMetadata {	3✔
147	package_name,	3✔
148	messages,	3✔
149	import_libs,	3✔
150	lower_name: self.schema_name.to_lowercase(),	3✔
151	plural_pascal_name: self.schema_name.to_pascal_case().to_plural(),	3✔
152	pascal_name: self.schema_name.to_pascal_case().to_singular(),	3✔
153	enable_token: self.security.is_some(),	3✔
154	enable_on_event: self.flags.to_owned().unwrap_or_default().push_events,	3✔
155	};	3✔
156	Ok(metadata)	3✔
157	}	3✔
158
159	pub fn _generate_proto(&self) -> Result<(String, PathBuf), GenerationError> {	3✔
160	if !Path::new(&self.folder_path).exists() {	3✔
161	return Err(GenerationError::DirPathNotExist);	×
162	}	3✔
163
164	let metadata = self.get_metadata()?;	3✔
165
166	let types_proto = include_str!("../../../protos/types.proto");	3✔
167
168	let resource_proto = self	3✔
169	.handlebars	3✔
170	.render("main", &metadata)	3✔
171	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	3✔
172
173	// Copy types proto file
174	let mut types_file = std::fs::File::create(self.folder_path.join("types.proto"))	3✔
175	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	3✔
176
177	let resource_path = self	3✔
178	.folder_path	3✔
179	.join(format!("{}.proto", self.schema_name.to_lowercase()));	3✔
180	let mut resource_file = std::fs::File::create(resource_path.clone())	3✔
181	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	3✔
182
183	std::io::Write::write_all(&mut types_file, types_proto.as_bytes())	3✔
184	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	3✔
185
186	std::io::Write::write_all(&mut resource_file, resource_proto.as_bytes())	3✔
187	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	3✔
188
189	Ok((resource_proto, resource_path))	3✔
190	}	3✔
191
192	pub fn copy_common(folder_path: &Path) -> Result<(), GenerationError> {	×
193	let common_proto = include_str!("../../../protos/api.proto");	×
194	let mut common_file = std::fs::File::create(folder_path.join("common.proto"))	×
195	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	×
196
197	std::io::Write::write_all(&mut common_file, common_proto.as_bytes())	×
198	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	×
199
200	Ok(())	×
201	}	×
202
203	pub fn generate(	3✔
204	folder_path: &Path,	3✔
205	details: PipelineDetails,	3✔
206	security: &Option<ApiSecurity>,	3✔
207	flags: &Option<Flags>,	3✔
208	) -> Result<(), GenerationError> {	3✔
209	let generator = ProtoGenerator::new(details, folder_path, security, flags)?;	3✔
210	generator._generate_proto()?;	3✔
211	Ok(())	3✔
212	}	3✔
213
214	pub fn generate_descriptor(	×
215	folder_path: &Path,	×
216	resources: Vec<String>,	×
217	) -> Result<ProtoResponse, GenerationError> {	×
218	let descriptor_path = create_descriptor_set(folder_path, &resources)	×
219	.map_err(\|e\| GenerationError::InternalError(Box::new(e)))?;	×
220
221	let (descriptor_bytes, descriptor) = get_proto_descriptor(&descriptor_path)?;	×
222
223	Ok(ProtoResponse {	×
224	descriptor,	×
225	descriptor_bytes,	×
226	})	×
227	}	×
228
229	pub fn read(folder_path: &Path) -> Result<ProtoResponse, GenerationError> {	×
230	let descriptor_path = folder_path.join("file_descriptor_set.bin");	×
231	let (descriptor_bytes, descriptor) = get_proto_descriptor(&descriptor_path)?;	×
232
233	Ok(ProtoResponse {	×
234	descriptor,	×
235	descriptor_bytes,	×
236	})	×
237	}	×
238	}
239
240	fn convert_dozer_type_to_proto_type(field_type: FieldType) -> Result<String, GenerationError> {	30✔
241	match field_type {	30✔
242	FieldType::UInt => Ok("uint64".to_owned()),	12✔
243	FieldType::Int => Ok("int64".to_owned()),	×
244	FieldType::Float => Ok("double".to_owned()),	6✔
245	FieldType::Boolean => Ok("bool".to_owned()),	×
246	FieldType::String => Ok("string".to_owned()),	6✔
247	FieldType::Decimal => Ok("double".to_owned()),	×
248	FieldType::Timestamp => Ok("google.protobuf.Timestamp".to_owned()),	6✔
249	FieldType::Date => Ok("string".to_owned()),	×
250	FieldType::Bson => Ok("google.protobuf.Any".to_owned()),	×
251	_ => Err(GenerationError::DozerToProtoTypeNotSupported(format!(	×
252	"{field_type:?}"	×
253	))),	×
254	}	×
255	}	30✔

getdozer / dozer / 4020902227

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous