6299724219

Committed 25 Sep 2023 12:58PM UTC coverage: 77.81% (+0.5%) from 77.275%

Build # 6299724219

Build Type

push

github

Committed by

chubei

Commit Message

fix: Add `BINDGEN_EXTRA_CLANG_ARGS` to cross compile rocksdb

Run Details

50223 of 64546 relevant lines covered (77.81%)

148909.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/dozer-cli/src/main.rs

use clap::Parser;
#[cfg(feature = "cloud")]
use dozer_cli::cli::cloud::CloudCommands;
use dozer_cli::cli::generate_config_repl;
use dozer_cli::cli::types::{Cli, Commands, ConnectorCommand, RunCommands, SecurityCommands};
use dozer_cli::cli::{init_dozer, list_sources, LOGO};
use dozer_cli::errors::{CliError, CloudError, OrchestrationError};
use dozer_cli::simple::SimpleOrchestrator;
#[cfg(feature = "cloud")]
use dozer_cli::CloudOrchestrator;
use dozer_cli::{live, set_ctrl_handler, set_panic_hook, shutdown};
use dozer_tracing::LabelsAndProgress;
use dozer_types::models::telemetry::{TelemetryConfig, TelemetryMetricsConfig};
use dozer_types::serde::Deserialize;
use dozer_types::tracing::{error, error_span, info};
use tokio::runtime::Runtime;
use tokio::time;

use clap::CommandFactory;
#[cfg(feature = "cloud")]
use dozer_cli::cloud_app_context::CloudAppContext;
use std::cmp::Ordering;
use std::sync::Arc;

use dozer_types::log::{debug, warn};
use std::time::Duration;
use std::{env, process};

fn main() {
    if let Err(e) = run() {
        display_error(&e);
        process::exit(1);
    }
}

fn render_logo() {
    const VERSION: &str = env!("CARGO_PKG_VERSION");

    println!("{LOGO}");
    println!("\nDozer Version: {VERSION}\n");
}

#[derive(Deserialize, Debug)]
#[serde(crate = "dozer_types::serde")]
struct DozerPackage {
    #[serde(rename(deserialize = "latestVersion"))]
    pub latest_version: String,
    #[serde(rename(deserialize = "availableAssets"))]
    pub _available_assets: Vec<String>,
    pub link: String,
}

fn version_to_vector(version: &str) -> Vec<i32> {
    version.split('.').map(|s| s.parse().unwrap()).collect()
}

fn compare_versions(v1: Vec<i32>, v2: Vec<i32>) -> bool {
    for i in 0..v1.len() {
        match v1.get(i).cmp(&v2.get(i)) {
            Ordering::Greater => return true,
            Ordering::Less => return false,
            Ordering::Equal => continue,
        }
    }
    false
}

async fn check_update() {
    const VERSION: &str = env!("CARGO_PKG_VERSION");
    let dozer_env = std::env::var("DOZER_ENV").unwrap_or("local".to_string());
    let dozer_dev = std::env::var("DOZER_DEV").unwrap_or("ext".to_string());
    let query = vec![
        ("version", VERSION),
        ("build", std::env::consts::ARCH),
        ("os", std::env::consts::OS),
        ("env", &dozer_env),
        ("dev", &dozer_dev),
    ];

    let request_url = "https://metadata.dev.getdozer.io/";

    let client = reqwest::Client::new();

    let mut printed = false;

    loop {
        let response = client
            .get(&request_url.to_string())
            .query(&query)
            .send()
            .await;

        match response {
            Ok(r) => {
                if !printed {
                    let package: DozerPackage = r.json().await.unwrap();
                    let current = version_to_vector(VERSION);
                    let remote = version_to_vector(&package.latest_version);

                    if compare_versions(remote, current) {
                        info!("A new version of Dozer is available.");
                        info!(
                            "You can download v{}, from {}.",
                            package.latest_version, package.link
                        );
                        printed = true;
                    }
                }
            }
            Err(e) => {
                // We dont show error if error is connection error, because mostly it happens
                // when main thread is shutting down before request completes.
                if !e.is_connect() {
                    warn!("Unable to fetch the latest metadata");
                }

                debug!("Updates check error: {}", e);
            }
        }
        time::sleep(Duration::from_secs(2 * 60 * 60)).await;
    }
}

fn run() -> Result<(), OrchestrationError> {
    // Reloading trace layer seems impossible, so we are running Cli::parse in a closure
    // and then initializing it after reading the configuration. This is a hacky workaround, but it works.

    let cli = parse_and_generate()?;
    let mut dozer = init_orchestrator(&cli)?;
    let (shutdown_sender, shutdown_receiver) = shutdown::new(&dozer.runtime);
    set_ctrl_handler(shutdown_sender);

    // Now we have access to telemetry configuration. Telemetry must be initialized in tokio runtime.
    let app_id = dozer
        .config
        .cloud
        .app_id
        .as_deref()
        .unwrap_or(&dozer.config.app_name);

    // We always enable telemetry when running live.
    let telemetry_config = if matches!(cli.cmd, Commands::Live(_)) {
        TelemetryConfig {
            trace: None,
            metrics: Some(TelemetryMetricsConfig::Prometheus),
        }
    } else {
        dozer.config.telemetry.clone()
    };

    let _telemetry = dozer
        .runtime
        .block_on(async { Telemetry::new(Some(app_id), &telemetry_config) });

    set_panic_hook();

    // run individual servers
    (match cli.cmd {
        Commands::Run(run) => match run.command {
            Some(RunCommands::Api) => {
                render_logo();

                dozer.run_api(shutdown_receiver)
            }
            Some(RunCommands::App) => {
                render_logo();

                dozer.run_apps(shutdown_receiver, None)
            }
            None => {
                render_logo();
                dozer.run_all(shutdown_receiver, run.locked)
            }
        },
        Commands::Security(security) => match security.command {
            SecurityCommands::GenerateToken => {
                let token = dozer.generate_token(None)?;
                info!("token: {:?} ", token);
                Ok(())
            }
        },
        Commands::Build(build) => {
            let force = build.force.is_some();

            dozer.build(force, shutdown_receiver, build.locked)
        }
        Commands::Connectors(ConnectorCommand { filter }) => dozer.runtime.block_on(list_sources(
            dozer.runtime.clone(),
            cli.config_paths,
            cli.config_token,
            cli.config_overrides,
            cli.ignore_pipe,
            filter,
        )),
        Commands::Clean => dozer.clean(),
        #[cfg(feature = "cloud")]
        Commands::Cloud(cloud) => {
            render_logo();

            match cloud.command.clone() {
                CloudCommands::Deploy(deploy) => dozer.deploy(cloud, deploy, cli.config_paths),
                CloudCommands::Login {
                    organisation_slug,
                    profile_name,
                    client_id,
                    client_secret,
                } => dozer.login(
                    cloud,
                    organisation_slug,
                    profile_name,
                    client_id,
                    client_secret,
                ),
                CloudCommands::Secrets(command) => dozer.execute_secrets_command(cloud, command),
                CloudCommands::Delete => dozer.delete(cloud),
                CloudCommands::Status => dozer.status(cloud),
                CloudCommands::Monitor => dozer.monitor(cloud),
                CloudCommands::Logs(logs) => dozer.trace_logs(cloud, logs),
                CloudCommands::Version(version) => dozer.version(cloud, version),
                CloudCommands::List(list) => dozer.list(cloud, list),
                CloudCommands::SetApp { app_id } => {
                    CloudAppContext::save_app_id(app_id.clone())?;
                    info!("Using \"{app_id}\" app");
                    Ok(())
                }
                CloudCommands::ApiRequestSamples { endpoint } => {
                    dozer.print_api_request_samples(cloud, endpoint)
                }
            }
        }
        Commands::Init => {
            panic!("This should not happen as it is handled in parse_and_generate");
        }
        Commands::Live(live_flags) => {
            render_logo();
            dozer.runtime.block_on(live::start_live_server(
                &dozer.runtime,
                shutdown_receiver,
                live_flags,
            ))?;
            Ok(())
        }
    })
    .map_err(|e| {
        let _span = error_span!("OrchestrationError", error = %e);

        e
    })
}

// Some commands dont need to initialize the orchestrator
// This function is used to run those commands
fn parse_and_generate() -> Result<Cli, OrchestrationError> {
    dozer_tracing::init_telemetry_closure(
        None,
        &Default::default(),
        || -> Result<Cli, OrchestrationError> {
            let cli = Cli::parse();

            if let Commands::Init = cli.cmd {
                Telemetry::new(None, &Default::default());
                if let Err(e) = generate_config_repl() {
                    error!("{}", e);
                    Err(e)
                } else {
                    // We need to exit here, otherwise the orchestrator will be initialized
                    process::exit(0);
                }
            } else {
                Ok(cli)
            }
        },
    )
}

fn init_orchestrator(cli: &Cli) -> Result<SimpleOrchestrator, CliError> {
    dozer_tracing::init_telemetry_closure(
        None,
        &Default::default(),
        || -> Result<SimpleOrchestrator, CliError> {
            let runtime = Arc::new(Runtime::new().map_err(CliError::FailedToCreateTokioRuntime)?);
            let res = runtime.block_on(init_dozer(
                runtime.clone(),
                cli.config_paths.clone(),
                cli.config_token.clone(),
                cli.config_overrides.clone(),
                cli.ignore_pipe,
                LabelsAndProgress::new(Default::default(), cli.enable_progress),
            ));

            match res {
                Ok(dozer) => {
                    dozer.runtime.spawn(check_update());
                    Ok(dozer)
                }
                Err(e) => {
                    if let CliError::FailedToFindConfigurationFiles(_) = &e {
                        let description = "Dozer was not able to find configuration files. \n\n\
                    Please use \"dozer init\" to create project or \"dozer -c {path}\" with path to your configuration.\n\
                    Configuration documentation can be found in https://getdozer.io/docs/configuration";

                        let mut command = Cli::command();
                        command = command.about(format!("\n\n\n{} \n {}", LOGO, description));

                        println!("{}", command.render_help());
                    }

                    error!("{}", e);
                    Err(e)
                }
            }
        },
    )
}

fn display_error(e: &OrchestrationError) {
    if let OrchestrationError::CloudError(CloudError::ApplicationNotFound) = &e {
        let description = "Dozer cloud service was not able to find application. \n\n\
        Please check your application id in `dozer-config.cloud.yaml` file.\n\
        To change it, you can manually update file or use \"dozer cloud set-app {app_id}\".";

        error!("{}", description);
    } else {
        error!("{}", e);
    }
}

struct Telemetry();

impl Telemetry {
    fn new(app_name: Option<&str>, config: &TelemetryConfig) -> Self {
        dozer_tracing::init_telemetry(app_name, config);
        Self()
    }
}

impl Drop for Telemetry {
    fn drop(&mut self) {
        dozer_tracing::shutdown_telemetry();
    }
}

1	use clap::Parser;
2	#[cfg(feature = "cloud")]
3	use dozer_cli::cli::cloud::CloudCommands;
4	use dozer_cli::cli::generate_config_repl;
5	use dozer_cli::cli::types::{Cli, Commands, ConnectorCommand, RunCommands, SecurityCommands};
6	use dozer_cli::cli::{init_dozer, list_sources, LOGO};
7	use dozer_cli::errors::{CliError, CloudError, OrchestrationError};
8	use dozer_cli::simple::SimpleOrchestrator;
9	#[cfg(feature = "cloud")]
10	use dozer_cli::CloudOrchestrator;
11	use dozer_cli::{live, set_ctrl_handler, set_panic_hook, shutdown};
12	use dozer_tracing::LabelsAndProgress;
13	use dozer_types::models::telemetry::{TelemetryConfig, TelemetryMetricsConfig};
14	use dozer_types::serde::Deserialize;
15	use dozer_types::tracing::{error, error_span, info};
16	use tokio::runtime::Runtime;
17	use tokio::time;
18
19	use clap::CommandFactory;
20	#[cfg(feature = "cloud")]
21	use dozer_cli::cloud_app_context::CloudAppContext;
22	use std::cmp::Ordering;
23	use std::sync::Arc;
24
25	use dozer_types::log::{debug, warn};
26	use std::time::Duration;
27	use std::{env, process};
28
29	fn main() {	×
30	if let Err(e) = run() {	×
31	display_error(&e);	×
32	process::exit(1);	×
33	}	×
34	}	×
35
36	fn render_logo() {	×
37	const VERSION: &str = env!("CARGO_PKG_VERSION");	×
38		×
39	println!("{LOGO}");	×
40	println!("\nDozer Version: {VERSION}\n");	×
41	}	×
42
43	#[derive(Deserialize, Debug)]	×
44	#[serde(crate = "dozer_types::serde")]
45	struct DozerPackage {
46	#[serde(rename(deserialize = "latestVersion"))]
47	pub latest_version: String,
48	#[serde(rename(deserialize = "availableAssets"))]
49	pub _available_assets: Vec<String>,
50	pub link: String,
51	}
52
53	fn version_to_vector(version: &str) -> Vec<i32> {	×
54	version.split('.').map(\|s\| s.parse().unwrap()).collect()	×
55	}	×
56
57	fn compare_versions(v1: Vec<i32>, v2: Vec<i32>) -> bool {	×
58	for i in 0..v1.len() {	×
59	match v1.get(i).cmp(&v2.get(i)) {	×
60	Ordering::Greater => return true,	×
61	Ordering::Less => return false,	×
62	Ordering::Equal => continue,	×
63	}
64	}
65	false	×
66	}	×
67
68	async fn check_update() {	×
69	const VERSION: &str = env!("CARGO_PKG_VERSION");	×
70	let dozer_env = std::env::var("DOZER_ENV").unwrap_or("local".to_string());	×
71	let dozer_dev = std::env::var("DOZER_DEV").unwrap_or("ext".to_string());	×
72	let query = vec![	×
73	("version", VERSION),	×
74	("build", std::env::consts::ARCH),	×
75	("os", std::env::consts::OS),	×
76	("env", &dozer_env),	×
77	("dev", &dozer_dev),	×
78	];	×
79		×
80	let request_url = "https://metadata.dev.getdozer.io/";	×
81		×
82	let client = reqwest::Client::new();	×
83		×
84	let mut printed = false;	×
85
86	loop {
87	let response = client	×
88	.get(&request_url.to_string())	×
89	.query(&query)	×
90	.send()	×
91	.await;	×
92
93	match response {	×
94	Ok(r) => {	×
95	if !printed {	×
96	let package: DozerPackage = r.json().await.unwrap();	×
97	let current = version_to_vector(VERSION);	×
98	let remote = version_to_vector(&package.latest_version);	×
99		×
100	if compare_versions(remote, current) {	×
101	info!("A new version of Dozer is available.");	×
102	info!(	×
103	"You can download v{}, from {}.",	×
104	package.latest_version, package.link	×
105	);	×
106	printed = true;	×
107	}	×
108	}	×
109	}
110	Err(e) => {	×
111	// We dont show error if error is connection error, because mostly it happens	×
112	// when main thread is shutting down before request completes.	×
113	if !e.is_connect() {	×
114	warn!("Unable to fetch the latest metadata");	×
115	}	×
116
117	debug!("Updates check error: {}", e);	×
118	}
119	}
120	time::sleep(Duration::from_secs(2 * 60 * 60)).await;	×
121	}
122	}
123
124	fn run() -> Result<(), OrchestrationError> {	×
125	// Reloading trace layer seems impossible, so we are running Cli::parse in a closure
126	// and then initializing it after reading the configuration. This is a hacky workaround, but it works.
127
128	let cli = parse_and_generate()?;	×
129	let mut dozer = init_orchestrator(&cli)?;	×
130	let (shutdown_sender, shutdown_receiver) = shutdown::new(&dozer.runtime);	×
131	set_ctrl_handler(shutdown_sender);	×
132		×
133	// Now we have access to telemetry configuration. Telemetry must be initialized in tokio runtime.	×
134	let app_id = dozer	×
135	.config	×
136	.cloud	×
137	.app_id	×
138	.as_deref()	×
139	.unwrap_or(&dozer.config.app_name);	×
140
141	// We always enable telemetry when running live.
142	let telemetry_config = if matches!(cli.cmd, Commands::Live(_)) {	×
143	TelemetryConfig {	×
144	trace: None,	×
145	metrics: Some(TelemetryMetricsConfig::Prometheus),	×
146	}	×
147	} else {
148	dozer.config.telemetry.clone()	×
149	};
150
151	let _telemetry = dozer	×
152	.runtime	×
153	.block_on(async { Telemetry::new(Some(app_id), &telemetry_config) });	×
154		×
155	set_panic_hook();	×
156		×
157	// run individual servers	×
158	(match cli.cmd {	×
159	Commands::Run(run) => match run.command {	×
160	Some(RunCommands::Api) => {
161	render_logo();	×
162		×
163	dozer.run_api(shutdown_receiver)	×
164	}
165	Some(RunCommands::App) => {
166	render_logo();	×
167		×
168	dozer.run_apps(shutdown_receiver, None)	×
169	}
170	None => {
171	render_logo();	×
172	dozer.run_all(shutdown_receiver, run.locked)	×
173	}
174	},
175	Commands::Security(security) => match security.command {	×
176	SecurityCommands::GenerateToken => {
177	let token = dozer.generate_token(None)?;	×
178	info!("token: {:?} ", token);	×
179	Ok(())	×
180	}
181	},
182	Commands::Build(build) => {	×
183	let force = build.force.is_some();	×
184		×
185	dozer.build(force, shutdown_receiver, build.locked)	×
186	}
187	Commands::Connectors(ConnectorCommand { filter }) => dozer.runtime.block_on(list_sources(	×
188	dozer.runtime.clone(),	×
189	cli.config_paths,	×
190	cli.config_token,	×
191	cli.config_overrides,	×
192	cli.ignore_pipe,	×
193	filter,	×
194	)),	×
195	Commands::Clean => dozer.clean(),	×
196	#[cfg(feature = "cloud")]
197	Commands::Cloud(cloud) => {
198	render_logo();
199
200	match cloud.command.clone() {
201	CloudCommands::Deploy(deploy) => dozer.deploy(cloud, deploy, cli.config_paths),
202	CloudCommands::Login {
203	organisation_slug,
204	profile_name,
205	client_id,
206	client_secret,
207	} => dozer.login(
208	cloud,
209	organisation_slug,
210	profile_name,
211	client_id,
212	client_secret,
213	),
214	CloudCommands::Secrets(command) => dozer.execute_secrets_command(cloud, command),
215	CloudCommands::Delete => dozer.delete(cloud),
216	CloudCommands::Status => dozer.status(cloud),
217	CloudCommands::Monitor => dozer.monitor(cloud),
218	CloudCommands::Logs(logs) => dozer.trace_logs(cloud, logs),
219	CloudCommands::Version(version) => dozer.version(cloud, version),
220	CloudCommands::List(list) => dozer.list(cloud, list),
221	CloudCommands::SetApp { app_id } => {
222	CloudAppContext::save_app_id(app_id.clone())?;
223	info!("Using \"{app_id}\" app");
224	Ok(())
225	}
226	CloudCommands::ApiRequestSamples { endpoint } => {
227	dozer.print_api_request_samples(cloud, endpoint)
228	}
229	}
230	}
231	Commands::Init => {
232	panic!("This should not happen as it is handled in parse_and_generate");	×
233	}
234	Commands::Live(live_flags) => {	×
235	render_logo();	×
236	dozer.runtime.block_on(live::start_live_server(	×
237	&dozer.runtime,	×
238	shutdown_receiver,	×
239	live_flags,	×
240	))?;	×
241	Ok(())	×
242	}
243	})
244	.map_err(\|e\| {	×
245	let _span = error_span!("OrchestrationError", error = %e);	×
246
247	e	×
248	})	×
249	}	×
250
251	// Some commands dont need to initialize the orchestrator
252	// This function is used to run those commands
253	fn parse_and_generate() -> Result<Cli, OrchestrationError> {	×
254	dozer_tracing::init_telemetry_closure(	×
255	None,	×
256	&Default::default(),	×
257	\|\| -> Result<Cli, OrchestrationError> {	×
258	let cli = Cli::parse();	×
259		×
260	if let Commands::Init = cli.cmd {	×
261	Telemetry::new(None, &Default::default());	×
262	if let Err(e) = generate_config_repl() {	×
263	error!("{}", e);	×
264	Err(e)	×
265	} else {
266	// We need to exit here, otherwise the orchestrator will be initialized
267	process::exit(0);	×
268	}
269	} else {
270	Ok(cli)	×
271	}
272	},	×
273	)	×
274	}	×
275
276	fn init_orchestrator(cli: &Cli) -> Result<SimpleOrchestrator, CliError> {	×
277	dozer_tracing::init_telemetry_closure(	×
278	None,	×
279	&Default::default(),	×
280	\|\| -> Result<SimpleOrchestrator, CliError> {	×
281	let runtime = Arc::new(Runtime::new().map_err(CliError::FailedToCreateTokioRuntime)?);	×
282	let res = runtime.block_on(init_dozer(	×
283	runtime.clone(),	×
284	cli.config_paths.clone(),	×
285	cli.config_token.clone(),	×
286	cli.config_overrides.clone(),	×
287	cli.ignore_pipe,	×
288	LabelsAndProgress::new(Default::default(), cli.enable_progress),	×
289	));	×
290		×
291	match res {	×
292	Ok(dozer) => {	×
293	dozer.runtime.spawn(check_update());	×
294	Ok(dozer)	×
295	}
296	Err(e) => {	×
297	if let CliError::FailedToFindConfigurationFiles(_) = &e {	×
298	let description = "Dozer was not able to find configuration files. \n\n\	×
299	Please use \"dozer init\" to create project or \"dozer -c {path}\" with path to your configuration.\n\	×
300	Configuration documentation can be found in https://getdozer.io/docs/configuration";	×
301		×
302	let mut command = Cli::command();	×
303	command = command.about(format!("\n\n\n{} \n {}", LOGO, description));	×
304		×
305	println!("{}", command.render_help());	×
306	}	×
307
308	error!("{}", e);	×
309	Err(e)	×
310	}
311	}
312	},	×
313	)	×
314	}	×
315
316	fn display_error(e: &OrchestrationError) {
317	if let OrchestrationError::CloudError(CloudError::ApplicationNotFound) = &e {	×
318	let description = "Dozer cloud service was not able to find application. \n\n\	×
319	Please check your application id in `dozer-config.cloud.yaml` file.\n\	×
320	To change it, you can manually update file or use \"dozer cloud set-app {app_id}\".";	×
321		×
322	error!("{}", description);	×
323	} else {
324	error!("{}", e);	×
325	}
326	}	×
327
328	struct Telemetry();
329
330	impl Telemetry {
331	fn new(app_name: Option<&str>, config: &TelemetryConfig) -> Self {	×
332	dozer_tracing::init_telemetry(app_name, config);	×
333	Self()	×
334	}	×
335	}
336
337	impl Drop for Telemetry {
338	fn drop(&mut self) {	×
339	dozer_tracing::shutdown_telemetry();	×
340	}	×
341	}

getdozer / dozer / 6299724219

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous