5869050642

pending completion

Build # 5869050642

Build Type

Pull #1858

github

Committed by

supergi0

Commit Message

updated readme

Pull Request Pull Request #1858: feat: Implement graph for dozer-live ui

Run Details

419 of 419 new or added lines in 15 files covered. (100.0%)

46002 of 59761 relevant lines covered (76.98%)

52423.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/dozer-cli/src/main.rs

use clap::Parser;
#[cfg(feature = "cloud")]
use dozer_cli::cli::cloud::CloudCommands;
use dozer_cli::cli::generate_config_repl;
use dozer_cli::cli::types::{Cli, Commands, ConnectorCommand, RunCommands, SecurityCommands};
use dozer_cli::cli::{init_dozer, list_sources, LOGO};
use dozer_cli::errors::{CliError, CloudError, OrchestrationError};
use dozer_cli::simple::SimpleOrchestrator;
#[cfg(feature = "cloud")]
use dozer_cli::CloudOrchestrator;
use dozer_cli::{live, set_ctrl_handler, set_panic_hook, shutdown};
use dozer_types::models::telemetry::{TelemetryConfig, TelemetryMetricsConfig};
use dozer_types::tracing::{error, info};
use serde::Deserialize;
use tokio::time;

use clap::CommandFactory;
#[cfg(feature = "cloud")]
use dozer_cli::cloud_app_context::CloudAppContext;
use std::cmp::Ordering;

use dozer_types::log::{debug, warn};
use std::time::Duration;
use std::{env, process};

fn main() {
    set_panic_hook();

    if let Err(e) = run() {
        display_error(&e);
        process::exit(1);
    }
}

fn render_logo() {
    const VERSION: &str = env!("CARGO_PKG_VERSION");

    println!("{LOGO}");
    println!("\nDozer Version: {VERSION}\n");
}

#[derive(Deserialize, Debug)]
struct DozerPackage {
    #[serde(rename(deserialize = "latestVersion"))]
    pub latest_version: String,
    #[serde(rename(deserialize = "availableAssets"))]
    pub _available_assets: Vec<String>,
    pub link: String,
}

fn version_to_vector(version: &str) -> Vec<i32> {
    version.split('.').map(|s| s.parse().unwrap()).collect()
}

fn compare_versions(v1: Vec<i32>, v2: Vec<i32>) -> bool {
    for i in 0..v1.len() {
        match v1.get(i).cmp(&v2.get(i)) {
            Ordering::Greater => return true,
            Ordering::Less => return false,
            Ordering::Equal => continue,
        }
    }
    false
}

async fn check_update() {
    const VERSION: &str = env!("CARGO_PKG_VERSION");
    let dozer_env = std::env::var("DOZER_ENV").unwrap_or("local".to_string());
    let dozer_dev = std::env::var("DOZER_DEV").unwrap_or("ext".to_string());
    let query = vec![
        ("version", VERSION),
        ("build", std::env::consts::ARCH),
        ("os", std::env::consts::OS),
        ("env", &dozer_env),
        ("dev", &dozer_dev),
    ];

    let request_url = "https://metadata.dev.getdozer.io/";

    let client = reqwest::Client::new();

    let mut printed = false;

    loop {
        let response = client
            .get(&request_url.to_string())
            .query(&query)
            .send()
            .await;

        match response {
            Ok(r) => {
                if !printed {
                    let package: DozerPackage = r.json().await.unwrap();
                    let current = version_to_vector(VERSION);
                    let remote = version_to_vector(&package.latest_version);

                    if compare_versions(remote, current) {
                        info!("A new version of Dozer is available.");
                        info!(
                            "You can download v{}, from {}.",
                            package.latest_version, package.link
                        );
                        printed = true;
                    }
                }
            }
            Err(e) => {
                // We dont show error if error is connection error, because mostly it happens
                // when main thread is shutting down before request completes.
                if !e.is_connect() {
                    warn!("Unable to fetch the latest metadata");
                }

                debug!("Updates check error: {}", e);
            }
        }
        time::sleep(Duration::from_secs(2 * 60 * 60)).await;
    }
}

fn run() -> Result<(), OrchestrationError> {
    // Reloading trace layer seems impossible, so we are running Cli::parse in a closure
    // and then initializing it after reading the configuration. This is a hacky workaround, but it works.

    let cli = parse_and_generate()?;
    let mut dozer = init_orchestrator(&cli)?;
    let (shutdown_sender, shutdown_receiver) = shutdown::new(&dozer.runtime);
    set_ctrl_handler(shutdown_sender);

    // Now we have access to telemetry configuration. Telemetry must be initialized in tokio runtime.
    let app_name = dozer.config.app_name.clone();
    let app_id = dozer
        .config
        .cloud
        .as_ref()
        .map(|cloud| cloud.app_id.clone().unwrap_or(app_name));

    // We always enable telemetry when running live.
    let telemetry_config = if matches!(cli.cmd, Some(Commands::Live)) {
        Some(TelemetryConfig {
            trace: None,
            metrics: Some(TelemetryMetricsConfig::Prometheus(())),
        })
    } else {
        dozer.config.telemetry.clone()
    };
    let _telemetry = dozer
        .runtime
        .block_on(async { Telemetry::new(app_id.as_deref(), telemetry_config) });

    if let Some(cmd) = cli.cmd {
        // run individual servers
        match cmd {
            Commands::Run(run) => match run.command {
                RunCommands::Api => {
                    render_logo();

                    dozer.run_api(shutdown_receiver)
                }
                RunCommands::App => {
                    render_logo();

                    dozer.run_apps(shutdown_receiver, None)
                }
            },
            Commands::Security(security) => match security.command {
                SecurityCommands::GenerateToken => {
                    let token = dozer.generate_token()?;
                    info!("token: {:?} ", token);
                    Ok(())
                }
            },
            Commands::Build(build) => {
                let force = build.force.is_some();

                dozer.build(force)
            }
            Commands::Connectors(ConnectorCommand { filter }) => list_sources(
                cli.config_paths,
                cli.config_token,
                cli.config_overrides,
                cli.ignore_pipe,
                filter,
            ),
            Commands::Clean => dozer.clean(),
            #[cfg(feature = "cloud")]
            Commands::Cloud(cloud) => {
                render_logo();

                match cloud.command.clone() {
                    CloudCommands::Deploy(deploy) => dozer.deploy(cloud, deploy, cli.config_paths),
                    CloudCommands::Api(api) => dozer.api(cloud, api),
                    CloudCommands::Login {
                        organisation_slug,
                        profile_name,
                        client_id,
                        client_secret,
                    } => dozer.login(
                        cloud,
                        organisation_slug,
                        profile_name,
                        client_id,
                        client_secret,
                    ),
                    CloudCommands::Secrets(command) => {
                        dozer.execute_secrets_command(cloud, command)
                    }
                    CloudCommands::Delete => dozer.delete(cloud),
                    CloudCommands::Status => dozer.status(cloud),
                    CloudCommands::Monitor => dozer.monitor(cloud),
                    CloudCommands::Logs(logs) => dozer.trace_logs(cloud, logs),
                    CloudCommands::Version(version) => dozer.version(cloud, version),
                    CloudCommands::List(list) => dozer.list(cloud, list),
                    CloudCommands::SetApp { app_id } => {
                        CloudAppContext::save_app_id(app_id.clone())?;
                        info!("Using \"{app_id}\" app");
                        Ok(())
                    }
                }
            }
            Commands::Init => {
                panic!("This should not happen as it is handled in parse_and_generate");
            }
            Commands::Live => {
                render_logo();
                live::start_live_server(dozer.runtime.clone(), shutdown_receiver)?;
                Ok(())
            }
        }
    } else {
        render_logo();

        dozer.run_all(shutdown_receiver)
    }
}

// Some commands dont need to initialize the orchestrator
// This function is used to run those commands
fn parse_and_generate() -> Result<Cli, OrchestrationError> {
    dozer_tracing::init_telemetry_closure(None, None, || -> Result<Cli, OrchestrationError> {
        let cli = Cli::parse();

        if let Some(Commands::Init) = cli.cmd {
            Telemetry::new(None, None);
            if let Err(e) = generate_config_repl() {
                error!("{}", e);
                Err(e)
            } else {
                // We need to exit here, otherwise the orchestrator will be initialized
                process::exit(0);
            }
        } else {
            Ok(cli)
        }
    })
}

fn init_orchestrator(cli: &Cli) -> Result<SimpleOrchestrator, CliError> {
    dozer_tracing::init_telemetry_closure(None, None, || -> Result<SimpleOrchestrator, CliError> {
        let res = init_dozer(
            cli.config_paths.clone(),
            cli.config_token.clone(),
            cli.config_overrides.clone(),
            cli.ignore_pipe,
            cli.enable_progress.clone(),
        );

        match res {
            Ok(dozer) => {
                dozer.runtime.spawn(check_update());
                Ok(dozer)
            }
            Err(e) => {
                if let CliError::FailedToFindConfigurationFiles(_) = &e {
                    let description = "Dozer was not able to find configuration files. \n\n\
                    Please use \"dozer init\" to create project or \"dozer -c {path}\" with path to your configuration.\n\
                    Configuration documentation can be found in https://getdozer.io/docs/configuration";

                    let mut command = Cli::command();
                    command = command.about(format!("\n\n\n{} \n {}", LOGO, description));

                    println!("{}", command.render_help());
                }

                error!("{}", e);
                Err(e)
            }
        }
    })
}

fn display_error(e: &OrchestrationError) {
    if let OrchestrationError::CloudError(CloudError::ApplicationNotFound) = &e {
        let description = "Dozer cloud service was not able to find application. \n\n\
        Please check your application id in `dozer-config.cloud.yaml` file.\n\
        To change it, you can manually update file or use \"dozer cloud set-app {app_id}\".";

        error!("{}", description);
    } else {
        error!("{}", e);
    }
}

struct Telemetry();

impl Telemetry {
    fn new(app_name: Option<&str>, config: Option<TelemetryConfig>) -> Self {
        dozer_tracing::init_telemetry(app_name, config);
        Self()
    }
}

impl Drop for Telemetry {
    fn drop(&mut self) {
        dozer_tracing::shutdown_telemetry();
    }
}

1	use clap::Parser;
2	#[cfg(feature = "cloud")]
3	use dozer_cli::cli::cloud::CloudCommands;
4	use dozer_cli::cli::generate_config_repl;
5	use dozer_cli::cli::types::{Cli, Commands, ConnectorCommand, RunCommands, SecurityCommands};
6	use dozer_cli::cli::{init_dozer, list_sources, LOGO};
7	use dozer_cli::errors::{CliError, CloudError, OrchestrationError};
8	use dozer_cli::simple::SimpleOrchestrator;
9	#[cfg(feature = "cloud")]
10	use dozer_cli::CloudOrchestrator;
11	use dozer_cli::{live, set_ctrl_handler, set_panic_hook, shutdown};
12	use dozer_types::models::telemetry::{TelemetryConfig, TelemetryMetricsConfig};
13	use dozer_types::tracing::{error, info};
14	use serde::Deserialize;
15	use tokio::time;
16
17	use clap::CommandFactory;
18	#[cfg(feature = "cloud")]
19	use dozer_cli::cloud_app_context::CloudAppContext;
20	use std::cmp::Ordering;
21
22	use dozer_types::log::{debug, warn};
23	use std::time::Duration;
24	use std::{env, process};
25
26	fn main() {	×
27	set_panic_hook();	×
28
29	if let Err(e) = run() {	×
30	display_error(&e);	×
31	process::exit(1);	×
32	}	×
33	}	×
34
35	fn render_logo() {	×
36	const VERSION: &str = env!("CARGO_PKG_VERSION");	×
37		×
38	println!("{LOGO}");	×
39	println!("\nDozer Version: {VERSION}\n");	×
40	}	×
41
42	#[derive(Deserialize, Debug)]	×
43	struct DozerPackage {
44	#[serde(rename(deserialize = "latestVersion"))]
45	pub latest_version: String,
46	#[serde(rename(deserialize = "availableAssets"))]
47	pub _available_assets: Vec<String>,
48	pub link: String,
49	}
50
51	fn version_to_vector(version: &str) -> Vec<i32> {	×
52	version.split('.').map(\|s\| s.parse().unwrap()).collect()	×
53	}	×
54
55	fn compare_versions(v1: Vec<i32>, v2: Vec<i32>) -> bool {	×
56	for i in 0..v1.len() {	×
57	match v1.get(i).cmp(&v2.get(i)) {	×
58	Ordering::Greater => return true,	×
59	Ordering::Less => return false,	×
60	Ordering::Equal => continue,	×
61	}
62	}
63	false	×
64	}	×
65
66	async fn check_update() {	×
67	const VERSION: &str = env!("CARGO_PKG_VERSION");	×
68	let dozer_env = std::env::var("DOZER_ENV").unwrap_or("local".to_string());	×
69	let dozer_dev = std::env::var("DOZER_DEV").unwrap_or("ext".to_string());	×
70	let query = vec![	×
71	("version", VERSION),	×
72	("build", std::env::consts::ARCH),	×
73	("os", std::env::consts::OS),	×
74	("env", &dozer_env),	×
75	("dev", &dozer_dev),	×
76	];	×
77		×
78	let request_url = "https://metadata.dev.getdozer.io/";	×
79		×
80	let client = reqwest::Client::new();	×
81		×
82	let mut printed = false;	×
83
84	loop {
85	let response = client	×
86	.get(&request_url.to_string())	×
87	.query(&query)	×
88	.send()	×
89	.await;	×
90
91	match response {	×
92	Ok(r) => {	×
93	if !printed {	×
94	let package: DozerPackage = r.json().await.unwrap();	×
95	let current = version_to_vector(VERSION);	×
96	let remote = version_to_vector(&package.latest_version);	×
97		×
98	if compare_versions(remote, current) {	×
99	info!("A new version of Dozer is available.");	×
100	info!(	×
101	"You can download v{}, from {}.",	×
102	package.latest_version, package.link	×
103	);	×
104	printed = true;	×
105	}	×
106	}	×
107	}
108	Err(e) => {	×
109	// We dont show error if error is connection error, because mostly it happens	×
110	// when main thread is shutting down before request completes.	×
111	if !e.is_connect() {	×
112	warn!("Unable to fetch the latest metadata");	×
113	}	×
114
115	debug!("Updates check error: {}", e);	×
116	}
117	}
118	time::sleep(Duration::from_secs(2 * 60 * 60)).await;	×
119	}
120	}
121
122	fn run() -> Result<(), OrchestrationError> {	×
123	// Reloading trace layer seems impossible, so we are running Cli::parse in a closure
124	// and then initializing it after reading the configuration. This is a hacky workaround, but it works.
125
126	let cli = parse_and_generate()?;	×
127	let mut dozer = init_orchestrator(&cli)?;	×
128	let (shutdown_sender, shutdown_receiver) = shutdown::new(&dozer.runtime);	×
129	set_ctrl_handler(shutdown_sender);	×
130		×
131	// Now we have access to telemetry configuration. Telemetry must be initialized in tokio runtime.	×
132	let app_name = dozer.config.app_name.clone();	×
133	let app_id = dozer	×
134	.config	×
135	.cloud	×
136	.as_ref()	×
137	.map(\|cloud\| cloud.app_id.clone().unwrap_or(app_name));	×
138		×
139	// We always enable telemetry when running live.	×
140	let telemetry_config = if matches!(cli.cmd, Some(Commands::Live)) {	×
141	Some(TelemetryConfig {	×
142	trace: None,	×
143	metrics: Some(TelemetryMetricsConfig::Prometheus(())),	×
144	})	×
145	} else {	×
146	dozer.config.telemetry.clone()	×
147	};	×
148	let _telemetry = dozer	×
149	.runtime	×
150	.block_on(async { Telemetry::new(app_id.as_deref(), telemetry_config) });	×
151
152	if let Some(cmd) = cli.cmd {	×
153	// run individual servers	×
154	match cmd {	×
155	Commands::Run(run) => match run.command {	×
156	RunCommands::Api => {
157	render_logo();	×
158		×
159	dozer.run_api(shutdown_receiver)	×
160	}	×
161	RunCommands::App => {	×
162	render_logo();	×
163		×
164	dozer.run_apps(shutdown_receiver, None)	×
165	}	×
166	},	×
167	Commands::Security(security) => match security.command {	×
168	SecurityCommands::GenerateToken => {
169	let token = dozer.generate_token()?;	×
170	info!("token: {:?} ", token);	×
171	Ok(())	×
172	}	×
173	},	×
174	Commands::Build(build) => {	×
175	let force = build.force.is_some();	×
176		×
177	dozer.build(force)	×
178	}
179	Commands::Connectors(ConnectorCommand { filter }) => list_sources(	×
180	cli.config_paths,	×
181	cli.config_token,	×
182	cli.config_overrides,	×
183	cli.ignore_pipe,	×
184	filter,	×
185	),	×
186	Commands::Clean => dozer.clean(),	×
187	#[cfg(feature = "cloud")]
188	Commands::Cloud(cloud) => {
189	render_logo();
190
191	match cloud.command.clone() {
192	CloudCommands::Deploy(deploy) => dozer.deploy(cloud, deploy, cli.config_paths),
193	CloudCommands::Api(api) => dozer.api(cloud, api),
194	CloudCommands::Login {
195	organisation_slug,
196	profile_name,
197	client_id,
198	client_secret,
199	} => dozer.login(
200	cloud,
201	organisation_slug,
202	profile_name,
203	client_id,
204	client_secret,
205	),
206	CloudCommands::Secrets(command) => {
207	dozer.execute_secrets_command(cloud, command)
208	}
209	CloudCommands::Delete => dozer.delete(cloud),
210	CloudCommands::Status => dozer.status(cloud),
211	CloudCommands::Monitor => dozer.monitor(cloud),
212	CloudCommands::Logs(logs) => dozer.trace_logs(cloud, logs),
213	CloudCommands::Version(version) => dozer.version(cloud, version),	×
214	CloudCommands::List(list) => dozer.list(cloud, list),
215	CloudCommands::SetApp { app_id } => {
216	CloudAppContext::save_app_id(app_id.clone())?;	×
217	info!("Using \"{app_id}\" app");	×
218	Ok(())	×
219	}
220	}
221	}
222	Commands::Init => {	×
223	panic!("This should not happen as it is handled in parse_and_generate");	×
224	}	×
225	Commands::Live => {
226	render_logo();	×
227	live::start_live_server(dozer.runtime.clone(), shutdown_receiver)?;	×
228	Ok(())	×
229	}
230	}	×
231	} else {	×
232	render_logo();	×
233		×
234	dozer.run_all(shutdown_receiver)	×
235	}	×
236	}	×
237		×
238	// Some commands dont need to initialize the orchestrator	×
239	// This function is used to run those commands
240	fn parse_and_generate() -> Result<Cli, OrchestrationError> {	×
241	dozer_tracing::init_telemetry_closure(None, None, \|\| -> Result<Cli, OrchestrationError> {	×
242	let cli = Cli::parse();	×
243
244	if let Some(Commands::Init) = cli.cmd {	×
245	Telemetry::new(None, None);	×
246	if let Err(e) = generate_config_repl() {	×
247	error!("{}", e);	×
248	Err(e)	×
249	} else {	×
250	// We need to exit here, otherwise the orchestrator will be initialized	×
251	process::exit(0);	×
252	}	×
253	} else {	×
254	Ok(cli)	×
255	}	×
256	})	×
257	}	×
258		×
259	fn init_orchestrator(cli: &Cli) -> Result<SimpleOrchestrator, CliError> {	×
260	dozer_tracing::init_telemetry_closure(None, None, \|\| -> Result<SimpleOrchestrator, CliError> {	×
261	let res = init_dozer(	×
262	cli.config_paths.clone(),	×
263	cli.config_token.clone(),	×
264	cli.config_overrides.clone(),	×
265	cli.ignore_pipe,	×
266	cli.enable_progress.clone(),	×
267	);	×
268		×
269	match res {	×
270	Ok(dozer) => {	×
271	dozer.runtime.spawn(check_update());	×
272	Ok(dozer)	×
273	}	×
274	Err(e) => {	×
275	if let CliError::FailedToFindConfigurationFiles(_) = &e {	×
276	let description = "Dozer was not able to find configuration files. \n\n\	×
277	Please use \"dozer init\" to create project or \"dozer -c {path}\" with path to your configuration.\n\	×
278	Configuration documentation can be found in https://getdozer.io/docs/configuration";	×
279		×
280	let mut command = Cli::command();	×
281	command = command.about(format!("\n\n\n{} \n {}", LOGO, description));	×
282		×
283	println!("{}", command.render_help());	×
284	}	×
285		×
286	error!("{}", e);	×
287	Err(e)	×
288	}	×
289	}
290	})	×
291	}	×
292		×
293	fn display_error(e: &OrchestrationError) {
294	if let OrchestrationError::CloudError(CloudError::ApplicationNotFound) = &e {	×
295	let description = "Dozer cloud service was not able to find application. \n\n\	×
296	Please check your application id in `dozer-config.cloud.yaml` file.\n\	×
297	To change it, you can manually update file or use \"dozer cloud set-app {app_id}\".";	×
298		×
299	error!("{}", description);	×
300	} else {	×
301	error!("{}", e);	×
302	}
303	}	×
304		×
305	struct Telemetry();	×
306		×
307	impl Telemetry {
308	fn new(app_name: Option<&str>, config: Option<TelemetryConfig>) -> Self {	×
309	dozer_tracing::init_telemetry(app_name, config);	×
310	Self()	×
311	}	×
312	}
313
314	impl Drop for Telemetry {
315	fn drop(&mut self) {	×
316	dozer_tracing::shutdown_telemetry();	×
317	}	×
318	}

getdozer / dozer / 5869050642

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous