• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

epazote / epazote / 13476719813

22 Feb 2025 06:11PM UTC coverage: 82.699% (+0.9%) from 81.814%
13476719813

push

github

nbari
test for metrics

118 of 123 new or added lines in 2 files covered. (95.93%)

1 existing line in 1 file now uncovered.

1716 of 2075 relevant lines covered (82.7%)

6.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.39
/src/cli/actions/run.rs
1
use crate::cli::{
2
    actions::{
3
        client::build_client,
4
        execute_fallback_command, execute_fallback_http,
5
        metrics::{metrics_server, ServiceMetrics},
6
        request::{build_http_request, handle_http_response},
7
        should_continue_fallback,
8
        ssl::check_ssl_certificate,
9
        Action,
10
    },
11
    config::{Config, ServiceDetails},
12
};
13
use anyhow::{anyhow, Result};
14
use reqwest::Client;
15
use rustls::crypto::CryptoProvider;
16
use std::{collections::HashMap, sync::Arc, time::Duration};
17
use tokio::{
18
    sync::Mutex,
19
    time::{interval, Instant},
20
};
21
use tracing::{debug, error, info, instrument};
22

23
enum ServiceAction {
24
    Url(Client),
25
    Command(String),
26
}
27

28
/// Handle the create action
29
#[instrument(skip(action))]
30
pub async fn handle(action: Action) -> Result<()> {
31
    // rustls requires a cryptographic provider
32
    CryptoProvider::install_default(rustls::crypto::ring::default_provider())
33
        .map_err(|e| anyhow!("Failed to install default crypto provider: {:?}", e))?;
×
34

35
    let Action::Run { config, port } = action;
36

37
    let config_path = config;
38

39
    let config = Config::new(config_path)?;
40

41
    // Create service metrics
42
    let service_metrics = Arc::new(ServiceMetrics::new()?);
43

44
    let mut service_handles = Vec::new();
45

46
    let service_counters: Arc<Mutex<HashMap<String, usize>>> = Arc::new(Mutex::new(HashMap::new()));
47

48
    for (service_name, service) in &config.services {
49
        let service_name = service_name.clone();
50
        let service_details = service.clone();
51
        let counters = service_counters.clone();
52

53
        let action = if let Some(ref command) = service_details.test {
54
            ServiceAction::Command(command.clone())
55
        } else {
56
            let (builder, _client_config) = build_client(&service_details)?;
57
            let client = builder.build()?;
58

59
            ServiceAction::Url(client)
60
        };
61

62
        // Clone the metrics for this task
63
        let metrics = service_metrics.clone();
64

65
        // Spawn a task for each service
66
        let handle = tokio::spawn(async move {
×
67
            let every = service_details.every;
×
68
            run_service(
×
69
                service_name,
×
70
                service_details,
×
71
                action,
×
72
                metrics,
×
73
                every,
×
74
                counters,
×
75
            )
×
76
            .await;
×
77
        });
×
78

79
        service_handles.push(handle);
80
    }
81

82
    // Spawn metrics server
83
    let metrics_server_handle = tokio::spawn(async move {
×
84
        if let Err(e) = metrics_server(service_metrics, port).await {
×
85
            error!("Metrics server error: {}", e);
×
86
        }
×
87
    });
×
88

89
    info!("Epazote 🌿 is running");
90

91
    // Wait for all tasks to complete
92
    tokio::select! {
93
        _ = futures::future::join_all(service_handles) => {
94
            error!("All service monitoring tasks completed unexpectedly");
95
        },
96
        _ = metrics_server_handle => {
97
            error!("Metrics server stopped unexpectedly");
98
        }
99
    }
100

101
    Ok(())
102
}
103

104
/// Runs the task for a single service
105
async fn run_service(
1✔
106
    service_name: String,
1✔
107
    service_details: ServiceDetails,
1✔
108
    action: ServiceAction,
1✔
109
    metrics: Arc<ServiceMetrics>,
1✔
110
    interval_duration: Duration,
1✔
111
    counters: Arc<Mutex<HashMap<String, usize>>>,
1✔
112
) {
1✔
113
    let mut interval_timer = interval(interval_duration);
1✔
114

115
    loop {
116
        interval_timer.tick().await; // Wait for the next interval
6✔
117

118
        debug!("Running scan for service: {}", service_name);
5✔
119

120
        // Perform the service scan
121
        match scan_service(
5✔
122
            &service_name,
5✔
123
            &service_details,
5✔
124
            &action,
5✔
125
            &metrics,
5✔
126
            counters.clone(),
5✔
127
        )
5✔
128
        .await
5✔
129
        {
130
            Ok(_) => (),
5✔
131
            Err(e) => {
×
132
                // Increment failure counter
×
133
                metrics
×
134
                    .epazote_failures_total
×
135
                    .with_label_values(&[&service_name])
×
136
                    .inc();
×
137

×
NEW
138
                metrics
×
NEW
139
                    .epazote_status
×
NEW
140
                    .with_label_values(&[&service_name])
×
NEW
141
                    .set(0);
×
NEW
142

×
UNCOV
143
                error!("Error scanning service '{}': {}", &service_name, e);
×
144
            }
145
        }
146
    }
147
}
148

149
/// scan_service performs the actual scan of the service
150
async fn scan_service(
111✔
151
    service_name: &str,
111✔
152
    service_details: &ServiceDetails,
111✔
153
    action: &ServiceAction,
111✔
154
    metrics: &ServiceMetrics,
111✔
155
    counters: Arc<Mutex<HashMap<String, usize>>>,
111✔
156
) -> Result<()> {
111✔
157
    let start_time = Instant::now();
111✔
158

111✔
159
    match action {
111✔
160
        ServiceAction::Url(client) => {
5✔
161
            let request_builder = build_http_request(client, service_details)?;
5✔
162

163
            let request = request_builder.build()?;
5✔
164

165
            let url = request.url().to_string();
5✔
166

5✔
167
            if url.starts_with("https://") {
5✔
168
                check_ssl_certificate(&url, service_name, metrics).await?;
×
169
            }
5✔
170

171
            debug!("HTTP request: {:?}", request);
5✔
172

173
            // Make the request
174
            let response = client.execute(request).await?;
5✔
175

176
            // Record response time
177
            let response_time = start_time.elapsed().as_secs_f64();
5✔
178
            metrics
5✔
179
                .epazote_response_time
5✔
180
                .with_label_values(&[service_name])
5✔
181
                .observe(response_time);
5✔
182

5✔
183
            // Handle the response
5✔
184
            handle_http_response(service_name, service_details, response, metrics, counters)
5✔
185
                .await?;
5✔
186
        }
187

188
        ServiceAction::Command(command) => {
106✔
189
            debug!("Executing command: {}", command);
106✔
190

191
            let exit_status = execute_fallback_command(command).await.unwrap_or(1);
106✔
192

106✔
193
            if exit_status != service_details.expect.status as i32 {
106✔
194
                if let Some(action) = &service_details.expect.if_not {
105✔
195
                    if should_continue_fallback(service_name, &counters, action).await {
105✔
196
                        if let Some(cmd) = &action.cmd {
104✔
197
                            let exit_code = execute_fallback_command(cmd).await?;
104✔
198
                            debug!("Fallback action executed with exit code: {}", exit_code);
104✔
199
                        }
×
200

201
                        if let Some(http) = &action.http {
104✔
202
                            let status = execute_fallback_http(http).await?;
×
203
                            info!(
×
204
                                "Executed fallback HTTP request for {} with status code {}",
×
205
                                service_name, status
206
                            );
207
                        }
104✔
208
                    }
1✔
209
                }
×
210
            }
1✔
211
        }
212
    }
213

214
    Ok(())
111✔
215
}
111✔
216

217
#[cfg(test)]
218
mod tests {
219
    use super::*;
220
    use crate::cli::config::{Action, Expect, HttpMethod};
221
    use mockito::Server;
222
    use reqwest::StatusCode;
223
    use std::sync::Arc;
224
    use tokio::process::Command;
225
    use tokio::runtime::Runtime;
226
    use tokio::time::Duration;
227

228
    /// Helper Function: Create Mock ServiceDetails
229
    fn mock_service_details(
5✔
230
        test_cmd: Option<&str>,
5✔
231
        expect_status: u16,
5✔
232
        if_not: Option<&str>,
5✔
233
    ) -> ServiceDetails {
5✔
234
        ServiceDetails {
5✔
235
            every: Duration::from_secs(1),
5✔
236
            expect: Expect {
5✔
237
                status: expect_status,
5✔
238
                header: None,
5✔
239
                body: None,
5✔
240
                if_not: if_not.map(|cmd| Action {
5✔
241
                    cmd: Some(cmd.to_string()),
4✔
242
                    ..Default::default()
4✔
243
                }),
5✔
244
            },
5✔
245
            follow_redirects: Some(true),
5✔
246
            headers: None,
5✔
247
            if_header: None,
5✔
248
            if_status: None,
5✔
249
            insecure: None,
5✔
250
            read_limit: None,
5✔
251
            test: test_cmd.map(|cmd| cmd.to_string()),
5✔
252
            timeout: Duration::from_secs(5),
5✔
253
            url: None,
5✔
254
            method: HttpMethod::Get,
5✔
255
            body: None,
5✔
256
        }
5✔
257
    }
5✔
258

259
    /// Helper Function: Create Mock Action
260
    fn mock_action(test_cmd: &str) -> ServiceAction {
5✔
261
        ServiceAction::Command(test_cmd.to_string())
5✔
262
    }
5✔
263

264
    /// Test: Verify Shell Command Exit Codes
265
    async fn run_command(cmd: &str) -> i32 {
2✔
266
        let shell = std::env::var("SHELL").unwrap_or_else(|_| "sh".to_string());
2✔
267
        let output = Command::new(shell)
2✔
268
            .arg("-c")
2✔
269
            .arg(cmd)
2✔
270
            .output()
2✔
271
            .await
2✔
272
            .expect("Failed to execute command");
2✔
273

2✔
274
        output.status.code().unwrap_or(1) // Default to 1 if no exit code
2✔
275
    }
2✔
276

277
    #[test]
278
    // this test is only for the test run_command function, not the actual code
279
    fn test_command_exit_status() {
1✔
280
        let rt = Runtime::new().unwrap();
1✔
281

1✔
282
        let exit_code_0 = rt.block_on(run_command("exit 0"));
1✔
283
        assert_eq!(exit_code_0, 0, "Command `exit 0` should return exit code 0");
1✔
284

285
        let exit_code_1 = rt.block_on(run_command("exit 1"));
1✔
286
        assert_eq!(exit_code_1, 1, "Command `exit 1` should return exit code 1");
1✔
287
    }
1✔
288

289
    /// Test: Successful HTTP Service with Expected Status
290
    #[tokio::test]
291
    async fn test_http_service_expect_status() {
1✔
292
        let mut server = Server::new_async().await;
1✔
293
        let _m = server
1✔
294
            .mock("GET", "/test")
1✔
295
            .with_status(200)
1✔
296
            .create_async()
1✔
297
            .await;
1✔
298

1✔
299
        let url = format!("{}/test", server.url());
1✔
300
        let client = Client::new();
1✔
301
        let response = client.get(&url).send().await.unwrap();
1✔
302
        let status = response.status();
1✔
303

1✔
304
        assert_eq!(status, StatusCode::OK, "Expected status 200 OK");
1✔
305
    }
1✔
306

307
    /// Test: Scan Service Command - Success
308
    #[tokio::test]
309
    async fn test_scan_service_command_success() {
1✔
310
        let service_details = mock_service_details(Some("exit 0"), 0, None);
1✔
311
        let action = mock_action("exit 0");
1✔
312
        let metrics = Arc::new(ServiceMetrics::new().unwrap());
1✔
313
        let counters: Arc<Mutex<HashMap<String, usize>>> = Arc::new(Mutex::new(HashMap::new()));
1✔
314

1✔
315
        let result = scan_service(
1✔
316
            "test-service",
1✔
317
            &service_details,
1✔
318
            &action,
1✔
319
            &metrics,
1✔
320
            counters,
1✔
321
        )
1✔
322
        .await;
1✔
323

1✔
324
        assert!(
1✔
325
            result.is_ok(),
1✔
326
            "Scan service should succeed for a successful command"
1✔
327
        );
1✔
328
    }
1✔
329

330
    /// Test: Scan Service Command - Failure with Fallback
331
    #[tokio::test]
332
    async fn test_scan_service_command_failure_with_fallback() {
1✔
333
        let service_details = mock_service_details(Some("exit 1"), 0, Some("echo 'Fallback'"));
1✔
334
        let action = mock_action("exit 1");
1✔
335
        let metrics = Arc::new(ServiceMetrics::new().unwrap());
1✔
336
        let counters: Arc<Mutex<HashMap<String, usize>>> = Arc::new(Mutex::new(HashMap::new()));
1✔
337

1✔
338
        let result = scan_service(
1✔
339
            "test-service",
1✔
340
            &service_details,
1✔
341
            &action,
1✔
342
            &metrics,
1✔
343
            Arc::clone(&counters),
1✔
344
        )
1✔
345
        .await;
1✔
346

1✔
347
        assert!(
1✔
348
            result.is_ok(),
1✔
349
            "Scan service should execute fallback for failed command"
1✔
350
        );
1✔
351

1✔
352
        let counters_locked = counters.lock().await;
1✔
353
        let count = counters_locked.get("test-service").copied().unwrap_or(0);
1✔
354

1✔
355
        assert_eq!(count, 1, "Counter should have been incremented");
1✔
356
    }
1✔
357

358
    /// Test: Scan Service Command - Stops after 2 failures
359
    #[tokio::test]
360
    async fn test_scan_service_command_failure_with_stop_after_2_attempts() {
1✔
361
        let mut service_details = mock_service_details(Some("exit 1"), 0, Some("echo 'Fallback'"));
1✔
362
        let action = mock_action("exit 1");
1✔
363

1✔
364
        // Set stop condition to 2
1✔
365
        service_details.expect.if_not.as_mut().unwrap().stop = Some(2);
1✔
366

1✔
367
        let metrics = Arc::new(ServiceMetrics::new().unwrap());
1✔
368
        let counters: Arc<Mutex<HashMap<String, usize>>> = Arc::new(Mutex::new(HashMap::new()));
1✔
369

1✔
370
        // First attempt
1✔
371
        let result1 = scan_service(
1✔
372
            "test-service",
1✔
373
            &service_details,
1✔
374
            &action,
1✔
375
            &metrics,
1✔
376
            Arc::clone(&counters),
1✔
377
        )
1✔
378
        .await;
1✔
379

1✔
380
        assert!(result1.is_ok(), "First attempt should allow fallback");
1✔
381

1✔
382
        // Check counter after first attempt
1✔
383
        let count1 = {
1✔
384
            let counters_locked = counters.lock().await;
1✔
385
            *counters_locked.get("test-service").unwrap_or(&0)
1✔
386
        };
1✔
387
        assert_eq!(count1, 1, "Counter should be 1 after first attempt");
1✔
388

1✔
389
        // Second attempt
1✔
390
        let result2 = scan_service(
1✔
391
            "test-service",
1✔
392
            &service_details,
1✔
393
            &action,
1✔
394
            &metrics,
1✔
395
            Arc::clone(&counters),
1✔
396
        )
1✔
397
        .await;
1✔
398

1✔
399
        assert!(result2.is_ok(), "Second attempt should allow fallback");
1✔
400

1✔
401
        // Check counter after second attempt
1✔
402
        let count2 = {
1✔
403
            let counters_locked = counters.lock().await;
1✔
404
            *counters_locked.get("test-service").unwrap_or(&0)
1✔
405
        };
1✔
406
        assert_eq!(count2, 2, "Counter should be 2 after second attempt");
1✔
407

1✔
408
        // Third attempt (should NOT execute fallback)
1✔
409
        let result3 = scan_service(
1✔
410
            "test-service",
1✔
411
            &service_details,
1✔
412
            &action,
1✔
413
            &metrics,
1✔
414
            Arc::clone(&counters),
1✔
415
        )
1✔
416
        .await;
1✔
417

1✔
418
        assert!(
1✔
419
            result3.is_ok(),
1✔
420
            "Third attempt should skip fallback due to stop limit"
1✔
421
        );
1✔
422

1✔
423
        // Check counter after third attempt (should remain at 2)
1✔
424
        let count3 = {
1✔
425
            let counters_locked = counters.lock().await;
1✔
426
            *counters_locked.get("test-service").unwrap_or(&0)
1✔
427
        };
1✔
428
        assert_eq!(count3, 2, "Counter should remain at 2 after third attempt");
1✔
429
    }
1✔
430

431
    /// Test: Scan Service Command - Ensure counter can reach 1000 when no stop condition is set
432
    #[tokio::test]
433
    async fn test_scan_service_command_runs_1000_times_without_stop() {
1✔
434
        let mut service_details = mock_service_details(Some("exit 1"), 0, Some("echo 'Fallback'"));
1✔
435
        let action = mock_action("exit 1");
1✔
436

1✔
437
        // Ensure no stop limit is set
1✔
438
        service_details.expect.if_not.as_mut().unwrap().stop = None;
1✔
439

1✔
440
        let metrics = Arc::new(ServiceMetrics::new().unwrap());
1✔
441
        let counters: Arc<Mutex<HashMap<String, usize>>> = Arc::new(Mutex::new(HashMap::new()));
1✔
442

1✔
443
        // Run scan_service 100 times
1✔
444
        for _ in 0..100 {
101✔
445
            let _ = scan_service(
100✔
446
                "test-service",
100✔
447
                &service_details,
100✔
448
                &action,
100✔
449
                &metrics,
100✔
450
                Arc::clone(&counters),
100✔
451
            )
100✔
452
            .await;
100✔
453
        }
1✔
454

1✔
455
        // Check that counter reached 1000
1✔
456
        let final_count = {
1✔
457
            let counters_locked = counters.lock().await;
1✔
458
            *counters_locked.get("test-service").unwrap_or(&0)
1✔
459
        };
1✔
460

1✔
461
        assert_eq!(
1✔
462
            final_count, 100,
1✔
463
            "Counter should reach 100 when no stop is set"
1✔
464
        );
1✔
465
    }
1✔
466

467
    /// Test: Scan Service Command - Failure with Fallback and Stop
468
    #[tokio::test]
469
    async fn test_scan_service_command_failure_with_fallback_and_stop() {
1✔
470
        let service_details = mock_service_details(Some("exit 1"), 0, Some("echo 'Fallback'"));
1✔
471
        let action = mock_action("exit 1");
1✔
472
        let metrics = Arc::new(ServiceMetrics::new().unwrap());
1✔
473
        let counters: Arc<Mutex<HashMap<String, usize>>> = Arc::new(Mutex::new(HashMap::new()));
1✔
474

1✔
475
        let result = scan_service(
1✔
476
            "test-service",
1✔
477
            &service_details,
1✔
478
            &action,
1✔
479
            &metrics,
1✔
480
            counters,
1✔
481
        )
1✔
482
        .await;
1✔
483
        assert!(
1✔
484
            result.is_ok(),
1✔
485
            "Scan service should execute fallback for failed command"
1✔
486
        );
1✔
487
    }
1✔
488

489
    /// Test: Run Service - URL Success
490
    #[tokio::test]
491
    async fn test_run_service_http_success() {
1✔
492
        let mut server = Server::new_async().await;
1✔
493
        let _m = server
1✔
494
            .mock("GET", "/health")
1✔
495
            .with_status(200)
1✔
496
            .create_async()
1✔
497
            .await;
1✔
498

1✔
499
        let service_details = ServiceDetails {
1✔
500
            every: Duration::from_secs(1),
1✔
501
            expect: Expect {
1✔
502
                status: 200,
1✔
503
                header: None,
1✔
504
                body: None,
1✔
505
                if_not: None,
1✔
506
            },
1✔
507
            follow_redirects: Some(true),
1✔
508
            headers: None,
1✔
509
            if_header: None,
1✔
510
            if_status: None,
1✔
511
            insecure: None,
1✔
512
            read_limit: None,
1✔
513
            test: None,
1✔
514
            timeout: Duration::from_secs(5),
1✔
515
            url: Some(format!("{}/health", server.url())),
1✔
516
            method: HttpMethod::Get,
1✔
517
            body: None,
1✔
518
        };
1✔
519

1✔
520
        let action = ServiceAction::Url(Client::new());
1✔
521
        let metrics = Arc::new(ServiceMetrics::new().unwrap());
1✔
522
        let counters: Arc<Mutex<HashMap<String, usize>>> = Arc::new(Mutex::new(HashMap::new()));
1✔
523

1✔
524
        tokio::spawn(async move {
1✔
525
            run_service(
1✔
526
                "http-service".to_string(),
1✔
527
                service_details,
1✔
528
                action,
1✔
529
                metrics,
1✔
530
                Duration::from_millis(100),
1✔
531
                counters,
1✔
532
            )
1✔
533
            .await;
1✔
534
        });
1✔
535

1✔
536
        tokio::time::sleep(Duration::from_millis(500)).await;
1✔
537
        assert!(
1✔
538
            true,
1✔
539
            "Run service should execute multiple times in test interval"
1✔
540
        );
1✔
541
    }
1✔
542
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc