27228748356

Committed 09 Jun 2026 06:56PM UTC coverage: 85.0%. First build

Build # 27228748356

Build Type

Pull #6

github

Committed by

web-flow

Commit Message

Merge 52a443ec8 into e9de08346

Pull Request Pull Request #6: New flow

Coverage Stats

542 of 638 new or added lines in 5 files covered. (84.95%)

544 of 640 relevant lines covered (85.0%)

2.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.87

/src/types.rs

use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use url::Url;

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub enum ResourceType {
    HTML,
    CSS,
    JavaScript,
    Image,
    Video,
    PDF,
    Font,
    Other,
}

impl std::fmt::Display for ResourceType {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.download_filter_key())
    }
}

impl ResourceType {
    /// CLI filter name used by `--download-only` (e.g. `js`, not `javascript`).
    pub fn download_filter_key(&self) -> &'static str {
        match self {
            ResourceType::HTML => "html",
            ResourceType::CSS => "css",
            ResourceType::JavaScript => "js",
            ResourceType::Image => "images",
            ResourceType::Video => "video",
            ResourceType::PDF => "pdf",
            ResourceType::Font => "fonts",
            ResourceType::Other => "other",
        }
    }
}

impl From<&str> for ResourceType {
    fn from(mime_type: &str) -> Self {
        match mime_type {
            "text/html" => ResourceType::HTML,
            "text/css" => ResourceType::CSS,
            "application/javascript" | "text/javascript" => ResourceType::JavaScript,
            mime if mime.starts_with("image/") => ResourceType::Image,
            mime if mime.starts_with("video/") => ResourceType::Video,
            "application/pdf" => ResourceType::PDF,
            mime if mime.starts_with("font/") || mime == "application/font-woff" => {
                ResourceType::Font
            }
            _ => ResourceType::Other,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Resource {
    pub url: Url,
    pub local_path: String,
    pub resource_type: ResourceType,
    pub mime_type: String,
    pub size: Option<u64>,
    pub downloaded: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Page {
    pub url: Url,
    pub local_path: String,
    pub depth: usize,
    pub resources: Vec<Resource>,
    pub processed: bool,
}

#[derive(Debug, Clone)]
pub struct DownloadStats {
    pub total_pages: usize,
    pub processed_pages: usize,
    pub total_resources: usize,
    pub downloaded_resources: usize,
    pub total_size: u64,
    pub start_time: std::time::Instant,
}

impl Default for DownloadStats {
    fn default() -> Self {
        Self::new()
    }
}

impl DownloadStats {
    pub fn new() -> Self {
        Self {
            total_pages: 0,
            processed_pages: 0,
            total_resources: 0,
            downloaded_resources: 0,
            total_size: 0,
            start_time: std::time::Instant::now(),
        }
    }

    pub fn progress_percentage(&self) -> f64 {
        if self.total_pages == 0 {
            0.0
        } else {
            (self.processed_pages as f64 / self.total_pages as f64) * 100.0
        }
    }

    pub fn elapsed_time(&self) -> std::time::Duration {
        self.start_time.elapsed()
    }
}

#[derive(Debug, Clone)]
pub struct WorkQueue {
    pub pages: Vec<Url>,
    pub resources: Vec<Url>,
    pub visited_urls: HashMap<String, bool>,
}

impl Default for WorkQueue {
    fn default() -> Self {
        Self::new()
    }
}

impl WorkQueue {
    pub fn new() -> Self {
        Self {
            pages: Vec::new(),
            resources: Vec::new(),
            visited_urls: HashMap::new(),
        }
    }

    pub fn add_page(&mut self, url: Url) {
        let normalized = self.normalize_url(&url);
        if !self.visited_urls.contains_key(&normalized) {
            self.pages.push(url);
            self.visited_urls.insert(normalized, true);
        }
    }

    pub fn add_resource(&mut self, url: Url) {
        // Resources can be added multiple times (they might be referenced from different pages)
        // Only check if we already have this exact resource to avoid duplicates
        let normalized = self.normalize_url(&url);
        if !self
            .resources
            .iter()
            .any(|r| self.normalize_url(r) == normalized)
        {
            self.resources.push(url);
        }
    }

    pub fn normalize_url(&self, url: &Url) -> String {
        let mut normalized = url.clone();
        normalized.set_fragment(None);
        normalized.set_query(None);
        normalized.to_string()
    }

    pub fn get_next_page(&mut self) -> Option<Url> {
        self.pages.pop()
    }

    pub fn get_next_resource(&mut self) -> Option<Url> {
        self.resources.pop()
    }

    pub fn is_empty(&self) -> bool {
        self.pages.is_empty() && self.resources.is_empty()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn download_filter_key_matches_cli_values() {
        assert_eq!(ResourceType::JavaScript.download_filter_key(), "js");
        assert_eq!(ResourceType::Image.download_filter_key(), "images");
        assert_eq!(ResourceType::Font.download_filter_key(), "fonts");
    }

    #[test]
    fn display_uses_download_filter_key() {
        assert_eq!(ResourceType::JavaScript.to_string(), "js");
    }

    #[test]
    fn download_stats_progress_percentage() {
        let mut stats = DownloadStats::new();
        assert_eq!(stats.progress_percentage(), 0.0);

        stats.total_pages = 4;
        stats.processed_pages = 2;
        assert_eq!(stats.progress_percentage(), 50.0);
    }

    #[test]
    fn resource_type_from_mime() {
        assert!(matches!(ResourceType::from("text/css"), ResourceType::CSS));
        assert!(matches!(
            ResourceType::from("image/png"),
            ResourceType::Image
        ));
        assert!(matches!(
            ResourceType::from("application/octet-stream"),
            ResourceType::Other
        ));
    }

    #[test]
    fn work_queue_fifo_and_empty() {
        let mut queue = WorkQueue::new();
        assert!(queue.is_empty());

        queue.add_page(Url::parse("https://example.com/a").unwrap());
        queue.add_resource(Url::parse("https://example.com/b.js").unwrap());
        assert!(!queue.is_empty());

        assert_eq!(queue.get_next_page().unwrap().path(), "/a");
        assert_eq!(queue.get_next_resource().unwrap().path(), "/b.js");
        assert!(queue.is_empty());
    }
}

1	use serde::{Deserialize, Serialize};
2	use std::collections::HashMap;
3	use url::Url;
4
5	#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
6	pub enum ResourceType {
7	HTML,
8	CSS,
9	JavaScript,
10	Image,
11	Video,
12	PDF,
13	Font,
14	Other,
15	}
16
17	impl std::fmt::Display for ResourceType {
18	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {	1✔
19	write!(f, "{}", self.download_filter_key())	1✔
20	}
21	}
22
23	impl ResourceType {
24	/// CLI filter name used by `--download-only` (e.g. `js`, not `javascript`).
25	pub fn download_filter_key(&self) -> &'static str {	3✔
26	match self {	3✔
27	ResourceType::HTML => "html",	2✔
28	ResourceType::CSS => "css",	2✔
29	ResourceType::JavaScript => "js",	3✔
30	ResourceType::Image => "images",	3✔
NEW 31	ResourceType::Video => "video",	×
NEW 32	ResourceType::PDF => "pdf",	×
33	ResourceType::Font => "fonts",	2✔
NEW 34	ResourceType::Other => "other",	×
35	}
36	}
37	}
38
NEW 39	impl From<&str> for ResourceType {	×
40	fn from(mime_type: &str) -> Self {	1✔
41	match mime_type {
42	"text/html" => ResourceType::HTML,	1✔
43	"text/css" => ResourceType::CSS,	2✔
44	"application/javascript" \| "text/javascript" => ResourceType::JavaScript,	2✔
45	mime if mime.starts_with("image/") => ResourceType::Image,	2✔
46	mime if mime.starts_with("video/") => ResourceType::Video,	1✔
47	"application/pdf" => ResourceType::PDF,	1✔
48	mime if mime.starts_with("font/") \|\| mime == "application/font-woff" => {	2✔
NEW 49	ResourceType::Font	×
50	}
51	_ => ResourceType::Other,	1✔
52	}
53	}
54	}
55
56	#[derive(Debug, Clone, Serialize, Deserialize)]
57	pub struct Resource {
58	pub url: Url,
59	pub local_path: String,
60	pub resource_type: ResourceType,
61	pub mime_type: String,
62	pub size: Option<u64>,
63	pub downloaded: bool,
64	}
65
66	#[derive(Debug, Clone, Serialize, Deserialize)]
67	pub struct Page {
68	pub url: Url,
69	pub local_path: String,
70	pub depth: usize,
71	pub resources: Vec<Resource>,
72	pub processed: bool,
73	}
74
75	#[derive(Debug, Clone)]
76	pub struct DownloadStats {
77	pub total_pages: usize,
78	pub processed_pages: usize,
79	pub total_resources: usize,
80	pub downloaded_resources: usize,
81	pub total_size: u64,
82	pub start_time: std::time::Instant,
83	}
84
85	impl Default for DownloadStats {
NEW 86	fn default() -> Self {	×
NEW 87	Self::new()	×
88	}
89	}
90
NEW 91	impl DownloadStats {	×
92	pub fn new() -> Self {	2✔
93	Self {
94	total_pages: 0,
95	processed_pages: 0,
96	total_resources: 0,
97	downloaded_resources: 0,
98	total_size: 0,
99	start_time: std::time::Instant::now(),	2✔
100	}
101	}
102
103	pub fn progress_percentage(&self) -> f64 {	1✔
104	if self.total_pages == 0 {	2✔
105	0.0	1✔
106	} else {
107	(self.processed_pages as f64 / self.total_pages as f64) * 100.0	1✔
108	}
109	}
110
111	pub fn elapsed_time(&self) -> std::time::Duration {	1✔
112	self.start_time.elapsed()	1✔
113	}
114	}
115
116	#[derive(Debug, Clone)]
117	pub struct WorkQueue {
118	pub pages: Vec<Url>,
119	pub resources: Vec<Url>,
120	pub visited_urls: HashMap<String, bool>,
121	}
122
123	impl Default for WorkQueue {
NEW 124	fn default() -> Self {	×
NEW 125	Self::new()	×
126	}
127	}
128
129	impl WorkQueue {
130	pub fn new() -> Self {	3✔
131	Self {
132	pages: Vec::new(),	3✔
133	resources: Vec::new(),	3✔
134	visited_urls: HashMap::new(),	3✔
135	}
136	}
137
138	pub fn add_page(&mut self, url: Url) {	3✔
139	let normalized = self.normalize_url(&url);	6✔
140	if !self.visited_urls.contains_key(&normalized) {	6✔
141	self.pages.push(url);	3✔
142	self.visited_urls.insert(normalized, true);	3✔
143	}
144	}
145
146	pub fn add_resource(&mut self, url: Url) {	3✔
147	// Resources can be added multiple times (they might be referenced from different pages)
148	// Only check if we already have this exact resource to avoid duplicates
149	let normalized = self.normalize_url(&url);	3✔
150	if !self	9✔
151	.resources
152	.iter()	3✔
153	.any(\|r\| self.normalize_url(r) == normalized)	7✔
154	{
155	self.resources.push(url);	6✔
156	}
157	}
158
159	pub fn normalize_url(&self, url: &Url) -> String {	3✔
160	let mut normalized = url.clone();	3✔
161	normalized.set_fragment(None);	3✔
162	normalized.set_query(None);	3✔
163	normalized.to_string()	3✔
164	}
165
166	pub fn get_next_page(&mut self) -> Option<Url> {	1✔
167	self.pages.pop()	1✔
168	}
169
170	pub fn get_next_resource(&mut self) -> Option<Url> {	1✔
171	self.resources.pop()	1✔
172	}
173
174	pub fn is_empty(&self) -> bool {	1✔
175	self.pages.is_empty() && self.resources.is_empty()	1✔
176	}
177	}
178
179	#[cfg(test)]
180	mod tests {
181	use super::*;
182
183	#[test]
184	fn download_filter_key_matches_cli_values() {
185	assert_eq!(ResourceType::JavaScript.download_filter_key(), "js");
186	assert_eq!(ResourceType::Image.download_filter_key(), "images");
187	assert_eq!(ResourceType::Font.download_filter_key(), "fonts");
188	}
189
190	#[test]
191	fn display_uses_download_filter_key() {
192	assert_eq!(ResourceType::JavaScript.to_string(), "js");
193	}
194
195	#[test]
196	fn download_stats_progress_percentage() {
197	let mut stats = DownloadStats::new();
198	assert_eq!(stats.progress_percentage(), 0.0);
199
200	stats.total_pages = 4;
201	stats.processed_pages = 2;
202	assert_eq!(stats.progress_percentage(), 50.0);
203	}
204
205	#[test]
206	fn resource_type_from_mime() {
207	assert!(matches!(ResourceType::from("text/css"), ResourceType::CSS));
208	assert!(matches!(
209	ResourceType::from("image/png"),
210	ResourceType::Image
211	));
212	assert!(matches!(
213	ResourceType::from("application/octet-stream"),
214	ResourceType::Other
215	));
216	}
217
218	#[test]
219	fn work_queue_fifo_and_empty() {
220	let mut queue = WorkQueue::new();
221	assert!(queue.is_empty());
222
223	queue.add_page(Url::parse("https://example.com/a").unwrap());
224	queue.add_resource(Url::parse("https://example.com/b.js").unwrap());
225	assert!(!queue.is_empty());
226
227	assert_eq!(queue.get_next_page().unwrap().path(), "/a");
228	assert_eq!(queue.get_next_resource().unwrap().path(), "/b.js");
229	assert!(queue.is_empty());
230	}
231	}

thanos / petrify / 27228748356

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous