18520354496

Committed 15 Oct 2025 06:50AM UTC coverage: 35.446% (+0.1%) from 35.315%

Build # 18520354496

Build Type

push

github

Committed by

0xmichalis

Commit Message

fix: error categorization when processing tokens

Run Details

11 of 14 new or added lines in 2 files covered. (78.57%)

1 existing line in 1 file now uncovered.

1387 of 3913 relevant lines covered (35.45%)

5.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.7

/src/httpclient/mod.rs

use std::path::{Path, PathBuf};
use tracing::info;

use crate::content::get_filename;
use crate::content::{extensions, try_exists, write_and_postprocess_file, Options};
use crate::httpclient::fetch::try_fetch_response;
use crate::httpclient::retry::{retry_operation, should_retry};
use crate::httpclient::stream::stream_http_to_file;
use crate::ipfs::config::{IpfsGatewayConfig, IpfsGatewayType, IPFS_GATEWAYS};
use crate::url::{get_data_url, is_data_url, resolve_url_with_gateways};

pub mod fetch;
pub mod retry;
pub mod stream;

#[derive(Clone, Debug)]
pub struct HttpClient {
    pub(crate) ipfs_gateways: Vec<IpfsGatewayConfig>,
    pub(crate) max_retries: u32,
}

impl HttpClient {
    pub fn new() -> Self {
        let ipfs_gateways = IPFS_GATEWAYS.to_vec();
        Self {
            ipfs_gateways,
            max_retries: 5,
        }
    }

    pub fn with_gateways(mut self, gateways: Vec<(String, IpfsGatewayType)>) -> Self {
        self.ipfs_gateways = gateways
            .into_iter()
            .map(|(url, gateway_type)| IpfsGatewayConfig {
                url: Box::leak(url.into_boxed_str()),
                gateway_type,
            })
            .collect();
        self
    }

    pub fn with_max_retries(mut self, max_retries: u32) -> Self {
        self.max_retries = max_retries;
        self
    }

    pub async fn fetch(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        if is_data_url(url) {
            return get_data_url(url)
                .ok_or_else(|| anyhow::anyhow!("Failed to parse data URL: {}", url));
        }

        let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
        let gateways = self.ipfs_gateways.clone();

        retry_operation(
            || {
                let url = resolved_url.clone();
                let gateways = gateways.clone();
                Box::pin(async move {
                    match try_fetch_response(&url, &gateways).await {
                        (Ok(response), status) => match response.bytes().await {
                            Ok(b) => (Ok(b.to_vec()), status),
                            Err(e) => (Err(anyhow::anyhow!(e)), status),
                        },
                        (Err(err), status) => (Err(err), status),
                    }
                })
            },
            self.max_retries,
            should_retry,
            &resolved_url,
        )
        .await
    }

    pub async fn fetch_and_write(
        &self,
        url: &str,
        token: &impl crate::chain::common::ContractTokenInfo,
        output_path: &Path,
        options: Options,
    ) -> anyhow::Result<PathBuf> {
        let mut file_path = get_filename(url, token, output_path, options).await?;

        if let Some(existing_path) = try_exists(&file_path).await? {
            tracing::debug!(
                "File already exists at {} (skipping download)",
                existing_path.display()
            );
            return Ok(existing_path);
        }

        let parent = file_path.parent().ok_or_else(|| {
            anyhow::anyhow!("File path has no parent directory: {}", file_path.display())
        })?;
        tokio::fs::create_dir_all(parent).await.map_err(|e| {
            anyhow::anyhow!("Failed to create directory {}: {}", parent.display(), e)
        })?;

        if is_data_url(url) {
            let content = get_data_url(url)
                .ok_or_else(|| anyhow::anyhow!("Failed to parse data URL: {}", url))?;
            if !extensions::has_known_extension(&file_path) {
                if let Some(detected_ext) = extensions::detect_media_extension(&content) {
                    let current_path_str = file_path.to_string_lossy();
                    tracing::debug!("Appending detected media extension: {}", detected_ext);
                    file_path = PathBuf::from(format!("{current_path_str}.{detected_ext}"));
                }
            }

            info!("Saving {} (data url)", file_path.display());
            write_and_postprocess_file(&file_path, &content, url).await?;
            info!("Saved {} (data url)", file_path.display());
            return Ok(file_path);
        }

        let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
        if url == resolved_url {
            info!("Saving {} (url: {})", file_path.display(), url);
        } else {
            info!(
                "Saving {} (original: {}, resolved: {})",
                file_path.display(),
                url,
                resolved_url
            );
        }
        let file_path = self
            .fetch_and_stream_to_file(&resolved_url, &file_path, self.max_retries)
            .await?;

        write_and_postprocess_file(&file_path, &[], url).await?;
        if url == resolved_url {
            info!("Saved {} (url: {})", file_path.display(), url);
        } else {
            info!(
                "Saved {} (original: {}, resolved: {})",
                file_path.display(),
                url,
                resolved_url
            );
        }
        Ok(file_path)
    }

    pub async fn try_fetch_response(
        &self,
        url: &str,
    ) -> (
        anyhow::Result<reqwest::Response>,
        Option<reqwest::StatusCode>,
    ) {
        try_fetch_response(url, &self.ipfs_gateways).await
    }

    pub(crate) async fn fetch_and_stream_to_file(
        &self,
        url: &str,
        file_path: &Path,
        max_retries: u32,
    ) -> anyhow::Result<PathBuf> {
        fetch_and_stream_to_file(url, file_path, max_retries, &self.ipfs_gateways).await
    }
}

async fn fetch_and_stream_to_file(
    url: &str,
    file_path: &Path,
    max_retries: u32,
    gateways: &[IpfsGatewayConfig],
) -> anyhow::Result<PathBuf> {
    retry_operation(
        || {
            let url = url.to_string();
            let file_path = file_path.to_path_buf();
            let gateways = gateways.to_owned();
            Box::pin(async move {
                match try_fetch_response(&url, &gateways).await {
                    (Ok(response), status) => {
                        (stream_http_to_file(response, &file_path).await, status)
                    }
                    (Err(err), status) => (Err(err), status),
                }
            })
        },
        max_retries,
        should_retry,
        url,
    )
    .await
}

impl Default for HttpClient {
    fn default() -> Self {
        Self::new()
    }
}

1	use std::path::{Path, PathBuf};
2	use tracing::info;
3
4	use crate::content::get_filename;
5	use crate::content::{extensions, try_exists, write_and_postprocess_file, Options};
6	use crate::httpclient::fetch::try_fetch_response;
7	use crate::httpclient::retry::{retry_operation, should_retry};
8	use crate::httpclient::stream::stream_http_to_file;
9	use crate::ipfs::config::{IpfsGatewayConfig, IpfsGatewayType, IPFS_GATEWAYS};
10	use crate::url::{get_data_url, is_data_url, resolve_url_with_gateways};
11
12	pub mod fetch;
13	pub mod retry;
14	pub mod stream;
15
16	#[derive(Clone, Debug)]
17	pub struct HttpClient {
18	pub(crate) ipfs_gateways: Vec<IpfsGatewayConfig>,
19	pub(crate) max_retries: u32,
20	}
21
22	impl HttpClient {
23	pub fn new() -> Self {	30✔
24	let ipfs_gateways = IPFS_GATEWAYS.to_vec();	90✔
25	Self {
26	ipfs_gateways,
27	max_retries: 5,
28	}
29	}
30
31	pub fn with_gateways(mut self, gateways: Vec<(String, IpfsGatewayType)>) -> Self {	10✔
32	self.ipfs_gateways = gateways	20✔
33	.into_iter()	10✔
34	.map(\|(url, gateway_type)\| IpfsGatewayConfig {	10✔
35	url: Box::leak(url.into_boxed_str()),	16✔
36	gateway_type,	8✔
37	})
38	.collect();	10✔
39	self	10✔
40	}
41
42	pub fn with_max_retries(mut self, max_retries: u32) -> Self {	8✔
43	self.max_retries = max_retries;	8✔
44	self	8✔
45	}
46
47	pub async fn fetch(&self, url: &str) -> anyhow::Result<Vec<u8>> {	×
48	if is_data_url(url) {	×
49	return get_data_url(url)	×
50	.ok_or_else(\|\| anyhow::anyhow!("Failed to parse data URL: {}", url));	×
51	}
52
53	let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
54	let gateways = self.ipfs_gateways.clone();
55
56	retry_operation(
UNCOV 57	\|\| {	×
58	let url = resolved_url.clone();	×
59	let gateways = gateways.clone();	×
60	Box::pin(async move {	×
61	match try_fetch_response(&url, &gateways).await {	×
62	(Ok(response), status) => match response.bytes().await {	×
63	Ok(b) => (Ok(b.to_vec()), status),	×
64	Err(e) => (Err(anyhow::anyhow!(e)), status),	×
65	},
66	(Err(err), status) => (Err(err), status),	×
67	}
68	})
69	},
70	self.max_retries,
71	should_retry,
72	&resolved_url,
73	)
74	.await
75	}
76
77	pub async fn fetch_and_write(	21✔
78	&self,
79	url: &str,
80	token: &impl crate::chain::common::ContractTokenInfo,
81	output_path: &Path,
82	options: Options,
83	) -> anyhow::Result<PathBuf> {
84	let mut file_path = get_filename(url, token, output_path, options).await?;	126✔
85
86	if let Some(existing_path) = try_exists(&file_path).await? {	7✔
87	tracing::debug!(
88	"File already exists at {} (skipping download)",	×
89	existing_path.display()	×
90	);
91	return Ok(existing_path);
92	}
93
94	let parent = file_path.parent().ok_or_else(\|\| {	42✔
95	anyhow::anyhow!("File path has no parent directory: {}", file_path.display())	×
96	})?;
97	tokio::fs::create_dir_all(parent).await.map_err(\|e\| {	14✔
98	anyhow::anyhow!("Failed to create directory {}: {}", parent.display(), e)	×
99	})?;
100
101	if is_data_url(url) {	14✔
102	let content = get_data_url(url)	24✔
103	.ok_or_else(\|\| anyhow::anyhow!("Failed to parse data URL: {}", url))?;	8✔
104	if !extensions::has_known_extension(&file_path) {
105	if let Some(detected_ext) = extensions::detect_media_extension(&content) {	16✔
106	let current_path_str = file_path.to_string_lossy();
107	tracing::debug!("Appending detected media extension: {}", detected_ext);	×
108	file_path = PathBuf::from(format!("{current_path_str}.{detected_ext}"));
109	}
110	}
111
112	info!("Saving {} (data url)", file_path.display());	×
113	write_and_postprocess_file(&file_path, &content, url).await?;	×
114	info!("Saved {} (data url)", file_path.display());	8✔
115	return Ok(file_path);
116	}
117
118	let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
119	if url == resolved_url {
120	info!("Saving {} (url: {})", file_path.display(), url);	1✔
121	} else {
122	info!(	5✔
123	"Saving {} (original: {}, resolved: {})",	×
124	file_path.display(),	×
125	url,
126	resolved_url
127	);
128	}
129	let file_path = self	×
130	.fetch_and_stream_to_file(&resolved_url, &file_path, self.max_retries)
131	.await?;	6✔
132
133	write_and_postprocess_file(&file_path, &[], url).await?;	×
134	if url == resolved_url {	×
135	info!("Saved {} (url: {})", file_path.display(), url);	×
136	} else {
137	info!(	×
138	"Saved {} (original: {}, resolved: {})",	×
139	file_path.display(),	×
140	url,
141	resolved_url
142	);
143	}
144	Ok(file_path)
145	}
146
147	pub async fn try_fetch_response(	×
148	&self,
149	url: &str,
150	) -> (
151	anyhow::Result<reqwest::Response>,
152	Option<reqwest::StatusCode>,
153	) {
154	try_fetch_response(url, &self.ipfs_gateways).await	×
155	}
156
157	pub(crate) async fn fetch_and_stream_to_file(	6✔
158	&self,
159	url: &str,
160	file_path: &Path,
161	max_retries: u32,
162	) -> anyhow::Result<PathBuf> {
163	fetch_and_stream_to_file(url, file_path, max_retries, &self.ipfs_gateways).await	30✔
164	}
165	}
166
167	async fn fetch_and_stream_to_file(	6✔
168	url: &str,
169	file_path: &Path,
170	max_retries: u32,
171	gateways: &[IpfsGatewayConfig],
172	) -> anyhow::Result<PathBuf> {
173	retry_operation(
174	\|\| {	6✔
175	let url = url.to_string();	18✔
176	let file_path = file_path.to_path_buf();	18✔
177	let gateways = gateways.to_owned();	18✔
178	Box::pin(async move {	12✔
179	match try_fetch_response(&url, &gateways).await {	24✔
180	(Ok(response), status) => {	×
181	(stream_http_to_file(response, &file_path).await, status)	×
182	}
183	(Err(err), status) => (Err(err), status),	18✔
184	}
185	})
186	},
187	max_retries,	6✔
188	should_retry,
189	url,	6✔
190	)
191	.await	6✔
192	}
193
194	impl Default for HttpClient {
195	fn default() -> Self {	×
196	Self::new()	×
197	}
198	}

0xmichalis / nftbk / 18520354496

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous