18723518392

Committed 22 Oct 2025 04:47PM UTC coverage: 45.087% (+0.2%) from 44.922%

Build # 18723518392

Build Type

push

github

Committed by

0xmichalis

Commit Message

refactor: extract complex closure into its own function

Run Details

14 of 36 new or added lines in 1 file covered. (38.89%)

61 existing lines in 4 files now uncovered.

2184 of 4844 relevant lines covered (45.09%)

7.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.63

/src/httpclient/mod.rs

use std::path::{Path, PathBuf};
use tracing::info;

use crate::content::get_filename;
use crate::content::{extensions, try_exists, write_and_postprocess_file, Options};
use crate::httpclient::fetch::try_fetch_response;
use crate::httpclient::retry::{retry_operation, should_retry};
use crate::httpclient::stream::stream_http_to_file;
use crate::ipfs::config::{IpfsGatewayConfig, IpfsGatewayType, IPFS_GATEWAYS};
use crate::url::{get_data_url, is_data_url, resolve_url_with_gateways};

pub mod fetch;
pub mod retry;
pub mod stream;

#[derive(Clone, Debug)]
pub struct HttpClient {
    pub(crate) ipfs_gateways: Vec<IpfsGatewayConfig>,
    pub(crate) max_retries: u32,
}

impl HttpClient {
    pub fn new() -> Self {
        let ipfs_gateways = IPFS_GATEWAYS.to_vec();
        Self {
            ipfs_gateways,
            max_retries: 5,
        }
    }

    pub fn with_gateways(mut self, gateways: Vec<(String, IpfsGatewayType)>) -> Self {
        self.ipfs_gateways = gateways
            .into_iter()
            .map(|(url, gateway_type)| IpfsGatewayConfig {
                url: Box::leak(url.into_boxed_str()),
                gateway_type,
                bearer_token_env: None,
            })
            .collect();
        self
    }

    pub fn with_max_retries(mut self, max_retries: u32) -> Self {
        self.max_retries = max_retries;
        self
    }

    pub async fn fetch(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        if is_data_url(url) {
            return get_data_url(url)
                .ok_or_else(|| anyhow::anyhow!("Failed to parse data URL: {}", url));
        }

        let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
        let gateways = self.ipfs_gateways.clone();

        let (result, _status) = retry_operation(
            || {
                let url = resolved_url.clone();
                let gateways = gateways.clone();
                Box::pin(async move {
                    match try_fetch_response(&url, &gateways).await {
                        (Ok(response), status) => match response.bytes().await {
                            Ok(b) => (Ok(b.to_vec()), status),
                            Err(e) => (Err(anyhow::anyhow!(e)), status),
                        },
                        (Err(err), status) => (Err(err), status),
                    }
                })
            },
            self.max_retries,
            should_retry,
            &resolved_url,
        )
        .await;
        result
    }

    pub async fn fetch_and_write(
        &self,
        url: &str,
        token: &impl crate::chain::common::ContractTokenInfo,
        output_path: &Path,
        options: Options,
    ) -> anyhow::Result<PathBuf> {
        let mut file_path = get_filename(url, token, output_path, options).await?;

        if let Some(existing_path) = try_exists(&file_path).await? {
            tracing::debug!(
                "File already exists at {} (skipping download)",
                existing_path.display()
            );
            return Ok(existing_path);
        }

        let parent = file_path.parent().ok_or_else(|| {
            anyhow::anyhow!("File path has no parent directory: {}", file_path.display())
        })?;
        tokio::fs::create_dir_all(parent).await.map_err(|e| {
            anyhow::anyhow!("Failed to create directory {}: {}", parent.display(), e)
        })?;

        if is_data_url(url) {
            let content = get_data_url(url)
                .ok_or_else(|| anyhow::anyhow!("Failed to parse data URL: {}", url))?;
            if !extensions::has_known_extension(&file_path) {
                if let Some(detected_ext) = extensions::detect_media_extension(&content) {
                    let current_path_str = file_path.to_string_lossy();
                    tracing::debug!("Appending detected media extension: {}", detected_ext);
                    file_path = PathBuf::from(format!("{current_path_str}.{detected_ext}"));
                }
            }

            info!("Saving {} (data url)", file_path.display());
            write_and_postprocess_file(&file_path, &content, url).await?;
            info!("Saved {} (data url)", file_path.display());
            return Ok(file_path);
        }

        let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
        if url == resolved_url {
            info!("Saving {} (url: {})", file_path.display(), url);
        } else {
            info!(
                "Saving {} (original: {}, resolved: {})",
                file_path.display(),
                url,
                resolved_url
            );
        }
        let file_path = self
            .fetch_and_stream_to_file(&resolved_url, &file_path, self.max_retries)
            .await?;

        write_and_postprocess_file(&file_path, &[], url).await?;
        if url == resolved_url {
            info!("Saved {} (url: {})", file_path.display(), url);
        } else {
            info!(
                "Saved {} (original: {}, resolved: {})",
                file_path.display(),
                url,
                resolved_url
            );
        }
        Ok(file_path)
    }

    pub async fn try_fetch_response(
        &self,
        url: &str,
    ) -> (
        anyhow::Result<reqwest::Response>,
        Option<reqwest::StatusCode>,
    ) {
        try_fetch_response(url, &self.ipfs_gateways).await
    }

    pub(crate) async fn fetch_and_stream_to_file(
        &self,
        url: &str,
        file_path: &Path,
        max_retries: u32,
    ) -> anyhow::Result<PathBuf> {
        fetch_and_stream_to_file(url, file_path, max_retries, &self.ipfs_gateways).await
    }
}

async fn fetch_and_stream_to_file(
    url: &str,
    file_path: &Path,
    max_retries: u32,
    gateways: &[IpfsGatewayConfig],
) -> anyhow::Result<PathBuf> {
    let (result, _status) = retry_operation(
        || {
            let url = url.to_string();
            let file_path = file_path.to_path_buf();
            let gateways = gateways.to_owned();
            Box::pin(async move {
                match try_fetch_response(&url, &gateways).await {
                    (Ok(response), status) => {
                        (stream_http_to_file(response, &file_path).await, status)
                    }
                    (Err(err), status) => (Err(err), status),
                }
            })
        },
        max_retries,
        should_retry,
        url,
    )
    .await;
    result
}

impl Default for HttpClient {
    fn default() -> Self {
        Self::new()
    }
}

1	use std::path::{Path, PathBuf};
2	use tracing::info;
3
4	use crate::content::get_filename;
5	use crate::content::{extensions, try_exists, write_and_postprocess_file, Options};
6	use crate::httpclient::fetch::try_fetch_response;
7	use crate::httpclient::retry::{retry_operation, should_retry};
8	use crate::httpclient::stream::stream_http_to_file;
9	use crate::ipfs::config::{IpfsGatewayConfig, IpfsGatewayType, IPFS_GATEWAYS};
10	use crate::url::{get_data_url, is_data_url, resolve_url_with_gateways};
11
12	pub mod fetch;
13	pub mod retry;
14	pub mod stream;
15
16	#[derive(Clone, Debug)]
17	pub struct HttpClient {
18	pub(crate) ipfs_gateways: Vec<IpfsGatewayConfig>,
19	pub(crate) max_retries: u32,
20	}
21
22	impl HttpClient {
23	pub fn new() -> Self {	31✔
24	let ipfs_gateways = IPFS_GATEWAYS.to_vec();	93✔
25	Self {
26	ipfs_gateways,
27	max_retries: 5,
28	}
29	}
30
31	pub fn with_gateways(mut self, gateways: Vec<(String, IpfsGatewayType)>) -> Self {	10✔
32	self.ipfs_gateways = gateways	20✔
33	.into_iter()	10✔
34	.map(\|(url, gateway_type)\| IpfsGatewayConfig {	10✔
35	url: Box::leak(url.into_boxed_str()),	16✔
36	gateway_type,	8✔
37	bearer_token_env: None,	8✔
38	})
39	.collect();	10✔
40	self	10✔
41	}
42
43	pub fn with_max_retries(mut self, max_retries: u32) -> Self {	8✔
44	self.max_retries = max_retries;	8✔
45	self	8✔
46	}
47
48	pub async fn fetch(&self, url: &str) -> anyhow::Result<Vec<u8>> {	×
49	if is_data_url(url) {	×
50	return get_data_url(url)	×
51	.ok_or_else(\|\| anyhow::anyhow!("Failed to parse data URL: {}", url));	×
52	}
53
54	let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
55	let gateways = self.ipfs_gateways.clone();
56
57	let (result, _status) = retry_operation(
58	\|\| {	×
59	let url = resolved_url.clone();	×
60	let gateways = gateways.clone();	×
61	Box::pin(async move {	×
62	match try_fetch_response(&url, &gateways).await {	×
63	(Ok(response), status) => match response.bytes().await {	×
64	Ok(b) => (Ok(b.to_vec()), status),	×
65	Err(e) => (Err(anyhow::anyhow!(e)), status),	×
66	},
67	(Err(err), status) => (Err(err), status),	×
68	}
69	})
70	},
71	self.max_retries,
72	should_retry,
73	&resolved_url,
74	)
75	.await;
UNCOV 76	result	×
77	}
78
79	pub async fn fetch_and_write(	21✔
80	&self,
81	url: &str,
82	token: &impl crate::chain::common::ContractTokenInfo,
83	output_path: &Path,
84	options: Options,
85	) -> anyhow::Result<PathBuf> {
86	let mut file_path = get_filename(url, token, output_path, options).await?;	126✔
87
88	if let Some(existing_path) = try_exists(&file_path).await? {	7✔
89	tracing::debug!(
90	"File already exists at {} (skipping download)",	×
UNCOV 91	existing_path.display()	×
92	);
93	return Ok(existing_path);
94	}
95
96	let parent = file_path.parent().ok_or_else(\|\| {	42✔
UNCOV 97	anyhow::anyhow!("File path has no parent directory: {}", file_path.display())	×
98	})?;
99	tokio::fs::create_dir_all(parent).await.map_err(\|e\| {	14✔
UNCOV 100	anyhow::anyhow!("Failed to create directory {}: {}", parent.display(), e)	×
101	})?;
102
103	if is_data_url(url) {	14✔
104	let content = get_data_url(url)	24✔
105	.ok_or_else(\|\| anyhow::anyhow!("Failed to parse data URL: {}", url))?;	8✔
106	if !extensions::has_known_extension(&file_path) {
107	if let Some(detected_ext) = extensions::detect_media_extension(&content) {	16✔
108	let current_path_str = file_path.to_string_lossy();
UNCOV 109	tracing::debug!("Appending detected media extension: {}", detected_ext);	×
110	file_path = PathBuf::from(format!("{current_path_str}.{detected_ext}"));
111	}
112	}
113
114	info!("Saving {} (data url)", file_path.display());	×
UNCOV 115	write_and_postprocess_file(&file_path, &content, url).await?;	×
116	info!("Saved {} (data url)", file_path.display());	8✔
117	return Ok(file_path);
118	}
119
120	let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
121	if url == resolved_url {
122	info!("Saving {} (url: {})", file_path.display(), url);	1✔
123	} else {
124	info!(	5✔
125	"Saving {} (original: {}, resolved: {})",	×
UNCOV 126	file_path.display(),	×
127	url,
128	resolved_url
129	);
130	}
UNCOV 131	let file_path = self	×
132	.fetch_and_stream_to_file(&resolved_url, &file_path, self.max_retries)
133	.await?;	6✔
134
135	write_and_postprocess_file(&file_path, &[], url).await?;	×
136	if url == resolved_url {	×
UNCOV 137	info!("Saved {} (url: {})", file_path.display(), url);	×
138	} else {
139	info!(	×
140	"Saved {} (original: {}, resolved: {})",	×
UNCOV 141	file_path.display(),	×
142	url,
143	resolved_url
144	);
145	}
146	Ok(file_path)
147	}
148
UNCOV 149	pub async fn try_fetch_response(	×
150	&self,
151	url: &str,
152	) -> (
153	anyhow::Result<reqwest::Response>,
154	Option<reqwest::StatusCode>,
155	) {
UNCOV 156	try_fetch_response(url, &self.ipfs_gateways).await	×
157	}
158
159	pub(crate) async fn fetch_and_stream_to_file(	6✔
160	&self,
161	url: &str,
162	file_path: &Path,
163	max_retries: u32,
164	) -> anyhow::Result<PathBuf> {
165	fetch_and_stream_to_file(url, file_path, max_retries, &self.ipfs_gateways).await	30✔
166	}
167	}
168
169	async fn fetch_and_stream_to_file(	6✔
170	url: &str,
171	file_path: &Path,
172	max_retries: u32,
173	gateways: &[IpfsGatewayConfig],
174	) -> anyhow::Result<PathBuf> {
175	let (result, _status) = retry_operation(
176	\|\| {	6✔
177	let url = url.to_string();	18✔
178	let file_path = file_path.to_path_buf();	18✔
179	let gateways = gateways.to_owned();	18✔
180	Box::pin(async move {	12✔
181	match try_fetch_response(&url, &gateways).await {	24✔
182	(Ok(response), status) => {	×
UNCOV 183	(stream_http_to_file(response, &file_path).await, status)	×
184	}
185	(Err(err), status) => (Err(err), status),	18✔
186	}
187	})
188	},
189	max_retries,	6✔
190	should_retry,
191	url,	6✔
192	)
193	.await;	6✔
194	result
195	}
196
197	impl Default for HttpClient {
UNCOV 198	fn default() -> Self {	×
UNCOV 199	Self::new()	×
200	}
201	}

0xmichalis / nftbk / 18723518392

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous