18378365690

Committed 09 Oct 2025 01:42PM UTC coverage: 34.076% (-31.8%) from 65.845%

Build # 18378365690

Build Type

push

github

Committed by

0xmichalis

Commit Message

ci: tarpaulin install check update

Run Details

887 of 2603 relevant lines covered (34.08%)

6.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.06

/src/httpclient/mod.rs

use std::path::{Path, PathBuf};
use tracing::info;

use crate::content::get_filename;
use crate::content::{extensions, try_exists, write_and_postprocess_file, Options};
use crate::httpclient::fetch::try_fetch_response;
use crate::httpclient::retry::{retry_operation, should_retry};
use crate::httpclient::stream::stream_http_to_file;
use crate::ipfs::config::{IpfsGatewayConfig, IpfsGatewayType, IPFS_GATEWAYS};
use crate::url::{get_data_url, is_data_url, resolve_url_with_gateways};

pub mod fetch;
pub mod retry;
pub mod stream;

#[derive(Clone, Debug)]
pub struct HttpClient {
    pub(crate) ipfs_gateways: Vec<IpfsGatewayConfig>,
}

impl HttpClient {
    pub fn new() -> Self {
        let ipfs_gateways = IPFS_GATEWAYS.to_vec();
        Self { ipfs_gateways }
    }

    pub fn new_with_gateways(gateways: Vec<(String, IpfsGatewayType)>) -> Self {
        let ipfs_gateways = gateways
            .into_iter()
            .map(|(url, gateway_type)| IpfsGatewayConfig {
                url: Box::leak(url.into_boxed_str()),
                gateway_type,
            })
            .collect();
        Self { ipfs_gateways }
    }

    pub async fn fetch(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        if is_data_url(url) {
            return get_data_url(url)
                .ok_or_else(|| anyhow::anyhow!("Failed to parse data URL: {}", url));
        }

        let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
        let gateways = self.ipfs_gateways.clone();

        const MAX_RETRIES: u32 = 5;
        retry_operation(
            || {
                let url = resolved_url.clone();
                let gateways = gateways.clone();
                Box::pin(async move {
                    match try_fetch_response(&url, &gateways).await {
                        (Ok(response), status) => match response.bytes().await {
                            Ok(b) => (Ok(b.to_vec()), status),
                            Err(e) => (Err(anyhow::anyhow!(e)), status),
                        },
                        (Err(err), status) => (Err(err), status),
                    }
                })
            },
            MAX_RETRIES,
            should_retry,
            &resolved_url,
        )
        .await
    }

    pub async fn fetch_and_write(
        &self,
        url: &str,
        token: &impl crate::chain::common::ContractTokenInfo,
        output_path: &Path,
        options: Options,
    ) -> anyhow::Result<PathBuf> {
        let mut file_path = get_filename(url, token, output_path, options).await?;

        if let Some(existing_path) = try_exists(&file_path).await? {
            tracing::debug!(
                "File already exists at {} (skipping download)",
                existing_path.display()
            );
            return Ok(existing_path);
        }

        let parent = file_path.parent().ok_or_else(|| {
            anyhow::anyhow!("File path has no parent directory: {}", file_path.display())
        })?;
        tokio::fs::create_dir_all(parent).await.map_err(|e| {
            anyhow::anyhow!("Failed to create directory {}: {}", parent.display(), e)
        })?;

        if is_data_url(url) {
            let content = get_data_url(url)
                .ok_or_else(|| anyhow::anyhow!("Failed to parse data URL: {}", url))?;
            if !extensions::has_known_extension(&file_path) {
                if let Some(detected_ext) = extensions::detect_media_extension(&content) {
                    let current_path_str = file_path.to_string_lossy();
                    tracing::debug!("Appending detected media extension: {}", detected_ext);
                    file_path = PathBuf::from(format!("{current_path_str}.{detected_ext}"));
                }
            }

            info!("Saving {} (data url)", file_path.display());
            write_and_postprocess_file(&file_path, &content, url).await?;
            info!("Saved {} (data url)", file_path.display());
            return Ok(file_path);
        }

        let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
        if url == resolved_url {
            info!("Saving {} (url: {})", file_path.display(), url);
        } else {
            info!(
                "Saving {} (original: {}, resolved: {})",
                file_path.display(),
                url,
                resolved_url
            );
        }
        let file_path = self
            .fetch_and_stream_to_file(&resolved_url, &file_path, 5)
            .await?;

        write_and_postprocess_file(&file_path, &[], url).await?;
        if url == resolved_url {
            info!("Saved {} (url: {})", file_path.display(), url);
        } else {
            info!(
                "Saved {} (original: {}, resolved: {})",
                file_path.display(),
                url,
                resolved_url
            );
        }
        Ok(file_path)
    }

    pub async fn try_fetch_response(
        &self,
        url: &str,
    ) -> (
        anyhow::Result<reqwest::Response>,
        Option<reqwest::StatusCode>,
    ) {
        try_fetch_response(url, &self.ipfs_gateways).await
    }

    pub(crate) async fn fetch_and_stream_to_file(
        &self,
        url: &str,
        file_path: &Path,
        max_retries: u32,
    ) -> anyhow::Result<PathBuf> {
        fetch_and_stream_to_file(url, file_path, max_retries, &self.ipfs_gateways).await
    }
}

async fn fetch_and_stream_to_file(
    url: &str,
    file_path: &Path,
    max_retries: u32,
    gateways: &[IpfsGatewayConfig],
) -> anyhow::Result<PathBuf> {
    retry_operation(
        || {
            let url = url.to_string();
            let file_path = file_path.to_path_buf();
            let gateways = gateways.to_owned();
            Box::pin(async move {
                match try_fetch_response(&url, &gateways).await {
                    (Ok(response), status) => {
                        (stream_http_to_file(response, &file_path).await, status)
                    }
                    (Err(err), status) => (Err(err), status),
                }
            })
        },
        max_retries,
        should_retry,
        url,
    )
    .await
}

impl Default for HttpClient {
    fn default() -> Self {
        Self::new()
    }
}

1	use std::path::{Path, PathBuf};
2	use tracing::info;
3
4	use crate::content::get_filename;
5	use crate::content::{extensions, try_exists, write_and_postprocess_file, Options};
6	use crate::httpclient::fetch::try_fetch_response;
7	use crate::httpclient::retry::{retry_operation, should_retry};
8	use crate::httpclient::stream::stream_http_to_file;
9	use crate::ipfs::config::{IpfsGatewayConfig, IpfsGatewayType, IPFS_GATEWAYS};
10	use crate::url::{get_data_url, is_data_url, resolve_url_with_gateways};
11
12	pub mod fetch;
13	pub mod retry;
14	pub mod stream;
15
16	#[derive(Clone, Debug)]
17	pub struct HttpClient {
18	pub(crate) ipfs_gateways: Vec<IpfsGatewayConfig>,
19	}
20
21	impl HttpClient {
22	pub fn new() -> Self {	18✔
23	let ipfs_gateways = IPFS_GATEWAYS.to_vec();	54✔
24	Self { ipfs_gateways }
25	}
26
27	pub fn new_with_gateways(gateways: Vec<(String, IpfsGatewayType)>) -> Self {	6✔
28	let ipfs_gateways = gateways	12✔
29	.into_iter()
30	.map(\|(url, gateway_type)\| IpfsGatewayConfig {	6✔
31	url: Box::leak(url.into_boxed_str()),	12✔
32	gateway_type,	6✔
33	})
34	.collect();
35	Self { ipfs_gateways }
36	}
37
38	pub async fn fetch(&self, url: &str) -> anyhow::Result<Vec<u8>> {	×
39	if is_data_url(url) {	×
40	return get_data_url(url)	×
41	.ok_or_else(\|\| anyhow::anyhow!("Failed to parse data URL: {}", url));	×
42	}
43
44	let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
45	let gateways = self.ipfs_gateways.clone();
46
47	const MAX_RETRIES: u32 = 5;
48	retry_operation(
49	\|\| {	×
50	let url = resolved_url.clone();	×
51	let gateways = gateways.clone();	×
52	Box::pin(async move {	×
53	match try_fetch_response(&url, &gateways).await {	×
54	(Ok(response), status) => match response.bytes().await {	×
55	Ok(b) => (Ok(b.to_vec()), status),	×
56	Err(e) => (Err(anyhow::anyhow!(e)), status),	×
57	},
58	(Err(err), status) => (Err(err), status),	×
59	}
60	})
61	},
62	MAX_RETRIES,
63	should_retry,
64	&resolved_url,
65	)
66	.await
67	}
68
69	pub async fn fetch_and_write(	18✔
70	&self,
71	url: &str,
72	token: &impl crate::chain::common::ContractTokenInfo,
73	output_path: &Path,
74	options: Options,
75	) -> anyhow::Result<PathBuf> {
76	let mut file_path = get_filename(url, token, output_path, options).await?;	108✔
77
78	if let Some(existing_path) = try_exists(&file_path).await? {	7✔
79	tracing::debug!(
80	"File already exists at {} (skipping download)",	×
81	existing_path.display()	×
82	);
83	return Ok(existing_path);
84	}
85
86	let parent = file_path.parent().ok_or_else(\|\| {	33✔
87	anyhow::anyhow!("File path has no parent directory: {}", file_path.display())	×
88	})?;
89	tokio::fs::create_dir_all(parent).await.map_err(\|e\| {	11✔
90	anyhow::anyhow!("Failed to create directory {}: {}", parent.display(), e)	×
91	})?;
92
93	if is_data_url(url) {	11✔
94	let content = get_data_url(url)	21✔
95	.ok_or_else(\|\| anyhow::anyhow!("Failed to parse data URL: {}", url))?;	7✔
96	if !extensions::has_known_extension(&file_path) {
97	if let Some(detected_ext) = extensions::detect_media_extension(&content) {	14✔
98	let current_path_str = file_path.to_string_lossy();
99	tracing::debug!("Appending detected media extension: {}", detected_ext);	×
100	file_path = PathBuf::from(format!("{current_path_str}.{detected_ext}"));
101	}
102	}
103
104	info!("Saving {} (data url)", file_path.display());	×
105	write_and_postprocess_file(&file_path, &content, url).await?;	×
106	info!("Saved {} (data url)", file_path.display());	7✔
107	return Ok(file_path);
108	}
109
110	let resolved_url = resolve_url_with_gateways(url, &self.ipfs_gateways);
111	if url == resolved_url {
112	info!("Saving {} (url: {})", file_path.display(), url);	×
113	} else {
114	info!(	4✔
115	"Saving {} (original: {}, resolved: {})",	×
116	file_path.display(),	×
117	url,
118	resolved_url
119	);
120	}
121	let file_path = self	×
122	.fetch_and_stream_to_file(&resolved_url, &file_path, 5)
123	.await?;	4✔
124
125	write_and_postprocess_file(&file_path, &[], url).await?;	×
126	if url == resolved_url {	×
127	info!("Saved {} (url: {})", file_path.display(), url);	×
128	} else {
129	info!(	×
130	"Saved {} (original: {}, resolved: {})",	×
131	file_path.display(),	×
132	url,
133	resolved_url
134	);
135	}
136	Ok(file_path)
137	}
138
139	pub async fn try_fetch_response(	×
140	&self,
141	url: &str,
142	) -> (
143	anyhow::Result<reqwest::Response>,
144	Option<reqwest::StatusCode>,
145	) {
146	try_fetch_response(url, &self.ipfs_gateways).await	×
147	}
148
149	pub(crate) async fn fetch_and_stream_to_file(	4✔
150	&self,
151	url: &str,
152	file_path: &Path,
153	max_retries: u32,
154	) -> anyhow::Result<PathBuf> {
155	fetch_and_stream_to_file(url, file_path, max_retries, &self.ipfs_gateways).await	20✔
156	}
157	}
158
159	async fn fetch_and_stream_to_file(	4✔
160	url: &str,
161	file_path: &Path,
162	max_retries: u32,
163	gateways: &[IpfsGatewayConfig],
164	) -> anyhow::Result<PathBuf> {
165	retry_operation(
166	\|\| {	4✔
167	let url = url.to_string();	12✔
168	let file_path = file_path.to_path_buf();	12✔
169	let gateways = gateways.to_owned();	12✔
170	Box::pin(async move {	8✔
171	match try_fetch_response(&url, &gateways).await {	16✔
172	(Ok(response), status) => {	×
173	(stream_http_to_file(response, &file_path).await, status)	×
174	}
175	(Err(err), status) => (Err(err), status),	12✔
176	}
177	})
178	},
179	max_retries,	4✔
180	should_retry,
181	url,	4✔
182	)
183	.await	4✔
184	}
185
186	impl Default for HttpClient {
187	fn default() -> Self {	×
188	Self::new()	×
189	}
190	}

0xmichalis / nftbk / 18378365690

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous