16105711226

Committed 07 Jul 2025 01:48AM UTC coverage: 47.797% (+0.2%) from 47.637%

Build # 16105711226

Build Type

push

github

Committed by

HebaruSan

Commit Message

Merge #4404 Refactor Net.Download exception handling

Run Details

3898 of 8735 branches covered (44.63%)

Branch coverage included in aggregate %.

4 of 27 new or added lines in 4 files covered. (14.81%)

7 existing lines in 2 files now uncovered.

8362 of 16915 relevant lines covered (49.44%)

1.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

12.62

/Netkan/Services/CachingHttpService.cs

using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.IO;

using LazyCache;
using log4net;

using CKAN.IO;
using CKAN.NetKAN.Model;

namespace CKAN.NetKAN.Services
{
    internal sealed class CachingHttpService : IHttpService
    {
        public CachingHttpService(NetFileCache cache,
                                  bool         overwrite = false,
                                  string?      userAgent = null)
        {
            _cache          = cache;
            _userAgent      = userAgent;
            _overwriteCache = overwrite;
        }

        public string? DownloadModule(Metadata metadata)
        {
            if (metadata.Download is not { Count: > 0 })
            {
                return null;
            }
            try
            {
                return DownloadPackage(metadata.Download[0],
                                       metadata.Identifier,
                                       metadata.ReleaseDate);
            }
            catch
            {
                var fallback = metadata.FallbackDownload;
                if (fallback == null)
                {
                    throw;
                }
                else
                {
                    log.InfoFormat("Trying fallback URL: {0}", fallback);
                    return DownloadPackage(fallback,
                                           metadata.Identifier,
                                           metadata.ReleaseDate,
                                           metadata.Download[0]);
                }
            }
        }

        private string DownloadPackage(Uri       url,
                                       string    identifier,
                                       DateTime? updated,
                                       Uri?      primaryUrl = null)
        {
            if (primaryUrl == null)
            {
                primaryUrl = url;
            }
            if (_overwriteCache && !_requestedURLs.ContainsKey(url))
            {
                // Discard cached file if command line says so,
                // but only the first time in each run
                _cache.Remove(url);
            }

            // There is no ConcurrentHashSet, so we have to store a null byte as the Value
            _requestedURLs.TryAdd(url, default);

            var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);

            if (cachedFile != null && !string.IsNullOrWhiteSpace(cachedFile))
            {
                return cachedFile;
            }
            else
            {
                var downloadedFile = Net.Download(url, out _, _userAgent,
                                                  _cache.GetInProgressFileName(url,
                                                                               $"netkan-{identifier}")
                                                        .FullName);

                string extension;

                switch (FileIdentifier.IdentifyFile(downloadedFile))
                {
                    case FileType.ASCII:
                        extension = "txt";
                        break;
                    case FileType.GZip:
                        extension = "gz";
                        break;
                    case FileType.Tar:
                        extension = "tar";
                        break;
                    case FileType.TarGz:
                        extension = "tar.gz";
                        break;
                    case FileType.Zip:
                        if (!NetModuleCache.ZipValid(downloadedFile, out string? invalidReason, null))
                        {
                            log.Debug($"{url} is not a valid ZIP file: {invalidReason}");
                            File.Delete(downloadedFile);
                            throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");
                        }
                        extension = "zip";
                        break;
                    default:
                        extension = "ckan-package";
                        break;
                }

                var destName = $"netkan-{identifier}.{extension}";

                try
                {
                    return _cache.Store(primaryUrl, downloadedFile,
                                        destName, move: true);
                }
                catch (IOException exc)
                {
                    // If cache is full, don't also fill /tmp
                    log.Debug($"Failed to store to cache: {exc.Message}");
                    File.Delete(downloadedFile);
                    throw;
                }
            }
        }

        public string? DownloadText(Uri     url,
                                    string? authToken = null,
                                    string? mimeType  = null)
            => _stringCache.GetOrAdd(url.OriginalString,
                                     () => Net.DownloadText(url,
                                                            _userAgent, authToken, mimeType,
                                                            10000),
                                     DateTimeOffset.Now + stringCacheLifetime);

        public IEnumerable<Uri> RequestedURLs => _requestedURLs.Keys;
        public void ClearRequestedURLs()
        {
            _requestedURLs.Clear();
        }

        public Uri? ResolveRedirect(Uri url, string? userAgent)
            => Net.ResolveRedirect(url, userAgent);

        private readonly NetFileCache                    _cache;
        private readonly string?                         _userAgent;
        // Microsoft declined to implement ConcurrentHashSet in favor of this
        private readonly ConcurrentDictionary<Uri, byte> _requestedURLs  = new ConcurrentDictionary<Uri, byte>();
        private readonly bool                            _overwriteCache = false;
        private readonly IAppCache                       _stringCache    = new CachingService();

        // Re-use string value URLs within 15 minutes
        private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);

        private static readonly ILog log = LogManager.GetLogger(typeof(CachingHttpService));
    }
}

1	using System;
2	using System.Collections.Generic;
3	using System.Collections.Concurrent;
4	using System.IO;
5
6	using LazyCache;
7	using log4net;
8
9	using CKAN.IO;
10	using CKAN.NetKAN.Model;
11
12	namespace CKAN.NetKAN.Services
13	{
14	internal sealed class CachingHttpService : IHttpService
15	{
16	public CachingHttpService(NetFileCache cache,	2✔
17	bool overwrite = false,
18	string? userAgent = null)
19	{	2✔
20	_cache = cache;	2✔
21	_userAgent = userAgent;	2✔
22	_overwriteCache = overwrite;	2✔
23	}	2✔
24
25	public string? DownloadModule(Metadata metadata)
26	{	×
27	if (metadata.Download is not { Count: > 0 })	×
28	{	×
29	return null;	×
30	}
31	try
32	{	×
33	return DownloadPackage(metadata.Download[0],	×
34	metadata.Identifier,
35	metadata.ReleaseDate);
36	}
37	catch	×
38	{	×
39	var fallback = metadata.FallbackDownload;	×
40	if (fallback == null)	×
41	{	×
42	throw;	×
43	}
44	else
45	{	×
46	log.InfoFormat("Trying fallback URL: {0}", fallback);	×
47	return DownloadPackage(fallback,	×
48	metadata.Identifier,
49	metadata.ReleaseDate,
50	metadata.Download[0]);
51	}
52	}
53	}	×
54
55	private string DownloadPackage(Uri url,
56	string identifier,
57	DateTime? updated,
58	Uri? primaryUrl = null)
59	{	×
60	if (primaryUrl == null)	×
61	{	×
62	primaryUrl = url;	×
63	}	×
64	if (_overwriteCache && !_requestedURLs.ContainsKey(url))	×
65	{	×
66	// Discard cached file if command line says so,
67	// but only the first time in each run
68	_cache.Remove(url);	×
69	}	×
70
71	// There is no ConcurrentHashSet, so we have to store a null byte as the Value
72	_requestedURLs.TryAdd(url, default);	×
73
74	var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);	×
75
76	if (cachedFile != null && !string.IsNullOrWhiteSpace(cachedFile))	×
77	{	×
78	return cachedFile;	×
79	}
80	else
81	{	×
NEW 82	var downloadedFile = Net.Download(url, out _, _userAgent,	×
83	_cache.GetInProgressFileName(url,
84	$"netkan-{identifier}")
85	.FullName);
86
87	string extension;
88
89	switch (FileIdentifier.IdentifyFile(downloadedFile))	×
90	{
91	case FileType.ASCII:
92	extension = "txt";	×
93	break;	×
94	case FileType.GZip:
95	extension = "gz";	×
96	break;	×
97	case FileType.Tar:
98	extension = "tar";	×
99	break;	×
100	case FileType.TarGz:
101	extension = "tar.gz";	×
102	break;	×
103	case FileType.Zip:
104	if (!NetModuleCache.ZipValid(downloadedFile, out string? invalidReason, null))	×
105	{	×
106	log.Debug($"{url} is not a valid ZIP file: {invalidReason}");	×
107	File.Delete(downloadedFile);	×
108	throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");	×
109	}
110	extension = "zip";	×
111	break;	×
112	default:
113	extension = "ckan-package";	×
114	break;	×
115	}
116
117	var destName = $"netkan-{identifier}.{extension}";	×
118
119	try
120	{	×
121	return _cache.Store(primaryUrl, downloadedFile,	×
122	destName, move: true);
123	}
124	catch (IOException exc)	×
125	{	×
126	// If cache is full, don't also fill /tmp
127	log.Debug($"Failed to store to cache: {exc.Message}");	×
128	File.Delete(downloadedFile);	×
129	throw;	×
130	}
131	}
132	}	×
133
134	public string? DownloadText(Uri url,
135	string? authToken = null,
136	string? mimeType = null)
137	=> _stringCache.GetOrAdd(url.OriginalString,	2✔
138	() => Net.DownloadText(url,	2✔
139	_userAgent, authToken, mimeType,
140	10000),
141	DateTimeOffset.Now + stringCacheLifetime);
142
143	public IEnumerable<Uri> RequestedURLs => _requestedURLs.Keys;	×
144	public void ClearRequestedURLs()
145	{	×
146	_requestedURLs.Clear();	×
147	}	×
148
149	public Uri? ResolveRedirect(Uri url, string? userAgent)
150	=> Net.ResolveRedirect(url, userAgent);	×
151
152	private readonly NetFileCache _cache;
153	private readonly string? _userAgent;
154	// Microsoft declined to implement ConcurrentHashSet in favor of this
155	private readonly ConcurrentDictionary<Uri, byte> _requestedURLs = new ConcurrentDictionary<Uri, byte>();	2✔
156	private readonly bool _overwriteCache = false;	2✔
157	private readonly IAppCache _stringCache = new CachingService();	2✔
158
159	// Re-use string value URLs within 15 minutes
160	private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);	2✔
161
162	private static readonly ILog log = LogManager.GetLogger(typeof(CachingHttpService));	2✔
163	}
164	}

KSP-CKAN / CKAN / 16105711226

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous