15834236157

Committed 23 Jun 2025 07:42PM UTC coverage: 42.232% (+0.1%) from 42.099%

Build # 15834236157

Build Type

push

github

Committed by

HebaruSan

Commit Message

Merge #4398 Exception handling revamp, parallel multi-host inflation

Run Details

3880 of 9479 branches covered (40.93%)

Branch coverage included in aggregate %.

48 of 137 new or added lines in 30 files covered. (35.04%)

12 existing lines in 6 files now uncovered.

8334 of 19442 relevant lines covered (42.87%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

12.26

/Netkan/Services/CachingHttpService.cs

using System;
using System.Collections.Generic;
using System.IO;

using LazyCache;
using log4net;

using CKAN.IO;
using CKAN.NetKAN.Model;

namespace CKAN.NetKAN.Services
{
    internal sealed class CachingHttpService : IHttpService
    {
        public CachingHttpService(NetFileCache cache,
                                  bool         overwrite = false,
                                  string?      userAgent = null)
        {
            _cache          = cache;
            _userAgent      = userAgent;
            _overwriteCache = overwrite;
        }

        public string? DownloadModule(Metadata metadata)
        {
            if (metadata.Download == null)
            {
                return null;
            }
            try
            {
                return DownloadPackage(metadata.Download[0],
                                       metadata.Identifier,
                                       metadata.ReleaseDate);
            }
            catch
            {
                var fallback = metadata.FallbackDownload;
                if (fallback == null)
                {
                    throw;
                }
                else
                {
                    log.InfoFormat("Trying fallback URL: {0}", fallback);
                    return DownloadPackage(fallback,
                                           metadata.Identifier,
                                           metadata.ReleaseDate,
                                           metadata.Download[0]);
                }
            }
        }

        private string DownloadPackage(Uri       url,
                                       string    identifier,
                                       DateTime? updated,
                                       Uri?      primaryUrl = null)
        {
            if (primaryUrl == null)
            {
                primaryUrl = url;
            }
            if (_overwriteCache && !_requestedURLs.Contains(url))
            {
                // Discard cached file if command line says so,
                // but only the first time in each run
                _cache.Remove(url);
            }

            _requestedURLs.Add(url);

            var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);

            if (cachedFile != null && !string.IsNullOrWhiteSpace(cachedFile))
            {
                return cachedFile;
            }
            else
            {
                var downloadedFile = Net.Download(url, _userAgent,
                                                  _cache.GetInProgressFileName(url,
                                                                               $"netkan-{identifier}")
                                                        .FullName);

                string extension;

                switch (FileIdentifier.IdentifyFile(downloadedFile))
                {
                    case FileType.ASCII:
                        extension = "txt";
                        break;
                    case FileType.GZip:
                        extension = "gz";
                        break;
                    case FileType.Tar:
                        extension = "tar";
                        break;
                    case FileType.TarGz:
                        extension = "tar.gz";
                        break;
                    case FileType.Zip:
                        if (!NetModuleCache.ZipValid(downloadedFile, out string? invalidReason, null))
                        {
                            log.Debug($"{url} is not a valid ZIP file: {invalidReason}");
                            File.Delete(downloadedFile);
                            throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");
                        }
                        extension = "zip";
                        break;
                    default:
                        extension = "ckan-package";
                        break;
                }

                var destName = $"netkan-{identifier}.{extension}";

                try
                {
                    return _cache.Store(primaryUrl, downloadedFile,
                                        destName, move: true);
                }
                catch (IOException exc)
                {
                    // If cache is full, don't also fill /tmp
                    log.Debug($"Failed to store to cache: {exc.Message}");
                    File.Delete(downloadedFile);
                    throw;
                }
            }
        }

        public string? DownloadText(Uri     url,
                                    string? authToken = null,
                                    string? mimeType  = null)
            => _stringCache.GetOrAdd(url.OriginalString,
                                     () => Net.DownloadText(url,
                                                            _userAgent, authToken, mimeType,
                                                            10000),
                                     DateTimeOffset.Now + stringCacheLifetime);

        public IEnumerable<Uri> RequestedURLs => _requestedURLs;
        public void ClearRequestedURLs()
        {
            _requestedURLs?.Clear();
        }

        public Uri? ResolveRedirect(Uri url, string? userAgent)
            => Net.ResolveRedirect(url, userAgent);

        private readonly NetFileCache _cache;
        private readonly string?      _userAgent;
        private readonly HashSet<Uri> _requestedURLs  = new HashSet<Uri>();
        private readonly bool         _overwriteCache = false;
        private readonly IAppCache    _stringCache    = new CachingService();

        // Re-use string value URLs within 15 minutes
        private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);

        private static readonly ILog log = LogManager.GetLogger(typeof(CachingHttpService));
    }
}

1	using System;
2	using System.Collections.Generic;
3	using System.IO;
4
5	using LazyCache;
6	using log4net;
7
8	using CKAN.IO;
9	using CKAN.NetKAN.Model;
10
11	namespace CKAN.NetKAN.Services
12	{
13	internal sealed class CachingHttpService : IHttpService
14	{
15	public CachingHttpService(NetFileCache cache,	2✔
16	bool overwrite = false,
17	string? userAgent = null)
18	{	2✔
19	_cache = cache;	2✔
20	_userAgent = userAgent;	2✔
21	_overwriteCache = overwrite;	2✔
22	}	2✔
23
24	public string? DownloadModule(Metadata metadata)
25	{	×
26	if (metadata.Download == null)	×
27	{	×
28	return null;	×
29	}
30	try
31	{	×
32	return DownloadPackage(metadata.Download[0],	×
33	metadata.Identifier,
34	metadata.ReleaseDate);
35	}
NEW 36	catch	×
37	{	×
38	var fallback = metadata.FallbackDownload;	×
39	if (fallback == null)	×
40	{	×
41	throw;	×
42	}
43	else
44	{	×
45	log.InfoFormat("Trying fallback URL: {0}", fallback);	×
46	return DownloadPackage(fallback,	×
47	metadata.Identifier,
48	metadata.ReleaseDate,
49	metadata.Download[0]);
50	}
51	}
52	}	×
53
54	private string DownloadPackage(Uri url,
55	string identifier,
56	DateTime? updated,
57	Uri? primaryUrl = null)
58	{	×
59	if (primaryUrl == null)	×
60	{	×
61	primaryUrl = url;	×
62	}	×
63	if (_overwriteCache && !_requestedURLs.Contains(url))	×
64	{	×
65	// Discard cached file if command line says so,
66	// but only the first time in each run
67	_cache.Remove(url);	×
68	}	×
69
70	_requestedURLs.Add(url);	×
71
72	var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);	×
73
74	if (cachedFile != null && !string.IsNullOrWhiteSpace(cachedFile))	×
75	{	×
76	return cachedFile;	×
77	}
78	else
79	{	×
80	var downloadedFile = Net.Download(url, _userAgent,	×
81	_cache.GetInProgressFileName(url,
82	$"netkan-{identifier}")
83	.FullName);
84
85	string extension;
86
87	switch (FileIdentifier.IdentifyFile(downloadedFile))	×
88	{
89	case FileType.ASCII:
90	extension = "txt";	×
91	break;	×
92	case FileType.GZip:
93	extension = "gz";	×
94	break;	×
95	case FileType.Tar:
96	extension = "tar";	×
97	break;	×
98	case FileType.TarGz:
99	extension = "tar.gz";	×
100	break;	×
101	case FileType.Zip:
102	if (!NetModuleCache.ZipValid(downloadedFile, out string? invalidReason, null))	×
103	{	×
104	log.Debug($"{url} is not a valid ZIP file: {invalidReason}");	×
105	File.Delete(downloadedFile);	×
106	throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");	×
107	}
108	extension = "zip";	×
109	break;	×
110	default:
111	extension = "ckan-package";	×
112	break;	×
113	}
114
115	var destName = $"netkan-{identifier}.{extension}";	×
116
117	try
118	{	×
119	return _cache.Store(primaryUrl, downloadedFile,	×
120	destName, move: true);
121	}
122	catch (IOException exc)	×
123	{	×
124	// If cache is full, don't also fill /tmp
125	log.Debug($"Failed to store to cache: {exc.Message}");	×
126	File.Delete(downloadedFile);	×
127	throw;	×
128	}
129	}
130	}	×
131
132	public string? DownloadText(Uri url,
133	string? authToken = null,
134	string? mimeType = null)
135	=> _stringCache.GetOrAdd(url.OriginalString,	2✔
136	() => Net.DownloadText(url,	2✔
137	_userAgent, authToken, mimeType,
138	10000),
139	DateTimeOffset.Now + stringCacheLifetime);
140
141	public IEnumerable<Uri> RequestedURLs => _requestedURLs;	×
142	public void ClearRequestedURLs()
143	{	×
144	_requestedURLs?.Clear();	×
145	}	×
146
147	public Uri? ResolveRedirect(Uri url, string? userAgent)
148	=> Net.ResolveRedirect(url, userAgent);	×
149
150	private readonly NetFileCache _cache;
151	private readonly string? _userAgent;
152	private readonly HashSet<Uri> _requestedURLs = new HashSet<Uri>();	2✔
153	private readonly bool _overwriteCache = false;	2✔
154	private readonly IAppCache _stringCache = new CachingService();	2✔
155
156	// Re-use string value URLs within 15 minutes
157	private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);	2✔
158
159	private static readonly ILog log = LogManager.GetLogger(typeof(CachingHttpService));	2✔
160	}
161	}

KSP-CKAN / CKAN / 15834236157

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous