15655026327

Committed 14 Jun 2025 06:40PM UTC coverage: 27.152% (-3.2%) from 30.329%

Build # 15655026327

Build Type

Pull #4392

github

Committed by

web-flow

Commit Message

Merge bb1dadfef into 1b4a54286

Pull Request Pull Request #4392: Writethrough when saving files, add Netkan tests

Run Details

3703 of 12085 branches covered (30.64%)

Branch coverage included in aggregate %.

19 of 32 new or added lines in 18 files covered. (59.38%)

9 existing lines in 7 files now uncovered.

8041 of 31168 relevant lines covered (25.8%)

0.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

24.82

/Netkan/Services/CachingHttpService.cs

using System;
using System.Collections.Generic;
using System.IO;

using log4net;

using CKAN.IO;
using CKAN.NetKAN.Model;

namespace CKAN.NetKAN.Services
{
    internal sealed class CachingHttpService : IHttpService
    {
        private readonly NetFileCache _cache;
        private readonly string?      _userAgent;
        private readonly HashSet<Uri> _requestedURLs  = new HashSet<Uri>();
        private readonly bool         _overwriteCache = false;
        private readonly Dictionary<Uri, StringCacheEntry> _stringCache = new Dictionary<Uri, StringCacheEntry>();

        // Re-use string value URLs within 15 minutes
        private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);

        public CachingHttpService(NetFileCache cache, bool overwrite = false, string? userAgent = null)
        {
            _cache          = cache;
            _userAgent      = userAgent;
            _overwriteCache = overwrite;
        }

        public string? DownloadModule(Metadata metadata)
        {
            if (metadata.Download == null)
            {
                return null;
            }
            try
            {
                return DownloadPackage(metadata.Download[0],
                                       metadata.Identifier,
                                       metadata.ReleaseDate);
            }
            catch (Exception)
            {
                var fallback = metadata.FallbackDownload;
                if (fallback == null)
                {
                    throw;
                }
                else
                {
                    log.InfoFormat("Trying fallback URL: {0}", fallback);
                    return DownloadPackage(fallback,
                                           metadata.Identifier,
                                           metadata.ReleaseDate,
                                           metadata.Download[0]);
                }
            }
        }

        private string DownloadPackage(Uri       url,
                                       string    identifier,
                                       DateTime? updated,
                                       Uri?      primaryUrl = null)
        {
            if (primaryUrl == null)
            {
                primaryUrl = url;
            }
            if (_overwriteCache && !_requestedURLs.Contains(url))
            {
                // Discard cached file if command line says so,
                // but only the first time in each run
                _cache.Remove(url);
            }

            _requestedURLs.Add(url);

            var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);

            if (cachedFile != null && !string.IsNullOrWhiteSpace(cachedFile))
            {
                return cachedFile;
            }
            else
            {
                var downloadedFile = Net.Download(url, _userAgent,
                                                  _cache.GetInProgressFileName(url,
                                                                               $"netkan-{identifier}")
                                                        .FullName);

                string extension;

                switch (FileIdentifier.IdentifyFile(downloadedFile))
                {
                    case FileType.ASCII:
                        extension = "txt";
                        break;
                    case FileType.GZip:
                        extension = "gz";
                        break;
                    case FileType.Tar:
                        extension = "tar";
                        break;
                    case FileType.TarGz:
                        extension = "tar.gz";
                        break;
                    case FileType.Zip:
                        if (!NetModuleCache.ZipValid(downloadedFile, out string? invalidReason, null))
                        {
                            log.Debug($"{url} is not a valid ZIP file: {invalidReason}");
                            File.Delete(downloadedFile);
                            throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");
                        }
                        extension = "zip";
                        break;
                    default:
                        extension = "ckan-package";
                        break;
                }

                var destName = $"netkan-{identifier}.{extension}";

                try
                {
                    return _cache.Store(primaryUrl, downloadedFile,
                                        destName, move: true);
                }
                catch (IOException exc)
                {
                    // If cache is full, don't also fill /tmp
                    log.Debug($"Failed to store to cache: {exc.Message}");
                    File.Delete(downloadedFile);
                    throw;
                }
            }
        }

        public string? DownloadText(Uri url)
        {
            return TryGetCached(url, () => Net.DownloadText(url, _userAgent, timeout: 10000));
        }
        public string? DownloadText(Uri url, string? authToken, string? mimeType = null)
        {
            return TryGetCached(url, () => Net.DownloadText(url, _userAgent, authToken, mimeType, 10000));
        }

        private string? TryGetCached(Uri url, Func<string?> uncached)
        {
            if (_stringCache.TryGetValue(url, out StringCacheEntry? entry))
            {
                if (DateTime.Now - entry.Timestamp < stringCacheLifetime)
                {
                    // Re-use recent cached request of this URL
                    return entry.Value;
                }
                else
                {
                    // Too old, purge it
                    _stringCache.Remove(url);
                }
            }
            var val = uncached();
            if (val is not null)
            {
                _stringCache.Add(url, new StringCacheEntry(val, DateTime.Now));
            }
            return val;
        }

        public IEnumerable<Uri> RequestedURLs => _requestedURLs;
        public void ClearRequestedURLs()
        {
            _requestedURLs?.Clear();
        }

        public Uri? ResolveRedirect(Uri url, string? userAgent)
            => Net.ResolveRedirect(url, userAgent);

        private static readonly ILog log = LogManager.GetLogger(typeof(CachingHttpService));
    }

    public class StringCacheEntry
    {
        public StringCacheEntry(string val, DateTime stamp)
        {
            Value     = val;
            Timestamp = stamp;
        }

        public string   Value;
        public DateTime Timestamp;
    }

}

1	using System;
2	using System.Collections.Generic;
3	using System.IO;
4
5	using log4net;
6
7	using CKAN.IO;
8	using CKAN.NetKAN.Model;
9
10	namespace CKAN.NetKAN.Services
11	{
12	internal sealed class CachingHttpService : IHttpService
13	{
14	private readonly NetFileCache _cache;
15	private readonly string? _userAgent;
16	private readonly HashSet<Uri> _requestedURLs = new HashSet<Uri>();	2✔
17	private readonly bool _overwriteCache = false;	2✔
18	private readonly Dictionary<Uri, StringCacheEntry> _stringCache = new Dictionary<Uri, StringCacheEntry>();	2✔
19
20	// Re-use string value URLs within 15 minutes
21	private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);	2✔
22
23	public CachingHttpService(NetFileCache cache, bool overwrite = false, string? userAgent = null)	2✔
24	{	2✔
25	_cache = cache;	2✔
26	_userAgent = userAgent;	2✔
27	_overwriteCache = overwrite;	2✔
28	}	2✔
29
30	public string? DownloadModule(Metadata metadata)
31	{	×
32	if (metadata.Download == null)	×
33	{	×
34	return null;	×
35	}
36	try
37	{	×
38	return DownloadPackage(metadata.Download[0],	×
39	metadata.Identifier,
40	metadata.ReleaseDate);
41	}
42	catch (Exception)	×
43	{	×
44	var fallback = metadata.FallbackDownload;	×
45	if (fallback == null)	×
46	{	×
47	throw;	×
48	}
49	else
50	{	×
51	log.InfoFormat("Trying fallback URL: {0}", fallback);	×
52	return DownloadPackage(fallback,	×
53	metadata.Identifier,
54	metadata.ReleaseDate,
55	metadata.Download[0]);
56	}
57	}
58	}	×
59
60	private string DownloadPackage(Uri url,
61	string identifier,
62	DateTime? updated,
63	Uri? primaryUrl = null)
64	{	×
65	if (primaryUrl == null)	×
66	{	×
67	primaryUrl = url;	×
68	}	×
69	if (_overwriteCache && !_requestedURLs.Contains(url))	×
70	{	×
71	// Discard cached file if command line says so,
72	// but only the first time in each run
73	_cache.Remove(url);	×
74	}	×
75
76	_requestedURLs.Add(url);	×
77
78	var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);	×
79
80	if (cachedFile != null && !string.IsNullOrWhiteSpace(cachedFile))	×
81	{	×
82	return cachedFile;	×
83	}
84	else
85	{	×
86	var downloadedFile = Net.Download(url, _userAgent,	×
87	_cache.GetInProgressFileName(url,
88	$"netkan-{identifier}")
89	.FullName);
90
91	string extension;
92
93	switch (FileIdentifier.IdentifyFile(downloadedFile))	×
94	{
95	case FileType.ASCII:
96	extension = "txt";	×
97	break;	×
98	case FileType.GZip:
99	extension = "gz";	×
100	break;	×
101	case FileType.Tar:
102	extension = "tar";	×
103	break;	×
104	case FileType.TarGz:
105	extension = "tar.gz";	×
106	break;	×
107	case FileType.Zip:
108	if (!NetModuleCache.ZipValid(downloadedFile, out string? invalidReason, null))	×
109	{	×
110	log.Debug($"{url} is not a valid ZIP file: {invalidReason}");	×
111	File.Delete(downloadedFile);	×
112	throw new Kraken($"{url} is not a valid ZIP file: {invalidReason}");	×
113	}
114	extension = "zip";	×
115	break;	×
116	default:
117	extension = "ckan-package";	×
118	break;	×
119	}
120
121	var destName = $"netkan-{identifier}.{extension}";	×
122
123	try
124	{	×
125	return _cache.Store(primaryUrl, downloadedFile,	×
126	destName, move: true);
127	}
128	catch (IOException exc)	×
129	{	×
130	// If cache is full, don't also fill /tmp
131	log.Debug($"Failed to store to cache: {exc.Message}");	×
132	File.Delete(downloadedFile);	×
133	throw;	×
134	}
135	}
136	}	×
137
138	public string? DownloadText(Uri url)
139	{	2✔
140	return TryGetCached(url, () => Net.DownloadText(url, _userAgent, timeout: 10000));	2✔
141	}	2✔
142	public string? DownloadText(Uri url, string? authToken, string? mimeType = null)
143	{	2✔
144	return TryGetCached(url, () => Net.DownloadText(url, _userAgent, authToken, mimeType, 10000));	2✔
145	}	2✔
146
147	private string? TryGetCached(Uri url, Func<string?> uncached)
148	{	2✔
149	if (_stringCache.TryGetValue(url, out StringCacheEntry? entry))	2!
150	{	×
151	if (DateTime.Now - entry.Timestamp < stringCacheLifetime)	×
152	{	×
153	// Re-use recent cached request of this URL
154	return entry.Value;	×
155	}
156	else
157	{	×
158	// Too old, purge it
159	_stringCache.Remove(url);	×
160	}	×
161	}	×
162	var val = uncached();	2✔
163	if (val is not null)	2!
164	{	2✔
165	_stringCache.Add(url, new StringCacheEntry(val, DateTime.Now));	2✔
166	}	2✔
167	return val;	2✔
168	}	2✔
169
170	public IEnumerable<Uri> RequestedURLs => _requestedURLs;	×
171	public void ClearRequestedURLs()
172	{	×
173	_requestedURLs?.Clear();	×
174	}	×
175
176	public Uri? ResolveRedirect(Uri url, string? userAgent)
NEW 177	=> Net.ResolveRedirect(url, userAgent);	×
178
179	private static readonly ILog log = LogManager.GetLogger(typeof(CachingHttpService));	2✔
180	}
181
182	public class StringCacheEntry
183	{
184	public StringCacheEntry(string val, DateTime stamp)	2✔
185	{	2✔
186	Value = val;	2✔
187	Timestamp = stamp;	2✔
188	}	2✔
189
190	public string Value;
191	public DateTime Timestamp;
192	}
193
194	}

KSP-CKAN / CKAN / 15655026327

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous