18959752962

Committed 31 Oct 2025 01:19AM UTC coverage: 85.329% (+3.5%) from 81.873%

Build # 18959752962

Build Type

Pull #4454

github

Committed by

HebaruSan

Commit Message

Build on Windows, upload multi-platform coverage

Pull Request Pull Request #4454: Build on Windows, upload multi-platform coverage

Run Details

2005 of 2167 branches covered (92.52%)

Branch coverage included in aggregate %.

3 of 3 new or added lines in 1 file covered. (100.0%)

27 existing lines in 19 files now uncovered.

11971 of 14212 relevant lines covered (84.23%)

1.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.33

/Core/Net/NetAsyncModulesDownloader.cs

using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Threading.Tasks;
using System.IO;
using System.Linq;
using System.Threading;
using System.Security.Cryptography;

using log4net;
using Autofac;

using CKAN.Configuration;
using CKAN.Extensions;

namespace CKAN
{
    /// <summary>
    /// Download lots of files at once!
    /// </summary>
    public class NetAsyncModulesDownloader : IDownloader
    {
        public event Action<CkanModule, long, long>? DownloadProgress;
        public event Action<ByteRateCounter>?        OverallDownloadProgress;
        public event Action<CkanModule, long, long>? StoreProgress;
        public event Action<CkanModule>?             OneComplete;
        public event Action?                         AllComplete;

        /// <summary>
        /// Returns a perfectly boring NetAsyncModulesDownloader.
        /// </summary>
        public NetAsyncModulesDownloader(IUser             user,
                                         NetModuleCache    cache,
                                         string?           userAgent   = null,
                                         CancellationToken cancelToken = default)
        {
            modules    = new List<CkanModule>();
            downloader = new NetAsyncDownloader(user, SHA256.Create, userAgent, cancelToken);
            // Schedule us to process each module on completion.
            downloader.onOneCompleted += ModuleDownloadComplete;
            downloader.TargetProgress += (target, remaining, total) =>
            {
                if (targetModules?[target].First() is CkanModule mod)
                {
                    DownloadProgress?.Invoke(mod, remaining, total);
                }
            };
            downloader.OverallProgress += brc => OverallDownloadProgress?.Invoke(brc);
            this.cache = cache;
            this.cancelToken = cancelToken;
        }

        internal NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(
                HashSet<CkanModule> group,
                string?[]           preferredHosts)
            => TargetFromModuleGroup(group, group.OrderBy(m => m.identifier).First(), preferredHosts);

        private NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup(
                HashSet<CkanModule> group,
                CkanModule          first,
                string?[]           preferredHosts)
            => new NetAsyncDownloader.DownloadTargetFile(
                group.SelectMany(mod => mod.download ?? Enumerable.Empty<Uri>())
                     .Concat(group.Select(mod => mod.InternetArchiveDownload)
                                  .OfType<Uri>()
                                  .OrderBy(uri => uri.ToString()))
                     .Distinct()
                     .OrderBy(u => u,
                              new PreferredHostUriComparer(preferredHosts))
                     .ToList(),
                cache.GetInProgressFileName(first)?.FullName,
                first.download_size,
                string.IsNullOrEmpty(first.download_content_type)
                    ? defaultMimeType
                    : $"{first.download_content_type};q=1.0,{defaultMimeType};q=0.9");

        /// <summary>
        /// <see cref="IDownloader.DownloadModules"/>
        /// </summary>
        public void DownloadModules(IEnumerable<CkanModule> modules)
        {
            var activeURLs = this.modules.SelectMany(m => m.download ?? Enumerable.Empty<Uri>())
                                         .OfType<Uri>()
                                         .ToHashSet();
            var moduleGroups = CkanModule.GroupByDownloads(modules);
            // Make sure we have enough space to download and cache
            cache.CheckFreeSpace(moduleGroups.Sum(grp => grp.First().download_size));
            // Add all the requested modules
            this.modules.AddRange(moduleGroups.SelectMany(grp => grp));

            var preferredHosts = ServiceLocator.Container.Resolve<IConfiguration>().PreferredHosts;
            targetModules = moduleGroups
                // Skip any group that already has a URL in progress
                .Where(grp => grp.All(mod => mod.download?.All(dlUri => !activeURLs.Contains(dlUri)) ?? false))
                // Each group gets one target containing all the URLs
                .ToDictionary(grp => TargetFromModuleGroup(grp, preferredHosts),
                              grp => grp.ToArray());
            try
            {
                // Start the downloads!
                downloader.DownloadAndWait(targetModules.Keys);
                this.modules.Clear();
                targetModules.Clear();
                AllComplete?.Invoke();
            }
            catch (DownloadErrorsKraken kraken)
            {
                // Associate the errors with the affected modules
                var exc = new ModuleDownloadErrorsKraken(
                    kraken.Exceptions
                          .SelectMany(kvp => targetModules[kvp.Key]
                                             .Select(m => new KeyValuePair<CkanModule, Exception>(
                                                              m, kvp.Value.GetBaseException())))
                          .ToList());
                // Clear this.modules because we're done with these
                this.modules.Clear();
                targetModules.Clear();
                throw exc;
            }
        }

        public IEnumerable<CkanModule> ModulesAsTheyFinish(IReadOnlyCollection<CkanModule> cached,
                                                           IReadOnlyCollection<CkanModule> toDownload)
        {
            var (dlTask, blockingQueue) = DownloadsCollection(toDownload);
            return ModulesAsTheyFinish(cached, dlTask, blockingQueue);
        }

        private static IEnumerable<CkanModule> ModulesAsTheyFinish(IReadOnlyCollection<CkanModule> cached,
                                                                   Task                            dlTask,
                                                                   BlockingCollection<CkanModule>  blockingQueue)
        {
            foreach (var m in cached)
            {
                yield return m;
            }
            foreach (var m in blockingQueue.GetConsumingEnumerable())
            {
                yield return m;
            }
            blockingQueue.Dispose();
            try
            {
                dlTask.Wait();
                if (dlTask.Exception is AggregateException agExc)
                {
                    agExc.RethrowInner();
                }
            }
            catch (AggregateException agExc)
            {
                agExc.RethrowInner();
            }
        }

        private (Task dlTask, BlockingCollection<CkanModule> blockingQueue) DownloadsCollection(IReadOnlyCollection<CkanModule> toDownload)
        {
            var blockingQueue = new BlockingCollection<CkanModule>(new ConcurrentQueue<CkanModule>());
            Action<CkanModule> oneComplete = m => blockingQueue.Add(m);
            OneComplete += oneComplete;
            return (Task.Run(() => DownloadModules(toDownload))
                        .ContinueWith(t =>
                                      {
                                          blockingQueue.CompleteAdding();
                                          OneComplete -= oneComplete;
                                          if (t.Exception is AggregateException agExc)
                                          {
                                              agExc.RethrowInner();
                                          }
                                      }),
                    blockingQueue);
        }

        private void ModuleDownloadComplete(NetAsyncDownloader.DownloadTarget target,
                                            Exception?                        error,
                                            string?                           etag,
                                            string?                           sha256)
        {
            if (target is NetAsyncDownloader.DownloadTargetFile fileTarget && targetModules != null)
            {
                var url      = fileTarget.urls.First();
                var filename = fileTarget.filename;

                log.DebugFormat("Received download completion: {0}, {1}, {2}",
                                url, filename, error?.Message);
                if (error != null)
                {
                    // If there was an error in DOWNLOADING, keep the file so we can retry it later
                    log.Info(error.Message);
                }
                else
                {
                    // Cache if this download succeeded
                    CkanModule? module = null;
                    try
                    {
                        var completedMods = targetModules[fileTarget];
                        module = completedMods.First();

                        // Check hash if defined in module
                        if (module.download_hash?.sha256 != null
                            && sha256 != module.download_hash.sha256)
                        {
                            throw new InvalidModuleFileKraken(
                                module, filename,
                                string.Format(Properties.Resources.NetModuleCacheMismatchSHA256,
                                              module, filename,
                                              sha256, module.download_hash.sha256));
                        }

                        User.RaiseMessage(Properties.Resources.NetAsyncDownloaderValidating,
                                          module.name);
                        var fileSize = new FileInfo(filename).Length;
                        cache.Store(module, filename,
                                    new ProgressImmediate<long>(bytes => StoreProgress?.Invoke(module,
                                                                                               fileSize - bytes,
                                                                                               fileSize)),
                                    module.StandardName(),
                                    false,
                                    cancelToken);
                        File.Delete(filename);
                        foreach (var m in completedMods)
                        {
                            OneComplete?.Invoke(m);
                        }
                    }
                    catch (InvalidModuleFileKraken)
                    {
                        if (module != null)
                        {
                            // Finish out the progress bar
                            StoreProgress?.Invoke(module, 0, 100);
                        }
                        // If there was an error in STORING, delete the file so we can try it from scratch later
                        File.Delete(filename);

                        // Tell downloader there is a problem with this file
                        throw;
                    }
                    catch (OperationCanceledException exc)
                    {
                        log.WarnFormat("Cancellation token threw, validation incomplete: {0}", filename);
                        User.RaiseMessage("{0}", exc.Message);
                        if (module != null)
                        {
                            // Finish out the progress bar
                            StoreProgress?.Invoke(module, 0, 100);
                        }
                        // Don't delete because there might be nothing wrong
                    }
                    catch (FileNotFoundException e)
                    {
                        log.WarnFormat("cache.Store(): FileNotFoundException: {0}", e.Message);
                    }
                    catch (InvalidOperationException)
                    {
                        log.WarnFormat("No module found for completed URL: {0}", url);
                    }
                }
            }
        }

        private static readonly ILog log = LogManager.GetLogger(typeof(NetAsyncModulesDownloader));

        private const    string                  defaultMimeType = "application/octet-stream";

        private readonly List<CkanModule>         modules;
        private          Dictionary<NetAsyncDownloader.DownloadTarget, CkanModule[]>? targetModules;
        private readonly NetAsyncDownloader       downloader;
        private          IUser                    User => downloader.User;
        private readonly NetModuleCache           cache;
        private readonly CancellationToken        cancelToken;
    }
}

1	using System;
2	using System.Collections.Generic;
3	using System.Collections.Concurrent;
4	using System.Threading.Tasks;
5	using System.IO;
6	using System.Linq;
7	using System.Threading;
8	using System.Security.Cryptography;
9
10	using log4net;
11	using Autofac;
12
13	using CKAN.Configuration;
14	using CKAN.Extensions;
15
16	namespace CKAN
17	{
18	/// <summary>
19	/// Download lots of files at once!
20	/// </summary>
21	public class NetAsyncModulesDownloader : IDownloader
22	{
23	public event Action<CkanModule, long, long>? DownloadProgress;
24	public event Action<ByteRateCounter>? OverallDownloadProgress;
25	public event Action<CkanModule, long, long>? StoreProgress;
26	public event Action<CkanModule>? OneComplete;
27	public event Action? AllComplete;
28
29	/// <summary>
30	/// Returns a perfectly boring NetAsyncModulesDownloader.
31	/// </summary>
32	public NetAsyncModulesDownloader(IUser user,	2✔
33	NetModuleCache cache,
34	string? userAgent = null,
35	CancellationToken cancelToken = default)
36	{	2✔
37	modules = new List<CkanModule>();	2✔
38	downloader = new NetAsyncDownloader(user, SHA256.Create, userAgent, cancelToken);	2✔
39	// Schedule us to process each module on completion.
40	downloader.onOneCompleted += ModuleDownloadComplete;	2✔
41	downloader.TargetProgress += (target, remaining, total) =>	2✔
42	{	2✔
43	if (targetModules?[target].First() is CkanModule mod)	2✔
44	{	2✔
45	DownloadProgress?.Invoke(mod, remaining, total);	2✔
46	}	2✔
47	};	2✔
48	downloader.OverallProgress += brc => OverallDownloadProgress?.Invoke(brc);	2✔
49	this.cache = cache;	2✔
50	this.cancelToken = cancelToken;	2✔
51	}	2✔
52
53	internal NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(
54	HashSet<CkanModule> group,
55	string?[] preferredHosts)
56	=> TargetFromModuleGroup(group, group.OrderBy(m => m.identifier).First(), preferredHosts);	2✔
57
58	private NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup(
59	HashSet<CkanModule> group,
60	CkanModule first,
61	string?[] preferredHosts)
62	=> new NetAsyncDownloader.DownloadTargetFile(	2✔
63	group.SelectMany(mod => mod.download ?? Enumerable.Empty<Uri>())	2✔
64	.Concat(group.Select(mod => mod.InternetArchiveDownload)	2✔
65	.OfType<Uri>()
66	.OrderBy(uri => uri.ToString()))	2✔
67	.Distinct()
68	.OrderBy(u => u,	2✔
69	new PreferredHostUriComparer(preferredHosts))
70	.ToList(),
71	cache.GetInProgressFileName(first)?.FullName,
72	first.download_size,
73	string.IsNullOrEmpty(first.download_content_type)
74	? defaultMimeType
75	: $"{first.download_content_type};q=1.0,{defaultMimeType};q=0.9");
76
77	/// <summary>
78	/// <see cref="IDownloader.DownloadModules"/>
79	/// </summary>
80	public void DownloadModules(IEnumerable<CkanModule> modules)
81	{	2✔
UNCOV 82	var activeURLs = this.modules.SelectMany(m => m.download ?? Enumerable.Empty<Uri>())	×
83	.OfType<Uri>()
84	.ToHashSet();
85	var moduleGroups = CkanModule.GroupByDownloads(modules);	2✔
86	// Make sure we have enough space to download and cache
87	cache.CheckFreeSpace(moduleGroups.Sum(grp => grp.First().download_size));	2✔
88	// Add all the requested modules
89	this.modules.AddRange(moduleGroups.SelectMany(grp => grp));	2✔
90
91	var preferredHosts = ServiceLocator.Container.Resolve<IConfiguration>().PreferredHosts;	2✔
92	targetModules = moduleGroups	2✔
93	// Skip any group that already has a URL in progress
94	.Where(grp => grp.All(mod => mod.download?.All(dlUri => !activeURLs.Contains(dlUri)) ?? false))	2✔
95	// Each group gets one target containing all the URLs
96	.ToDictionary(grp => TargetFromModuleGroup(grp, preferredHosts),	2✔
97	grp => grp.ToArray());	2✔
98	try
99	{	2✔
100	// Start the downloads!
101	downloader.DownloadAndWait(targetModules.Keys);	2✔
102	this.modules.Clear();	2✔
103	targetModules.Clear();	2✔
104	AllComplete?.Invoke();	2✔
105	}	2✔
106	catch (DownloadErrorsKraken kraken)	2✔
107	{	2✔
108	// Associate the errors with the affected modules
109	var exc = new ModuleDownloadErrorsKraken(	2✔
110	kraken.Exceptions
111	.SelectMany(kvp => targetModules[kvp.Key]	2✔
112	.Select(m => new KeyValuePair<CkanModule, Exception>(	2✔
113	m, kvp.Value.GetBaseException())))
114	.ToList());
115	// Clear this.modules because we're done with these
116	this.modules.Clear();	2✔
117	targetModules.Clear();	2✔
118	throw exc;	2✔
119	}
120	}	2✔
121
122	public IEnumerable<CkanModule> ModulesAsTheyFinish(IReadOnlyCollection<CkanModule> cached,
123	IReadOnlyCollection<CkanModule> toDownload)
124	{	2✔
125	var (dlTask, blockingQueue) = DownloadsCollection(toDownload);	2✔
126	return ModulesAsTheyFinish(cached, dlTask, blockingQueue);	2✔
127	}	2✔
128
129	private static IEnumerable<CkanModule> ModulesAsTheyFinish(IReadOnlyCollection<CkanModule> cached,
130	Task dlTask,
131	BlockingCollection<CkanModule> blockingQueue)
132	{	2✔
133	foreach (var m in cached)	6✔
134	{	×
135	yield return m;	×
136	}	×
137	foreach (var m in blockingQueue.GetConsumingEnumerable())	8✔
138	{	2✔
139	yield return m;	2✔
140	}	2✔
141	blockingQueue.Dispose();	2✔
142	try
143	{	2✔
144	dlTask.Wait();	2✔
145	if (dlTask.Exception is AggregateException agExc)	2✔
146	{	×
147	agExc.RethrowInner();	×
148	}	×
149	}	2✔
150	catch (AggregateException agExc)	2✔
151	{	2✔
152	agExc.RethrowInner();	2✔
153	}	×
154	}	2✔
155
156	private (Task dlTask, BlockingCollection<CkanModule> blockingQueue) DownloadsCollection(IReadOnlyCollection<CkanModule> toDownload)
157	{	2✔
158	var blockingQueue = new BlockingCollection<CkanModule>(new ConcurrentQueue<CkanModule>());	2✔
159	Action<CkanModule> oneComplete = m => blockingQueue.Add(m);	2✔
160	OneComplete += oneComplete;	2✔
161	return (Task.Run(() => DownloadModules(toDownload))	2✔
162	.ContinueWith(t =>
163	{	2✔
164	blockingQueue.CompleteAdding();	2✔
165	OneComplete -= oneComplete;	2✔
166	if (t.Exception is AggregateException agExc)	2✔
167	{	2✔
168	agExc.RethrowInner();	2✔
169	}	×
170	}),	2✔
171	blockingQueue);
172	}	2✔
173
174	private void ModuleDownloadComplete(NetAsyncDownloader.DownloadTarget target,
175	Exception? error,
176	string? etag,
177	string? sha256)
178	{	2✔
179	if (target is NetAsyncDownloader.DownloadTargetFile fileTarget && targetModules != null)	2✔
180	{	2✔
181	var url = fileTarget.urls.First();	2✔
182	var filename = fileTarget.filename;	2✔
183
184	log.DebugFormat("Received download completion: {0}, {1}, {2}",	2✔
185	url, filename, error?.Message);
186	if (error != null)	2✔
187	{	2✔
188	// If there was an error in DOWNLOADING, keep the file so we can retry it later
189	log.Info(error.Message);	2✔
190	}	2✔
191	else
192	{	2✔
193	// Cache if this download succeeded
194	CkanModule? module = null;	2✔
195	try
196	{	2✔
197	var completedMods = targetModules[fileTarget];	2✔
198	module = completedMods.First();	2✔
199
200	// Check hash if defined in module
201	if (module.download_hash?.sha256 != null	2✔
202	&& sha256 != module.download_hash.sha256)
203	{	×
204	throw new InvalidModuleFileKraken(	×
205	module, filename,
206	string.Format(Properties.Resources.NetModuleCacheMismatchSHA256,
207	module, filename,
208	sha256, module.download_hash.sha256));
209	}
210
211	User.RaiseMessage(Properties.Resources.NetAsyncDownloaderValidating,	2✔
212	module.name);
213	var fileSize = new FileInfo(filename).Length;	2✔
214	cache.Store(module, filename,	2✔
215	new ProgressImmediate<long>(bytes => StoreProgress?.Invoke(module,	2✔
216	fileSize - bytes,
217	fileSize)),
218	module.StandardName(),
219	false,
220	cancelToken);
221	File.Delete(filename);	2✔
222	foreach (var m in completedMods)	8✔
223	{	2✔
224	OneComplete?.Invoke(m);	2✔
225	}	2✔
226	}	2✔
227	catch (InvalidModuleFileKraken)	2✔
228	{	2✔
229	if (module != null)	2✔
230	{	2✔
231	// Finish out the progress bar
232	StoreProgress?.Invoke(module, 0, 100);	2✔
233	}	2✔
234	// If there was an error in STORING, delete the file so we can try it from scratch later
235	File.Delete(filename);	2✔
236
237	// Tell downloader there is a problem with this file
238	throw;	2✔
239	}
240	catch (OperationCanceledException exc)	×
241	{	×
242	log.WarnFormat("Cancellation token threw, validation incomplete: {0}", filename);	×
243	User.RaiseMessage("{0}", exc.Message);	×
244	if (module != null)	×
245	{	×
246	// Finish out the progress bar
247	StoreProgress?.Invoke(module, 0, 100);	×
248	}	×
249	// Don't delete because there might be nothing wrong
250	}	×
251	catch (FileNotFoundException e)	×
252	{	×
253	log.WarnFormat("cache.Store(): FileNotFoundException: {0}", e.Message);	×
254	}	×
255	catch (InvalidOperationException)	×
256	{	×
257	log.WarnFormat("No module found for completed URL: {0}", url);	×
258	}	×
259	}	2✔
260	}	2✔
261	}	2✔
262
263	private static readonly ILog log = LogManager.GetLogger(typeof(NetAsyncModulesDownloader));	2✔
264
265	private const string defaultMimeType = "application/octet-stream";
266
267	private readonly List<CkanModule> modules;
268	private Dictionary<NetAsyncDownloader.DownloadTarget, CkanModule[]>? targetModules;
269	private readonly NetAsyncDownloader downloader;
270	private IUser User => downloader.User;	2✔
271	private readonly NetModuleCache cache;
272	private readonly CancellationToken cancelToken;
273	}
274	}

KSP-CKAN / CKAN / 18959752962

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous