15833572481

Committed 23 Jun 2025 07:42PM UTC coverage: 42.239% (+0.1%) from 42.099%

Build # 15833572481

Build Type

push

github

Committed by

HebaruSan

Commit Message

Merge #4398 Exception handling revamp, parallel multi-host inflation

Run Details

3882 of 9479 branches covered (40.95%)

Branch coverage included in aggregate %.

48 of 137 new or added lines in 30 files covered. (35.04%)

12 existing lines in 6 files now uncovered.

8334 of 19442 relevant lines covered (42.87%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

55.38

/Core/Extensions/EnumerableExtensions.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Text.RegularExpressions;
using System.Runtime.ExceptionServices;

namespace CKAN.Extensions
{
    public static class EnumerableExtensions
    {
        public static ConcurrentDictionary<K, V> ToConcurrentDictionary<K, V>(this IEnumerable<KeyValuePair<K, V>> pairs) where K: class
            => new ConcurrentDictionary<K, V>(pairs);

        public static IEnumerable<T> AsParallelIf<T>(this IEnumerable<T> source,
                                                     bool                parallel)
            => parallel ? source.AsParallel()
                        : source;

        // https://stackoverflow.com/a/55591477/2422988
        public static ParallelQuery<T> WithProgress<T>(this ParallelQuery<T> source,
                                                       long                  totalCount,
                                                       IProgress<int>?       progress)
        {
            long count       = 0;
            int  prevPercent = -1;
            return progress == null
                ? source
                : source.Select(item =>
                {
                    var percent = (int)(100 * Interlocked.Increment(ref count) / totalCount);
                    if (percent > prevPercent)
                    {
                        progress.Report(percent);
                        prevPercent = percent;
                    }
                    return item;
                });
        }

        public static IEnumerable<T> Memoize<T>(this IEnumerable<T> source)
            => source is Memoized<T>
                   // Already memoized, don't wrap another layer
                   ? source
                   : new Memoized<T>(source);

        public static void RemoveWhere<K, V>(this Dictionary<K, V> source,
                                             Func<KeyValuePair<K, V>, bool> predicate) where K: class
        {
            var pairs = source.ToList();
            foreach (var kvp in pairs)
            {
                if (predicate(kvp))
                {
                    source.Remove(kvp.Key);
                }
            }
        }

        /// <summary>
        /// Sum a sequence of TimeSpans.
        /// Mysteriously not defined standardly.
        /// </summary>
        /// <param name="source">Sequence of TimeSpans to sum</param>
        /// <returns>
        /// Sum of the TimeSpans
        /// </returns>
        public static TimeSpan Sum(this IEnumerable<TimeSpan> source)
            => source.Aggregate(TimeSpan.Zero,
                                (a, b) => a + b);

        /// <summary>
        /// Select : SelectMany :: Zip : ZipMany
        /// </summary>
        /// <param name="seq1">Sequence from which to get first values</param>
        /// <param name="seq2">Sequence from which to get second values</param>
        /// <param name="func">Function to transform a value from each input sequence into a sequence of multiple outputs</param>
        /// <returns>Flattened sequence of values from func applies to seq1 and seq2</returns>
        public static IEnumerable<V> ZipMany<T, U, V>(this IEnumerable<T> seq1, IEnumerable<U> seq2, Func<T, U, IEnumerable<V>> func)
            => seq1.Zip(seq2, func).SelectMany(seqs => seqs);

        /// <summary>
        /// Zip a sequence with a sequence generated from the first sequence using the given function
        /// </summary>
        /// <typeparam name="T">Source sequence type</typeparam>
        /// <typeparam name="V">Type of elements returned by func</typeparam>
        /// <param name="source">Source sequence</param>
        /// <param name="func">Function to generate values of second sequence given source</param>
        /// <returns>Sequence of tuples containing pairs from each sequence</returns>
        public static IEnumerable<(T First, V Second)> ZipBy<T, V>(this IEnumerable<T> source, Func<IEnumerable<T>, IEnumerable<V>> func)
            => source.ToArray() is T[] array
                   ? array.Zip(func(array))
                   : Enumerable.Empty<(T First, V Second)>();

        /// <summary>
        /// Insert new elements between consecutive pairs of existing elements,
        /// preserving the original elements in order, and using null to
        /// represent the elements before the beginning and after the end.
        /// </summary>
        /// <param name="source">Sequence into which to inject</param>
        /// <param name="inBetween">Function to generate the new elements</param>
        /// <returns>Sequence with new elements in it</returns>
        public static IEnumerable<T> Inject<T>(this IEnumerable<T> source,
                                               Func<T?, T?, T>     inBetween)
            where T : class
        {
            using (var e = source.GetEnumerator())
            {
                if (e.MoveNext())
                {
                    yield return inBetween(null, e.Current);
                    yield return e.Current;
                    var prev = e.Current;
                    while (e.MoveNext())
                    {
                        yield return inBetween(prev, e.Current);
                        yield return e.Current;
                        prev = e.Current;
                    }
                    yield return inBetween(prev, null);
                }
                else
                {
                    yield return inBetween(null, null);
                }
            }
        }

        /// <summary>
        /// Poor man's PLINQ, a trivially parallelized SelectMany that
        /// runs one process per item in the source sequence.
        /// For short sequences and long-running functions,
        /// when you don't feel like fighting with Partitioner.Create
        /// over how many items should be in each partition.
        /// </summary>
        /// <param name="source">The sequence to process</param>
        /// <param name="func">The function to apply to each item in the sequence</param>
        /// <returns>Sequence of all values from the function</returns>
        public static IEnumerable<V> SelectManyTasks<T, V>(this ICollection<T>     source,
                                                           Func<T, IEnumerable<V>> func)
        {
            if (source.Count <= 1)
            {
                return source.SelectMany(func);
            }
            else
            {
                var tasks = source.Select(item => Task.Run(() => func(item).ToArray()))
                                  // Force non-lazy creation of tasks
                                  .ToArray();
                // Without this, later tasks don't finish if an earlier one throws
                Task.WaitAll(tasks);
                try
                {
                    return tasks.SelectMany(task => task.Result);
                }
                catch (AggregateException agExc) when (agExc is { InnerException: Exception exc })
                {
                    ExceptionDispatchInfo.Capture(exc).Throw();
                    throw;
                }
            }
        }

        /// <summary>
        /// Generate a sequence from a linked list
        /// </summary>
        /// <param name="start">The first node</param>
        /// <param name="getNext">Function to go from one node to the next</param>
        /// <returns>All the nodes in the list as a sequence</returns>
        public static IEnumerable<T> TraverseNodes<T>(this T start, Func<T, T?> getNext)
            where T : class
        {
            for (T? t = start; t != null; t = Utilities.DefaultIfThrows(() => getNext(t)))
            {
                yield return t;
            }
        }

        /// <summary>
        /// Try matching a regex against a series of strings and return the Match objects
        /// </summary>
        /// <param name="source">Sequence of strings to scan</param>
        /// <param name="pattern">Pattern to match</param>
        /// <returns>Sequence of Match objects</returns>
        public static IEnumerable<Match> WithMatches(this IEnumerable<string> source, Regex pattern)
            => source.Select(val => pattern.TryMatch(val, out Match? match) ? match : null)
                     .OfType<Match>();

        /// <summary>
        /// Apply a function to a sequence and handle any exceptions that are thrown
        /// </summary>
        /// <typeparam name="TSrc">Type of source sequence</typeparam>
        /// <typeparam name="TDest">Type of destination sequence</typeparam>
        /// <param name="source">Source sequence</param>
        /// <param name="func">Function to apply to each item</param>
        /// <param name="onThrow">Function to call if there's an exception</param>
        /// <returns>Sequence of return values of given function</returns>
        public static IEnumerable<TDest?> SelectWithCatch<TSrc, TDest>(this IEnumerable<TSrc>       source,
                                                                      Func<TSrc, TDest>             func,
                                                                      Func<TSrc, Exception, TDest?> onThrow)
                where TDest : class
            => source.Select(item => Utilities.DefaultIfThrows(()  => func(item),
                                                               exc => onThrow(item, exc)));

        /// <summary>
        /// Get a hash code for a sequence with a variable number of elements
        /// </summary>
        /// <typeparam name="T">Type of the elements in the sequence</typeparam>
        /// <param name="source">The sequence</param>
        /// <returns></returns>
        public static int ToSequenceHashCode<T>(this IEnumerable<T> source)
            => source.Aggregate(new HashCode(),
                                (hc, item) =>
                                {
                                    hc.Add(item);
                                    return hc;
                                },
                                hc => hc.ToHashCode());

        /// <summary>
        /// Accumulate a sequence of values based on a seed value and a function,
        /// similar to Aggregate but including intermediate values
        /// </summary>
        /// <typeparam name="TSource"></typeparam>
        /// <typeparam name="TResult"></typeparam>
        /// <param name="source">Input sequence</param>
        /// <param name="seed">First intermediate value, not included in return sequence</param>
        /// <param name="func">Function to transform a previous result and a next input sequence element into the next result</param>
        /// <returns></returns>
        public static IEnumerable<TResult> Accumulate<TSource, TResult>(this IEnumerable<TSource>       source,
                                                                        TResult                         seed,
                                                                        Func<TResult, TSource, TResult> func)
        {
            var result = seed;
            foreach (var item in source)
            {
                result = func(result, item);
                yield return result;
            }
        }
    }

    /// <summary>
    /// Memoized lazy evaluation in C#!
    /// From https://stackoverflow.com/a/12428250/2422988
    /// </summary>
    public class Memoized<T> : IEnumerable<T>
    {
        public Memoized(IEnumerable<T> source)
        {
            this.source = source;
        }

        IEnumerator IEnumerable.GetEnumerator()
            => GetEnumerator();

        public IEnumerator<T> GetEnumerator()
        {
            lock (gate)
            {
                if (isCacheComplete)
                {
                    return cache.GetEnumerator();
                }
                else
                {
                    enumerator ??= source.GetEnumerator();
                }
            }
            return GetMemoizingEnumerator();
        }

        private IEnumerator<T> GetMemoizingEnumerator()
        {
            for (int index = 0; TryGetItem(index, out T? item); ++index)
            {
                yield return item;
            }
        }

        private bool TryGetItem(int index,
                                out T item)
        {
            lock (gate)
            {
                if (enumerator is not null && !IsItemInCache(index))
                {
                    // The iteration may have completed while waiting for the lock
                    #nullable disable
                    if (isCacheComplete)
                    {
                        item = default;
                        return false;
                    }
                    if (!enumerator.MoveNext())
                    {
                        item = default;
                        isCacheComplete = true;
                        enumerator.Dispose();
                        return false;
                    }
                    #nullable enable
                    cache.Add(enumerator.Current);
                }
                item = cache[index];
                return true;
            }
        }

        private bool IsItemInCache(int index)
            => index < cache.Count;

        private readonly IEnumerable<T>  source;
        private          IEnumerator<T>? enumerator;
        private readonly List<T>         cache = new List<T>();
        private          bool            isCacheComplete;
        private readonly object          gate = new object();
    }
}

1	using System;
2	using System.Collections;
3	using System.Collections.Generic;
4	using System.Collections.Concurrent;
5	using System.Linq;
6	using System.Threading;
7	using System.Threading.Tasks;
8	using System.Text.RegularExpressions;
9	using System.Runtime.ExceptionServices;
10
11	namespace CKAN.Extensions
12	{
13	public static class EnumerableExtensions
14	{
15	public static ConcurrentDictionary<K, V> ToConcurrentDictionary<K, V>(this IEnumerable<KeyValuePair<K, V>> pairs) where K: class
16	=> new ConcurrentDictionary<K, V>(pairs);	2✔
17
18	public static IEnumerable<T> AsParallelIf<T>(this IEnumerable<T> source,
19	bool parallel)
20	=> parallel ? source.AsParallel()	2✔
21	: source;
22
23	// https://stackoverflow.com/a/55591477/2422988
24	public static ParallelQuery<T> WithProgress<T>(this ParallelQuery<T> source,
25	long totalCount,
26	IProgress<int>? progress)
27	{	2✔
28	long count = 0;	2✔
29	int prevPercent = -1;	2✔
30	return progress == null	2!
31	? source
32	: source.Select(item =>
33	{	×
34	var percent = (int)(100 * Interlocked.Increment(ref count) / totalCount);	×
35	if (percent > prevPercent)	×
36	{	×
37	progress.Report(percent);	×
38	prevPercent = percent;	×
39	}	×
40	return item;	×
41	});	×
42	}	2✔
43
44	public static IEnumerable<T> Memoize<T>(this IEnumerable<T> source)
45	=> source is Memoized<T>	2!
46	// Already memoized, don't wrap another layer
47	? source
48	: new Memoized<T>(source);
49
50	public static void RemoveWhere<K, V>(this Dictionary<K, V> source,
51	Func<KeyValuePair<K, V>, bool> predicate) where K: class
52	{	2✔
53	var pairs = source.ToList();	2✔
54	foreach (var kvp in pairs)	5✔
55	{	2✔
56	if (predicate(kvp))	2✔
57	{	2✔
58	source.Remove(kvp.Key);	2✔
59	}	2✔
60	}	2✔
61	}	2✔
62
63	/// <summary>
64	/// Sum a sequence of TimeSpans.
65	/// Mysteriously not defined standardly.
66	/// </summary>
67	/// <param name="source">Sequence of TimeSpans to sum</param>
68	/// <returns>
69	/// Sum of the TimeSpans
70	/// </returns>
71	public static TimeSpan Sum(this IEnumerable<TimeSpan> source)
72	=> source.Aggregate(TimeSpan.Zero,	×
73	(a, b) => a + b);	×
74
75	/// <summary>
76	/// Select : SelectMany :: Zip : ZipMany
77	/// </summary>
78	/// <param name="seq1">Sequence from which to get first values</param>
79	/// <param name="seq2">Sequence from which to get second values</param>
80	/// <param name="func">Function to transform a value from each input sequence into a sequence of multiple outputs</param>
81	/// <returns>Flattened sequence of values from func applies to seq1 and seq2</returns>
82	public static IEnumerable<V> ZipMany<T, U, V>(this IEnumerable<T> seq1, IEnumerable<U> seq2, Func<T, U, IEnumerable<V>> func)
83	=> seq1.Zip(seq2, func).SelectMany(seqs => seqs);	×
84
85	/// <summary>
86	/// Zip a sequence with a sequence generated from the first sequence using the given function
87	/// </summary>
88	/// <typeparam name="T">Source sequence type</typeparam>
89	/// <typeparam name="V">Type of elements returned by func</typeparam>
90	/// <param name="source">Source sequence</param>
91	/// <param name="func">Function to generate values of second sequence given source</param>
92	/// <returns>Sequence of tuples containing pairs from each sequence</returns>
93	public static IEnumerable<(T First, V Second)> ZipBy<T, V>(this IEnumerable<T> source, Func<IEnumerable<T>, IEnumerable<V>> func)
94	=> source.ToArray() is T[] array	2!
95	? array.Zip(func(array))
96	: Enumerable.Empty<(T First, V Second)>();
97
98	/// <summary>
99	/// Insert new elements between consecutive pairs of existing elements,
100	/// preserving the original elements in order, and using null to
101	/// represent the elements before the beginning and after the end.
102	/// </summary>
103	/// <param name="source">Sequence into which to inject</param>
104	/// <param name="inBetween">Function to generate the new elements</param>
105	/// <returns>Sequence with new elements in it</returns>
106	public static IEnumerable<T> Inject<T>(this IEnumerable<T> source,
107	Func<T?, T?, T> inBetween)
108	where T : class
109	{	×
110	using (var e = source.GetEnumerator())	×
111	{	×
112	if (e.MoveNext())	×
113	{	×
114	yield return inBetween(null, e.Current);	×
115	yield return e.Current;	×
116	var prev = e.Current;	×
117	while (e.MoveNext())	×
118	{	×
119	yield return inBetween(prev, e.Current);	×
120	yield return e.Current;	×
121	prev = e.Current;	×
122	}	×
123	yield return inBetween(prev, null);	×
124	}	×
125	else
126	{	×
127	yield return inBetween(null, null);	×
128	}	×
129	}	×
130	}	×
131
132	/// <summary>
133	/// Poor man's PLINQ, a trivially parallelized SelectMany that
134	/// runs one process per item in the source sequence.
135	/// For short sequences and long-running functions,
136	/// when you don't feel like fighting with Partitioner.Create
137	/// over how many items should be in each partition.
138	/// </summary>
139	/// <param name="source">The sequence to process</param>
140	/// <param name="func">The function to apply to each item in the sequence</param>
141	/// <returns>Sequence of all values from the function</returns>
142	public static IEnumerable<V> SelectManyTasks<T, V>(this ICollection<T> source,
143	Func<T, IEnumerable<V>> func)
NEW 144	{	×
NEW 145	if (source.Count <= 1)	×
NEW 146	{	×
NEW 147	return source.SelectMany(func);	×
148	}
149	else
NEW 150	{	×
NEW 151	var tasks = source.Select(item => Task.Run(() => func(item).ToArray()))	×
152	// Force non-lazy creation of tasks
153	.ToArray();
154	// Without this, later tasks don't finish if an earlier one throws
NEW 155	Task.WaitAll(tasks);	×
156	try
NEW 157	{	×
NEW 158	return tasks.SelectMany(task => task.Result);	×
159	}
NEW 160	catch (AggregateException agExc) when (agExc is { InnerException: Exception exc })	×
NEW 161	{	×
NEW 162	ExceptionDispatchInfo.Capture(exc).Throw();	×
NEW 163	throw;	×
164	}
165	}
NEW 166	}	×
167
168	/// <summary>
169	/// Generate a sequence from a linked list
170	/// </summary>
171	/// <param name="start">The first node</param>
172	/// <param name="getNext">Function to go from one node to the next</param>
173	/// <returns>All the nodes in the list as a sequence</returns>
174	public static IEnumerable<T> TraverseNodes<T>(this T start, Func<T, T?> getNext)
175	where T : class
176	{	2✔
177	for (T? t = start; t != null; t = Utilities.DefaultIfThrows(() => getNext(t)))	4✔
178	{	2✔
179	yield return t;	2✔
180	}	2✔
181	}	2✔
182
183	/// <summary>
184	/// Try matching a regex against a series of strings and return the Match objects
185	/// </summary>
186	/// <param name="source">Sequence of strings to scan</param>
187	/// <param name="pattern">Pattern to match</param>
188	/// <returns>Sequence of Match objects</returns>
189	public static IEnumerable<Match> WithMatches(this IEnumerable<string> source, Regex pattern)
190	=> source.Select(val => pattern.TryMatch(val, out Match? match) ? match : null)	×
191	.OfType<Match>();
192
193	/// <summary>
194	/// Apply a function to a sequence and handle any exceptions that are thrown
195	/// </summary>
196	/// <typeparam name="TSrc">Type of source sequence</typeparam>
197	/// <typeparam name="TDest">Type of destination sequence</typeparam>
198	/// <param name="source">Source sequence</param>
199	/// <param name="func">Function to apply to each item</param>
200	/// <param name="onThrow">Function to call if there's an exception</param>
201	/// <returns>Sequence of return values of given function</returns>
202	public static IEnumerable<TDest?> SelectWithCatch<TSrc, TDest>(this IEnumerable<TSrc> source,
203	Func<TSrc, TDest> func,
204	Func<TSrc, Exception, TDest?> onThrow)
205	where TDest : class
206	=> source.Select(item => Utilities.DefaultIfThrows(() => func(item),	2✔
207	exc => onThrow(item, exc)));	2✔
208
209	/// <summary>
210	/// Get a hash code for a sequence with a variable number of elements
211	/// </summary>
212	/// <typeparam name="T">Type of the elements in the sequence</typeparam>
213	/// <param name="source">The sequence</param>
214	/// <returns></returns>
215	public static int ToSequenceHashCode<T>(this IEnumerable<T> source)
216	=> source.Aggregate(new HashCode(),	×
217	(hc, item) =>
218	{	×
219	hc.Add(item);	×
220	return hc;	×
221	},	×
222	hc => hc.ToHashCode());	×
223
224	/// <summary>
225	/// Accumulate a sequence of values based on a seed value and a function,
226	/// similar to Aggregate but including intermediate values
227	/// </summary>
228	/// <typeparam name="TSource"></typeparam>
229	/// <typeparam name="TResult"></typeparam>
230	/// <param name="source">Input sequence</param>
231	/// <param name="seed">First intermediate value, not included in return sequence</param>
232	/// <param name="func">Function to transform a previous result and a next input sequence element into the next result</param>
233	/// <returns></returns>
234	public static IEnumerable<TResult> Accumulate<TSource, TResult>(this IEnumerable<TSource> source,
235	TResult seed,
236	Func<TResult, TSource, TResult> func)
237	{	×
238	var result = seed;	×
239	foreach (var item in source)	×
240	{	×
241	result = func(result, item);	×
242	yield return result;	×
243	}	×
244	}	×
245	}
246
247	/// <summary>
248	/// Memoized lazy evaluation in C#!
249	/// From https://stackoverflow.com/a/12428250/2422988
250	/// </summary>
251	public class Memoized<T> : IEnumerable<T>
252	{
253	public Memoized(IEnumerable<T> source)	2✔
254	{	2✔
255	this.source = source;	2✔
256	}	2✔
257
258	IEnumerator IEnumerable.GetEnumerator()
259	=> GetEnumerator();	×
260
261	public IEnumerator<T> GetEnumerator()
262	{	2✔
263	lock (gate)	2✔
264	{	2✔
265	if (isCacheComplete)	2✔
266	{	2✔
267	return cache.GetEnumerator();	2✔
268	}
269	else
270	{	2✔
271	enumerator ??= source.GetEnumerator();	2!
272	}	2✔
273	}	2✔
274	return GetMemoizingEnumerator();	2✔
275	}	2✔
276
277	private IEnumerator<T> GetMemoizingEnumerator()
278	{	2✔
279	for (int index = 0; TryGetItem(index, out T? item); ++index)	4✔
280	{	2✔
281	yield return item;	2✔
282	}	2✔
283	}	2✔
284
285	private bool TryGetItem(int index,
286	out T item)
287	{	2✔
288	lock (gate)	2✔
289	{	2✔
290	if (enumerator is not null && !IsItemInCache(index))	2!
291	{	2✔
292	// The iteration may have completed while waiting for the lock
293	#nullable disable
294	if (isCacheComplete)	2!
295	{	×
296	item = default;	×
297	return false;	×
298	}
299	if (!enumerator.MoveNext())	2✔
300	{	2✔
301	item = default;	2✔
302	isCacheComplete = true;	2✔
303	enumerator.Dispose();	2✔
304	return false;	2✔
305	}
306	#nullable enable
307	cache.Add(enumerator.Current);	2✔
308	}	2✔
309	item = cache[index];	2✔
310	return true;	2✔
311	}
312	}	2✔
313
314	private bool IsItemInCache(int index)
315	=> index < cache.Count;	2✔
316
317	private readonly IEnumerable<T> source;
318	private IEnumerator<T>? enumerator;
319	private readonly List<T> cache = new List<T>();	2✔
320	private bool isCacheComplete;
321	private readonly object gate = new object();	2✔
322	}
323	}

KSP-CKAN / CKAN / 15833572481

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous