16536011796

Committed 26 Jul 2025 04:19AM UTC coverage: 56.351% (+8.5%) from 47.804%

Build # 16536011796

Build Type

Pull #4408

github

Committed by

web-flow

Commit Message

Merge b51164c06 into 99ebf468c

Pull Request Pull Request #4408: Add tests for CmdLine

Run Details

4558 of 8422 branches covered (54.12%)

Branch coverage included in aggregate %.

148 of 273 new or added lines in 28 files covered. (54.21%)

18 existing lines in 5 files now uncovered.

9719 of 16914 relevant lines covered (57.46%)

1.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.89

/Core/Net/Net.cs

using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;

using Autofac;
using ChinhDo.Transactions.FileManager;
using log4net;

using CKAN.Extensions;
using CKAN.Configuration;

namespace CKAN
{
    /// <summary>
    /// Doing something with the network? Do it here.
    /// </summary>

    public static class Net
    {
        /// <summary>
        /// Make a HEAD request to get the ETag of a URL without downloading it
        /// </summary>
        /// <param name="url">Remote URL to check</param>
        /// <returns>
        /// ETag value of the URL if any, otherwise null, see
        /// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
        /// </returns>
        public static string? CurrentETag(Uri url)
        {
            // HttpClient apparently is worse than what it was supposed to replace
            #pragma warning disable SYSLIB0014
            WebRequest req = WebRequest.Create(url);
            #pragma warning restore SYSLIB0014
            req.Method = "HEAD";
            try
            {
                HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
                var val = resp.Headers["ETag"]?.Replace("\"", "");
                resp.Close();
                return val;
            }
            catch (WebException exc)
            {
                // Let the calling code keep going to get the actual problem
                log.Debug($"Failed to get ETag from {url}", exc);
                return null;
            }
        }

        /// <summary>
        /// Downloads the specified url, and stores it in the filename given.
        /// If no filename is supplied, a temporary file will be generated.
        /// Returns the filename the file was saved to on success.
        /// Throws an exception on failure.
        /// Throws a MissingCertificateException *and* prints a message to the
        /// console if we detect missing certificates (common on a fresh Linux/mono install)
        /// </summary>
        public static string Download(Uri         url,
                                      out string? etag,
                                      string?     userAgent = null,
                                      string?     filename  = null,
                                      IUser?      user      = null)
        {
            user?.RaiseMessage(Properties.Resources.NetDownloading, url);
            var FileTransaction = new TxFileManager();

            // Generate a temporary file if none is provided.
            filename ??= FileTransaction.GetTempFileName();

            log.DebugFormat("Downloading {0} to {1}", url, filename);

            try
            {
                // This WebClient child class does some complicated stuff, let's keep using it for now
                #pragma warning disable SYSLIB0014
                var agent = new RedirectingTimeoutWebClient(userAgent ?? UserAgentString);
                #pragma warning restore SYSLIB0014
                agent.DownloadFile(url, filename);
                etag = agent.ResponseHeaders?.Get("ETag")?.Replace("\"", "");
            }
            catch (WebException wex)
            when (wex is
                  {
                      Status:   WebExceptionStatus.ProtocolError,
                      Response: HttpWebResponse
                                {
                                    StatusCode: HttpStatusCode.Redirect
                                }
                                response,
                  })
            {
                // Get redirect if redirected.
                // This is needed when redirecting from HTTPS to HTTP on .NET Core.
                var loc = new Uri(response.GetResponseHeader("Location"));
                log.InfoFormat("Redirected to {0}", loc);
                return Download(loc, out etag, userAgent, filename, user);
            }
            catch (WebException wex)
            when (wex is { Status: WebExceptionStatus.SecureChannelFailure })
            {
                throw new MissingCertificateKraken(url, null, wex);
            }
            catch (WebException wex)
            when (wex is { InnerException: Exception inner })
            {
                throw new DownloadErrorsKraken(new NetAsyncDownloader.DownloadTargetFile(url),
                                               inner);
            }
            // Otherwise it's a valid failure from the server (probably a 404), keep it
            catch
            {
                try
                {
                    // Clean up our file, it's unlikely to be complete.
                    // We do this even though we're using transactional files, as we may not be in a transaction.
                    log.DebugFormat("Removing {0} after web error failure", filename);
                    FileTransaction.Delete(filename);
                }
                catch
                {
                    // It's okay if this fails.
                }
                throw;
            }

            return filename;
        }

        /// <summary>
        /// Download a string from a URL
        /// </summary>
        /// <param name="url">URL to download from</param>
        /// <param name="userAgent">User agent to send with the request</param>
        /// <param name="authToken">An authentication token sent with the "Authorization" header.
        ///                         Attempted to be looked up from the configuraiton if not specified</param>
        /// <param name="mimeType">A mime type sent with the "Accept" header</param>
        /// <param name="timeout">Timeout for the request in milliseconds, defaulting to 100 000 (=100 seconds)</param>
        /// <returns>The text content returned by the server</returns>
        public static string? DownloadText(Uri     url,
                                           string? userAgent = null,
                                           string? authToken = "",
                                           string? mimeType = null,
                                           int     timeout = 100000)
        {
            log.DebugFormat("About to download {0}", url.OriginalString);

            #pragma warning disable SYSLIB0014
            WebClient agent = new RedirectingTimeoutWebClient(userAgent ?? UserAgentString,
                                                              timeout, mimeType ?? "");
            #pragma warning restore SYSLIB0014

            // Check whether to use an auth token for this host
            if (!string.IsNullOrEmpty(authToken)
                || (ServiceLocator.Container.Resolve<IConfiguration>().TryGetAuthToken(url.Host, out authToken)
                    && !string.IsNullOrEmpty(authToken)))
            {
                log.InfoFormat("Using auth token for {0}", url.Host);
                // Send our auth token to the GitHub API (or whoever else needs one)
                agent.Headers.Add("Authorization", $"token {authToken}");
            }

            for (int whichAttempt = 0; whichAttempt < MaxRetries + 1; ++whichAttempt)
            {
                try
                {
                    var content = Utilities.WithRethrowInner(() => agent.DownloadString(url));
                    var header  = agent.ResponseHeaders?.ToString();

                    log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);

                    return content;
                }
                catch (WebException wex)
                when (wex.Status == WebExceptionStatus.Timeout)
                {
                    throw new RequestTimedOutKraken(url, wex);
                }
                catch (WebException wex)
                when (wex.Status != WebExceptionStatus.ProtocolError
                      && whichAttempt < MaxRetries)
                {
                    log.DebugFormat("Web request failed with non-protocol error, retrying in {0} milliseconds: {1}", RetryDelayMilliseconds * whichAttempt, wex.Message);
                    // Exponential backoff with jitter
                    Thread.Sleep((int)(RetryDelayMilliseconds
                                 * (Math.Pow(2, whichAttempt) + random.NextDouble())));
                }
            }
            // Should never get here, because we don't catch any exceptions
            // in the final iteration of the above for loop. They should be
            // thrown to the calling code, or the call should succeed.
            return null;
        }

        public static Uri? ResolveRedirect(Uri     url,
                                           string? userAgent,
                                           int     maxRedirects = 6)
        {
            var urls = url.TraverseNodes(u => new RedirectWebClient(userAgent ?? UserAgentString) is RedirectWebClient rwClient
                                              && rwClient.OpenRead(u) is Stream s && DisposeStream(s)
                                              && rwClient.ResponseHeaders is WebHeaderCollection headers
                                              && headers["Location"] is string location
                                                  ? Uri.IsWellFormedUriString(location, UriKind.Absolute)
                                                      ? new Uri(location)
                                                      : Uri.IsWellFormedUriString(location, UriKind.Relative)
                                                          ? new Uri(u, location)
                                                          : throw new Kraken(string.Format(Properties.Resources.NetInvalidLocation,
                                                                                           location))
                                                  : null)
                          // The first element is the input, so e.g. if we want two redirects, that's three elements
                          .Take(maxRedirects + 1)
                          .ToArray();
            if (log.IsDebugEnabled)
            {
                foreach ((Uri from, Uri to) in urls.Zip(urls.Skip(1)))
                {
                    log.DebugFormat("Redirected {0} to {1}", from, to);
                }
            }
            return urls.LastOrDefault();
        }

        private static bool DisposeStream(Stream s)
        {
            s.Dispose();
            return true;
        }

        /// <summary>
        /// Provide an escaped version of the given Uri string, including converting
        /// square brackets to their escaped forms.
        /// </summary>
        /// <returns>
        /// <c>null</c> if the string is not a valid <see cref="Uri"/>, otherwise its normalized form.
        /// </returns>
        public static string? NormalizeUri(string uri)
        {
            // Uri.EscapeUriString has been deprecated because its purpose was ambiguous.
            // Is it supposed to turn a "&" into part of the content of a form field,
            // or is it supposed to assume that it separates different form fields?
            // https://github.com/dotnet/runtime/issues/31387
            // So now we have to just substitute certain characters ourselves one by one.

            // Square brackets are "reserved characters" that should not appear
            // in strings to begin with, so C# doesn't try to escape them in case
            // they're being used in a special way. They're not; some mod authors
            // just have crazy ideas as to what should be in a URL, and SD doesn't
            // escape them in its API. There's probably more in RFC 3986.
            var escaped = UriEscapeAll(uri.Replace(" ", "+"),
                                       '"', '<', '>', '^', '`',
                                       '(', ')',
                                       '{', '|', '}', '[', ']');

            // Make sure we have a "http://" or "https://" start.
            if (!Regex.IsMatch(escaped, "(?i)^(http|https)://"))
            {
                // Prepend "http://", as we do not know if the site supports https.
                escaped = "http://" + escaped;
            }

            return Uri.IsWellFormedUriString(escaped, UriKind.Absolute)
                       ? escaped
                       : null;
        }

        private static string UriEscapeAll(string orig, params char[] characters)
            => characters.Aggregate(orig,
                                    (s, c) => s.Replace(c.ToString(),
                                                        Uri.HexEscape(c)));

        /// <summary>
        /// Translate a URL into a form that returns the raw contents of a file
        /// Only changes GitHub URLs, others are left as-is
        /// </summary>
        /// <param name="remoteUri">URL to handle</param>
        /// <returns>
        /// URL pointing to the raw contents of the input URL
        /// </returns>
        public static Uri GetRawUri(Uri remoteUri)
        {
            // Authors may use the URI of the GitHub file page instead of the URL to the actual raw file.
            // Detect that case and automatically transform the remote URL to one we can use.
            // This is hacky and fragile but it's basically what KSP-AVC itself does in its
            // FormatCompatibleUrl(string) method so we have to go along with the flow:
            // https://github.com/CYBUTEK/KSPAddonVersionChecker/blob/ff94000144a666c8ff637c71b802e1baee9c15cd/KSP-AVC/AddonInfo.cs#L199
            // However, this implementation is more robust as it actually parses the URI rather than doing
            // basic string replacements.
            if (string.Compare(remoteUri.Host, "github.com", StringComparison.OrdinalIgnoreCase) == 0)
            {
                // We expect a non-raw URI to be in one of these forms:
                //  1. https://github.com/<USER>/<PROJECT>/blob/<BRANCH>/<PATH>
                //  2. https://github.com/<USER>/<PROJECT>/tree/<BRANCH>/<PATH>
                //
                // Therefore, we expect at least six segments in the path:
                //  1. "/"
                //  2. "<USER>/"
                //  3. "<PROJECT>/"
                //  4. "blob/" or "tree/"
                //  5. "<BRANCH>/"
                //  6+. "<PATH>"
                //
                // And that the fourth segment (index 3) is either "blob/" or "tree/"

                // Check that the path is what we expect
                var segments = remoteUri.Segments.ToList();

                if (//segments is [_, _, _, "raw/", ..]
                    segments.Count > 3
                    && segments[3] is "raw/")
                {
                    log.InfoFormat("Remote GitHub URL is in raw format, using as is.");
                    return remoteUri;
                }
                if (//segments is [_, _, _, "releases/", "latest/", "download/", ..]
                    segments.Count > 6
                    && segments[3] is "releases/"
                    && segments[4] is "latest/"
                    && segments[5] is "download/")
                {
                    log.InfoFormat("Remote GitHub URL is in release asset format, using as is.");
                    return remoteUri;
                }
                if (//segments is not [_, _, _, "blob/" or "tree/", _, _, ..]
                    segments.Count < 6
                    || segments[3] is not ("blob/" or "tree/"))
                {
                    log.WarnFormat("Remote non-raw GitHub URL is in an unknown format, using as is.");
                    return remoteUri;
                }

                var remoteUriBuilder = new UriBuilder(remoteUri)
                {
                    // Replace host with raw host
                    Host = "raw.githubusercontent.com",
                    // Remove "blob/" or "tree/" segment from raw URI
                    Path = string.Join("", segments.Take(3)
                                                   .Concat(segments.Skip(4))),
                };

                log.InfoFormat("Canonicalized non-raw GitHub URL to: {0}",
                               remoteUriBuilder.Uri);

                return remoteUriBuilder.Uri;
            }
            else
            {
                return remoteUri;
            }
        }

        // The user agent that we report to web sites
        // Maybe overwritten by command line args
        public static readonly string UserAgentString = $"Mozilla/5.0 (compatible; CKAN/{Meta.ReleaseVersion})";

        private const int MaxRetries             = 5;
        private const int RetryDelayMilliseconds = 100;

        private static readonly ILog log = LogManager.GetLogger(typeof(Net));

        private static readonly Random random = new Random();

        public static readonly Dictionary<string, Uri> ThrottledHosts = new Dictionary<string, Uri>()
        {
            {
                "api.github.com",
                new Uri(HelpURLs.AuthTokens)
            }
        };
    }
}

1	using System;
2	using System.IO;
3	using System.Collections.Generic;
4	using System.Linq;
5	using System.Net;
6	using System.Text.RegularExpressions;
7	using System.Threading;
8
9	using Autofac;
10	using ChinhDo.Transactions.FileManager;
11	using log4net;
12
13	using CKAN.Extensions;
14	using CKAN.Configuration;
15
16	namespace CKAN
17	{
18	/// <summary>
19	/// Doing something with the network? Do it here.
20	/// </summary>
21
22	public static class Net
23	{
24	/// <summary>
25	/// Make a HEAD request to get the ETag of a URL without downloading it
26	/// </summary>
27	/// <param name="url">Remote URL to check</param>
28	/// <returns>
29	/// ETag value of the URL if any, otherwise null, see
30	/// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
31	/// </returns>
32	public static string? CurrentETag(Uri url)
33	{	2✔
34	// HttpClient apparently is worse than what it was supposed to replace
35	#pragma warning disable SYSLIB0014
36	WebRequest req = WebRequest.Create(url);	2✔
37	#pragma warning restore SYSLIB0014
38	req.Method = "HEAD";	2✔
39	try
40	{	2✔
41	HttpWebResponse resp = (HttpWebResponse)req.GetResponse();	2✔
42	var val = resp.Headers["ETag"]?.Replace("\"", "");	2✔
43	resp.Close();	2✔
44	return val;	2✔
45	}
46	catch (WebException exc)	×
47	{	×
48	// Let the calling code keep going to get the actual problem
49	log.Debug($"Failed to get ETag from {url}", exc);	×
50	return null;	×
51	}
52	}	2✔
53
54	/// <summary>
55	/// Downloads the specified url, and stores it in the filename given.
56	/// If no filename is supplied, a temporary file will be generated.
57	/// Returns the filename the file was saved to on success.
58	/// Throws an exception on failure.
59	/// Throws a MissingCertificateException and prints a message to the
60	/// console if we detect missing certificates (common on a fresh Linux/mono install)
61	/// </summary>
62	public static string Download(Uri url,
63	out string? etag,
64	string? userAgent = null,
65	string? filename = null,
66	IUser? user = null)
67	{	2✔
68	user?.RaiseMessage(Properties.Resources.NetDownloading, url);	2!
69	var FileTransaction = new TxFileManager();	2✔
70
71	// Generate a temporary file if none is provided.
72	filename ??= FileTransaction.GetTempFileName();	2✔
73
74	log.DebugFormat("Downloading {0} to {1}", url, filename);	2✔
75
76	try
77	{	2✔
78	// This WebClient child class does some complicated stuff, let's keep using it for now
79	#pragma warning disable SYSLIB0014
80	var agent = new RedirectingTimeoutWebClient(userAgent ?? UserAgentString);	2✔
81	#pragma warning restore SYSLIB0014
82	agent.DownloadFile(url, filename);	2✔
83	etag = agent.ResponseHeaders?.Get("ETag")?.Replace("\"", "");	2✔
84	}	2!
85	catch (WebException wex)
86	when (wex is	×
87	{
88	Status: WebExceptionStatus.ProtocolError,
89	Response: HttpWebResponse
90	{
91	StatusCode: HttpStatusCode.Redirect
92	}
93	response,
94	})
95	{	×
96	// Get redirect if redirected.
97	// This is needed when redirecting from HTTPS to HTTP on .NET Core.
98	var loc = new Uri(response.GetResponseHeader("Location"));	×
99	log.InfoFormat("Redirected to {0}", loc);	×
100	return Download(loc, out etag, userAgent, filename, user);	×
101	}
102	catch (WebException wex)
103	when (wex is { Status: WebExceptionStatus.SecureChannelFailure })	×
104	{	×
105	throw new MissingCertificateKraken(url, null, wex);	×
106	}
107	catch (WebException wex)
108	when (wex is { InnerException: Exception inner })	×
109	{	×
110	throw new DownloadErrorsKraken(new NetAsyncDownloader.DownloadTargetFile(url),	×
111	inner);
112	}
113	// Otherwise it's a valid failure from the server (probably a 404), keep it
114	catch	×
115	{	×
116	try
117	{	×
118	// Clean up our file, it's unlikely to be complete.
119	// We do this even though we're using transactional files, as we may not be in a transaction.
120	log.DebugFormat("Removing {0} after web error failure", filename);	×
121	FileTransaction.Delete(filename);	×
122	}	×
123	catch	×
124	{	×
125	// It's okay if this fails.
126	}	×
127	throw;	×
128	}
129
130	return filename;	2✔
131	}	2✔
132
133	/// <summary>
134	/// Download a string from a URL
135	/// </summary>
136	/// <param name="url">URL to download from</param>
137	/// <param name="userAgent">User agent to send with the request</param>
138	/// <param name="authToken">An authentication token sent with the "Authorization" header.
139	/// Attempted to be looked up from the configuraiton if not specified</param>
140	/// <param name="mimeType">A mime type sent with the "Accept" header</param>
141	/// <param name="timeout">Timeout for the request in milliseconds, defaulting to 100 000 (=100 seconds)</param>
142	/// <returns>The text content returned by the server</returns>
143	public static string? DownloadText(Uri url,
144	string? userAgent = null,
145	string? authToken = "",
146	string? mimeType = null,
147	int timeout = 100000)
148	{	2✔
149	log.DebugFormat("About to download {0}", url.OriginalString);	2✔
150
151	#pragma warning disable SYSLIB0014
152	WebClient agent = new RedirectingTimeoutWebClient(userAgent ?? UserAgentString,	2✔
153	timeout, mimeType ?? "");
154	#pragma warning restore SYSLIB0014
155
156	// Check whether to use an auth token for this host
157	if (!string.IsNullOrEmpty(authToken)	2!
158	\|\| (ServiceLocator.Container.Resolve<IConfiguration>().TryGetAuthToken(url.Host, out authToken)
159	&& !string.IsNullOrEmpty(authToken)))
160	{	×
161	log.InfoFormat("Using auth token for {0}", url.Host);	×
162	// Send our auth token to the GitHub API (or whoever else needs one)
163	agent.Headers.Add("Authorization", $"token {authToken}");	×
164	}	×
165
166	for (int whichAttempt = 0; whichAttempt < MaxRetries + 1; ++whichAttempt)	3!
167	{	2✔
168	try
169	{	2✔
170	var content = Utilities.WithRethrowInner(() => agent.DownloadString(url));	2✔
171	var header = agent.ResponseHeaders?.ToString();	2✔
172
173	log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);	2✔
174
175	return content;	2!
176	}
177	catch (WebException wex)
178	when (wex.Status == WebExceptionStatus.Timeout)	×
179	{	×
180	throw new RequestTimedOutKraken(url, wex);	×
181	}
182	catch (WebException wex)
183	when (wex.Status != WebExceptionStatus.ProtocolError	×
184	&& whichAttempt < MaxRetries)
185	{	×
186	log.DebugFormat("Web request failed with non-protocol error, retrying in {0} milliseconds: {1}", RetryDelayMilliseconds * whichAttempt, wex.Message);	×
187	// Exponential backoff with jitter
NEW 188	Thread.Sleep((int)(RetryDelayMilliseconds	×
189	* (Math.Pow(2, whichAttempt) + random.NextDouble())));
190	}	×
191	}	×
192	// Should never get here, because we don't catch any exceptions
193	// in the final iteration of the above for loop. They should be
194	// thrown to the calling code, or the call should succeed.
195	return null;	×
196	}	2✔
197
198	public static Uri? ResolveRedirect(Uri url,
199	string? userAgent,
200	int maxRedirects = 6)
201	{	2✔
202	var urls = url.TraverseNodes(u => new RedirectWebClient(userAgent ?? UserAgentString) is RedirectWebClient rwClient	2!
203	&& rwClient.OpenRead(u) is Stream s && DisposeStream(s)
204	&& rwClient.ResponseHeaders is WebHeaderCollection headers
205	&& headers["Location"] is string location
206	? Uri.IsWellFormedUriString(location, UriKind.Absolute)
207	? new Uri(location)
208	: Uri.IsWellFormedUriString(location, UriKind.Relative)
209	? new Uri(u, location)
210	: throw new Kraken(string.Format(Properties.Resources.NetInvalidLocation,
211	location))
212	: null)
213	// The first element is the input, so e.g. if we want two redirects, that's three elements
214	.Take(maxRedirects + 1)
215	.ToArray();
216	if (log.IsDebugEnabled)	2!
217	{	×
218	foreach ((Uri from, Uri to) in urls.Zip(urls.Skip(1)))	×
219	{	×
220	log.DebugFormat("Redirected {0} to {1}", from, to);	×
221	}	×
222	}	×
223	return urls.LastOrDefault();	2✔
224	}	2✔
225
226	private static bool DisposeStream(Stream s)
227	{	2✔
228	s.Dispose();	2✔
229	return true;	2✔
230	}	2✔
231
232	/// <summary>
233	/// Provide an escaped version of the given Uri string, including converting
234	/// square brackets to their escaped forms.
235	/// </summary>
236	/// <returns>
237	/// <c>null</c> if the string is not a valid <see cref="Uri"/>, otherwise its normalized form.
238	/// </returns>
239	public static string? NormalizeUri(string uri)
240	{	2✔
241	// Uri.EscapeUriString has been deprecated because its purpose was ambiguous.
242	// Is it supposed to turn a "&" into part of the content of a form field,
243	// or is it supposed to assume that it separates different form fields?
244	// https://github.com/dotnet/runtime/issues/31387
245	// So now we have to just substitute certain characters ourselves one by one.
246
247	// Square brackets are "reserved characters" that should not appear
248	// in strings to begin with, so C# doesn't try to escape them in case
249	// they're being used in a special way. They're not; some mod authors
250	// just have crazy ideas as to what should be in a URL, and SD doesn't
251	// escape them in its API. There's probably more in RFC 3986.
252	var escaped = UriEscapeAll(uri.Replace(" ", "+"),	2✔
253	'"', '<', '>', '^', '`',
254	'(', ')',
255	'{', '\|', '}', '[', ']');
256
257	// Make sure we have a "http://" or "https://" start.
258	if (!Regex.IsMatch(escaped, "(?i)^(http\|https)://"))	2✔
259	{	2✔
260	// Prepend "http://", as we do not know if the site supports https.
261	escaped = "http://" + escaped;	2✔
262	}	2✔
263
264	return Uri.IsWellFormedUriString(escaped, UriKind.Absolute)	2✔
265	? escaped
266	: null;
267	}	2✔
268
269	private static string UriEscapeAll(string orig, params char[] characters)
270	=> characters.Aggregate(orig,	2✔
271	(s, c) => s.Replace(c.ToString(),	2✔
272	Uri.HexEscape(c)));
273
274	/// <summary>
275	/// Translate a URL into a form that returns the raw contents of a file
276	/// Only changes GitHub URLs, others are left as-is
277	/// </summary>
278	/// <param name="remoteUri">URL to handle</param>
279	/// <returns>
280	/// URL pointing to the raw contents of the input URL
281	/// </returns>
282	public static Uri GetRawUri(Uri remoteUri)
283	{	2✔
284	// Authors may use the URI of the GitHub file page instead of the URL to the actual raw file.
285	// Detect that case and automatically transform the remote URL to one we can use.
286	// This is hacky and fragile but it's basically what KSP-AVC itself does in its
287	// FormatCompatibleUrl(string) method so we have to go along with the flow:
288	// https://github.com/CYBUTEK/KSPAddonVersionChecker/blob/ff94000144a666c8ff637c71b802e1baee9c15cd/KSP-AVC/AddonInfo.cs#L199
289	// However, this implementation is more robust as it actually parses the URI rather than doing
290	// basic string replacements.
291	if (string.Compare(remoteUri.Host, "github.com", StringComparison.OrdinalIgnoreCase) == 0)	2✔
292	{	2✔
293	// We expect a non-raw URI to be in one of these forms:
294	// 1. https://github.com/<USER>/<PROJECT>/blob/<BRANCH>/<PATH>
295	// 2. https://github.com/<USER>/<PROJECT>/tree/<BRANCH>/<PATH>
296	//
297	// Therefore, we expect at least six segments in the path:
298	// 1. "/"
299	// 2. "<USER>/"
300	// 3. "<PROJECT>/"
301	// 4. "blob/" or "tree/"
302	// 5. "<BRANCH>/"
303	// 6+. "<PATH>"
304	//
305	// And that the fourth segment (index 3) is either "blob/" or "tree/"
306
307	// Check that the path is what we expect
308	var segments = remoteUri.Segments.ToList();	2✔
309
310	if (//segments is [_, _, _, "raw/", ..]	2✔
311	segments.Count > 3
312	&& segments[3] is "raw/")
313	{	2✔
314	log.InfoFormat("Remote GitHub URL is in raw format, using as is.");	2✔
315	return remoteUri;	2✔
316	}
317	if (//segments is [_, _, _, "releases/", "latest/", "download/", ..]	2!
318	segments.Count > 6
319	&& segments[3] is "releases/"
320	&& segments[4] is "latest/"
321	&& segments[5] is "download/")
322	{	×
323	log.InfoFormat("Remote GitHub URL is in release asset format, using as is.");	×
324	return remoteUri;	×
325	}
326	if (//segments is not [_, _, _, "blob/" or "tree/", _, _, ..]	2✔
327	segments.Count < 6
328	\|\| segments[3] is not ("blob/" or "tree/"))
329	{	2✔
330	log.WarnFormat("Remote non-raw GitHub URL is in an unknown format, using as is.");	2✔
331	return remoteUri;	2✔
332	}
333
334	var remoteUriBuilder = new UriBuilder(remoteUri)	2✔
335	{
336	// Replace host with raw host
337	Host = "raw.githubusercontent.com",
338	// Remove "blob/" or "tree/" segment from raw URI
339	Path = string.Join("", segments.Take(3)
340	.Concat(segments.Skip(4))),
341	};
342
343	log.InfoFormat("Canonicalized non-raw GitHub URL to: {0}",	2✔
344	remoteUriBuilder.Uri);
345
346	return remoteUriBuilder.Uri;	2✔
347	}
348	else
349	{	2✔
350	return remoteUri;	2✔
351	}
352	}	2✔
353
354	// The user agent that we report to web sites
355	// Maybe overwritten by command line args
356	public static readonly string UserAgentString = $"Mozilla/5.0 (compatible; CKAN/{Meta.ReleaseVersion})";	2✔
357
358	private const int MaxRetries = 5;
359	private const int RetryDelayMilliseconds = 100;
360
361	private static readonly ILog log = LogManager.GetLogger(typeof(Net));	2✔
362
363	private static readonly Random random = new Random();	2✔
364
365	public static readonly Dictionary<string, Uri> ThrottledHosts = new Dictionary<string, Uri>()	2✔
366	{
367	{
368	"api.github.com",
369	new Uri(HelpURLs.AuthTokens)
370	}
371	};
372	}
373	}

KSP-CKAN / CKAN / 16536011796

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous