6237307473

Committed 19 Sep 2023 04:02PM UTC coverage: 57.015% (-0.4%) from 57.44%

Build # 6237307473

Build Type

push

github

Committed by

web-flow

Commit Message

Feature/rc4 (#1570)

* Syntax tidying
* Dependency updates
* Event handling singletons (ThrowImmediately and co)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: James A Sutherland <>
Co-authored-by: James Friel <jfriel001@dundee.ac.uk>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Run Details

10734 of 20259 branches covered (0.0%)

Branch coverage included in aggregate %.

5922 of 5922 new or added lines in 565 files covered. (100.0%)

30687 of 52390 relevant lines covered (58.57%)

7361.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/Web/WebFileDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using FAnsi.Discovery;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;
using MissingFieldException = System.MissingFieldException;

namespace Rdmp.Core.DataLoad.Modules.Web;

/// <summary>
/// Data load component which downloads a file from a remote URL (e.g. http) into the ForLoading directory of the load.
/// </summary>
public class WebFileDownloader : IPluginDataProvider
{
    [DemandsInitialization(
        "The full URI to a file that will be downloaded into project ForLoading directory, must be a valid Uri",
        Mandatory = true)]
    public Uri UriToFile { get; set; }

    [DemandsInitialization(
        "Optional Username/password to use for network Websense challenges, these will be provided to the WebRequest as a NetworkCredential")]
    public DataAccessCredentials WebsenseCredentials { get; set; }

    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
    {
    }

    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        var t = Stopwatch.StartNew();
        var destinationFile =
            new FileInfo(Path.Combine(job.LoadDirectory.ForLoading.FullName, Path.GetFileName(UriToFile.LocalPath)));
        DownloadFileWhilstPretendingToBeFirefox(destinationFile, job);
        job.OnProgress(this,
            new ProgressEventArgs(destinationFile.FullName,
                new ProgressMeasurement((int)(destinationFile.Length / 1000), ProgressType.Kilobytes), t.Elapsed));
        return ExitCodeType.Success;
    }

    private void DownloadFileWhilstPretendingToBeFirefox(FileInfo destinationFile, IDataLoadJob job)
    {
        NetworkCredential credentials;
        try
        {
            credentials =
                new NetworkCredential(WebsenseCredentials.Username, WebsenseCredentials.GetDecryptedPassword());
        }
        catch (Exception)
        {
            credentials = null;
        }

        FetchRequest(File.Create(destinationFile.FullName), UriToFile.AbsoluteUri, credentials);
    }

    private static void FetchRequest(Stream output, string url, ICredentials credentials = null,
        bool useCredentials = false)
    {
        using var httpClientHandler = new HttpClientHandler();
        if (useCredentials && credentials is not null)
            httpClientHandler.Credentials = credentials;
        using var httpClient = new HttpClient(httpClientHandler, false)
        {
            Timeout = TimeSpan.FromSeconds(60)
        };
        httpClient.DefaultRequestHeaders.Add("User-Agent",
            "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36");
        using var response = httpClient.GetAsync(url).Result;
        if (response.IsSuccessStatusCode)
        {
            response.Content.ReadAsStreamAsync().Result.CopyTo(output);
            return;
        }

        // Failed - retry with credentials?
        if (!useCredentials && response.Headers.WwwAuthenticate.Any(static h =>
                h.Scheme.Equals("basic", StringComparison.OrdinalIgnoreCase) &&
                h.Parameter?.Equals("realm=\"Websense\"", StringComparison.OrdinalIgnoreCase) == true))
            FetchRequest(output, response.Headers.Location?.AbsoluteUri, credentials, true);
        else
            throw new Exception(
                $"Could not get response from {url} - {response.StatusCode} - {response.ReasonPhrase}");
    }

    public string GetDescription() => throw new NotImplementedException();

    public IDataProvider Clone() => throw new NotImplementedException();

    public bool Validate(ILoadDirectory _) =>
        string.IsNullOrWhiteSpace(UriToFile?.PathAndQuery)
            ? throw new MissingFieldException(
                "PathToFile is null or white space - should be populated externally as a parameter")
            : true;


    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
    }


    public void Check(ICheckNotifier notifier)
    {
        notifier.OnCheckPerformed(UriToFile == null
            ? new CheckEventArgs("No URI has been specified", CheckResult.Fail)
            : new CheckEventArgs($"URI is:{UriToFile}", CheckResult.Success));
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Diagnostics;
9	using System.IO;
10	using System.Linq;
11	using System.Net;
12	using System.Net.Http;
13	using FAnsi.Discovery;
14	using Rdmp.Core.Curation;
15	using Rdmp.Core.Curation.Data;
16	using Rdmp.Core.DataFlowPipeline;
17	using Rdmp.Core.DataLoad.Engine.DataProvider;
18	using Rdmp.Core.DataLoad.Engine.Job;
19	using Rdmp.Core.ReusableLibraryCode.Checks;
20	using Rdmp.Core.ReusableLibraryCode.Progress;
21	using MissingFieldException = System.MissingFieldException;
22
23	namespace Rdmp.Core.DataLoad.Modules.Web;
24
25	/// <summary>
26	/// Data load component which downloads a file from a remote URL (e.g. http) into the ForLoading directory of the load.
27	/// </summary>
28	public class WebFileDownloader : IPluginDataProvider
29	{
30	[DemandsInitialization(
31	"The full URI to a file that will be downloaded into project ForLoading directory, must be a valid Uri",
32	Mandatory = true)]
33	public Uri UriToFile { get; set; }	×
34
35	[DemandsInitialization(
36	"Optional Username/password to use for network Websense challenges, these will be provided to the WebRequest as a NetworkCredential")]
37	public DataAccessCredentials WebsenseCredentials { get; set; }	×
38
39	public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
40	{
41	}	×
42
43	public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
44	{
45	var t = Stopwatch.StartNew();	×
46	var destinationFile =	×
47	new FileInfo(Path.Combine(job.LoadDirectory.ForLoading.FullName, Path.GetFileName(UriToFile.LocalPath)));	×
48	DownloadFileWhilstPretendingToBeFirefox(destinationFile, job);	×
49	job.OnProgress(this,	×
50	new ProgressEventArgs(destinationFile.FullName,	×
51	new ProgressMeasurement((int)(destinationFile.Length / 1000), ProgressType.Kilobytes), t.Elapsed));	×
52	return ExitCodeType.Success;	×
53	}
54
55	private void DownloadFileWhilstPretendingToBeFirefox(FileInfo destinationFile, IDataLoadJob job)
56	{
57	NetworkCredential credentials;
58	try
59	{
60	credentials =	×
61	new NetworkCredential(WebsenseCredentials.Username, WebsenseCredentials.GetDecryptedPassword());	×
62	}	×
63	catch (Exception)	×
64	{
65	credentials = null;	×
66	}	×
67
68	FetchRequest(File.Create(destinationFile.FullName), UriToFile.AbsoluteUri, credentials);	×
69	}	×
70
71	private static void FetchRequest(Stream output, string url, ICredentials credentials = null,
72	bool useCredentials = false)
73	{
74	using var httpClientHandler = new HttpClientHandler();	×
75	if (useCredentials && credentials is not null)	×
76	httpClientHandler.Credentials = credentials;	×
77	using var httpClient = new HttpClient(httpClientHandler, false)	×
78	{	×
79	Timeout = TimeSpan.FromSeconds(60)	×
80	};	×
81	httpClient.DefaultRequestHeaders.Add("User-Agent",	×
82	"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36");	×
83	using var response = httpClient.GetAsync(url).Result;	×
84	if (response.IsSuccessStatusCode)	×
85	{
86	response.Content.ReadAsStreamAsync().Result.CopyTo(output);	×
87	return;	×
88	}
89
90	// Failed - retry with credentials?
91	if (!useCredentials && response.Headers.WwwAuthenticate.Any(static h =>	×
92	h.Scheme.Equals("basic", StringComparison.OrdinalIgnoreCase) &&	×
93	h.Parameter?.Equals("realm=\"Websense\"", StringComparison.OrdinalIgnoreCase) == true))	×
94	FetchRequest(output, response.Headers.Location?.AbsoluteUri, credentials, true);	×
95	else
96	throw new Exception(	×
97	$"Could not get response from {url} - {response.StatusCode} - {response.ReasonPhrase}");	×
98	}	×
99
100	public string GetDescription() => throw new NotImplementedException();	×
101
102	public IDataProvider Clone() => throw new NotImplementedException();	×
103
104	public bool Validate(ILoadDirectory _) =>
105	string.IsNullOrWhiteSpace(UriToFile?.PathAndQuery)	×
106	? throw new MissingFieldException(	×
107	"PathToFile is null or white space - should be populated externally as a parameter")	×
108	: true;	×
109
110
111	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
112	{
113	}	×
114
115
116	public void Check(ICheckNotifier notifier)
117	{
118	notifier.OnCheckPerformed(UriToFile == null	×
119	? new CheckEventArgs("No URI has been specified", CheckResult.Fail)	×
120	: new CheckEventArgs($"URI is:{UriToFile}", CheckResult.Success));	×
121	}	×
122	}

HicServices / RDMP / 6237307473

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous