22626536046

Committed 03 Mar 2026 02:04PM UTC coverage: 57.18% (-0.004%) from 57.184%

Build # 22626536046

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #2318 from HicServices/bugfix/RDMP-340-hashing-issue

fix file lock in web downloader

Run Details

11513 of 21629 branches covered (53.23%)

Branch coverage included in aggregate %.

1 of 12 new or added lines in 2 files covered. (8.33%)

3 existing lines in 1 file now uncovered.

32592 of 55505 relevant lines covered (58.72%)

17686.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/Web/WebFileDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using FAnsi.Discovery;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;
using MissingFieldException = System.MissingFieldException;

namespace Rdmp.Core.DataLoad.Modules.Web;

/// <summary>
/// Data load component which downloads a file from a remote URL (e.g. http) into the ForLoading directory of the load.
/// </summary>
public class WebFileDownloader : IPluginDataProvider
{
    [DemandsInitialization(
        "The full URI to a file that will be downloaded into project ForLoading directory, must be a valid Uri",
        Mandatory = true)]
    public Uri UriToFile { get; set; }

    [DemandsInitialization(
        "Optional Username/password to use for network Websense challenges, these will be provided to the WebRequest as a NetworkCredential")]
    public DataAccessCredentials WebsenseCredentials { get; set; }

    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
    {
    }

    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        var t = Stopwatch.StartNew();
        var destinationFile =
            new FileInfo(Path.Combine(job.LoadDirectory.ForLoading.FullName, Path.GetFileName(UriToFile.LocalPath)));
        DownloadFileWhilstPretendingToBeFirefox(destinationFile, job);
        job.OnProgress(this,
            new ProgressEventArgs(destinationFile.FullName,
                new ProgressMeasurement((int)(destinationFile.Length / 1000), ProgressType.Kilobytes), t.Elapsed));
        return ExitCodeType.Success;
    }

    private void DownloadFileWhilstPretendingToBeFirefox(FileInfo destinationFile, IDataLoadJob job)
    {
        NetworkCredential credentials = null;
        if (WebsenseCredentials is not null)
        {
            try
            {
                credentials =
                    new NetworkCredential(WebsenseCredentials.Username, WebsenseCredentials.GetDecryptedPassword());
            }
            catch (Exception)
            {
                credentials = null;
            }
        }
        using (var fs = File.Create(destinationFile.FullName)) {
            FetchRequest(fs, UriToFile.AbsoluteUri, credentials);
        }
    }

    private static void FetchRequest(Stream output, string url, ICredentials credentials = null,
        bool useCredentials = false)
    {
        using var httpClientHandler = new HttpClientHandler();
        if (useCredentials && credentials is not null)
            httpClientHandler.Credentials = credentials;
        using var httpClient = new HttpClient(httpClientHandler, false)
        {
            Timeout = TimeSpan.FromSeconds(60)
        };
        httpClient.DefaultRequestHeaders.Add("User-Agent",
            "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36");
        using var response = httpClient.GetAsync(url).Result;
        if (response.IsSuccessStatusCode)
        {
            response.Content.ReadAsStream().CopyTo(output);
            return;
        }

        // Failed - retry with credentials?
        if (!useCredentials && response.Headers.WwwAuthenticate.Any(static h =>
                h.Scheme.Equals("basic", StringComparison.OrdinalIgnoreCase) &&
                h.Parameter?.Equals("realm=\"Websense\"", StringComparison.OrdinalIgnoreCase) == true))
            FetchRequest(output, response.Headers.Location?.AbsoluteUri, credentials, true);
        else
        {
            throw new Exception(
                $"Could not get response from {url} - {response.StatusCode} - {response.ReasonPhrase}");
        }
    }

    public string GetDescription() => throw new NotImplementedException();

    public IDataProvider Clone() => throw new NotImplementedException();

    public bool Validate(ILoadDirectory _) =>
        string.IsNullOrWhiteSpace(UriToFile?.PathAndQuery)
            ? throw new MissingFieldException(
                "PathToFile is null or white space - should be populated externally as a parameter")
            : true;


    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
    }


    public void Check(ICheckNotifier notifier)
    {
        notifier.OnCheckPerformed(UriToFile == null
            ? new CheckEventArgs("No URI has been specified", CheckResult.Fail)
            : new CheckEventArgs($"URI is:{UriToFile}", CheckResult.Success));
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Diagnostics;
9	using System.IO;
10	using System.Linq;
11	using System.Net;
12	using System.Net.Http;
13	using System.Threading.Tasks;
14	using FAnsi.Discovery;
15	using Rdmp.Core.Curation;
16	using Rdmp.Core.Curation.Data;
17	using Rdmp.Core.DataFlowPipeline;
18	using Rdmp.Core.DataLoad.Engine.DataProvider;
19	using Rdmp.Core.DataLoad.Engine.Job;
20	using Rdmp.Core.ReusableLibraryCode.Checks;
21	using Rdmp.Core.ReusableLibraryCode.Progress;
22	using MissingFieldException = System.MissingFieldException;
23
24	namespace Rdmp.Core.DataLoad.Modules.Web;
25
26	/// <summary>
27	/// Data load component which downloads a file from a remote URL (e.g. http) into the ForLoading directory of the load.
28	/// </summary>
29	public class WebFileDownloader : IPluginDataProvider
30	{
31	[DemandsInitialization(
32	"The full URI to a file that will be downloaded into project ForLoading directory, must be a valid Uri",
33	Mandatory = true)]
34	public Uri UriToFile { get; set; }	×
35
36	[DemandsInitialization(
37	"Optional Username/password to use for network Websense challenges, these will be provided to the WebRequest as a NetworkCredential")]
38	public DataAccessCredentials WebsenseCredentials { get; set; }	×
39
40	public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
41	{
42	}	×
43
44	public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
45	{
46	var t = Stopwatch.StartNew();	×
47	var destinationFile =	×
48	new FileInfo(Path.Combine(job.LoadDirectory.ForLoading.FullName, Path.GetFileName(UriToFile.LocalPath)));	×
49	DownloadFileWhilstPretendingToBeFirefox(destinationFile, job);	×
50	job.OnProgress(this,	×
51	new ProgressEventArgs(destinationFile.FullName,	×
52	new ProgressMeasurement((int)(destinationFile.Length / 1000), ProgressType.Kilobytes), t.Elapsed));	×
53	return ExitCodeType.Success;	×
54	}
55
56	private void DownloadFileWhilstPretendingToBeFirefox(FileInfo destinationFile, IDataLoadJob job)
57	{
NEW 58	NetworkCredential credentials = null;	×
NEW 59	if (WebsenseCredentials is not null)	×
60	{
61	try
62	{
NEW 63	credentials =	×
NEW 64	new NetworkCredential(WebsenseCredentials.Username, WebsenseCredentials.GetDecryptedPassword());	×
NEW 65	}	×
NEW 66	catch (Exception)	×
67	{
NEW 68	credentials = null;	×
NEW 69	}	×
70	}
NEW 71	using (var fs = File.Create(destinationFile.FullName)) {	×
NEW 72	FetchRequest(fs, UriToFile.AbsoluteUri, credentials);	×
UNCOV 73	}	×
UNCOV 74	}	×
75
76	private static void FetchRequest(Stream output, string url, ICredentials credentials = null,
77	bool useCredentials = false)
78	{
79	using var httpClientHandler = new HttpClientHandler();	×
80	if (useCredentials && credentials is not null)	×
81	httpClientHandler.Credentials = credentials;	×
82	using var httpClient = new HttpClient(httpClientHandler, false)	×
83	{	×
84	Timeout = TimeSpan.FromSeconds(60)	×
85	};	×
86	httpClient.DefaultRequestHeaders.Add("User-Agent",	×
87	"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36");	×
88	using var response = httpClient.GetAsync(url).Result;	×
89	if (response.IsSuccessStatusCode)	×
90	{
NEW 91	response.Content.ReadAsStream().CopyTo(output);	×
92	return;	×
93	}
94
95	// Failed - retry with credentials?
96	if (!useCredentials && response.Headers.WwwAuthenticate.Any(static h =>	×
97	h.Scheme.Equals("basic", StringComparison.OrdinalIgnoreCase) &&	×
98	h.Parameter?.Equals("realm=\"Websense\"", StringComparison.OrdinalIgnoreCase) == true))	×
99	FetchRequest(output, response.Headers.Location?.AbsoluteUri, credentials, true);	×
100	else
101	{
102	throw new Exception(	×
103	$"Could not get response from {url} - {response.StatusCode} - {response.ReasonPhrase}");	×
104	}
UNCOV 105	}	×
106
107	public string GetDescription() => throw new NotImplementedException();	×
108
109	public IDataProvider Clone() => throw new NotImplementedException();	×
110
111	public bool Validate(ILoadDirectory _) =>
112	string.IsNullOrWhiteSpace(UriToFile?.PathAndQuery)	×
113	? throw new MissingFieldException(	×
114	"PathToFile is null or white space - should be populated externally as a parameter")	×
115	: true;	×
116
117
118	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
119	{
120	}	×
121
122
123	public void Check(ICheckNotifier notifier)
124	{
125	notifier.OnCheckPerformed(UriToFile == null	×
126	? new CheckEventArgs("No URI has been specified", CheckResult.Fail)	×
127	: new CheckEventArgs($"URI is:{UriToFile}", CheckResult.Success));	×
128	}	×
129	}

HicServices / RDMP / 22626536046

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous