• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 22626536046

03 Mar 2026 02:04PM UTC coverage: 57.18% (-0.004%) from 57.184%
22626536046

push

github

web-flow
Merge pull request #2318 from HicServices/bugfix/RDMP-340-hashing-issue

fix file lock in web downloader

11513 of 21629 branches covered (53.23%)

Branch coverage included in aggregate %.

1 of 12 new or added lines in 2 files covered. (8.33%)

3 existing lines in 1 file now uncovered.

32592 of 55505 relevant lines covered (58.72%)

17686.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Rdmp.Core/DataLoad/Modules/Web/WebFileDownloader.cs
1
// Copyright (c) The University of Dundee 2018-2019
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
using System;
8
using System.Diagnostics;
9
using System.IO;
10
using System.Linq;
11
using System.Net;
12
using System.Net.Http;
13
using System.Threading.Tasks;
14
using FAnsi.Discovery;
15
using Rdmp.Core.Curation;
16
using Rdmp.Core.Curation.Data;
17
using Rdmp.Core.DataFlowPipeline;
18
using Rdmp.Core.DataLoad.Engine.DataProvider;
19
using Rdmp.Core.DataLoad.Engine.Job;
20
using Rdmp.Core.ReusableLibraryCode.Checks;
21
using Rdmp.Core.ReusableLibraryCode.Progress;
22
using MissingFieldException = System.MissingFieldException;
23

24
namespace Rdmp.Core.DataLoad.Modules.Web;
25

26
/// <summary>
27
/// Data load component which downloads a file from a remote URL (e.g. http) into the ForLoading directory of the load.
28
/// </summary>
29
public class WebFileDownloader : IPluginDataProvider
30
{
31
    [DemandsInitialization(
32
        "The full URI to a file that will be downloaded into project ForLoading directory, must be a valid Uri",
33
        Mandatory = true)]
34
    public Uri UriToFile { get; set; }
×
35

36
    [DemandsInitialization(
37
        "Optional Username/password to use for network Websense challenges, these will be provided to the WebRequest as a NetworkCredential")]
38
    public DataAccessCredentials WebsenseCredentials { get; set; }
×
39

40
    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
41
    {
42
    }
×
43

44
    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
45
    {
46
        var t = Stopwatch.StartNew();
×
47
        var destinationFile =
×
48
            new FileInfo(Path.Combine(job.LoadDirectory.ForLoading.FullName, Path.GetFileName(UriToFile.LocalPath)));
×
49
        DownloadFileWhilstPretendingToBeFirefox(destinationFile, job);
×
50
        job.OnProgress(this,
×
51
            new ProgressEventArgs(destinationFile.FullName,
×
52
                new ProgressMeasurement((int)(destinationFile.Length / 1000), ProgressType.Kilobytes), t.Elapsed));
×
53
        return ExitCodeType.Success;
×
54
    }
55

56
    private void DownloadFileWhilstPretendingToBeFirefox(FileInfo destinationFile, IDataLoadJob job)
57
    {
NEW
58
        NetworkCredential credentials = null;
×
NEW
59
        if (WebsenseCredentials is not null)
×
60
        {
61
            try
62
            {
NEW
63
                credentials =
×
NEW
64
                    new NetworkCredential(WebsenseCredentials.Username, WebsenseCredentials.GetDecryptedPassword());
×
NEW
65
            }
×
NEW
66
            catch (Exception)
×
67
            {
NEW
68
                credentials = null;
×
NEW
69
            }
×
70
        }
NEW
71
        using (var fs = File.Create(destinationFile.FullName)) {
×
NEW
72
            FetchRequest(fs, UriToFile.AbsoluteUri, credentials);
×
UNCOV
73
        }
×
UNCOV
74
    }
×
75

76
    private static void FetchRequest(Stream output, string url, ICredentials credentials = null,
77
        bool useCredentials = false)
78
    {
79
        using var httpClientHandler = new HttpClientHandler();
×
80
        if (useCredentials && credentials is not null)
×
81
            httpClientHandler.Credentials = credentials;
×
82
        using var httpClient = new HttpClient(httpClientHandler, false)
×
83
        {
×
84
            Timeout = TimeSpan.FromSeconds(60)
×
85
        };
×
86
        httpClient.DefaultRequestHeaders.Add("User-Agent",
×
87
            "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36");
×
88
        using var response = httpClient.GetAsync(url).Result;
×
89
        if (response.IsSuccessStatusCode)
×
90
        {
NEW
91
            response.Content.ReadAsStream().CopyTo(output);
×
92
            return;
×
93
        }
94

95
        // Failed - retry with credentials?
96
        if (!useCredentials && response.Headers.WwwAuthenticate.Any(static h =>
×
97
                h.Scheme.Equals("basic", StringComparison.OrdinalIgnoreCase) &&
×
98
                h.Parameter?.Equals("realm=\"Websense\"", StringComparison.OrdinalIgnoreCase) == true))
×
99
            FetchRequest(output, response.Headers.Location?.AbsoluteUri, credentials, true);
×
100
        else
101
        {
102
            throw new Exception(
×
103
                $"Could not get response from {url} - {response.StatusCode} - {response.ReasonPhrase}");
×
104
        }
UNCOV
105
    }
×
106

107
    public string GetDescription() => throw new NotImplementedException();
×
108

109
    public IDataProvider Clone() => throw new NotImplementedException();
×
110

111
    public bool Validate(ILoadDirectory _) =>
112
        string.IsNullOrWhiteSpace(UriToFile?.PathAndQuery)
×
113
            ? throw new MissingFieldException(
×
114
                "PathToFile is null or white space - should be populated externally as a parameter")
×
115
            : true;
×
116

117

118
    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
119
    {
120
    }
×
121

122

123
    public void Check(ICheckNotifier notifier)
124
    {
125
        notifier.OnCheckPerformed(UriToFile == null
×
126
            ? new CheckEventArgs("No URI has been specified", CheckResult.Fail)
×
127
            : new CheckEventArgs($"URI is:{UriToFile}", CheckResult.Success));
×
128
    }
×
129
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc