• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 6237307473

19 Sep 2023 04:02PM UTC coverage: 57.015% (-0.4%) from 57.44%
6237307473

push

github

web-flow
Feature/rc4 (#1570)

* Syntax tidying
* Dependency updates
* Event handling singletons (ThrowImmediately and co)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: James A Sutherland <>
Co-authored-by: James Friel <jfriel001@dundee.ac.uk>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

10734 of 20259 branches covered (0.0%)

Branch coverage included in aggregate %.

5922 of 5922 new or added lines in 565 files covered. (100.0%)

30687 of 52390 relevant lines covered (58.57%)

7361.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs
1
// Copyright (c) The University of Dundee 2018-2019
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
using System;
8
using System.Collections.Generic;
9
using System.Diagnostics;
10
using System.IO;
11
using System.Linq;
12
using System.Net;
13
using System.Net.Security;
14
using System.Security.Cryptography.X509Certificates;
15
using System.Text;
16
using System.Text.RegularExpressions;
17
using FAnsi.Discovery;
18
using Rdmp.Core.Curation;
19
using Rdmp.Core.Curation.Data;
20
using Rdmp.Core.DataFlowPipeline;
21
using Rdmp.Core.DataLoad.Engine.DataProvider;
22
using Rdmp.Core.DataLoad.Engine.Job;
23
using Rdmp.Core.ReusableLibraryCode.Checks;
24
using Rdmp.Core.ReusableLibraryCode.Progress;
25

26
namespace Rdmp.Core.DataLoad.Modules.FTP;
27

28
/// <summary>
29
/// load component which downloads files from a remote FTP server to the ForLoading directory
30
/// 
31
/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
32
///  root folder - if not then configure redirection on the FTP so you land in the correct directory).  Files are downloaded into the ForLoading folder</para>
33
/// </summary>
34
public class FTPDownloader : IPluginDataProvider
35
{
36
    protected string _host;
37
    protected string _username;
38
    protected string _password;
39

40
    private bool _useSSL = false;
41

42
    protected List<string> _filesRetrieved = new();
×
43
    private ILoadDirectory _directory;
44

45
    [DemandsInitialization(
46
        "Determines the behaviour of the system when no files are found on the server.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
47
    public bool SendLoadNotRequiredIfFileNotFound { get; set; }
×
48

49
    [DemandsInitialization(
50
        "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
51
    public Regex FilePattern { get; set; }
×
52

53
    [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
54
    public int TimeoutInSeconds { get; set; }
×
55

56
    [DemandsInitialization(
57
        "Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state).  This will only delete the files if they were actually fetched from the FTP server.  If the files were already in forLoading then the remote files are not deleted")]
58
    public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }
×
59

60
    [DemandsInitialization(
61
        "The FTP server to connect to.  Server should be specified with only IP:Port e.g. 127.0.0.1:20.  You do not have to specify ftp:// at the start",
62
        Mandatory = true)]
63
    public ExternalDatabaseServer FTPServer { get; set; }
×
64

65
    [DemandsInitialization("The directory on the FTP server that you want to download files from")]
66
    public string RemoteDirectory { get; set; }
×
67

68
    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
69
    public bool KeepAlive { get; set; }
×
70

71

72
    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
73
    {
74
        _directory = directory;
×
75
    }
×
76

77
    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
78
    {
79
        SetupFTP();
×
80
        return DownloadFilesOnFTP(_directory, job);
×
81
    }
82

83
    public static string GetDescription() => "See Description attribute of class";
×
84

85
    public static IDataProvider Clone() => new FTPDownloader();
×
86

87
    public bool Validate(ILoadDirectory destination)
88
    {
89
        SetupFTP();
×
90
        return GetFileList().Any();
×
91
    }
92

93
    private void SetupFTP()
94
    {
95
        _host = FTPServer.Server;
×
96
        _username = FTPServer.Username ?? "anonymous";
×
97
        _password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
×
98

99
        if (string.IsNullOrWhiteSpace(_host))
×
100
            throw new NullReferenceException(
×
101
                $"FTPServer is not set up correctly it must have Server property filled in{FTPServer}");
×
102
    }
×
103

104
    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
105
    {
106
        var files = GetFileList();
×
107

108
        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, files.Aggregate(
×
109
            "Identified the following files on the FTP server:", (s, f) =>
×
110
                $"{f},").TrimEnd(',')));
×
111

112
        var forLoadingContainedCachedFiles = false;
×
113

114
        foreach (var file in files)
×
115
        {
116
            var action = GetSkipActionForFile(file, destination);
×
117

118
            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
×
119
                $"File {file} was evaluated as {action}"));
×
120
            if (action == SkipReason.DoNotSkip)
×
121
            {
122
                listener.OnNotify(this,
×
123
                    new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
×
124
                Download(file, destination, listener);
×
125
            }
126

127
            if (action == SkipReason.InForLoading)
×
128
                forLoadingContainedCachedFiles = true;
×
129
        }
130

131
        //if no files were downloaded (and there were none skiped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
132
        if (!forLoadingContainedCachedFiles && !_filesRetrieved.Any() && SendLoadNotRequiredIfFileNotFound)
×
133
        {
134
            listener.OnNotify(this,
×
135
                new NotifyEventArgs(ProgressEventType.Information,
×
136
                    "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
×
137
            return ExitCodeType.OperationNotRequired;
×
138
        }
139

140
        //otherwise it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
141
        return ExitCodeType.Success;
×
142
    }
143

144
    protected enum SkipReason
145
    {
146
        DoNotSkip,
147
        InForLoading,
148
        DidNotMatchPattern,
149
        IsImaginaryFile
150
    }
151

152
    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
153
    {
154
        if (file.StartsWith("."))
×
155
            return SkipReason.IsImaginaryFile;
×
156

157
        //if there is a regex pattern
158
        if (FilePattern != null)
×
159
            if (!FilePattern.IsMatch(file)) //and it does not match
×
160
                return SkipReason.DidNotMatchPattern; //skip because it did not match pattern
×
161

162
        //if the file on the FTP already exists in the forLoading directory, skip it
163
        return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;
×
164
    }
165

166

167
    private bool ValidateServerCertificate(object sender, X509Certificate certificate, X509Chain chain,
168
        SslPolicyErrors sslpolicyerrors) => true; //any cert will do! yay
×
169

170

171
    protected virtual string[] GetFileList()
172
    {
173
        var result = new StringBuilder();
×
174
        WebResponse response = null;
×
175
        StreamReader reader = null;
×
176
        try
177
        {
178
            var uri = !string.IsNullOrWhiteSpace(RemoteDirectory)
×
179
                ? $"ftp://{_host}/{RemoteDirectory}"
×
180
                : $"ftp://{_host}";
×
181

182
#pragma warning disable SYSLIB0014 // Type or member is obsolete
183
            var reqFTP = (FtpWebRequest)WebRequest.Create(new Uri(uri));
×
184
#pragma warning restore SYSLIB0014 // Type or member is obsolete
185
            reqFTP.UseBinary = true;
×
186
            reqFTP.Credentials = new NetworkCredential(_username, _password);
×
187
            reqFTP.Method = WebRequestMethods.Ftp.ListDirectory;
×
188
            reqFTP.Timeout = TimeoutInSeconds * 1000;
×
189
            reqFTP.KeepAlive = KeepAlive;
×
190

191
            reqFTP.Proxy = null;
×
192
            reqFTP.KeepAlive = false;
×
193
            reqFTP.UsePassive = true;
×
194
            reqFTP.EnableSsl = _useSSL;
×
195

196
            //accept any certificates
197
            ServicePointManager.ServerCertificateValidationCallback = ValidateServerCertificate;
×
198
            response = reqFTP.GetResponse();
×
199

200
            reader = new StreamReader(response.GetResponseStream());
×
201
            var line = reader.ReadLine();
×
202
            while (line != null)
×
203
            {
204
                result.Append(line);
×
205
                result.Append('\n');
×
206
                line = reader.ReadLine();
×
207
            }
208

209
            // to remove the trailing '\n'
210
            result.Remove(result.ToString().LastIndexOf('\n'), 1);
×
211
            return result.ToString().Split('\n');
×
212
        }
213
        finally
214
        {
215
            reader?.Close();
×
216

217
            response?.Close();
×
218
        }
×
219
    }
×
220

221
    protected virtual void Download(string file, ILoadDirectory destination, IDataLoadEventListener job)
222
    {
223
        var s = new Stopwatch();
×
224
        s.Start();
×
225

226
        var uri = !string.IsNullOrWhiteSpace(RemoteDirectory)
×
227
            ? $"ftp://{_host}/{RemoteDirectory}/{file}"
×
228
            : $"ftp://{_host}/{file}";
×
229

230
        if (_useSSL)
×
231
            uri = $"s{uri}";
×
232

233
        var serverUri = new Uri(uri);
×
234
        if (serverUri.Scheme != Uri.UriSchemeFtp) return;
×
235

236
#pragma warning disable SYSLIB0014 // Type or member is obsolete
237
        var reqFTP = (FtpWebRequest)WebRequest.Create(new Uri(uri));
×
238
#pragma warning restore SYSLIB0014 // Type or member is obsolete
239
        reqFTP.Credentials = new NetworkCredential(_username, _password);
×
240
        reqFTP.KeepAlive = false;
×
241
        reqFTP.Method = WebRequestMethods.Ftp.DownloadFile;
×
242
        reqFTP.UseBinary = true;
×
243
        reqFTP.Proxy = null;
×
244
        reqFTP.UsePassive = true;
×
245
        reqFTP.EnableSsl = _useSSL;
×
246
        reqFTP.Timeout = TimeoutInSeconds * 1000;
×
247

248
        var response = (FtpWebResponse)reqFTP.GetResponse();
×
249
        var responseStream = response.GetResponseStream();
×
250
        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
×
251

252
        using (var writeStream = new FileStream(destinationFileName, FileMode.Create))
×
253
        {
254
            responseStream.CopyTo(writeStream);
×
255
            writeStream.Close();
×
256
        }
×
257

258
        response.Close();
×
259

260
        _filesRetrieved.Add(serverUri.ToString());
×
261
        s.Stop();
×
262
    }
×
263

264
    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
265
    {
266
        if (exitCode == ExitCodeType.Success && DeleteFilesOffFTPServerAfterSuccesfulDataLoad)
×
267
            foreach (var file in _filesRetrieved)
×
268
            {
269
#pragma warning disable SYSLIB0014
270
                // Type or member is obsolete
271
                var reqFTP = (FtpWebRequest)WebRequest.Create(new Uri(file));
×
272
#pragma warning restore SYSLIB0014
273
                // Type or member is obsolete
274
                reqFTP.Credentials = new NetworkCredential(_username, _password);
×
275
                reqFTP.KeepAlive = false;
×
276
                reqFTP.Method = WebRequestMethods.Ftp.DeleteFile;
×
277
                reqFTP.UseBinary = true;
×
278
                reqFTP.Proxy = null;
×
279
                reqFTP.UsePassive = true;
×
280
                reqFTP.EnableSsl = _useSSL;
×
281

282
                var response = (FtpWebResponse)reqFTP.GetResponse();
×
283

284
                if (response.StatusCode != FtpStatusCode.FileActionOK)
×
285
                    postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
×
286
                        $"Attempt to delete file at URI {file} resulted in response with StatusCode = {response.StatusCode}"));
×
287
                else
288
                    postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
×
289
                        $"Deleted FTP file at URI {file} status code was {response.StatusCode}"));
×
290

291
                response.Close();
×
292
            }
293
    }
×
294

295

296
    public void Check(ICheckNotifier notifier)
297
    {
298
        try
299
        {
300
            SetupFTP();
×
301
        }
×
302
        catch (Exception e)
×
303
        {
304
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
×
305
        }
×
306
    }
×
307
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc