• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 10195230065

01 Aug 2024 08:57AM UTC coverage: 57.282% (-0.01%) from 57.295%
10195230065

push

github

JFriel
force timeout

11072 of 20796 branches covered (53.24%)

Branch coverage included in aggregate %.

0 of 23 new or added lines in 1 file covered. (0.0%)

2 existing lines in 1 file now uncovered.

31313 of 53197 relevant lines covered (58.86%)

7884.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs
1
// Copyright (c) The University of Dundee 2018-2019
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
#nullable enable
8

9
using System;
10
using System.Collections.Generic;
11
using System.IO;
12
using System.Linq;
13
using System.Net.Security;
14
using System.Security.Cryptography.X509Certificates;
15
using System.Text.RegularExpressions;
16
using System.Threading;
17
using FAnsi.Discovery;
18
using FluentFTP;
19
using FluentFTP.Model.Functions;
20
using Rdmp.Core.Curation;
21
using Rdmp.Core.Curation.Data;
22
using Rdmp.Core.DataFlowPipeline;
23
using Rdmp.Core.DataLoad.Engine.DataProvider;
24
using Rdmp.Core.DataLoad.Engine.Job;
25
using Rdmp.Core.ReusableLibraryCode.Checks;
26
using Rdmp.Core.ReusableLibraryCode.Progress;
27

28
namespace Rdmp.Core.DataLoad.Modules.FTP;
29

30
/// <summary>
31
/// load component which downloads files from a remote FTP server to the ForLoading directory
32
/// 
33
/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
34
///  root folder - if not then configure redirection on the FTP, so you land in the correct directory).  Files are downloaded into the ForLoading folder</para>
35
/// </summary>
36
public class FTPDownloader : IPluginDataProvider
37
{
38
    private readonly Lazy<FtpClient> _connection;
39
    protected readonly List<string> _filesRetrieved = new();
×
40
    private ILoadDirectory? _directory;
41

42
    public FTPDownloader()
×
43
    {
44
        _connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);
×
45
    }
×
46

47
    [DemandsInitialization(
48
        "Determines the behaviour of the system when no files are found on the server.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
49
    public bool SendLoadNotRequiredIfFileNotFound { get; set; }
×
50

51
    [DemandsInitialization(
52
        "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
53
    public Regex? FilePattern { get; set; }
×
54

55
    [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
56
    public int TimeoutInSeconds { get; set; }
×
57

58
    [DemandsInitialization(
59
        "Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state).  This will only delete the files if they were actually fetched from the FTP server.  If the files were already in forLoading then the remote files are not deleted")]
60
    public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }
×
61

62
    [DemandsInitialization(
63
        "The FTP server to connect to.  Server should be specified with only IP:Port e.g. 127.0.0.1:20.  You do not have to specify ftp:// at the start",
64
        Mandatory = true)]
65
    public ExternalDatabaseServer? FTPServer { get; set; }
×
66

67
    [DemandsInitialization("The directory on the FTP server that you want to download files from")]
68
    public string? RemoteDirectory { get; set; }
×
69

70
    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
71
    public bool KeepAlive { get; set; }
×
72

73

74
    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
75
    {
76
        _directory = directory;
×
77
    }
×
78

79
    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
80
    {
NEW
81
        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);
×
82
    }
83

84
    private FtpClient SetupFtp()
85
    {
86
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
×
87
        var username = FTPServer.Username ?? "anonymous";
×
88
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
×
89
        var c = new FtpClient(host, username, password);
×
90
        if (TimeoutInSeconds > 0)
×
91
        {
92
            c.Config.ConnectTimeout = TimeoutInSeconds * 1000;
×
93
            c.Config.ReadTimeout = TimeoutInSeconds * 1000;
×
94
            c.Config.DataConnectionConnectTimeout = TimeoutInSeconds * 1000;
×
95
            c.Config.DataConnectionReadTimeout = TimeoutInSeconds * 1000;
×
96
        }
97
        // Enable periodic NOOP keepalive operations to keep connection active until we're done
98
        c.Config.Noop = true;
×
NEW
99
        List<FtpProfile> profileLists = new();
×
NEW
100
        if (TimeoutInSeconds is 0)
×
101
        {
NEW
102
            c.AutoConnect();
×
103

104
        }
105
        else
106
        {
NEW
107
            profileLists = c.AutoDetect(new FtpAutoDetectConfig
×
NEW
108
            {
×
NEW
109
                FirstOnly = true,
×
NEW
110
                CloneConnection = false
×
NEW
111
            });
×
NEW
112
            if (profileLists.Count > 0)
×
113
            {
NEW
114
                FtpProfile ftpProfile = profileLists[0];
×
NEW
115
                ftpProfile.Timeout = TimeoutInSeconds * 1000;
×
NEW
116
                c.LoadProfile(ftpProfile);
×
117
            }
118
        }
119

120

UNCOV
121
        return c;
×
122
    }
123

124
    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
125
    {
126
        var files = GetFileList().ToArray();
×
127

NEW
128
        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
×
NEW
129
            $"Identified the following files on the FTP server:{string.Join(',', files)}"));
×
130

131
        var forLoadingContainedCachedFiles = false;
×
132

133
        foreach (var file in files)
×
134
        {
NEW
135
            var action = GetSkipActionForFile(file, destination);
×
136

NEW
137
            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
×
138
                $"File {file} was evaluated as {action}"));
×
139

140
            switch (action)
141
            {
142
                case SkipReason.DoNotSkip:
143
                    listener.OnNotify(this,
×
NEW
144
                        new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
×
NEW
145
                    Download(file, destination);
×
UNCOV
146
                    break;
×
147
                case SkipReason.InForLoading:
148
                    forLoadingContainedCachedFiles = true;
×
149
                    break;
150
            }
151
        }
152

153
        // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
154
        if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound)
×
155
            return ExitCodeType.Success;
×
156

157
        // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
158
        listener.OnNotify(this,
×
159
            new NotifyEventArgs(ProgressEventType.Information,
×
160
                "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
×
161
        return ExitCodeType.OperationNotRequired;
×
162
    }
163

164
    protected enum SkipReason
165
    {
166
        DoNotSkip,
167
        InForLoading,
168
        DidNotMatchPattern,
169
        IsImaginaryFile
170
    }
171

172
    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
173
    {
NEW
174
        if (file.StartsWith(".", StringComparison.Ordinal))
×
175
            return SkipReason.IsImaginaryFile;
×
176

177
        //if there is a regex pattern
178
        if (FilePattern?.IsMatch(file) == false) //and it does not match
×
179
            return SkipReason.DidNotMatchPattern; //skip because it did not match pattern
×
180

181
        //if the file on the FTP already exists in the forLoading directory, skip it
182
        return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;
×
183
    }
184

185

186
    private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
187
        SslPolicyErrors _4) => true; //any cert will do! yay
×
188

189

190
    protected virtual IEnumerable<string> GetFileList()
191
    {
192
        return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
×
193
    }
194

195
    protected virtual void Download(string file, ILoadDirectory destination)
196
    {
197
        var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
×
198
            ? $"{RemoteDirectory}/{file}"
×
199
            : file;
×
200

NEW
201
        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
×
NEW
202
        _connection.Value.DownloadFile(destinationFileName, remotePath);
×
203
        _filesRetrieved.Add(remotePath);
×
204
    }
×
205

206
    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
207
    {
208
        if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;
×
209

210
        // Force a reconnection attempt if we got cut off
211
        if (!_connection.Value.IsStillConnected())
×
212
            _connection.Value.Connect(true);
×
213
        foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
×
214
    }
×
215

216

217
    public void Check(ICheckNotifier notifier)
218
    {
219
        try
220
        {
221
            SetupFtp();
×
222
        }
×
223
        catch (Exception e)
×
224
        {
NEW
225
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
×
226
        }
×
227
    }
×
228
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc