10195230065

Committed 01 Aug 2024 08:57AM UTC coverage: 57.282% (-0.01%) from 57.295%

Build # 10195230065

Build Type

push

github

Committed by

JFriel

Commit Message

force timeout

Run Details

11072 of 20796 branches covered (53.24%)

Branch coverage included in aggregate %.

0 of 23 new or added lines in 1 file covered. (0.0%)

2 existing lines in 1 file now uncovered.

31313 of 53197 relevant lines covered (58.86%)

7884.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

#nullable enable

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
using System.Threading;
using FAnsi.Discovery;
using FluentFTP;
using FluentFTP.Model.Functions;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote FTP server to the ForLoading directory
/// 
/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
///  root folder - if not then configure redirection on the FTP, so you land in the correct directory).  Files are downloaded into the ForLoading folder</para>
/// </summary>
public class FTPDownloader : IPluginDataProvider
{
    private readonly Lazy<FtpClient> _connection;
    protected readonly List<string> _filesRetrieved = new();
    private ILoadDirectory? _directory;

    public FTPDownloader()
    {
        _connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);
    }

    [DemandsInitialization(
        "Determines the behaviour of the system when no files are found on the server.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
    public bool SendLoadNotRequiredIfFileNotFound { get; set; }

    [DemandsInitialization(
        "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
    public Regex? FilePattern { get; set; }

    [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
    public int TimeoutInSeconds { get; set; }

    [DemandsInitialization(
        "Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state).  This will only delete the files if they were actually fetched from the FTP server.  If the files were already in forLoading then the remote files are not deleted")]
    public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }

    [DemandsInitialization(
        "The FTP server to connect to.  Server should be specified with only IP:Port e.g. 127.0.0.1:20.  You do not have to specify ftp:// at the start",
        Mandatory = true)]
    public ExternalDatabaseServer? FTPServer { get; set; }

    [DemandsInitialization("The directory on the FTP server that you want to download files from")]
    public string? RemoteDirectory { get; set; }

    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
    public bool KeepAlive { get; set; }


    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
    {
        _directory = directory;
    }

    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);
    }

    private FtpClient SetupFtp()
    {
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
        var username = FTPServer.Username ?? "anonymous";
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
        var c = new FtpClient(host, username, password);
        if (TimeoutInSeconds > 0)
        {
            c.Config.ConnectTimeout = TimeoutInSeconds * 1000;
            c.Config.ReadTimeout = TimeoutInSeconds * 1000;
            c.Config.DataConnectionConnectTimeout = TimeoutInSeconds * 1000;
            c.Config.DataConnectionReadTimeout = TimeoutInSeconds * 1000;
        }
        // Enable periodic NOOP keepalive operations to keep connection active until we're done
        c.Config.Noop = true;
        List<FtpProfile> profileLists = new();
        if (TimeoutInSeconds is 0)
        {
            c.AutoConnect();

        }
        else
        {
            profileLists = c.AutoDetect(new FtpAutoDetectConfig
            {
                FirstOnly = true,
                CloneConnection = false
            });
            if (profileLists.Count > 0)
            {
                FtpProfile ftpProfile = profileLists[0];
                ftpProfile.Timeout = TimeoutInSeconds * 1000;
                c.LoadProfile(ftpProfile);
            }
        }


        return c;
    }

    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
    {
        var files = GetFileList().ToArray();

        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"Identified the following files on the FTP server:{string.Join(',', files)}"));

        var forLoadingContainedCachedFiles = false;

        foreach (var file in files)
        {
            var action = GetSkipActionForFile(file, destination);

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"File {file} was evaluated as {action}"));

            switch (action)
            {
                case SkipReason.DoNotSkip:
                    listener.OnNotify(this,
                        new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
                    Download(file, destination);
                    break;
                case SkipReason.InForLoading:
                    forLoadingContainedCachedFiles = true;
                    break;
            }
        }

        // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
        if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound)
            return ExitCodeType.Success;

        // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
        listener.OnNotify(this,
            new NotifyEventArgs(ProgressEventType.Information,
                "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
        return ExitCodeType.OperationNotRequired;
    }

    protected enum SkipReason
    {
        DoNotSkip,
        InForLoading,
        DidNotMatchPattern,
        IsImaginaryFile
    }

    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
    {
        if (file.StartsWith(".", StringComparison.Ordinal))
            return SkipReason.IsImaginaryFile;

        //if there is a regex pattern
        if (FilePattern?.IsMatch(file) == false) //and it does not match
            return SkipReason.DidNotMatchPattern; //skip because it did not match pattern

        //if the file on the FTP already exists in the forLoading directory, skip it
        return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;
    }


    private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
        SslPolicyErrors _4) => true; //any cert will do! yay


    protected virtual IEnumerable<string> GetFileList()
    {
        return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
    }

    protected virtual void Download(string file, ILoadDirectory destination)
    {
        var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
            ? $"{RemoteDirectory}/{file}"
            : file;

        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
        _connection.Value.DownloadFile(destinationFileName, remotePath);
        _filesRetrieved.Add(remotePath);
    }

    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;

        // Force a reconnection attempt if we got cut off
        if (!_connection.Value.IsStillConnected())
            _connection.Value.Connect(true);
        foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
    }


    public void Check(ICheckNotifier notifier)
    {
        try
        {
            SetupFtp();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	#nullable enable
8
9	using System;
10	using System.Collections.Generic;
11	using System.IO;
12	using System.Linq;
13	using System.Net.Security;
14	using System.Security.Cryptography.X509Certificates;
15	using System.Text.RegularExpressions;
16	using System.Threading;
17	using FAnsi.Discovery;
18	using FluentFTP;
19	using FluentFTP.Model.Functions;
20	using Rdmp.Core.Curation;
21	using Rdmp.Core.Curation.Data;
22	using Rdmp.Core.DataFlowPipeline;
23	using Rdmp.Core.DataLoad.Engine.DataProvider;
24	using Rdmp.Core.DataLoad.Engine.Job;
25	using Rdmp.Core.ReusableLibraryCode.Checks;
26	using Rdmp.Core.ReusableLibraryCode.Progress;
27
28	namespace Rdmp.Core.DataLoad.Modules.FTP;
29
30	/// <summary>
31	/// load component which downloads files from a remote FTP server to the ForLoading directory
32	///
33	/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
34	/// root folder - if not then configure redirection on the FTP, so you land in the correct directory). Files are downloaded into the ForLoading folder</para>
35	/// </summary>
36	public class FTPDownloader : IPluginDataProvider
37	{
38	private readonly Lazy<FtpClient> _connection;
39	protected readonly List<string> _filesRetrieved = new();	×
40	private ILoadDirectory? _directory;
41
42	public FTPDownloader()	×
43	{
44	_connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);	×
45	}	×
46
47	[DemandsInitialization(
48	"Determines the behaviour of the system when no files are found on the server. If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
49	public bool SendLoadNotRequiredIfFileNotFound { get; set; }	×
50
51	[DemandsInitialization(
52	"The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
53	public Regex? FilePattern { get; set; }	×
54
55	[DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
56	public int TimeoutInSeconds { get; set; }	×
57
58	[DemandsInitialization(
59	"Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state). This will only delete the files if they were actually fetched from the FTP server. If the files were already in forLoading then the remote files are not deleted")]
60	public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }	×
61
62	[DemandsInitialization(
63	"The FTP server to connect to. Server should be specified with only IP:Port e.g. 127.0.0.1:20. You do not have to specify ftp:// at the start",
64	Mandatory = true)]
65	public ExternalDatabaseServer? FTPServer { get; set; }	×
66
67	[DemandsInitialization("The directory on the FTP server that you want to download files from")]
68	public string? RemoteDirectory { get; set; }	×
69
70	[DemandsInitialization("True to set keep alive", DefaultValue = true)]
71	public bool KeepAlive { get; set; }	×
72
73
74	public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
75	{
76	_directory = directory;	×
77	}	×
78
79	public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
80	{
NEW 81	return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);	×
82	}
83
84	private FtpClient SetupFtp()
85	{
86	var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");	×
87	var username = FTPServer.Username ?? "anonymous";	×
88	var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();	×
89	var c = new FtpClient(host, username, password);	×
90	if (TimeoutInSeconds > 0)	×
91	{
92	c.Config.ConnectTimeout = TimeoutInSeconds * 1000;	×
93	c.Config.ReadTimeout = TimeoutInSeconds * 1000;	×
94	c.Config.DataConnectionConnectTimeout = TimeoutInSeconds * 1000;	×
95	c.Config.DataConnectionReadTimeout = TimeoutInSeconds * 1000;	×
96	}
97	// Enable periodic NOOP keepalive operations to keep connection active until we're done
98	c.Config.Noop = true;	×
NEW 99	List<FtpProfile> profileLists = new();	×
NEW 100	if (TimeoutInSeconds is 0)	×
101	{
NEW 102	c.AutoConnect();	×
103
104	}
105	else
106	{
NEW 107	profileLists = c.AutoDetect(new FtpAutoDetectConfig	×
NEW 108	{	×
NEW 109	FirstOnly = true,	×
NEW 110	CloneConnection = false	×
NEW 111	});	×
NEW 112	if (profileLists.Count > 0)	×
113	{
NEW 114	FtpProfile ftpProfile = profileLists[0];	×
NEW 115	ftpProfile.Timeout = TimeoutInSeconds * 1000;	×
NEW 116	c.LoadProfile(ftpProfile);	×
117	}
118	}
119
120
UNCOV 121	return c;	×
122	}
123
124	private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
125	{
126	var files = GetFileList().ToArray();	×
127
NEW 128	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
NEW 129	$"Identified the following files on the FTP server:{string.Join(',', files)}"));	×
130
131	var forLoadingContainedCachedFiles = false;	×
132
133	foreach (var file in files)	×
134	{
NEW 135	var action = GetSkipActionForFile(file, destination);	×
136
NEW 137	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
138	$"File {file} was evaluated as {action}"));	×
139
140	switch (action)
141	{
142	case SkipReason.DoNotSkip:
143	listener.OnNotify(this,	×
NEW 144	new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));	×
NEW 145	Download(file, destination);	×
UNCOV 146	break;	×
147	case SkipReason.InForLoading:
148	forLoadingContainedCachedFiles = true;	×
149	break;
150	}
151	}
152
153	// it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
154	if (forLoadingContainedCachedFiles \|\| _filesRetrieved.Count != 0 \|\| !SendLoadNotRequiredIfFileNotFound)	×
155	return ExitCodeType.Success;	×
156
157	// if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
158	listener.OnNotify(this,	×
159	new NotifyEventArgs(ProgressEventType.Information,	×
160	"Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));	×
161	return ExitCodeType.OperationNotRequired;	×
162	}
163
164	protected enum SkipReason
165	{
166	DoNotSkip,
167	InForLoading,
168	DidNotMatchPattern,
169	IsImaginaryFile
170	}
171
172	protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
173	{
NEW 174	if (file.StartsWith(".", StringComparison.Ordinal))	×
175	return SkipReason.IsImaginaryFile;	×
176
177	//if there is a regex pattern
178	if (FilePattern?.IsMatch(file) == false) //and it does not match	×
179	return SkipReason.DidNotMatchPattern; //skip because it did not match pattern	×
180
181	//if the file on the FTP already exists in the forLoading directory, skip it
182	return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;	×
183	}
184
185
186	private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
187	SslPolicyErrors _4) => true; //any cert will do! yay	×
188
189
190	protected virtual IEnumerable<string> GetFileList()
191	{
192	return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);	×
193	}
194
195	protected virtual void Download(string file, ILoadDirectory destination)
196	{
197	var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)	×
198	? $"{RemoteDirectory}/{file}"	×
199	: file;	×
200
NEW 201	var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);	×
NEW 202	_connection.Value.DownloadFile(destinationFileName, remotePath);	×
203	_filesRetrieved.Add(remotePath);	×
204	}	×
205
206	public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
207	{
208	if (exitCode != ExitCodeType.Success \|\| !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;	×
209
210	// Force a reconnection attempt if we got cut off
211	if (!_connection.Value.IsStillConnected())	×
212	_connection.Value.Connect(true);	×
213	foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);	×
214	}	×
215
216
217	public void Check(ICheckNotifier notifier)
218	{
219	try
220	{
221	SetupFtp();	×
222	}	×
223	catch (Exception e)	×
224	{
NEW 225	notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));	×
226	}	×
227	}	×
228	}

HicServices / RDMP / 10195230065

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous