9521283302

Committed 14 Jun 2024 07:19PM UTC coverage: 56.904% (-0.009%) from 56.913%

Build # 9521283302

Build Type

Pull #1856

github

Committed by

jas88

Commit Message

FTP, SFTP timeouts and liveness checks

Pull Request Pull Request #1856: FTP, SFTP timeouts and liveness checks

Run Details

10817 of 20488 branches covered (52.8%)

Branch coverage included in aggregate %.

0 of 17 new or added lines in 2 files covered. (0.0%)

3 existing lines in 2 files now uncovered.

30827 of 52695 relevant lines covered (58.5%)

7418.92 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

#nullable enable

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
using System.Threading;
using FAnsi.Discovery;
using FluentFTP;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote FTP server to the ForLoading directory
/// 
/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
///  root folder - if not then configure redirection on the FTP, so you land in the correct directory).  Files are downloaded into the ForLoading folder</para>
/// </summary>
public class FTPDownloader : IPluginDataProvider
{
    private readonly Lazy<FtpClient> _connection;
    protected readonly List<string> _filesRetrieved = new();
    private ILoadDirectory? _directory;

    public FTPDownloader()
    {
        _connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);
    }

    [DemandsInitialization(
        "Determines the behaviour of the system when no files are found on the server.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
    public bool SendLoadNotRequiredIfFileNotFound { get; set; }

    [DemandsInitialization(
        "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
    public Regex? FilePattern { get; set; }

    [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
    public int TimeoutInSeconds { get; set; }

    [DemandsInitialization(
        "Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state).  This will only delete the files if they were actually fetched from the FTP server.  If the files were already in forLoading then the remote files are not deleted")]
    public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }

    [DemandsInitialization(
        "The FTP server to connect to.  Server should be specified with only IP:Port e.g. 127.0.0.1:20.  You do not have to specify ftp:// at the start",
        Mandatory = true)]
    public ExternalDatabaseServer? FTPServer { get; set; }

    [DemandsInitialization("The directory on the FTP server that you want to download files from")]
    public string? RemoteDirectory { get; set; }

    [DemandsInitialization("True to set keep alive",DefaultValue = true)]
    public bool KeepAlive { get; set; }


    public void Initialize(ILoadDirectory directory,DiscoveredDatabase dbInfo)
    {
        _directory = directory;
    }

    public ExitCodeType Fetch(IDataLoadJob job,GracefulCancellationToken cancellationToken)
    {
        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"),job);
    }

    private FtpClient SetupFtp()
    {
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
        var username = FTPServer.Username ?? "anonymous";
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
        var c = new FtpClient(host, username, password);

        // Enable periodic NOOP keepalive operations to keep connection active until we're done
        c.Config.Noop = true;
        c.AutoConnect();
        return c;
    }

    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination,IDataLoadEventListener listener)
    {
        var files = GetFileList().ToArray();

        listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,
            $"Identified the following files on the FTP server:{string.Join(',',files)}"));

        var forLoadingContainedCachedFiles = false;

        foreach (var file in files)
        {
            var action = GetSkipActionForFile(file,destination);

            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,
                $"File {file} was evaluated as {action}"));

            switch (action)
            {
                case SkipReason.DoNotSkip:
                    listener.OnNotify(this,
                        new NotifyEventArgs(ProgressEventType.Information,$"About to download {file}"));
                    Download(file,destination);
                    break;
                case SkipReason.InForLoading:
                    forLoadingContainedCachedFiles = true;
                    break;
            }
        }

        // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
        if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound)
            return ExitCodeType.Success;

        // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
        listener.OnNotify(this,
            new NotifyEventArgs(ProgressEventType.Information,
                "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
        return ExitCodeType.OperationNotRequired;
    }

    protected enum SkipReason
    {
        DoNotSkip,
        InForLoading,
        DidNotMatchPattern,
        IsImaginaryFile
    }

    protected SkipReason GetSkipActionForFile(string file,ILoadDirectory destination)
    {
        if (file.StartsWith(".",StringComparison.Ordinal))
            return SkipReason.IsImaginaryFile;

        //if there is a regex pattern
        if (FilePattern?.IsMatch(file) == false) //and it does not match
            return SkipReason.DidNotMatchPattern; //skip because it did not match pattern

        //if the file on the FTP already exists in the forLoading directory, skip it
        return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;
    }


    private static bool ValidateServerCertificate(object _1,X509Certificate _2,X509Chain _3,
        SslPolicyErrors _4) => true; //any cert will do! yay


    protected virtual IEnumerable<string> GetFileList()
    {
        return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
    }

    protected virtual void Download(string file,ILoadDirectory destination)
    {
        var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
            ? $"{RemoteDirectory}/{file}"
            : file;

        var destinationFileName = Path.Combine(destination.ForLoading.FullName,file);
        _connection.Value.DownloadFile(destinationFileName,remotePath);
        _filesRetrieved.Add(remotePath);
    }

    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode,IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;

        // Force a reconnection attempt if we got cut off
        if (!_connection.Value.IsStillConnected())
            _connection.Value.Connect(true);
        foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
    }


    public void Check(ICheckNotifier notifier)
    {
        try
        {
            SetupFtp();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP",CheckResult.Fail,e));
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	#nullable enable
8
9	using System;
10	using System.Collections.Generic;
11	using System.IO;
12	using System.Linq;
13	using System.Net.Security;
14	using System.Security.Cryptography.X509Certificates;
15	using System.Text.RegularExpressions;
16	using System.Threading;
17	using FAnsi.Discovery;
18	using FluentFTP;
19	using Rdmp.Core.Curation;
20	using Rdmp.Core.Curation.Data;
21	using Rdmp.Core.DataFlowPipeline;
22	using Rdmp.Core.DataLoad.Engine.DataProvider;
23	using Rdmp.Core.DataLoad.Engine.Job;
24	using Rdmp.Core.ReusableLibraryCode.Checks;
25	using Rdmp.Core.ReusableLibraryCode.Progress;
26
27	namespace Rdmp.Core.DataLoad.Modules.FTP;
28
29	/// <summary>
30	/// load component which downloads files from a remote FTP server to the ForLoading directory
31	///
32	/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
33	/// root folder - if not then configure redirection on the FTP, so you land in the correct directory). Files are downloaded into the ForLoading folder</para>
34	/// </summary>
35	public class FTPDownloader : IPluginDataProvider
36	{
37	private readonly Lazy<FtpClient> _connection;
38	protected readonly List<string> _filesRetrieved = new();	×
39	private ILoadDirectory? _directory;
40
41	public FTPDownloader()	×
42	{
43	_connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);	×
44	}	×
45
46	[DemandsInitialization(
47	"Determines the behaviour of the system when no files are found on the server. If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
48	public bool SendLoadNotRequiredIfFileNotFound { get; set; }	×
49
50	[DemandsInitialization(
51	"The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
52	public Regex? FilePattern { get; set; }	×
53
54	[DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
55	public int TimeoutInSeconds { get; set; }	×
56
57	[DemandsInitialization(
58	"Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state). This will only delete the files if they were actually fetched from the FTP server. If the files were already in forLoading then the remote files are not deleted")]
59	public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }	×
60
61	[DemandsInitialization(
62	"The FTP server to connect to. Server should be specified with only IP:Port e.g. 127.0.0.1:20. You do not have to specify ftp:// at the start",
63	Mandatory = true)]
64	public ExternalDatabaseServer? FTPServer { get; set; }	×
65
66	[DemandsInitialization("The directory on the FTP server that you want to download files from")]
67	public string? RemoteDirectory { get; set; }	×
68
69	[DemandsInitialization("True to set keep alive",DefaultValue = true)]
70	public bool KeepAlive { get; set; }	×
71
72
73	public void Initialize(ILoadDirectory directory,DiscoveredDatabase dbInfo)
74	{
75	_directory = directory;	×
76	}	×
77
78	public ExitCodeType Fetch(IDataLoadJob job,GracefulCancellationToken cancellationToken)
79	{
NEW 80	return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"),job);	×
81	}
82
83	private FtpClient SetupFtp()
84	{
85	var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");	×
86	var username = FTPServer.Username ?? "anonymous";	×
87	var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();	×
88	var c = new FtpClient(host, username, password);	×
89
90	// Enable periodic NOOP keepalive operations to keep connection active until we're done
NEW 91	c.Config.Noop = true;	×
92	c.AutoConnect();	×
93	return c;	×
94	}
95
96	private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination,IDataLoadEventListener listener)
97	{
98	var files = GetFileList().ToArray();	×
99
NEW 100	listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,	×
101	$"Identified the following files on the FTP server:{string.Join(',',files)}"));	×
102
103	var forLoadingContainedCachedFiles = false;	×
104
105	foreach (var file in files)	×
106	{
NEW 107	var action = GetSkipActionForFile(file,destination);	×
108
NEW 109	listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,	×
110	$"File {file} was evaluated as {action}"));	×
111
112	switch (action)
113	{
114	case SkipReason.DoNotSkip:
115	listener.OnNotify(this,	×
NEW 116	new NotifyEventArgs(ProgressEventType.Information,$"About to download {file}"));	×
NEW 117	Download(file,destination);	×
UNCOV 118	break;	×
119	case SkipReason.InForLoading:
120	forLoadingContainedCachedFiles = true;	×
121	break;
122	}
123	}
124
125	// it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
NEW 126	if (forLoadingContainedCachedFiles \|\| _filesRetrieved.Count != 0 \|\| !SendLoadNotRequiredIfFileNotFound)	×
127	return ExitCodeType.Success;	×
128
129	// if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
130	listener.OnNotify(this,	×
131	new NotifyEventArgs(ProgressEventType.Information,	×
132	"Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));	×
133	return ExitCodeType.OperationNotRequired;	×
134	}
135
136	protected enum SkipReason
137	{
138	DoNotSkip,
139	InForLoading,
140	DidNotMatchPattern,
141	IsImaginaryFile
142	}
143
144	protected SkipReason GetSkipActionForFile(string file,ILoadDirectory destination)
145	{
146	if (file.StartsWith(".",StringComparison.Ordinal))	×
147	return SkipReason.IsImaginaryFile;	×
148
149	//if there is a regex pattern
150	if (FilePattern?.IsMatch(file) == false) //and it does not match	×
151	return SkipReason.DidNotMatchPattern; //skip because it did not match pattern	×
152
153	//if the file on the FTP already exists in the forLoading directory, skip it
154	return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;	×
155	}
156
157
158	private static bool ValidateServerCertificate(object _1,X509Certificate _2,X509Chain _3,
159	SslPolicyErrors _4) => true; //any cert will do! yay	×
160
161
162	protected virtual IEnumerable<string> GetFileList()
163	{
164	return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);	×
165	}
166
167	protected virtual void Download(string file,ILoadDirectory destination)
168	{
169	var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)	×
170	? $"{RemoteDirectory}/{file}"	×
171	: file;	×
172
NEW 173	var destinationFileName = Path.Combine(destination.ForLoading.FullName,file);	×
NEW 174	_connection.Value.DownloadFile(destinationFileName,remotePath);	×
175	_filesRetrieved.Add(remotePath);	×
176	}	×
177
178	public virtual void LoadCompletedSoDispose(ExitCodeType exitCode,IDataLoadEventListener postLoadEventListener)
179	{
180	if (exitCode != ExitCodeType.Success \|\| !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;	×
181
182	// Force a reconnection attempt if we got cut off
NEW 183	if (!_connection.Value.IsStillConnected())	×
NEW 184	_connection.Value.Connect(true);	×
185	foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);	×
186	}	×
187
188
189	public void Check(ICheckNotifier notifier)
190	{
191	try
192	{
193	SetupFtp();	×
194	}	×
195	catch (Exception e)	×
196	{
NEW 197	notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP",CheckResult.Fail,e));	×
198	}	×
199	}	×
200	}

HicServices / RDMP / 9521283302

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous