10246247756

Committed 05 Aug 2024 09:28AM UTC coverage: 57.283% (-0.01%) from 57.295%

Build # 10246247756

Build Type

Pull #1913

github

Committed by

JFriel

Commit Message

tidy up

Pull Request Pull Request #1913: Fix SFTP Timeout Issue

Run Details

11090 of 20824 branches covered (53.26%)

Branch coverage included in aggregate %.

0 of 28 new or added lines in 2 files covered. (0.0%)

2 existing lines in 1 file now uncovered.

31368 of 53296 relevant lines covered (58.86%)

7931.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

#nullable enable

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
using System.Threading;
using FAnsi.Discovery;
using FluentFTP;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote FTP server to the ForLoading directory
/// 
/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
///  root folder - if not then configure redirection on the FTP, so you land in the correct directory).  Files are downloaded into the ForLoading folder</para>
/// </summary>
public class FTPDownloader : IPluginDataProvider
{
    private readonly Lazy<FtpClient> _connection;
    protected readonly List<string> _filesRetrieved = new();
    private ILoadDirectory? _directory;

    public FTPDownloader()
    {
        _connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);
    }

    [DemandsInitialization(
        "Determines the behaviour of the system when no files are found on the server.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
    public bool SendLoadNotRequiredIfFileNotFound { get; set; }

    [DemandsInitialization(
        "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
    public Regex? FilePattern { get; set; }

    [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
    public int TimeoutInSeconds { get; set; }

    [DemandsInitialization(
        "Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state).  This will only delete the files if they were actually fetched from the FTP server.  If the files were already in forLoading then the remote files are not deleted")]
    public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }

    [DemandsInitialization(
        "The FTP server to connect to.  Server should be specified with only IP:Port e.g. 127.0.0.1:20.  You do not have to specify ftp:// at the start",
        Mandatory = true)]
    public ExternalDatabaseServer? FTPServer { get; set; }

    [DemandsInitialization("The directory on the FTP server that you want to download files from")]
    public string? RemoteDirectory { get; set; }

    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
    public bool KeepAlive { get; set; }


    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
    {
        _directory = directory;
    }

    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);
    }

    private FtpClient SetupFtp()
    {
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
        var username = FTPServer.Username ?? "anonymous";
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
        var c = new FtpClient(host, username, password);
        if (TimeoutInSeconds > 0)
        {
            c.Config.ConnectTimeout = TimeoutInSeconds * 1000;
            c.Config.ReadTimeout = TimeoutInSeconds * 1000;
            c.Config.DataConnectionConnectTimeout = TimeoutInSeconds * 1000;
            c.Config.DataConnectionReadTimeout = TimeoutInSeconds * 1000;
        }
        // Enable periodic NOOP keepalive operations to keep connection active until we're done
        c.Config.Noop = true;
        c.AutoConnect();

        return c;
    }

    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
    {
        var files = GetFileList().ToArray();

        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"Identified the following files on the FTP server:{string.Join(',', files)}"));

        var forLoadingContainedCachedFiles = false;

        foreach (var file in files)
        {
            var action = GetSkipActionForFile(file, destination);

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"File {file} was evaluated as {action}"));

            switch (action)
            {
                case SkipReason.DoNotSkip:
                    listener.OnNotify(this,
                        new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
                    Download(file, destination);
                    break;
                case SkipReason.InForLoading:
                    forLoadingContainedCachedFiles = true;
                    break;
            }
        }

        // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
        if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound)
            return ExitCodeType.Success;

        // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
        listener.OnNotify(this,
            new NotifyEventArgs(ProgressEventType.Information,
                "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
        return ExitCodeType.OperationNotRequired;
    }

    protected enum SkipReason
    {
        DoNotSkip,
        InForLoading,
        DidNotMatchPattern,
        IsImaginaryFile
    }

    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
    {
        if (file.StartsWith(".", StringComparison.Ordinal))
            return SkipReason.IsImaginaryFile;

        //if there is a regex pattern
        if (FilePattern?.IsMatch(file) == false) //and it does not match
            return SkipReason.DidNotMatchPattern; //skip because it did not match pattern

        //if the file on the FTP already exists in the forLoading directory, skip it
        return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;
    }


    private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
        SslPolicyErrors _4) => true; //any cert will do! yay


    protected virtual IEnumerable<string> GetFileList()
    {
        return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
    }

    protected virtual void Download(string file, ILoadDirectory destination)
    {
        var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
            ? $"{RemoteDirectory}/{file}"
            : file;

        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
        _connection.Value.DownloadFile(destinationFileName, remotePath);
        _filesRetrieved.Add(remotePath);
    }

    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;

        // Force a reconnection attempt if we got cut off
        if (!_connection.Value.IsStillConnected())
            _connection.Value.Connect(true);
        foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
    }


    public virtual void Check(ICheckNotifier notifier)
    {
        try
        {
            SetupFtp();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	#nullable enable
8
9	using System;
10	using System.Collections.Generic;
11	using System.IO;
12	using System.Linq;
13	using System.Net.Security;
14	using System.Security.Cryptography.X509Certificates;
15	using System.Text.RegularExpressions;
16	using System.Threading;
17	using FAnsi.Discovery;
18	using FluentFTP;
19	using Rdmp.Core.Curation;
20	using Rdmp.Core.Curation.Data;
21	using Rdmp.Core.DataFlowPipeline;
22	using Rdmp.Core.DataLoad.Engine.DataProvider;
23	using Rdmp.Core.DataLoad.Engine.Job;
24	using Rdmp.Core.ReusableLibraryCode.Checks;
25	using Rdmp.Core.ReusableLibraryCode.Progress;
26
27	namespace Rdmp.Core.DataLoad.Modules.FTP;
28
29	/// <summary>
30	/// load component which downloads files from a remote FTP server to the ForLoading directory
31	///
32	/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
33	/// root folder - if not then configure redirection on the FTP, so you land in the correct directory). Files are downloaded into the ForLoading folder</para>
34	/// </summary>
35	public class FTPDownloader : IPluginDataProvider
36	{
37	private readonly Lazy<FtpClient> _connection;
38	protected readonly List<string> _filesRetrieved = new();	×
39	private ILoadDirectory? _directory;
40
41	public FTPDownloader()	×
42	{
43	_connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);	×
44	}	×
45
46	[DemandsInitialization(
47	"Determines the behaviour of the system when no files are found on the server. If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
48	public bool SendLoadNotRequiredIfFileNotFound { get; set; }	×
49
50	[DemandsInitialization(
51	"The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
52	public Regex? FilePattern { get; set; }	×
53
54	[DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
55	public int TimeoutInSeconds { get; set; }	×
56
57	[DemandsInitialization(
58	"Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state). This will only delete the files if they were actually fetched from the FTP server. If the files were already in forLoading then the remote files are not deleted")]
59	public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }	×
60
61	[DemandsInitialization(
62	"The FTP server to connect to. Server should be specified with only IP:Port e.g. 127.0.0.1:20. You do not have to specify ftp:// at the start",
63	Mandatory = true)]
64	public ExternalDatabaseServer? FTPServer { get; set; }	×
65
66	[DemandsInitialization("The directory on the FTP server that you want to download files from")]
67	public string? RemoteDirectory { get; set; }	×
68
69	[DemandsInitialization("True to set keep alive", DefaultValue = true)]
70	public bool KeepAlive { get; set; }	×
71
72
73	public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
74	{
75	_directory = directory;	×
76	}	×
77
78	public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
79	{
NEW 80	return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);	×
81	}
82
83	private FtpClient SetupFtp()
84	{
85	var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");	×
86	var username = FTPServer.Username ?? "anonymous";	×
87	var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();	×
88	var c = new FtpClient(host, username, password);	×
NEW 89	if (TimeoutInSeconds > 0)	×
90	{
NEW 91	c.Config.ConnectTimeout = TimeoutInSeconds * 1000;	×
NEW 92	c.Config.ReadTimeout = TimeoutInSeconds * 1000;	×
NEW 93	c.Config.DataConnectionConnectTimeout = TimeoutInSeconds * 1000;	×
NEW 94	c.Config.DataConnectionReadTimeout = TimeoutInSeconds * 1000;	×
95	}
96	// Enable periodic NOOP keepalive operations to keep connection active until we're done
97	c.Config.Noop = true;	×
98	c.AutoConnect();	×
99
UNCOV 100	return c;	×
101	}
102
103	private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
104	{
105	var files = GetFileList().ToArray();	×
106
NEW 107	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
NEW 108	$"Identified the following files on the FTP server:{string.Join(',', files)}"));	×
109
110	var forLoadingContainedCachedFiles = false;	×
111
112	foreach (var file in files)	×
113	{
NEW 114	var action = GetSkipActionForFile(file, destination);	×
115
NEW 116	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
117	$"File {file} was evaluated as {action}"));	×
118
119	switch (action)
120	{
121	case SkipReason.DoNotSkip:
122	listener.OnNotify(this,	×
NEW 123	new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));	×
NEW 124	Download(file, destination);	×
UNCOV 125	break;	×
126	case SkipReason.InForLoading:
127	forLoadingContainedCachedFiles = true;	×
128	break;
129	}
130	}
131
132	// it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
133	if (forLoadingContainedCachedFiles \|\| _filesRetrieved.Count != 0 \|\| !SendLoadNotRequiredIfFileNotFound)	×
134	return ExitCodeType.Success;	×
135
136	// if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
137	listener.OnNotify(this,	×
138	new NotifyEventArgs(ProgressEventType.Information,	×
139	"Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));	×
140	return ExitCodeType.OperationNotRequired;	×
141	}
142
143	protected enum SkipReason
144	{
145	DoNotSkip,
146	InForLoading,
147	DidNotMatchPattern,
148	IsImaginaryFile
149	}
150
151	protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
152	{
NEW 153	if (file.StartsWith(".", StringComparison.Ordinal))	×
154	return SkipReason.IsImaginaryFile;	×
155
156	//if there is a regex pattern
157	if (FilePattern?.IsMatch(file) == false) //and it does not match	×
158	return SkipReason.DidNotMatchPattern; //skip because it did not match pattern	×
159
160	//if the file on the FTP already exists in the forLoading directory, skip it
161	return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;	×
162	}
163
164
165	private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
166	SslPolicyErrors _4) => true; //any cert will do! yay	×
167
168
169	protected virtual IEnumerable<string> GetFileList()
170	{
171	return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);	×
172	}
173
174	protected virtual void Download(string file, ILoadDirectory destination)
175	{
176	var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)	×
177	? $"{RemoteDirectory}/{file}"	×
178	: file;	×
179
NEW 180	var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);	×
NEW 181	_connection.Value.DownloadFile(destinationFileName, remotePath);	×
182	_filesRetrieved.Add(remotePath);	×
183	}	×
184
185	public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
186	{
187	if (exitCode != ExitCodeType.Success \|\| !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;	×
188
189	// Force a reconnection attempt if we got cut off
190	if (!_connection.Value.IsStillConnected())	×
191	_connection.Value.Connect(true);	×
192	foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);	×
193	}	×
194
195
196	public virtual void Check(ICheckNotifier notifier)
197	{
198	try
199	{
200	SetupFtp();	×
201	}	×
202	catch (Exception e)	×
203	{
NEW 204	notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));	×
205	}	×
206	}	×
207	}

HicServices / RDMP / 10246247756

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous