9413524711

Committed 07 Jun 2024 07:46AM UTC coverage: 56.924% (+0.01%) from 56.914%

Build # 9413524711

Build Type

Pull #1844

github

Committed by

JFriel

Commit Message

herge branch 'develop' of https://github.com/HicServices/RDMP into release/8.1.7

Pull Request Pull Request #1844: Release/8.1.7

Run Details

10819 of 20482 branches covered (52.82%)

Branch coverage included in aggregate %.

11 of 12 new or added lines in 2 files covered. (91.67%)

41 existing lines in 2 files now uncovered.

30838 of 52698 relevant lines covered (58.52%)

7416.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

#nullable enable

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
using System.Threading;
using FAnsi.Discovery;
using FluentFTP;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote FTP server to the ForLoading directory
/// 
/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
///  root folder - if not then configure redirection on the FTP, so you land in the correct directory).  Files are downloaded into the ForLoading folder</para>
/// </summary>
public class FTPDownloader : IPluginDataProvider
{
    private readonly Lazy<FtpClient> _connection;
    protected readonly List<string> _filesRetrieved = new();
    private ILoadDirectory? _directory;

    public FTPDownloader()
    {
        _connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);
    }

    [DemandsInitialization(
        "Determines the behaviour of the system when no files are found on the server.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
    public bool SendLoadNotRequiredIfFileNotFound { get; set; }

    [DemandsInitialization(
        "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
    public Regex? FilePattern { get; set; }

    [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
    public int TimeoutInSeconds { get; set; }

    [DemandsInitialization(
        "Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state).  This will only delete the files if they were actually fetched from the FTP server.  If the files were already in forLoading then the remote files are not deleted")]
    public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }

    [DemandsInitialization(
        "The FTP server to connect to.  Server should be specified with only IP:Port e.g. 127.0.0.1:20.  You do not have to specify ftp:// at the start",
        Mandatory = true)]
    public ExternalDatabaseServer? FTPServer { get; set; }

    [DemandsInitialization("The directory on the FTP server that you want to download files from")]
    public string? RemoteDirectory { get; set; }

    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
    public bool KeepAlive { get; set; }


    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
    {
        _directory = directory;
    }

    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);
    }

    private FtpClient SetupFtp()
    {
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
        var username = FTPServer.Username ?? "anonymous";
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
        var c = new FtpClient(host, username, password);
        c.AutoConnect();
        return c;
    }

    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
    {
        var files = GetFileList().ToArray();

        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"Identified the following files on the FTP server:{string.Join(',',files)}"));

        var forLoadingContainedCachedFiles = false;

        foreach (var file in files)
        {
            var action = GetSkipActionForFile(file, destination);

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"File {file} was evaluated as {action}"));

            switch (action)
            {
                case SkipReason.DoNotSkip:
                    listener.OnNotify(this,
                        new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
                    Download(file, destination);
                    break;
                case SkipReason.InForLoading:
                    forLoadingContainedCachedFiles = true;
                    break;
            }
        }

        // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
        if (forLoadingContainedCachedFiles || _filesRetrieved.Any() || !SendLoadNotRequiredIfFileNotFound)
            return ExitCodeType.Success;

        // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
        listener.OnNotify(this,
            new NotifyEventArgs(ProgressEventType.Information,
                "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
        return ExitCodeType.OperationNotRequired;
    }

    protected enum SkipReason
    {
        DoNotSkip,
        InForLoading,
        DidNotMatchPattern,
        IsImaginaryFile
    }

    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
    {
        if (file.StartsWith(".",StringComparison.Ordinal))
            return SkipReason.IsImaginaryFile;

        //if there is a regex pattern
        if (FilePattern?.IsMatch(file) == false) //and it does not match
            return SkipReason.DidNotMatchPattern; //skip because it did not match pattern

        //if the file on the FTP already exists in the forLoading directory, skip it
        return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;
    }


    private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
        SslPolicyErrors _4) => true; //any cert will do! yay


    protected virtual IEnumerable<string> GetFileList()
    {
        return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
    }

    protected virtual void Download(string file, ILoadDirectory destination)
    {
        var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
            ? $"{RemoteDirectory}/{file}"
            : file;

        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
        _connection.Value.DownloadFile(destinationFileName, remotePath);
        _filesRetrieved.Add(remotePath);
    }

    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;

        foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
    }


    public void Check(ICheckNotifier notifier)
    {
        try
        {
            SetupFtp();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	#nullable enable
8
9	using System;
10	using System.Collections.Generic;
11	using System.IO;
12	using System.Linq;
13	using System.Net.Security;
14	using System.Security.Cryptography.X509Certificates;
15	using System.Text.RegularExpressions;
16	using System.Threading;
17	using FAnsi.Discovery;
18	using FluentFTP;
19	using Rdmp.Core.Curation;
20	using Rdmp.Core.Curation.Data;
21	using Rdmp.Core.DataFlowPipeline;
22	using Rdmp.Core.DataLoad.Engine.DataProvider;
23	using Rdmp.Core.DataLoad.Engine.Job;
24	using Rdmp.Core.ReusableLibraryCode.Checks;
25	using Rdmp.Core.ReusableLibraryCode.Progress;
26
27	namespace Rdmp.Core.DataLoad.Modules.FTP;
28
29	/// <summary>
30	/// load component which downloads files from a remote FTP server to the ForLoading directory
31	///
32	/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
33	/// root folder - if not then configure redirection on the FTP, so you land in the correct directory). Files are downloaded into the ForLoading folder</para>
34	/// </summary>
35	public class FTPDownloader : IPluginDataProvider
36	{
37	private readonly Lazy<FtpClient> _connection;
38	protected readonly List<string> _filesRetrieved = new();	×
39	private ILoadDirectory? _directory;
40
41	public FTPDownloader()	×
42	{
43	_connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);	×
44	}	×
45
46	[DemandsInitialization(
47	"Determines the behaviour of the system when no files are found on the server. If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
48	public bool SendLoadNotRequiredIfFileNotFound { get; set; }	×
49
50	[DemandsInitialization(
51	"The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
52	public Regex? FilePattern { get; set; }	×
53
54	[DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
55	public int TimeoutInSeconds { get; set; }	×
56
57	[DemandsInitialization(
58	"Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state). This will only delete the files if they were actually fetched from the FTP server. If the files were already in forLoading then the remote files are not deleted")]
59	public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }	×
60
61	[DemandsInitialization(
62	"The FTP server to connect to. Server should be specified with only IP:Port e.g. 127.0.0.1:20. You do not have to specify ftp:// at the start",
63	Mandatory = true)]
64	public ExternalDatabaseServer? FTPServer { get; set; }	×
65
66	[DemandsInitialization("The directory on the FTP server that you want to download files from")]
67	public string? RemoteDirectory { get; set; }	×
68
69	[DemandsInitialization("True to set keep alive", DefaultValue = true)]
70	public bool KeepAlive { get; set; }	×
71
72
73	public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
74	{
75	_directory = directory;	×
76	}	×
77
78	public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
79	{
80	return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);	×
81	}
82
83	private FtpClient SetupFtp()
84	{
85	var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");	×
86	var username = FTPServer.Username ?? "anonymous";	×
87	var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();	×
88	var c = new FtpClient(host, username, password);	×
89	c.AutoConnect();	×
90	return c;	×
91	}
92
93	private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
94	{
UNCOV 95	var files = GetFileList().ToArray();	×
96
UNCOV 97	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
98	$"Identified the following files on the FTP server:{string.Join(',',files)}"));	×
99
100	var forLoadingContainedCachedFiles = false;	×
101
UNCOV 102	foreach (var file in files)	×
103	{
UNCOV 104	var action = GetSkipActionForFile(file, destination);	×
105
UNCOV 106	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
107	$"File {file} was evaluated as {action}"));	×
108
109	switch (action)
110	{
111	case SkipReason.DoNotSkip:
UNCOV 112	listener.OnNotify(this,	×
UNCOV 113	new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));	×
UNCOV 114	Download(file, destination);	×
115	break;	×
116	case SkipReason.InForLoading:
117	forLoadingContainedCachedFiles = true;	×
118	break;
119	}
120	}
121
122	// it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
UNCOV 123	if (forLoadingContainedCachedFiles \|\| _filesRetrieved.Any() \|\| !SendLoadNotRequiredIfFileNotFound)	×
UNCOV 124	return ExitCodeType.Success;	×
125
126	// if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
127	listener.OnNotify(this,	×
UNCOV 128	new NotifyEventArgs(ProgressEventType.Information,	×
UNCOV 129	"Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));	×
130	return ExitCodeType.OperationNotRequired;	×
131	}
132
133	protected enum SkipReason
134	{
135	DoNotSkip,
136	InForLoading,
137	DidNotMatchPattern,
138	IsImaginaryFile
139	}
140
141	protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
142	{
UNCOV 143	if (file.StartsWith(".",StringComparison.Ordinal))	×
144	return SkipReason.IsImaginaryFile;	×
145
146	//if there is a regex pattern
147	if (FilePattern?.IsMatch(file) == false) //and it does not match	×
UNCOV 148	return SkipReason.DidNotMatchPattern; //skip because it did not match pattern	×
149
150	//if the file on the FTP already exists in the forLoading directory, skip it
151	return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;	×
152	}
153
154
155	private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
UNCOV 156	SslPolicyErrors _4) => true; //any cert will do! yay	×
157
158
159	protected virtual IEnumerable<string> GetFileList()
160	{
UNCOV 161	return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);	×
162	}
163
164	protected virtual void Download(string file, ILoadDirectory destination)
165	{
UNCOV 166	var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)	×
167	? $"{RemoteDirectory}/{file}"	×
UNCOV 168	: file;	×
169
170	var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);	×
171	_connection.Value.DownloadFile(destinationFileName, remotePath);	×
UNCOV 172	_filesRetrieved.Add(remotePath);	×
173	}	×
174
175	public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
176	{
UNCOV 177	if (exitCode != ExitCodeType.Success \|\| !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;	×
178
UNCOV 179	foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);	×
180	}	×
181
182
183	public void Check(ICheckNotifier notifier)
184	{
185	try
186	{
UNCOV 187	SetupFtp();	×
UNCOV 188	}	×
UNCOV 189	catch (Exception e)	×
190	{
UNCOV 191	notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));	×
UNCOV 192	}	×
193	}	×
194	}

HicServices / RDMP / 9413524711

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous