7194961165

Committed 13 Dec 2023 12:07PM UTC coverage: 56.776% (-0.2%) from 57.013%

Build # 7194961165

Build Type

push

github

Committed by

web-flow

Commit Message

Merge Latest Release into main (#1702)

* Bump YamlDotNet from 13.3.1 to 13.4.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 13.3.1 to 13.4.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v13.3.1...v13.4.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump shogo82148/actions-setup-perl from 1.23.1 to 1.24.1

Bumps [shogo82148/actions-setup-perl](https://github.com/shogo82148/actions-setup-perl) from 1.23.1 to 1.24.1.
- [Release notes](https://github.com/shogo82148/actions-setup-perl/releases)
- [Commits](https://github.com/shogo82148/actions-setup-perl/compare/v1.23.1...v1.24.1)

---
updated-dependencies:
- dependency-name: shogo82148/actions-setup-perl
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix checkbox issue

* improve confirmation text (#1639)

* improve confirmation text
* Loop tidyup, use var where possible

---------

Co-authored-by: jas88 <j.a.sutherland@dundee.ac.uk>

* correct typo in create logging sql (#1640)

* Feature/ci codescan (#1641)

* Move SecurityCodescan.VS2019 to run on Github CI alone, integrate results with CodeQL
* Remove SecurityCodescan from Packages.md, no longer used via Nuget

---------

Co-authored-by: James A Sutherland <j@sutherland.pw>

* hide source control when not available

* Remove old Plugin object bits, tidy up (#1636)

* Remove old Plugin object bits, tidy up

* Purge remaining bits of AllExpiredPluginsNode

* Fix plugin display name in tree

* Update CreateNewDataExtractionProjectUI.cs

Casting fix

* Feature/rdmp42 delete plugins (#1642)

* add ui plugin delete functionality

* Warning and inherita... (continued)

Run Details

10722 of 20351 branches covered (0.0%)

Branch coverage included in aggregate %.

215 of 789 new or added lines in 63 files covered. (27.25%)

39 existing lines in 16 files now uncovered.

30650 of 52518 relevant lines covered (58.36%)

7294.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

#nullable enable

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
using System.Threading;
using FAnsi.Discovery;
using FluentFTP;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote FTP server to the ForLoading directory
/// 
/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
///  root folder - if not then configure redirection on the FTP, so you land in the correct directory).  Files are downloaded into the ForLoading folder</para>
/// </summary>
public class FTPDownloader : IPluginDataProvider
{
    private readonly Lazy<FtpClient> _connection;
    protected readonly List<string> _filesRetrieved = new();
    private ILoadDirectory? _directory;

    public FTPDownloader()
    {
        _connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);
    }

    [DemandsInitialization(
        "Determines the behaviour of the system when no files are found on the server.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
    public bool SendLoadNotRequiredIfFileNotFound { get; set; }

    [DemandsInitialization(
        "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
    public Regex? FilePattern { get; set; }

    [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
    public int TimeoutInSeconds { get; set; }

    [DemandsInitialization(
        "Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state).  This will only delete the files if they were actually fetched from the FTP server.  If the files were already in forLoading then the remote files are not deleted")]
    public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }

    [DemandsInitialization(
        "The FTP server to connect to.  Server should be specified with only IP:Port e.g. 127.0.0.1:20.  You do not have to specify ftp:// at the start",
        Mandatory = true)]
    public ExternalDatabaseServer? FTPServer { get; set; }

    [DemandsInitialization("The directory on the FTP server that you want to download files from")]
    public string? RemoteDirectory { get; set; }

    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
    public bool KeepAlive { get; set; }


    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
    {
        _directory = directory;
    }

    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);
    }

    private FtpClient SetupFtp()
    {
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
        var username = FTPServer.Username ?? "anonymous";
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
        var c = new FtpClient(host, username, password);
        c.AutoConnect();
        return c;
    }

    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
    {
        var files = GetFileList().ToArray();

        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"Identified the following files on the FTP server:{string.Join(',',files)}"));

        var forLoadingContainedCachedFiles = false;

        foreach (var file in files)
        {
            var action = GetSkipActionForFile(file, destination);

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"File {file} was evaluated as {action}"));

            switch (action)
            {
                case SkipReason.DoNotSkip:
                    listener.OnNotify(this,
                        new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
                    Download(file, destination);
                    break;
                case SkipReason.InForLoading:
                    forLoadingContainedCachedFiles = true;
                    break;
            }
        }

        // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
        if (forLoadingContainedCachedFiles || _filesRetrieved.Any() || !SendLoadNotRequiredIfFileNotFound)
            return ExitCodeType.Success;

        // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
        listener.OnNotify(this,
            new NotifyEventArgs(ProgressEventType.Information,
                "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
        return ExitCodeType.OperationNotRequired;
    }

    protected enum SkipReason
    {
        DoNotSkip,
        InForLoading,
        DidNotMatchPattern,
        IsImaginaryFile
    }

    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
    {
        if (file.StartsWith(".",StringComparison.Ordinal))
            return SkipReason.IsImaginaryFile;

        //if there is a regex pattern
        if (FilePattern?.IsMatch(file) == false) //and it does not match
            return SkipReason.DidNotMatchPattern; //skip because it did not match pattern

        //if the file on the FTP already exists in the forLoading directory, skip it
        return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;
    }


    private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
        SslPolicyErrors _4) => true; //any cert will do! yay


    protected virtual IEnumerable<string> GetFileList()
    {
        return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
    }

    protected virtual void Download(string file, ILoadDirectory destination)
    {
        var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
            ? $"{RemoteDirectory}/{file}"
            : file;

        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
        _connection.Value.DownloadFile(destinationFileName, remotePath);
        _filesRetrieved.Add(remotePath);
    }

    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;

        foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
    }


    public void Check(ICheckNotifier notifier)
    {
        try
        {
            SetupFtp();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	#nullable enable
8
9	using System;
10	using System.Collections.Generic;
11	using System.IO;
12	using System.Linq;
13	using System.Net.Security;
14	using System.Security.Cryptography.X509Certificates;
15	using System.Text.RegularExpressions;
16	using System.Threading;
17	using FAnsi.Discovery;
18	using FluentFTP;
19	using Rdmp.Core.Curation;
20	using Rdmp.Core.Curation.Data;
21	using Rdmp.Core.DataFlowPipeline;
22	using Rdmp.Core.DataLoad.Engine.DataProvider;
23	using Rdmp.Core.DataLoad.Engine.Job;
24	using Rdmp.Core.ReusableLibraryCode.Checks;
25	using Rdmp.Core.ReusableLibraryCode.Progress;
26
27	namespace Rdmp.Core.DataLoad.Modules.FTP;
28
29	/// <summary>
30	/// load component which downloads files from a remote FTP server to the ForLoading directory
31	///
32	/// <para>Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the
33	/// root folder - if not then configure redirection on the FTP, so you land in the correct directory). Files are downloaded into the ForLoading folder</para>
34	/// </summary>
35	public class FTPDownloader : IPluginDataProvider
36	{
37	private readonly Lazy<FtpClient> _connection;
NEW 38	protected readonly List<string> _filesRetrieved = new();	×
39	private ILoadDirectory? _directory;
40
NEW 41	public FTPDownloader()	×
42	{
NEW 43	_connection = new Lazy<FtpClient>(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication);	×
NEW 44	}	×
45
46	[DemandsInitialization(
47	"Determines the behaviour of the system when no files are found on the server. If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
48	public bool SendLoadNotRequiredIfFileNotFound { get; set; }	×
49
50	[DemandsInitialization(
51	"The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")]
NEW 52	public Regex? FilePattern { get; set; }	×
53
54	[DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")]
55	public int TimeoutInSeconds { get; set; }	×
56
57	[DemandsInitialization(
58	"Tick to delete files from the FTP server when the load is successful (ends with .Success not .OperationNotRequired - which happens when LoadNotRequired state). This will only delete the files if they were actually fetched from the FTP server. If the files were already in forLoading then the remote files are not deleted")]
59	public bool DeleteFilesOffFTPServerAfterSuccesfulDataLoad { get; set; }	×
60
61	[DemandsInitialization(
62	"The FTP server to connect to. Server should be specified with only IP:Port e.g. 127.0.0.1:20. You do not have to specify ftp:// at the start",
63	Mandatory = true)]
NEW 64	public ExternalDatabaseServer? FTPServer { get; set; }	×
65
66	[DemandsInitialization("The directory on the FTP server that you want to download files from")]
NEW 67	public string? RemoteDirectory { get; set; }	×
68
69	[DemandsInitialization("True to set keep alive", DefaultValue = true)]
70	public bool KeepAlive { get; set; }	×
71
72
73	public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
74	{
75	_directory = directory;	×
76	}	×
77
78	public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
79	{
NEW 80	return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);	×
81	}
82
83	private FtpClient SetupFtp()
84	{
NEW 85	var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");	×
NEW 86	var username = FTPServer.Username ?? "anonymous";	×
NEW 87	var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();	×
NEW 88	var c = new FtpClient(host, username, password);	×
NEW 89	c.AutoConnect();	×
NEW 90	return c;	×
91	}
92
93	private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
94	{
NEW 95	var files = GetFileList().ToArray();	×
96
NEW 97	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
NEW 98	$"Identified the following files on the FTP server:{string.Join(',',files)}"));	×
99
100	var forLoadingContainedCachedFiles = false;	×
101
102	foreach (var file in files)	×
103	{
104	var action = GetSkipActionForFile(file, destination);	×
105
106	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
107	$"File {file} was evaluated as {action}"));	×
108
109	switch (action)
110	{
111	case SkipReason.DoNotSkip:
NEW 112	listener.OnNotify(this,	×
NEW 113	new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));	×
NEW 114	Download(file, destination);	×
NEW 115	break;	×
116	case SkipReason.InForLoading:
NEW 117	forLoadingContainedCachedFiles = true;	×
118	break;
119	}
120	}
121
122	// it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
NEW 123	if (forLoadingContainedCachedFiles \|\| _filesRetrieved.Any() \|\| !SendLoadNotRequiredIfFileNotFound)	×
NEW 124	return ExitCodeType.Success;	×
125
126	// if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
NEW 127	listener.OnNotify(this,	×
NEW 128	new NotifyEventArgs(ProgressEventType.Information,	×
NEW 129	"Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));	×
NEW 130	return ExitCodeType.OperationNotRequired;	×
131	}
132
133	protected enum SkipReason
134	{
135	DoNotSkip,
136	InForLoading,
137	DidNotMatchPattern,
138	IsImaginaryFile
139	}
140
141	protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
142	{
NEW 143	if (file.StartsWith(".",StringComparison.Ordinal))	×
144	return SkipReason.IsImaginaryFile;	×
145
146	//if there is a regex pattern
NEW 147	if (FilePattern?.IsMatch(file) == false) //and it does not match	×
NEW 148	return SkipReason.DidNotMatchPattern; //skip because it did not match pattern	×
149
150	//if the file on the FTP already exists in the forLoading directory, skip it
151	return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip;	×
152	}
153
154
155	private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
NEW 156	SslPolicyErrors _4) => true; //any cert will do! yay	×
157
158
159	protected virtual IEnumerable<string> GetFileList()
160	{
NEW 161	return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);	×
162	}
163
164	protected virtual void Download(string file, ILoadDirectory destination)
165	{
NEW 166	var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)	×
NEW 167	? $"{RemoteDirectory}/{file}"	×
NEW 168	: file;	×
169
NEW 170	var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);	×
NEW 171	_connection.Value.DownloadFile(destinationFileName, remotePath);	×
NEW 172	_filesRetrieved.Add(remotePath);	×
UNCOV 173	}	×
174
175	public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
176	{
NEW 177	if (exitCode != ExitCodeType.Success \|\| !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;	×
178
NEW 179	foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);	×
UNCOV 180	}	×
181
182
183	public void Check(ICheckNotifier notifier)
184	{
185	try
186	{
NEW 187	SetupFtp();	×
188	}	×
189	catch (Exception e)	×
190	{
191	notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));	×
192	}	×
193	}	×
194	}

HicServices / RDMP / 7194961165

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous