7194961165

Committed 13 Dec 2023 12:07PM UTC coverage: 56.776% (-0.2%) from 57.013%

Build # 7194961165

Build Type

push

github

Committed by

web-flow

Commit Message

Merge Latest Release into main (#1702)

* Bump YamlDotNet from 13.3.1 to 13.4.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 13.3.1 to 13.4.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v13.3.1...v13.4.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump shogo82148/actions-setup-perl from 1.23.1 to 1.24.1

Bumps [shogo82148/actions-setup-perl](https://github.com/shogo82148/actions-setup-perl) from 1.23.1 to 1.24.1.
- [Release notes](https://github.com/shogo82148/actions-setup-perl/releases)
- [Commits](https://github.com/shogo82148/actions-setup-perl/compare/v1.23.1...v1.24.1)

---
updated-dependencies:
- dependency-name: shogo82148/actions-setup-perl
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix checkbox issue

* improve confirmation text (#1639)

* improve confirmation text
* Loop tidyup, use var where possible

---------

Co-authored-by: jas88 <j.a.sutherland@dundee.ac.uk>

* correct typo in create logging sql (#1640)

* Feature/ci codescan (#1641)

* Move SecurityCodescan.VS2019 to run on Github CI alone, integrate results with CodeQL
* Remove SecurityCodescan from Packages.md, no longer used via Nuget

---------

Co-authored-by: James A Sutherland <j@sutherland.pw>

* hide source control when not available

* Remove old Plugin object bits, tidy up (#1636)

* Remove old Plugin object bits, tidy up

* Purge remaining bits of AllExpiredPluginsNode

* Fix plugin display name in tree

* Update CreateNewDataExtractionProjectUI.cs

Casting fix

* Feature/rdmp42 delete plugins (#1642)

* add ui plugin delete functionality

* Warning and inherita... (continued)

Run Details

10722 of 20351 branches covered (0.0%)

Branch coverage included in aggregate %.

215 of 789 new or added lines in 63 files covered. (27.25%)

39 existing lines in 16 files now uncovered.

30650 of 52518 relevant lines covered (58.36%)

7294.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.IO;
using System.Linq;
using System.Threading;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.ReusableLibraryCode.Progress;
using Renci.SshNet;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote SFTP (Secure File Transfer Protocol) server to the ForLoading directory
/// 
/// <para>Operates in the same way as <see cref="FTPDownloader"/> except that it uses SSH.  In addition, this
/// class will not bother downloading any files that already exist in the forLoading directory (have the same name - file size is NOT checked)</para>
/// </summary>
public class SFTPDownloader : FTPDownloader
{
    [DemandsInitialization("The keep-alive interval.  In milliseconds.  Requires KeepAlive to be set to take effect.")]
    public int KeepAliveIntervalMilliseconds { get; set; }

    private readonly Lazy<SftpClient> _connection;

    public SFTPDownloader(Lazy<SftpClient> connection)
    {
        _connection = new Lazy<SftpClient>(SetupSftp,LazyThreadSafetyMode.ExecutionAndPublication);
    }

    private SftpClient SetupSftp()
    {
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
        var username = FTPServer.Username ?? "anonymous";
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
        var c = new SftpClient(host, username, password);
        c.Connect();
        return c;
    }


    protected override void Download(string file, ILoadDirectory destination)
    {
        if (file.Contains('/') || file.Contains('\\'))
            throw new Exception("Was not expecting a relative path here");

        //if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList())
        var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file;

        var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file);

        using (var dest=File.Create(destinationFilePath))
            _connection.Value.DownloadFile(fullFilePath,dest);
        _filesRetrieved.Add(fullFilePath);
    }


    public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success) return;

        foreach (var retrievedFiles in _filesRetrieved)
            try
            {
                _connection.Value.DeleteFile(retrievedFiles);
                postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                    $"Deleted SFTP file {retrievedFiles} from SFTP server"));
            }
            catch (Exception e)
            {
                postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,
                    $"Could not delete SFTP file {retrievedFiles} from SFTP server", e));
            }
    }


    protected override string[] GetFileList()
    {
        var directory = string.IsNullOrWhiteSpace(RemoteDirectory) ? "." : RemoteDirectory;

        return _connection.Value.ListDirectory(directory).Select(static d => d.Name).ToArray();
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.IO;
9	using System.Linq;
10	using System.Threading;
11	using Rdmp.Core.Curation;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.ReusableLibraryCode.Progress;
14	using Renci.SshNet;
15
16	namespace Rdmp.Core.DataLoad.Modules.FTP;
17
18	/// <summary>
19	/// load component which downloads files from a remote SFTP (Secure File Transfer Protocol) server to the ForLoading directory
20	///
21	/// <para>Operates in the same way as <see cref="FTPDownloader"/> except that it uses SSH. In addition, this
22	/// class will not bother downloading any files that already exist in the forLoading directory (have the same name - file size is NOT checked)</para>
23	/// </summary>
24	public class SFTPDownloader : FTPDownloader
25	{
26	[DemandsInitialization("The keep-alive interval. In milliseconds. Requires KeepAlive to be set to take effect.")]
27	public int KeepAliveIntervalMilliseconds { get; set; }	×
28
29	private readonly Lazy<SftpClient> _connection;
30
NEW 31	public SFTPDownloader(Lazy<SftpClient> connection)	×
32	{
NEW 33	_connection = new Lazy<SftpClient>(SetupSftp,LazyThreadSafetyMode.ExecutionAndPublication);	×
NEW 34	}	×
35
36	private SftpClient SetupSftp()
37	{
NEW 38	var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");	×
NEW 39	var username = FTPServer.Username ?? "anonymous";	×
NEW 40	var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();	×
NEW 41	var c = new SftpClient(host, username, password);	×
NEW 42	c.Connect();	×
NEW 43	return c;	×
44	}
45
46
47	protected override void Download(string file, ILoadDirectory destination)
48	{
NEW 49	if (file.Contains('/') \|\| file.Contains('\\'))	×
NEW 50	throw new Exception("Was not expecting a relative path here");	×
51
52	//if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList())
NEW 53	var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file;	×
54
NEW 55	var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file);	×
56
NEW 57	using (var dest=File.Create(destinationFilePath))	×
NEW 58	_connection.Value.DownloadFile(fullFilePath,dest);	×
NEW 59	_filesRetrieved.Add(fullFilePath);	×
UNCOV 60	}	×
61
62
63	public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
64	{
65	if (exitCode != ExitCodeType.Success) return;	×
66
67	foreach (var retrievedFiles in _filesRetrieved)	×
68	try
69	{
NEW 70	_connection.Value.DeleteFile(retrievedFiles);	×
71	postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
72	$"Deleted SFTP file {retrievedFiles} from SFTP server"));	×
73	}	×
74	catch (Exception e)	×
75	{
76	postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,	×
77	$"Could not delete SFTP file {retrievedFiles} from SFTP server", e));	×
78	}	×
79	}	×
80
81
82	protected override string[] GetFileList()
83	{
NEW 84	var directory = string.IsNullOrWhiteSpace(RemoteDirectory) ? "." : RemoteDirectory;	×
85
NEW 86	return _connection.Value.ListDirectory(directory).Select(static d => d.Name).ToArray();	×
87	}
88	}

HicServices / RDMP / 7194961165

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous