6237307473

Committed 19 Sep 2023 04:02PM UTC coverage: 57.015% (-0.4%) from 57.44%

Build # 6237307473

Build Type

push

github

Committed by

web-flow

Commit Message

Feature/rc4 (#1570)

* Syntax tidying
* Dependency updates
* Event handling singletons (ThrowImmediately and co)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: James A Sutherland <>
Co-authored-by: James Friel <jfriel001@dundee.ac.uk>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Run Details

10734 of 20259 branches covered (0.0%)

Branch coverage included in aggregate %.

5922 of 5922 new or added lines in 565 files covered. (100.0%)

30687 of 52390 relevant lines covered (58.57%)

7361.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.ReusableLibraryCode.Progress;
using Renci.SshNet;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote SFTP (Secure File Transfer Protocol) server to the ForLoading directory
/// 
/// <para>Operates in the same way as <see cref="FTPDownloader"/> except that it uses SSH.  In addition this
/// class will not bother downloading any files that already exist in the forLoading directory (have the same name - file size is NOT checked)</para>
/// </summary>
public class SFTPDownloader : FTPDownloader
{
    [DemandsInitialization("The keep-alive interval.  In milliseconds.  Requires KeepAlive to be set to take effect.")]
    public int KeepAliveIntervalMilliseconds { get; set; }

    protected override void Download(string file, ILoadDirectory destination, IDataLoadEventListener job)
    {
        if (file.Contains('/') || file.Contains('\\'))
            throw new Exception("Was not expecting a relative path here");

        var s = new Stopwatch();
        s.Start();

        using (var sftp = new SftpClient(_host, _username, _password))
        {
            if (KeepAlive && KeepAliveIntervalMilliseconds > 0)
                sftp.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds);

            sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);
            sftp.Connect();

            //if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList())
            var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file;

            var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file);

            //register for events
            void Callback(ulong totalBytes) => job.OnProgress(this,
                new ProgressEventArgs(destinationFilePath,
                    new ProgressMeasurement((int)(totalBytes * 0.001), ProgressType.Kilobytes), s.Elapsed));

            using (var fs = new FileStream(destinationFilePath, FileMode.CreateNew))
            {
                //download
                sftp.DownloadFile(fullFilePath, fs, Callback);
                fs.Close();
            }

            _filesRetrieved.Add(fullFilePath);
        }

        s.Stop();
    }


    public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success) return;
        using var sftp = new SftpClient(_host, _username, _password);
        sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);
        sftp.Connect();

        foreach (var retrievedFiles in _filesRetrieved)
            try
            {
                sftp.DeleteFile(retrievedFiles);
                postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                    $"Deleted SFTP file {retrievedFiles} from SFTP server"));
            }
            catch (Exception e)
            {
                postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,
                    $"Could not delete SFTP file {retrievedFiles} from SFTP server", e));
            }
    }


    protected override string[] GetFileList()
    {
        using var sftp = new SftpClient(_host, _username, _password);
        sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);
        sftp.Connect();

        var directory = RemoteDirectory;

        if (string.IsNullOrWhiteSpace(directory))
            directory = ".";

        return sftp.ListDirectory(directory).Select(d => d.Name).ToArray();
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Diagnostics;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.ReusableLibraryCode.Progress;
14	using Renci.SshNet;
15
16	namespace Rdmp.Core.DataLoad.Modules.FTP;
17
18	/// <summary>
19	/// load component which downloads files from a remote SFTP (Secure File Transfer Protocol) server to the ForLoading directory
20	///
21	/// <para>Operates in the same way as <see cref="FTPDownloader"/> except that it uses SSH. In addition this
22	/// class will not bother downloading any files that already exist in the forLoading directory (have the same name - file size is NOT checked)</para>
23	/// </summary>
24	public class SFTPDownloader : FTPDownloader
25	{
26	[DemandsInitialization("The keep-alive interval. In milliseconds. Requires KeepAlive to be set to take effect.")]
27	public int KeepAliveIntervalMilliseconds { get; set; }	×
28
29	protected override void Download(string file, ILoadDirectory destination, IDataLoadEventListener job)
30	{
31	if (file.Contains('/') \|\| file.Contains('\\'))	×
32	throw new Exception("Was not expecting a relative path here");	×
33
34	var s = new Stopwatch();	×
35	s.Start();	×
36
37	using (var sftp = new SftpClient(_host, _username, _password))	×
38	{
39	if (KeepAlive && KeepAliveIntervalMilliseconds > 0)	×
40	sftp.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds);	×
41
42	sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);	×
43	sftp.Connect();	×
44
45	//if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList())
46	var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file;	×
47
48	var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file);	×
49
50	//register for events
51	void Callback(ulong totalBytes) => job.OnProgress(this,	×
52	new ProgressEventArgs(destinationFilePath,	×
53	new ProgressMeasurement((int)(totalBytes * 0.001), ProgressType.Kilobytes), s.Elapsed));	×
54
55	using (var fs = new FileStream(destinationFilePath, FileMode.CreateNew))	×
56	{
57	//download
58	sftp.DownloadFile(fullFilePath, fs, Callback);	×
59	fs.Close();	×
60	}	×
61
62	_filesRetrieved.Add(fullFilePath);	×
63	}	×
64
65	s.Stop();	×
66	}	×
67
68
69	public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
70	{
71	if (exitCode != ExitCodeType.Success) return;	×
72	using var sftp = new SftpClient(_host, _username, _password);	×
73	sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);	×
74	sftp.Connect();	×
75
76	foreach (var retrievedFiles in _filesRetrieved)	×
77	try
78	{
79	sftp.DeleteFile(retrievedFiles);	×
80	postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
81	$"Deleted SFTP file {retrievedFiles} from SFTP server"));	×
82	}	×
83	catch (Exception e)	×
84	{
85	postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,	×
86	$"Could not delete SFTP file {retrievedFiles} from SFTP server", e));	×
87	}	×
88	}	×
89
90
91	protected override string[] GetFileList()
92	{
93	using var sftp = new SftpClient(_host, _username, _password);	×
94	sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);	×
95	sftp.Connect();	×
96
97	var directory = RemoteDirectory;	×
98
99	if (string.IsNullOrWhiteSpace(directory))	×
100	directory = ".";	×
101
102	return sftp.ListDirectory(directory).Select(d => d.Name).ToArray();	×
103	}	×
104	}

HicServices / RDMP / 6237307473

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous