10246789906

Committed 05 Aug 2024 10:05AM UTC coverage: 57.283% (-0.01%) from 57.295%

Build # 10246789906

Build Type

push

github

Committed by

web-flow

Commit Message

Fix SFTP Timeout Issue (#1913)

* add timeout setting

* update deps

* force timeout

* dont auto disconnect on timeout

* override ftp check

* use timespan correctly

* bump version

* tidy up

Run Details

11090 of 20824 branches covered (53.26%)

Branch coverage included in aggregate %.

0 of 28 new or added lines in 2 files covered. (0.0%)

2 existing lines in 1 file now uncovered.

31368 of 53296 relevant lines covered (58.86%)

7941.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.IO;
using System.Linq;
using System.Threading;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;
using Renci.SshNet;

namespace Rdmp.Core.DataLoad.Modules.FTP;

/// <summary>
/// load component which downloads files from a remote SFTP (Secure File Transfer Protocol) server to the ForLoading directory
/// 
/// <para>Operates in the same way as <see cref="FTPDownloader"/> except that it uses SSH.  In addition, this
/// class will not bother downloading any files that already exist in the forLoading directory (have the same name - file size is NOT checked)</para>
/// </summary>
public class SFTPDownloader : FTPDownloader
{
    [DemandsInitialization("The keep-alive interval.  In milliseconds.  Requires KeepAlive to be set to take effect.")]
    public int KeepAliveIntervalMilliseconds { get; set; }

    private readonly Lazy<SftpClient> _connection;

    public SFTPDownloader(Lazy<SftpClient> connection)
    {
        _connection = new Lazy<SftpClient>(SetupSftp, LazyThreadSafetyMode.ExecutionAndPublication);
    }

    public SFTPDownloader()
    {
        _connection = new Lazy<SftpClient>(SetupSftp, LazyThreadSafetyMode.ExecutionAndPublication);
    }

    private SftpClient SetupSftp()
    {
        var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");
        var username = FTPServer.Username ?? "anonymous";
        var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
        var c = new SftpClient(host, username, password);
        if (TimeoutInSeconds > 0)
        {
            c.OperationTimeout = TimeSpan.FromSeconds(TimeoutInSeconds);
            c.ConnectionInfo.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);
        }
        c.Connect();
        if (KeepAlive)
            c.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds);
        return c;
    }

    public override void Check(ICheckNotifier notifier)
    {
        try
        {
            SetupSftp();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupSFTP", CheckResult.Fail, e));
        }
    }


    protected override void Download(string file, ILoadDirectory destination)
    {
        if (file.Contains('/') || file.Contains('\\'))
            throw new Exception("Was not expecting a relative path here");

        //if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList())
        var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file;

        var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file);

        using (var dest = File.Create(destinationFilePath))
            _connection.Value.DownloadFile(fullFilePath, dest);
        _filesRetrieved.Add(fullFilePath);
    }


    public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        if (exitCode != ExitCodeType.Success) return;

        // Reconnect if we got cut off, for example due to idle timers
        if (!_connection.Value.IsConnected)
            _connection.Value.Connect();

        foreach (var retrievedFiles in _filesRetrieved)
            try
            {
                _connection.Value.DeleteFile(retrievedFiles);
                postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                    $"Deleted SFTP file {retrievedFiles} from SFTP server"));
            }
            catch (Exception e)
            {
                postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,
                    $"Could not delete SFTP file {retrievedFiles} from SFTP server", e));
            }
    }


    protected override string[] GetFileList()
    {
        var directory = string.IsNullOrWhiteSpace(RemoteDirectory) ? "." : RemoteDirectory;

        return _connection.Value.ListDirectory(directory).Select(static d => d.Name).ToArray();
    }
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.IO;
9	using System.Linq;
10	using System.Threading;
11	using Rdmp.Core.Curation;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.ReusableLibraryCode.Checks;
14	using Rdmp.Core.ReusableLibraryCode.Progress;
15	using Renci.SshNet;
16
17	namespace Rdmp.Core.DataLoad.Modules.FTP;
18
19	/// <summary>
20	/// load component which downloads files from a remote SFTP (Secure File Transfer Protocol) server to the ForLoading directory
21	///
22	/// <para>Operates in the same way as <see cref="FTPDownloader"/> except that it uses SSH. In addition, this
23	/// class will not bother downloading any files that already exist in the forLoading directory (have the same name - file size is NOT checked)</para>
24	/// </summary>
25	public class SFTPDownloader : FTPDownloader
26	{
27	[DemandsInitialization("The keep-alive interval. In milliseconds. Requires KeepAlive to be set to take effect.")]
28	public int KeepAliveIntervalMilliseconds { get; set; }	×
29
30	private readonly Lazy<SftpClient> _connection;
31
32	public SFTPDownloader(Lazy<SftpClient> connection)	×
33	{
NEW 34	_connection = new Lazy<SftpClient>(SetupSftp, LazyThreadSafetyMode.ExecutionAndPublication);	×
35	}	×
36
37	public SFTPDownloader()	×
38	{
39	_connection = new Lazy<SftpClient>(SetupSftp, LazyThreadSafetyMode.ExecutionAndPublication);	×
40	}	×
41
42	private SftpClient SetupSftp()
43	{
44	var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set");	×
45	var username = FTPServer.Username ?? "anonymous";	×
46	var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();	×
47	var c = new SftpClient(host, username, password);	×
NEW 48	if (TimeoutInSeconds > 0)	×
49	{
NEW 50	c.OperationTimeout = TimeSpan.FromSeconds(TimeoutInSeconds);	×
NEW 51	c.ConnectionInfo.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);	×
52	}
53	c.Connect();	×
54	if (KeepAlive)	×
55	c.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds);	×
56	return c;	×
57	}
58
59	public override void Check(ICheckNotifier notifier)
60	{
61	try
62	{
NEW 63	SetupSftp();	×
NEW 64	}	×
NEW 65	catch (Exception e)	×
66	{
NEW 67	notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupSFTP", CheckResult.Fail, e));	×
NEW 68	}	×
NEW 69	}	×
70
71
72	protected override void Download(string file, ILoadDirectory destination)
73	{
74	if (file.Contains('/') \|\| file.Contains('\\'))	×
75	throw new Exception("Was not expecting a relative path here");	×
76
77	//if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList())
78	var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file;	×
79
80	var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file);	×
81
NEW 82	using (var dest = File.Create(destinationFilePath))	×
NEW 83	_connection.Value.DownloadFile(fullFilePath, dest);	×
84	_filesRetrieved.Add(fullFilePath);	×
85	}	×
86
87
88	public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
89	{
90	if (exitCode != ExitCodeType.Success) return;	×
91
92	// Reconnect if we got cut off, for example due to idle timers
93	if (!_connection.Value.IsConnected)	×
94	_connection.Value.Connect();	×
95
96	foreach (var retrievedFiles in _filesRetrieved)	×
97	try
98	{
99	_connection.Value.DeleteFile(retrievedFiles);	×
100	postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
101	$"Deleted SFTP file {retrievedFiles} from SFTP server"));	×
102	}	×
103	catch (Exception e)	×
104	{
105	postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,	×
106	$"Could not delete SFTP file {retrievedFiles} from SFTP server", e));	×
107	}	×
108	}	×
109
110
111	protected override string[] GetFileList()
112	{
113	var directory = string.IsNullOrWhiteSpace(RemoteDirectory) ? "." : RemoteDirectory;	×
114
115	return _connection.Value.ListDirectory(directory).Select(static d => d.Name).ToArray();	×
116	}
117	}

HicServices / RDMP / 10246789906

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous