6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.45

/Rdmp.Core/DataLoad/Modules/DataProvider/ImportFilesDataProvider.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System.IO;
using FAnsi.Discovery;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.DataProvider;

/// <summary>
/// Data load component that copies files into the ForLoading directory from the remote directory (that match the file pattern e.g. *.csv).  A good use case
/// for this is if you want to expose a network location as a share for data provders to send you files to but want the DLE to take a copy of the files at
/// runtime for the purposes of loading.
///
/// <para>Optionally deletes files from the fetch location if the data load is successful</para>
/// </summary>
public class ImportFilesDataProvider : IPluginDataProvider
{
    private FileInfo[] _files;

    [DemandsInitialization("The path you want to copy files from", Mandatory = true)]
    public string DirectoryPath { get; set; }

    [DemandsInitialization("The file pattern to match on the DirectoryPath", Mandatory = true)]
    public string FilePattern { get; set; }

    [DemandsInitialization(
        "If true then at the end of a successful data load the files that were originally matched and copied to forLoading will be deleted from the remote DirectoryPath.  Note that only the files copied will be deleted, any new files that appear during the load will not be deleted")]
    public bool DeleteFilesOnsuccessfulLoad { get; set; }

    public void Check(ICheckNotifier notifier)
    {
        if (string.IsNullOrWhiteSpace(DirectoryPath))
            notifier.OnCheckPerformed(new CheckEventArgs(
                "No DirectoryPath has been specified, this should be set to the remote folder you want to copy files out of",
                CheckResult.Fail));

        if (string.IsNullOrWhiteSpace(FilePattern))
            notifier.OnCheckPerformed(new CheckEventArgs(
                "No FilePattern has been specified, this should be a pattern that matches files in the remote folder you want to copy files out of e.g. *.*",
                CheckResult.Fail));

        notifier.OnCheckPerformed(new DirectoryInfo(DirectoryPath).Exists
            ? new CheckEventArgs($"Path {DirectoryPath} was found", CheckResult.Success)
            : new CheckEventArgs($"Path {DirectoryPath} was not found", CheckResult.Fail));
    }

    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
    {
    }

    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        _files = new DirectoryInfo(DirectoryPath).GetFiles(FilePattern);

        foreach (var f in _files)
        {
            var to = Path.Combine(job.LoadDirectory.ForLoading.FullName, f.Name);
            job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"Copying file {f.FullName} to directory {to}"));
            f.CopyTo(to, true);
        }

        return ExitCodeType.Success;
    }

    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
    {
        if (exitCode == ExitCodeType.Success)
            if (DeleteFilesOnsuccessfulLoad)
                foreach (var f in _files)
                {
                    postLoadEventsListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                        $"About to delete {f.FullName}"));
                    f.Delete();
                }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System.IO;
8	using FAnsi.Discovery;
9	using Rdmp.Core.Curation;
10	using Rdmp.Core.Curation.Data;
11	using Rdmp.Core.DataFlowPipeline;
12	using Rdmp.Core.DataLoad.Engine.DataProvider;
13	using Rdmp.Core.DataLoad.Engine.Job;
14	using Rdmp.Core.ReusableLibraryCode.Checks;
15	using Rdmp.Core.ReusableLibraryCode.Progress;
16
17	namespace Rdmp.Core.DataLoad.Modules.DataProvider;
18
19	/// <summary>
20	/// Data load component that copies files into the ForLoading directory from the remote directory (that match the file pattern e.g. *.csv). A good use case
21	/// for this is if you want to expose a network location as a share for data provders to send you files to but want the DLE to take a copy of the files at
22	/// runtime for the purposes of loading.
23	///
24	/// <para>Optionally deletes files from the fetch location if the data load is successful</para>
25	/// </summary>
26	public class ImportFilesDataProvider : IPluginDataProvider
27	{
28	private FileInfo[] _files;
29
30	[DemandsInitialization("The path you want to copy files from", Mandatory = true)]
31	public string DirectoryPath { get; set; }	16✔
32
33	[DemandsInitialization("The file pattern to match on the DirectoryPath", Mandatory = true)]
34	public string FilePattern { get; set; }	12✔
35
36	[DemandsInitialization(
37	"If true then at the end of a successful data load the files that were originally matched and copied to forLoading will be deleted from the remote DirectoryPath. Note that only the files copied will be deleted, any new files that appear during the load will not be deleted")]
38	public bool DeleteFilesOnsuccessfulLoad { get; set; }	6✔
39
40	public void Check(ICheckNotifier notifier)
41	{
42	if (string.IsNullOrWhiteSpace(DirectoryPath))	6✔
43	notifier.OnCheckPerformed(new CheckEventArgs(	2✔
44	"No DirectoryPath has been specified, this should be set to the remote folder you want to copy files out of",	2✔
45	CheckResult.Fail));	2✔
46
47	if (string.IsNullOrWhiteSpace(FilePattern))	4✔
48	notifier.OnCheckPerformed(new CheckEventArgs(	2✔
49	"No FilePattern has been specified, this should be a pattern that matches files in the remote folder you want to copy files out of e.g. .",	2✔
50	CheckResult.Fail));	2✔
51
52	notifier.OnCheckPerformed(new DirectoryInfo(DirectoryPath).Exists	2!
53	? new CheckEventArgs($"Path {DirectoryPath} was found", CheckResult.Success)	2✔
54	: new CheckEventArgs($"Path {DirectoryPath} was not found", CheckResult.Fail));	2✔
55	}	2✔
56
57	public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
58	{
59	}	×
60
61	public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
62	{
63	_files = new DirectoryInfo(DirectoryPath).GetFiles(FilePattern);	4✔
64
65	foreach (var f in _files)	12✔
66	{
67	var to = Path.Combine(job.LoadDirectory.ForLoading.FullName, f.Name);	2✔
68	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	2✔
69	$"Copying file {f.FullName} to directory {to}"));	2✔
70	f.CopyTo(to, true);	2✔
71	}
72
73	return ExitCodeType.Success;	4✔
74	}
75
76	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
77	{
78	if (exitCode == ExitCodeType.Success)	8✔
79	if (DeleteFilesOnsuccessfulLoad)	4✔
80	foreach (var f in _files)	8✔
81	{
82	postLoadEventsListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	2✔
83	$"About to delete {f.FullName}"));	2✔
84	f.Delete();	2✔
85	}
86	}	8✔
87	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous