• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 6245535001

20 Sep 2023 07:44AM UTC coverage: 57.013%. First build
6245535001

push

github

web-flow
8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.12
/Rdmp.Core/DataExport/DataExtraction/Pipeline/SimpleFileExtractor.cs
1
// Copyright (c) The University of Dundee 2018-2019
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
using System;
8
using System.Collections.Generic;
9
using System.Data;
10
using System.IO;
11
using System.Linq;
12
using Rdmp.Core.Curation.Data;
13
using Rdmp.Core.DataExport.DataExtraction.Commands;
14
using Rdmp.Core.DataFlowPipeline;
15
using Rdmp.Core.ReusableLibraryCode;
16
using Rdmp.Core.ReusableLibraryCode.Checks;
17
using Rdmp.Core.ReusableLibraryCode.Progress;
18

19
namespace Rdmp.Core.DataExport.DataExtraction.Pipeline;
20

21
/// <summary>
22
/// <para>
23
/// Component for copying directory trees or top level files from a location on disk to the output directory
24
/// of a project extraction.  Supports substituting private identifiers for release identifiers in top level
25
/// file/directory names.
26
/// </para>
27
/// <para>IMPORTANT: File extractor operates as part of the 'Extract Globals' section of the extraction pipeline.
28
/// This means that you must enable globals in the extraction for the component to operate.</para>
29
/// </summary>
30
public class SimpleFileExtractor : FileExtractor
31
{
32
    [DemandsInitialization("Location of files on disk that should be copied to the output directory", Mandatory = true)]
33
    public DirectoryInfo LocationOfFiles { get; set; }
46✔
34

35
    [DemandsInitialization(
36
        "True if the LocationOfFiles contains a number of directories to be copied.  False if it contains files only (no subdirectories)",
37
        Mandatory = true, DefaultValue = true)]
38
    public bool Directories { get; set; }
90✔
39

40
    [DemandsInitialization(
41
        "True if there is 1 or more files/folders per patient (if so Pattern must contain $p).  False if there is one arbitrary file/folder that needs copied once only",
42
        Mandatory = true, DefaultValue = true)]
43
    public bool PerPatient { get; set; }
40✔
44

45
    [DemandsInitialization(
46
        "Expected naming pattern of files to be moved.  If PerPatient is true then this should include the symbol $p to indicate the private identifier value of each patient to be moved e.g. $p.txt.  This symbol will be replaced in the file/path names (but not file body)",
47
        Mandatory = true)]
48
    public string Pattern { get; set; } = "$p";
80✔
49

50
    [DemandsInitialization(@"Directory where files should be put 
51
$p - Project Extraction Directory (e.g. c:\MyProject\)
52
$n - Project Number (e.g. 234)
53
$c - Configuration Extraction Directory  (e.g. c:\MyProject\Extractions\Extr_16)
54
", Mandatory = true, DefaultValue = "$c\\Files\\")]
55
    public string OutputDirectoryName { get; set; } = "$c\\Files\\";
48✔
56

57
    [DemandsInitialization(
58
        "Determines behaviour when the destination file already exists either due to an old run or cohort private identifier aliases.  Set to true to overwrite or false to crash.",
59
        DefaultValue = true)]
60
    public bool Overwrite { get; set; } = true;
40✔
61

62
    public override void Check(ICheckNotifier notifier)
63
    {
64
        base.Check(notifier);
16✔
65

66
        if (PerPatient && !Pattern.Contains("$p"))
16!
67
            notifier.OnCheckPerformed(
×
68
                new CheckEventArgs($"PerPatient is true but Pattern {Pattern} did not contain token $p",
×
69
                    CheckResult.Fail));
×
70

71
        if (!PerPatient && Pattern.IndexOf("$p", StringComparison.Ordinal) != -1)
16!
72
            notifier.OnCheckPerformed(new CheckEventArgs(
×
73
                $"PerPatient is false but Pattern {Pattern} contains token $p.  This token will never be matched in MoveAll mode",
×
74
                CheckResult.Fail));
×
75

76
        try
77
        {
78
            notifier.OnCheckPerformed(new CheckEventArgs($"Output path is:{GetDestinationDirectory()}",
16✔
79
                CheckResult.Success));
16✔
80
        }
16✔
81
        catch (Exception ex)
×
82
        {
83
            throw new Exception(
×
84
                "Unable to to determine output directory from 'OutputDirectoryName'.  Perhaps pattern is bad", ex);
×
85
        }
86
    }
16✔
87

88
    protected override void MoveFiles(ExtractGlobalsCommand command, IDataLoadEventListener listener,
89
        GracefulCancellationToken cancellationToken)
90
    {
91
        if (!LocationOfFiles.Exists)
×
92
            throw new Exception($"LocationOfFiles {LocationOfFiles} did not exist");
×
93

94
        var destinationDirectory = GetDestinationDirectory();
×
95

96
        if (!destinationDirectory.Exists)
×
97
            destinationDirectory.Create();
×
98

99
        if (PerPatient)
×
100
        {
101
            var cohort = command.Configuration.Cohort;
×
102
            var cohortData = cohort.FetchEntireCohort();
×
103

104
            var priv = cohort.GetPrivateIdentifier(true);
×
105
            var rel = cohort.GetReleaseIdentifier(true);
×
106

107
            foreach (DataRow r in cohortData.Rows)
×
108
                MovePatient(r[priv], r[rel], destinationDirectory, listener, cancellationToken);
×
109
        }
110
        else
111
        {
112
            MoveAll(destinationDirectory, listener, cancellationToken);
×
113
        }
114
    }
×
115

116
    /// <summary>
117
    /// Resolves tokens (if any) in OutputDirectoryName into a single path
118
    /// </summary>
119
    /// <returns></returns>
120
    public DirectoryInfo GetDestinationDirectory()
121
    {
122
        var path = OutputDirectoryName;
16✔
123

124
        if (path.Contains("$p")) path = path.Replace("$p", _command.Project.ExtractionDirectory);
16!
125
        if (path.Contains("$n")) path = path.Replace("$n", _command.Project.ProjectNumber.ToString());
16!
126

127
        if (path.Contains("$c"))
16!
128
            path = path.Replace("$c",
×
129
                new ExtractionDirectory(_command.Project.ExtractionDirectory, _command.Configuration)
×
130
                    .ExtractionDirectoryInfo.FullName);
×
131

132
        return new DirectoryInfo(path);
16✔
133
    }
134

135
    /// <summary>
136
    /// Called when <see cref="PerPatient"/> is false.  Called once per extraction
137
    /// </summary>
138
    public virtual void MoveAll(DirectoryInfo destinationDirectory, IDataLoadEventListener listener,
139
        GracefulCancellationToken cancellationToken)
140
    {
141
        var atLeastOne = false;
8✔
142

143
        var infos = new List<FileSystemInfo>();
8✔
144

145
        if (Pattern.Contains('*'))
8!
146
        {
147
            infos.AddRange(LocationOfFiles.EnumerateFileSystemInfos(Pattern));
8✔
148
        }
149
        else
150
        {
151
            var f = LocationOfFiles.GetFiles()
×
152
                .FirstOrDefault(f => f.Name.Equals(Pattern, StringComparison.OrdinalIgnoreCase));
×
153

154
            if (f != null)
×
155
                infos.Add(f);
×
156

157
            var d = LocationOfFiles.GetDirectories()
×
158
                .FirstOrDefault(d => d.Name.Equals(Pattern, StringComparison.OrdinalIgnoreCase));
×
159

160
            if (d != null)
×
161
                infos.Add(d);
×
162
        }
163

164
        foreach (var e in infos)
64✔
165
        {
166
            if (Directories && e is DirectoryInfo dir)
24✔
167
            {
168
                var dest = Path.Combine(destinationDirectory.FullName, dir.Name);
6✔
169

170
                // Recursively copy all files from input path to destination path
171
                listener.OnNotify(this,
6✔
172
                    new NotifyEventArgs(ProgressEventType.Information,
6✔
173
                        $"Copying directory '{e.FullName}' to '{dest}'"));
6✔
174
                CopyFolder(e.FullName, dest);
6✔
175
                atLeastOne = true;
6✔
176
            }
177

178
            if (!Directories && e is FileInfo f)
24✔
179
            {
180
                var dest = Path.Combine(destinationDirectory.FullName, f.Name);
8✔
181
                listener.OnNotify(this,
8✔
182
                    new NotifyEventArgs(ProgressEventType.Information, $"Copying file '{f.FullName}' to '{dest}'"));
8✔
183
                File.Copy(f.FullName, dest, Overwrite);
8✔
184
                atLeastOne = true;
8✔
185
            }
186
        }
187

188
        if (!atLeastOne)
8!
189
            listener.OnNotify(this,
×
190
                new NotifyEventArgs(ProgressEventType.Warning,
×
191
                    $"No {(Directories ? "Directories" : "Files")} were found matching Pattern {Pattern} in {LocationOfFiles.FullName}"));
×
192
    }
8✔
193

194
    /// <summary>
195
    /// Called when <see cref="PerPatient"/> is true.  Called once per private identifier.  Note that it is possible for 2 private identifiers to map to the same release identifier - be careful
196
    /// </summary>
197
    public virtual void MovePatient(object privateIdentifier, object releaseIdentifier,
198
        DirectoryInfo destinationDirectory, IDataLoadEventListener listener,
199
        GracefulCancellationToken cancellationToken)
200
    {
201
        var atLeastOne = false;
16✔
202

203
        if (privateIdentifier == DBNull.Value || string.IsNullOrWhiteSpace(privateIdentifier?.ToString()))
16!
204
        {
205
            listener.OnNotify(this,
×
206
                new NotifyEventArgs(ProgressEventType.Warning,
×
207
                    "Skipped NULL private identifier found in cohort when trying to copy files"));
×
208
            return;
×
209
        }
210

211
        if (releaseIdentifier == DBNull.Value || string.IsNullOrWhiteSpace(releaseIdentifier?.ToString()))
16!
212
        {
213
            listener.OnNotify(this,
×
214
                new NotifyEventArgs(ProgressEventType.Error,
×
215
                    $"Found NULL release identifier in cohort when trying to copy files.  This is not allowed as it breaks file name substitutions.  Private identifier was {privateIdentifier}"));
×
216
            return;
×
217
        }
218

219
        // What we will be writing into the file/path names in place of the private identifier
220
        var releaseSub = UsefulStuff.RemoveIllegalFilenameCharacters(releaseIdentifier.ToString());
16✔
221

222
        var patternAfterTokenInsertion = Pattern.Replace("$p", privateIdentifier.ToString());
16✔
223

224
        foreach (var e in LocationOfFiles.EnumerateFileSystemInfos(patternAfterTokenInsertion))
52✔
225
        {
226
            if (Directories && e is DirectoryInfo dir)
10✔
227
            {
228
                var dest = Path.Combine(
6✔
229
                    destinationDirectory.FullName,
6✔
230
                    dir.Name.Replace(privateIdentifier.ToString(), releaseSub));
6✔
231

232
                // Recursively copy all files from input path to destination path
233
                listener.OnNotify(this,
6✔
234
                    new NotifyEventArgs(ProgressEventType.Information,
6✔
235
                        $"Copying directory '{e.FullName}' to '{dest}'"));
6✔
236
                CopyFolder(e.FullName, dest);
6✔
237
                atLeastOne = true;
6✔
238
            }
239

240
            if (!Directories && e is FileInfo f)
10✔
241
            {
242
                var dest = Path.Combine(
4✔
243
                    destinationDirectory.FullName,
4✔
244
                    f.Name.Replace(privateIdentifier.ToString(), releaseSub));
4✔
245

246
                listener.OnNotify(this,
4✔
247
                    new NotifyEventArgs(ProgressEventType.Information, $"Copying file '{f.FullName}' to '{dest}'"));
4✔
248
                File.Copy(f.FullName, dest, Overwrite);
4✔
249
                atLeastOne = true;
4✔
250
            }
251
        }
252

253
        if (!atLeastOne)
16✔
254
            listener.OnNotify(this,
6✔
255
                new NotifyEventArgs(ProgressEventType.Warning,
6✔
256
                    $"No {(Directories ? "Directories" : "Files")} were found matching Pattern {patternAfterTokenInsertion} in {LocationOfFiles.FullName}.  For private identifier '{privateIdentifier}'"));
6✔
257
    }
14✔
258

259
    protected void CopyFolder(string sourceFolder, string destFolder)
260
    {
261
        if (!Directory.Exists(destFolder))
12✔
262
            Directory.CreateDirectory(destFolder);
12✔
263
        var files = Directory.GetFiles(sourceFolder);
12✔
264
        foreach (var file in files)
48✔
265
        {
266
            var name = Path.GetFileName(file);
12✔
267
            var dest = Path.Combine(destFolder, name);
12✔
268
            File.Copy(file, dest, Overwrite);
12✔
269
        }
270

271
        var folders = Directory.GetDirectories(sourceFolder);
12✔
272
        foreach (var folder in folders)
24!
273
        {
274
            var name = Path.GetFileName(folder);
×
275
            var dest = Path.Combine(destFolder, name);
×
276
            CopyFolder(folder, dest);
×
277
        }
278
    }
12✔
279
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc