6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.78

/Rdmp.Core/DataExport/DataExtraction/ExtractionDirectory.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataExport.DataExtraction;

/// <summary>
/// The target directory for a given ExtractionConfiguration on a given day.  This is where linked anonymised project extracts will appear when
/// an ExtractionConfiguration is executed.  It is also the location where the Release Engine will pick them up from when it bundles together a
/// release package.
/// </summary>
public class ExtractionDirectory : IExtractionDirectory
{
    public const string EXTRACTION_SUB_FOLDER_NAME = "Extractions";
    public const string STANDARD_EXTRACTION_PREFIX = "Extr_";
    public const string GLOBALS_DATA_NAME = "Globals";
    public const string CUSTOM_COHORT_DATA_FOLDER_NAME = "CohortCustomData";
    public const string MASTER_DATA_FOLDER_NAME = "MasterData";
    public const string METADATA_FOLDER_NAME = "MetadataShareDefs";

    public DirectoryInfo ExtractionDirectoryInfo { get; private set; }

    public ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration)
        : this(rootExtractionDirectory, configuration, DateTime.Now)
    {
    }

    private ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration,
        DateTime extractionDate)
    {
        if (string.IsNullOrWhiteSpace(rootExtractionDirectory))
            throw new NullReferenceException("Extraction Directory not set");

        if (!rootExtractionDirectory.StartsWith("\\"))
            if (!Directory.Exists(rootExtractionDirectory))
                throw new DirectoryNotFoundException($"Root directory \"{rootExtractionDirectory}\" does not exist");

        var root = new DirectoryInfo(Path.Combine(rootExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME));
        if (!root.Exists)
            root.Create();

        var subdirectoryName = GetExtractionDirectoryPrefix(configuration);

        ExtractionDirectoryInfo = Directory.Exists(Path.Combine(root.FullName, subdirectoryName))
            ? new DirectoryInfo(Path.Combine(root.FullName, subdirectoryName))
            : root.CreateSubdirectory(subdirectoryName);
    }

    public static string GetExtractionDirectoryPrefix(IExtractionConfiguration configuration) =>
        STANDARD_EXTRACTION_PREFIX + configuration.ID;

    public DirectoryInfo GetDirectoryForDataset(IExtractableDataSet dataset)
    {
        if (dataset.ToString().Equals(CUSTOM_COHORT_DATA_FOLDER_NAME))
            throw new Exception(
                $"You cannot call a dataset '{CUSTOM_COHORT_DATA_FOLDER_NAME}' because this string is reserved for cohort custom data the system spits out itself");

        if (!Catalogue.IsAcceptableName(dataset.Catalogue.Name, out var reason))
            throw new NotSupportedException(
                $"Cannot extract dataset {dataset} because it points at Catalogue with an invalid name, name is invalid because:{reason}");

        var datasetDirectory = dataset.ToString();
        try
        {
            return ExtractionDirectoryInfo.CreateSubdirectory(datasetDirectory);
        }
        catch (Exception e)
        {
            throw new Exception(
                $"Could not create a directory called '{datasetDirectory}' as a subfolder of Project extraction directory {ExtractionDirectoryInfo.Root}",
                e);
        }
    }

    public DirectoryInfo GetGlobalsDirectory() => ExtractionDirectoryInfo.CreateSubdirectory(GLOBALS_DATA_NAME);

    public static bool IsOwnerOf(IExtractionConfiguration configuration, DirectoryInfo directory)
    {
        //they passed a root directory like c:\bob?
        if (directory.Parent == null)
            return false;

        //The configuration number matches but directory isn't the currently configured Project extraction directory
        var p = configuration.Project;

        return directory.Parent.FullName == Path.Combine(p.ExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME) &&
               directory.Name.StartsWith(STANDARD_EXTRACTION_PREFIX + configuration.ID);
    }

    public DirectoryInfo GetDirectoryForCohortCustomData() =>
        ExtractionDirectoryInfo.CreateSubdirectory(CUSTOM_COHORT_DATA_FOLDER_NAME);

    public DirectoryInfo GetDirectoryForMasterData() =>
        ExtractionDirectoryInfo.CreateSubdirectory(MASTER_DATA_FOLDER_NAME);

    public static void CleanupExtractionDirectory(object sender, string extractionDirectory,
        IEnumerable<IExtractionConfiguration> configurations, IDataLoadEventListener listener)
    {
        var projectExtractionDirectory =
            new DirectoryInfo(Path.Combine(extractionDirectory, EXTRACTION_SUB_FOLDER_NAME));
        var directoriesToDelete = new List<DirectoryInfo>();
        var filesToDelete = new List<FileInfo>();

        foreach (var extractionConfiguration in configurations)
        {
            var config = extractionConfiguration;
            var directoryInfos = projectExtractionDirectory.GetDirectories().Where(d => IsOwnerOf(config, d));

            foreach (var toCleanup in directoryInfos)
                AddDirectoryToCleanupList(toCleanup, true, directoriesToDelete, filesToDelete);
        }

        foreach (var fileInfo in filesToDelete)
        {
            listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,
                $"Deleting: {fileInfo.FullName}"));
            try
            {
                fileInfo.Delete();
            }
            catch (Exception e)
            {
                listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,
                    $"Error deleting: {fileInfo.FullName}", e));
            }
        }

        foreach (var directoryInfo in directoriesToDelete)
        {
            listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,
                $"Recursively deleting folder: {directoryInfo.FullName}"));
            try
            {
                directoryInfo.Delete(true);
            }
            catch (Exception e)
            {
                listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,
                    $"Error deleting: {directoryInfo.FullName}", e));
            }
        }
    }

    private static void AddDirectoryToCleanupList(DirectoryInfo toCleanup, bool isRoot,
        List<DirectoryInfo> directoriesToDelete, List<FileInfo> filesToDelete)
    {
        //only add root folders to the delete queue
        if (isRoot)
            if (!directoriesToDelete.Any(dir =>
                    dir.FullName.Equals(toCleanup.FullName))) //don't add the same folder twice
                directoriesToDelete.Add(toCleanup);

        filesToDelete.AddRange(toCleanup.EnumerateFiles());

        foreach (var dir in toCleanup.EnumerateDirectories())
            AddDirectoryToCleanupList(dir, false, directoriesToDelete, filesToDelete);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.DataExport.Data;
13	using Rdmp.Core.ReusableLibraryCode.Progress;
14
15	namespace Rdmp.Core.DataExport.DataExtraction;
16
17	/// <summary>
18	/// The target directory for a given ExtractionConfiguration on a given day. This is where linked anonymised project extracts will appear when
19	/// an ExtractionConfiguration is executed. It is also the location where the Release Engine will pick them up from when it bundles together a
20	/// release package.
21	/// </summary>
22	public class ExtractionDirectory : IExtractionDirectory
23	{
24	public const string EXTRACTION_SUB_FOLDER_NAME = "Extractions";
25	public const string STANDARD_EXTRACTION_PREFIX = "Extr_";
26	public const string GLOBALS_DATA_NAME = "Globals";
27	public const string CUSTOM_COHORT_DATA_FOLDER_NAME = "CohortCustomData";
28	public const string MASTER_DATA_FOLDER_NAME = "MasterData";
29	public const string METADATA_FOLDER_NAME = "MetadataShareDefs";
30
31	public DirectoryInfo ExtractionDirectoryInfo { get; private set; }	340✔
32
33	public ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration)
34	: this(rootExtractionDirectory, configuration, DateTime.Now)	186✔
35	{
36	}	186✔
37
38	private ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration,	186✔
39	DateTime extractionDate)	186✔
40	{
41	if (string.IsNullOrWhiteSpace(rootExtractionDirectory))	186!
42	throw new NullReferenceException("Extraction Directory not set");	×
43
44	if (!rootExtractionDirectory.StartsWith("\\"))	186✔
45	if (!Directory.Exists(rootExtractionDirectory))	186!
46	throw new DirectoryNotFoundException($"Root directory \"{rootExtractionDirectory}\" does not exist");	×
47
48	var root = new DirectoryInfo(Path.Combine(rootExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME));	186✔
49	if (!root.Exists)	186✔
50	root.Create();	2✔
51
52	var subdirectoryName = GetExtractionDirectoryPrefix(configuration);	186✔
53
54	ExtractionDirectoryInfo = Directory.Exists(Path.Combine(root.FullName, subdirectoryName))	186✔
55	? new DirectoryInfo(Path.Combine(root.FullName, subdirectoryName))	186✔
56	: root.CreateSubdirectory(subdirectoryName);	186✔
57	}	186✔
58
59	public static string GetExtractionDirectoryPrefix(IExtractionConfiguration configuration) =>
60	STANDARD_EXTRACTION_PREFIX + configuration.ID;	190✔
61
62	public DirectoryInfo GetDirectoryForDataset(IExtractableDataSet dataset)
63	{
64	if (dataset.ToString().Equals(CUSTOM_COHORT_DATA_FOLDER_NAME))	150!
65	throw new Exception(	×
66	$"You cannot call a dataset '{CUSTOM_COHORT_DATA_FOLDER_NAME}' because this string is reserved for cohort custom data the system spits out itself");	×
67
68	if (!Catalogue.IsAcceptableName(dataset.Catalogue.Name, out var reason))	150✔
69	throw new NotSupportedException(	2✔
70	$"Cannot extract dataset {dataset} because it points at Catalogue with an invalid name, name is invalid because:{reason}");	2✔
71
72	var datasetDirectory = dataset.ToString();	148✔
73	try
74	{
75	return ExtractionDirectoryInfo.CreateSubdirectory(datasetDirectory);	148✔
76	}
77	catch (Exception e)	×
78	{
79	throw new Exception(	×
80	$"Could not create a directory called '{datasetDirectory}' as a subfolder of Project extraction directory {ExtractionDirectoryInfo.Root}",	×
81	e);	×
82	}
83	}	148✔
84
85	public DirectoryInfo GetGlobalsDirectory() => ExtractionDirectoryInfo.CreateSubdirectory(GLOBALS_DATA_NAME);	6✔
86
87	public static bool IsOwnerOf(IExtractionConfiguration configuration, DirectoryInfo directory)
88	{
89	//they passed a root directory like c:\bob?
90	if (directory.Parent == null)	24!
91	return false;	×
92
93	//The configuration number matches but directory isn't the currently configured Project extraction directory
94	var p = configuration.Project;	24✔
95
96	return directory.Parent.FullName == Path.Combine(p.ExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME) &&	24!
97	directory.Name.StartsWith(STANDARD_EXTRACTION_PREFIX + configuration.ID);	24✔
98	}
99
100	public DirectoryInfo GetDirectoryForCohortCustomData() =>
101	ExtractionDirectoryInfo.CreateSubdirectory(CUSTOM_COHORT_DATA_FOLDER_NAME);	×
102
103	public DirectoryInfo GetDirectoryForMasterData() =>
104	ExtractionDirectoryInfo.CreateSubdirectory(MASTER_DATA_FOLDER_NAME);	×
105
106	public static void CleanupExtractionDirectory(object sender, string extractionDirectory,
107	IEnumerable<IExtractionConfiguration> configurations, IDataLoadEventListener listener)
108	{
109	var projectExtractionDirectory =	2✔
110	new DirectoryInfo(Path.Combine(extractionDirectory, EXTRACTION_SUB_FOLDER_NAME));	2✔
111	var directoriesToDelete = new List<DirectoryInfo>();	2✔
112	var filesToDelete = new List<FileInfo>();	2✔
113
114	foreach (var extractionConfiguration in configurations)	16✔
115	{
116	var config = extractionConfiguration;	6✔
117	var directoryInfos = projectExtractionDirectory.GetDirectories().Where(d => IsOwnerOf(config, d));	30✔
118
119	foreach (var toCleanup in directoryInfos)	24✔
120	AddDirectoryToCleanupList(toCleanup, true, directoriesToDelete, filesToDelete);	6✔
121	}
122
123	foreach (var fileInfo in filesToDelete)	92✔
124	{
125	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,	44✔
126	$"Deleting: {fileInfo.FullName}"));	44✔
127	try
128	{
129	fileInfo.Delete();	44✔
130	}	44✔
131	catch (Exception e)	×
132	{
133	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,	×
134	$"Error deleting: {fileInfo.FullName}", e));	×
135	}	×
136	}
137
138	foreach (var directoryInfo in directoriesToDelete)	16✔
139	{
140	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,	6✔
141	$"Recursively deleting folder: {directoryInfo.FullName}"));	6✔
142	try
143	{
144	directoryInfo.Delete(true);	6✔
145	}	6✔
146	catch (Exception e)	×
147	{
148	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,	×
149	$"Error deleting: {directoryInfo.FullName}", e));	×
150	}	×
151	}
152	}	2✔
153
154	private static void AddDirectoryToCleanupList(DirectoryInfo toCleanup, bool isRoot,
155	List<DirectoryInfo> directoriesToDelete, List<FileInfo> filesToDelete)
156	{
157	//only add root folders to the delete queue
158	if (isRoot)	50✔
159	if (!directoriesToDelete.Any(dir =>	6✔
160	dir.FullName.Equals(toCleanup.FullName))) //don't add the same folder twice	12✔
161	directoriesToDelete.Add(toCleanup);	6✔
162
163	filesToDelete.AddRange(toCleanup.EnumerateFiles());	50✔
164
165	foreach (var dir in toCleanup.EnumerateDirectories())	188✔
166	AddDirectoryToCleanupList(dir, false, directoriesToDelete, filesToDelete);	44✔
167	}	50✔
168	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous