9758065908

Committed 02 Jul 2024 09:03AM UTC coverage: 56.679% (-0.2%) from 56.914%

Build # 9758065908

Build Type

push

github

Committed by

web-flow

Commit Message

Release/8.2.0 (#1867)

* add extraction additions

* interim

* add test

* interim

* working dedupe

* improved checking

* add timestamp option

* fix extra looping

* add check

* start on tests

* tidy up code

* update link

* tidy up

* Rename executeFullExtractionToDatabaseMSSql.md to ExecuteFullExtractionToDatabaseMSSql.md

* fix typo

* add docs

* update

* update documentation

* attempt fix docs

* update docs

* tidy up code

* better tests

* add real test

* tidy up

* interim

* grab existiing entity

* no new data

* add basic tests

* attempt to fix test

* interim

* interim commit

* working clash

* add test

* fix test

* improved clash checker

* tidy up

* update test

* fix up test

* update from codeql

* tidy up code

* fix bad merge

* fix typo

* skip over for now

* revert change

* Task/RDMP-180 Add instance settings table (#1820)

* working settings interface

* add documentation

* add missing files

* update namespace

* add icon

* update from run

* make key unique

* add tests

* update tests

* update for tests

* fix unique name issue

* tidy up

* tidy up from review

* works

* nested deprications

* recursive deprication

* tidy up

* add newline

* Task/rdmp 174 dqe improvements (#1849)

* working scallable graph

* add changelog

* add axis override

* interim

* working increments

* working ui refresh

* update changelog

* tidy up code

* add missing file

* tidy up

* Task/rdmp 155 migrate catalogue tables (#1805)

* start of UI

* interim

* working switch

* improved ui

* fix build

* rename duped file

* imterim

* add checks

* start of tests

* local tests  working

* add tests

* improved ui

* tidy up

* add single item use

* broken test

* updated tests

* tidy up imports

* add some documentation

* fix docume... (continued)

Run Details

10912 of 20750 branches covered (52.59%)

Branch coverage included in aggregate %.

369 of 831 new or added lines in 38 files covered. (44.4%)

375 existing lines in 25 files now uncovered.

30965 of 53135 relevant lines covered (58.28%)

7845.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.33

/Rdmp.Core/DataExport/DataExtraction/ExtractionDirectory.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataExport.DataExtraction;

/// <summary>
/// The target directory for a given ExtractionConfiguration on a given day.  This is where linked anonymised project extracts will appear when
/// an ExtractionConfiguration is executed.  It is also the location where the Release Engine will pick them up from when it bundles together a
/// release package.
/// </summary>
public class ExtractionDirectory : IExtractionDirectory
{
    public const string EXTRACTION_SUB_FOLDER_NAME = "Extractions";
    public const string STANDARD_EXTRACTION_PREFIX = "Extr_";
    public const string GLOBALS_DATA_NAME = "Globals";
    public const string CUSTOM_COHORT_DATA_FOLDER_NAME = "CohortCustomData";
    public const string MASTER_DATA_FOLDER_NAME = "MasterData";
    public const string METADATA_FOLDER_NAME = "MetadataShareDefs";

    public DirectoryInfo ExtractionDirectoryInfo { get; private set; }

    public ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration)
        : this(rootExtractionDirectory, configuration, DateTime.Now)
    {
    }

    private ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration,
        DateTime extractionDate)
    {
        if (string.IsNullOrWhiteSpace(rootExtractionDirectory))
            throw new NullReferenceException("Extraction Directory not set");

        if (!rootExtractionDirectory.StartsWith("\\"))
            if (!Directory.Exists(rootExtractionDirectory))
                throw new DirectoryNotFoundException($"Root directory \"{rootExtractionDirectory}\" does not exist");

        var root = new DirectoryInfo(Path.Combine(rootExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME));
        if (!root.Exists)
            root.Create();

        var subdirectoryName = GetExtractionDirectoryPrefix(configuration);

        ExtractionDirectoryInfo = Directory.Exists(Path.Combine(root.FullName, subdirectoryName))
            ? new DirectoryInfo(Path.Combine(root.FullName, subdirectoryName))
            : root.CreateSubdirectory(subdirectoryName);
    }

    public static string GetExtractionDirectoryPrefix(IExtractionConfiguration configuration) =>
        STANDARD_EXTRACTION_PREFIX + configuration.ID;

    public DirectoryInfo GetDirectoryForDataset(IExtractableDataSet dataset)
    {
        if (dataset.ToString().Equals(CUSTOM_COHORT_DATA_FOLDER_NAME))
            throw new Exception(
                $"You cannot call a dataset '{CUSTOM_COHORT_DATA_FOLDER_NAME}' because this string is reserved for cohort custom data the system spits out itself");

        if (!Catalogue.IsAcceptableName(dataset.Catalogue.Name, out var reason))
            throw new NotSupportedException(
                $"Cannot extract dataset {dataset} because it points at Catalogue with an invalid name, name is invalid because:{reason}");

        var datasetDirectory = dataset.ToString();
        try
        {
            return ExtractionDirectoryInfo.CreateSubdirectory(datasetDirectory);
        }
        catch (Exception e)
        {
            throw new Exception(
                $"Could not create a directory called '{datasetDirectory}' as a subfolder of Project extraction directory {ExtractionDirectoryInfo.Root}",
                e);
        }
    }

    public DirectoryInfo GetGlobalsDirectory() => ExtractionDirectoryInfo.CreateSubdirectory(GLOBALS_DATA_NAME);

    public static bool IsOwnerOf(IExtractionConfiguration configuration, DirectoryInfo directory)
    {
        //they passed a root directory like c:\bob?
        if (directory.Parent == null)
            return false;

        //The configuration number matches but directory isn't the currently configured Project extraction directory
        var p = configuration.Project;

        return directory.Parent.FullName == Path.Combine(p.ExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME) &&
               directory.Name.StartsWith(STANDARD_EXTRACTION_PREFIX + configuration.ID);
    }

    public DirectoryInfo GetDirectoryForCohortCustomData() =>
        ExtractionDirectoryInfo.CreateSubdirectory(CUSTOM_COHORT_DATA_FOLDER_NAME);

    public DirectoryInfo GetDirectoryForMasterData() =>
        ExtractionDirectoryInfo.CreateSubdirectory(MASTER_DATA_FOLDER_NAME);

    public static void CleanupExtractionDirectory(object sender, string extractionDirectory,
        IEnumerable<IExtractionConfiguration> configurations, IDataLoadEventListener listener)
    {
        var projectExtractionDirectory =
            new DirectoryInfo(Path.Combine(extractionDirectory, EXTRACTION_SUB_FOLDER_NAME));
        var directoriesToDelete = new List<DirectoryInfo>();
        var filesToDelete = new List<FileInfo>();

        foreach (var extractionConfiguration in configurations)
        {
            var config = extractionConfiguration;
            var directoryInfos = projectExtractionDirectory.GetDirectories().Where(d => IsOwnerOf(config, d));

            foreach (var toCleanup in directoryInfos)
                AddDirectoryToCleanupList(toCleanup, true, directoriesToDelete, filesToDelete);
        }

        foreach (var fileInfo in filesToDelete)
        {
            listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,
                $"Deleting: {fileInfo.FullName}"));
            try
            {
                fileInfo.Delete();
            }
            catch (Exception e)
            {
                listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,
                    $"Error deleting: {fileInfo.FullName}", e));
            }
        }

        foreach (var directoryInfo in directoriesToDelete)
        {
            listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,
                $"Recursively deleting folder: {directoryInfo.FullName}"));
            try
            {
                directoryInfo.Delete(true);
            }
            catch (Exception e)
            {
                listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,
                    $"Error deleting: {directoryInfo.FullName}", e));
            }
        }
    }

    private static void AddDirectoryToCleanupList(DirectoryInfo toCleanup, bool isRoot,
        List<DirectoryInfo> directoriesToDelete, List<FileInfo> filesToDelete)
    {
        //only add root folders to the delete queue
        if (isRoot)
            if (!directoriesToDelete.Any(dir =>
                    dir.FullName.Equals(toCleanup.FullName))) //don't add the same folder twice
                directoriesToDelete.Add(toCleanup);

        filesToDelete.AddRange(toCleanup.EnumerateFiles());

        foreach (var dir in toCleanup.EnumerateDirectories())
            AddDirectoryToCleanupList(dir, false, directoriesToDelete, filesToDelete);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.DataExport.Data;
13	using Rdmp.Core.ReusableLibraryCode.Progress;
14
15	namespace Rdmp.Core.DataExport.DataExtraction;
16
17	/// <summary>
18	/// The target directory for a given ExtractionConfiguration on a given day. This is where linked anonymised project extracts will appear when
19	/// an ExtractionConfiguration is executed. It is also the location where the Release Engine will pick them up from when it bundles together a
20	/// release package.
21	/// </summary>
22	public class ExtractionDirectory : IExtractionDirectory
23	{
24	public const string EXTRACTION_SUB_FOLDER_NAME = "Extractions";
25	public const string STANDARD_EXTRACTION_PREFIX = "Extr_";
26	public const string GLOBALS_DATA_NAME = "Globals";
27	public const string CUSTOM_COHORT_DATA_FOLDER_NAME = "CohortCustomData";
28	public const string MASTER_DATA_FOLDER_NAME = "MasterData";
29	public const string METADATA_FOLDER_NAME = "MetadataShareDefs";
30
31	public DirectoryInfo ExtractionDirectoryInfo { get; private set; }	518✔
32
33	public ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration)
34	: this(rootExtractionDirectory, configuration, DateTime.Now)	266✔
35	{
36	}	266✔
37
38	private ExtractionDirectory(string rootExtractionDirectory, IExtractionConfiguration configuration,	266✔
39	DateTime extractionDate)	266✔
40	{
41	if (string.IsNullOrWhiteSpace(rootExtractionDirectory))	266!
42	throw new NullReferenceException("Extraction Directory not set");	×
43
44	if (!rootExtractionDirectory.StartsWith("\\"))	266✔
45	if (!Directory.Exists(rootExtractionDirectory))	266!
46	throw new DirectoryNotFoundException($"Root directory \"{rootExtractionDirectory}\" does not exist");	×
47
48	var root = new DirectoryInfo(Path.Combine(rootExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME));	266✔
49	if (!root.Exists)	266✔
50	root.Create();	4✔
51
52	var subdirectoryName = GetExtractionDirectoryPrefix(configuration);	266✔
53
54	ExtractionDirectoryInfo = Directory.Exists(Path.Combine(root.FullName, subdirectoryName))	266✔
55	? new DirectoryInfo(Path.Combine(root.FullName, subdirectoryName))	266✔
56	: root.CreateSubdirectory(subdirectoryName);	266✔
57	}	266✔
58
59	public static string GetExtractionDirectoryPrefix(IExtractionConfiguration configuration) =>
60	STANDARD_EXTRACTION_PREFIX + configuration.ID;	270✔
61
62	public DirectoryInfo GetDirectoryForDataset(IExtractableDataSet dataset)
63	{
64	if (dataset.ToString().Equals(CUSTOM_COHORT_DATA_FOLDER_NAME))	188!
65	throw new Exception(	×
66	$"You cannot call a dataset '{CUSTOM_COHORT_DATA_FOLDER_NAME}' because this string is reserved for cohort custom data the system spits out itself");	×
67
68	if (!Catalogue.IsAcceptableName(dataset.Catalogue.Name, out var reason))	188✔
69	throw new NotSupportedException(	2✔
70	$"Cannot extract dataset {dataset} because it points at Catalogue with an invalid name, name is invalid because:{reason}");	2✔
71
72	var datasetDirectory = dataset.ToString();	186✔
73	try
74	{
75	return ExtractionDirectoryInfo.CreateSubdirectory(datasetDirectory);	186✔
76	}
77	catch (Exception e)	×
78	{
79	throw new Exception(	×
80	$"Could not create a directory called '{datasetDirectory}' as a subfolder of Project extraction directory {ExtractionDirectoryInfo.Root}",	×
81	e);	×
82	}
83	}	186✔
84
85	public DirectoryInfo GetGlobalsDirectory() => ExtractionDirectoryInfo.CreateSubdirectory(GLOBALS_DATA_NAME);	66✔
86
87	public static bool IsOwnerOf(IExtractionConfiguration configuration, DirectoryInfo directory)
88	{
89	//they passed a root directory like c:\bob?
UNCOV 90	if (directory.Parent == null)	×
91	return false;	×
92
93	//The configuration number matches but directory isn't the currently configured Project extraction directory
UNCOV 94	var p = configuration.Project;	×
95
UNCOV 96	return directory.Parent.FullName == Path.Combine(p.ExtractionDirectory, EXTRACTION_SUB_FOLDER_NAME) &&	×
UNCOV 97	directory.Name.StartsWith(STANDARD_EXTRACTION_PREFIX + configuration.ID);	×
98	}
99
100	public DirectoryInfo GetDirectoryForCohortCustomData() =>
101	ExtractionDirectoryInfo.CreateSubdirectory(CUSTOM_COHORT_DATA_FOLDER_NAME);	×
102
103	public DirectoryInfo GetDirectoryForMasterData() =>
104	ExtractionDirectoryInfo.CreateSubdirectory(MASTER_DATA_FOLDER_NAME);	×
105
106	public static void CleanupExtractionDirectory(object sender, string extractionDirectory,
107	IEnumerable<IExtractionConfiguration> configurations, IDataLoadEventListener listener)
108	{
UNCOV 109	var projectExtractionDirectory =	×
UNCOV 110	new DirectoryInfo(Path.Combine(extractionDirectory, EXTRACTION_SUB_FOLDER_NAME));	×
UNCOV 111	var directoriesToDelete = new List<DirectoryInfo>();	×
UNCOV 112	var filesToDelete = new List<FileInfo>();	×
113
UNCOV 114	foreach (var extractionConfiguration in configurations)	×
115	{
UNCOV 116	var config = extractionConfiguration;	×
UNCOV 117	var directoryInfos = projectExtractionDirectory.GetDirectories().Where(d => IsOwnerOf(config, d));	×
118
UNCOV 119	foreach (var toCleanup in directoryInfos)	×
UNCOV 120	AddDirectoryToCleanupList(toCleanup, true, directoriesToDelete, filesToDelete);	×
121	}
122
UNCOV 123	foreach (var fileInfo in filesToDelete)	×
124	{
UNCOV 125	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,	×
UNCOV 126	$"Deleting: {fileInfo.FullName}"));	×
127	try
128	{
UNCOV 129	fileInfo.Delete();	×
UNCOV 130	}	×
131	catch (Exception e)	×
132	{
133	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,	×
134	$"Error deleting: {fileInfo.FullName}", e));	×
135	}	×
136	}
137
UNCOV 138	foreach (var directoryInfo in directoriesToDelete)	×
139	{
UNCOV 140	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Information,	×
UNCOV 141	$"Recursively deleting folder: {directoryInfo.FullName}"));	×
142	try
143	{
UNCOV 144	directoryInfo.Delete(true);	×
UNCOV 145	}	×
146	catch (Exception e)	×
147	{
148	listener.OnNotify(sender, new NotifyEventArgs(ProgressEventType.Error,	×
149	$"Error deleting: {directoryInfo.FullName}", e));	×
150	}	×
151	}
UNCOV 152	}	×
153
154	private static void AddDirectoryToCleanupList(DirectoryInfo toCleanup, bool isRoot,
155	List<DirectoryInfo> directoriesToDelete, List<FileInfo> filesToDelete)
156	{
157	//only add root folders to the delete queue
UNCOV 158	if (isRoot)	×
UNCOV 159	if (!directoriesToDelete.Any(dir =>	×
UNCOV 160	dir.FullName.Equals(toCleanup.FullName))) //don't add the same folder twice	×
UNCOV 161	directoriesToDelete.Add(toCleanup);	×
162
UNCOV 163	filesToDelete.AddRange(toCleanup.EnumerateFiles());	×
164
UNCOV 165	foreach (var dir in toCleanup.EnumerateDirectories())	×
UNCOV 166	AddDirectoryToCleanupList(dir, false, directoriesToDelete, filesToDelete);	×
UNCOV 167	}	×
168	}

HicServices / RDMP / 9758065908

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous