9957226774

Committed 16 Jul 2024 12:53PM UTC coverage: 56.677% (-0.002%) from 56.679%

Build # 9957226774

Build Type

push

github

Committed by

web-flow

Commit Message

Bugfix/rdmp 210: Improve LoadMetadata directory checking (#1884)

* add folders check

* add load directory checks

* add changelog

* update imports

---------

Co-authored-by: James A Sutherland <j.a.sutherland@dundee.ac.uk>

Run Details

10933 of 20780 branches covered (52.61%)

Branch coverage included in aggregate %.

13 of 24 new or added lines in 3 files covered. (54.17%)

1 existing line in 1 file now uncovered.

30981 of 53172 relevant lines covered (58.27%)

7851.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.57

/Rdmp.Core/DataLoad/Engine/LoadExecution/Components/LoadFiles.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System.Collections.Generic;
using System.Linq;
using Rdmp.Core.Curation;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Runtime;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components;

/// <summary>
/// DLE component responsible for the LoadStage.GetFiles.  This includes running any user configured ProcessTask (which will be RuntimeTasks in _components).
/// Also pushes DeleteForLoadingFilesOperation onto the disposal stack so that any files accumulated in ForLoading are cleared at the end of the DLE run (After
/// archiving).
/// </summary>
public class LoadFiles : CompositeDataLoadComponent
{
    public LoadFiles(List<IRuntimeTask> collection) : base(collection.Cast<IDataLoadComponent>().ToList())
    {
        Description = Description = "LoadFiles";
    }

    public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        if (Skip(job))
            return ExitCodeType.Error;

        var
            toReturn = ExitCodeType
                .Success; //This default will be returned unless there is an explicit DataProvider or collection of runtime tasks to run which return a different result (See below)

        // Figure out where we are getting the source files from
        try
        {
            if (Components.Any())
                toReturn = base.Run(job, cancellationToken);
            else
            {
                if (!((LoadDirectory)job.LoadDirectory).AllSubdirectoriesExist())
                {
                    job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,
                            $"One or more of the Load Metadata directories does not exist. Check the following locations exits: {string.Join(", ", [job.LoadDirectory.ForLoading.FullName, job.LoadDirectory.ForArchiving.FullName, job.LoadDirectory.Cache.FullName, job.LoadDirectory.ExecutablesPath.FullName])}"));
                    return ExitCodeType.Error;
                }
                else if (job.LoadDirectory.ForLoading.EnumerateFileSystemInfos().Any())
                    job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                        $"Using existing files in '{job.LoadDirectory.ForLoading.FullName}', there are no GetFiles processes or DataProviders configured"));
                else
                    job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                        $"There are no GetFiles tasks and there are no files in the ForLoading directory ({job.LoadDirectory.ForLoading.FullName})"));
            }

        }
        finally
        {
            // We can only clean up ForLoading after the job is finished, so give it the necessary disposal operation
            job.PushForDisposal(new DeleteForLoadingFilesOperation(job));
        }

        return toReturn;
    }

    public void DeleteAllFilesInForLoading(LoadDirectory directory, DataLoadJob job)
    {
        job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"Deleting files in ForLoading ({directory.ForLoading.FullName})"));
        directory.ForLoading.EnumerateFiles().ToList().ForEach(info => info.Delete());
        directory.ForLoading.EnumerateDirectories().ToList().ForEach(info => info.Delete(true));
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System.Collections.Generic;
8	using System.Linq;
9	using Rdmp.Core.Curation;
10	using Rdmp.Core.DataFlowPipeline;
11	using Rdmp.Core.DataLoad.Engine.Job;
12	using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Runtime;
13	using Rdmp.Core.ReusableLibraryCode.Progress;
14
15	namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components;
16
17	/// <summary>
18	/// DLE component responsible for the LoadStage.GetFiles. This includes running any user configured ProcessTask (which will be RuntimeTasks in _components).
19	/// Also pushes DeleteForLoadingFilesOperation onto the disposal stack so that any files accumulated in ForLoading are cleared at the end of the DLE run (After
20	/// archiving).
21	/// </summary>
22	public class LoadFiles : CompositeDataLoadComponent
23	{
24	public LoadFiles(List<IRuntimeTask> collection) : base(collection.Cast<IDataLoadComponent>().ToList())	46✔
25	{
26	Description = Description = "LoadFiles";	46✔
27	}	46✔
28
29	public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
30	{
31	if (Skip(job))	46!
32	return ExitCodeType.Error;	×
33
34	var	46✔
35	toReturn = ExitCodeType	46✔
36	.Success; //This default will be returned unless there is an explicit DataProvider or collection of runtime tasks to run which return a different result (See below)	46✔
37
38	// Figure out where we are getting the source files from
39	try
40	{
41	if (Components.Any())	46✔
42	toReturn = base.Run(job, cancellationToken);	2✔
43	else
44	{
45	if (!((LoadDirectory)job.LoadDirectory).AllSubdirectoriesExist())	44!
46	{
NEW 47	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,	×
NEW 48	$"One or more of the Load Metadata directories does not exist. Check the following locations exits: {string.Join(", ", [job.LoadDirectory.ForLoading.FullName, job.LoadDirectory.ForArchiving.FullName, job.LoadDirectory.Cache.FullName, job.LoadDirectory.ExecutablesPath.FullName])}"));	×
NEW 49	return ExitCodeType.Error;	×
50	}
51	else if (job.LoadDirectory.ForLoading.EnumerateFileSystemInfos().Any())	44✔
52	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	42✔
53	$"Using existing files in '{job.LoadDirectory.ForLoading.FullName}', there are no GetFiles processes or DataProviders configured"));	42✔
54	else
55	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	2✔
56	$"There are no GetFiles tasks and there are no files in the ForLoading directory ({job.LoadDirectory.ForLoading.FullName})"));	2✔
57	}
58
59	}	2✔
60	finally
61	{
62	// We can only clean up ForLoading after the job is finished, so give it the necessary disposal operation
63	job.PushForDisposal(new DeleteForLoadingFilesOperation(job));	46✔
64	}	46✔
65
66	return toReturn;	46✔
UNCOV 67	}	×
68
69	public void DeleteAllFilesInForLoading(LoadDirectory directory, DataLoadJob job)
70	{
71	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
72	$"Deleting files in ForLoading ({directory.ForLoading.FullName})"));	×
73	directory.ForLoading.EnumerateFiles().ToList().ForEach(info => info.Delete());	×
74	directory.ForLoading.EnumerateDirectories().ToList().ForEach(info => info.Delete(true));	×
75	}	×
76	}

HicServices / RDMP / 9957226774

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous