6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.94

/Rdmp.Core/DataExport/DataRelease/Pipeline/BasicDataReleaseDestination.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.DataExport.DataExtraction;
using Rdmp.Core.DataExport.DataRelease.Audit;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataFlowPipeline.Requirements;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataExport.DataRelease.Pipeline;

/// <summary>
/// Default release pipeline destination implementation wraps Release Engine for the supplied ReleaseData.
/// </summary>
public class BasicDataReleaseDestination : IPluginDataFlowComponent<ReleaseAudit>, IDataFlowDestination<ReleaseAudit>,
    IPipelineRequirement<Project>, IPipelineRequirement<ReleaseData>
{
    [DemandsNestedInitialization] public ReleaseEngineSettings ReleaseSettings { get; set; }

    private ReleaseData _releaseData;
    private Project _project;
    private DirectoryInfo _destinationFolder;
    private ReleaseEngine _engine;
    private List<IExtractionConfiguration> _configurationReleased;

    public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEventListener listener,
        GracefulCancellationToken cancellationToken)
    {
        if (releaseAudit == null)
            return null;

        if (releaseAudit.ReleaseFolder == null)
            throw new ArgumentException(
                "This component needs a destination folder! Did you forget to introduce and initialize the ReleaseFolderProvider in the pipeline?");

        if (_releaseData.ReleaseState == ReleaseState.DoingPatch)
        {
            listener.OnNotify(this,
                new NotifyEventArgs(ProgressEventType.Information,
                    "CumulativeExtractionResults for datasets not included in the Patch will now be erased."));

            var recordsDeleted = 0;

            foreach (var (configuration, potentials) in _releaseData.ConfigurationsForRelease)
                //foreach existing CumulativeExtractionResults if it is not included in the patch then it should be deleted
                foreach (var redundantResult in configuration.CumulativeExtractionResults.Where(r =>
                             potentials.All(rp => rp.DataSet.ID != r.ExtractableDataSet_ID)))
                {
                    redundantResult.DeleteInDatabase();
                    recordsDeleted++;
                }

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"Deleted {recordsDeleted} old CumulativeExtractionResults (That were not included in the final Patch you are preparing)"));
        }

        _engine = new ReleaseEngine(_project, ReleaseSettings, listener, releaseAudit);

        _engine.DoRelease(_releaseData.ConfigurationsForRelease, _releaseData.EnvironmentPotentials,
            _releaseData.ReleaseState == ReleaseState.DoingPatch);

        _destinationFolder = _engine.ReleaseAudit.ReleaseFolder;
        _configurationReleased = _engine.ConfigurationsReleased;

        return null;
    }

    public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
    {
        if (pipelineFailureExceptionIfAny != null && _releaseData != null)
            try
            {
                var remnantsDeleted = 0;

                foreach (ExtractionConfiguration configuration in _releaseData.ConfigurationsForRelease.Keys)
                    foreach (ReleaseLog remnant in configuration.ReleaseLog)
                    {
                        remnant.DeleteInDatabase();
                        remnantsDeleted++;
                    }

                if (remnantsDeleted > 0)
                    listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                        $"Because release failed we are deleting ReleaseLogEntries, this resulted in {remnantsDeleted} deleted records, you will likely need to re-extract these datasets or retrieve them from the Release directory"));
            }
            catch (Exception e1)
            {
                listener.OnNotify(this,
                    new NotifyEventArgs(ProgressEventType.Error,
                        "Error occurred when trying to clean up remnant ReleaseLogEntries", e1));
            }

        if (pipelineFailureExceptionIfAny == null && _destinationFolder != null)
        {
            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"Data release succeeded into:{_destinationFolder}"));
            //mark configuration as released
            foreach (var config in _configurationReleased)
            {
                config.IsReleased = true;
                config.SaveToDatabase();
            }

            if (ReleaseSettings.DeleteFilesOnSuccess)
            {
                listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Cleaning up..."));
                ExtractionDirectory.CleanupExtractionDirectory(this, _project.ExtractionDirectory,
                    _configurationReleased, listener);
            }

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "All done!"));
        }
    }

    public void Abort(IDataLoadEventListener listener)
    {
        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "This component cannot Abort!"));
    }

    public void Check(ICheckNotifier notifier)
    {
        ((ICheckable)ReleaseSettings).Check(notifier);
    }

    public void PreInitialize(Project value, IDataLoadEventListener listener)
    {
        _project = value;
    }

    public void PreInitialize(ReleaseData value, IDataLoadEventListener listener)
    {
        _releaseData = value;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.DataExport.Data;
13	using Rdmp.Core.DataExport.DataExtraction;
14	using Rdmp.Core.DataExport.DataRelease.Audit;
15	using Rdmp.Core.DataFlowPipeline;
16	using Rdmp.Core.DataFlowPipeline.Requirements;
17	using Rdmp.Core.ReusableLibraryCode.Checks;
18	using Rdmp.Core.ReusableLibraryCode.Progress;
19
20	namespace Rdmp.Core.DataExport.DataRelease.Pipeline;
21
22	/// <summary>
23	/// Default release pipeline destination implementation wraps Release Engine for the supplied ReleaseData.
24	/// </summary>
25	public class BasicDataReleaseDestination : IPluginDataFlowComponent<ReleaseAudit>, IDataFlowDestination<ReleaseAudit>,
26	IPipelineRequirement<Project>, IPipelineRequirement<ReleaseData>
27	{
28	[DemandsNestedInitialization] public ReleaseEngineSettings ReleaseSettings { get; set; }	10✔
29
30	private ReleaseData _releaseData;
31	private Project _project;
32	private DirectoryInfo _destinationFolder;
33	private ReleaseEngine _engine;
34	private List<IExtractionConfiguration> _configurationReleased;
35
36	public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEventListener listener,
37	GracefulCancellationToken cancellationToken)
38	{
39	if (releaseAudit == null)	2!
40	return null;	×
41
42	if (releaseAudit.ReleaseFolder == null)	2!
43	throw new ArgumentException(	×
44	"This component needs a destination folder! Did you forget to introduce and initialize the ReleaseFolderProvider in the pipeline?");	×
45
46	if (_releaseData.ReleaseState == ReleaseState.DoingPatch)	2!
47	{
48	listener.OnNotify(this,	×
49	new NotifyEventArgs(ProgressEventType.Information,	×
50	"CumulativeExtractionResults for datasets not included in the Patch will now be erased."));	×
51
52	var recordsDeleted = 0;	×
53
54	foreach (var (configuration, potentials) in _releaseData.ConfigurationsForRelease)	×
55	//foreach existing CumulativeExtractionResults if it is not included in the patch then it should be deleted
56	foreach (var redundantResult in configuration.CumulativeExtractionResults.Where(r =>	×
57	potentials.All(rp => rp.DataSet.ID != r.ExtractableDataSet_ID)))	×
58	{
59	redundantResult.DeleteInDatabase();	×
60	recordsDeleted++;	×
61	}
62
63	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
64	$"Deleted {recordsDeleted} old CumulativeExtractionResults (That were not included in the final Patch you are preparing)"));	×
65	}
66
67	_engine = new ReleaseEngine(_project, ReleaseSettings, listener, releaseAudit);	2✔
68
69	_engine.DoRelease(_releaseData.ConfigurationsForRelease, _releaseData.EnvironmentPotentials,	2✔
70	_releaseData.ReleaseState == ReleaseState.DoingPatch);	2✔
71
72	_destinationFolder = _engine.ReleaseAudit.ReleaseFolder;	2✔
73	_configurationReleased = _engine.ConfigurationsReleased;	2✔
74
75	return null;	2✔
76	}
77
78	public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
79	{
80	if (pipelineFailureExceptionIfAny != null && _releaseData != null)	2!
81	try
82	{
83	var remnantsDeleted = 0;	×
84
85	foreach (ExtractionConfiguration configuration in _releaseData.ConfigurationsForRelease.Keys)	×
86	foreach (ReleaseLog remnant in configuration.ReleaseLog)	×
87	{
88	remnant.DeleteInDatabase();	×
89	remnantsDeleted++;	×
90	}
91
92	if (remnantsDeleted > 0)	×
93	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
94	$"Because release failed we are deleting ReleaseLogEntries, this resulted in {remnantsDeleted} deleted records, you will likely need to re-extract these datasets or retrieve them from the Release directory"));	×
95	}	×
96	catch (Exception e1)	×
97	{
98	listener.OnNotify(this,	×
99	new NotifyEventArgs(ProgressEventType.Error,	×
100	"Error occurred when trying to clean up remnant ReleaseLogEntries", e1));	×
101	}	×
102
103	if (pipelineFailureExceptionIfAny == null && _destinationFolder != null)	2✔
104	{
105	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	2✔
106	$"Data release succeeded into:{_destinationFolder}"));	2✔
107	//mark configuration as released
108	foreach (var config in _configurationReleased)	16✔
109	{
110	config.IsReleased = true;	6✔
111	config.SaveToDatabase();	6✔
112	}
113
114	if (ReleaseSettings.DeleteFilesOnSuccess)	2✔
115	{
116	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Cleaning up..."));	2✔
117	ExtractionDirectory.CleanupExtractionDirectory(this, _project.ExtractionDirectory,	2✔
118	_configurationReleased, listener);	2✔
119	}
120
121	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "All done!"));	2✔
122	}
123	}	2✔
124
125	public void Abort(IDataLoadEventListener listener)
126	{
127	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "This component cannot Abort!"));	×
128	}	×
129
130	public void Check(ICheckNotifier notifier)
131	{
132	((ICheckable)ReleaseSettings).Check(notifier);	×
133	}	×
134
135	public void PreInitialize(Project value, IDataLoadEventListener listener)
136	{
137	_project = value;	4✔
138	}	4✔
139
140	public void PreInitialize(ReleaseData value, IDataLoadEventListener listener)
141	{
142	_releaseData = value;	4✔
143	}	4✔
144	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous