9859858140

Committed 09 Jul 2024 03:24PM UTC coverage: 56.679% (-0.2%) from 56.916%

Build # 9859858140

Build Type

push

github

Committed by

JFriel

Commit Message

update

Run Details

10912 of 20750 branches covered (52.59%)

Branch coverage included in aggregate %.

30965 of 53135 relevant lines covered (58.28%)

7908.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.12

/Rdmp.Core/DataExport/DataRelease/Pipeline/BasicDataReleaseDestination.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.DataExport.DataExtraction;
using Rdmp.Core.DataExport.DataRelease.Audit;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataFlowPipeline.Requirements;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataExport.DataRelease.Pipeline;

/// <summary>
/// Default release pipeline destination implementation wraps Release Engine for the supplied ReleaseData.
/// </summary>
public class BasicDataReleaseDestination : IPluginDataFlowComponent<ReleaseAudit>, IDataFlowDestination<ReleaseAudit>,
    IPipelineRequirement<Project>, IPipelineRequirement<ReleaseData>
{
    [DemandsNestedInitialization] public ReleaseEngineSettings ReleaseSettings { get; set; }

    private ReleaseData _releaseData;
    private Project _project;
    private DirectoryInfo _destinationFolder;
    private ReleaseEngine _engine;
    private List<IExtractionConfiguration> _configurationReleased;

    public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEventListener listener,
        GracefulCancellationToken cancellationToken)
    {
        if (releaseAudit == null)
            return null;

        if (releaseAudit.ReleaseFolder == null)
            throw new ArgumentException(
                "This component needs a destination folder! Did you forget to introduce and initialize the ReleaseFolderProvider in the pipeline?");

        if (_releaseData.ReleaseState == ReleaseState.DoingPatch)
        {
            listener.OnNotify(this,
                new NotifyEventArgs(ProgressEventType.Information,
                    "CumulativeExtractionResults for datasets not included in the Patch will now be erased."));

            var recordsDeleted = 0;

            foreach (var (configuration, potentials) in _releaseData.ConfigurationsForRelease)
                //foreach existing CumulativeExtractionResults if it is not included in the patch then it should be deleted
                foreach (var redundantResult in configuration.CumulativeExtractionResults.Where(r =>
                             potentials.All(rp => rp.DataSet.ID != r.ExtractableDataSet_ID)))
                {
                    redundantResult.DeleteInDatabase();
                    recordsDeleted++;
                }

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"Deleted {recordsDeleted} old CumulativeExtractionResults (That were not included in the final Patch you are preparing)"));
        }

        _engine = new ReleaseEngine(_project, ReleaseSettings, listener, releaseAudit);

        _engine.DoRelease(_releaseData.ConfigurationsForRelease, _releaseData.EnvironmentPotentials,
            _releaseData.ReleaseState == ReleaseState.DoingPatch);

        _destinationFolder = _engine.ReleaseAudit.ReleaseFolder;
        _configurationReleased = _engine.ConfigurationsReleased;

        return null;
    }

    public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
    {
        if (pipelineFailureExceptionIfAny != null && _releaseData != null)
            try
            {
                var remnantsDeleted = 0;

                foreach (ExtractionConfiguration configuration in _releaseData.ConfigurationsForRelease.Keys)
                    foreach (ReleaseLog remnant in configuration.ReleaseLog)
                    {
                        remnant.DeleteInDatabase();
                        remnantsDeleted++;
                    }

                if (remnantsDeleted > 0)
                    listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                        $"Because release failed we are deleting ReleaseLogEntries, this resulted in {remnantsDeleted} deleted records, you will likely need to re-extract these datasets or retrieve them from the Release directory"));
            }
            catch (Exception e1)
            {
                listener.OnNotify(this,
                    new NotifyEventArgs(ProgressEventType.Error,
                        "Error occurred when trying to clean up remnant ReleaseLogEntries", e1));
            }

        if (pipelineFailureExceptionIfAny == null && _destinationFolder != null)
        {
            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"Data release succeeded into:{_destinationFolder}"));
            //mark configuration as released
            foreach (var config in _configurationReleased)
            {
                config.IsReleased = true;
                config.SaveToDatabase();
            }

            if (ReleaseSettings.DeleteFilesOnSuccess)
            {
                listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Cleaning up..."));
                ExtractionDirectory.CleanupExtractionDirectory(this, _project.ExtractionDirectory,
                    _configurationReleased, listener);
            }

            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "All done!"));
        }
    }

    public void Abort(IDataLoadEventListener listener)
    {
        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "This component cannot Abort!"));
    }

    public void Check(ICheckNotifier notifier)
    {
        ((ICheckable)ReleaseSettings).Check(notifier);
    }

    public void PreInitialize(Project value, IDataLoadEventListener listener)
    {
        _project = value;
    }

    public void PreInitialize(ReleaseData value, IDataLoadEventListener listener)
    {
        _releaseData = value;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.DataExport.Data;
13	using Rdmp.Core.DataExport.DataExtraction;
14	using Rdmp.Core.DataExport.DataRelease.Audit;
15	using Rdmp.Core.DataFlowPipeline;
16	using Rdmp.Core.DataFlowPipeline.Requirements;
17	using Rdmp.Core.ReusableLibraryCode.Checks;
18	using Rdmp.Core.ReusableLibraryCode.Progress;
19
20	namespace Rdmp.Core.DataExport.DataRelease.Pipeline;
21
22	/// <summary>
23	/// Default release pipeline destination implementation wraps Release Engine for the supplied ReleaseData.
24	/// </summary>
25	public class BasicDataReleaseDestination : IPluginDataFlowComponent<ReleaseAudit>, IDataFlowDestination<ReleaseAudit>,
26	IPipelineRequirement<Project>, IPipelineRequirement<ReleaseData>
27	{
28	[DemandsNestedInitialization] public ReleaseEngineSettings ReleaseSettings { get; set; }	4✔
29
30	private ReleaseData _releaseData;
31	private Project _project;
32	private DirectoryInfo _destinationFolder;
33	private ReleaseEngine _engine;
34	private List<IExtractionConfiguration> _configurationReleased;
35
36	public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEventListener listener,
37	GracefulCancellationToken cancellationToken)
38	{
39	if (releaseAudit == null)	×
40	return null;	×
41
42	if (releaseAudit.ReleaseFolder == null)	×
43	throw new ArgumentException(	×
44	"This component needs a destination folder! Did you forget to introduce and initialize the ReleaseFolderProvider in the pipeline?");	×
45
46	if (_releaseData.ReleaseState == ReleaseState.DoingPatch)	×
47	{
48	listener.OnNotify(this,	×
49	new NotifyEventArgs(ProgressEventType.Information,	×
50	"CumulativeExtractionResults for datasets not included in the Patch will now be erased."));	×
51
52	var recordsDeleted = 0;	×
53
54	foreach (var (configuration, potentials) in _releaseData.ConfigurationsForRelease)	×
55	//foreach existing CumulativeExtractionResults if it is not included in the patch then it should be deleted
56	foreach (var redundantResult in configuration.CumulativeExtractionResults.Where(r =>	×
57	potentials.All(rp => rp.DataSet.ID != r.ExtractableDataSet_ID)))	×
58	{
59	redundantResult.DeleteInDatabase();	×
60	recordsDeleted++;	×
61	}
62
63	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
64	$"Deleted {recordsDeleted} old CumulativeExtractionResults (That were not included in the final Patch you are preparing)"));	×
65	}
66
67	_engine = new ReleaseEngine(_project, ReleaseSettings, listener, releaseAudit);	×
68
69	_engine.DoRelease(_releaseData.ConfigurationsForRelease, _releaseData.EnvironmentPotentials,	×
70	_releaseData.ReleaseState == ReleaseState.DoingPatch);	×
71
72	_destinationFolder = _engine.ReleaseAudit.ReleaseFolder;	×
73	_configurationReleased = _engine.ConfigurationsReleased;	×
74
75	return null;	×
76	}
77
78	public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
79	{
80	if (pipelineFailureExceptionIfAny != null && _releaseData != null)	×
81	try
82	{
83	var remnantsDeleted = 0;	×
84
85	foreach (ExtractionConfiguration configuration in _releaseData.ConfigurationsForRelease.Keys)	×
86	foreach (ReleaseLog remnant in configuration.ReleaseLog)	×
87	{
88	remnant.DeleteInDatabase();	×
89	remnantsDeleted++;	×
90	}
91
92	if (remnantsDeleted > 0)	×
93	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
94	$"Because release failed we are deleting ReleaseLogEntries, this resulted in {remnantsDeleted} deleted records, you will likely need to re-extract these datasets or retrieve them from the Release directory"));	×
95	}	×
96	catch (Exception e1)	×
97	{
98	listener.OnNotify(this,	×
99	new NotifyEventArgs(ProgressEventType.Error,	×
100	"Error occurred when trying to clean up remnant ReleaseLogEntries", e1));	×
101	}	×
102
103	if (pipelineFailureExceptionIfAny == null && _destinationFolder != null)	×
104	{
105	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	×
106	$"Data release succeeded into:{_destinationFolder}"));	×
107	//mark configuration as released
108	foreach (var config in _configurationReleased)	×
109	{
110	config.IsReleased = true;	×
111	config.SaveToDatabase();	×
112	}
113
114	if (ReleaseSettings.DeleteFilesOnSuccess)	×
115	{
116	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Cleaning up..."));	×
117	ExtractionDirectory.CleanupExtractionDirectory(this, _project.ExtractionDirectory,	×
118	_configurationReleased, listener);	×
119	}
120
121	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "All done!"));	×
122	}
123	}	×
124
125	public void Abort(IDataLoadEventListener listener)
126	{
127	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "This component cannot Abort!"));	×
128	}	×
129
130	public void Check(ICheckNotifier notifier)
131	{
132	((ICheckable)ReleaseSettings).Check(notifier);	×
133	}	×
134
135	public void PreInitialize(Project value, IDataLoadEventListener listener)
136	{
137	_project = value;	×
138	}	×
139
140	public void PreInitialize(ReleaseData value, IDataLoadEventListener listener)
141	{
142	_releaseData = value;	×
143	}	×
144	}

HicServices / RDMP / 9859858140

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous