10250867621

Committed 05 Aug 2024 02:37PM UTC coverage: 57.281%. Remained the same

Build # 10250867621

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #1915 from HicServices/task/RDMP-231-dqe-runner-multiple-catalogues

DQE Post Run Loader Catalogue Looping

Run Details

11089 of 20826 branches covered (53.25%)

Branch coverage included in aggregate %.

0 of 19 new or added lines in 1 file covered. (0.0%)

1 existing line in 1 file now uncovered.

31368 of 53295 relevant lines covered (58.86%)

7972.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

7.14

/Rdmp.Core/DataLoad/Modules/Mutilators/DQEPostLoadRunner.cs

﻿// Copyright (c) The University of Dundee 2024-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using FAnsi.Discovery;
using Rdmp.Core.CommandExecution;
using Rdmp.Core.CommandLine.Options;
using Rdmp.Core.CommandLine.Runners;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.Curation.Data.Defaults;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.Mutilators;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;
using System;
using System.Linq;

namespace Rdmp.Core.DataLoad.Modules.Mutilators;

/// <summary>
/// This component will run the DQE engine on the associated catalogue after the data load has ran
/// </summary>
public class DQEPostLoadRunner : IMutilateDataTables
{

    public void Check(ICheckNotifier notifier)
    {
    }

    public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
    {
        if (loadStage != LoadStage.PostLoad)
        {
            throw new Exception("DQL Runner can only be done in the PostLoad stage.");
        }
    }

    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
    {
    }

    public ExitCodeType Mutilate(IDataLoadJob job)
    {
        var dqeServer = job.RepositoryLocator.CatalogueRepository.GetDefaultFor(PermissableDefaults.DQE);
        if (dqeServer == null)
        {
            job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
            "There is no DQE server. DQE will not be run."));
            return ExitCodeType.Success;
        }
        var lmdID = job.LoadMetadata.ID;
        var linkage = job.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", lmdID);
        foreach (var link in linkage)
        {
            var catalogue = job.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<Catalogue>("ID", link.CatalogueID).FirstOrDefault();
            if (catalogue is null) continue;
            if (catalogue.TimeCoverage_ExtractionInformation_ID == null)
            {
                job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
               $"Catalogue '{catalogue.Name}' does not have a Time Coverage column set. DQE will not be run"));
            }

            if (string.IsNullOrWhiteSpace(catalogue.ValidatorXML))
            {
                job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                 $"Catalogue '{catalogue.Name}' does not have any validation rules configured.DQE will not be run."));
            }

            DqeOptions options = new()
            {
                Catalogue = catalogue.ID.ToString(),
                Command = CommandLineActivity.run
            };
            var runner = RunnerFactory.CreateRunner(new ThrowImmediatelyActivator(job.RepositoryLocator), options);
            runner.Run(job.RepositoryLocator, ThrowImmediatelyDataLoadEventListener.Quiet, new AcceptAllCheckNotifier(),
                        new GracefulCancellationToken());
        }
        return ExitCodeType.Success;

    }
}

1	// Copyright (c) The University of Dundee 2024-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using FAnsi.Discovery;
8	using Rdmp.Core.CommandExecution;
9	using Rdmp.Core.CommandLine.Options;
10	using Rdmp.Core.CommandLine.Runners;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.Curation.Data.DataLoad;
13	using Rdmp.Core.Curation.Data.Defaults;
14	using Rdmp.Core.DataFlowPipeline;
15	using Rdmp.Core.DataLoad.Engine.Job;
16	using Rdmp.Core.DataLoad.Engine.Mutilators;
17	using Rdmp.Core.ReusableLibraryCode.Checks;
18	using Rdmp.Core.ReusableLibraryCode.Progress;
19	using System;
20	using System.Linq;
21
22	namespace Rdmp.Core.DataLoad.Modules.Mutilators;
23
24	/// <summary>
25	/// This component will run the DQE engine on the associated catalogue after the data load has ran
26	/// </summary>
27	public class DQEPostLoadRunner : IMutilateDataTables
28	{
29
30	public void Check(ICheckNotifier notifier)
31	{
32	}	×
33
34	public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
35	{
36	if (loadStage != LoadStage.PostLoad)	4!
37	{
38	throw new Exception("DQL Runner can only be done in the PostLoad stage.");	4✔
39	}
40	}	×
41
42	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
43	{
44	}	×
45
46	public ExitCodeType Mutilate(IDataLoadJob job)
47	{
48	var dqeServer = job.RepositoryLocator.CatalogueRepository.GetDefaultFor(PermissableDefaults.DQE);	×
49	if (dqeServer == null)	×
50	{
51	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	×
52	"There is no DQE server. DQE will not be run."));	×
53	return ExitCodeType.Success;	×
54	}
NEW 55	var lmdID = job.LoadMetadata.ID;	×
NEW 56	var linkage = job.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", lmdID);	×
NEW 57	foreach (var link in linkage)	×
58	{
NEW 59	var catalogue = job.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<Catalogue>("ID", link.CatalogueID).FirstOrDefault();	×
NEW 60	if (catalogue is null) continue;	×
NEW 61	if (catalogue.TimeCoverage_ExtractionInformation_ID == null)	×
62	{
NEW 63	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	×
NEW 64	$"Catalogue '{catalogue.Name}' does not have a Time Coverage column set. DQE will not be run"));	×
65	}
66
NEW 67	if (string.IsNullOrWhiteSpace(catalogue.ValidatorXML))	×
68	{
NEW 69	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	×
NEW 70	$"Catalogue '{catalogue.Name}' does not have any validation rules configured.DQE will not be run."));	×
71	}
72
NEW 73	DqeOptions options = new()	×
NEW 74	{	×
NEW 75	Catalogue = catalogue.ID.ToString(),	×
NEW 76	Command = CommandLineActivity.run	×
NEW 77	};	×
NEW 78	var runner = RunnerFactory.CreateRunner(new ThrowImmediatelyActivator(job.RepositoryLocator), options);	×
NEW 79	runner.Run(job.RepositoryLocator, ThrowImmediatelyDataLoadEventListener.Quiet, new AcceptAllCheckNotifier(),	×
NEW 80	new GracefulCancellationToken());	×
81	}
UNCOV 82	return ExitCodeType.Success;	×
83
84	}
85	}

HicServices / RDMP / 10250867621

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous