11457925069

pending completion

Build # 11457925069

Build Type

push

github

Committed by

JFriel

Commit Message

Merge branch 'develop' of https://github.com/HicServices/RDMP

Run Details

11207 of 21044 branches covered (53.26%)

Branch coverage included in aggregate %.

25 of 51 new or added lines in 3 files covered. (49.02%)

705 existing lines in 27 files now uncovered.

31728 of 53779 relevant lines covered (59.0%)

4115.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.67

/Rdmp.Core/DataLoad/Engine/LoadExecution/Components/PopulateRAW.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Runtime;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components;

/// <summary>
/// DLE component responsible for creating the RAW (first database in the RAW=>STAGING=>LIVE model of DLE loading) database (if required).  Also runs
/// all LoadStage.Mounting and components.
/// </summary>
public class PopulateRAW : CompositeDataLoadComponent
{
    private readonly HICDatabaseConfiguration _databaseConfiguration;

    public PopulateRAW(List<IRuntimeTask> collection, HICDatabaseConfiguration databaseConfiguration) : base(
        collection.Cast<IDataLoadComponent>().ToList())
    {
        _databaseConfiguration = databaseConfiguration;
        Description = "Populate RAW";
    }

    public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        if (Skip(job))
            return ExitCodeType.Error;

        // We may or may not need to create the raw database, depending on how we are getting the data
        CreateRawDatabaseIfRequired(job);

        var toReturn = base.Run(job, cancellationToken);

        if (toReturn == ExitCodeType.Success)
            // Verify that we have put something into the database
            VerifyExistenceOfRawData(job);

        return toReturn;
    }

    private bool MustCreateRawDatabase()
    {
        var attachingProcesses = Components.OfType<AttacherRuntimeTask>().ToArray();

        switch (attachingProcesses.Length)
        {
            //we do not have any attaching processes ... magically the data must appear somehow in our RAW database so better create it -- maybe an executable is going to populate it or something
            case 0:
                return true;
            case > 1:
                {
                    // if there are multiple attachers, ensure that they all agree on whether or not they require external database creation
                    var attachers = attachingProcesses.Select(runtime => runtime.Attacher).ToList();
                    var numAttachersRequiringDbCreation =
                        attachers.Count(attacher => attacher.RequestsExternalDatabaseCreation);

                    if (numAttachersRequiringDbCreation > 0 && numAttachersRequiringDbCreation < attachingProcesses.Length)
                        throw new Exception(
                            $"If there are multiple attachers then they should all agree on whether they require database creation or not: {attachers.Aggregate("", (s, attacher) => $"{s} {attacher.GetType().Name}:{attacher.RequestsExternalDatabaseCreation}")}");
                    break;
                }
        }

        return attachingProcesses[0].Attacher.RequestsExternalDatabaseCreation;
    }

    private void CreateRawDatabaseIfRequired(IDataLoadJob job)
    {
        // Ask the runtime process host if we need to create the RAW database
        if (!MustCreateRawDatabase()) return;

        job.OnNotify(this,
            new NotifyEventArgs(ProgressEventType.Information,
                "Determined that we must create the RAW database tables..."));

        var cloner = new DatabaseCloner(_databaseConfiguration);

        if (!job.PersistentRaw) cloner.CreateDatabaseForStage(LoadBubble.Raw);

        job.CreateTablesInStage(cloner, LoadBubble.Raw);
    }

    // Check that either Raw database exists and is populated, or that 'forLoading' is not empty
    private void VerifyExistenceOfRawData(IDataLoadJob job)
    {
        var raw = _databaseConfiguration.DeployInfo[LoadBubble.Raw];

        if (!raw.Exists())
            job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,
                $"The Mounting stage has not created the {raw.GetRuntimeName()} database."));

        var rawDbInfo = _databaseConfiguration.DeployInfo[LoadBubble.Raw];

        if (_databaseConfiguration.ExpectTables(job, LoadBubble.Raw, true).All(t => t.IsEmpty()))
        {
            var message = $"The Mounting stage has not populated the RAW database ({rawDbInfo}) with any data";
            job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, message));
            throw new Exception(message);
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Linq;
10	using Rdmp.Core.Curation.Data;
11	using Rdmp.Core.DataFlowPipeline;
12	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
13	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
14	using Rdmp.Core.DataLoad.Engine.Job;
15	using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Runtime;
16	using Rdmp.Core.ReusableLibraryCode.Progress;
17
18	namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components;
19
20	/// <summary>
21	/// DLE component responsible for creating the RAW (first database in the RAW=>STAGING=>LIVE model of DLE loading) database (if required). Also runs
22	/// all LoadStage.Mounting and components.
23	/// </summary>
24	public class PopulateRAW : CompositeDataLoadComponent
25	{
26	private readonly HICDatabaseConfiguration _databaseConfiguration;
27
28	public PopulateRAW(List<IRuntimeTask> collection, HICDatabaseConfiguration databaseConfiguration) : base(	46✔
29	collection.Cast<IDataLoadComponent>().ToList())	46✔
30	{
31	_databaseConfiguration = databaseConfiguration;	46✔
32	Description = "Populate RAW";	46✔
33	}	46✔
34
35	public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
36	{
37	if (Skip(job))	46!
38	return ExitCodeType.Error;	×
39
40	// We may or may not need to create the raw database, depending on how we are getting the data
41	CreateRawDatabaseIfRequired(job);	46✔
42
43	var toReturn = base.Run(job, cancellationToken);	44✔
44
45	if (toReturn == ExitCodeType.Success)	44✔
46	// Verify that we have put something into the database
47	VerifyExistenceOfRawData(job);	42✔
48
49	return toReturn;	44✔
50	}
51
52	private bool MustCreateRawDatabase()
53	{
54	var attachingProcesses = Components.OfType<AttacherRuntimeTask>().ToArray();	46✔
55
56	switch (attachingProcesses.Length)	46!
57	{
58	//we do not have any attaching processes ... magically the data must appear somehow in our RAW database so better create it -- maybe an executable is going to populate it or something
59	case 0:
60	return true;	×
61	case > 1:
62	{
63	// if there are multiple attachers, ensure that they all agree on whether or not they require external database creation
64	var attachers = attachingProcesses.Select(runtime => runtime.Attacher).ToList();	12✔
65	var numAttachersRequiringDbCreation =	4✔
66	attachers.Count(attacher => attacher.RequestsExternalDatabaseCreation);	12✔
67
68	if (numAttachersRequiringDbCreation > 0 && numAttachersRequiringDbCreation < attachingProcesses.Length)	4!
69	throw new Exception(	×
70	$"If there are multiple attachers then they should all agree on whether they require database creation or not: {attachers.Aggregate("", (s, attacher) => $"{s} {attacher.GetType().Name}:{attacher.RequestsExternalDatabaseCreation}")}");	×
71	break;
72	}
73	}
74
75	return attachingProcesses[0].Attacher.RequestsExternalDatabaseCreation;	46✔
76	}
77
78	private void CreateRawDatabaseIfRequired(IDataLoadJob job)
79	{
80	// Ask the runtime process host if we need to create the RAW database
81	if (!MustCreateRawDatabase()) return;	48✔
82
83	job.OnNotify(this,	44✔
84	new NotifyEventArgs(ProgressEventType.Information,	44✔
85	"Determined that we must create the RAW database tables..."));	44✔
86
87	var cloner = new DatabaseCloner(_databaseConfiguration);	44✔
88
89	if (!job.PersistentRaw) cloner.CreateDatabaseForStage(LoadBubble.Raw);	88✔
90
91	job.CreateTablesInStage(cloner, LoadBubble.Raw);	42✔
92	}	42✔
93
94	// Check that either Raw database exists and is populated, or that 'forLoading' is not empty
95	private void VerifyExistenceOfRawData(IDataLoadJob job)
96	{
97	var raw = _databaseConfiguration.DeployInfo[LoadBubble.Raw];	42✔
98
99	if (!raw.Exists())	42!
UNCOV 100	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,	×
101	$"The Mounting stage has not created the {raw.GetRuntimeName()} database."));	×
102
103	var rawDbInfo = _databaseConfiguration.DeployInfo[LoadBubble.Raw];	42✔
104
105	if (_databaseConfiguration.ExpectTables(job, LoadBubble.Raw, true).All(t => t.IsEmpty()))	84!
106	{
UNCOV 107	var message = $"The Mounting stage has not populated the RAW database ({rawDbInfo}) with any data";	×
108	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, message));	×
109	throw new Exception(message);	×
110	}
111	}	42✔
112	}

HicServices / RDMP / 11457925069

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous