6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.78

/Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Standard/MigrateRAWToStaging.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System.Collections.Generic;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.LoadProcess;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Standard;

/// <summary>
/// DLE component resonsible for streaming data off the RAW database and writing it to the STAGING database.  Happens one table at a time with the actual
/// implementation of moving data in MigrateRAWTableToStaging (See MigrateRAWTableToStaging).
/// </summary>
public class MigrateRAWToStaging : DataLoadComponent
{
    private readonly HICDatabaseConfiguration _databaseConfiguration;

    private readonly Stack<IDisposeAfterDataLoad> _toDispose = new();

    public MigrateRAWToStaging(HICDatabaseConfiguration databaseConfiguration,
        HICLoadConfigurationFlags loadConfigurationFlags)
    {
        _databaseConfiguration = databaseConfiguration;

        Description = "Migrate RAW to Staging";
        SkipComponent = !loadConfigurationFlags.DoLoadToStaging;
    }

    public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        if (Skip(job))
            return ExitCodeType.Error;


        // To be on the safe side, we will create/destroy the staging tables on a per-load basis
        if (_databaseConfiguration.RequiresStagingTableCreation)
            CreateStagingTables(job);

        DoMigration(job, cancellationToken);

        return ExitCodeType.Success;
    }

    public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
    {
        while (_toDispose.Any())
            _toDispose.Pop().LoadCompletedSoDispose(exitCode, postLoadEventListener);
    }

    private void DoMigration(IDataLoadJob job, GracefulCancellationToken cancellationToken)
    {
        foreach (var regularTableInfo in job.RegularTablesToLoad)
            MigrateRAWTableToStaging(job, regularTableInfo, false, cancellationToken);

        foreach (var lookupTableInfo in job.LookupTablesToLoad)
            MigrateRAWTableToStaging(job, lookupTableInfo, true, cancellationToken);
    }


    private void MigrateRAWTableToStaging(IDataLoadJob job, ITableInfo tableInfo, bool isLookupTable,
        GracefulCancellationToken cancellationToken)
    {
        var component = new MigrateRAWTableToStaging(tableInfo, isLookupTable, _databaseConfiguration);
        component.Run(job, cancellationToken);
    }

    private void CreateStagingTables(IDataLoadJob job)
    {
        var cloner = new DatabaseCloner(_databaseConfiguration);
        job.CreateTablesInStage(cloner, LoadBubble.Staging);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System.Collections.Generic;
8	using System.Linq;
9	using Rdmp.Core.Curation.Data;
10	using Rdmp.Core.DataFlowPipeline;
11	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
12	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
13	using Rdmp.Core.DataLoad.Engine.Job;
14	using Rdmp.Core.DataLoad.Engine.LoadProcess;
15	using Rdmp.Core.ReusableLibraryCode.Progress;
16
17	namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Standard;
18
19	/// <summary>
20	/// DLE component resonsible for streaming data off the RAW database and writing it to the STAGING database. Happens one table at a time with the actual
21	/// implementation of moving data in MigrateRAWTableToStaging (See MigrateRAWTableToStaging).
22	/// </summary>
23	public class MigrateRAWToStaging : DataLoadComponent
24	{
25	private readonly HICDatabaseConfiguration _databaseConfiguration;
26
27	private readonly Stack<IDisposeAfterDataLoad> _toDispose = new();	46✔
28
29	public MigrateRAWToStaging(HICDatabaseConfiguration databaseConfiguration,	46✔
30	HICLoadConfigurationFlags loadConfigurationFlags)	46✔
31	{
32	_databaseConfiguration = databaseConfiguration;	46✔
33
34	Description = "Migrate RAW to Staging";	46✔
35	SkipComponent = !loadConfigurationFlags.DoLoadToStaging;	46✔
36	}	46✔
37
38	public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
39	{
40	if (Skip(job))	42!
41	return ExitCodeType.Error;	×
42
43
44	// To be on the safe side, we will create/destroy the staging tables on a per-load basis
45	if (_databaseConfiguration.RequiresStagingTableCreation)	42✔
46	CreateStagingTables(job);	42✔
47
48	DoMigration(job, cancellationToken);	42✔
49
50	return ExitCodeType.Success;	42✔
51	}
52
53	public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
54	{
55	while (_toDispose.Any())	42!
56	_toDispose.Pop().LoadCompletedSoDispose(exitCode, postLoadEventListener);	×
57	}	42✔
58
59	private void DoMigration(IDataLoadJob job, GracefulCancellationToken cancellationToken)
60	{
61	foreach (var regularTableInfo in job.RegularTablesToLoad)	176✔
62	MigrateRAWTableToStaging(job, regularTableInfo, false, cancellationToken);	46✔
63
64	foreach (var lookupTableInfo in job.LookupTablesToLoad)	84!
65	MigrateRAWTableToStaging(job, lookupTableInfo, true, cancellationToken);	×
66	}	42✔
67
68
69	private void MigrateRAWTableToStaging(IDataLoadJob job, ITableInfo tableInfo, bool isLookupTable,
70	GracefulCancellationToken cancellationToken)
71	{
72	var component = new MigrateRAWTableToStaging(tableInfo, isLookupTable, _databaseConfiguration);	46✔
73	component.Run(job, cancellationToken);	46✔
74	}	46✔
75
76	private void CreateStagingTables(IDataLoadJob job)
77	{
78	var cloner = new DatabaseCloner(_databaseConfiguration);	42✔
79	job.CreateTablesInStage(cloner, LoadBubble.Staging);	42✔
80	}	42✔
81	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous