6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.68

/Rdmp.Core/DataLoad/Engine/LoadExecution/HICDataLoadFactory.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.LoadExecution.Components;
using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Arguments;
using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Standard;
using Rdmp.Core.DataLoad.Engine.LoadProcess;
using Rdmp.Core.Logging;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.LoadExecution;

/// <summary>
/// This is factored out more for documentation's sake. It is a description of the HIC data load pipeline, in factory form!
/// </summary>
public class HICDataLoadFactory
{
    private readonly HICDatabaseConfiguration _databaseConfiguration;
    private readonly HICLoadConfigurationFlags _loadConfigurationFlags;
    private readonly ICatalogueRepository _repository;
    private readonly ILogManager _logManager;
    private readonly IList<ICatalogue> _cataloguesToLoad;

    public ILoadMetadata LoadMetadata { get; private set; }

    public HICDataLoadFactory(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration,
        HICLoadConfigurationFlags loadConfigurationFlags, ICatalogueRepository repository, ILogManager logManager)
    {
        _databaseConfiguration = databaseConfiguration;
        _loadConfigurationFlags = loadConfigurationFlags;
        _repository = repository;
        _logManager = logManager;
        LoadMetadata = loadMetadata;

        // If we are not supplied any catalogues to load, it is expected that we will load all catalogues associated with the provided ILoadMetadata
        _cataloguesToLoad = LoadMetadata.GetAllCatalogues().ToList();
        if (!_cataloguesToLoad.Any())
            throw new InvalidOperationException(
                $"LoadMetadata {LoadMetadata.ID} is not related to any Catalogues, there is nothing to load");
    }

    public IDataLoadExecution Create(IDataLoadEventListener postLoadEventListener)
    {
        var loadArgsDictionary = new LoadArgsDictionary(LoadMetadata, _databaseConfiguration.DeployInfo);

        //warn user about disabled tasks
        var processTasks = LoadMetadata.ProcessTasks.ToList();
        foreach (var task in processTasks
                     .Where(p => p.IsDisabled))
            postLoadEventListener.OnNotify(this,
                new NotifyEventArgs(ProgressEventType.Warning, $"Found disabled ProcessTask{task}"));

        //Get all the runtime tasks which are not disabled
        var factory = new RuntimeTaskPackager(processTasks.Where(p => !p.IsDisabled), loadArgsDictionary.LoadArgs,
            _cataloguesToLoad, _repository);

        var getFiles = new LoadFiles(factory.GetRuntimeTasksForStage(LoadStage.GetFiles));

        var mounting = new PopulateRAW(factory.GetRuntimeTasksForStage(LoadStage.Mounting), _databaseConfiguration);

        var adjustRaw = factory.CreateCompositeDataLoadComponentFor(LoadStage.AdjustRaw, "Adjust RAW");

        var migrateToStaging = new MigrateRAWToStaging(_databaseConfiguration, _loadConfigurationFlags);

        var adjustStaging = factory.CreateCompositeDataLoadComponentFor(LoadStage.AdjustStaging, "Adjust Staging");

        var migrateStagingToLive = new MigrateStagingToLive(_databaseConfiguration, _loadConfigurationFlags);

        var postLoad = factory.CreateCompositeDataLoadComponentFor(LoadStage.PostLoad, "Post Load");

        var archiveFiles = new ArchiveFiles(_loadConfigurationFlags);

        var loadStagingDatabase = new CompositeDataLoadComponent(new List<IDataLoadComponent>
        {
            mounting,
            adjustRaw,
            migrateToStaging
        });

        var adjustStagingAndMigrateToLive = new CompositeDataLoadComponent(new List<IDataLoadComponent>
        {
            loadStagingDatabase,
            adjustStaging,
            migrateStagingToLive,
            postLoad
        });

        var components = new List<IDataLoadComponent>
        {
            getFiles,
            adjustStagingAndMigrateToLive,
            archiveFiles
        };

        return new SingleJobExecution(components);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Linq;
10	using Rdmp.Core.Curation.Data;
11	using Rdmp.Core.Curation.Data.DataLoad;
12	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
13	using Rdmp.Core.DataLoad.Engine.LoadExecution.Components;
14	using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Arguments;
15	using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Standard;
16	using Rdmp.Core.DataLoad.Engine.LoadProcess;
17	using Rdmp.Core.Logging;
18	using Rdmp.Core.Repositories;
19	using Rdmp.Core.ReusableLibraryCode.Progress;
20
21	namespace Rdmp.Core.DataLoad.Engine.LoadExecution;
22
23	/// <summary>
24	/// This is factored out more for documentation's sake. It is a description of the HIC data load pipeline, in factory form!
25	/// </summary>
26	public class HICDataLoadFactory
27	{
28	private readonly HICDatabaseConfiguration _databaseConfiguration;
29	private readonly HICLoadConfigurationFlags _loadConfigurationFlags;
30	private readonly ICatalogueRepository _repository;
31	private readonly ILogManager _logManager;
32	private readonly IList<ICatalogue> _cataloguesToLoad;
33
34	public ILoadMetadata LoadMetadata { get; private set; }	184✔
35
36	public HICDataLoadFactory(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration,	46✔
37	HICLoadConfigurationFlags loadConfigurationFlags, ICatalogueRepository repository, ILogManager logManager)	46✔
38	{
39	_databaseConfiguration = databaseConfiguration;	46✔
40	_loadConfigurationFlags = loadConfigurationFlags;	46✔
41	_repository = repository;	46✔
42	_logManager = logManager;	46✔
43	LoadMetadata = loadMetadata;	46✔
44
45	// If we are not supplied any catalogues to load, it is expected that we will load all catalogues associated with the provided ILoadMetadata
46	_cataloguesToLoad = LoadMetadata.GetAllCatalogues().ToList();	46✔
47	if (!_cataloguesToLoad.Any())	46!
48	throw new InvalidOperationException(	×
49	$"LoadMetadata {LoadMetadata.ID} is not related to any Catalogues, there is nothing to load");	×
50	}	46✔
51
52	public IDataLoadExecution Create(IDataLoadEventListener postLoadEventListener)
53	{
54	var loadArgsDictionary = new LoadArgsDictionary(LoadMetadata, _databaseConfiguration.DeployInfo);	46✔
55
56	//warn user about disabled tasks
57	var processTasks = LoadMetadata.ProcessTasks.ToList();	46✔
58	foreach (var task in processTasks	92!
59	.Where(p => p.IsDisabled))	98✔
60	postLoadEventListener.OnNotify(this,	×
61	new NotifyEventArgs(ProgressEventType.Warning, $"Found disabled ProcessTask{task}"));	×
62
63	//Get all the runtime tasks which are not disabled
64	var factory = new RuntimeTaskPackager(processTasks.Where(p => !p.IsDisabled), loadArgsDictionary.LoadArgs,	306✔
65	_cataloguesToLoad, _repository);	46✔
66
67	var getFiles = new LoadFiles(factory.GetRuntimeTasksForStage(LoadStage.GetFiles));	46✔
68
69	var mounting = new PopulateRAW(factory.GetRuntimeTasksForStage(LoadStage.Mounting), _databaseConfiguration);	46✔
70
71	var adjustRaw = factory.CreateCompositeDataLoadComponentFor(LoadStage.AdjustRaw, "Adjust RAW");	46✔
72
73	var migrateToStaging = new MigrateRAWToStaging(_databaseConfiguration, _loadConfigurationFlags);	46✔
74
75	var adjustStaging = factory.CreateCompositeDataLoadComponentFor(LoadStage.AdjustStaging, "Adjust Staging");	46✔
76
77	var migrateStagingToLive = new MigrateStagingToLive(_databaseConfiguration, _loadConfigurationFlags);	46✔
78
79	var postLoad = factory.CreateCompositeDataLoadComponentFor(LoadStage.PostLoad, "Post Load");	46✔
80
81	var archiveFiles = new ArchiveFiles(_loadConfigurationFlags);	46✔
82
83	var loadStagingDatabase = new CompositeDataLoadComponent(new List<IDataLoadComponent>	46✔
84	{	46✔
85	mounting,	46✔
86	adjustRaw,	46✔
87	migrateToStaging	46✔
88	});	46✔
89
90	var adjustStagingAndMigrateToLive = new CompositeDataLoadComponent(new List<IDataLoadComponent>	46✔
91	{	46✔
92	loadStagingDatabase,	46✔
93	adjustStaging,	46✔
94	migrateStagingToLive,	46✔
95	postLoad	46✔
96	});	46✔
97
98	var components = new List<IDataLoadComponent>	46✔
99	{	46✔
100	getFiles,	46✔
101	adjustStagingAndMigrateToLive,	46✔
102	archiveFiles	46✔
103	};	46✔
104
105	return new SingleJobExecution(components);	46✔
106	}
107	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous