6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.46

/Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
using Rdmp.Core.DataLoad.Engine.LoadProcess;
using Rdmp.Core.Logging;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.Job;

/// <inheritdoc/>
public class DataLoadJob : IDataLoadJob
{
    public string Description { get; private set; }
    public IDataLoadInfo DataLoadInfo { get; private set; }
    public ILoadDirectory LoadDirectory { get; set; }
    private readonly IDataLoadEventListener _listener;

    public int JobID { get; set; }

    private readonly ILogManager _logManager;
    public ILoadMetadata LoadMetadata { get; private set; }

    public List<ITableInfo> RegularTablesToLoad { get; private set; }
    public List<ITableInfo> LookupTablesToLoad { get; private set; }
    public IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; private set; }

    private Stack<IDisposeAfterDataLoad> _disposalStack = new();

    public HICDatabaseConfiguration Configuration { get; set; }
    public object Payload { get; set; }

    public bool PersistentRaw { get; set; }


    private List<NotifyEventArgs> _crashAtEnd = new();

    public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();


    private string _loggingTask;

    public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description,
        ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener,
        HICDatabaseConfiguration configuration)
    {
        _logManager = logManager;
        RepositoryLocator = repositoryLocator;
        LoadMetadata = loadMetadata;
        LoadDirectory = directory;
        Configuration = configuration;
        _listener = listener;
        Description = description;

        var catalogues = LoadMetadata.GetAllCatalogues().ToList();

        if (LoadMetadata != null)
            _loggingTask = GetLoggingTask(catalogues);

        RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
        LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
    }

    private string GetLoggingTask(IEnumerable<ICatalogue> cataloguesToLoad)
    {
        var distinctLoggingTasks = cataloguesToLoad.Select(catalogue => catalogue.LoggingDataTask).Distinct().ToList();
        if (distinctLoggingTasks.Count > 1)
            throw new Exception(
                $"The catalogues to be loaded do not share the same logging task: {string.Join(", ", distinctLoggingTasks)}");

        _loggingTask = distinctLoggingTasks.First();
        return string.IsNullOrWhiteSpace(_loggingTask)
            ? throw new Exception("There is no logging task specified for this load (the name is blank)")
            : _loggingTask;
    }

    private void CreateDataLoadInfo()
    {
        if (string.IsNullOrWhiteSpace(Description))
            throw new Exception(
                "The data load description (for the DataLoadInfo object) must not be empty, please provide a relevant description");

        DataLoadInfo = _logManager.CreateDataLoadInfo(_loggingTask, nameof(DataLoadProcess), Description, "", false);

        if (DataLoadInfo == null)
            throw new Exception("DataLoadInfo is null");

        JobID = DataLoadInfo.ID;
    }

    public void LogProgress(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnProgress, senderName, message);
    }

    public void LogError(string message, Exception exception)
    {
        // we are bailing out before the load process has had a chance to create a DataLoadInfo object
        if (DataLoadInfo == null)
            CreateDataLoadInfo();

        DataLoadInfo.LogFatalError(nameof(DataLoadProcess),
            message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true));
        DataLoadInfo.CloseAndMarkComplete();
    }

    public void StartLogging()
    {
        CreateDataLoadInfo();
    }

    public void CloseLogging()
    {
        DataLoadInfo?.CloseAndMarkComplete();
    }

    public void LogInformation(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, senderName, message);
    }

    public void LogWarning(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message);
    }

    public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
    {
        foreach (TableInfo regularTableInfo in RegularTablesToLoad)
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage);

        foreach (TableInfo lookupTableInfo in LookupTablesToLoad)
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage);

        PushForDisposal(cloner);
    }

    public void PushForDisposal(IDisposeAfterDataLoad disposeable)
    {
        _disposalStack.Push(disposeable);
    }

    public void OnNotify(object sender, NotifyEventArgs e)
    {
        if (DataLoadInfo != null)
            switch (e.ProgressEventType)
            {
                case ProgressEventType.Trace:
                case ProgressEventType.Debug:
                    break;
                case ProgressEventType.Information:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation,
                        sender.GetType().Name, e.Message + (e.Exception != null
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
                            : ""));
                    break;
                case ProgressEventType.Warning:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name,
                        e.Message + (e.Exception != null
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
                            : ""));
                    break;
                case ProgressEventType.Error:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name,
                        e.Message);
                    DataLoadInfo.LogFatalError(sender.GetType().Name,
                        e.Exception != null
                            ? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)
                            : e.Message);
                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }

        _listener.OnNotify(sender, e);
    }

    public void OnProgress(object sender, ProgressEventArgs e)
    {
        _listener.OnProgress(sender, e);
    }

    public string ArchiveFilepath => Path.Combine(LoadDirectory.ForArchiving.FullName, $"{DataLoadInfo.ID}.zip");

    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
    {
        while (_disposalStack.Any())
        {
            var disposable = _disposalStack.Pop();
            disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);
        }
    }

    public ColumnInfo[] GetAllColumns()
    {
        return RegularTablesToLoad.SelectMany(t => t.ColumnInfos)
            .Union(LookupTablesToLoad.SelectMany(t => t.ColumnInfos)).Distinct().ToArray();
    }

    public void CrashAtEnd(NotifyEventArgs because)
    {
        _crashAtEnd.Add(because);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.Curation.Data.DataLoad;
14	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
15	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
16	using Rdmp.Core.DataLoad.Engine.LoadProcess;
17	using Rdmp.Core.Logging;
18	using Rdmp.Core.Repositories;
19	using Rdmp.Core.ReusableLibraryCode;
20	using Rdmp.Core.ReusableLibraryCode.Progress;
21
22	namespace Rdmp.Core.DataLoad.Engine.Job;
23
24	/// <inheritdoc/>
25	public class DataLoadJob : IDataLoadJob
26	{
27	public string Description { get; private set; }	198✔
28	public IDataLoadInfo DataLoadInfo { get; private set; }	3,088✔
29	public ILoadDirectory LoadDirectory { get; set; }	476✔
30	private readonly IDataLoadEventListener _listener;
31
32	public int JobID { get; set; }	104✔
33
34	private readonly ILogManager _logManager;
35	public ILoadMetadata LoadMetadata { get; private set; }	438✔
36
37	public List<ITableInfo> RegularTablesToLoad { get; private set; }	368✔
38	public List<ITableInfo> LookupTablesToLoad { get; private set; }	326✔
39	public IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; private set; }	74✔
40
41	private Stack<IDisposeAfterDataLoad> _disposalStack = new();	74✔
42
43	public HICDatabaseConfiguration Configuration { get; set; }	116✔
44	public object Payload { get; set; }	20✔
45
46	public bool PersistentRaw { get; set; }	60✔
47
48
49	private List<NotifyEventArgs> _crashAtEnd = new();	74✔
50
51	public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();	42✔
52
53
54	private string _loggingTask;
55
56	public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description,	74✔
57	ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener,	74✔
58	HICDatabaseConfiguration configuration)	74✔
59	{
60	_logManager = logManager;	74✔
61	RepositoryLocator = repositoryLocator;	74✔
62	LoadMetadata = loadMetadata;	74✔
63	LoadDirectory = directory;	74✔
64	Configuration = configuration;	74✔
65	_listener = listener;	74✔
66	Description = description;	74✔
67
68	var catalogues = LoadMetadata.GetAllCatalogues().ToList();	74✔
69
70	if (LoadMetadata != null)	74✔
71	_loggingTask = GetLoggingTask(catalogues);	74✔
72
73	RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();	152✔
74	LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();	152✔
75	}	74✔
76
77	private string GetLoggingTask(IEnumerable<ICatalogue> cataloguesToLoad)
78	{
79	var distinctLoggingTasks = cataloguesToLoad.Select(catalogue => catalogue.LoggingDataTask).Distinct().ToList();	152✔
80	if (distinctLoggingTasks.Count > 1)	74!
81	throw new Exception(	×
82	$"The catalogues to be loaded do not share the same logging task: {string.Join(", ", distinctLoggingTasks)}");	×
83
84	_loggingTask = distinctLoggingTasks.First();	74✔
85	return string.IsNullOrWhiteSpace(_loggingTask)	74!
86	? throw new Exception("There is no logging task specified for this load (the name is blank)")	74✔
87	: _loggingTask;	74✔
88	}
89
90	private void CreateDataLoadInfo()
91	{
92	if (string.IsNullOrWhiteSpace(Description))	62!
93	throw new Exception(	×
94	"The data load description (for the DataLoadInfo object) must not be empty, please provide a relevant description");	×
95
96	DataLoadInfo = _logManager.CreateDataLoadInfo(_loggingTask, nameof(DataLoadProcess), Description, "", false);	62✔
97
98	if (DataLoadInfo == null)	62!
99	throw new Exception("DataLoadInfo is null");	×
100
101	JobID = DataLoadInfo.ID;	62✔
102	}	62✔
103
104	public void LogProgress(string senderName, string message)
105	{
106	if (DataLoadInfo == null)	×
107	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
108
109	if (!DataLoadInfo.IsClosed)	×
110	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnProgress, senderName, message);	×
111	}	×
112
113	public void LogError(string message, Exception exception)
114	{
115	// we are bailing out before the load process has had a chance to create a DataLoadInfo object
116	if (DataLoadInfo == null)	×
117	CreateDataLoadInfo();	×
118
119	DataLoadInfo.LogFatalError(nameof(DataLoadProcess),	×
120	message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true));	×
121	DataLoadInfo.CloseAndMarkComplete();	×
122	}	×
123
124	public void StartLogging()
125	{
126	CreateDataLoadInfo();	62✔
127	}	62✔
128
129	public void CloseLogging()
130	{
131	DataLoadInfo?.CloseAndMarkComplete();	86!
132	}	86✔
133
134	public void LogInformation(string senderName, string message)
135	{
136	if (DataLoadInfo == null)	×
137	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
138
139	if (!DataLoadInfo.IsClosed)	×
140	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, senderName, message);	×
141	}	×
142
143	public void LogWarning(string senderName, string message)
144	{
145	if (DataLoadInfo == null)	×
146	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
147
148	if (!DataLoadInfo.IsClosed)	×
149	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message);	×
150	}	×
151
152	public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
153	{
154	foreach (TableInfo regularTableInfo in RegularTablesToLoad)	352✔
155	cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage);	92✔
156
157	foreach (TableInfo lookupTableInfo in LookupTablesToLoad)	168!
158	cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage);	×
159
160	PushForDisposal(cloner);	84✔
161	}	84✔
162
163	public void PushForDisposal(IDisposeAfterDataLoad disposeable)
164	{
165	_disposalStack.Push(disposeable);	618✔
166	}	618✔
167
168	public void OnNotify(object sender, NotifyEventArgs e)
169	{
170	if (DataLoadInfo != null)	1,502✔
171	switch (e.ProgressEventType)	1,472!
172	{
173	case ProgressEventType.Trace:
174	case ProgressEventType.Debug:
175	break;
176	case ProgressEventType.Information:
177	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation,	1,002!
178	sender.GetType().Name, e.Message + (e.Exception != null	1,002✔
179	? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"	1,002✔
180	: ""));	1,002✔
181	break;	1,002✔
182	case ProgressEventType.Warning:
183	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name,	82✔
184	e.Message + (e.Exception != null	82✔
185	? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"	82✔
186	: ""));	82✔
187	break;	82✔
188	case ProgressEventType.Error:
189	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name,	2✔
190	e.Message);	2✔
191	DataLoadInfo.LogFatalError(sender.GetType().Name,	2!
192	e.Exception != null	2✔
193	? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)	2✔
194	: e.Message);	2✔
195	break;	2✔
196	default:
197	throw new ArgumentOutOfRangeException();	×
198	}
199
200	_listener.OnNotify(sender, e);	1,502✔
201	}	1,500✔
202
203	public void OnProgress(object sender, ProgressEventArgs e)
204	{
205	_listener.OnProgress(sender, e);	266✔
206	}	266✔
207
208	public string ArchiveFilepath => Path.Combine(LoadDirectory.ForArchiving.FullName, $"{DataLoadInfo.ID}.zip");	42✔
209
210	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
211	{
212	while (_disposalStack.Any())	694✔
213	{
214	var disposable = _disposalStack.Pop();	608✔
215	disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);	608✔
216	}
217	}	86✔
218
219	public ColumnInfo[] GetAllColumns()
220	{
221	return RegularTablesToLoad.SelectMany(t => t.ColumnInfos)	88✔
222	.Union(LookupTablesToLoad.SelectMany(t => t.ColumnInfos)).Distinct().ToArray();	42✔
223	}
224
225	public void CrashAtEnd(NotifyEventArgs because)
226	{
227	_crashAtEnd.Add(because);	×
228	}	×
229	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous