10943038862

Committed 19 Sep 2024 02:26PM UTC coverage: 57.373% (-0.02%) from 57.395%

Build # 10943038862

Build Type

push

github

Committed by

web-flow

Commit Message

Merge branch 'develop' into task/rdmp-247-mdf-sequentials

Run Details

11193 of 21020 branches covered (53.25%)

Branch coverage included in aggregate %.

6 of 29 new or added lines in 6 files covered. (20.69%)

1 existing line in 1 file now uncovered.

31677 of 53701 relevant lines covered (58.99%)

8166.42 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.08

/Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
using Rdmp.Core.DataLoad.Engine.LoadProcess;
using Rdmp.Core.Logging;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.Job;

/// <inheritdoc/>
public class DataLoadJob : IDataLoadJob
{
    public string Description { get; private set; }
    public IDataLoadInfo DataLoadInfo { get; private set; }
    public ILoadDirectory LoadDirectory { get; set; }
    private readonly IDataLoadEventListener _listener;

    public int JobID { get; set; }

    private readonly ILogManager _logManager;
    public ILoadMetadata LoadMetadata { get; private set; }

    public List<ITableInfo> RegularTablesToLoad { get; private set; }
    public List<ITableInfo> LookupTablesToLoad { get; private set; }
    public IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; private set; }

    private Stack<IDisposeAfterDataLoad> _disposalStack = new();

    public HICDatabaseConfiguration Configuration { get; set; }
    public object Payload { get; set; }

    public bool PersistentRaw { get; set; }


    private List<NotifyEventArgs> _crashAtEnd = new();

    public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();


    private string _loggingTask;

    public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description,
        ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener,
        HICDatabaseConfiguration configuration)
    {
        _logManager = logManager;
        RepositoryLocator = repositoryLocator;
        LoadMetadata = loadMetadata;
        LoadDirectory = directory;
        Configuration = configuration;
        _listener = listener;
        Description = description;

        var catalogues = LoadMetadata.GetAllCatalogues().ToList();

        if (LoadMetadata != null)
            _loggingTask = GetLoggingTask(catalogues);

        RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
        LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
    }

    private string GetLoggingTask(IEnumerable<ICatalogue> cataloguesToLoad)
    {
        var distinctLoggingTasks = cataloguesToLoad.Select(catalogue => catalogue.LoggingDataTask).Distinct().ToList();
        if (distinctLoggingTasks.Count > 1)
            throw new Exception(
                $"The catalogues to be loaded do not share the same logging task: {string.Join(", ", distinctLoggingTasks)}");

        _loggingTask = distinctLoggingTasks.First();
        return string.IsNullOrWhiteSpace(_loggingTask)
            ? throw new Exception("There is no logging task specified for this load (the name is blank)")
            : _loggingTask;
    }

    private void CreateDataLoadInfo()
    {
        if (string.IsNullOrWhiteSpace(Description))
            throw new Exception(
                "The data load description (for the DataLoadInfo object) must not be empty, please provide a relevant description");

        DataLoadInfo = _logManager.CreateDataLoadInfo(_loggingTask, nameof(DataLoadProcess), Description, "", false);

        if (DataLoadInfo == null)
            throw new Exception("DataLoadInfo is null");

        JobID = DataLoadInfo.ID;
    }

    public void LogProgress(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnProgress, senderName, message);
    }

    public void LogError(string message, Exception exception)
    {
        // we are bailing out before the load process has had a chance to create a DataLoadInfo object
        if (DataLoadInfo == null)
            CreateDataLoadInfo();

        DataLoadInfo.LogFatalError(nameof(DataLoadProcess),
            message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true));
        DataLoadInfo.CloseAndMarkComplete();
    }

    public void StartLogging()
    {
        CreateDataLoadInfo();
    }

    public void CloseLogging()
    {
        DataLoadInfo?.CloseAndMarkComplete();
    }

    public void LogInformation(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, senderName, message);
    }

    public void LogWarning(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message);
    }

    public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
    {
        bool allowReservedPrefixColumns = stage == LoadBubble.Raw ? LoadMetadata.AllowReservedPrefix : true;
        foreach (TableInfo regularTableInfo in RegularTablesToLoad)
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage, allowReservedPrefixColumns);

        foreach (TableInfo lookupTableInfo in LookupTablesToLoad)
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage, allowReservedPrefixColumns);

        PushForDisposal(cloner);
    }

    public void PushForDisposal(IDisposeAfterDataLoad disposeable)
    {
        _disposalStack.Push(disposeable);
    }

    public void OnNotify(object sender, NotifyEventArgs e)
    {
        if (DataLoadInfo != null)
            switch (e.ProgressEventType)
            {
                case ProgressEventType.Trace:
                case ProgressEventType.Debug:
                    break;
                case ProgressEventType.Information:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation,
                        sender.GetType().Name, e.Message + (e.Exception != null
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
                            : ""));
                    break;
                case ProgressEventType.Warning:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name,
                        e.Message + (e.Exception != null
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
                            : ""));
                    break;
                case ProgressEventType.Error:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name,
                        e.Message);
                    DataLoadInfo.LogFatalError(sender.GetType().Name,
                        e.Exception != null
                            ? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)
                            : e.Message);
                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }

        _listener.OnNotify(sender, e);
    }

    public void OnProgress(object sender, ProgressEventArgs e)
    {
        _listener.OnProgress(sender, e);
    }

    public string ArchiveFilepath => Path.Combine(LoadDirectory.ForArchiving.FullName, $"{DataLoadInfo.ID}.zip");

    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
    {
        while (_disposalStack.Any())
        {
            var disposable = _disposalStack.Pop();
            disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);
        }
    }

    public ColumnInfo[] GetAllColumns()
    {
        return RegularTablesToLoad.SelectMany(t => t.ColumnInfos)
            .Union(LookupTablesToLoad.SelectMany(t => t.ColumnInfos)).Distinct().ToArray();
    }

    public void CrashAtEnd(NotifyEventArgs because)
    {
        _crashAtEnd.Add(because);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.Curation.Data.DataLoad;
14	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
15	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
16	using Rdmp.Core.DataLoad.Engine.LoadProcess;
17	using Rdmp.Core.Logging;
18	using Rdmp.Core.Repositories;
19	using Rdmp.Core.ReusableLibraryCode;
20	using Rdmp.Core.ReusableLibraryCode.Progress;
21
22	namespace Rdmp.Core.DataLoad.Engine.Job;
23
24	/// <inheritdoc/>
25	public class DataLoadJob : IDataLoadJob
26	{
27	public string Description { get; private set; }	276✔
28	public IDataLoadInfo DataLoadInfo { get; private set; }	3,582✔
29	public ILoadDirectory LoadDirectory { get; set; }	546✔
30	private readonly IDataLoadEventListener _listener;
31
32	public int JobID { get; set; }	130✔
33
34	private readonly ILogManager _logManager;
35	public ILoadMetadata LoadMetadata { get; private set; }	608✔
36
37	public List<ITableInfo> RegularTablesToLoad { get; private set; }	394✔
38	public List<ITableInfo> LookupTablesToLoad { get; private set; }	352✔
39	public IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; private set; }	100✔
40
41	private Stack<IDisposeAfterDataLoad> _disposalStack = new();	100✔
42
43	public HICDatabaseConfiguration Configuration { get; set; }	142✔
44	public object Payload { get; set; }	20✔
45
46	public bool PersistentRaw { get; set; }	60✔
47
48
49	private List<NotifyEventArgs> _crashAtEnd = new();	100✔
50
51	public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();	42✔
52
53
54	private string _loggingTask;
55
56	public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description,	100✔
57	ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener,	100✔
58	HICDatabaseConfiguration configuration)	100✔
59	{
60	_logManager = logManager;	100✔
61	RepositoryLocator = repositoryLocator;	100✔
62	LoadMetadata = loadMetadata;	100✔
63	LoadDirectory = directory;	100✔
64	Configuration = configuration;	100✔
65	_listener = listener;	100✔
66	Description = description;	100✔
67
68	var catalogues = LoadMetadata.GetAllCatalogues().ToList();	100✔
69
70	if (LoadMetadata != null)	100✔
71	_loggingTask = GetLoggingTask(catalogues);	100✔
72
73	RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();	204✔
74	LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();	204✔
75	}	100✔
76
77	private string GetLoggingTask(IEnumerable<ICatalogue> cataloguesToLoad)
78	{
79	var distinctLoggingTasks = cataloguesToLoad.Select(catalogue => catalogue.LoggingDataTask).Distinct().ToList();	204✔
80	if (distinctLoggingTasks.Count > 1)	100!
81	throw new Exception(	×
82	$"The catalogues to be loaded do not share the same logging task: {string.Join(", ", distinctLoggingTasks)}");	×
83
84	_loggingTask = distinctLoggingTasks.First();	100✔
85	return string.IsNullOrWhiteSpace(_loggingTask)	100!
86	? throw new Exception("There is no logging task specified for this load (the name is blank)")	100✔
87	: _loggingTask;	100✔
88	}
89
90	private void CreateDataLoadInfo()
91	{
92	if (string.IsNullOrWhiteSpace(Description))	88!
93	throw new Exception(	×
94	"The data load description (for the DataLoadInfo object) must not be empty, please provide a relevant description");	×
95
96	DataLoadInfo = _logManager.CreateDataLoadInfo(_loggingTask, nameof(DataLoadProcess), Description, "", false);	88✔
97
98	if (DataLoadInfo == null)	88!
99	throw new Exception("DataLoadInfo is null");	×
100
101	JobID = DataLoadInfo.ID;	88✔
102	}	88✔
103
104	public void LogProgress(string senderName, string message)
105	{
106	if (DataLoadInfo == null)	×
107	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
108
109	if (!DataLoadInfo.IsClosed)	×
110	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnProgress, senderName, message);	×
111	}	×
112
113	public void LogError(string message, Exception exception)
114	{
115	// we are bailing out before the load process has had a chance to create a DataLoadInfo object
116	if (DataLoadInfo == null)	×
117	CreateDataLoadInfo();	×
118
119	DataLoadInfo.LogFatalError(nameof(DataLoadProcess),	×
120	message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true));	×
121	DataLoadInfo.CloseAndMarkComplete();	×
122	}	×
123
124	public void StartLogging()
125	{
126	CreateDataLoadInfo();	88✔
127	}	88✔
128
129	public void CloseLogging()
130	{
131	DataLoadInfo?.CloseAndMarkComplete();	86!
132	}	86✔
133
134	public void LogInformation(string senderName, string message)
135	{
136	if (DataLoadInfo == null)	×
137	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
138
139	if (!DataLoadInfo.IsClosed)	×
140	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, senderName, message);	×
141	}	×
142
143	public void LogWarning(string senderName, string message)
144	{
145	if (DataLoadInfo == null)	×
146	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
147
148	if (!DataLoadInfo.IsClosed)	×
149	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message);	×
150	}	×
151
152	public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
153	{
154	bool allowReservedPrefixColumns = stage == LoadBubble.Raw ? LoadMetadata.AllowReservedPrefix : true;	84✔
155	foreach (TableInfo regularTableInfo in RegularTablesToLoad)	352✔
156	cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage, allowReservedPrefixColumns);	92✔
157
158	foreach (TableInfo lookupTableInfo in LookupTablesToLoad)	168!
NEW 159	cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage, allowReservedPrefixColumns);	×
160
161	PushForDisposal(cloner);	84✔
162	}	84✔
163
164	public void PushForDisposal(IDisposeAfterDataLoad disposeable)
165	{
166	_disposalStack.Push(disposeable);	618✔
167	}	618✔
168
169	public void OnNotify(object sender, NotifyEventArgs e)
170	{
171	if (DataLoadInfo != null)	1,762✔
172	switch (e.ProgressEventType)	1,732!
173	{
174	case ProgressEventType.Trace:
175	case ProgressEventType.Debug:
176	break;
177	case ProgressEventType.Information:
178	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation,	1,132!
179	sender.GetType().Name, e.Message + (e.Exception != null	1,132✔
180	? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"	1,132✔
181	: ""));	1,132✔
182	break;	1,132✔
183	case ProgressEventType.Warning:
184	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name,	82✔
185	e.Message + (e.Exception != null	82✔
186	? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"	82✔
187	: ""));	82✔
188	break;	82✔
189	case ProgressEventType.Error:
190	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name,	2✔
191	e.Message);	2✔
192	DataLoadInfo.LogFatalError(sender.GetType().Name,	2!
193	e.Exception != null	2✔
194	? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)	2✔
195	: e.Message);	2✔
196	break;	2✔
197	default:
198	throw new ArgumentOutOfRangeException();	×
199	}
200
201	_listener.OnNotify(sender, e);	1,762✔
202	}	1,760✔
203
204	public void OnProgress(object sender, ProgressEventArgs e)
205	{
206	_listener.OnProgress(sender, e);	344✔
207	}	344✔
208
209	public string ArchiveFilepath => Path.Combine(LoadDirectory.ForArchiving.FullName, $"{DataLoadInfo.ID}.zip");	42✔
210
211	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
212	{
213	while (_disposalStack.Any())	694✔
214	{
215	var disposable = _disposalStack.Pop();	608✔
216	disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);	608✔
217	}
218	}	86✔
219
220	public ColumnInfo[] GetAllColumns()
221	{
222	return RegularTablesToLoad.SelectMany(t => t.ColumnInfos)	88✔
223	.Union(LookupTablesToLoad.SelectMany(t => t.ColumnInfos)).Distinct().ToArray();	42✔
224	}
225
226	public void CrashAtEnd(NotifyEventArgs because)
227	{
228	_crashAtEnd.Add(because);	×
229	}	×
230	}

HicServices / RDMP / 10943038862

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous