23054871216

Committed 13 Mar 2026 02:13PM UTC coverage: 57.063% (-0.06%) from 57.12%

Build # 23054871216

Build Type

Pull #2323

github

Committed by

web-flow

Commit Message

Merge branch 'develop' into task/RDMP-365-better-load-metadata-logging

Pull Request Pull Request #2323: Task/rdmp 365 better load metadata logging

Run Details

11528 of 21729 branches covered (53.05%)

Branch coverage included in aggregate %.

28 of 41 new or added lines in 7 files covered. (68.29%)

33 existing lines in 4 files now uncovered.

32629 of 55654 relevant lines covered (58.63%)

18134.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.08

/Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
using Rdmp.Core.DataLoad.Engine.LoadProcess;
using Rdmp.Core.Logging;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.Job;

/// <inheritdoc/>
public class DataLoadJob : IDataLoadJob
{
    public string Description { get; private set; }
    public IDataLoadInfo DataLoadInfo { get; private set; }
    public ILoadDirectory LoadDirectory { get; set; }
    private readonly IDataLoadEventListener _listener;

    public int JobID { get; set; }

    private readonly ILogManager _logManager;
    public ILoadMetadata LoadMetadata { get; private set; }

    public List<ITableInfo> RegularTablesToLoad { get; private set; }
    public List<ITableInfo> LookupTablesToLoad { get; private set; }
    public IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; private set; }

    private Stack<IDisposeAfterDataLoad> _disposalStack = new();

    public HICDatabaseConfiguration Configuration { get; set; }
    public object Payload { get; set; }

    public bool PersistentRaw { get; set; }


    private List<NotifyEventArgs> _crashAtEnd = new();

    public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();


    private string _loggingTask;

    public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description,
        ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener,
        HICDatabaseConfiguration configuration)
    {
        _logManager = logManager;
        RepositoryLocator = repositoryLocator;
        LoadMetadata = loadMetadata;
        LoadDirectory = directory;
        Configuration = configuration;
        _listener = listener;
        Description = description;

        var catalogues = LoadMetadata.GetAllCatalogues().ToList();

        if (LoadMetadata != null)
            _loggingTask = GetLoggingTask(catalogues);

        RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
        LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
    }

    private string GetLoggingTask(IEnumerable<ICatalogue> cataloguesToLoad)
    {
        var distinctLoggingTasks = cataloguesToLoad.SelectMany(catalogue => catalogue.LoggingDataTasks).DistinctBy(l => l.Name ).ToList();
        if (distinctLoggingTasks.Count > 1)
            throw new Exception(
                $"The catalogues to be loaded do not share the same logging task: {string.Join(", ", distinctLoggingTasks.Select(l => l.Name))}");

        _loggingTask = distinctLoggingTasks.First().Name;
        return string.IsNullOrWhiteSpace(_loggingTask)
            ? throw new Exception("There is no logging task specified for this load (the name is blank)")
            : _loggingTask;
    }

    private void CreateDataLoadInfo()
    {
        if (string.IsNullOrWhiteSpace(Description))
            throw new Exception(
                "The data load description (for the DataLoadInfo object) must not be empty, please provide a relevant description");

        DataLoadInfo = _logManager.CreateDataLoadInfo(_loggingTask, nameof(DataLoadProcess), Description, "", false);

        if (DataLoadInfo == null)
            throw new Exception("DataLoadInfo is null");

        JobID = DataLoadInfo.ID;
    }

    public void LogProgress(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnProgress, senderName, message);
    }

    public void LogError(string message, Exception exception)
    {
        // we are bailing out before the load process has had a chance to create a DataLoadInfo object
        if (DataLoadInfo == null)
            CreateDataLoadInfo();

        DataLoadInfo.LogFatalError(nameof(DataLoadProcess),
            message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true));
        DataLoadInfo.CloseAndMarkComplete();
    }

    public void StartLogging()
    {
        CreateDataLoadInfo();
    }

    public void CloseLogging()
    {
        DataLoadInfo?.CloseAndMarkComplete();
    }

    public void LogInformation(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, senderName, message);
    }

    public void LogWarning(string senderName, string message)
    {
        if (DataLoadInfo == null)
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");

        if (!DataLoadInfo.IsClosed)
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message);
    }

    public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
    {
        bool allowReservedPrefixColumns = stage == LoadBubble.Raw ? LoadMetadata.AllowReservedPrefix : true;
        foreach (TableInfo regularTableInfo in RegularTablesToLoad)
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage, allowReservedPrefixColumns);

        foreach (TableInfo lookupTableInfo in LookupTablesToLoad)
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage, allowReservedPrefixColumns);

        PushForDisposal(cloner);
    }

    public void PushForDisposal(IDisposeAfterDataLoad disposeable)
    {
        _disposalStack.Push(disposeable);
    }

    public void OnNotify(object sender, NotifyEventArgs e)
    {
        if (DataLoadInfo != null)
            switch (e.ProgressEventType)
            {
                case ProgressEventType.Trace:
                case ProgressEventType.Debug:
                    break;
                case ProgressEventType.Information:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation,
                        sender.GetType().Name, e.Message + (e.Exception != null
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
                            : ""));
                    break;
                case ProgressEventType.Warning:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name,
                        e.Message + (e.Exception != null
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
                            : ""));
                    break;
                case ProgressEventType.Error:
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name,
                        e.Message);
                    DataLoadInfo.LogFatalError(sender.GetType().Name,
                        e.Exception != null
                            ? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)
                            : e.Message);
                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }

        _listener.OnNotify(sender, e);
    }

    public void OnProgress(object sender, ProgressEventArgs e)
    {
        _listener.OnProgress(sender, e);
    }

    public string ArchiveFilepath => Path.Combine(LoadDirectory.ForArchiving.FullName, $"{DataLoadInfo.ID}.zip");

    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
    {
        while (_disposalStack.Any())
        {
            var disposable = _disposalStack.Pop();
            disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);
        }
    }

    public ColumnInfo[] GetAllColumns()
    {
        return RegularTablesToLoad.SelectMany(t => t.ColumnInfos)
            .Union(LookupTablesToLoad.SelectMany(t => t.ColumnInfos)).Distinct().ToArray();
    }

    public void CrashAtEnd(NotifyEventArgs because)
    {
        _crashAtEnd.Add(because);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.Curation;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.Curation.Data.DataLoad;
14	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
15	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
16	using Rdmp.Core.DataLoad.Engine.LoadProcess;
17	using Rdmp.Core.Logging;
18	using Rdmp.Core.Repositories;
19	using Rdmp.Core.ReusableLibraryCode;
20	using Rdmp.Core.ReusableLibraryCode.Progress;
21
22	namespace Rdmp.Core.DataLoad.Engine.Job;
23
24	/// <inheritdoc/>
25	public class DataLoadJob : IDataLoadJob
26	{
27	public string Description { get; private set; }	318✔
28	public IDataLoadInfo DataLoadInfo { get; private set; }	4,684✔
29	public ILoadDirectory LoadDirectory { get; set; }	700✔
30	private readonly IDataLoadEventListener _listener;
31
32	public int JobID { get; set; }	158✔
33
34	private readonly ILogManager _logManager;
35	public ILoadMetadata LoadMetadata { get; private set; }	734✔
36
37	public List<ITableInfo> RegularTablesToLoad { get; private set; }	520✔
38	public List<ITableInfo> LookupTablesToLoad { get; private set; }	450✔
39	public IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; private set; }	122✔
40
41	private Stack<IDisposeAfterDataLoad> _disposalStack = new();	114✔
42
43	public HICDatabaseConfiguration Configuration { get; set; }	184✔
44	public object Payload { get; set; }	20✔
45
46	public bool PersistentRaw { get; set; }	74✔
47
48
49	private List<NotifyEventArgs> _crashAtEnd = new();	114✔
50
51	public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();	56✔
52
53
54	private string _loggingTask;
55
56	public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description,	114✔
57	ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener,	114✔
58	HICDatabaseConfiguration configuration)	114✔
59	{
60	_logManager = logManager;	114✔
61	RepositoryLocator = repositoryLocator;	114✔
62	LoadMetadata = loadMetadata;	114✔
63	LoadDirectory = directory;	114✔
64	Configuration = configuration;	114✔
65	_listener = listener;	114✔
66	Description = description;	114✔
67
68	var catalogues = LoadMetadata.GetAllCatalogues().ToList();	114✔
69
70	if (LoadMetadata != null)	114✔
71	_loggingTask = GetLoggingTask(catalogues);	114✔
72
73	RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();	232✔
74	LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();	232✔
75	}	114✔
76
77	private string GetLoggingTask(IEnumerable<ICatalogue> cataloguesToLoad)
78	{
79	var distinctLoggingTasks = cataloguesToLoad.SelectMany(catalogue => catalogue.LoggingDataTasks).DistinctBy(l => l.Name ).ToList();	350✔
80	if (distinctLoggingTasks.Count > 1)	114!
81	throw new Exception(	×
NEW 82	$"The catalogues to be loaded do not share the same logging task: {string.Join(", ", distinctLoggingTasks.Select(l => l.Name))}");	×
83
84	_loggingTask = distinctLoggingTasks.First().Name;	114✔
85	return string.IsNullOrWhiteSpace(_loggingTask)	114!
86	? throw new Exception("There is no logging task specified for this load (the name is blank)")	114✔
87	: _loggingTask;	114✔
88	}
89
90	private void CreateDataLoadInfo()
91	{
92	if (string.IsNullOrWhiteSpace(Description))	102!
93	throw new Exception(	×
94	"The data load description (for the DataLoadInfo object) must not be empty, please provide a relevant description");	×
95
96	DataLoadInfo = _logManager.CreateDataLoadInfo(_loggingTask, nameof(DataLoadProcess), Description, "", false);	102✔
97
98	if (DataLoadInfo == null)	102!
99	throw new Exception("DataLoadInfo is null");	×
100
101	JobID = DataLoadInfo.ID;	102✔
102	}	102✔
103
104	public void LogProgress(string senderName, string message)
105	{
106	if (DataLoadInfo == null)	×
107	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
108
109	if (!DataLoadInfo.IsClosed)	×
110	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnProgress, senderName, message);	×
111	}	×
112
113	public void LogError(string message, Exception exception)
114	{
115	// we are bailing out before the load process has had a chance to create a DataLoadInfo object
116	if (DataLoadInfo == null)	×
117	CreateDataLoadInfo();	×
118
119	DataLoadInfo.LogFatalError(nameof(DataLoadProcess),	×
120	message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true));	×
121	DataLoadInfo.CloseAndMarkComplete();	×
122	}	×
123
124	public void StartLogging()
125	{
126	CreateDataLoadInfo();	102✔
127	}	102✔
128
129	public void CloseLogging()
130	{
131	DataLoadInfo?.CloseAndMarkComplete();	114!
132	}	114✔
133
134	public void LogInformation(string senderName, string message)
135	{
136	if (DataLoadInfo == null)	×
137	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
138
139	if (!DataLoadInfo.IsClosed)	×
140	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, senderName, message);	×
141	}	×
142
143	public void LogWarning(string senderName, string message)
144	{
145	if (DataLoadInfo == null)	×
146	throw new Exception("Logging hasn't been started for this job (call StartLogging first)");	×
147
148	if (!DataLoadInfo.IsClosed)	×
149	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message);	×
150	}	×
151
152	public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
153	{
154	bool allowReservedPrefixColumns = stage == LoadBubble.Raw ? LoadMetadata.AllowReservedPrefix : true;	112✔
155	foreach (TableInfo regularTableInfo in RegularTablesToLoad)	464✔
156	cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage, allowReservedPrefixColumns);	120✔
157
158	foreach (TableInfo lookupTableInfo in LookupTablesToLoad)	224!
159	cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage, allowReservedPrefixColumns);	×
160
161	PushForDisposal(cloner);	112✔
162	}	112✔
163
164	public void PushForDisposal(IDisposeAfterDataLoad disposeable)
165	{
166	_disposalStack.Push(disposeable);	828✔
167	}	828✔
168
169	public void OnNotify(object sender, NotifyEventArgs e)
170	{
171	if (DataLoadInfo != null)	2,292✔
172	switch (e.ProgressEventType)	2,262!
173	{
174	case ProgressEventType.Trace:
175	case ProgressEventType.Debug:
176	break;
177	case ProgressEventType.Information:
178	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation,	1,548!
179	sender.GetType().Name, e.Message + (e.Exception != null	1,548✔
180	? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"	1,548✔
181	: ""));	1,548✔
182	break;	1,548✔
183	case ProgressEventType.Warning:
184	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name,	98✔
185	e.Message + (e.Exception != null	98✔
186	? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"	98✔
187	: ""));	98✔
188	break;	98✔
189	case ProgressEventType.Error:
190	DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name,	2✔
191	e.Message);	2✔
192	DataLoadInfo.LogFatalError(sender.GetType().Name,	2!
193	e.Exception != null	2✔
194	? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)	2✔
195	: e.Message);	2✔
196	break;	2✔
197	default:
198	throw new ArgumentOutOfRangeException();	×
199	}
200
201	_listener.OnNotify(sender, e);	2,292✔
202	}	2,290✔
203
204	public void OnProgress(object sender, ProgressEventArgs e)
205	{
206	_listener.OnProgress(sender, e);	614✔
207	}	614✔
208
209	public string ArchiveFilepath => Path.Combine(LoadDirectory.ForArchiving.FullName, $"{DataLoadInfo.ID}.zip");	56✔
210
211	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
212	{
213	while (_disposalStack.Any())	932✔
214	{
215	var disposable = _disposalStack.Pop();	818✔
216	disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);	818✔
217	}
218	}	114✔
219
220	public ColumnInfo[] GetAllColumns()
221	{
222	return RegularTablesToLoad.SelectMany(t => t.ColumnInfos)	116✔
223	.Union(LookupTablesToLoad.SelectMany(t => t.ColumnInfos)).Distinct().ToArray();	56✔
224	}
225
226	public void CrashAtEnd(NotifyEventArgs because)
227	{
228	_crashAtEnd.Add(because);	×
229	}	×
230	}

HicServices / RDMP / 23054871216

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous