• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 23054871216

13 Mar 2026 02:13PM UTC coverage: 57.063% (-0.06%) from 57.12%
23054871216

Pull #2323

github

web-flow
Merge branch 'develop' into task/RDMP-365-better-load-metadata-logging
Pull Request #2323: Task/rdmp 365 better load metadata logging

11528 of 21729 branches covered (53.05%)

Branch coverage included in aggregate %.

28 of 41 new or added lines in 7 files covered. (68.29%)

33 existing lines in 4 files now uncovered.

32629 of 55654 relevant lines covered (58.63%)

18134.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.08
/Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs
1
// Copyright (c) The University of Dundee 2018-2019
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
using System;
8
using System.Collections.Generic;
9
using System.IO;
10
using System.Linq;
11
using Rdmp.Core.Curation;
12
using Rdmp.Core.Curation.Data;
13
using Rdmp.Core.Curation.Data.DataLoad;
14
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
15
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.Operations;
16
using Rdmp.Core.DataLoad.Engine.LoadProcess;
17
using Rdmp.Core.Logging;
18
using Rdmp.Core.Repositories;
19
using Rdmp.Core.ReusableLibraryCode;
20
using Rdmp.Core.ReusableLibraryCode.Progress;
21

22
namespace Rdmp.Core.DataLoad.Engine.Job;
23

24
/// <inheritdoc/>
25
public class DataLoadJob : IDataLoadJob
26
{
27
    public string Description { get; private set; }
318✔
28
    public IDataLoadInfo DataLoadInfo { get; private set; }
4,684✔
29
    public ILoadDirectory LoadDirectory { get; set; }
700✔
30
    private readonly IDataLoadEventListener _listener;
31

32
    public int JobID { get; set; }
158✔
33

34
    private readonly ILogManager _logManager;
35
    public ILoadMetadata LoadMetadata { get; private set; }
734✔
36

37
    public List<ITableInfo> RegularTablesToLoad { get; private set; }
520✔
38
    public List<ITableInfo> LookupTablesToLoad { get; private set; }
450✔
39
    public IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; private set; }
122✔
40

41
    private Stack<IDisposeAfterDataLoad> _disposalStack = new();
114✔
42

43
    public HICDatabaseConfiguration Configuration { get; set; }
184✔
44
    public object Payload { get; set; }
20✔
45

46
    public bool PersistentRaw { get; set; }
74✔
47

48

49
    private List<NotifyEventArgs> _crashAtEnd = new();
114✔
50

51
    public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();
56✔
52

53

54
    private string _loggingTask;
55

56
    public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description,
114✔
57
        ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener,
114✔
58
        HICDatabaseConfiguration configuration)
114✔
59
    {
60
        _logManager = logManager;
114✔
61
        RepositoryLocator = repositoryLocator;
114✔
62
        LoadMetadata = loadMetadata;
114✔
63
        LoadDirectory = directory;
114✔
64
        Configuration = configuration;
114✔
65
        _listener = listener;
114✔
66
        Description = description;
114✔
67

68
        var catalogues = LoadMetadata.GetAllCatalogues().ToList();
114✔
69

70
        if (LoadMetadata != null)
114✔
71
            _loggingTask = GetLoggingTask(catalogues);
114✔
72

73
        RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
232✔
74
        LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
232✔
75
    }
114✔
76

77
    private string GetLoggingTask(IEnumerable<ICatalogue> cataloguesToLoad)
78
    {
79
        var distinctLoggingTasks = cataloguesToLoad.SelectMany(catalogue => catalogue.LoggingDataTasks).DistinctBy(l => l.Name ).ToList();
350✔
80
        if (distinctLoggingTasks.Count > 1)
114!
81
            throw new Exception(
×
NEW
82
                $"The catalogues to be loaded do not share the same logging task: {string.Join(", ", distinctLoggingTasks.Select(l => l.Name))}");
×
83

84
        _loggingTask = distinctLoggingTasks.First().Name;
114✔
85
        return string.IsNullOrWhiteSpace(_loggingTask)
114!
86
            ? throw new Exception("There is no logging task specified for this load (the name is blank)")
114✔
87
            : _loggingTask;
114✔
88
    }
89

90
    private void CreateDataLoadInfo()
91
    {
92
        if (string.IsNullOrWhiteSpace(Description))
102!
93
            throw new Exception(
×
94
                "The data load description (for the DataLoadInfo object) must not be empty, please provide a relevant description");
×
95

96
        DataLoadInfo = _logManager.CreateDataLoadInfo(_loggingTask, nameof(DataLoadProcess), Description, "", false);
102✔
97

98
        if (DataLoadInfo == null)
102!
99
            throw new Exception("DataLoadInfo is null");
×
100

101
        JobID = DataLoadInfo.ID;
102✔
102
    }
102✔
103

104
    public void LogProgress(string senderName, string message)
105
    {
106
        if (DataLoadInfo == null)
×
107
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");
×
108

109
        if (!DataLoadInfo.IsClosed)
×
110
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnProgress, senderName, message);
×
111
    }
×
112

113
    public void LogError(string message, Exception exception)
114
    {
115
        // we are bailing out before the load process has had a chance to create a DataLoadInfo object
116
        if (DataLoadInfo == null)
×
117
            CreateDataLoadInfo();
×
118

119
        DataLoadInfo.LogFatalError(nameof(DataLoadProcess),
×
120
            message + Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(exception, true));
×
121
        DataLoadInfo.CloseAndMarkComplete();
×
122
    }
×
123

124
    public void StartLogging()
125
    {
126
        CreateDataLoadInfo();
102✔
127
    }
102✔
128

129
    public void CloseLogging()
130
    {
131
        DataLoadInfo?.CloseAndMarkComplete();
114!
132
    }
114✔
133

134
    public void LogInformation(string senderName, string message)
135
    {
136
        if (DataLoadInfo == null)
×
137
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");
×
138

139
        if (!DataLoadInfo.IsClosed)
×
140
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, senderName, message);
×
141
    }
×
142

143
    public void LogWarning(string senderName, string message)
144
    {
145
        if (DataLoadInfo == null)
×
146
            throw new Exception("Logging hasn't been started for this job (call StartLogging first)");
×
147

148
        if (!DataLoadInfo.IsClosed)
×
149
            DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, senderName, message);
×
150
    }
×
151

152
    public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
153
    {
154
        bool allowReservedPrefixColumns = stage == LoadBubble.Raw ? LoadMetadata.AllowReservedPrefix : true;
112✔
155
        foreach (TableInfo regularTableInfo in RegularTablesToLoad)
464✔
156
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage, allowReservedPrefixColumns);
120✔
157

158
        foreach (TableInfo lookupTableInfo in LookupTablesToLoad)
224!
159
            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage, allowReservedPrefixColumns);
×
160

161
        PushForDisposal(cloner);
112✔
162
    }
112✔
163

164
    public void PushForDisposal(IDisposeAfterDataLoad disposeable)
165
    {
166
        _disposalStack.Push(disposeable);
828✔
167
    }
828✔
168

169
    public void OnNotify(object sender, NotifyEventArgs e)
170
    {
171
        if (DataLoadInfo != null)
2,292✔
172
            switch (e.ProgressEventType)
2,262!
173
            {
174
                case ProgressEventType.Trace:
175
                case ProgressEventType.Debug:
176
                    break;
177
                case ProgressEventType.Information:
178
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation,
1,548!
179
                        sender.GetType().Name, e.Message + (e.Exception != null
1,548✔
180
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
1,548✔
181
                            : ""));
1,548✔
182
                    break;
1,548✔
183
                case ProgressEventType.Warning:
184
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.GetType().Name,
98✔
185
                        e.Message + (e.Exception != null
98✔
186
                            ? $"Exception={ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)}"
98✔
187
                            : ""));
98✔
188
                    break;
98✔
189
                case ProgressEventType.Error:
190
                    DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnTaskFailed, sender.GetType().Name,
2✔
191
                        e.Message);
2✔
192
                    DataLoadInfo.LogFatalError(sender.GetType().Name,
2!
193
                        e.Exception != null
2✔
194
                            ? ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)
2✔
195
                            : e.Message);
2✔
196
                    break;
2✔
197
                default:
198
                    throw new ArgumentOutOfRangeException();
×
199
            }
200

201
        _listener.OnNotify(sender, e);
2,292✔
202
    }
2,290✔
203

204
    public void OnProgress(object sender, ProgressEventArgs e)
205
    {
206
        _listener.OnProgress(sender, e);
614✔
207
    }
614✔
208

209
    public string ArchiveFilepath => Path.Combine(LoadDirectory.ForArchiving.FullName, $"{DataLoadInfo.ID}.zip");
56✔
210

211
    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
212
    {
213
        while (_disposalStack.Any())
932✔
214
        {
215
            var disposable = _disposalStack.Pop();
818✔
216
            disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);
818✔
217
        }
218
    }
114✔
219

220
    public ColumnInfo[] GetAllColumns()
221
    {
222
        return RegularTablesToLoad.SelectMany(t => t.ColumnInfos)
116✔
223
            .Union(LookupTablesToLoad.SelectMany(t => t.ColumnInfos)).Distinct().ToArray();
56✔
224
    }
225

226
    public void CrashAtEnd(NotifyEventArgs because)
227
    {
228
        _crashAtEnd.Add(because);
×
229
    }
×
230
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc