6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.79

/Rdmp.Core/DataExport/Data/ExtractionProgress.cs

﻿// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.Threading.Tasks;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataExport.Data;

/// <inheritdoc cref="IExtractionProgress"/>
public class ExtractionProgress : DatabaseEntity, IExtractionProgress
{
    #region Database Properties

    private int _selectedDataSets_ID;
    private DateTime? _progress;
    private int _extractionInformation_ID;
    private DateTime? _startDate;
    private DateTime? _endDate;
    private int _numberOfDaysPerBatch;
    private string _name;
    private RetryStrategy _retry;

    #endregion

    /// <inheritdoc/>
    public int SelectedDataSets_ID
    {
        get => _selectedDataSets_ID;
        set => SetField(ref _selectedDataSets_ID, value);
    }

    /// <inheritdoc/>
    public DateTime? ProgressDate
    {
        get => _progress;
        set => SetField(ref _progress, value);
    }

    /// <inheritdoc/>
    public DateTime? StartDate
    {
        get => _startDate;
        set => SetField(ref _startDate, value);
    }

    /// <inheritdoc/>
    public DateTime? EndDate
    {
        get => _endDate;
        set => SetField(ref _endDate, value);
    }

    /// <inheritdoc/>
    public int NumberOfDaysPerBatch
    {
        get => _numberOfDaysPerBatch;
        set => SetField(ref _numberOfDaysPerBatch, value);
    }

    /// <inheritdoc/>
    public int ExtractionInformation_ID
    {
        get => _extractionInformation_ID;
        set => SetField(ref _extractionInformation_ID, value);
    }

    /// <inheritdoc/>
    public string Name
    {
        get => _name;
        set => SetField(ref _name, value);
    }

    public RetryStrategy Retry
    {
        get => _retry;
        set => SetField(ref _retry, value);
    }

    #region Relationships

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ISelectedDataSets SelectedDataSets =>
        DataExportRepository.GetObjectByID<SelectedDataSets>(SelectedDataSets_ID);

    public static void ValidateSelectedColumn(ICheckNotifier notifier, ExtractionInformation ei)
    {
        if (ei == null)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("A date column on which to split batches must be selected",
                CheckResult.Fail));
            return;
        }

        var col = ei.ColumnInfo;

        if (col == null)
        {
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"ExtractionInformation '{ei}' is an orphan with no associated ColumnInfo", CheckResult.Fail));
            return;
        }

        try
        {
            if (col.GetQuerySyntaxHelper().TypeTranslater.GetCSharpTypeForSQLDBType(col.Data_type) != typeof(DateTime))
                notifier.OnCheckPerformed(new CheckEventArgs(ErrorCodes.ExtractionProgressColumnProbablyNotADate, ei,
                    col.Data_type));
        }
        catch (Exception ex)
        {
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"Could not determine datatype of ColumnInfo {col} ('{col?.Data_type}')", CheckResult.Fail, ex));
        }
    }

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ExtractionInformation ExtractionInformation =>
        DataExportRepository.CatalogueRepository.GetObjectByID<ExtractionInformation>(ExtractionInformation_ID);

    #endregion

    public ExtractionProgress()
    {
    }

    public ExtractionProgress(IDataExportRepository repository, ISelectedDataSets sds, DateTime? startDate,
        DateTime? endDate, int numberOfDaysPerBatch, string name, int extractionInformation_ID)
    {
        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "SelectedDataSets_ID", sds.ID },
            { "ExtractionInformation_ID", extractionInformation_ID },
            { "NumberOfDaysPerBatch", numberOfDaysPerBatch },
            { "StartDate", startDate },
            { "EndDate", endDate },
            { "Name", name },
            { "Retry", RetryStrategy.NoRetry }
        });

        if (ID == 0 || Repository != repository)
            throw new ArgumentException("Repository failed to properly hydrate this class");
    }

    public ExtractionProgress(IDataExportRepository repository, ISelectedDataSets sds)
    {
        var cata = sds.GetCatalogue();
        var coverageColId = cata?.TimeCoverage_ExtractionInformation_ID;

        if (sds.ExtractionProgressIfAny != null)
            throw new Exception($"There is already an ExtractionProgress associated with {sds}");

        if (!coverageColId.HasValue)
            throw new ArgumentException(
                $"Cannot create ExtractionProgress because Catalogue {cata} does not have a time coverage column");

        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "SelectedDataSets_ID", sds.ID },
            { "ExtractionInformation_ID", coverageColId },
            { "NumberOfDaysPerBatch", 365 },
            { "Name", $"ExtractionProgress{Guid.NewGuid()}" },
            { "Retry", RetryStrategy.NoRetry }
        });

        if (ID == 0 || Repository != repository)
            throw new ArgumentException("Repository failed to properly hydrate this class");
    }

    public ExtractionProgress(IDataExportRepository repository, DbDataReader r) : base(repository, r)
    {
        SelectedDataSets_ID = Convert.ToInt32(r["SelectedDataSets_ID"]);
        ProgressDate = ObjectToNullableDateTime(r["ProgressDate"]);
        StartDate = ObjectToNullableDateTime(r["StartDate"]);
        EndDate = ObjectToNullableDateTime(r["EndDate"]);
        ExtractionInformation_ID = Convert.ToInt32(r["ExtractionInformation_ID"]);
        NumberOfDaysPerBatch = Convert.ToInt32(r["NumberOfDaysPerBatch"]);
        Name = r["Name"].ToString();
        Retry = (RetryStrategy)Enum.Parse(typeof(RetryStrategy), r["Retry"].ToString());
    }

    public override string ToString() => Name;

    public bool MoreToFetch() => ProgressDate < EndDate;

    public bool ApplyRetryWaitStrategy(GracefulCancellationToken token, IDataLoadEventListener listener,
        int totalFailureCount, int consecutiveFailureCount)
    {
        return Retry switch
        {
            RetryStrategy.NoRetry => false,
            RetryStrategy.IterativeBackoff1Hour => IterativeBackoff1Hour(token, listener, totalFailureCount),
            _ => throw new ArgumentOutOfRangeException($"Unknown retry strategy {Retry}")
        };
    }

    private bool IterativeBackoff1Hour(GracefulCancellationToken token, IDataLoadEventListener listener,
        int totalFailureCount)
    {
        token.ThrowIfAbortRequested();

        int[] waitTimes = { 0, 1, 2, 3, 5, 8, 13, 21, 34 };

        if (totalFailureCount > waitTimes.Length) return false;

        // sleep for however many minutes we are up to
        var mins = waitTimes[totalFailureCount];
        listener.OnNotify(this,
            new NotifyEventArgs(ProgressEventType.Information, $"Waiting {mins} mins before retry"));

        // wait for the minutes but cancel if abort is hit
        Task.Delay((int)TimeSpan.FromMinutes(mins).TotalMilliseconds, token.AbortToken).Wait();

        token.ThrowIfAbortRequested();

        // then retry
        return true;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.Threading.Tasks;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.DataFlowPipeline;
13	using Rdmp.Core.MapsDirectlyToDatabaseTable;
14	using Rdmp.Core.Repositories;
15	using Rdmp.Core.ReusableLibraryCode.Checks;
16	using Rdmp.Core.ReusableLibraryCode.Progress;
17
18	namespace Rdmp.Core.DataExport.Data;
19
20	/// <inheritdoc cref="IExtractionProgress"/>
21	public class ExtractionProgress : DatabaseEntity, IExtractionProgress
22	{
23	#region Database Properties
24
25	private int _selectedDataSets_ID;
26	private DateTime? _progress;
27	private int _extractionInformation_ID;
28	private DateTime? _startDate;
29	private DateTime? _endDate;
30	private int _numberOfDaysPerBatch;
31	private string _name;
32	private RetryStrategy _retry;
33
34	#endregion
35
36	/// <inheritdoc/>
37	public int SelectedDataSets_ID
38	{
39	get => _selectedDataSets_ID;	104✔
40	set => SetField(ref _selectedDataSets_ID, value);	142✔
41	}
42
43	/// <inheritdoc/>
44	public DateTime? ProgressDate
45	{
46	get => _progress;	146✔
47	set => SetField(ref _progress, value);	146✔
48	}
49
50	/// <inheritdoc/>
51	public DateTime? StartDate
52	{
53	get => _startDate;	98✔
54	set => SetField(ref _startDate, value);	138✔
55	}
56
57	/// <inheritdoc/>
58	public DateTime? EndDate
59	{
60	get => _endDate;	126✔
61	set => SetField(ref _endDate, value);	138✔
62	}
63
64	/// <inheritdoc/>
65	public int NumberOfDaysPerBatch
66	{
67	get => _numberOfDaysPerBatch;	120✔
68	set => SetField(ref _numberOfDaysPerBatch, value);	144✔
69	}
70
71	/// <inheritdoc/>
72	public int ExtractionInformation_ID
73	{
74	get => _extractionInformation_ID;	112✔
75	set => SetField(ref _extractionInformation_ID, value);	142✔
76	}
77
78	/// <inheritdoc/>
79	public string Name
80	{
81	get => _name;	118✔
82	set => SetField(ref _name, value);	142✔
83	}
84
85	public RetryStrategy Retry
86	{
87	get => _retry;	94✔
88	set => SetField(ref _retry, value);	144✔
89	}
90
91	#region Relationships
92
93	/// <inheritdoc/>
94	[NoMappingToDatabase]
95	public ISelectedDataSets SelectedDataSets =>
96	DataExportRepository.GetObjectByID<SelectedDataSets>(SelectedDataSets_ID);	8✔
97
98	public static void ValidateSelectedColumn(ICheckNotifier notifier, ExtractionInformation ei)
99	{
100	if (ei == null)	2!
101	{
102	notifier.OnCheckPerformed(new CheckEventArgs("A date column on which to split batches must be selected",	×
103	CheckResult.Fail));	×
104	return;	×
105	}
106
107	var col = ei.ColumnInfo;	2✔
108
109	if (col == null)	2!
110	{
111	notifier.OnCheckPerformed(new CheckEventArgs(	×
112	$"ExtractionInformation '{ei}' is an orphan with no associated ColumnInfo", CheckResult.Fail));	×
113	return;	×
114	}
115
116	try
117	{
118	if (col.GetQuerySyntaxHelper().TypeTranslater.GetCSharpTypeForSQLDBType(col.Data_type) != typeof(DateTime))	2!
119	notifier.OnCheckPerformed(new CheckEventArgs(ErrorCodes.ExtractionProgressColumnProbablyNotADate, ei,	×
120	col.Data_type));	×
121	}	2✔
122	catch (Exception ex)	×
123	{
124	notifier.OnCheckPerformed(new CheckEventArgs(	×
125	$"Could not determine datatype of ColumnInfo {col} ('{col?.Data_type}')", CheckResult.Fail, ex));	×
126	}	×
127	}	2✔
128
129	/// <inheritdoc/>
130	[NoMappingToDatabase]
131	public ExtractionInformation ExtractionInformation =>
132	DataExportRepository.CatalogueRepository.GetObjectByID<ExtractionInformation>(ExtractionInformation_ID);	20✔
133
134	#endregion
135
136	public ExtractionProgress()	×
137	{
138	}	×
139
140	public ExtractionProgress(IDataExportRepository repository, ISelectedDataSets sds, DateTime? startDate,	12✔
141	DateTime? endDate, int numberOfDaysPerBatch, string name, int extractionInformation_ID)	12✔
142	{
143	repository.InsertAndHydrate(this, new Dictionary<string, object>	12✔
144	{	12✔
145	{ "SelectedDataSets_ID", sds.ID },	12✔
146	{ "ExtractionInformation_ID", extractionInformation_ID },	12✔
147	{ "NumberOfDaysPerBatch", numberOfDaysPerBatch },	12✔
148	{ "StartDate", startDate },	12✔
149	{ "EndDate", endDate },	12✔
150	{ "Name", name },	12✔
151	{ "Retry", RetryStrategy.NoRetry }	12✔
152	});	12✔
153
154	if (ID == 0 \|\| Repository != repository)	12!
155	throw new ArgumentException("Repository failed to properly hydrate this class");	×
156	}	12✔
157
158	public ExtractionProgress(IDataExportRepository repository, ISelectedDataSets sds)	26✔
159	{
160	var cata = sds.GetCatalogue();	26✔
161	var coverageColId = cata?.TimeCoverage_ExtractionInformation_ID;	26!
162
163	if (sds.ExtractionProgressIfAny != null)	26✔
164	throw new Exception($"There is already an ExtractionProgress associated with {sds}");	2✔
165
166	if (!coverageColId.HasValue)	24✔
167	throw new ArgumentException(	2✔
168	$"Cannot create ExtractionProgress because Catalogue {cata} does not have a time coverage column");	2✔
169
170	repository.InsertAndHydrate(this, new Dictionary<string, object>	22✔
171	{	22✔
172	{ "SelectedDataSets_ID", sds.ID },	22✔
173	{ "ExtractionInformation_ID", coverageColId },	22✔
174	{ "NumberOfDaysPerBatch", 365 },	22✔
175	{ "Name", $"ExtractionProgress{Guid.NewGuid()}" },	22✔
176	{ "Retry", RetryStrategy.NoRetry }	22✔
177	});	22✔
178
179	if (ID == 0 \|\| Repository != repository)	22!
180	throw new ArgumentException("Repository failed to properly hydrate this class");	×
181	}	22✔
182
183	public ExtractionProgress(IDataExportRepository repository, DbDataReader r) : base(repository, r)	106✔
184	{
185	SelectedDataSets_ID = Convert.ToInt32(r["SelectedDataSets_ID"]);	106✔
186	ProgressDate = ObjectToNullableDateTime(r["ProgressDate"]);	106✔
187	StartDate = ObjectToNullableDateTime(r["StartDate"]);	106✔
188	EndDate = ObjectToNullableDateTime(r["EndDate"]);	106✔
189	ExtractionInformation_ID = Convert.ToInt32(r["ExtractionInformation_ID"]);	106✔
190	NumberOfDaysPerBatch = Convert.ToInt32(r["NumberOfDaysPerBatch"]);	106✔
191	Name = r["Name"].ToString();	106✔
192	Retry = (RetryStrategy)Enum.Parse(typeof(RetryStrategy), r["Retry"].ToString());	106✔
193	}	106✔
194
195	public override string ToString() => Name;	22✔
196
197	public bool MoreToFetch() => ProgressDate < EndDate;	2!
198
199	public bool ApplyRetryWaitStrategy(GracefulCancellationToken token, IDataLoadEventListener listener,
200	int totalFailureCount, int consecutiveFailureCount)
201	{
202	return Retry switch	×
203	{	×
204	RetryStrategy.NoRetry => false,	×
205	RetryStrategy.IterativeBackoff1Hour => IterativeBackoff1Hour(token, listener, totalFailureCount),	×
206	_ => throw new ArgumentOutOfRangeException($"Unknown retry strategy {Retry}")	×
207	};	×
208	}
209
210	private bool IterativeBackoff1Hour(GracefulCancellationToken token, IDataLoadEventListener listener,
211	int totalFailureCount)
212	{
213	token.ThrowIfAbortRequested();	×
214
215	int[] waitTimes = { 0, 1, 2, 3, 5, 8, 13, 21, 34 };	×
216
217	if (totalFailureCount > waitTimes.Length) return false;	×
218
219	// sleep for however many minutes we are up to
220	var mins = waitTimes[totalFailureCount];	×
221	listener.OnNotify(this,	×
222	new NotifyEventArgs(ProgressEventType.Information, $"Waiting {mins} mins before retry"));	×
223
224	// wait for the minutes but cancel if abort is hit
225	Task.Delay((int)TimeSpan.FromMinutes(mins).TotalMilliseconds, token.AbortToken).Wait();	×
226
227	token.ThrowIfAbortRequested();	×
228
229	// then retry
230	return true;	×
231	}
232	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous