6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Engine/LoadProcess/Scheduling/IterativeScheduledDataLoadProcess.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System.Linq;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.Job.Scheduling;
using Rdmp.Core.DataLoad.Engine.LoadExecution;
using Rdmp.Core.DataLoad.Engine.LoadProcess.Scheduling.Strategy;
using Rdmp.Core.Logging;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.LoadProcess.Scheduling;

/// <summary>
/// DataLoadProcess for LoadMetadata's which have one or more LoadProgresses (See ScheduledDataLoadProcess).  This version of ScheduledDataLoadProcess will
/// run loads one after another until all LoadProgresses are exhausted (they have loaded all data up to today / when available data stops).  For example
/// if you have a load 'Load biochemistry records' with a LoadProgress which 'loads 5 days at a time' and is currently at LoadProgress.DataLoadProgress of
/// 2001-01-01 it will keep running data loads iteratively until the IDataLoadExecution returns OperationNotRequired (i.e. the load is up-to-date).
/// </summary>
public class IterativeScheduledDataLoadProcess : ScheduledDataLoadProcess
{
    // todo: refactor to cut down on ctor params
    public IterativeScheduledDataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator,
        ILoadMetadata loadMetadata, ICheckable preExecutionChecker, IDataLoadExecution loadExecution,
        JobDateGenerationStrategyFactory jobDateGenerationStrategyFactory,
        ILoadProgressSelectionStrategy loadProgressSelectionStrategy, int? overrideNumberOfDaysToLoad,
        ILogManager logManager, IDataLoadEventListener dataLoadEventsreceiver, HICDatabaseConfiguration configuration)
        : base(repositoryLocator, loadMetadata, preExecutionChecker, loadExecution, jobDateGenerationStrategyFactory,
            loadProgressSelectionStrategy, overrideNumberOfDaysToLoad, logManager, dataLoadEventsreceiver,
            configuration)
    {
    }

    public override ExitCodeType Run(GracefulCancellationToken loadCancellationToken, object payload = null)
    {
        // grab all the load schedules we can and lock them
        var loadProgresses = LoadProgressSelectionStrategy.GetAllLoadProgresses();
        if (!loadProgresses.Any())
            return ExitCodeType.OperationNotRequired;

        // create job factory
        var progresses = loadProgresses.ToDictionary(loadProgress => loadProgress,
            loadProgress => JobDateGenerationStrategyFactory.Create(loadProgress, DataLoadEventListener));
        var jobProvider =
            new MultipleScheduleJobFactory(progresses, OverrideNumberOfDaysToLoad, LoadMetadata, LogManager);

        // check if the factory will produce any jobs, if not we can stop here
        if (!jobProvider.HasJobs())
            return ExitCodeType.OperationNotRequired;

        // Run the data load process
        JobProvider = jobProvider;

        //Do a data load
        ExitCodeType result;
        while ((result = base.Run(loadCancellationToken, payload)) ==
               ExitCodeType.Success) //stop if it said not required
        {
            //or if between executions the token is set
            if (loadCancellationToken.IsAbortRequested)
                return ExitCodeType.Abort;

            if (loadCancellationToken.IsCancellationRequested)
                return ExitCodeType.Success;
        }

        //should be Operation Not Required or Error since the token inside handles stopping
        return result;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System.Linq;
8	using Rdmp.Core.Curation.Data.DataLoad;
9	using Rdmp.Core.DataFlowPipeline;
10	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
11	using Rdmp.Core.DataLoad.Engine.Job.Scheduling;
12	using Rdmp.Core.DataLoad.Engine.LoadExecution;
13	using Rdmp.Core.DataLoad.Engine.LoadProcess.Scheduling.Strategy;
14	using Rdmp.Core.Logging;
15	using Rdmp.Core.Repositories;
16	using Rdmp.Core.ReusableLibraryCode.Checks;
17	using Rdmp.Core.ReusableLibraryCode.Progress;
18
19	namespace Rdmp.Core.DataLoad.Engine.LoadProcess.Scheduling;
20
21	/// <summary>
22	/// DataLoadProcess for LoadMetadata's which have one or more LoadProgresses (See ScheduledDataLoadProcess). This version of ScheduledDataLoadProcess will
23	/// run loads one after another until all LoadProgresses are exhausted (they have loaded all data up to today / when available data stops). For example
24	/// if you have a load 'Load biochemistry records' with a LoadProgress which 'loads 5 days at a time' and is currently at LoadProgress.DataLoadProgress of
25	/// 2001-01-01 it will keep running data loads iteratively until the IDataLoadExecution returns OperationNotRequired (i.e. the load is up-to-date).
26	/// </summary>
27	public class IterativeScheduledDataLoadProcess : ScheduledDataLoadProcess
28	{
29	// todo: refactor to cut down on ctor params
30	public IterativeScheduledDataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator,
31	ILoadMetadata loadMetadata, ICheckable preExecutionChecker, IDataLoadExecution loadExecution,
32	JobDateGenerationStrategyFactory jobDateGenerationStrategyFactory,
33	ILoadProgressSelectionStrategy loadProgressSelectionStrategy, int? overrideNumberOfDaysToLoad,
34	ILogManager logManager, IDataLoadEventListener dataLoadEventsreceiver, HICDatabaseConfiguration configuration)
35	: base(repositoryLocator, loadMetadata, preExecutionChecker, loadExecution, jobDateGenerationStrategyFactory,	×
36	loadProgressSelectionStrategy, overrideNumberOfDaysToLoad, logManager, dataLoadEventsreceiver,	×
37	configuration)	×
38	{
39	}	×
40
41	public override ExitCodeType Run(GracefulCancellationToken loadCancellationToken, object payload = null)
42	{
43	// grab all the load schedules we can and lock them
44	var loadProgresses = LoadProgressSelectionStrategy.GetAllLoadProgresses();	×
45	if (!loadProgresses.Any())	×
46	return ExitCodeType.OperationNotRequired;	×
47
48	// create job factory
49	var progresses = loadProgresses.ToDictionary(loadProgress => loadProgress,	×
50	loadProgress => JobDateGenerationStrategyFactory.Create(loadProgress, DataLoadEventListener));	×
51	var jobProvider =	×
52	new MultipleScheduleJobFactory(progresses, OverrideNumberOfDaysToLoad, LoadMetadata, LogManager);	×
53
54	// check if the factory will produce any jobs, if not we can stop here
55	if (!jobProvider.HasJobs())	×
56	return ExitCodeType.OperationNotRequired;	×
57
58	// Run the data load process
59	JobProvider = jobProvider;	×
60
61	//Do a data load
62	ExitCodeType result;
63	while ((result = base.Run(loadCancellationToken, payload)) ==	×
64	ExitCodeType.Success) //stop if it said not required	×
65	{
66	//or if between executions the token is set
67	if (loadCancellationToken.IsAbortRequested)	×
68	return ExitCodeType.Abort;	×
69
70	if (loadCancellationToken.IsCancellationRequested)	×
71	return ExitCodeType.Success;	×
72	}
73
74	//should be Operation Not Required or Error since the token inside handles stopping
75	return result;	×
76	}
77	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous