6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.1

/Rdmp.Core/DataLoad/Engine/Job/Scheduling/SingleScheduleCacheDateTrackingStrategy.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Linq;
using Rdmp.Core.Caching.Layouts;
using Rdmp.Core.Caching.Pipeline;
using Rdmp.Core.Caching.Pipeline.Destinations;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.Cache;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.Job.Scheduling;

/// <summary>
/// This returns dates by examining cache filenames whilst honouring the Load and Cache Progress information
/// </summary>
public class SingleScheduleCacheDateTrackingStrategy : IJobDateGenerationStrategy
{
    private readonly Queue<DateTime> _availableDates = new();
    private readonly DateTime _lastDateForLoading;

    public SingleScheduleCacheDateTrackingStrategy(ICacheLayout cacheLayout, ILoadProgress loadProgress,
        IDataLoadEventListener listener)
    {
        // no null check needed as the contract ensures that both DataLoadProgress and OriginDate can't simultaneously be null
        var lastAssignedLoadDate = loadProgress.DataLoadProgress ?? loadProgress.OriginDate.Value;

        // This is all the dates in the cache, but we want to start from _lastAssignedLoadDate
        // todo: must be efficient, revisit
        _availableDates = cacheLayout.GetSortedDateQueue(listener);

        while (_availableDates.Any() && _availableDates.Peek() <= lastAssignedLoadDate)
            _availableDates.Dequeue();


        _lastDateForLoading = CalculateLastLoadDate(loadProgress);
    }

    /// <summary>
    /// 
    /// </summary>
    /// <param name="loadProgress"></param>
    /// <returns></returns>
    /// <exception cref="InvalidOperationException">Caching has not been configured correctly or the caching process has not begun</exception>
    public static DateTime CalculateLastLoadDate(ILoadProgress loadProgress)
    {
        // Compute the last cache date from the CacheFillProgress date
        // CacheFillProgress is the date up to which caching has been performed, and is therefore the date from which caching will next begin.
        var cacheProgress = loadProgress.CacheProgress ?? throw new InvalidOperationException(
            $"Could not retrieve the CacheProgress from LoadProgress {loadProgress.ID} (ensure caching is configured on this load before using this strategy)");
        if (cacheProgress.CacheFillProgress == null)
            throw new InvalidOperationException(
                $"Caching has not begun for this CacheProgress ({cacheProgress.ID}), so there is nothing to load and this strategy should not be used.");

        // We don't want to load partially filled cache files, so use the CacheFileGranularity to calculate the latest file we can safely load
        // CacheFileGranularity is a caching pipeline component argument, so need to get the runtime pipeline
        var cachedFileRetriever = GetCacheDestinationPipelineComponent(cacheProgress);
        var layout = cachedFileRetriever.CreateCacheLayout();

        return CalculateLastLoadDate(layout.CacheFileGranularity, cacheProgress.CacheFillProgress.Value);
    }

    /// <summary>
    /// Retrieves the destination component from the caching pipeline associated with the ICacheProgress object. The destination component is required to be an ICacheFileSystemDestination.
    /// </summary>
    /// <param name="cacheProgress"></param>
    /// <returns></returns>
    /// <exception cref="InvalidOperationException">Caching pipeline is not configured properly/doesn't exist</exception>
    private static ICacheFileSystemDestination GetCacheDestinationPipelineComponent(ICacheProgress cacheProgress)
    {
        if (cacheProgress.Pipeline_ID == null)
            throw new InvalidOperationException(
                "This CacheProgress does not have a caching pipeline, please configure one.");

        var factory = new CachingPipelineUseCase(cacheProgress);

        try
        {
            return factory.CreateDestinationOnly(ThrowImmediatelyDataLoadEventListener.Quiet);
        }
        catch (Exception e)
        {
            throw new Exception(
                $"We identified that your cache uses pipeline {cacheProgress.Pipeline} but we could not instantiate the Pipeline's Destination instance, make sure the pipeline is intact in PipelineDiagramUI.  See inner exception for details",
                e);
        }
    }

    public static DateTime CalculateLastLoadDate(CacheFileGranularity cacheFileGranularity, DateTime nextDateToBeCached)
    {
        switch (cacheFileGranularity)
        {
            case CacheFileGranularity.Hour:
                var priorDate = nextDateToBeCached.AddHours(-1);
                return priorDate.Date.AddHours(priorDate.Hour); // to get rid of any minutes
            case CacheFileGranularity.Day:
                // Get the beginning of the day prior to nextDateToBeCached
                return nextDateToBeCached.AddDays(-1).Date;
            default:
                throw new ArgumentOutOfRangeException(nameof(cacheFileGranularity), cacheFileGranularity,
                    "CacheFileGranularity must either be Hour or Day.");
        }
    }

    // todo: figure out how allowLoadingFutureDates fits into this
    public List<DateTime> GetDates(int batchSize, bool allowLoadingFutureDates)
    {
        var dates = new List<DateTime>();

        for (var i = 0; i < batchSize; i++)
        {
            if (!_availableDates.Any())
                break;

            var nextDate = _availableDates.Dequeue();
            if (nextDate > _lastDateForLoading)
                break;

            dates.Add(nextDate);
        }

        return dates;
    }

    public int GetTotalNumberOfJobs(int batchSize, bool allowLoadingFutureDates)
    {
        var totalNumberOfValidDates = _availableDates.Count(time => time <= _lastDateForLoading);
        var totalNumberOfJobs = totalNumberOfValidDates / batchSize;
        if (totalNumberOfValidDates % batchSize > 0) ++totalNumberOfJobs;
        return totalNumberOfJobs;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Linq;
10	using Rdmp.Core.Caching.Layouts;
11	using Rdmp.Core.Caching.Pipeline;
12	using Rdmp.Core.Caching.Pipeline.Destinations;
13	using Rdmp.Core.Curation.Data;
14	using Rdmp.Core.Curation.Data.Cache;
15	using Rdmp.Core.ReusableLibraryCode.Progress;
16
17	namespace Rdmp.Core.DataLoad.Engine.Job.Scheduling;
18
19	/// <summary>
20	/// This returns dates by examining cache filenames whilst honouring the Load and Cache Progress information
21	/// </summary>
22	public class SingleScheduleCacheDateTrackingStrategy : IJobDateGenerationStrategy
23	{
24	private readonly Queue<DateTime> _availableDates = new();	8✔
25	private readonly DateTime _lastDateForLoading;
26
27	public SingleScheduleCacheDateTrackingStrategy(ICacheLayout cacheLayout, ILoadProgress loadProgress,	8✔
28	IDataLoadEventListener listener)	8✔
29	{
30	// no null check needed as the contract ensures that both DataLoadProgress and OriginDate can't simultaneously be null
31	var lastAssignedLoadDate = loadProgress.DataLoadProgress ?? loadProgress.OriginDate.Value;	8!
32
33	// This is all the dates in the cache, but we want to start from _lastAssignedLoadDate
34	// todo: must be efficient, revisit
35	_availableDates = cacheLayout.GetSortedDateQueue(listener);	8✔
36
37	while (_availableDates.Any() && _availableDates.Peek() <= lastAssignedLoadDate)	8!
38	_availableDates.Dequeue();	×
39
40
41	_lastDateForLoading = CalculateLastLoadDate(loadProgress);	8✔
42	}	6✔
43
44	/// <summary>
45	///
46	/// </summary>
47	/// <param name="loadProgress"></param>
48	/// <returns></returns>
49	/// <exception cref="InvalidOperationException">Caching has not been configured correctly or the caching process has not begun</exception>
50	public static DateTime CalculateLastLoadDate(ILoadProgress loadProgress)
51	{
52	// Compute the last cache date from the CacheFillProgress date
53	// CacheFillProgress is the date up to which caching has been performed, and is therefore the date from which caching will next begin.
54	var cacheProgress = loadProgress.CacheProgress ?? throw new InvalidOperationException(	8!
55	$"Could not retrieve the CacheProgress from LoadProgress {loadProgress.ID} (ensure caching is configured on this load before using this strategy)");	8✔
56	if (cacheProgress.CacheFillProgress == null)	8✔
57	throw new InvalidOperationException(	2✔
58	$"Caching has not begun for this CacheProgress ({cacheProgress.ID}), so there is nothing to load and this strategy should not be used.");	2✔
59
60	// We don't want to load partially filled cache files, so use the CacheFileGranularity to calculate the latest file we can safely load
61	// CacheFileGranularity is a caching pipeline component argument, so need to get the runtime pipeline
62	var cachedFileRetriever = GetCacheDestinationPipelineComponent(cacheProgress);	6✔
63	var layout = cachedFileRetriever.CreateCacheLayout();	6✔
64
65	return CalculateLastLoadDate(layout.CacheFileGranularity, cacheProgress.CacheFillProgress.Value);	6✔
66	}
67
68	/// <summary>
69	/// Retrieves the destination component from the caching pipeline associated with the ICacheProgress object. The destination component is required to be an ICacheFileSystemDestination.
70	/// </summary>
71	/// <param name="cacheProgress"></param>
72	/// <returns></returns>
73	/// <exception cref="InvalidOperationException">Caching pipeline is not configured properly/doesn't exist</exception>
74	private static ICacheFileSystemDestination GetCacheDestinationPipelineComponent(ICacheProgress cacheProgress)
75	{
76	if (cacheProgress.Pipeline_ID == null)	6!
77	throw new InvalidOperationException(	×
78	"This CacheProgress does not have a caching pipeline, please configure one.");	×
79
80	var factory = new CachingPipelineUseCase(cacheProgress);	6✔
81
82	try
83	{
84	return factory.CreateDestinationOnly(ThrowImmediatelyDataLoadEventListener.Quiet);	6✔
85	}
86	catch (Exception e)	×
87	{
88	throw new Exception(	×
89	$"We identified that your cache uses pipeline {cacheProgress.Pipeline} but we could not instantiate the Pipeline's Destination instance, make sure the pipeline is intact in PipelineDiagramUI. See inner exception for details",	×
90	e);	×
91	}
92	}	6✔
93
94	public static DateTime CalculateLastLoadDate(CacheFileGranularity cacheFileGranularity, DateTime nextDateToBeCached)
95	{
96	switch (cacheFileGranularity)
97	{
98	case CacheFileGranularity.Hour:
99	var priorDate = nextDateToBeCached.AddHours(-1);	2!
100	return priorDate.Date.AddHours(priorDate.Hour); // to get rid of any minutes	2✔
101	case CacheFileGranularity.Day:
102	// Get the beginning of the day prior to nextDateToBeCached
103	return nextDateToBeCached.AddDays(-1).Date;	10✔
104	default:
105	throw new ArgumentOutOfRangeException(nameof(cacheFileGranularity), cacheFileGranularity,	×
106	"CacheFileGranularity must either be Hour or Day.");	×
107	}
108	}
109
110	// todo: figure out how allowLoadingFutureDates fits into this
111	public List<DateTime> GetDates(int batchSize, bool allowLoadingFutureDates)
112	{
113	var dates = new List<DateTime>();	6✔
114
115	for (var i = 0; i < batchSize; i++)	28✔
116	{
117	if (!_availableDates.Any())	14✔
118	break;
119
120	var nextDate = _availableDates.Dequeue();	8✔
121	if (nextDate > _lastDateForLoading)	8✔
122	break;
123
124	dates.Add(nextDate);	8✔
125	}
126
127	return dates;	6✔
128	}
129
130	public int GetTotalNumberOfJobs(int batchSize, bool allowLoadingFutureDates)
131	{
132	var totalNumberOfValidDates = _availableDates.Count(time => time <= _lastDateForLoading);	4✔
133	var totalNumberOfJobs = totalNumberOfValidDates / batchSize;	2✔
134	if (totalNumberOfValidDates % batchSize > 0) ++totalNumberOfJobs;	4✔
135	return totalNumberOfJobs;	2✔
136	}
137	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous