6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.33

/Rdmp.Core/DataLoad/Modules/LoadProgressUpdating/DataLoadProgressUpdateInfo.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Text;
using FAnsi.Discovery;
using Microsoft.Data.SqlClient;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.Job.Scheduling;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.DataAccess;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.LoadProgressUpdating;

/// <summary>
/// Represents a user made descision about how to upload a LoadProgress after a succesful data load.  LoadProgress has a field DataLoadProgress which stores
/// the last date that was loaded.  However you can overreach during a load e.g. run a load for 30 days but find only 5 days worth of data streamed through
/// the load, in such cases you might want to update the DataLoadProgress to the 5 day mark on the assumption that there is a delay in data provision and it
/// will arrive later.  There are multiple ways to determine what dates were actually loaded during a data load (See DataLoadProgressUpdateStrategy).
/// 
/// <para>You can declare a [DemandsInitialization] decorated property of this Type in a data load component (IAttacher) etc in order to illicit a decision about
/// what to update the DataLoadProgress with from the user at design time.</para>
/// </summary>
public class DataLoadProgressUpdateInfo : ICustomUIDrivenClass, ICheckable
{
    public DataLoadProgressUpdateStrategy Strategy { get; set; }
    public string ExecuteScalarSQL { get; set; }
    public int Timeout { get; set; }

    #region Serialization

    public void RestoreStateFrom(string value)
    {
        if (string.IsNullOrWhiteSpace(value))
            return;

        var lines = value.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);

        if (lines.Length > 0)
        {
            var fields = lines[0].Split(';');
            if (fields.Length > 0)
                if (Enum.TryParse(fields[0], out DataLoadProgressUpdateStrategy strat))
                    Strategy = strat;

            if (fields.Length > 1)
                Timeout = int.Parse(fields[1]);
        }

        ExecuteScalarSQL = "";

        for (var i = 1; i < lines.Length; i++)
            ExecuteScalarSQL += lines[i] + Environment.NewLine;
    }

    public string SaveStateToString()
    {
        var sb = new StringBuilder();
        sb.AppendLine($"{Strategy};{Timeout}");
        sb.AppendLine(ExecuteScalarSQL ?? "");

        return sb.ToString();
    }

    #endregion


    /// <summary>
    /// Only call this method when you hav finished populating RAW (since the strategy ExecuteScalarSQLInRAW requires to calculate date from populated RAW database right now and it is known that RAW won't even exist post load time)
    /// </summary>
    /// <param name="job"></param>
    /// <param name="rawDatabase"></param>
    public IUpdateLoadProgress AddAppropriateDisposeStep(ScheduledDataLoadJob job, DiscoveredDatabase rawDatabase)
    {
        IUpdateLoadProgress added;
        Check(ThrowImmediatelyCheckNotifier.Quiet);

        switch (Strategy)
        {
            case DataLoadProgressUpdateStrategy.UseMaxRequestedDay:
                added = new UpdateProgressIfLoadsuccessful(job);
                break;
            case DataLoadProgressUpdateStrategy.ExecuteScalarSQLInLIVE:

                added = new UpdateProgressToResultOfDelegate(job, () => GetMaxDate(GetLiveServer(job), job));
                break;
            case DataLoadProgressUpdateStrategy.ExecuteScalarSQLInRAW:
                try
                {
                    var dt = GetMaxDate(rawDatabase.Server, job);
                    added = new UpdateProgressToSpecificValueIfLoadsuccessful(job, dt);
                }
                catch (SqlException e)
                {
                    throw new DataLoadProgressUpdateException(
                        $"Failed to execute the following SQL in the RAW database:{ExecuteScalarSQL}", e);
                }

                break;
            case DataLoadProgressUpdateStrategy.DoNothing:
                //Do not add any post load update i.e. do nothing
                return null;
            default:
                throw new ArgumentOutOfRangeException();
        }

        job.PushForDisposal(added);
        return added;
    }

    private static DiscoveredServer GetLiveServer(ScheduledDataLoadJob job) =>
        DataAccessPortal.ExpectDistinctServer(job.RegularTablesToLoad.ToArray(), DataAccessContext.DataLoad, false);

    private DateTime GetMaxDate(DiscoveredServer server, IDataLoadEventListener listener)
    {
        using var con = server.GetConnection();
        con.Open();

        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"About to execute SQL to determine the maximum date for data loaded:{ExecuteScalarSQL}"));

        using var cmd = server.GetCommand(ExecuteScalarSQL, con);
        var scalarValue = cmd.ExecuteScalar();

        if (scalarValue == null || scalarValue == DBNull.Value)
            throw new DataLoadProgressUpdateException(
                "ExecuteScalarSQL specified for determining the maximum date of data loaded returned null when executed");

        try
        {
            return Convert.ToDateTime(scalarValue);
        }
        catch (Exception e)
        {
            throw new DataLoadProgressUpdateException(
                $"ExecuteScalarSQL specified for determining the maximum date of data loaded returned a value that was not a Date:{scalarValue}",
                e);
        }
    }

    public void Check(ICheckNotifier notifier)
    {
        if (Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInLIVE ||
            Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInRAW)
            if (string.IsNullOrWhiteSpace(ExecuteScalarSQL))
                notifier.OnCheckPerformed(
                    new CheckEventArgs(
                        $"Strategy is {Strategy} but there is no ExecuteScalarSQL, ExecuteScalarSQL should be a SELECT statement that returns a specific value that reflects the maximum date in the load e.g. Select MAX(MyDate) FROM MyTable",
                        CheckResult.Fail));

        // TODO: These checks are VERY Sql Server specific!

        if (Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInRAW)
            if (ExecuteScalarSQL.Contains("..") || ExecuteScalarSQL.Contains(".[dbo].") ||
                ExecuteScalarSQL.Contains(".dbo."))
                notifier.OnCheckPerformed(
                    new CheckEventArgs(
                        $"Strategy is {Strategy} but the SQL looks like it references explicit tables, In general RAW queries should use unqualified table names i.e. 'Select MAX(dt) FROM MyTable' NOT 'Select MAX(dt) FROM [MyLIVEDatabase]..[MyTable]'",
                        CheckResult.Warning));

        if (Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInLIVE)
            if (!(ExecuteScalarSQL.Contains("..") || ExecuteScalarSQL.Contains(".[dbo].") ||
                  ExecuteScalarSQL.Contains(".dbo.")))
                notifier.OnCheckPerformed(
                    new CheckEventArgs(
                        $"Strategy is {Strategy} but the SQL does not contain '..' or '.[dbo].' or '.dbo.', LIVE update queries should use fully table names i.e. 'Select MAX(dt) FROM [MyLIVEDatabase]..[MyTable]' NOT 'Select MAX(dt) FROM MyTable'",
                        CheckResult.Warning));
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Text;
9	using FAnsi.Discovery;
10	using Microsoft.Data.SqlClient;
11	using Rdmp.Core.Curation.Data.DataLoad;
12	using Rdmp.Core.DataLoad.Engine.Job.Scheduling;
13	using Rdmp.Core.ReusableLibraryCode.Checks;
14	using Rdmp.Core.ReusableLibraryCode.DataAccess;
15	using Rdmp.Core.ReusableLibraryCode.Progress;
16
17	namespace Rdmp.Core.DataLoad.Modules.LoadProgressUpdating;
18
19	/// <summary>
20	/// Represents a user made descision about how to upload a LoadProgress after a succesful data load. LoadProgress has a field DataLoadProgress which stores
21	/// the last date that was loaded. However you can overreach during a load e.g. run a load for 30 days but find only 5 days worth of data streamed through
22	/// the load, in such cases you might want to update the DataLoadProgress to the 5 day mark on the assumption that there is a delay in data provision and it
23	/// will arrive later. There are multiple ways to determine what dates were actually loaded during a data load (See DataLoadProgressUpdateStrategy).
24	///
25	/// <para>You can declare a [DemandsInitialization] decorated property of this Type in a data load component (IAttacher) etc in order to illicit a decision about
26	/// what to update the DataLoadProgress with from the user at design time.</para>
27	/// </summary>
28	public class DataLoadProgressUpdateInfo : ICustomUIDrivenClass, ICheckable
29	{
30	public DataLoadProgressUpdateStrategy Strategy { get; set; }	106✔
31	public string ExecuteScalarSQL { get; set; }	60✔
32	public int Timeout { get; set; }	×
33
34	#region Serialization
35
36	public void RestoreStateFrom(string value)
37	{
38	if (string.IsNullOrWhiteSpace(value))	×
39	return;	×
40
41	var lines = value.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);	×
42
43	if (lines.Length > 0)	×
44	{
45	var fields = lines[0].Split(';');	×
46	if (fields.Length > 0)	×
47	if (Enum.TryParse(fields[0], out DataLoadProgressUpdateStrategy strat))	×
48	Strategy = strat;	×
49
50	if (fields.Length > 1)	×
51	Timeout = int.Parse(fields[1]);	×
52	}
53
54	ExecuteScalarSQL = "";	×
55
56	for (var i = 1; i < lines.Length; i++)	×
57	ExecuteScalarSQL += lines[i] + Environment.NewLine;	×
58	}	×
59
60	public string SaveStateToString()
61	{
62	var sb = new StringBuilder();	×
63	sb.AppendLine($"{Strategy};{Timeout}");	×
64	sb.AppendLine(ExecuteScalarSQL ?? "");	×
65
66	return sb.ToString();	×
67	}
68
69	#endregion
70
71
72	/// <summary>
73	/// Only call this method when you hav finished populating RAW (since the strategy ExecuteScalarSQLInRAW requires to calculate date from populated RAW database right now and it is known that RAW won't even exist post load time)
74	/// </summary>
75	/// <param name="job"></param>
76	/// <param name="rawDatabase"></param>
77	public IUpdateLoadProgress AddAppropriateDisposeStep(ScheduledDataLoadJob job, DiscoveredDatabase rawDatabase)
78	{
79	IUpdateLoadProgress added;
80	Check(ThrowImmediatelyCheckNotifier.Quiet);	18✔
81
82	switch (Strategy)	16!
83	{
84	case DataLoadProgressUpdateStrategy.UseMaxRequestedDay:
85	added = new UpdateProgressIfLoadsuccessful(job);	4✔
86	break;	2✔
87	case DataLoadProgressUpdateStrategy.ExecuteScalarSQLInLIVE:
88
89	added = new UpdateProgressToResultOfDelegate(job, () => GetMaxDate(GetLiveServer(job), job));	×
90	break;	×
91	case DataLoadProgressUpdateStrategy.ExecuteScalarSQLInRAW:
92	try
93	{
94	var dt = GetMaxDate(rawDatabase.Server, job);	8✔
95	added = new UpdateProgressToSpecificValueIfLoadsuccessful(job, dt);	2✔
96	}	2✔
97	catch (SqlException e)	2✔
98	{
99	throw new DataLoadProgressUpdateException(	2✔
100	$"Failed to execute the following SQL in the RAW database:{ExecuteScalarSQL}", e);	2✔
101	}
102
103	break;
104	case DataLoadProgressUpdateStrategy.DoNothing:
105	//Do not add any post load update i.e. do nothing
106	return null;	4✔
107	default:
108	throw new ArgumentOutOfRangeException();	×
109	}
110
111	job.PushForDisposal(added);	4✔
112	return added;	4✔
113	}
114
115	private static DiscoveredServer GetLiveServer(ScheduledDataLoadJob job) =>
116	DataAccessPortal.ExpectDistinctServer(job.RegularTablesToLoad.ToArray(), DataAccessContext.DataLoad, false);	×
117
118	private DateTime GetMaxDate(DiscoveredServer server, IDataLoadEventListener listener)
119	{
120	using var con = server.GetConnection();	8✔
121	con.Open();	8✔
122
123	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	8✔
124	$"About to execute SQL to determine the maximum date for data loaded:{ExecuteScalarSQL}"));	8✔
125
126	using var cmd = server.GetCommand(ExecuteScalarSQL, con);	8✔
127	var scalarValue = cmd.ExecuteScalar();	8✔
128
129	if (scalarValue == null \|\| scalarValue == DBNull.Value)	6✔
130	throw new DataLoadProgressUpdateException(	2✔
131	"ExecuteScalarSQL specified for determining the maximum date of data loaded returned null when executed");	2✔
132
133	try
134	{
135	return Convert.ToDateTime(scalarValue);	4✔
136	}
137	catch (Exception e)	2✔
138	{
139	throw new DataLoadProgressUpdateException(	2✔
140	$"ExecuteScalarSQL specified for determining the maximum date of data loaded returned a value that was not a Date:{scalarValue}",	2✔
141	e);	2✔
142	}
143	}	2✔
144
145	public void Check(ICheckNotifier notifier)
146	{
147	if (Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInLIVE \|\|	18✔
148	Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInRAW)	18✔
149	if (string.IsNullOrWhiteSpace(ExecuteScalarSQL))	10✔
150	notifier.OnCheckPerformed(	2✔
151	new CheckEventArgs(	2✔
152	$"Strategy is {Strategy} but there is no ExecuteScalarSQL, ExecuteScalarSQL should be a SELECT statement that returns a specific value that reflects the maximum date in the load e.g. Select MAX(MyDate) FROM MyTable",	2✔
153	CheckResult.Fail));	2✔
154
155	// TODO: These checks are VERY Sql Server specific!
156
157	if (Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInRAW)	16✔
158	if (ExecuteScalarSQL.Contains("..") \|\| ExecuteScalarSQL.Contains(".[dbo].") \|\|	8!
159	ExecuteScalarSQL.Contains(".dbo."))	8✔
160	notifier.OnCheckPerformed(	×
161	new CheckEventArgs(	×
162	$"Strategy is {Strategy} but the SQL looks like it references explicit tables, In general RAW queries should use unqualified table names i.e. 'Select MAX(dt) FROM MyTable' NOT 'Select MAX(dt) FROM [MyLIVEDatabase]..[MyTable]'",	×
163	CheckResult.Warning));	×
164
165	if (Strategy == DataLoadProgressUpdateStrategy.ExecuteScalarSQLInLIVE)	16!
166	if (!(ExecuteScalarSQL.Contains("..") \|\| ExecuteScalarSQL.Contains(".[dbo].") \|\|	×
167	ExecuteScalarSQL.Contains(".dbo.")))	×
168	notifier.OnCheckPerformed(	×
169	new CheckEventArgs(	×
170	$"Strategy is {Strategy} but the SQL does not contain '..' or '.[dbo].' or '.dbo.', LIVE update queries should use fully table names i.e. 'Select MAX(dt) FROM [MyLIVEDatabase]..[MyTable]' NOT 'Select MAX(dt) FROM MyTable'",	×
171	CheckResult.Warning));	×
172	}	16✔
173	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous