6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.13

/Rdmp.Core/DataLoad/Modules/Mutilators/SafePrimaryKeyCollisionResolverMutilation.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System.Linq;
using FAnsi.Discovery;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.Mutilators;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.Mutilators;

/// <summary>
/// Attempts to resolves primary key collisions by ordering on a specific column and deleting rows which differ on this column
/// </summary>
public class SafePrimaryKeyCollisionResolverMutilation : IPluginMutilateDataTables
{
    private DiscoveredDatabase _database;
    private LoadStage _loadStage;

    [DemandsInitialization(
        "The non primary key column to be used for deduplication.  This must contain different values for the same primary key and you must only want to keep one e.g. DataAge")]
    public ColumnInfo ColumnToResolveOn { get; set; }

    [DemandsInitialization(
        @"Determines behaviour when a primary key collision is the result of one record having null and another not.
True - Delete the non null record
False - Delete the null record")]
    public bool PreferNulls { get; set; }

    [DemandsInitialization(@"Determines which record is deleted when two values of ColumnToResolveOn are conflicting
True - Delete the smaller value
False - Delete the larger value")]
    public bool PreferLargerValues { get; set; }

    [DemandsInitialization("Timeout in seconds to allow the operation to run for", DefaultValue = 600)]
    public int Timeout { get; set; }

    public void DeleteRows(DiscoveredTable tbl, ColumnInfo[] primaryKeys, IDataLoadEventListener listener)
    {
        var join = string.Join(" AND ", primaryKeys.Select(k => $"t1.{k.GetRuntimeName()}=t2.{k.GetRuntimeName()}"));


        var t1DotColumn = $"t1.{ColumnToResolveOn.GetRuntimeName(_loadStage)}";
        var t2DotColumn = $"t2.{ColumnToResolveOn.GetRuntimeName(_loadStage)}";

        //FYI - we are considering whether to delete records from table {0}

        var deleteConditional =
            PreferNulls
                ?
                //delete rows {0} where {0} is not null and {1} is null - leaving only the null records {1}
                $"({t1DotColumn} IS NOT NULL AND {t2DotColumn} IS NULL)"
                : string.Format("({1} IS NOT NULL AND {0} IS NULL)", t1DotColumn, t2DotColumn);

        deleteConditional += " OR ";

        deleteConditional += PreferLargerValues
            //delete rows {0} where {0} less than {1} - leaving only the greater records {1}
            ? string.Format("({0} <> {1} AND {0} < {1})", t1DotColumn, t2DotColumn)
            : string.Format("({0} <> {1} AND {1} < {0})", t1DotColumn, t2DotColumn);


        var sql = string.Format(@"DELETE t1 FROM {0} t1
  JOIN {0} t2
  ON {1}
  AND ({2})",
            tbl.GetRuntimeName(), join, deleteConditional);

        using var con = tbl.Database.Server.GetConnection();
        con.Open();
        using var cmd = tbl.Database.Server.GetCommand(sql, con);
        cmd.CommandTimeout = Timeout;

        var affectedRows = cmd.ExecuteNonQuery();
        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"Deleted {affectedRows} rows"));
    }

    public void Check(ICheckNotifier notifier)
    {
        if (ColumnToResolveOn is { IsPrimaryKey: true })
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"You cannot use {ColumnToResolveOn} to resolve primary key collisions because it is part of the primary key",
                CheckResult.Fail));
    }

    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
    {
    }

    public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
    {
        _database = dbInfo;
        _loadStage = loadStage;
    }

    public ExitCodeType Mutilate(IDataLoadJob job)
    {
        var tbl = _database.ExpectTable(
            ColumnToResolveOn.TableInfo.GetRuntimeName(_loadStage, job.Configuration.DatabaseNamer));
        var pks = ColumnToResolveOn.TableInfo.ColumnInfos.Where(ci => ci.IsPrimaryKey).ToArray();

        DeleteRows(tbl, pks, job);

        return ExitCodeType.Success;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System.Linq;
8	using FAnsi.Discovery;
9	using Rdmp.Core.Curation.Data;
10	using Rdmp.Core.Curation.Data.DataLoad;
11	using Rdmp.Core.DataLoad.Engine.Job;
12	using Rdmp.Core.DataLoad.Engine.Mutilators;
13	using Rdmp.Core.ReusableLibraryCode.Checks;
14	using Rdmp.Core.ReusableLibraryCode.Progress;
15
16	namespace Rdmp.Core.DataLoad.Modules.Mutilators;
17
18	/// <summary>
19	/// Attempts to resolves primary key collisions by ordering on a specific column and deleting rows which differ on this column
20	/// </summary>
21	public class SafePrimaryKeyCollisionResolverMutilation : IPluginMutilateDataTables
22	{
23	private DiscoveredDatabase _database;
24	private LoadStage _loadStage;
25
26	[DemandsInitialization(
27	"The non primary key column to be used for deduplication. This must contain different values for the same primary key and you must only want to keep one e.g. DataAge")]
28	public ColumnInfo ColumnToResolveOn { get; set; }	228✔
29
30	[DemandsInitialization(
31	@"Determines behaviour when a primary key collision is the result of one record having null and another not.
32	True - Delete the non null record
33	False - Delete the null record")]
34	public bool PreferNulls { get; set; }	92✔
35
36	[DemandsInitialization(@"Determines which record is deleted when two values of ColumnToResolveOn are conflicting
37	True - Delete the smaller value
38	False - Delete the larger value")]
39	public bool PreferLargerValues { get; set; }	92✔
40
41	[DemandsInitialization("Timeout in seconds to allow the operation to run for", DefaultValue = 600)]
42	public int Timeout { get; set; }	48✔
43
44	public void DeleteRows(DiscoveredTable tbl, ColumnInfo[] primaryKeys, IDataLoadEventListener listener)
45	{
46	var join = string.Join(" AND ", primaryKeys.Select(k => $"t1.{k.GetRuntimeName()}=t2.{k.GetRuntimeName()}"));	96✔
47
48
49	var t1DotColumn = $"t1.{ColumnToResolveOn.GetRuntimeName(_loadStage)}";	44✔
50	var t2DotColumn = $"t2.{ColumnToResolveOn.GetRuntimeName(_loadStage)}";	44✔
51
52	//FYI - we are considering whether to delete records from table {0}
53
54	var deleteConditional =	44✔
55	PreferNulls	44✔
56	?	44✔
57	//delete rows {0} where {0} is not null and {1} is null - leaving only the null records {1}	44✔
58	$"({t1DotColumn} IS NOT NULL AND {t2DotColumn} IS NULL)"	44✔
59	: string.Format("({1} IS NOT NULL AND {0} IS NULL)", t1DotColumn, t2DotColumn);	44✔
60
61	deleteConditional += " OR ";	44✔
62
63	deleteConditional += PreferLargerValues	44✔
64	//delete rows {0} where {0} less than {1} - leaving only the greater records {1}	44✔
65	? string.Format("({0} <> {1} AND {0} < {1})", t1DotColumn, t2DotColumn)	44✔
66	: string.Format("({0} <> {1} AND {1} < {0})", t1DotColumn, t2DotColumn);	44✔
67
68
69	var sql = string.Format(@"DELETE t1 FROM {0} t1	44✔
70	JOIN {0} t2	44✔
71	ON {1}	44✔
72	AND ({2})",	44✔
73	tbl.GetRuntimeName(), join, deleteConditional);	44✔
74
75	using var con = tbl.Database.Server.GetConnection();	44✔
76	con.Open();	44✔
77	using var cmd = tbl.Database.Server.GetCommand(sql, con);	44✔
78	cmd.CommandTimeout = Timeout;	44✔
79
80	var affectedRows = cmd.ExecuteNonQuery();	44✔
81	listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	44✔
82	$"Deleted {affectedRows} rows"));	44✔
83	}	88✔
84
85	public void Check(ICheckNotifier notifier)
86	{
87	if (ColumnToResolveOn is { IsPrimaryKey: true })	×
88	notifier.OnCheckPerformed(new CheckEventArgs(	×
89	$"You cannot use {ColumnToResolveOn} to resolve primary key collisions because it is part of the primary key",	×
90	CheckResult.Fail));	×
91	}	×
92
93	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
94	{
95	}	×
96
97	public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
98	{
99	_database = dbInfo;	48✔
100	_loadStage = loadStage;	48✔
101	}	48✔
102
103	public ExitCodeType Mutilate(IDataLoadJob job)
104	{
105	var tbl = _database.ExpectTable(	44✔
106	ColumnToResolveOn.TableInfo.GetRuntimeName(_loadStage, job.Configuration.DatabaseNamer));	44✔
107	var pks = ColumnToResolveOn.TableInfo.ColumnInfos.Where(ci => ci.IsPrimaryKey).ToArray();	184✔
108
109	DeleteRows(tbl, pks, job);	44✔
110
111	return ExitCodeType.Success;	44✔
112	}
113	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous