6237307473

Committed 19 Sep 2023 04:02PM UTC coverage: 57.015% (-0.4%) from 57.44%

Build # 6237307473

Build Type

push

github

Committed by

web-flow

Commit Message

Feature/rc4 (#1570)

* Syntax tidying
* Dependency updates
* Event handling singletons (ThrowImmediately and co)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: James A Sutherland <>
Co-authored-by: James Friel <jfriel001@dundee.ac.uk>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Run Details

10734 of 20259 branches covered (0.0%)

Branch coverage included in aggregate %.

5922 of 5922 new or added lines in 565 files covered. (100.0%)

30687 of 52390 relevant lines covered (58.57%)

7361.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

29.79

/Rdmp.Core/DataLoad/Modules/Mutilators/PrimaryKeyCollisionResolverMutilation.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using FAnsi.Discovery;
using Microsoft.Data.SqlClient;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.Mutilators;
using Rdmp.Core.QueryBuilding;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.Mutilators;

/// <summary>
/// Data load component which can delete records in an (unconstrained) RAW table to enforce uniqueness of the primary key field (as it is configured in LIVE).
/// This lets you resolve non-exact duplication based on column order (e.g. if there is a collision where one has an later 'DataAge' field then use the later
/// one and discard the earlier one.
/// 
///  <para>This is a very dangerous operation which uses the primary key collision resolution order (Accessible through CatalogueManager by right clicking a
/// TableInfo and choosing 'Configure Primary Key Collision Resolution') to delete records in a preferred order, fully eliminating primary key collisions.
/// It is a very good idea to not have this task until you are absolutely certain that your primary key is correct and that the duplicate records being deleted
/// are the correct decisions e.g. delete an older record in a given load batch and not simply erasing vast swathes of data!.  The Data Load Engine will tell
/// you with a warning when records are deleted and how many.  If you notice a lot of deletion then try removing this component and manually inspecting the data
/// in the RAW database after the data load fails (due to unresolved primary key conflicts)</para>
/// 
/// <para>This component requires that a collision resolution order has been configured on the TableInfo (See ConfigurePrimaryKeyCollisionResolution)</para>
/// 
/// </summary>
public class PrimaryKeyCollisionResolverMutilation : IPluginMutilateDataTables
{
    [DemandsInitialization(
        "The table on which to resolve primary key collisions, must have PrimaryKeyCollision resolution setup for it in the Data Catalogue",
        Mandatory = true)]
    public TableInfo TargetTable { get; set; }

    public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
    {
    }

    private DiscoveredDatabase _dbInfo;

    public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
    {
        if (loadStage != LoadStage.AdjustRaw)
            throw new Exception(
                $"Primary key collisions can only be resolved in a RAW environment, current load stage is:{loadStage} (The reason for this is because there should be primary keys in the database level in STAGING and LIVE making primary key collisions IMPOSSIBLE)");

        _dbInfo = dbInfo;
    }

    public ExitCodeType Mutilate(IDataLoadJob job)
    {
        ResolvePrimaryKeyConflicts(job);
        return ExitCodeType.Success;
    }


    private void ResolvePrimaryKeyConflicts(IDataLoadEventListener job)
    {
        using var con = (SqlConnection)_dbInfo.Server.GetConnection();
        con.Open();

        var resolver = new PrimaryKeyCollisionResolver(TargetTable);
        var cmdAreTherePrimaryKeyCollisions = new SqlCommand(resolver.GenerateCollisionDetectionSQL(), con)
        {
            CommandTimeout = 5000
        };

        //if there are no primary key collisions
        if (cmdAreTherePrimaryKeyCollisions.ExecuteScalar().ToString().Equals("0"))
        {
            job.OnNotify(this,
                new NotifyEventArgs(ProgressEventType.Information, "No primary key collisions detected"));
            return;
        }

        //there are primary key collisions so resolve them
        job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Primary key collisions detected"));

        var cmdResolve = new SqlCommand(resolver.GenerateSQL(), con)
        {
            CommandTimeout = 5000
        };
        var affectedRows = cmdResolve.ExecuteNonQuery();

        job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
            $"Primary key collisions resolved by deleting {affectedRows} rows"));
    }

    public void Check(ICheckNotifier notifier)
    {
        if (TargetTable == null)
            notifier.OnCheckPerformed(new CheckEventArgs(
                "Target table is null, a table must be specified upon which to resolve primary key duplication (that TableInfo must have a primary key collision resolution order)",
                CheckResult.Fail, null));

        try
        {
            var resolver = new PrimaryKeyCollisionResolver(TargetTable);
            var sql = resolver.GenerateSQL();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"Failed to check PrimaryKeyCollisionResolver on {TargetTable}", CheckResult.Fail, e));
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using FAnsi.Discovery;
9	using Microsoft.Data.SqlClient;
10	using Rdmp.Core.Curation.Data;
11	using Rdmp.Core.Curation.Data.DataLoad;
12	using Rdmp.Core.DataLoad.Engine.Job;
13	using Rdmp.Core.DataLoad.Engine.Mutilators;
14	using Rdmp.Core.QueryBuilding;
15	using Rdmp.Core.ReusableLibraryCode.Checks;
16	using Rdmp.Core.ReusableLibraryCode.Progress;
17
18	namespace Rdmp.Core.DataLoad.Modules.Mutilators;
19
20	/// <summary>
21	/// Data load component which can delete records in an (unconstrained) RAW table to enforce uniqueness of the primary key field (as it is configured in LIVE).
22	/// This lets you resolve non-exact duplication based on column order (e.g. if there is a collision where one has an later 'DataAge' field then use the later
23	/// one and discard the earlier one.
24	///
25	/// <para>This is a very dangerous operation which uses the primary key collision resolution order (Accessible through CatalogueManager by right clicking a
26	/// TableInfo and choosing 'Configure Primary Key Collision Resolution') to delete records in a preferred order, fully eliminating primary key collisions.
27	/// It is a very good idea to not have this task until you are absolutely certain that your primary key is correct and that the duplicate records being deleted
28	/// are the correct decisions e.g. delete an older record in a given load batch and not simply erasing vast swathes of data!. The Data Load Engine will tell
29	/// you with a warning when records are deleted and how many. If you notice a lot of deletion then try removing this component and manually inspecting the data
30	/// in the RAW database after the data load fails (due to unresolved primary key conflicts)</para>
31	///
32	/// <para>This component requires that a collision resolution order has been configured on the TableInfo (See ConfigurePrimaryKeyCollisionResolution)</para>
33	///
34	/// </summary>
35	public class PrimaryKeyCollisionResolverMutilation : IPluginMutilateDataTables
36	{
37	[DemandsInitialization(
38	"The table on which to resolve primary key collisions, must have PrimaryKeyCollision resolution setup for it in the Data Catalogue",
39	Mandatory = true)]
40	public TableInfo TargetTable { get; set; }	16✔
41
42	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
43	{
44	}	×
45
46	private DiscoveredDatabase _dbInfo;
47
48	public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
49	{
50	if (loadStage != LoadStage.AdjustRaw)	×
51	throw new Exception(	×
52	$"Primary key collisions can only be resolved in a RAW environment, current load stage is:{loadStage} (The reason for this is because there should be primary keys in the database level in STAGING and LIVE making primary key collisions IMPOSSIBLE)");	×
53
54	_dbInfo = dbInfo;	×
55	}	×
56
57	public ExitCodeType Mutilate(IDataLoadJob job)
58	{
59	ResolvePrimaryKeyConflicts(job);	×
60	return ExitCodeType.Success;	×
61	}
62
63
64	private void ResolvePrimaryKeyConflicts(IDataLoadEventListener job)
65	{
66	using var con = (SqlConnection)_dbInfo.Server.GetConnection();	×
67	con.Open();	×
68
69	var resolver = new PrimaryKeyCollisionResolver(TargetTable);	×
70	var cmdAreTherePrimaryKeyCollisions = new SqlCommand(resolver.GenerateCollisionDetectionSQL(), con)	×
71	{	×
72	CommandTimeout = 5000	×
73	};	×
74
75	//if there are no primary key collisions
76	if (cmdAreTherePrimaryKeyCollisions.ExecuteScalar().ToString().Equals("0"))	×
77	{
78	job.OnNotify(this,	×
79	new NotifyEventArgs(ProgressEventType.Information, "No primary key collisions detected"));	×
80	return;	×
81	}
82
83	//there are primary key collisions so resolve them
84	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Primary key collisions detected"));	×
85
86	var cmdResolve = new SqlCommand(resolver.GenerateSQL(), con)	×
87	{	×
88	CommandTimeout = 5000	×
89	};	×
90	var affectedRows = cmdResolve.ExecuteNonQuery();	×
91
92	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	×
93	$"Primary key collisions resolved by deleting {affectedRows} rows"));	×
94	}	×
95
96	public void Check(ICheckNotifier notifier)
97	{
98	if (TargetTable == null)	6✔
99	notifier.OnCheckPerformed(new CheckEventArgs(	2✔
100	"Target table is null, a table must be specified upon which to resolve primary key duplication (that TableInfo must have a primary key collision resolution order)",	2✔
101	CheckResult.Fail, null));	2✔
102
103	try
104	{
105	var resolver = new PrimaryKeyCollisionResolver(TargetTable);	4✔
106	var sql = resolver.GenerateSQL();	4✔
107	}	2✔
108	catch (Exception e)	2✔
109	{
110	notifier.OnCheckPerformed(new CheckEventArgs(	2✔
111	$"Failed to check PrimaryKeyCollisionResolver on {TargetTable}", CheckResult.Fail, e));	2✔
112	}	×
113	}	2✔
114	}

HicServices / RDMP / 6237307473

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous