6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.13

/Rdmp.Core/DataLoad/Engine/Migration/MigrationColumnSet.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Linq;
using FAnsi.Discovery;
using Rdmp.Core.DataLoad.Triggers;

namespace Rdmp.Core.DataLoad.Engine.Migration;

/// <summary>
/// Defines the role of every field involved in a STAGING to LIVE migration during DLE execution.  When performing a selective UPDATE it is important not to
/// overwrite current records with new records where the 'newness' is an artifact of data loading rather than source data.  For example the field
/// hic_dataLoadRunID will always be different between STAGING and LIVE.  This class stores which columns should be used to identify records which exist
/// in both (PrimaryKeys), which columns indicate significant change and should be promoted (FieldsToDiff) and which are not significant changes but should
/// be copied across anyway in the event that the row is new or there is a difference in another significant field in that record (FieldsToUpdate).
/// </summary>
public class MigrationColumnSet
{
    public DiscoveredTable SourceTable { get; set; }
    public DiscoveredTable DestinationTable { get; set; }

    public DiscoveredColumn[] PrimaryKeys { get; private set; }

    public static List<string> GetStandardColumnNames() =>
        new() { SpecialFieldNames.DataLoadRunID, SpecialFieldNames.ValidFrom };

    /// <summary>
    /// Fields that will have their values compared for change, to decide whether to overwrite destination data with source data. (some fields might not matter
    /// if they are different e.g. dataLoadRunID)
    /// </summary>
    public List<DiscoveredColumn> FieldsToDiff { get; set; }

    /// <summary>
    /// Fields that will have their values copied across to the new table (this is a superset of fields to diff, and also includes all primary keys).  Note
    /// that the non-standard columns (data load run and valid from do not appear in this list, you are intended to handle their update yourself)
    /// </summary>
    public List<DiscoveredColumn> FieldsToUpdate { get; set; }

    public MigrationColumnSet(DiscoveredTable from, DiscoveredTable to,
        IMigrationFieldProcessor migrationFieldProcessor)
    {
        var fromCols = from.DiscoverColumns();
        var toCols = to.DiscoverColumns();

        migrationFieldProcessor.ValidateFields(fromCols, toCols);

        SourceTable = from;
        DestinationTable = to;

        PrimaryKeys = fromCols.Where(c => c.IsPrimaryKey).ToArray();
        FieldsToDiff = new List<DiscoveredColumn>();
        FieldsToUpdate = new List<DiscoveredColumn>();

        foreach (var pk in PrimaryKeys)
            if (!toCols.Any(f =>
                    f.GetRuntimeName().Equals(pk.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase)))
                throw new MissingFieldException($"Column {pk} is missing from either the destination table");

        if (!PrimaryKeys.Any())
            throw new Exception($"There are no primary keys declared in table {from}");

        //figure out things to migrate and whether they matter to diffing
        foreach (var field in fromCols)
        {
            if (
                field.GetRuntimeName()
                    .Equals(SpecialFieldNames.DataLoadRunID, StringComparison.CurrentCultureIgnoreCase) ||
                field.GetRuntimeName().Equals(SpecialFieldNames.ValidFrom, StringComparison.CurrentCultureIgnoreCase))
                continue;

            if (!toCols.Any(c =>
                    c.GetRuntimeName().Equals(field.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase)))
                throw new MissingFieldException($"Field {field} is missing from destination table");

            migrationFieldProcessor.AssignFieldsForProcessing(field, FieldsToDiff, FieldsToUpdate);
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Linq;
10	using FAnsi.Discovery;
11	using Rdmp.Core.DataLoad.Triggers;
12
13	namespace Rdmp.Core.DataLoad.Engine.Migration;
14
15	/// <summary>
16	/// Defines the role of every field involved in a STAGING to LIVE migration during DLE execution. When performing a selective UPDATE it is important not to
17	/// overwrite current records with new records where the 'newness' is an artifact of data loading rather than source data. For example the field
18	/// hic_dataLoadRunID will always be different between STAGING and LIVE. This class stores which columns should be used to identify records which exist
19	/// in both (PrimaryKeys), which columns indicate significant change and should be promoted (FieldsToDiff) and which are not significant changes but should
20	/// be copied across anyway in the event that the row is new or there is a difference in another significant field in that record (FieldsToUpdate).
21	/// </summary>
22	public class MigrationColumnSet
23	{
24	public DiscoveredTable SourceTable { get; set; }	342✔
25	public DiscoveredTable DestinationTable { get; set; }	858✔
26
27	public DiscoveredColumn[] PrimaryKeys { get; private set; }	702✔
28
29	public static List<string> GetStandardColumnNames() =>
30	new() { SpecialFieldNames.DataLoadRunID, SpecialFieldNames.ValidFrom };	×
31
32	/// <summary>
33	/// Fields that will have their values compared for change, to decide whether to overwrite destination data with source data. (some fields might not matter
34	/// if they are different e.g. dataLoadRunID)
35	/// </summary>
36	public List<DiscoveredColumn> FieldsToDiff { get; set; }	636✔
37
38	/// <summary>
39	/// Fields that will have their values copied across to the new table (this is a superset of fields to diff, and also includes all primary keys). Note
40	/// that the non-standard columns (data load run and valid from do not appear in this list, you are intended to handle their update yourself)
41	/// </summary>
42	public List<DiscoveredColumn> FieldsToUpdate { get; set; }	802✔
43
44	public MigrationColumnSet(DiscoveredTable from, DiscoveredTable to,	110✔
45	IMigrationFieldProcessor migrationFieldProcessor)	110✔
46	{
47	var fromCols = from.DiscoverColumns();	110✔
48	var toCols = to.DiscoverColumns();	110✔
49
50	migrationFieldProcessor.ValidateFields(fromCols, toCols);	110✔
51
52	SourceTable = from;	110✔
53	DestinationTable = to;	110✔
54
55	PrimaryKeys = fromCols.Where(c => c.IsPrimaryKey).ToArray();	712✔
56	FieldsToDiff = new List<DiscoveredColumn>();	110✔
57	FieldsToUpdate = new List<DiscoveredColumn>();	110✔
58
59	foreach (var pk in PrimaryKeys)	520✔
60	if (!toCols.Any(f =>	150!
61	f.GetRuntimeName().Equals(pk.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase)))	419✔
62	throw new MissingFieldException($"Column {pk} is missing from either the destination table");	×
63
64	if (!PrimaryKeys.Any())	110✔
65	throw new Exception($"There are no primary keys declared in table {from}");	2✔
66
67	//figure out things to migrate and whether they matter to diffing
68	foreach (var field in fromCols)	1,416✔
69	{
70	if (	600✔
71	field.GetRuntimeName()	600✔
72	.Equals(SpecialFieldNames.DataLoadRunID, StringComparison.CurrentCultureIgnoreCase) \|\|	600✔
73	field.GetRuntimeName().Equals(SpecialFieldNames.ValidFrom, StringComparison.CurrentCultureIgnoreCase))	600✔
74	continue;
75
76	if (!toCols.Any(c =>	484!
77	c.GetRuntimeName().Equals(field.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase)))	4,206✔
78	throw new MissingFieldException($"Field {field} is missing from destination table");	×
79
80	migrationFieldProcessor.AssignFieldsForProcessing(field, FieldsToDiff, FieldsToUpdate);	484✔
81	}
82	}	108✔
83	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous