6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.31

/Rdmp.Core/DataLoad/Engine/Migration/StagingToLiveMigrationFieldProcessor.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using FAnsi.Discovery;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataLoad.Triggers;

namespace Rdmp.Core.DataLoad.Engine.Migration;

/// <summary>
/// Checks that LIVE has appropriate fields to support the migration of records from STAGING to LIVE and assigns fields roles such that artifact fields
/// that are generated as part of the load (i.e. computed columns) denoted by the prefix hic_ are not treated as differences in the dataset.  This means
/// that records in STAGING with a new hic_dataLoadRunID (all of them because each load gets a unique number) will not be identified as UPDATES to the
/// LIVE data table and will be ignored (assuming that there are no differences in other fields that are Diffed).
/// </summary>
public class StagingToLiveMigrationFieldProcessor : IMigrationFieldProcessor
{
    private readonly Regex _updateButDoNotDiffExtended;
    private readonly Regex _ignore;
    private readonly ColumnInfo[] _alsoIgnore;

    /// <inheritdoc/>
    public bool NoBackupTrigger { get; set; }

    public StagingToLiveMigrationFieldProcessor(Regex updateButDoNotDiff = null, Regex ignore = null,
        ColumnInfo[] alsoIgnore = null)
    {
        _updateButDoNotDiffExtended = updateButDoNotDiff;
        _ignore = ignore;
        _alsoIgnore = alsoIgnore;
    }

    public void ValidateFields(DiscoveredColumn[] sourceFields, DiscoveredColumn[] destinationFields)
    {
        if (NoBackupTrigger)
            return;

        if (!destinationFields.Any(f =>
                f.GetRuntimeName().Equals(SpecialFieldNames.DataLoadRunID, StringComparison.CurrentCultureIgnoreCase)))
            throw new MissingFieldException(
                $"Destination (Live) database table is missing field:{SpecialFieldNames.DataLoadRunID}");

        if (!destinationFields.Any(f =>
                f.GetRuntimeName().Equals(SpecialFieldNames.ValidFrom, StringComparison.CurrentCultureIgnoreCase)))
            throw new MissingFieldException(
                $"Destination (Live) database table is missing field:{SpecialFieldNames.ValidFrom}");
    }

    public void AssignFieldsForProcessing(DiscoveredColumn field, List<DiscoveredColumn> fieldsToDiff,
        List<DiscoveredColumn> fieldsToUpdate)
    {
        if (IgnoreColumnInfo(field))
            return;

        if (Ignore(field))
            return;

        //it is a hic internal field but not one of the overwritten, standard ones
        if (SpecialFieldNames.IsHicPrefixed(field)
            ||
            UpdateOnly(field))

        {
            fieldsToUpdate.Add(field);
        }
        else
        {
            //it is not a hic internal field
            fieldsToDiff.Add(field);
            fieldsToUpdate.Add(field);
        }
    }

    private bool IgnoreColumnInfo(DiscoveredColumn field)
    {
        if (_alsoIgnore == null)
            return false;

        // TODO: if a load targets 2 tables with the same name in different databases and one has a column marked ignore it could be ignored in both during load.  Note that `field` parameter is the from column not the to column

        //its a column we were told to ignore
        var match = _alsoIgnore.FirstOrDefault(c =>
            c.GetRuntimeName().Equals(field.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase) &&
            c.TableInfo.GetRuntimeName().Equals(field.Table.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase)
        );

        return match != null && field.IsPrimaryKey
            ? throw new NotSupportedException(
                $"ColumnInfo {match} is marked {nameof(ColumnInfo.IgnoreInLoads)} but is a Primary Key column this is not permitted")
            : match != null;
    }

    private bool Ignore(DiscoveredColumn field)
    {
        if (_ignore == null)
            return false;

        //its a global ignore based on regex ignore pattern?
        var match = _ignore.IsMatch(field.GetRuntimeName());

        return match && field.IsPrimaryKey
            ? throw new NotSupportedException(
                $"Ignore Pattern {_ignore} matched Primary Key column '{field.GetRuntimeName()}' this is not permitted")
            : match;
    }

    private bool UpdateOnly(DiscoveredColumn field)
    {
        if (_updateButDoNotDiffExtended == null)
            return false;

        //its a supplemental ignore e.g. MessageGuid
        var match = _updateButDoNotDiffExtended.IsMatch(field.GetRuntimeName());

        return match && field.IsPrimaryKey
            ? throw new NotSupportedException(
                $"UpdateButDoNotDiff Pattern {_updateButDoNotDiffExtended} matched Primary Key column '{field.GetRuntimeName()}' this is not permitted")
            : match;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Linq;
10	using System.Text.RegularExpressions;
11	using FAnsi.Discovery;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.DataLoad.Triggers;
14
15	namespace Rdmp.Core.DataLoad.Engine.Migration;
16
17	/// <summary>
18	/// Checks that LIVE has appropriate fields to support the migration of records from STAGING to LIVE and assigns fields roles such that artifact fields
19	/// that are generated as part of the load (i.e. computed columns) denoted by the prefix hic_ are not treated as differences in the dataset. This means
20	/// that records in STAGING with a new hic_dataLoadRunID (all of them because each load gets a unique number) will not be identified as UPDATES to the
21	/// LIVE data table and will be ignored (assuming that there are no differences in other fields that are Diffed).
22	/// </summary>
23	public class StagingToLiveMigrationFieldProcessor : IMigrationFieldProcessor
24	{
25	private readonly Regex _updateButDoNotDiffExtended;
26	private readonly Regex _ignore;
27	private readonly ColumnInfo[] _alsoIgnore;
28
29	/// <inheritdoc/>
30	public bool NoBackupTrigger { get; set; }	96✔
31
32	public StagingToLiveMigrationFieldProcessor(Regex updateButDoNotDiff = null, Regex ignore = null,	46✔
33	ColumnInfo[] alsoIgnore = null)	46✔
34	{
35	_updateButDoNotDiffExtended = updateButDoNotDiff;	46✔
36	_ignore = ignore;	46✔
37	_alsoIgnore = alsoIgnore;	46✔
38	}	46✔
39
40	public void ValidateFields(DiscoveredColumn[] sourceFields, DiscoveredColumn[] destinationFields)
41	{
42	if (NoBackupTrigger)	50✔
43	return;	6✔
44
45	if (!destinationFields.Any(f =>	44!
46	f.GetRuntimeName().Equals(SpecialFieldNames.DataLoadRunID, StringComparison.CurrentCultureIgnoreCase)))	362✔
47	throw new MissingFieldException(	×
48	$"Destination (Live) database table is missing field:{SpecialFieldNames.DataLoadRunID}");	×
49
50	if (!destinationFields.Any(f =>	44!
51	f.GetRuntimeName().Equals(SpecialFieldNames.ValidFrom, StringComparison.CurrentCultureIgnoreCase)))	406✔
52	throw new MissingFieldException(	×
53	$"Destination (Live) database table is missing field:{SpecialFieldNames.ValidFrom}");	×
54	}	44✔
55
56	public void AssignFieldsForProcessing(DiscoveredColumn field, List<DiscoveredColumn> fieldsToDiff,
57	List<DiscoveredColumn> fieldsToUpdate)
58	{
59	if (IgnoreColumnInfo(field))	302!
60	return;	×
61
62	if (Ignore(field))	302!
63	return;	×
64
65	//it is a hic internal field but not one of the overwritten, standard ones
66	if (SpecialFieldNames.IsHicPrefixed(field)	302✔
67	\|\|	302✔
68	UpdateOnly(field))	302✔
69
70	{
71	fieldsToUpdate.Add(field);	2✔
72	}
73	else
74	{
75	//it is not a hic internal field
76	fieldsToDiff.Add(field);	300✔
77	fieldsToUpdate.Add(field);	300✔
78	}
79	}	300✔
80
81	private bool IgnoreColumnInfo(DiscoveredColumn field)
82	{
83	if (_alsoIgnore == null)	302!
84	return false;	×
85
86	// TODO: if a load targets 2 tables with the same name in different databases and one has a column marked ignore it could be ignored in both during load. Note that `field` parameter is the from column not the to column
87
88	//its a column we were told to ignore
89	var match = _alsoIgnore.FirstOrDefault(c =>	302✔
90	c.GetRuntimeName().Equals(field.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase) &&	24!
91	c.TableInfo.GetRuntimeName().Equals(field.Table.GetRuntimeName(), StringComparison.CurrentCultureIgnoreCase)	24✔
92	);	302✔
93
94	return match != null && field.IsPrimaryKey	302!
95	? throw new NotSupportedException(	302✔
96	$"ColumnInfo {match} is marked {nameof(ColumnInfo.IgnoreInLoads)} but is a Primary Key column this is not permitted")	302✔
97	: match != null;	302✔
98	}
99
100	private bool Ignore(DiscoveredColumn field)
101	{
102	if (_ignore == null)	302✔
103	return false;	290✔
104
105	//its a global ignore based on regex ignore pattern?
106	var match = _ignore.IsMatch(field.GetRuntimeName());	12✔
107
108	return match && field.IsPrimaryKey	12!
109	? throw new NotSupportedException(	12✔
110	$"Ignore Pattern {_ignore} matched Primary Key column '{field.GetRuntimeName()}' this is not permitted")	12✔
111	: match;	12✔
112	}
113
114	private bool UpdateOnly(DiscoveredColumn field)
115	{
116	if (_updateButDoNotDiffExtended == null)	302✔
117	return false;	296✔
118
119	//its a supplemental ignore e.g. MessageGuid
120	var match = _updateButDoNotDiffExtended.IsMatch(field.GetRuntimeName());	6✔
121
122	return match && field.IsPrimaryKey	6!
123	? throw new NotSupportedException(	6✔
124	$"UpdateButDoNotDiff Pattern {_updateButDoNotDiffExtended} matched Primary Key column '{field.GetRuntimeName()}' this is not permitted")	6✔
125	: match;	6✔
126	}
127	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous