6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataExport/DataExtraction/ExtractionTimeValidator.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.QueryBuilding;
using Rdmp.Core.Validation;

namespace Rdmp.Core.DataExport.DataExtraction;

/// <summary>
/// Applies Catalogue.ValidationXML to rows extracted during a Data Extraction Pipeline (See ExecuteDatasetExtractionSource).  Because the columns which
/// are extracted can be a subset of the columns in the Catalogue and can include transforms the validation rules have to be adjusted (some are not applied).
///
/// <para>A count of the number of rows failing validation is stored in VerboseValidationResults (divided by column) and is available for writing to the word
/// metadata document that accompanies the extracted records (See WordDataWriter). </para>
///
/// <para>This is similar to CatalogueConstraintReport (DQE) but is applied to a researchers extract instead of the Catalogue as a whole.</para>
/// </summary>
public class ExtractionTimeValidator
{
    private readonly List<IColumn> _columnsToExtract;

    private bool _initialized;

    public Validator Validator { get; set; }
    public VerboseValidationResults Results { get; set; }

    public List<ItemValidator> IgnoredBecauseColumnHashed { get; private set; }

    public ExtractionTimeValidator(ICatalogue catalogue, List<IColumn> columnsToExtract)
    {
        _columnsToExtract = columnsToExtract;

        Validator = Validator.LoadFromXml(catalogue.ValidatorXML);

        if (string.IsNullOrWhiteSpace(catalogue.ValidatorXML))
            throw new ArgumentException($"No validations are configured for catalogue {catalogue.Name}");

        IgnoredBecauseColumnHashed = new List<ItemValidator>();
    }

    public void Validate(DataTable dt, string validationColumnToPopulateIfAny)
    {
        if (!_initialized)
            Initialize(dt);
        dt.BeginLoadData();
        foreach (DataRow r in dt.Rows)
        {
            //additive validation results, Results is a class that wraps DictionaryOfFailure which is an array of columns and each element is another array of consequences (with a row count for each consequence)
            //think of it like a 2D array with X columns and Y consquences and a number in each box which is how many values in that column failed validation with that consequence
            Results = Validator.ValidateVerboseAdditive(r, Results, out var consequenceOnLastRowProcessed);


            if (validationColumnToPopulateIfAny != null)
                r[validationColumnToPopulateIfAny] = consequenceOnLastRowProcessed;
        }
        dt.EndLoadData();
    }

    private void Initialize(DataTable dt)
    {
        var toDiscard = new List<ItemValidator>();

        //discard any item validators that don't exist in our colmn collection (from schema) - These are likely just columns that are not used during validation
        foreach (var iv in Validator.ItemValidators)
            if (!dt.Columns.Contains(iv.TargetProperty)) //if target property is not in the column collection
            {
                toDiscard.Add(iv);
            }
            else
            {
                //also discard any that have an underlying column that is Hashed as they will not match validation constraints post hash (hashing is done in SQL so we will never see original value)
                if (_columnsToExtract.Exists(c => c.ToString().Equals(iv.TargetProperty)))
                {
                    var ec = _columnsToExtract.First(c => c.ToString().Equals(iv.TargetProperty));
                    if (ec.HashOnDataRelease)
                    {
                        IgnoredBecauseColumnHashed.Add(iv);
                        toDiscard.Add(iv);
                    }
                }
                else //also discard any CHI validations as the CHI column will be swapped for a PROCHI
                if (iv.TargetProperty.Equals("CHI", StringComparison.InvariantCultureIgnoreCase))
                {
                    IgnoredBecauseColumnHashed.Add(iv);
                    toDiscard.Add(iv);
                }
            }

        foreach (var itemValidator in toDiscard)
            Validator.ItemValidators.Remove(itemValidator);

        _initialized = true;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.QueryBuilding;
13	using Rdmp.Core.Validation;
14
15	namespace Rdmp.Core.DataExport.DataExtraction;
16
17	/// <summary>
18	/// Applies Catalogue.ValidationXML to rows extracted during a Data Extraction Pipeline (See ExecuteDatasetExtractionSource). Because the columns which
19	/// are extracted can be a subset of the columns in the Catalogue and can include transforms the validation rules have to be adjusted (some are not applied).
20	///
21	/// <para>A count of the number of rows failing validation is stored in VerboseValidationResults (divided by column) and is available for writing to the word
22	/// metadata document that accompanies the extracted records (See WordDataWriter). </para>
23	///
24	/// <para>This is similar to CatalogueConstraintReport (DQE) but is applied to a researchers extract instead of the Catalogue as a whole.</para>
25	/// </summary>
26	public class ExtractionTimeValidator
27	{
28	private readonly List<IColumn> _columnsToExtract;
29
30	private bool _initialized;
31
32	public Validator Validator { get; set; }	×
33	public VerboseValidationResults Results { get; set; }	×
34
35	public List<ItemValidator> IgnoredBecauseColumnHashed { get; private set; }	×
36
37	public ExtractionTimeValidator(ICatalogue catalogue, List<IColumn> columnsToExtract)	×
38	{
39	_columnsToExtract = columnsToExtract;	×
40
41	Validator = Validator.LoadFromXml(catalogue.ValidatorXML);	×
42
43	if (string.IsNullOrWhiteSpace(catalogue.ValidatorXML))	×
44	throw new ArgumentException($"No validations are configured for catalogue {catalogue.Name}");	×
45
46	IgnoredBecauseColumnHashed = new List<ItemValidator>();	×
47	}	×
48
49	public void Validate(DataTable dt, string validationColumnToPopulateIfAny)
50	{
51	if (!_initialized)	×
52	Initialize(dt);	×
53	dt.BeginLoadData();	×
54	foreach (DataRow r in dt.Rows)	×
55	{
56	//additive validation results, Results is a class that wraps DictionaryOfFailure which is an array of columns and each element is another array of consequences (with a row count for each consequence)
57	//think of it like a 2D array with X columns and Y consquences and a number in each box which is how many values in that column failed validation with that consequence
58	Results = Validator.ValidateVerboseAdditive(r, Results, out var consequenceOnLastRowProcessed);	×
59
60
61	if (validationColumnToPopulateIfAny != null)	×
62	r[validationColumnToPopulateIfAny] = consequenceOnLastRowProcessed;	×
63	}
64	dt.EndLoadData();	×
65	}	×
66
67	private void Initialize(DataTable dt)
68	{
69	var toDiscard = new List<ItemValidator>();	×
70
71	//discard any item validators that don't exist in our colmn collection (from schema) - These are likely just columns that are not used during validation
72	foreach (var iv in Validator.ItemValidators)	×
73	if (!dt.Columns.Contains(iv.TargetProperty)) //if target property is not in the column collection	×
74	{
75	toDiscard.Add(iv);	×
76	}
77	else
78	{
79	//also discard any that have an underlying column that is Hashed as they will not match validation constraints post hash (hashing is done in SQL so we will never see original value)
80	if (_columnsToExtract.Exists(c => c.ToString().Equals(iv.TargetProperty)))	×
81	{
82	var ec = _columnsToExtract.First(c => c.ToString().Equals(iv.TargetProperty));	×
83	if (ec.HashOnDataRelease)	×
84	{
85	IgnoredBecauseColumnHashed.Add(iv);	×
86	toDiscard.Add(iv);	×
87	}
88	}
89	else //also discard any CHI validations as the CHI column will be swapped for a PROCHI
90	if (iv.TargetProperty.Equals("CHI", StringComparison.InvariantCultureIgnoreCase))	×
91	{
92	IgnoredBecauseColumnHashed.Add(iv);	×
93	toDiscard.Add(iv);	×
94	}
95	}
96
97	foreach (var itemValidator in toDiscard)	×
98	Validator.ItemValidators.Remove(itemValidator);	×
99
100	_initialized = true;	×
101	}	×
102	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous