6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.51

/Rdmp.Core/DataQualityEngine/Reports/DQEStateOverDataLoadRunId.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataQualityEngine.Data;
using Rdmp.Core.QueryBuilding;
using Rdmp.Core.Validation;
using Rdmp.Core.Validation.Constraints;

namespace Rdmp.Core.DataQualityEngine.Reports;

/// <summary>
/// Records the total number of validation failures that occur for each column.  Results are calculated for each novel DataLoadRunId found.  The counts
/// for a column will always add up to the row count even if there are multiple validation rules broken (The worst consequence only is counted).
/// </summary>
public class DQEStateOverDataLoadRunId
{
    private readonly string _pivotCategory;

    //column level results - validation failures
    public Dictionary<int, VerboseValidationResults> ColumnValidationFailuresByDataLoadRunID { get; set; }

    //column level results - everything including unconstrained columns and columns which never had any validation failures at all
    public Dictionary<int, ColumnState[]> AllColumnStates { get; set; }

    public Dictionary<int, Dictionary<Consequence, int>> WorstConsequencesByDataLoadRunID { get; set; }
    public Dictionary<int, int> RowsPassingValidationByDataLoadRunID { get; set; }

    public DQEStateOverDataLoadRunId(string pivotCategory)
    {
        _pivotCategory = pivotCategory;
        InitializeDictionaries();
    }

    public void InitializeDictionaries()
    {
        //column level
        ColumnValidationFailuresByDataLoadRunID = new Dictionary<int, VerboseValidationResults>();
        AllColumnStates = new Dictionary<int, ColumnState[]>();

        //row level
        WorstConsequencesByDataLoadRunID = new Dictionary<int, Dictionary<Consequence, int>>();
        RowsPassingValidationByDataLoadRunID = new Dictionary<int, int>();
    }

    public void AddKeyToDictionaries(int dataLoadRunID, Validator validator, QueryBuilder queryBuilder)
    {
        //ensure keys exit (if it is a novel data load run ID then we will add it to the dictionaries

        //column level
        //ensure validation failures contain it
        if (!ColumnValidationFailuresByDataLoadRunID.ContainsKey(dataLoadRunID))
            ColumnValidationFailuresByDataLoadRunID.Add(dataLoadRunID,
                new VerboseValidationResults(validator.ItemValidators.ToArray()));

        //ensure unconstrained columns have it
        if (!AllColumnStates.ContainsKey(dataLoadRunID))
        {
            var allColumns = new List<ColumnState>();

            foreach (var col in queryBuilder.SelectColumns.Select(s => s.IColumn))
            {
                var runtimeName = col.GetRuntimeName();
                var validationXML = "";

                var itemValidator =
                    validator.ItemValidators.SingleOrDefault(iv => iv.TargetProperty.Equals(runtimeName));

                //if it is a constrained column it is likely to have child ColumnConstraints results but whatever - the important thing is we should document the state of the ItemValidator for this col
                if (itemValidator != null)
                    validationXML = itemValidator.SaveToXml();
                //else it is an unconstrained column, ah well still interesting

                //add the state regardless
                allColumns.Add(new ColumnState(runtimeName, dataLoadRunID, validationXML));
            }

            //and add it to our dictionary under the load batch
            AllColumnStates.Add(dataLoadRunID, allColumns.ToArray());
        }

        //row level
        //ensure key exists in failing rows
        if (!WorstConsequencesByDataLoadRunID.ContainsKey(dataLoadRunID))
        {
            //add the data load run id key
            WorstConsequencesByDataLoadRunID.Add(dataLoadRunID, new Dictionary<Consequence, int>());

            //add each possible consequence as a key too
            WorstConsequencesByDataLoadRunID[dataLoadRunID].Add(Consequence.Wrong, 0);
            WorstConsequencesByDataLoadRunID[dataLoadRunID].Add(Consequence.Missing, 0);
            WorstConsequencesByDataLoadRunID[dataLoadRunID].Add(Consequence.InvalidatesRow, 0);
        }

        //ensure key exists in passing rows
        RowsPassingValidationByDataLoadRunID.TryAdd(dataLoadRunID, 0);
    }

    private bool _correctValuesCalculated;

    /// <summary>
    /// Calculates the final counts for each Column based on the validation failures documented to date.  You can only call this method once and it
    /// must be called before committing to database.
    /// </summary>
    /// <exception cref="Exception"></exception>
    public void CalculateFinalValues()
    {
        if (_correctValuesCalculated)
            throw new Exception("Correct values have already been calculated");

        _correctValuesCalculated = true;

        //adjust the counts for each data load run id \ column according to the dictionary of validation failures
        //per run id
        foreach (var dataLoadRunID in AllColumnStates.Keys)
            //per column
            foreach (var column in AllColumnStates[dataLoadRunID])
                //if it is a constrained column
                if (ColumnValidationFailuresByDataLoadRunID[dataLoadRunID]
                    .DictionaryOfFailure.TryGetValue(column.TargetProperty, out var consequence))
                {
                    //adjust our correct value downwards according to the results of the dictionary of failure
                    column.CountMissing = consequence[Consequence.Missing];
                    column.CountWrong = consequence[Consequence.Wrong];
                    column.CountInvalidatesRow = consequence[Consequence.InvalidatesRow];

                    column.CountCorrect -= consequence[Consequence.Missing];
                    column.CountCorrect -= consequence[Consequence.Wrong];
                    column.CountCorrect -= consequence[Consequence.InvalidatesRow];
                }
    }

    public void CommitToDatabase(Evaluation evaluation, ICatalogue catalogue, DbConnection con,
        DbTransaction transaction)
    {
        if (!_correctValuesCalculated)
            throw new Exception("You must call CalculateFinalValues before committing to the database");

        IEnumerable<int> novelDataLoadRunIDs = RowsPassingValidationByDataLoadRunID.Keys;

        //now for every load batch we encountered in our evaluations
        foreach (var dataLoadRunID in novelDataLoadRunIDs)
        {
            //record the row states calculation (how many total rows are good/bad/ugly etc)
            evaluation.AddRowState(dataLoadRunID,
                RowsPassingValidationByDataLoadRunID[dataLoadRunID],
                WorstConsequencesByDataLoadRunID[dataLoadRunID][Consequence.Missing],
                WorstConsequencesByDataLoadRunID[dataLoadRunID][Consequence.Wrong],
                WorstConsequencesByDataLoadRunID[dataLoadRunID][Consequence.InvalidatesRow],
                catalogue.ValidatorXML,
                _pivotCategory,
                con,
                transaction
            );

            //record the column states calculations (how many total values in column x are good/bad/ugly etc)
            foreach (var columnState in AllColumnStates[dataLoadRunID])
                columnState.Commit(evaluation, _pivotCategory, con, transaction);
        }
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.DataQualityEngine.Data;
13	using Rdmp.Core.QueryBuilding;
14	using Rdmp.Core.Validation;
15	using Rdmp.Core.Validation.Constraints;
16
17	namespace Rdmp.Core.DataQualityEngine.Reports;
18
19	/// <summary>
20	/// Records the total number of validation failures that occur for each column. Results are calculated for each novel DataLoadRunId found. The counts
21	/// for a column will always add up to the row count even if there are multiple validation rules broken (The worst consequence only is counted).
22	/// </summary>
23	public class DQEStateOverDataLoadRunId
24	{
25	private readonly string _pivotCategory;
26
27	//column level results - validation failures
28	public Dictionary<int, VerboseValidationResults> ColumnValidationFailuresByDataLoadRunID { get; set; }	81,768✔
29
30	//column level results - everything including unconstrained columns and columns which never had any validation failures at all
31	public Dictionary<int, ColumnState[]> AllColumnStates { get; set; }	80,144✔
32
33	public Dictionary<int, Dictionary<Consequence, int>> WorstConsequencesByDataLoadRunID { get; set; }	41,600✔
34	public Dictionary<int, int> RowsPassingValidationByDataLoadRunID { get; set; }	78,772✔
35
36	public DQEStateOverDataLoadRunId(string pivotCategory)	8✔
37	{
38	_pivotCategory = pivotCategory;	8✔
39	InitializeDictionaries();	8✔
40	}	8✔
41
42	public void InitializeDictionaries()
43	{
44	//column level
45	ColumnValidationFailuresByDataLoadRunID = new Dictionary<int, VerboseValidationResults>();	8✔
46	AllColumnStates = new Dictionary<int, ColumnState[]>();	8✔
47
48	//row level
49	WorstConsequencesByDataLoadRunID = new Dictionary<int, Dictionary<Consequence, int>>();	8✔
50	RowsPassingValidationByDataLoadRunID = new Dictionary<int, int>();	8✔
51	}	8✔
52
53	public void AddKeyToDictionaries(int dataLoadRunID, Validator validator, QueryBuilder queryBuilder)
54	{
55	//ensure keys exit (if it is a novel data load run ID then we will add it to the dictionaries
56
57	//column level
58	//ensure validation failures contain it
59	if (!ColumnValidationFailuresByDataLoadRunID.ContainsKey(dataLoadRunID))	40,000✔
60	ColumnValidationFailuresByDataLoadRunID.Add(dataLoadRunID,	44✔
61	new VerboseValidationResults(validator.ItemValidators.ToArray()));	44✔
62
63	//ensure unconstrained columns have it
64	if (!AllColumnStates.ContainsKey(dataLoadRunID))	40,000✔
65	{
66	var allColumns = new List<ColumnState>();	44✔
67
68	foreach (var col in queryBuilder.SelectColumns.Select(s => s.IColumn))	5,236✔
69	{
70	var runtimeName = col.GetRuntimeName();	1,716✔
71	var validationXML = "";	1,716✔
72
73	var itemValidator =	1,716✔
74	validator.ItemValidators.SingleOrDefault(iv => iv.TargetProperty.Equals(runtimeName));	5,148✔
75
76	//if it is a constrained column it is likely to have child ColumnConstraints results but whatever - the important thing is we should document the state of the ItemValidator for this col
77	if (itemValidator != null)	1,716✔
78	validationXML = itemValidator.SaveToXml();	88✔
79	//else it is an unconstrained column, ah well still interesting
80
81	//add the state regardless
82	allColumns.Add(new ColumnState(runtimeName, dataLoadRunID, validationXML));	1,716✔
83	}
84
85	//and add it to our dictionary under the load batch
86	AllColumnStates.Add(dataLoadRunID, allColumns.ToArray());	44✔
87	}
88
89	//row level
90	//ensure key exists in failing rows
91	if (!WorstConsequencesByDataLoadRunID.ContainsKey(dataLoadRunID))	40,000✔
92	{
93	//add the data load run id key
94	WorstConsequencesByDataLoadRunID.Add(dataLoadRunID, new Dictionary<Consequence, int>());	44✔
95
96	//add each possible consequence as a key too
97	WorstConsequencesByDataLoadRunID[dataLoadRunID].Add(Consequence.Wrong, 0);	44✔
98	WorstConsequencesByDataLoadRunID[dataLoadRunID].Add(Consequence.Missing, 0);	44✔
99	WorstConsequencesByDataLoadRunID[dataLoadRunID].Add(Consequence.InvalidatesRow, 0);	44✔
100	}
101
102	//ensure key exists in passing rows
103	RowsPassingValidationByDataLoadRunID.TryAdd(dataLoadRunID, 0);	40,000✔
104	}	40,000✔
105
106	private bool _correctValuesCalculated;
107
108	/// <summary>
109	/// Calculates the final counts for each Column based on the validation failures documented to date. You can only call this method once and it
110	/// must be called before committing to database.
111	/// </summary>
112	/// <exception cref="Exception"></exception>
113	public void CalculateFinalValues()
114	{
115	if (_correctValuesCalculated)	4!
116	throw new Exception("Correct values have already been calculated");	×
117
118	_correctValuesCalculated = true;	4✔
119
120	//adjust the counts for each data load run id \ column according to the dictionary of validation failures
121	//per run id
122	foreach (var dataLoadRunID in AllColumnStates.Keys)	96✔
123	//per column
124	foreach (var column in AllColumnStates[dataLoadRunID])	3,520✔
125	//if it is a constrained column
126	if (ColumnValidationFailuresByDataLoadRunID[dataLoadRunID]	1,716✔
127	.DictionaryOfFailure.TryGetValue(column.TargetProperty, out var consequence))	1,716✔
128	{
129	//adjust our correct value downwards according to the results of the dictionary of failure
130	column.CountMissing = consequence[Consequence.Missing];	88✔
131	column.CountWrong = consequence[Consequence.Wrong];	88✔
132	column.CountInvalidatesRow = consequence[Consequence.InvalidatesRow];	88✔
133
134	column.CountCorrect -= consequence[Consequence.Missing];	88✔
135	column.CountCorrect -= consequence[Consequence.Wrong];	88✔
136	column.CountCorrect -= consequence[Consequence.InvalidatesRow];	88✔
137	}
138	}	4✔
139
140	public void CommitToDatabase(Evaluation evaluation, ICatalogue catalogue, DbConnection con,
141	DbTransaction transaction)
142	{
143	if (!_correctValuesCalculated)	4!
144	throw new Exception("You must call CalculateFinalValues before committing to the database");	×
145
146	IEnumerable<int> novelDataLoadRunIDs = RowsPassingValidationByDataLoadRunID.Keys;	4✔
147
148	//now for every load batch we encountered in our evaluations
149	foreach (var dataLoadRunID in novelDataLoadRunIDs)	96✔
150	{
151	//record the row states calculation (how many total rows are good/bad/ugly etc)
152	evaluation.AddRowState(dataLoadRunID,	44✔
153	RowsPassingValidationByDataLoadRunID[dataLoadRunID],	44✔
154	WorstConsequencesByDataLoadRunID[dataLoadRunID][Consequence.Missing],	44✔
155	WorstConsequencesByDataLoadRunID[dataLoadRunID][Consequence.Wrong],	44✔
156	WorstConsequencesByDataLoadRunID[dataLoadRunID][Consequence.InvalidatesRow],	44✔
157	catalogue.ValidatorXML,	44✔
158	_pivotCategory,	44✔
159	con,	44✔
160	transaction	44✔
161	);	44✔
162
163	//record the column states calculations (how many total values in column x are good/bad/ugly etc)
164	foreach (var columnState in AllColumnStates[dataLoadRunID])	3,520✔
165	columnState.Commit(evaluation, _pivotCategory, con, transaction);	1,716✔
166	}
167	}	4✔
168	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous