6003738864

Committed 28 Aug 2023 06:50PM UTC coverage: 57.442% (+0.01%) from 57.432%

Build # 6003738864

Build Type

push

github

Committed by

web-flow

Commit Message

Feature/RDMP-28 Add BeginLoadData & EndLoadData to Datatables (#1598)

* partial fix

* add row peaker update

* fix up whitespace

* add a lot of daat begin loads

* more data load

* fix typo

Run Details

10908 of 20572 branches covered (0.0%)

Branch coverage included in aggregate %.

65 of 65 new or added lines in 20 files covered. (100.0%)

31683 of 53574 relevant lines covered (59.14%)

8443.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataExport/DataExtraction/ExtractionTimeValidator.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.QueryBuilding;
using Rdmp.Core.Validation;
using Rdmp.Core.Validation.Constraints;

namespace Rdmp.Core.DataExport.DataExtraction;

/// <summary>
/// Applies Catalogue.ValidationXML to rows extracted during a Data Extraction Pipeline (See ExecuteDatasetExtractionSource).  Because the columns which
/// are extracted can be a subset of the columns in the Catalogue and can include transforms the validation rules have to be adjusted (some are not applied).
///
/// <para>A count of the number of rows failing validation is stored in VerboseValidationResults (divided by column) and is available for writing to the word
/// metadata document that accompanies the extracted records (See WordDataWriter). </para>
///
/// <para>This is similar to CatalogueConstraintReport (DQE) but is applied to a researchers extract instead of the Catalogue as a whole.</para>
/// </summary>
public class ExtractionTimeValidator
{
    private readonly ICatalogue _catalogue;
    private readonly List<IColumn> _columnsToExtract;

    private bool _initialized = false;

    public Validator Validator { get; set; }
    public VerboseValidationResults Results { get; set; }

    public List<ItemValidator> IgnoredBecauseColumnHashed { get; private set; }

    public ExtractionTimeValidator(ICatalogue catalogue, List<IColumn> columnsToExtract)
    {
        _catalogue = catalogue;
        _columnsToExtract = columnsToExtract;

        Validator = Validator.LoadFromXml(_catalogue.ValidatorXML);

        if (string.IsNullOrWhiteSpace(_catalogue.ValidatorXML))
            throw new ArgumentException($"No validations are configured for catalogue {catalogue.Name}");

        IgnoredBecauseColumnHashed = new List<ItemValidator>();
    }

    public void Validate(DataTable dt, string validationColumnToPopulateIfAny)
    {
        if (!_initialized)
            Initialize(dt);
        dt.BeginLoadData();
        foreach (DataRow r in dt.Rows)
        {
            //additive validation results, Results is a class that wraps DictionaryOfFailure which is an array of columns and each element is another array of consequences (with a row count for each consequence)
            //think of it like a 2D array with X columns and Y consquences and a number in each box which is how many values in that column failed validation with that consequence
            Results = Validator.ValidateVerboseAdditive(r, Results, out var consequenceOnLastRowProcessed);


            if (validationColumnToPopulateIfAny != null)
                r[validationColumnToPopulateIfAny] = consequenceOnLastRowProcessed;
        }
        dt.EndLoadData();
    }

    private void Initialize(DataTable dt)
    {
        var toDiscard = new List<ItemValidator>();

        //discard any item validators that don't exist in our colmn collection (from schema) - These are likely just columns that are not used during validation
        foreach (var iv in Validator.ItemValidators)
            if (!dt.Columns.Contains(iv.TargetProperty)) //if target property is not in the column collection
            {
                toDiscard.Add(iv);
            }
            else
            {
                //also discard any that have an underlying column that is Hashed as they will not match validation constraints post hash (hashing is done in SQL so we will never see original value)
                if (_columnsToExtract.Exists(c => c.ToString().Equals(iv.TargetProperty)))
                {
                    var ec = _columnsToExtract.First(c => c.ToString().Equals(iv.TargetProperty));
                    if (ec.HashOnDataRelease)
                    {
                        IgnoredBecauseColumnHashed.Add(iv);
                        toDiscard.Add(iv);
                    }
                }
                else //also discard any CHI validations as the CHI column will be swapped for a PROCHI
                if (iv.TargetProperty.Equals("CHI", StringComparison.InvariantCultureIgnoreCase))
                {
                    IgnoredBecauseColumnHashed.Add(iv);
                    toDiscard.Add(iv);
                }
            }

        foreach (var itemValidator in toDiscard)
            Validator.ItemValidators.Remove(itemValidator);

        _initialized = true;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.QueryBuilding;
13	using Rdmp.Core.Validation;
14	using Rdmp.Core.Validation.Constraints;
15
16	namespace Rdmp.Core.DataExport.DataExtraction;
17
18	/// <summary>
19	/// Applies Catalogue.ValidationXML to rows extracted during a Data Extraction Pipeline (See ExecuteDatasetExtractionSource). Because the columns which
20	/// are extracted can be a subset of the columns in the Catalogue and can include transforms the validation rules have to be adjusted (some are not applied).
21	///
22	/// <para>A count of the number of rows failing validation is stored in VerboseValidationResults (divided by column) and is available for writing to the word
23	/// metadata document that accompanies the extracted records (See WordDataWriter). </para>
24	///
25	/// <para>This is similar to CatalogueConstraintReport (DQE) but is applied to a researchers extract instead of the Catalogue as a whole.</para>
26	/// </summary>
27	public class ExtractionTimeValidator
28	{
29	private readonly ICatalogue _catalogue;
30	private readonly List<IColumn> _columnsToExtract;
31
32	private bool _initialized = false;
33
34	public Validator Validator { get; set; }	×
35	public VerboseValidationResults Results { get; set; }	×
36
37	public List<ItemValidator> IgnoredBecauseColumnHashed { get; private set; }	×
38
39	public ExtractionTimeValidator(ICatalogue catalogue, List<IColumn> columnsToExtract)	×
40	{
41	_catalogue = catalogue;	×
42	_columnsToExtract = columnsToExtract;	×
43
44	Validator = Validator.LoadFromXml(_catalogue.ValidatorXML);	×
45
46	if (string.IsNullOrWhiteSpace(_catalogue.ValidatorXML))	×
47	throw new ArgumentException($"No validations are configured for catalogue {catalogue.Name}");	×
48
49	IgnoredBecauseColumnHashed = new List<ItemValidator>();	×
50	}	×
51
52	public void Validate(DataTable dt, string validationColumnToPopulateIfAny)
53	{
54	if (!_initialized)	×
55	Initialize(dt);	×
56	dt.BeginLoadData();	×
57	foreach (DataRow r in dt.Rows)	×
58	{
59	//additive validation results, Results is a class that wraps DictionaryOfFailure which is an array of columns and each element is another array of consequences (with a row count for each consequence)
60	//think of it like a 2D array with X columns and Y consquences and a number in each box which is how many values in that column failed validation with that consequence
61	Results = Validator.ValidateVerboseAdditive(r, Results, out var consequenceOnLastRowProcessed);	×
62
63
64	if (validationColumnToPopulateIfAny != null)	×
65	r[validationColumnToPopulateIfAny] = consequenceOnLastRowProcessed;	×
66	}
67	dt.EndLoadData();	×
68	}	×
69
70	private void Initialize(DataTable dt)
71	{
72	var toDiscard = new List<ItemValidator>();	×
73
74	//discard any item validators that don't exist in our colmn collection (from schema) - These are likely just columns that are not used during validation
75	foreach (var iv in Validator.ItemValidators)	×
76	if (!dt.Columns.Contains(iv.TargetProperty)) //if target property is not in the column collection	×
77	{
78	toDiscard.Add(iv);	×
79	}
80	else
81	{
82	//also discard any that have an underlying column that is Hashed as they will not match validation constraints post hash (hashing is done in SQL so we will never see original value)
83	if (_columnsToExtract.Exists(c => c.ToString().Equals(iv.TargetProperty)))	×
84	{
85	var ec = _columnsToExtract.First(c => c.ToString().Equals(iv.TargetProperty));	×
86	if (ec.HashOnDataRelease)	×
87	{
88	IgnoredBecauseColumnHashed.Add(iv);	×
89	toDiscard.Add(iv);	×
90	}
91	}
92	else //also discard any CHI validations as the CHI column will be swapped for a PROCHI
93	if (iv.TargetProperty.Equals("CHI", StringComparison.InvariantCultureIgnoreCase))	×
94	{
95	IgnoredBecauseColumnHashed.Add(iv);	×
96	toDiscard.Add(iv);	×
97	}
98	}
99
100	foreach (var itemValidator in toDiscard)	×
101	Validator.ItemValidators.Remove(itemValidator);	×
102
103	_initialized = true;	×
104	}	×
105	}

HicServices / RDMP / 6003738864

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous