6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataExport/DataExtraction/ExtractionTimeTimeCoverageAggregator.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data;
using System.Data.Common;
using System.Linq;
using System.Text.RegularExpressions;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;

namespace Rdmp.Core.DataExport.DataExtraction;

/// <summary>
/// Counts the number of unique patients / records encountered while executing an ExtractionConfiguration's dataset (linked to the project cohort).  The input
/// to this is a DataRow rather than an SQL query because the class is used at extraction time as we are writing records out to the ExtractionDirectory.
/// </summary>
public partial class ExtractionTimeTimeCoverageAggregator
{
    public int countOfBrokenDates { get; private set; }
    public int countOfNullsSeen { get; private set; }
    public Dictionary<DateTime, ExtractionTimeTimeCoverageAggregatorBucket> Buckets { get; private set; }

    private readonly ICatalogue _catalogue;
    private string _expectedTimeFieldInOutputBuffer;
    private string _expectedExtractionIdentifierInOutputBuffer;

    private bool haveCheckedTimeFieldExists;


    private const ExtractionTimeTimeCoverageAggregatorBucket.BucketSize BucketSize =
        ExtractionTimeTimeCoverageAggregatorBucket.BucketSize.Month;

    public ExtractionTimeTimeCoverageAggregator(ICatalogue catalogue, IExtractableCohort cohort)
    {
        _catalogue = catalogue;

        Buckets = new Dictionary<DateTime, ExtractionTimeTimeCoverageAggregatorBucket>();

        if (!catalogue.TimeCoverage_ExtractionInformation_ID.HasValue)
            return;
        try
        {
            _expectedTimeFieldInOutputBuffer = catalogue.TimeCoverage_ExtractionInformation.GetRuntimeName();
        }
        catch (Exception e)
        {
            throw new Exception(
                $"Could not resolve TimeCoverage_ExtractionInformation for Catalogue '{catalogue}',time coverage ExtractionInformationID was:{catalogue.TimeCoverage_ExtractionInformation_ID}",
                e);
        }

        _expectedExtractionIdentifierInOutputBuffer =
            cohort.GetQuerySyntaxHelper().GetRuntimeName(cohort.GetReleaseIdentifier());
    }

    private bool firstRow = true;

    public void ProcessRow(DataRow row)
    {
        //there is no configured time period field for the dataset, we have already thrown so just ignore calls
        if (string.IsNullOrWhiteSpace(_expectedTimeFieldInOutputBuffer))
            return;

        //check that the listed extraction field that has timeliness information in it is actually included in the column collection
        if (!haveCheckedTimeFieldExists)
            if (!row.Table.Columns.Contains(_expectedTimeFieldInOutputBuffer))
                throw new MissingFieldException(
                    $"Catalogue {_catalogue.Name} specifies that the time periodicity field of the dataset is called {_expectedTimeFieldInOutputBuffer} but the pipeline did not contain a field with this name, the fields in the pipeline are ({string.Join(",", row.Table.Columns.Cast<DataColumn>().Select(c => c.ColumnName))})");
            else
                haveCheckedTimeFieldExists = true;

        var value = row[_expectedTimeFieldInOutputBuffer];

        if (value == DBNull.Value)
        {
            countOfNullsSeen++;
            return;
        }

        DateTime key;

        object identifier = null;

        try
        {
            if (_expectedExtractionIdentifierInOutputBuffer != null)
                identifier = row[_expectedExtractionIdentifierInOutputBuffer];
        }
        catch (Exception)
        {
            //could not find the identifier in the output buffer, could be that there are multiple CHI columns e.g. CHI_Baby1, CHI_Baby2 or something
            //only swallow this exception (and abandon counting of distinct release identifiers) if it is the first output row.
            if (firstRow)
                _expectedExtractionIdentifierInOutputBuffer =
                    null; //give up trying to work out the extraction identifier
            else
                throw;
        }

        firstRow = false;

        try
        {
            if (value is string s)
            {
                if (string.IsNullOrWhiteSpace(s))
                {
                    countOfNullsSeen++;
                    return;
                }

                var valueAsString = s;

                //trim off times
                if (TimeRegex().IsMatch(valueAsString))
                    valueAsString = valueAsString[..^"00:00:00".Length].Trim();

                key = DateTime.ParseExact(valueAsString, "dd/MM/yyyy", null);
            }
            else if (value is DateTime time)
            {
                key = time;
            }
            else
            {
                key = Convert.ToDateTime(value);
            }
        }
        catch (Exception)
        {
            countOfBrokenDates++;
            return;
        }

        //drop the time part
        key = ExtractionTimeTimeCoverageAggregatorBucket.RoundDateTimeDownToNearestBucketFloor(key, BucketSize);

        if (!Buckets.Any())
        {
            //no buckets yet, create a bucket here
            Buckets.Add(key, new ExtractionTimeTimeCoverageAggregatorBucket(key));
            Buckets[key].SawIdentifier(identifier);
        }
        else
        {
            if (!Buckets.ContainsKey(key))
                ExtendBucketsToEncompas(key);

            Buckets[key].SawIdentifier(identifier);
        }
    }

    /// <summary>
    /// we want a consistent line of buckets with difference of BucketSize.  Use this method when you identify a DateTime that is (after
    /// rounding to the BucketSize, not in Buckets.  This method will add additional buckets up until the key has been reached (either
    /// adding new higher buckets if the key is > than the current maximum or adding additional lower buckets if it is lower.
    /// </summary>
    /// <param name="key"></param>
    private void ExtendBucketsToEncompas(DateTime key)
    {
        var addCount = 0;
        var somethingHasGoneWrongIfAddedThisManyBuckets = 100000;

        if (Buckets.Keys.Max() < key)
        {
            //extend upwards
            var toAdd = Buckets.Keys.Max();

            while (toAdd != key && addCount < somethingHasGoneWrongIfAddedThisManyBuckets)
            {
                toAdd = ExtractionTimeTimeCoverageAggregatorBucket.IncreaseDateTimeBy(toAdd, BucketSize);
                Buckets.Add(toAdd, new ExtractionTimeTimeCoverageAggregatorBucket(toAdd));
                addCount++;
            }
        }
        else if (Buckets.Keys.Min() > key)
        {
            //extend downwards
            var toAdd = Buckets.Keys.Min();

            while (toAdd != key && addCount < somethingHasGoneWrongIfAddedThisManyBuckets)
            {
                toAdd = ExtractionTimeTimeCoverageAggregatorBucket.DecreaseDateTimeBy(toAdd, BucketSize);
                Buckets.Add(toAdd, new ExtractionTimeTimeCoverageAggregatorBucket(toAdd));
                addCount++;
            }
        }
        else
        {
            throw new Exception(
                $"Could not work out how to extend aggregation buckets where dictionary of datetimes had a max of {Buckets.Keys.Max()} and a min of{Buckets.Keys.Min()} which could not be extended in either direction to encompass {key}");
        }

        if (addCount == somethingHasGoneWrongIfAddedThisManyBuckets)
            throw new Exception($"Added {addCount} buckets without reaching the key... oops");
    }

    public static bool HasColumn(DbDataReader Reader, string ColumnName)
    {
        return Reader.GetSchemaTable().Rows.Cast<DataRow>().Any(row => row["ColumnName"].ToString() == ColumnName);
    }

    [GeneratedRegex("[0-2][0-9]:[0-5][0-9]:[0-5][0-9]")]
    private static partial Regex TimeRegex();
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data;
10	using System.Data.Common;
11	using System.Linq;
12	using System.Text.RegularExpressions;
13	using Rdmp.Core.Curation.Data;
14	using Rdmp.Core.DataExport.Data;
15
16	namespace Rdmp.Core.DataExport.DataExtraction;
17
18	/// <summary>
19	/// Counts the number of unique patients / records encountered while executing an ExtractionConfiguration's dataset (linked to the project cohort). The input
20	/// to this is a DataRow rather than an SQL query because the class is used at extraction time as we are writing records out to the ExtractionDirectory.
21	/// </summary>
22	public partial class ExtractionTimeTimeCoverageAggregator
23	{
24	public int countOfBrokenDates { get; private set; }	×
25	public int countOfNullsSeen { get; private set; }	×
26	public Dictionary<DateTime, ExtractionTimeTimeCoverageAggregatorBucket> Buckets { get; private set; }	×
27
28	private readonly ICatalogue _catalogue;
29	private string _expectedTimeFieldInOutputBuffer;
30	private string _expectedExtractionIdentifierInOutputBuffer;
31
32	private bool haveCheckedTimeFieldExists;
33
34
35	private const ExtractionTimeTimeCoverageAggregatorBucket.BucketSize BucketSize =
36	ExtractionTimeTimeCoverageAggregatorBucket.BucketSize.Month;
37
38	public ExtractionTimeTimeCoverageAggregator(ICatalogue catalogue, IExtractableCohort cohort)	×
39	{
40	_catalogue = catalogue;	×
41
42	Buckets = new Dictionary<DateTime, ExtractionTimeTimeCoverageAggregatorBucket>();	×
43
44	if (!catalogue.TimeCoverage_ExtractionInformation_ID.HasValue)	×
45	return;	×
46	try
47	{
48	_expectedTimeFieldInOutputBuffer = catalogue.TimeCoverage_ExtractionInformation.GetRuntimeName();	×
49	}	×
50	catch (Exception e)	×
51	{
52	throw new Exception(	×
53	$"Could not resolve TimeCoverage_ExtractionInformation for Catalogue '{catalogue}',time coverage ExtractionInformationID was:{catalogue.TimeCoverage_ExtractionInformation_ID}",	×
54	e);	×
55	}
56
57	_expectedExtractionIdentifierInOutputBuffer =	×
58	cohort.GetQuerySyntaxHelper().GetRuntimeName(cohort.GetReleaseIdentifier());	×
59	}	×
60
61	private bool firstRow = true;	×
62
63	public void ProcessRow(DataRow row)
64	{
65	//there is no configured time period field for the dataset, we have already thrown so just ignore calls
66	if (string.IsNullOrWhiteSpace(_expectedTimeFieldInOutputBuffer))	×
67	return;	×
68
69	//check that the listed extraction field that has timeliness information in it is actually included in the column collection
70	if (!haveCheckedTimeFieldExists)	×
71	if (!row.Table.Columns.Contains(_expectedTimeFieldInOutputBuffer))	×
72	throw new MissingFieldException(	×
73	$"Catalogue {_catalogue.Name} specifies that the time periodicity field of the dataset is called {_expectedTimeFieldInOutputBuffer} but the pipeline did not contain a field with this name, the fields in the pipeline are ({string.Join(",", row.Table.Columns.Cast<DataColumn>().Select(c => c.ColumnName))})");	×
74	else
75	haveCheckedTimeFieldExists = true;	×
76
77	var value = row[_expectedTimeFieldInOutputBuffer];	×
78
79	if (value == DBNull.Value)	×
80	{
81	countOfNullsSeen++;	×
82	return;	×
83	}
84
85	DateTime key;
86
87	object identifier = null;	×
88
89	try
90	{
91	if (_expectedExtractionIdentifierInOutputBuffer != null)	×
92	identifier = row[_expectedExtractionIdentifierInOutputBuffer];	×
93	}	×
94	catch (Exception)	×
95	{
96	//could not find the identifier in the output buffer, could be that there are multiple CHI columns e.g. CHI_Baby1, CHI_Baby2 or something
97	//only swallow this exception (and abandon counting of distinct release identifiers) if it is the first output row.
98	if (firstRow)	×
99	_expectedExtractionIdentifierInOutputBuffer =	×
100	null; //give up trying to work out the extraction identifier	×
101	else
102	throw;	×
103	}	×
104
105	firstRow = false;	×
106
107	try
108	{
109	if (value is string s)	×
110	{
111	if (string.IsNullOrWhiteSpace(s))	×
112	{
113	countOfNullsSeen++;	×
114	return;	×
115	}
116
117	var valueAsString = s;	×
118
119	//trim off times
120	if (TimeRegex().IsMatch(valueAsString))	×
121	valueAsString = valueAsString[..^"00:00:00".Length].Trim();	×
122
123	key = DateTime.ParseExact(valueAsString, "dd/MM/yyyy", null);	×
124	}
125	else if (value is DateTime time)	×
126	{
127	key = time;	×
128	}
129	else
130	{
131	key = Convert.ToDateTime(value);	×
132	}
133	}	×
134	catch (Exception)	×
135	{
136	countOfBrokenDates++;	×
137	return;	×
138	}
139
140	//drop the time part
141	key = ExtractionTimeTimeCoverageAggregatorBucket.RoundDateTimeDownToNearestBucketFloor(key, BucketSize);	×
142
143	if (!Buckets.Any())	×
144	{
145	//no buckets yet, create a bucket here
146	Buckets.Add(key, new ExtractionTimeTimeCoverageAggregatorBucket(key));	×
147	Buckets[key].SawIdentifier(identifier);	×
148	}
149	else
150	{
151	if (!Buckets.ContainsKey(key))	×
152	ExtendBucketsToEncompas(key);	×
153
154	Buckets[key].SawIdentifier(identifier);	×
155	}
156	}	×
157
158	/// <summary>
159	/// we want a consistent line of buckets with difference of BucketSize. Use this method when you identify a DateTime that is (after
160	/// rounding to the BucketSize, not in Buckets. This method will add additional buckets up until the key has been reached (either
161	/// adding new higher buckets if the key is > than the current maximum or adding additional lower buckets if it is lower.
162	/// </summary>
163	/// <param name="key"></param>
164	private void ExtendBucketsToEncompas(DateTime key)
165	{
166	var addCount = 0;	×
167	var somethingHasGoneWrongIfAddedThisManyBuckets = 100000;	×
168
169	if (Buckets.Keys.Max() < key)	×
170	{
171	//extend upwards
172	var toAdd = Buckets.Keys.Max();	×
173
174	while (toAdd != key && addCount < somethingHasGoneWrongIfAddedThisManyBuckets)	×
175	{
176	toAdd = ExtractionTimeTimeCoverageAggregatorBucket.IncreaseDateTimeBy(toAdd, BucketSize);	×
177	Buckets.Add(toAdd, new ExtractionTimeTimeCoverageAggregatorBucket(toAdd));	×
178	addCount++;	×
179	}
180	}
181	else if (Buckets.Keys.Min() > key)	×
182	{
183	//extend downwards
184	var toAdd = Buckets.Keys.Min();	×
185
186	while (toAdd != key && addCount < somethingHasGoneWrongIfAddedThisManyBuckets)	×
187	{
188	toAdd = ExtractionTimeTimeCoverageAggregatorBucket.DecreaseDateTimeBy(toAdd, BucketSize);	×
189	Buckets.Add(toAdd, new ExtractionTimeTimeCoverageAggregatorBucket(toAdd));	×
190	addCount++;	×
191	}
192	}
193	else
194	{
195	throw new Exception(	×
196	$"Could not work out how to extend aggregation buckets where dictionary of datetimes had a max of {Buckets.Keys.Max()} and a min of{Buckets.Keys.Min()} which could not be extended in either direction to encompass {key}");	×
197	}
198
199	if (addCount == somethingHasGoneWrongIfAddedThisManyBuckets)	×
200	throw new Exception($"Added {addCount} buckets without reaching the key... oops");	×
201	}	×
202
203	public static bool HasColumn(DbDataReader Reader, string ColumnName)
204	{
205	return Reader.GetSchemaTable().Rows.Cast<DataRow>().Any(row => row["ColumnName"].ToString() == ColumnName);	×
206	}
207
208	[GeneratedRegex("[0-2][0-9]:[0-5][0-9]:[0-5][0-9]")]
209	private static partial Regex TimeRegex();
210	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous