6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

40.43

/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ColumnForbidder.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Data;
using System.Linq;
using System.Text.RegularExpressions;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations;

/// <summary>
/// Pipeline component designed to prevent unwanted data existing within DataTables passing through the pipeline.  The component will crash the entire pipeline
/// if it sees columns which match the forbidlist.  Use cases for this include when the user wants to prevent private identifiers being accidentally released
/// due to system misconfiguration e.g. you might forbidlist all columns containing the strings starting "Patient" on the grounds that they are likely to be
/// identifiable (PatientName, PatientDob etc).
/// 
/// <para>Crashes the pipeline if any column matches the regex e.g. '^(mCHI)|(chi)$'</para>
/// </summary>
public class ColumnForbidder : IPluginDataFlowComponent<DataTable>
{
    private Regex _crashIfAnyColumnMatches;

    [DemandsInitialization("Crashes the load if any column name matches this regex")]
    public Regex CrashIfAnyColumnMatches
    {
        get => _crashIfAnyColumnMatches;
        set => _crashIfAnyColumnMatches = new Regex(value.ToString(), value.Options | RegexOptions.IgnoreCase);
    }

    [DemandsInitialization(
        "Alternative to specifying a Regex pattern in CrashIfAnyColumnMatches.  Select an existing StandardRegex.  This has the advantage of centralising the concept.  See StandardRegexUI for configuring StandardRegexes")]
    public StandardRegex StandardRegex { get; set; }

    [DemandsInitialization(
        "Crash message (if any) to explain why columns matching the Regex are a problem e.g. 'Patient telephone numbers should never be extracted!'")]
    public string Rationale { get; set; }

    private Regex _reCache;

    public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener,
        GracefulCancellationToken cancellationToken)
    {
        var checkPattern = _reCache ??= _crashIfAnyColumnMatches ?? new Regex(GetPattern(), RegexOptions.IgnoreCase);

        foreach (var c in toProcess.Columns.Cast<DataColumn>().Select(c => c.ColumnName))
            if (checkPattern.IsMatch(c))
                if (string.IsNullOrWhiteSpace(Rationale))
                    throw new Exception($"Column {c} matches forbidlist regex");
                else
                    throw new Exception(
                        $"{Rationale}{Environment.NewLine}Exception generated because Column {c} matches forbidlist regex");

        return toProcess;
    }

    public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
    {
    }

    public void Abort(IDataLoadEventListener listener)
    {
    }

    public void Check(ICheckNotifier notifier)
    {
        try
        {
            var p = GetPattern();
            _ = new Regex(p);
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Problem occurred getting Regex pattern for forbidlist",
                CheckResult.Fail, e));
        }
    }

    private string GetPattern()
    {
        string pattern = null;

        if (CrashIfAnyColumnMatches != null)
            pattern = CrashIfAnyColumnMatches.ToString();
        else if (StandardRegex != null)
            pattern = StandardRegex.Regex;


        return string.IsNullOrWhiteSpace(pattern)
            ? throw new Exception(
                "You must specify either a pattern in CrashIfAnyColumnMatches or pick an existing StandardRegex with a pattern to match on")
            : pattern;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Data;
9	using System.Linq;
10	using System.Text.RegularExpressions;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.DataFlowPipeline;
13	using Rdmp.Core.ReusableLibraryCode.Checks;
14	using Rdmp.Core.ReusableLibraryCode.Progress;
15
16	namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations;
17
18	/// <summary>
19	/// Pipeline component designed to prevent unwanted data existing within DataTables passing through the pipeline. The component will crash the entire pipeline
20	/// if it sees columns which match the forbidlist. Use cases for this include when the user wants to prevent private identifiers being accidentally released
21	/// due to system misconfiguration e.g. you might forbidlist all columns containing the strings starting "Patient" on the grounds that they are likely to be
22	/// identifiable (PatientName, PatientDob etc).
23	///
24	/// <para>Crashes the pipeline if any column matches the regex e.g. '^(mCHI)\|(chi)$'</para>
25	/// </summary>
26	public class ColumnForbidder : IPluginDataFlowComponent<DataTable>
27	{
28	private Regex _crashIfAnyColumnMatches;
29
30	[DemandsInitialization("Crashes the load if any column name matches this regex")]
31	public Regex CrashIfAnyColumnMatches
32	{
33	get => _crashIfAnyColumnMatches;	×
34	set => _crashIfAnyColumnMatches = new Regex(value.ToString(), value.Options \| RegexOptions.IgnoreCase);	2✔
35	}
36
37	[DemandsInitialization(
38	"Alternative to specifying a Regex pattern in CrashIfAnyColumnMatches. Select an existing StandardRegex. This has the advantage of centralising the concept. See StandardRegexUI for configuring StandardRegexes")]
39	public StandardRegex StandardRegex { get; set; }	×
40
41	[DemandsInitialization(
42	"Crash message (if any) to explain why columns matching the Regex are a problem e.g. 'Patient telephone numbers should never be extracted!'")]
43	public string Rationale { get; set; }	8✔
44
45	private Regex _reCache;
46
47	public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener,
48	GracefulCancellationToken cancellationToken)
49	{
50	var checkPattern = _reCache ??= _crashIfAnyColumnMatches ?? new Regex(GetPattern(), RegexOptions.IgnoreCase);	6!
51
52	foreach (var c in toProcess.Columns.Cast<DataColumn>().Select(c => c.ColumnName))	38✔
53	if (checkPattern.IsMatch(c))	10✔
54	if (string.IsNullOrWhiteSpace(Rationale))	4✔
55	throw new Exception($"Column {c} matches forbidlist regex");	2✔
56	else
57	throw new Exception(	2✔
58	$"{Rationale}{Environment.NewLine}Exception generated because Column {c} matches forbidlist regex");	2✔
59
60	return toProcess;	2✔
61	}
62
63	public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
64	{
65	}	×
66
67	public void Abort(IDataLoadEventListener listener)
68	{
69	}	×
70
71	public void Check(ICheckNotifier notifier)
72	{
73	try
74	{
75	var p = GetPattern();	×
76	_ = new Regex(p);	×
77	}	×
78	catch (Exception e)	×
79	{
80	notifier.OnCheckPerformed(new CheckEventArgs("Problem occurred getting Regex pattern for forbidlist",	×
81	CheckResult.Fail, e));	×
82	}	×
83	}	×
84
85	private string GetPattern()
86	{
87	string pattern = null;	×
88
89	if (CrashIfAnyColumnMatches != null)	×
90	pattern = CrashIfAnyColumnMatches.ToString();	×
91	else if (StandardRegex != null)	×
92	pattern = StandardRegex.Regex;	×
93
94
95	return string.IsNullOrWhiteSpace(pattern)	×
96	? throw new Exception(	×
97	"You must specify either a pattern in CrashIfAnyColumnMatches or pick an existing StandardRegex with a pattern to match on")	×
98	: pattern;	×
99	}
100	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous