6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.73

/Rdmp.Core/DataLoad/Modules/DataFlowOperations/SetNull.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.


using System;
using System.Data;
using System.Diagnostics;
using System.Text.RegularExpressions;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations;

internal class SetNull : IPluginDataFlowComponent<DataTable>
{
    [DemandsInitialization("Looks for a column with exactly this name", Mandatory = true)]
    public string ColumnNameToFind { get; set; }

    [DemandsInitialization(
        "Deletes all rows where the values in the specified ColumnNameToFind match the StandardRegex")]
    public StandardRegex NullCellsWhereValuesMatchStandard { get; set; }

    [DemandsInitialization("Deletes all rows where the values in the specified ColumnNameToFind match the Regex")]
    public Regex NullCellsWhereValuesMatch { get; set; }

    private int _changes;
    private Stopwatch _sw = new();

    public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener,
        GracefulCancellationToken cancellationToken)
    {
        _sw.Start();

        var regex = NullCellsWhereValuesMatch ?? new Regex(NullCellsWhereValuesMatchStandard.Regex);

        foreach (DataRow row in toProcess.Rows)
        {
            var val = row[ColumnNameToFind];

            //keep nulls, dbnulls or anything where ToString doesn't match the regex
            if (val != null && val != DBNull.Value && regex.IsMatch(val.ToString()))
            {
                row[ColumnNameToFind] = DBNull.Value;
                _changes++;
            }
        }

        listener.OnProgress(this,
            new ProgressEventArgs("SetNull Rows", new ProgressMeasurement(_changes, ProgressType.Records),
                _sw.Elapsed));

        _sw.Stop();
        return toProcess;
    }

    public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
    {
        listener.OnNotify(this,
            new NotifyEventArgs(_changes > 0 ? ProgressEventType.Warning : ProgressEventType.Information,
                $"Total SetNull operations for ColumnNameToFind '{ColumnNameToFind}' was {_changes}"));
    }

    public void Abort(IDataLoadEventListener listener)
    {
    }

    public void Check(ICheckNotifier notifier)
    {
        if (NullCellsWhereValuesMatch == null && NullCellsWhereValuesMatchStandard == null)
            notifier.OnCheckPerformed(new CheckEventArgs("You must specify a Regex for value selection",
                CheckResult.Fail));
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7
8	using System;
9	using System.Data;
10	using System.Diagnostics;
11	using System.Text.RegularExpressions;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.DataFlowPipeline;
14	using Rdmp.Core.ReusableLibraryCode.Checks;
15	using Rdmp.Core.ReusableLibraryCode.Progress;
16
17	namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations;
18
19	internal class SetNull : IPluginDataFlowComponent<DataTable>
20	{
21	[DemandsInitialization("Looks for a column with exactly this name", Mandatory = true)]
22	public string ColumnNameToFind { get; set; }	12✔
23
24	[DemandsInitialization(
25	"Deletes all rows where the values in the specified ColumnNameToFind match the StandardRegex")]
26	public StandardRegex NullCellsWhereValuesMatchStandard { get; set; }	×
27
28	[DemandsInitialization("Deletes all rows where the values in the specified ColumnNameToFind match the Regex")]
29	public Regex NullCellsWhereValuesMatch { get; set; }	4✔
30
31	private int _changes;
32	private Stopwatch _sw = new();	2✔
33
34	public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener,
35	GracefulCancellationToken cancellationToken)
36	{
37	_sw.Start();	2✔
38
39	var regex = NullCellsWhereValuesMatch ?? new Regex(NullCellsWhereValuesMatchStandard.Regex);	2!
40
41	foreach (DataRow row in toProcess.Rows)	16✔
42	{
43	var val = row[ColumnNameToFind];	6✔
44
45	//keep nulls, dbnulls or anything where ToString doesn't match the regex
46	if (val != null && val != DBNull.Value && regex.IsMatch(val.ToString()))	6✔
47	{
48	row[ColumnNameToFind] = DBNull.Value;	2✔
49	_changes++;	2✔
50	}
51	}
52
53	listener.OnProgress(this,	2✔
54	new ProgressEventArgs("SetNull Rows", new ProgressMeasurement(_changes, ProgressType.Records),	2✔
55	_sw.Elapsed));	2✔
56
57	_sw.Stop();	2✔
58	return toProcess;	2✔
59	}
60
61	public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
62	{
63	listener.OnNotify(this,	2!
64	new NotifyEventArgs(_changes > 0 ? ProgressEventType.Warning : ProgressEventType.Information,	2✔
65	$"Total SetNull operations for ColumnNameToFind '{ColumnNameToFind}' was {_changes}"));	2✔
66	}	2✔
67
68	public void Abort(IDataLoadEventListener listener)
69	{
70	}	×
71
72	public void Check(ICheckNotifier notifier)
73	{
74	if (NullCellsWhereValuesMatch == null && NullCellsWhereValuesMatchStandard == null)	×
75	notifier.OnCheckPerformed(new CheckEventArgs("You must specify a Regex for value selection",	×
76	CheckResult.Fail));	×
77	}	×
78	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous