6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.49

/Rdmp.Core/DataLoad/Modules/DataFlowOperations/Transposer.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Data;
using FAnsi.Discovery;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Modules.DataFlowSources;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations;

/// <summary>
/// Pipeline component which rotates DataTables flowing through it by 90 degrees such that the first column becomes the new headers.  Only use this if you have
/// been given a file in which proper headers are vertical down the first column and records are subsequent columns (i.e. adding new records results in the
/// DataTable growing horizontally).
///
/// <para>IMPORTANT: Only works with a single load batch if you have a chunked pipeline you cannot use this component unless you set the chunk size large enough
/// to read the entire file in one go
/// </para>
/// </summary>
public class Transposer : IPluginDataFlowComponent<DataTable>
{
    private bool _haveServedResult;

    [DemandsInitialization(DelimitedFlatFileDataFlowSource.MakeHeaderNamesSane_DemandDescription,
        DemandType.Unspecified, true)]
    public bool MakeHeaderNamesSane { get; set; }

    public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener,
        GracefulCancellationToken cancellationToken)
    {
        if (toProcess == null)
            return null;

        if (_haveServedResult)
            throw new NotSupportedException(
                "Error, we received multiple batches, Transposer only works when all the data arrives in a single DataTable");

        if (toProcess.Rows.Count == 0 || toProcess.Columns.Count == 0)
            throw new NotSupportedException(
                $"DataTable toProcess had {toProcess.Rows.Count} rows and {toProcess.Columns.Count} columns, thus it cannot be transposed");

        _haveServedResult = true;

        return GenerateTransposedTable(toProcess);
    }

    public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
    {
    }

    public void Abort(IDataLoadEventListener listener)
    {
    }

    public void Check(ICheckNotifier notifier)
    {
    }

    private DataTable GenerateTransposedTable(DataTable inputTable)
    {
        var outputTable = new DataTable();
        outputTable.BeginLoadData();
        // Add columns by looping rows

        // Header row's first column is same as in inputTable
        outputTable.Columns.Add(inputTable.Columns[0].ColumnName);

        // Header row's second column onwards, 'inputTable's first column taken
        foreach (DataRow inRow in inputTable.Rows)
        {
            var newColName = inRow[0].ToString();

            if (MakeHeaderNamesSane)
                newColName = QuerySyntaxHelper.MakeHeaderNameSensible(newColName);

            outputTable.Columns.Add(newColName);
        }

        // Add rows by looping columns
        for (var rCount = 1; rCount <= inputTable.Columns.Count - 1; rCount++)
        {
            var newRow = outputTable.NewRow();

            // First column is inputTable's Header row's second column
            newRow[0] = inputTable.Columns[rCount].ColumnName;
            for (var cCount = 0; cCount <= inputTable.Rows.Count - 1; cCount++)
            {
                var colValue = inputTable.Rows[cCount][rCount].ToString();
                newRow[cCount + 1] = colValue;
            }

            outputTable.Rows.Add(newRow);
        }
        outputTable.EndLoadData();
        return outputTable;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Data;
9	using FAnsi.Discovery;
10	using Rdmp.Core.Curation.Data;
11	using Rdmp.Core.DataFlowPipeline;
12	using Rdmp.Core.DataLoad.Modules.DataFlowSources;
13	using Rdmp.Core.ReusableLibraryCode.Checks;
14	using Rdmp.Core.ReusableLibraryCode.Progress;
15
16	namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations;
17
18	/// <summary>
19	/// Pipeline component which rotates DataTables flowing through it by 90 degrees such that the first column becomes the new headers. Only use this if you have
20	/// been given a file in which proper headers are vertical down the first column and records are subsequent columns (i.e. adding new records results in the
21	/// DataTable growing horizontally).
22	///
23	/// <para>IMPORTANT: Only works with a single load batch if you have a chunked pipeline you cannot use this component unless you set the chunk size large enough
24	/// to read the entire file in one go
25	/// </para>
26	/// </summary>
27	public class Transposer : IPluginDataFlowComponent<DataTable>
28	{
29	private bool _haveServedResult;
30
31	[DemandsInitialization(DelimitedFlatFileDataFlowSource.MakeHeaderNamesSane_DemandDescription,
32	DemandType.Unspecified, true)]
33	public bool MakeHeaderNamesSane { get; set; }	646✔
34
35	public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener,
36	GracefulCancellationToken cancellationToken)
37	{
38	if (toProcess == null)	18!
39	return null;	×
40
41	if (_haveServedResult)	18✔
42	throw new NotSupportedException(	2✔
43	"Error, we received multiple batches, Transposer only works when all the data arrives in a single DataTable");	2✔
44
45	if (toProcess.Rows.Count == 0 \|\| toProcess.Columns.Count == 0)	16✔
46	throw new NotSupportedException(	2✔
47	$"DataTable toProcess had {toProcess.Rows.Count} rows and {toProcess.Columns.Count} columns, thus it cannot be transposed");	2✔
48
49	_haveServedResult = true;	14✔
50
51	return GenerateTransposedTable(toProcess);	14✔
52	}
53
54	public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
55	{
56	}	8✔
57
58	public void Abort(IDataLoadEventListener listener)
59	{
60	}	×
61
62	public void Check(ICheckNotifier notifier)
63	{
64	}	×
65
66	private DataTable GenerateTransposedTable(DataTable inputTable)
67	{
68	var outputTable = new DataTable();	14✔
69	outputTable.BeginLoadData();	14✔
70	// Add columns by looping rows
71
72	// Header row's first column is same as in inputTable
73	outputTable.Columns.Add(inputTable.Columns[0].ColumnName);	14✔
74
75	// Header row's second column onwards, 'inputTable's first column taken
76	foreach (DataRow inRow in inputTable.Rows)	1,300✔
77	{
78	var newColName = inRow[0].ToString();	636✔
79
80	if (MakeHeaderNamesSane)	636✔
81	newColName = QuerySyntaxHelper.MakeHeaderNameSensible(newColName);	8✔
82
83	outputTable.Columns.Add(newColName);	636✔
84	}
85
86	// Add rows by looping columns
87	for (var rCount = 1; rCount <= inputTable.Columns.Count - 1; rCount++)	112✔
88	{
89	var newRow = outputTable.NewRow();	42✔
90
91	// First column is inputTable's Header row's second column
92	newRow[0] = inputTable.Columns[rCount].ColumnName;	42✔
93	for (var cCount = 0; cCount <= inputTable.Rows.Count - 1; cCount++)	3,900✔
94	{
95	var colValue = inputTable.Rows[cCount][rCount].ToString();	1,908✔
96	newRow[cCount + 1] = colValue;	1,908✔
97	}
98
99	outputTable.Rows.Add(newRow);	42✔
100	}
101	outputTable.EndLoadData();	14✔
102	return outputTable;	14✔
103	}
104	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous