6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.61

/Rdmp.Core/DataLoad/Modules/Attachers/FixedWidthFormatFile.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Data;
using System.IO;
using System.Linq;
using Rdmp.Core.DataLoad.Modules.Exceptions;

namespace Rdmp.Core.DataLoad.Modules.Attachers;

/// <summary>
/// User generated file describing the layout of a fixed width file (See FixedWidthAttacher).  Includes the character positions of each named field and date
/// format (where applicable).
/// </summary>
public class FixedWidthFormatFile
{
    private readonly FileInfo _pathToFormatFile;

    public FixedWidthColumn[] FormatColumns { get; }

    public FixedWidthFormatFile(FileInfo pathToFormatFile)
    {
        _pathToFormatFile = pathToFormatFile;
        var readAllLines = File.ReadAllLines(_pathToFormatFile.FullName);

        var headers = readAllLines[0];

        EnsureHeaderIntact(headers);

        //get rid of blank lines
        readAllLines = readAllLines.Where(l => !string.IsNullOrWhiteSpace(l)).ToArray();

        //create a format field for each line in the format file
        FormatColumns = new FixedWidthColumn[readAllLines.Length - 1];

        //now add values
        for (var index = 0; index < readAllLines.Length - 1; index++)
        {
            //skip header line
            var cellsOnRowAsSplitString = readAllLines[index + 1].Split(',');

            FormatColumns[index].From = int.Parse(cellsOnRowAsSplitString[0]);
            FormatColumns[index].To = int.Parse(cellsOnRowAsSplitString[1]);
            FormatColumns[index].Field = cellsOnRowAsSplitString[2];
            FormatColumns[index].Size = int.Parse(cellsOnRowAsSplitString[3]);

            //It's ok to omit this column for specific rows (that aren't dates)
            if (cellsOnRowAsSplitString.Length > 4)
                FormatColumns[index].DateFormat =
                    cellsOnRowAsSplitString[4]
                        .Replace("ccyy",
                            "yyyy"); //some people think that ccyy is a valid way of expressing year formats... they are wrong

            if (FormatColumns[index].From + FormatColumns[index].Size - 1 != FormatColumns[index].To)
                throw new FlatFileLoadException(
                    $"Problem with format of field {FormatColumns[index].Field} From + Size -1 does not equal To");

            if (!string.IsNullOrWhiteSpace(FormatColumns[index].DateFormat))
                try
                {
                    DateTime.Now.ToString(FormatColumns[index].DateFormat);
                }
                catch (Exception e)
                {
                    throw new FlatFileLoadException(
                        $"Problem with flat file format which announced the date format as {FormatColumns[index].DateFormat} which C# says isn't a valid format",
                        e);
                }
        }
    }


    public DataTable GetDataTableFromFlatFile(FileInfo f)
    {
        //setup the table
        DataTable toReturn = new DataTable();

        toReturn.BeginLoadData();
        foreach (var fixedWidthColumn in FormatColumns)
        {
            var dataColumn = toReturn.Columns.Add(fixedWidthColumn.Field);

            if (!string.IsNullOrWhiteSpace(fixedWidthColumn.DateFormat))
                dataColumn.DataType = typeof(DateTime);

            dataColumn.AllowDBNull = true;
        }

        var lineNumber = 0;

        //populate the table
        //foreach line in file
        foreach (var readAllLine in File.ReadLines(f.FullName))
        {
            lineNumber++;

            //add a new row to data table
            var dataRow = toReturn.Rows.Add();

            //foreach expected fixed width column
            foreach (var fixedWidthColumn in FormatColumns)
            {
                if (readAllLine.Length < fixedWidthColumn.To)
                    throw new FlatFileLoadException(
                        $"Error on line {lineNumber} of file {f.Name}, the format file ({_pathToFormatFile.FullName}) specified that a column {fixedWidthColumn.Field} would be found between character positions {fixedWidthColumn.From} and {fixedWidthColumn.To} but the current line is only {readAllLine.Length} characters long");

                //substring in order to get cell data
                var value = readAllLine.Substring(fixedWidthColumn.From - 1, fixedWidthColumn.Size);

                //if it's a null
                if (string.IsNullOrWhiteSpace(value))
                    dataRow[fixedWidthColumn.Field] = DBNull.Value;
                else
                //it is a date column
                if (!string.IsNullOrWhiteSpace(fixedWidthColumn.DateFormat))
                    try
                    {
                        dataRow[fixedWidthColumn.Field] = DateTime.ParseExact(value, fixedWidthColumn.DateFormat, null);
                    }
                    catch (Exception e)
                    {
                        throw new Exception(
                            $"The value '{value}' was rejected by DateTime.ParseExact using the listed date time format '{fixedWidthColumn.DateFormat}'",
                            e);
                    }
                else //it's not a date
                    dataRow[fixedWidthColumn.Field] = value.Trim();
            }
        }

        toReturn.EndLoadData();
        return toReturn;
    }

    private void EnsureHeaderIntact(string header)
    {
        //From        To        Field        Size        DateFormat
        var expected = string.Join(",", typeof(FixedWidthColumn).GetFields().Select(f => f.Name));

        if (!header.TrimEnd().Equals(expected))
            throw new FlatFileLoadException(
                $"Format file headers in file {_pathToFormatFile.FullName} WAS: {header} WE EXPECTED: {expected}");
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Data;
9	using System.IO;
10	using System.Linq;
11	using Rdmp.Core.DataLoad.Modules.Exceptions;
12
13	namespace Rdmp.Core.DataLoad.Modules.Attachers;
14
15	/// <summary>
16	/// User generated file describing the layout of a fixed width file (See FixedWidthAttacher). Includes the character positions of each named field and date
17	/// format (where applicable).
18	/// </summary>
19	public class FixedWidthFormatFile
20	{
21	private readonly FileInfo _pathToFormatFile;
22
23	public FixedWidthColumn[] FormatColumns { get; }	486✔
24
25	public FixedWidthFormatFile(FileInfo pathToFormatFile)	10✔
26	{
27	_pathToFormatFile = pathToFormatFile;	10✔
28	var readAllLines = File.ReadAllLines(_pathToFormatFile.FullName);	10✔
29
30	var headers = readAllLines[0];	10✔
31
32	EnsureHeaderIntact(headers);	10✔
33
34	//get rid of blank lines
35	readAllLines = readAllLines.Where(l => !string.IsNullOrWhiteSpace(l)).ToArray();	58✔
36
37	//create a format field for each line in the format file
38	FormatColumns = new FixedWidthColumn[readAllLines.Length - 1];	10✔
39
40	//now add values
41	for (var index = 0; index < readAllLines.Length - 1; index++)	96✔
42	{
43	//skip header line
44	var cellsOnRowAsSplitString = readAllLines[index + 1].Split(',');	38✔
45
46	FormatColumns[index].From = int.Parse(cellsOnRowAsSplitString[0]);	38✔
47	FormatColumns[index].To = int.Parse(cellsOnRowAsSplitString[1]);	38✔
48	FormatColumns[index].Field = cellsOnRowAsSplitString[2];	38✔
49	FormatColumns[index].Size = int.Parse(cellsOnRowAsSplitString[3]);	38✔
50
51	//It's ok to omit this column for specific rows (that aren't dates)
52	if (cellsOnRowAsSplitString.Length > 4)	38✔
53	FormatColumns[index].DateFormat =	32✔
54	cellsOnRowAsSplitString[4]	32✔
55	.Replace("ccyy",	32✔
56	"yyyy"); //some people think that ccyy is a valid way of expressing year formats... they are wrong	32✔
57
58	if (FormatColumns[index].From + FormatColumns[index].Size - 1 != FormatColumns[index].To)	38!
59	throw new FlatFileLoadException(	×
60	$"Problem with format of field {FormatColumns[index].Field} From + Size -1 does not equal To");	×
61
62	if (!string.IsNullOrWhiteSpace(FormatColumns[index].DateFormat))	38✔
63	try
64	{
65	DateTime.Now.ToString(FormatColumns[index].DateFormat);	8✔
66	}	8✔
67	catch (Exception e)	×
68	{
69	throw new FlatFileLoadException(	×
70	$"Problem with flat file format which announced the date format as {FormatColumns[index].DateFormat} which C# says isn't a valid format",	×
71	e);	×
72	}
73	}
74	}	10✔
75
76
77	public DataTable GetDataTableFromFlatFile(FileInfo f)
78	{
79	//setup the table
80	DataTable toReturn = new DataTable();	8✔
81
82	toReturn.BeginLoadData();	8✔
83	foreach (var fixedWidthColumn in FormatColumns)	60✔
84	{
85	var dataColumn = toReturn.Columns.Add(fixedWidthColumn.Field);	22✔
86
87	if (!string.IsNullOrWhiteSpace(fixedWidthColumn.DateFormat))	22✔
88	dataColumn.DataType = typeof(DateTime);	4✔
89
90	dataColumn.AllowDBNull = true;	22✔
91	}
92
93	var lineNumber = 0;	8✔
94
95	//populate the table
96	//foreach line in file
97	foreach (var readAllLine in File.ReadLines(f.FullName))	46✔
98	{
99	lineNumber++;	16✔
100
101	//add a new row to data table
102	var dataRow = toReturn.Rows.Add();	16✔
103
104	//foreach expected fixed width column
105	foreach (var fixedWidthColumn in FormatColumns)	118✔
106	{
107	if (readAllLine.Length < fixedWidthColumn.To)	44✔
108	throw new FlatFileLoadException(	2✔
109	$"Error on line {lineNumber} of file {f.Name}, the format file ({_pathToFormatFile.FullName}) specified that a column {fixedWidthColumn.Field} would be found between character positions {fixedWidthColumn.From} and {fixedWidthColumn.To} but the current line is only {readAllLine.Length} characters long");	2✔
110
111	//substring in order to get cell data
112	var value = readAllLine.Substring(fixedWidthColumn.From - 1, fixedWidthColumn.Size);	42✔
113
114	//if it's a null
115	if (string.IsNullOrWhiteSpace(value))	42!
116	dataRow[fixedWidthColumn.Field] = DBNull.Value;	×
117	else
118	//it is a date column
119	if (!string.IsNullOrWhiteSpace(fixedWidthColumn.DateFormat))	42✔
120	try
121	{
122	dataRow[fixedWidthColumn.Field] = DateTime.ParseExact(value, fixedWidthColumn.DateFormat, null);	8✔
123	}	8✔
124	catch (Exception e)	×
125	{
126	throw new Exception(	×
127	$"The value '{value}' was rejected by DateTime.ParseExact using the listed date time format '{fixedWidthColumn.DateFormat}'",	×
128	e);	×
129	}
130	else //it's not a date
131	dataRow[fixedWidthColumn.Field] = value.Trim();	34✔
132	}
133	}
134
135	toReturn.EndLoadData();	6✔
136	return toReturn;	6✔
137	}
138
139	private void EnsureHeaderIntact(string header)
140	{
141	//From To Field Size DateFormat
142	var expected = string.Join(",", typeof(FixedWidthColumn).GetFields().Select(f => f.Name));	60✔
143
144	if (!header.TrimEnd().Equals(expected))	10!
145	throw new FlatFileLoadException(	×
146	$"Format file headers in file {_pathToFormatFile.FullName} WAS: {header} WE EXPECTED: {expected}");	×
147	}	10✔
148	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous