6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.84

/Rdmp.Core/DataLoad/Modules/DataFlowSources/SubComponents/FlatFileEventHandlers.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.IO;
using CsvHelper;
using Rdmp.Core.DataFlowPipeline.Requirements;
using Rdmp.Core.DataLoad.Modules.Exceptions;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Modules.DataFlowSources.SubComponents;

/// <summary>
/// This class is a sub component of <see cref="DelimitedFlatFileDataFlowSource"/>, it is responsible for responding to errors processing the file
/// being loaded according to the <see cref="BadDataHandlingStrategy"/>. It also includes settings for how to respond to empty files.
/// </summary>
public class FlatFileEventHandlers
{
    private readonly FlatFileToLoad _fileToLoad;
    private readonly FlatFileToDataTablePusher _dataPusher;
    private readonly bool _throwOnEmptyFiles;
    private readonly BadDataHandlingStrategy _strategy;
    private readonly IDataLoadEventListener _listener;
    private int _maximumErrorsToReport;
    private readonly bool _ignoreBadDataEvents;

    /// <summary>
    /// File where we put error rows
    /// </summary>
    public FileInfo DivertErrorsFile;

    public FlatFileEventHandlers(FlatFileToLoad fileToLoad, FlatFileToDataTablePusher dataPusher,
        bool throwOnEmptyFiles, BadDataHandlingStrategy strategy, IDataLoadEventListener listener,
        int maximumErrorsToReport, bool ignoreBadDataEvents)
    {
        _fileToLoad = fileToLoad;
        _dataPusher = dataPusher;
        _throwOnEmptyFiles = throwOnEmptyFiles;
        _strategy = strategy;
        _listener = listener;
        _maximumErrorsToReport = maximumErrorsToReport;
        _ignoreBadDataEvents = ignoreBadDataEvents;
    }

    public void FileIsEmpty()
    {
        if (_throwOnEmptyFiles)
            throw new FlatFileLoadException($"File {_fileToLoad} is empty");

        _listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"File {_fileToLoad} is empty"));
    }

    public bool ReadingExceptionOccurred(ReadingExceptionOccurredArgs args)
    {
        var line = new FlatFileLine(args.Exception.Context);

        switch (_strategy)
        {
            case BadDataHandlingStrategy.IgnoreRows:
                if (_maximumErrorsToReport-- > 0)
                    _listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                        $"Ignored ReadingException on {line.GetLineDescription()}", args.Exception));

                //move to next line
                _dataPusher.BadLines.Add(args.Exception.Context.Parser.RawRow);

                break;
            case BadDataHandlingStrategy.DivertRows:

                DivertErrorRow(new FlatFileLine(args.Exception.Context), args.Exception);
                break;

            case BadDataHandlingStrategy.ThrowException:
                throw new FlatFileLoadException($"Bad data found on li{line.GetLineDescription()}", args.Exception);

            default:
                throw new ArgumentOutOfRangeException();
        }

        //todo should this return true or false? not clear, this was an API change in CSV.
        return true;
    }

    public void BadDataFound(FlatFileLine line, bool isFromCsvHelper = false)
    {
        if (_ignoreBadDataEvents && isFromCsvHelper)
            if (_maximumErrorsToReport-- > 0)
            {
                _listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                    $"Ignoring CSVHelper internal bad data warning:{line.GetLineDescription()}"));
                return;
            }

        // if we have seen this bad line already
        if (_dataPusher.BadLines.Contains(line.LineNumber))
            return;

        switch (_strategy)
        {
            case BadDataHandlingStrategy.IgnoreRows:

                if (_maximumErrorsToReport-- > 0)
                    _listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                        $"Ignored BadData on {line.GetLineDescription()}"));

                //move to next line
                _dataPusher.BadLines.Add(line.LineNumber);

                break;
            case BadDataHandlingStrategy.DivertRows:
                DivertErrorRow(line, null);
                break;

            case BadDataHandlingStrategy.ThrowException:
                throw new FlatFileLoadException($"Bad data found on {line.GetLineDescription()}");


            default:
                throw new ArgumentOutOfRangeException();
        }
    }

    public void DivertErrorRow(FlatFileLine line, Exception ex)
    {
        if (DivertErrorsFile == null)
        {
            DivertErrorsFile = new FileInfo(Path.Combine(_fileToLoad.File.Directory.FullName,
                $"{Path.GetFileNameWithoutExtension(_fileToLoad.File.Name)}_Errors.txt"));

            //delete any old version
            if (DivertErrorsFile.Exists)
                DivertErrorsFile.Delete();
        }

        if (_maximumErrorsToReport-- > 0)
            _listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                $"Diverting Error on {line.GetLineDescription()} to '{DivertErrorsFile.FullName}'", ex));

        File.AppendAllText(DivertErrorsFile.FullName, line.RawRecord);

        //move to next line
        _dataPusher.BadLines.Add(line.LineNumber);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.IO;
9	using CsvHelper;
10	using Rdmp.Core.DataFlowPipeline.Requirements;
11	using Rdmp.Core.DataLoad.Modules.Exceptions;
12	using Rdmp.Core.ReusableLibraryCode.Progress;
13
14	namespace Rdmp.Core.DataLoad.Modules.DataFlowSources.SubComponents;
15
16	/// <summary>
17	/// This class is a sub component of <see cref="DelimitedFlatFileDataFlowSource"/>, it is responsible for responding to errors processing the file
18	/// being loaded according to the <see cref="BadDataHandlingStrategy"/>. It also includes settings for how to respond to empty files.
19	/// </summary>
20	public class FlatFileEventHandlers
21	{
22	private readonly FlatFileToLoad _fileToLoad;
23	private readonly FlatFileToDataTablePusher _dataPusher;
24	private readonly bool _throwOnEmptyFiles;
25	private readonly BadDataHandlingStrategy _strategy;
26	private readonly IDataLoadEventListener _listener;
27	private int _maximumErrorsToReport;
28	private readonly bool _ignoreBadDataEvents;
29
30	/// <summary>
31	/// File where we put error rows
32	/// </summary>
33	public FileInfo DivertErrorsFile;
34
35	public FlatFileEventHandlers(FlatFileToLoad fileToLoad, FlatFileToDataTablePusher dataPusher,	244✔
36	bool throwOnEmptyFiles, BadDataHandlingStrategy strategy, IDataLoadEventListener listener,	244✔
37	int maximumErrorsToReport, bool ignoreBadDataEvents)	244✔
38	{
39	_fileToLoad = fileToLoad;	244✔
40	_dataPusher = dataPusher;	244✔
41	_throwOnEmptyFiles = throwOnEmptyFiles;	244✔
42	_strategy = strategy;	244✔
43	_listener = listener;	244✔
44	_maximumErrorsToReport = maximumErrorsToReport;	244✔
45	_ignoreBadDataEvents = ignoreBadDataEvents;	244✔
46	}	244✔
47
48	public void FileIsEmpty()
49	{
50	if (_throwOnEmptyFiles)	16✔
51	throw new FlatFileLoadException($"File {_fileToLoad} is empty");	8✔
52
53	_listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"File {_fileToLoad} is empty"));	8✔
54	}	8✔
55
56	public bool ReadingExceptionOccurred(ReadingExceptionOccurredArgs args)
57	{
58	var line = new FlatFileLine(args.Exception.Context);	×
59
60	switch (_strategy)	×
61	{
62	case BadDataHandlingStrategy.IgnoreRows:
63	if (_maximumErrorsToReport-- > 0)	×
64	_listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	×
65	$"Ignored ReadingException on {line.GetLineDescription()}", args.Exception));	×
66
67	//move to next line
68	_dataPusher.BadLines.Add(args.Exception.Context.Parser.RawRow);	×
69
70	break;	×
71	case BadDataHandlingStrategy.DivertRows:
72
73	DivertErrorRow(new FlatFileLine(args.Exception.Context), args.Exception);	×
74	break;	×
75
76	case BadDataHandlingStrategy.ThrowException:
77	throw new FlatFileLoadException($"Bad data found on li{line.GetLineDescription()}", args.Exception);	×
78
79	default:
80	throw new ArgumentOutOfRangeException();	×
81	}
82
83	//todo should this return true or false? not clear, this was an API change in CSV.
84	return true;	×
85	}
86
87	public void BadDataFound(FlatFileLine line, bool isFromCsvHelper = false)
88	{
89	if (_ignoreBadDataEvents && isFromCsvHelper)	90✔
90	if (_maximumErrorsToReport-- > 0)	6✔
91	{
92	_listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	6✔
93	$"Ignoring CSVHelper internal bad data warning:{line.GetLineDescription()}"));	6✔
94	return;	6✔
95	}
96
97	// if we have seen this bad line already
98	if (_dataPusher.BadLines.Contains(line.LineNumber))	84!
99	return;	×
100
101	switch (_strategy)	84!
102	{
103	case BadDataHandlingStrategy.IgnoreRows:
104
105	if (_maximumErrorsToReport-- > 0)	30✔
106	_listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	30✔
107	$"Ignored BadData on {line.GetLineDescription()}"));	30✔
108
109	//move to next line
110	_dataPusher.BadLines.Add(line.LineNumber);	30✔
111
112	break;	30✔
113	case BadDataHandlingStrategy.DivertRows:
114	DivertErrorRow(line, null);	22✔
115	break;	22✔
116
117	case BadDataHandlingStrategy.ThrowException:
118	throw new FlatFileLoadException($"Bad data found on {line.GetLineDescription()}");	32✔
119
120
121	default:
122	throw new ArgumentOutOfRangeException();	×
123	}
124	}
125
126	public void DivertErrorRow(FlatFileLine line, Exception ex)
127	{
128	if (DivertErrorsFile == null)	22✔
129	{
130	DivertErrorsFile = new FileInfo(Path.Combine(_fileToLoad.File.Directory.FullName,	16✔
131	$"{Path.GetFileNameWithoutExtension(_fileToLoad.File.Name)}_Errors.txt"));	16✔
132
133	//delete any old version
134	if (DivertErrorsFile.Exists)	16✔
135	DivertErrorsFile.Delete();	15✔
136	}
137
138	if (_maximumErrorsToReport-- > 0)	22✔
139	_listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	22✔
140	$"Diverting Error on {line.GetLineDescription()} to '{DivertErrorsFile.FullName}'", ex));	22✔
141
142	File.AppendAllText(DivertErrorsFile.FullName, line.RawRecord);	22✔
143
144	//move to next line
145	_dataPusher.BadLines.Add(line.LineNumber);	22✔
146	}	22✔
147	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous