8448610096

Committed 27 Mar 2024 08:16AM UTC coverage: 56.272% (-0.6%) from 56.886%

Build # 8448610096

Build Type

Pull #1779

github

Committed by

JFriel

Commit Message

Merge branch 'develop' of https://github.com/HicServices/RDMP into task/RDMP-122-update-a-database-extraction

Pull Request Pull Request #1779: Task/rdmp 122 update a database extraction

Run Details

10712 of 20527 branches covered (52.18%)

Branch coverage included in aggregate %.

29 of 40 new or added lines in 3 files covered. (72.5%)

370 existing lines in 21 files now uncovered.

30518 of 52742 relevant lines covered (57.86%)

7356.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.14

/Rdmp.Core/DataExport/DataExtraction/FileOutputFormats/CSVOutputFormat.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace Rdmp.Core.DataExport.DataExtraction.FileOutputFormats;

/// <summary>
/// Helper class for writing data to CSV files.  This is a simplified version of Rfc4180Writer in that it simply strips out all problem fields rather
/// than applying proper escaping etc.  This is done because some researcher end point tools / scripts do not support the full specification of CSV and
/// it is easier to provide them with a file where problem symbols are not present than explain that they have to join multiple lines together when it is
/// bounded by quotes.
/// </summary>
public class CSVOutputFormat : FileOutputFormat
{
    public string Separator { get; set; }
    public string DateFormat { get; set; }

    public int SeparatorsStrippedOut { get; private set; }
    private static readonly string[] ThingsToStripOut = { "\r", "\n", "\t", "\"" };
    private StreamWriter _sw;
    private StringBuilder _sbWriteOutLinesBuffer;
    private const string _illegalCharactersReplacement = " ";

    public CSVOutputFormat(string outputFilename, string separator, string dateFormat) : base(outputFilename)
    {
        Separator = separator;
        DateFormat = dateFormat;
    }

    public override string GetFileExtension() => ".csv";

    public override void Open()
    {
        _sw = new StreamWriter(OutputFilename);
        _sbWriteOutLinesBuffer = new StringBuilder();
    }

    public override void Open(bool append)
    {
        _sw = new StreamWriter(OutputFilename, append);
        _sbWriteOutLinesBuffer = new StringBuilder();
    }

    public override void WriteHeaders(DataTable t)
    {
        //write headers separated by separator
        _sw.Write(string.Join(Separator, t.Columns.Cast<DataColumn>().Select(c => c.ColumnName).ToArray()));

        _sw.WriteLine();
    }

    public override void Append(DataRow r)
    {
        //write headers separated by separator
        _sbWriteOutLinesBuffer.Append(string.Join(Separator, r.ItemArray.Select(CleanString)));

        //write the new line
        _sbWriteOutLinesBuffer.Append(Environment.NewLine);
    }

    public override void Flush()
    {
        _sw.Write(_sbWriteOutLinesBuffer.ToString());
        _sbWriteOutLinesBuffer.Clear();
        _sw.Flush();
    }

    public override void Close()
    {
        _sw?.Flush();
        _sw?.Close();
        _sw?.Dispose();
    }

    public string CleanString(object o)
    {
        var toReturn = CleanString(o, Separator, out var numberOfSeparatorsStrippedOutThisPass, DateFormat,
            RoundFloatsTo);

        SeparatorsStrippedOut += numberOfSeparatorsStrippedOutThisPass;

        return toReturn;
    }


    public static string CleanString(object o, string separator, out int separatorsStrippedOut, string dateFormat,
        int? roundFloatsTo)
    {
        if (o is DateTime dateTime)
        {
            separatorsStrippedOut = 0;
            return dateTime.ToString(dateFormat);
        }

        if (roundFloatsTo.HasValue)
        {
            separatorsStrippedOut = 0;
            switch (o)
            {
                case float f: return f.ToString($"N{roundFloatsTo.Value}");
                case decimal dec: return dec.ToString($"N{roundFloatsTo.Value}");
                case double d: return d.ToString($"N{roundFloatsTo.Value}");
            }
        }

        //in order to kep a count
        var regexReplace = new Regex(Regex.Escape(separator));

        separatorsStrippedOut = 0;

        while (o.ToString().Contains(separator))
        {
            o = regexReplace.Replace(o.ToString(), _illegalCharactersReplacement, 1);
            separatorsStrippedOut++;
        }

        return o is string s
            ? ThingsToStripOut.Aggregate(s,
                (current, cToStripOut) => current.Replace(cToStripOut, _illegalCharactersReplacement)).Trim()
            : o.ToString().Trim();
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Data;
9	using System.IO;
10	using System.Linq;
11	using System.Text;
12	using System.Text.RegularExpressions;
13
14	namespace Rdmp.Core.DataExport.DataExtraction.FileOutputFormats;
15
16	/// <summary>
17	/// Helper class for writing data to CSV files. This is a simplified version of Rfc4180Writer in that it simply strips out all problem fields rather
18	/// than applying proper escaping etc. This is done because some researcher end point tools / scripts do not support the full specification of CSV and
19	/// it is easier to provide them with a file where problem symbols are not present than explain that they have to join multiple lines together when it is
20	/// bounded by quotes.
21	/// </summary>
22	public class CSVOutputFormat : FileOutputFormat
23	{
24	public string Separator { get; set; }	216✔
25	public string DateFormat { get; set; }	166✔
26
27	public int SeparatorsStrippedOut { get; private set; }	140✔
28	private static readonly string[] ThingsToStripOut = { "\r", "\n", "\t", "\"" };	2✔
29	private StreamWriter _sw;
30	private StringBuilder _sbWriteOutLinesBuffer;
31	private const string _illegalCharactersReplacement = " ";
32
33	public CSVOutputFormat(string outputFilename, string separator, string dateFormat) : base(outputFilename)	96✔
34	{
35	Separator = separator;	96✔
36	DateFormat = dateFormat;	96✔
37	}	96✔
38
39	public override string GetFileExtension() => ".csv";	×
40
41	public override void Open()
42	{
43	_sw = new StreamWriter(OutputFilename);	×
44	_sbWriteOutLinesBuffer = new StringBuilder();	×
45	}	×
46
47	public override void Open(bool append)
48	{
49	_sw = new StreamWriter(OutputFilename, append);	24✔
50	_sbWriteOutLinesBuffer = new StringBuilder();	24✔
51	}	24✔
52
53	public override void WriteHeaders(DataTable t)
54	{
55	//write headers separated by separator
56	_sw.Write(string.Join(Separator, t.Columns.Cast<DataColumn>().Select(c => c.ColumnName).ToArray()));	88✔
57
58	_sw.WriteLine();	24✔
59	}	24✔
60
61	public override void Append(DataRow r)
62	{
63	//write headers separated by separator
64	_sbWriteOutLinesBuffer.Append(string.Join(Separator, r.ItemArray.Select(CleanString)));	26✔
65
66	//write the new line
67	_sbWriteOutLinesBuffer.Append(Environment.NewLine);	26✔
68	}	26✔
69
70	public override void Flush()
71	{
72	_sw.Write(_sbWriteOutLinesBuffer.ToString());	24✔
73	_sbWriteOutLinesBuffer.Clear();	24✔
74	_sw.Flush();	24✔
75	}	24✔
76
77	public override void Close()
78	{
79	_sw?.Flush();	24!
80	_sw?.Close();	24!
81	_sw?.Dispose();	24!
82	}	24✔
83
84	public string CleanString(object o)
85	{
86	var toReturn = CleanString(o, Separator, out var numberOfSeparatorsStrippedOutThisPass, DateFormat,	70✔
87	RoundFloatsTo);	70✔
88
89	SeparatorsStrippedOut += numberOfSeparatorsStrippedOutThisPass;	70✔
90
91	return toReturn;	70✔
92	}
93
94
95	public static string CleanString(object o, string separator, out int separatorsStrippedOut, string dateFormat,
96	int? roundFloatsTo)
97	{
98	if (o is DateTime dateTime)	70✔
99	{
100	separatorsStrippedOut = 0;	18✔
101	return dateTime.ToString(dateFormat);	18✔
102	}
103
104	if (roundFloatsTo.HasValue)	52✔
105	{
106	separatorsStrippedOut = 0;	4!
107	switch (o)
108	{
109	case float f: return f.ToString($"N{roundFloatsTo.Value}");	×
110	case decimal dec: return dec.ToString($"N{roundFloatsTo.Value}");	4✔
111	case double d: return d.ToString($"N{roundFloatsTo.Value}");	×
112	}
113	}
114
115	//in order to kep a count
116	var regexReplace = new Regex(Regex.Escape(separator));	48✔
117
118	separatorsStrippedOut = 0;	48✔
119
120	while (o.ToString().Contains(separator))	48!
121	{
UNCOV 122	o = regexReplace.Replace(o.ToString(), _illegalCharactersReplacement, 1);	×
UNCOV 123	separatorsStrippedOut++;	×
124	}
125
126	return o is string s	48!
127	? ThingsToStripOut.Aggregate(s,	48✔
128	(current, cToStripOut) => current.Replace(cToStripOut, _illegalCharactersReplacement)).Trim()	192✔
129	: o.ToString().Trim();	48✔
130	}
131	}

HicServices / RDMP / 8448610096

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous