8685582570

Committed 15 Apr 2024 08:05AM UTC coverage: 77.354% (+1.1%) from 76.233%

Build # 8685582570

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #57 from robbery/wip-speedup

speedup

Run Details

345 of 446 relevant lines covered (77.35%)

28.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.82

csv-diff/CSVDiff.cs

using System;
using System.Collections.Generic;
using System.Linq;
using csv_diff.Interfaces;

namespace csv_diff
{
    public class CSVDiff : Algorithm
    {
        // CSVSource object containing details of the left/from input.
        public ISource Left { get; }
        // CSVSource object containing details of the right/to input.
        public ISource Right { get; }
        // An array of differences
        public Dictionary<string, Diff> Diffs { get; set; }

        // An array of field names that are compared in the diff process.
        public List<string> DiffFields { get; }
        // An array of field names of the key fields that uniquely identify each row.
        public List<string> KeyFields { get; }
        // An array of field names for the parent field(s).
        public List<string> ParentFields { get; }
        // An array of field names for the child field(s).
        public List<string> ChildFields { get; }
        // The options dictionary used for the diff.
        public Dictionary<string, object> Options { get; set; }

        // Constructor
        public CSVDiff(object left, object right, Dictionary<string, object> options = null)
        {
            if (options is null)
                options = new Dictionary<string, object>();
            
            Left = left is ISource leftSource ? leftSource : new CSVSource(left, options);
            if (Left.Lines == null)
            {
                Left.IndexSource();
            }
            if (Left.FieldNames == null || Left.FieldNames.Count == 0)
            {
                throw new Exception("No field names found in left (from) source");
            }

            Right = right is ISource rightSource ? rightSource : new CSVSource(right, options);
            if (Right.Lines == null)
            {
                Right.IndexSource();
            }
            if (Right.FieldNames == null || Right.FieldNames.Count == 0)
            {
                throw new Exception("No field names found in right (to) source");
            }

            Diffs = new Dictionary<string, Diff>();
            DiffFields = GetDiffFields(Left.FieldNames, Right.FieldNames, options);
            KeyFields = Left.KeyFields;
            Diff(options);
        }

        // Performs a diff with the specified options.
        public void Diff(Dictionary<string, object> options = null)
        {
            Options = options;
            Diffs = DiffSources(Left, Right, KeyFields.ToArray(), DiffFields.ToArray(), options);
        }

        // Returns a summary of the number of adds, deletes, moves, and updates.
        public Dictionary<string, int> Summary
        {
            get
            {
                var summary = new Dictionary<string, int>
                {
                    { "Add", 0 },
                    { "Delete", 0 },
                    { "Update", 0 },
                    { "Move", 0 }
                };
                foreach (var diff in Diffs)
                {
                    // summary[diff.Value.DiffType] += 1;
                    summary[(string)diff.Value["action"]] += 1;
                }
                summary["warning"] = Warnings.Count > 0 ? Warnings.Count : 0;
                return summary;
            }
        }

        // Returns an array of adds.
        public Dictionary<string, Diff> Adds
        {
            get
            {
                return Diffs.Where(diff => (string)diff.Value["action"] == "Add").ToDictionary(x => x.Key, x => x.Value);
            }
        }
        
        // Returns an array of deletes.
        public Dictionary<string, Diff> Deletes
        {
            get
            {
                return Diffs.Where(diff => (string)diff.Value["action"] == "Delete").ToDictionary(x => x.Key, x => x.Value);
            }
        }
        
        // Returns an array of updates.
        public Dictionary<string, Diff> Updates
        {
            get
            {
                return Diffs.Where(diff => (string)diff.Value["action"] == "Update").ToDictionary(x => x.Key, x => x.Value);
            }
        }
        
        // Returns an array of moves.
        public Dictionary<string, Diff> Moves
        {
            get
            {
                return Diffs.Where(diff => (string)diff.Value["action"] == "Move").ToDictionary(x => x.Key, x => x.Value);
            }
        }

        // Returns an array of warning messages generated from the sources and the diff process.
        public List<string> Warnings
        {
            get
            {
                var warnings = new List<string>();
                warnings.AddRange(Left.Warnings);
                warnings.AddRange(Right.Warnings);
                warnings.AddRange(_warnings);
                return warnings;
            }
        }

        // Returns an array of warning messages from the diff process.
        public List<string> DiffWarnings
        {
            get
            {
                return _warnings;
            }
        }

        private List<string> _warnings = new List<string>();

        // Given two sets of field names, determines the common set of fields present
        // in both, on which members can be diffed.
        private List<string> GetDiffFields(List<string> leftFields, List<string> rightFields, Dictionary<string, object> options)
        {
            var ignoreFields = options.ContainsKey("ignore_fields") ? options["ignore_fields"] : new List<object>();
            var ignoreFieldsList = new List<string>();
            if (ignoreFields is string ignoreFieldString)
            {
                ignoreFieldsList.Add(ignoreFieldString.ToUpper());
            }
            else if (ignoreFields is IEnumerable<object> ignoreFieldEnumerable)
            {
                ignoreFieldsList.AddRange(ignoreFieldEnumerable.Select(f => f.ToString().ToUpper()));
            }

            var diffFields = new List<string>();
            if (options.ContainsKey("diff_common_fields_only") && (bool)options["diff_common_fields_only"])
            {
                foreach (var field in rightFields)
                {
                    if (leftFields.Contains(field))
                    {
                        var upperCaseField = field.ToUpper();
                        if (!ignoreFieldsList.Contains(upperCaseField))
                        {
                            diffFields.Add(field);
                        }
                    }
                }
            }
            else
            {
                diffFields.AddRange(rightFields);
                diffFields.AddRange(leftFields);
                diffFields = diffFields.Distinct().Where(f => !ignoreFieldsList.Contains(f.ToUpper())).ToList();
            }

            return diffFields;
        }
    }
}

1	using System;
2	using System.Collections.Generic;
3	using System.Linq;
4	using csv_diff.Interfaces;
5
6	namespace csv_diff
7	{
8	public class CSVDiff : Algorithm
9	{
10	// CSVSource object containing details of the left/from input.
11	public ISource Left { get; }	33✔
12	// CSVSource object containing details of the right/to input.
13	public ISource Right { get; }	28✔
14	// An array of differences
15	public Dictionary<string, Diff> Diffs { get; set; }	27✔
16
17	// An array of field names that are compared in the diff process.
18	public List<string> DiffFields { get; }	5✔
19	// An array of field names of the key fields that uniquely identify each row.
20	public List<string> KeyFields { get; }	5✔
21	// An array of field names for the parent field(s).
22	public List<string> ParentFields { get; }	×
23	// An array of field names for the child field(s).
24	public List<string> ChildFields { get; }	×
25	// The options dictionary used for the diff.
26	public Dictionary<string, object> Options { get; set; }	5✔
27
28	// Constructor
29	public CSVDiff(object left, object right, Dictionary<string, object> options = null)	5✔
30	{
31	if (options is null)	5✔
32	options = new Dictionary<string, object>();	×
33
34	Left = left is ISource leftSource ? leftSource : new CSVSource(left, options);	5✔
35	if (Left.Lines == null)	5✔
36	{
37	Left.IndexSource();	1✔
38	}
39	if (Left.FieldNames == null \|\| Left.FieldNames.Count == 0)	5✔
40	{
41	throw new Exception("No field names found in left (from) source");	×
42	}
43
44	Right = right is ISource rightSource ? rightSource : new CSVSource(right, options);	5✔
45	if (Right.Lines == null)	5✔
46	{
47	Right.IndexSource();	1✔
48	}
49	if (Right.FieldNames == null \|\| Right.FieldNames.Count == 0)	5✔
50	{
51	throw new Exception("No field names found in right (to) source");	×
52	}
53
54	Diffs = new Dictionary<string, Diff>();	5✔
55	DiffFields = GetDiffFields(Left.FieldNames, Right.FieldNames, options);	5✔
56	KeyFields = Left.KeyFields;	5✔
57	Diff(options);	5✔
58	}	5✔
59
60	// Performs a diff with the specified options.
61	public void Diff(Dictionary<string, object> options = null)
62	{
63	Options = options;	5✔
64	Diffs = DiffSources(Left, Right, KeyFields.ToArray(), DiffFields.ToArray(), options);	5✔
65	}	5✔
66
67	// Returns a summary of the number of adds, deletes, moves, and updates.
68	public Dictionary<string, int> Summary
69	{
70	get
71	{
72	var summary = new Dictionary<string, int>	×
73	{	×
74	{ "Add", 0 },	×
75	{ "Delete", 0 },	×
76	{ "Update", 0 },	×
77	{ "Move", 0 }	×
78	};	×
79	foreach (var diff in Diffs)	×
80	{
81	// summary[diff.Value.DiffType] += 1;
82	summary[(string)diff.Value["action"]] += 1;	×
83	}
84	summary["warning"] = Warnings.Count > 0 ? Warnings.Count : 0;	×
85	return summary;	×
86	}
87	}
88
89	// Returns an array of adds.
90	public Dictionary<string, Diff> Adds
91	{
92	get
93	{
94	return Diffs.Where(diff => (string)diff.Value["action"] == "Add").ToDictionary(x => x.Key, x => x.Value);	76✔
95	}
96	}
97
98	// Returns an array of deletes.
99	public Dictionary<string, Diff> Deletes
100	{
101	get
102	{
103	return Diffs.Where(diff => (string)diff.Value["action"] == "Delete").ToDictionary(x => x.Key, x => x.Value);	66✔
104	}
105	}
106
107	// Returns an array of updates.
108	public Dictionary<string, Diff> Updates
109	{
110	get
111	{
112	return Diffs.Where(diff => (string)diff.Value["action"] == "Update").ToDictionary(x => x.Key, x => x.Value);	64✔
113	}
114	}
115
116	// Returns an array of moves.
117	public Dictionary<string, Diff> Moves
118	{
119	get
120	{
121	return Diffs.Where(diff => (string)diff.Value["action"] == "Move").ToDictionary(x => x.Key, x => x.Value);	29✔
122	}
123	}
124
125	// Returns an array of warning messages generated from the sources and the diff process.
126	public List<string> Warnings
127	{
128	get
129	{
130	var warnings = new List<string>();	×
131	warnings.AddRange(Left.Warnings);	×
132	warnings.AddRange(Right.Warnings);	×
133	warnings.AddRange(_warnings);	×
134	return warnings;	×
135	}
136	}
137
138	// Returns an array of warning messages from the diff process.
139	public List<string> DiffWarnings
140	{
141	get
142	{
143	return _warnings;	×
144	}
145	}
146
147	private List<string> _warnings = new List<string>();	5✔
148
149	// Given two sets of field names, determines the common set of fields present
150	// in both, on which members can be diffed.
151	private List<string> GetDiffFields(List<string> leftFields, List<string> rightFields, Dictionary<string, object> options)
152	{
153	var ignoreFields = options.ContainsKey("ignore_fields") ? options["ignore_fields"] : new List<object>();	5✔
154	var ignoreFieldsList = new List<string>();	5✔
155	if (ignoreFields is string ignoreFieldString)	5✔
156	{
157	ignoreFieldsList.Add(ignoreFieldString.ToUpper());	×
158	}
159	else if (ignoreFields is IEnumerable<object> ignoreFieldEnumerable)	5✔
160	{
161	ignoreFieldsList.AddRange(ignoreFieldEnumerable.Select(f => f.ToString().ToUpper()));	5✔
162	}
163
164	var diffFields = new List<string>();	5✔
165	if (options.ContainsKey("diff_common_fields_only") && (bool)options["diff_common_fields_only"])	5✔
166	{
167	foreach (var field in rightFields)	×
168	{
169	if (leftFields.Contains(field))	×
170	{
171	var upperCaseField = field.ToUpper();	×
172	if (!ignoreFieldsList.Contains(upperCaseField))	×
173	{
174	diffFields.Add(field);	×
175	}
176	}
177	}
178	}
179	else
180	{
181	diffFields.AddRange(rightFields);	5✔
182	diffFields.AddRange(leftFields);	5✔
183	diffFields = diffFields.Distinct().Where(f => !ignoreFieldsList.Contains(f.ToUpper())).ToList();	20✔
184	}
185
186	return diffFields;	5✔
187	}
188	}
189	}

WillemOpperman / csv-diff-dotnet / 8685582570

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous