8973726646

Committed 06 May 2024 06:14PM UTC coverage: 53.644% (-0.8%) from 54.48%

Build # 8973726646

Build Type

push

github

Committed by

pwelter34

Commit Message

fix test snapshots

Run Details

1165 of 2845 branches covered (40.95%)

Branch coverage included in aggregate %.

3686 of 6198 relevant lines covered (59.47%)

697.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.61

/src/FluentCommand.SqlServer/Import/ImportProcessor.cs

using System.Data;

using FluentCommand.Extensions;
using FluentCommand.Merge;

namespace FluentCommand.Import;

/// <summary>
/// A data import processor
/// </summary>
public class ImportProcessor : IImportProcessor
{
    private readonly IDataSession _dataSession;
    private readonly ImportFactory _importFactory;

    /// <summary>
    /// Initializes a new instance of the <see cref="ImportProcessor" /> class.
    /// </summary>
    /// <param name="dataSession">The data session.</param>
    /// <param name="importFactory">The service provider factory.</param>
    public ImportProcessor(IDataSession dataSession, ImportFactory importFactory)
    {
        _dataSession = dataSession;
        _importFactory = importFactory;
    }


    /// <summary>
    /// Import data using the specified <paramref name="importDefinition" /> and <paramref name="importData" />.
    /// </summary>
    /// <param name="importDefinition">The import definition.</param>
    /// <param name="importData">The import data.</param>
    /// <param name="username">The name of the user importing the data.</param>
    /// <param name="cancellationToken">The cancellation token.</param>
    /// <returns>The results of the import</returns>
    /// <exception cref="ArgumentNullException"><paramref name="importData" /> or <paramref name="importDefinition" /> is null</exception>
    public virtual async Task<ImportResult> ImportAsync(ImportDefinition importDefinition, ImportData importData, string username, CancellationToken cancellationToken = default)
    {
        if (importData == null)
            throw new ArgumentNullException(nameof(importData));
        if (importDefinition == null)
            throw new ArgumentNullException(nameof(importDefinition));

        var context = new ImportProcessContext(importDefinition, importData, username, _importFactory);

        var dataTable = CreateTable(context);
        await PopulateTable(context, dataTable);

        if (dataTable.Rows.Count == 0)
            return new ImportResult { Processed = 0, Errors = context.Errors };

        var mergeDefinition = CreateMergeDefinition(context);

        var result = await _dataSession
            .MergeData(mergeDefinition)
            .ExecuteAsync(dataTable, cancellationToken);

        return new ImportResult { Processed = result, Errors = context.Errors };
    }


    /// <summary>
    /// Create a <see cref="DataTable" /> instance using the specified <paramref name="importContext" />.
    /// </summary>
    /// <param name="importContext">The import context to create DataTable from.</param>
    /// <returns>
    /// An instance of <see cref="DataTable" />.
    /// </returns>
    /// <exception cref="ArgumentNullException">importContext is null</exception>
    protected virtual DataTable CreateTable(ImportProcessContext importContext)
    {
        if (importContext == null)
            throw new ArgumentNullException(nameof(importContext));

        var dataTable = new DataTable("#Import" + DateTime.Now.Ticks);

        foreach (var field in importContext.MappedFields)
        {
            var dataType = Nullable.GetUnderlyingType(field.Definition.DataType)
                           ?? field.Definition.DataType;

            var dataColumn = new DataColumn
            {
                ColumnName = field.Definition.Name,
                DataType = dataType
            };

            dataTable.Columns.Add(dataColumn);
        }

        return dataTable;
    }

    /// <summary>
    /// Populates the <see cref="DataTable" /> with the specified <paramref name="importContext" />.
    /// </summary>
    /// <param name="importContext">The import context.</param>
    /// <param name="dataTable">The data table to populate.</param>
    /// <returns>
    /// The <see cref="DataTable" /> with the populated data.
    /// </returns>
    /// <exception cref="ArgumentNullException">
    /// dataTable or importContext is null
    /// </exception>
    protected virtual async Task<DataTable> PopulateTable(ImportProcessContext importContext, DataTable dataTable)
    {
        if (dataTable == null)
            throw new ArgumentNullException(nameof(dataTable));

        if (importContext == null)
            throw new ArgumentNullException(nameof(importContext));

        var data = importContext.ImportData.Data;
        if (data == null || data.Length == 0)
            return dataTable;

        var rows = data.Length;
        var startIndex = importContext.ImportData.HasHeader ? 1 : 0;

        for (var index = startIndex; index < rows; index++)
        {
            var row = data[index];

            // skip empty row
            if (row.All(string.IsNullOrWhiteSpace))
                continue;

            var dataRow = dataTable.NewRow();

            var valid = await PopulateRow(importContext, dataRow, row);
            if (valid)
                dataTable.Rows.Add(dataRow);
        }

        return dataTable;
    }

    /// <summary>
    /// Populates the <see cref="DataRow" /> using the specified <paramref name="row" />.
    /// </summary>
    /// <param name="importContext">The import context.</param>
    /// <param name="dataRow">The data row to populate.</param>
    /// <param name="row">The imported source data row.</param>
    /// <returns>
    /// The <see cref="DataRow" /> with the populated data.
    /// </returns>
    protected virtual async Task<bool> PopulateRow(ImportProcessContext importContext, DataRow dataRow, string[] row)
    {
        try
        {
            foreach (var field in importContext.MappedFields)
            {
                if (field.Definition.Default.HasValue)
                {
                    dataRow[field.Definition.Name] = GetDefault(field.Definition, importContext.UserName);
                    continue;
                }

                var index = field.FieldMap.Index;
                if (!index.HasValue)
                    continue;

                var value = row[index.Value];

                var convertValue = await ConvertValue(importContext, field.Definition, value);

                dataRow[field.Definition.Name] = convertValue ?? DBNull.Value;
            }

            if (importContext.Definition.Validator == null)
                return true;

            var validator = importContext.GetService(importContext.Definition.Validator) as IImportValidator;
            if (validator == null)
                throw new InvalidOperationException($"Failed to create data row validator '{importContext.Definition.Validator}'");

            await validator.ValidateRow(importContext.Definition, dataRow);

            return true;
        }
        catch (Exception ex)
        {
            importContext.Errors.Add(ex);

            if (importContext.Errors.Count > importContext.Definition.MaxErrors)
                throw;

            return false;
        }
    }

    /// <summary>
    /// Converts the source string value into the correct data type using specified <paramref name="field" /> definition.
    /// </summary>
    /// <param name="importContext">The import context.</param>
    /// <param name="field">The field definition.</param>
    /// <param name="value">The source value.</param>
    /// <returns>
    /// The convert value.
    /// </returns>
    /// <exception cref="InvalidOperationException">Failed to create translator for field '{field.Name}'</exception>
    protected virtual async Task<object> ConvertValue(ImportProcessContext importContext, FieldDefinition field, string value)
    {
        if (field.Translator == null)
        {
            return ConvertExtensions.SafeConvert(field.DataType, value);
        }

        var translator = importContext.GetService(field.Translator) as IFieldTranslator;
        if (translator == null)
            throw new InvalidOperationException($"Failed to create translator for field '{field.Name}'");


        var translatedValue = await translator.Translate(value);
        return translatedValue;
    }

    /// <summary>
    /// Gets the default value for the specified <paramref name="fieldDefinition"/>.
    /// </summary>
    /// <param name="fieldDefinition">The field definition.</param>
    /// <param name="username">The username.</param>
    /// <returns></returns>
    protected virtual object GetDefault(FieldDefinition fieldDefinition, string username)
    {
        var fieldDefault = fieldDefinition?.Default;
        if (!fieldDefault.HasValue)
            return null;

        if (fieldDefault.Value == FieldDefault.CurrentDate)
            return DateTimeOffset.UtcNow;

        if (fieldDefault.Value == FieldDefault.Static)
            return fieldDefinition.DefaultValue;

        if (fieldDefault.Value == FieldDefault.UserName)
            return username;

        return null;
    }

    /// <summary>
    /// Creates a <see cref="DataMergeDefinition" /> from the specified <paramref name="importContext" />.
    /// </summary>
    /// <param name="importContext">The import context.</param>
    /// <returns>
    /// An instance of <see cref="DataMergeDefinition" />
    /// </returns>
    /// <exception cref="InvalidOperationException">Could not find matching field definition for data column</exception>
    protected virtual DataMergeDefinition CreateMergeDefinition(ImportProcessContext importContext)
    {
        var importDefinition = importContext.Definition;

        var mergeDefinition = new DataMergeDefinition();
        mergeDefinition.TargetTable = importDefinition.TargetTable;
        mergeDefinition.IncludeInsert = importDefinition.CanInsert;
        mergeDefinition.IncludeUpdate = importDefinition.CanUpdate;

        // fluent builder
        var mergeMapping = new DataMergeMapping(mergeDefinition);

        // map included columns
        foreach (var fieldMapping in importContext.MappedFields)
        {
            var fieldDefinition = fieldMapping.Definition;
            var nativeType = SqlTypeMapping.NativeType(fieldDefinition.DataType);

            mergeMapping
                .Column(fieldDefinition.Name)
                .Insert(fieldDefinition.CanInsert)
                .Update(fieldDefinition.CanUpdate)
                .Key(fieldDefinition.IsKey)
                .NativeType(nativeType);
        }

        return mergeDefinition;
    }
}

1	using System.Data;
2
3	using FluentCommand.Extensions;
4	using FluentCommand.Merge;
5
6	namespace FluentCommand.Import;
7
8	/// <summary>
9	/// A data import processor
10	/// </summary>
11	public class ImportProcessor : IImportProcessor
12	{
13	private readonly IDataSession _dataSession;
14	private readonly ImportFactory _importFactory;
15
16	/// <summary>
17	/// Initializes a new instance of the <see cref="ImportProcessor" /> class.
18	/// </summary>
19	/// <param name="dataSession">The data session.</param>
20	/// <param name="importFactory">The service provider factory.</param>
21	public ImportProcessor(IDataSession dataSession, ImportFactory importFactory)	42✔
22	{
23	_dataSession = dataSession;	42✔
24	_importFactory = importFactory;	42✔
25	}	42✔
26
27
28	/// <summary>
29	/// Import data using the specified <paramref name="importDefinition" /> and <paramref name="importData" />.
30	/// </summary>
31	/// <param name="importDefinition">The import definition.</param>
32	/// <param name="importData">The import data.</param>
33	/// <param name="username">The name of the user importing the data.</param>
34	/// <param name="cancellationToken">The cancellation token.</param>
35	/// <returns>The results of the import</returns>
36	/// <exception cref="ArgumentNullException"><paramref name="importData" /> or <paramref name="importDefinition" /> is null</exception>
37	public virtual async Task<ImportResult> ImportAsync(ImportDefinition importDefinition, ImportData importData, string username, CancellationToken cancellationToken = default)
38	{
39	if (importData == null)	×
40	throw new ArgumentNullException(nameof(importData));	×
41	if (importDefinition == null)	×
42	throw new ArgumentNullException(nameof(importDefinition));	×
43
44	var context = new ImportProcessContext(importDefinition, importData, username, _importFactory);	×
45
46	var dataTable = CreateTable(context);	×
47	await PopulateTable(context, dataTable);	×
48
49	if (dataTable.Rows.Count == 0)	×
50	return new ImportResult { Processed = 0, Errors = context.Errors };	×
51
52	var mergeDefinition = CreateMergeDefinition(context);	×
53
54	var result = await _dataSession	×
55	.MergeData(mergeDefinition)	×
56	.ExecuteAsync(dataTable, cancellationToken);	×
57
58	return new ImportResult { Processed = result, Errors = context.Errors };	×
59	}	×
60
61
62	/// <summary>
63	/// Create a <see cref="DataTable" /> instance using the specified <paramref name="importContext" />.
64	/// </summary>
65	/// <param name="importContext">The import context to create DataTable from.</param>
66	/// <returns>
67	/// An instance of <see cref="DataTable" />.
68	/// </returns>
69	/// <exception cref="ArgumentNullException">importContext is null</exception>
70	protected virtual DataTable CreateTable(ImportProcessContext importContext)
71	{
72	if (importContext == null)	6!
73	throw new ArgumentNullException(nameof(importContext));	×
74
75	var dataTable = new DataTable("#Import" + DateTime.Now.Ticks);	6✔
76
77	foreach (var field in importContext.MappedFields)	72✔
78	{
79	var dataType = Nullable.GetUnderlyingType(field.Definition.DataType)	30✔
80	?? field.Definition.DataType;	30✔
81
82	var dataColumn = new DataColumn	30✔
83	{	30✔
84	ColumnName = field.Definition.Name,	30✔
85	DataType = dataType	30✔
86	};	30✔
87
88	dataTable.Columns.Add(dataColumn);	30✔
89	}
90
91	return dataTable;	6✔
92	}
93
94	/// <summary>
95	/// Populates the <see cref="DataTable" /> with the specified <paramref name="importContext" />.
96	/// </summary>
97	/// <param name="importContext">The import context.</param>
98	/// <param name="dataTable">The data table to populate.</param>
99	/// <returns>
100	/// The <see cref="DataTable" /> with the populated data.
101	/// </returns>
102	/// <exception cref="ArgumentNullException">
103	/// dataTable or importContext is null
104	/// </exception>
105	protected virtual async Task<DataTable> PopulateTable(ImportProcessContext importContext, DataTable dataTable)
106	{
107	if (dataTable == null)	3!
108	throw new ArgumentNullException(nameof(dataTable));	×
109
110	if (importContext == null)	3!
111	throw new ArgumentNullException(nameof(importContext));	×
112
113	var data = importContext.ImportData.Data;	3✔
114	if (data == null \|\| data.Length == 0)	3!
115	return dataTable;	×
116
117	var rows = data.Length;	3✔
118	var startIndex = importContext.ImportData.HasHeader ? 1 : 0;	3✔
119
120	for (var index = startIndex; index < rows; index++)	24✔
121	{
122	var row = data[index];	9✔
123
124	// skip empty row
125	if (row.All(string.IsNullOrWhiteSpace))	9✔
126	continue;
127
128	var dataRow = dataTable.NewRow();	9✔
129
130	var valid = await PopulateRow(importContext, dataRow, row);	9✔
131	if (valid)	9✔
132	dataTable.Rows.Add(dataRow);	9✔
133	}	9✔
134
135	return dataTable;	3✔
136	}	3✔
137
138	/// <summary>
139	/// Populates the <see cref="DataRow" /> using the specified <paramref name="row" />.
140	/// </summary>
141	/// <param name="importContext">The import context.</param>
142	/// <param name="dataRow">The data row to populate.</param>
143	/// <param name="row">The imported source data row.</param>
144	/// <returns>
145	/// The <see cref="DataRow" /> with the populated data.
146	/// </returns>
147	protected virtual async Task<bool> PopulateRow(ImportProcessContext importContext, DataRow dataRow, string[] row)
148	{
149	try
150	{
151	foreach (var field in importContext.MappedFields)	108✔
152	{
153	if (field.Definition.Default.HasValue)	45!
154	{
155	dataRow[field.Definition.Name] = GetDefault(field.Definition, importContext.UserName);	×
156	continue;	×
157	}
158
159	var index = field.FieldMap.Index;	45✔
160	if (!index.HasValue)	45✔
161	continue;
162
163	var value = row[index.Value];	45✔
164
165	var convertValue = await ConvertValue(importContext, field.Definition, value);	45✔
166
167	dataRow[field.Definition.Name] = convertValue ?? DBNull.Value;	45✔
168	}	45✔
169
170	if (importContext.Definition.Validator == null)	9!
171	return true;	9✔
172
173	var validator = importContext.GetService(importContext.Definition.Validator) as IImportValidator;	×
174	if (validator == null)	×
175	throw new InvalidOperationException($"Failed to create data row validator '{importContext.Definition.Validator}'");	×
176
177	await validator.ValidateRow(importContext.Definition, dataRow);	×
178
179	return true;	×
180	}
181	catch (Exception ex)	×
182	{
183	importContext.Errors.Add(ex);	×
184
185	if (importContext.Errors.Count > importContext.Definition.MaxErrors)	×
186	throw;	×
187
188	return false;	×
189	}
190	}	9✔
191
192	/// <summary>
193	/// Converts the source string value into the correct data type using specified <paramref name="field" /> definition.
194	/// </summary>
195	/// <param name="importContext">The import context.</param>
196	/// <param name="field">The field definition.</param>
197	/// <param name="value">The source value.</param>
198	/// <returns>
199	/// The convert value.
200	/// </returns>
201	/// <exception cref="InvalidOperationException">Failed to create translator for field '{field.Name}'</exception>
202	protected virtual async Task<object> ConvertValue(ImportProcessContext importContext, FieldDefinition field, string value)
203	{
204	if (field.Translator == null)	72!
205	{
206	return ConvertExtensions.SafeConvert(field.DataType, value);	72✔
207	}
208
209	var translator = importContext.GetService(field.Translator) as IFieldTranslator;	×
210	if (translator == null)	×
211	throw new InvalidOperationException($"Failed to create translator for field '{field.Name}'");	×
212
213
214	var translatedValue = await translator.Translate(value);	×
215	return translatedValue;	×
216	}	72✔
217
218	/// <summary>
219	/// Gets the default value for the specified <paramref name="fieldDefinition"/>.
220	/// </summary>
221	/// <param name="fieldDefinition">The field definition.</param>
222	/// <param name="username">The username.</param>
223	/// <returns></returns>
224	protected virtual object GetDefault(FieldDefinition fieldDefinition, string username)
225	{
226	var fieldDefault = fieldDefinition?.Default;	9!
227	if (!fieldDefault.HasValue)	9!
228	return null;	×
229
230	if (fieldDefault.Value == FieldDefault.CurrentDate)	9!
231	return DateTimeOffset.UtcNow;	×
232
233	if (fieldDefault.Value == FieldDefault.Static)	9✔
234	return fieldDefinition.DefaultValue;	6✔
235
236	if (fieldDefault.Value == FieldDefault.UserName)	3!
237	return username;	3✔
238
239	return null;	×
240	}
241
242	/// <summary>
243	/// Creates a <see cref="DataMergeDefinition" /> from the specified <paramref name="importContext" />.
244	/// </summary>
245	/// <param name="importContext">The import context.</param>
246	/// <returns>
247	/// An instance of <see cref="DataMergeDefinition" />
248	/// </returns>
249	/// <exception cref="InvalidOperationException">Could not find matching field definition for data column</exception>
250	protected virtual DataMergeDefinition CreateMergeDefinition(ImportProcessContext importContext)
251	{
252	var importDefinition = importContext.Definition;	×
253
254	var mergeDefinition = new DataMergeDefinition();	×
255	mergeDefinition.TargetTable = importDefinition.TargetTable;	×
256	mergeDefinition.IncludeInsert = importDefinition.CanInsert;	×
257	mergeDefinition.IncludeUpdate = importDefinition.CanUpdate;	×
258
259	// fluent builder
260	var mergeMapping = new DataMergeMapping(mergeDefinition);	×
261
262	// map included columns
263	foreach (var fieldMapping in importContext.MappedFields)	×
264	{
265	var fieldDefinition = fieldMapping.Definition;	×
266	var nativeType = SqlTypeMapping.NativeType(fieldDefinition.DataType);	×
267
268	mergeMapping	×
269	.Column(fieldDefinition.Name)	×
270	.Insert(fieldDefinition.CanInsert)	×
271	.Update(fieldDefinition.CanUpdate)	×
272	.Key(fieldDefinition.IsKey)	×
273	.NativeType(nativeType);	×
274	}
275
276	return mergeDefinition;	×
277	}
278	}

loresoft / FluentCommand / 8973726646

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous