6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.64

/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecutePkSynthesizerDatasetExtractionSource.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.Spontaneous;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.DataExport.DataExtraction.Commands;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.QueryBuilding;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Sources;

/// <summary>
/// Extraction source which creates a PrimaryKey on the DataTable being extracted.  This is based on <see cref="IColumn.IsPrimaryKey"/> of the
/// columns extracted and is not garuanteed to actually be unique (depending on how you have configured the flags).
/// 
/// <para>The primary use case for this is when extracting to database where you want to have meaningful primary keys</para>
/// </summary>
public class ExecutePkSynthesizerDatasetExtractionSource : ExecuteDatasetExtractionSource
{
    private const string SYNTH_PK_COLUMN = "SynthesizedPk";
    private bool _synthesizePkCol;

    public override string HackExtractionSQL(string sql, IDataLoadEventListener listener)
    {
        // let's look for primary keys in the Extraction Information
        var cataloguePrimaryKeys = GetCatalogueItemPrimaryKeys();

        // if there are some they will be marked in the "GetChunk".
        // If there are none, then we need to synth a new column here.
        if (!cataloguePrimaryKeys.Any())
        {
            var primaryKeys = GetColumnInfoPrimaryKeys().ToArray();

            if (primaryKeys.Any())
            {
                string newSql;
                if (primaryKeys.Length > 1) // no need to do anything if there is only one.
                    newSql = $"CONCAT({string.Join(",'_',", primaryKeys.Select(apk => apk.ToString()))})";
                else
                    newSql = primaryKeys.Single().Name;

                var syntaxHelper = Request.Catalogue.GetQuerySyntaxHelper();

                Request.QueryBuilder.AddColumn(
                    new SpontaneouslyInventedColumn(new MemoryRepository(), SYNTH_PK_COLUMN,
                        syntaxHelper.HowDoWeAchieveMd5(newSql))
                    {
                        HashOnDataRelease = true,
                        IsPrimaryKey = true,
                        Order = -1
                    });
                _synthesizePkCol = true;
            }
        }

        return Request.QueryBuilder.SQL;
    }

    private IEnumerable<ITableInfo> GetProperTables()
    {
        if (Request.QueryBuilder.SQLOutOfDate)
            Request.QueryBuilder.RegenerateSQL();

        return Request.QueryBuilder.TablesUsedInQuery.Where(ti => !ti.IsLookupTable());
    }

    public override DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
    {
        var chunk = base.GetChunk(listener, cancellationToken);
        if (GlobalsRequest != null)
            return chunk;

        if (chunk == null)
            return null;

        var catalogueItemPkColumns = GetCatalogueItemPrimaryKeys().Select(c => c.GetRuntimeName()).ToArray();

        if (catalogueItemPkColumns.Any())
            chunk.PrimaryKey = chunk.Columns.Cast<DataColumn>().Where(c =>
                catalogueItemPkColumns.Contains(c.ColumnName, StringComparer.CurrentCultureIgnoreCase)).ToArray();
        else if (_synthesizePkCol)
            chunk.PrimaryKey = new[] { chunk.Columns[SYNTH_PK_COLUMN] };

        return chunk;
    }

    public override void Check(ICheckNotifier notifier)
    {
        base.Check(notifier);
        if (Request == null ||
            Request == ExtractDatasetCommand
                .EmptyCommand) // it is the globals, and there is no PK involved in there... although there should be...
            return;

        var cataloguePrimaryKeys = GetCatalogueItemPrimaryKeys().ToArray();
        if (!cataloguePrimaryKeys.Any())
        {
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"PKSynthesizer:No CatalogueItems marked IsPrimaryKey in '{Request.SelectedDataSets}'",
                CheckResult.Warning));

            var columnInfoPrimaryKeys = GetColumnInfoPrimaryKeys().ToArray();

            if (columnInfoPrimaryKeys.Any())
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"PKSynthesizer:Found ColumnInfo(s) marked IsPrimaryKey in '{Request.SelectedDataSets}'{string.Join(",", columnInfoPrimaryKeys.Select(c => c.Name))}",
                    CheckResult.Success));
            else
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"PKSynthesizer:No ColumnInfo marked IsPrimaryKey in '{Request.SelectedDataSets}'",
                    CheckResult.Fail));
        }
        else
        {
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"PKSynthesizer:Found CatalogueItem(s) marked IsPrimaryKey in '{Request.SelectedDataSets}'{string.Join(",", cataloguePrimaryKeys.Select(c => c.GetRuntimeName()))}",
                CheckResult.Success));
        }
    }

    private IEnumerable<IColumn> GetCatalogueItemPrimaryKeys()
    {
        foreach (var column in Request.ColumnsToExtract.Union(Request.ReleaseIdentifierSubstitutions))
            switch (column)
            {
                case ReleaseIdentifierSubstitution ri when ri.IsPrimaryKey || ri.OriginalDatasetColumn.IsPrimaryKey:
                    yield return ri;

                    break;
                case ExtractableColumn { IsPrimaryKey: true } ec:
                    yield return ec;

                    break;
            }
    }

    private IEnumerable<ColumnInfo> GetColumnInfoPrimaryKeys()
    {
        return GetProperTables().SelectMany(static t => t.ColumnInfos).Where(static column => column.IsPrimaryKey);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data;
10	using System.Linq;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.Curation.Data.Spontaneous;
13	using Rdmp.Core.DataExport.Data;
14	using Rdmp.Core.DataExport.DataExtraction.Commands;
15	using Rdmp.Core.DataFlowPipeline;
16	using Rdmp.Core.MapsDirectlyToDatabaseTable;
17	using Rdmp.Core.QueryBuilding;
18	using Rdmp.Core.ReusableLibraryCode.Checks;
19	using Rdmp.Core.ReusableLibraryCode.Progress;
20
21	namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Sources;
22
23	/// <summary>
24	/// Extraction source which creates a PrimaryKey on the DataTable being extracted. This is based on <see cref="IColumn.IsPrimaryKey"/> of the
25	/// columns extracted and is not garuanteed to actually be unique (depending on how you have configured the flags).
26	///
27	/// <para>The primary use case for this is when extracting to database where you want to have meaningful primary keys</para>
28	/// </summary>
29	public class ExecutePkSynthesizerDatasetExtractionSource : ExecuteDatasetExtractionSource
30	{
31	private const string SYNTH_PK_COLUMN = "SynthesizedPk";
32	private bool _synthesizePkCol;
33
34	public override string HackExtractionSQL(string sql, IDataLoadEventListener listener)
35	{
36	// let's look for primary keys in the Extraction Information
37	var cataloguePrimaryKeys = GetCatalogueItemPrimaryKeys();	10✔
38
39	// if there are some they will be marked in the "GetChunk".
40	// If there are none, then we need to synth a new column here.
41	if (!cataloguePrimaryKeys.Any())	10✔
42	{
43	var primaryKeys = GetColumnInfoPrimaryKeys().ToArray();	6✔
44
45	if (primaryKeys.Any())	6✔
46	{
47	string newSql;
48	if (primaryKeys.Length > 1) // no need to do anything if there is only one.	6✔
49	newSql = $"CONCAT({string.Join(",'_',", primaryKeys.Select(apk => apk.ToString()))})";	6✔
50	else
51	newSql = primaryKeys.Single().Name;	4✔
52
53	var syntaxHelper = Request.Catalogue.GetQuerySyntaxHelper();	6✔
54
55	Request.QueryBuilder.AddColumn(	6✔
56	new SpontaneouslyInventedColumn(new MemoryRepository(), SYNTH_PK_COLUMN,	6✔
57	syntaxHelper.HowDoWeAchieveMd5(newSql))	6✔
58	{	6✔
59	HashOnDataRelease = true,	6✔
60	IsPrimaryKey = true,	6✔
61	Order = -1	6✔
62	});	6✔
63	_synthesizePkCol = true;	6✔
64	}
65	}
66
67	return Request.QueryBuilder.SQL;	10✔
68	}
69
70	private IEnumerable<ITableInfo> GetProperTables()
71	{
72	if (Request.QueryBuilder.SQLOutOfDate)	6!
73	Request.QueryBuilder.RegenerateSQL();	×
74
75	return Request.QueryBuilder.TablesUsedInQuery.Where(ti => !ti.IsLookupTable());	18✔
76	}
77
78	public override DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
79	{
80	var chunk = base.GetChunk(listener, cancellationToken);	10✔
81	if (GlobalsRequest != null)	10!
82	return chunk;	×
83
84	if (chunk == null)	10!
85	return null;	×
86
87	var catalogueItemPkColumns = GetCatalogueItemPrimaryKeys().Select(c => c.GetRuntimeName()).ToArray();	16✔
88
89	if (catalogueItemPkColumns.Any())	10✔
90	chunk.PrimaryKey = chunk.Columns.Cast<DataColumn>().Where(c =>	4✔
91	catalogueItemPkColumns.Contains(c.ColumnName, StringComparer.CurrentCultureIgnoreCase)).ToArray();	16✔
92	else if (_synthesizePkCol)	6✔
93	chunk.PrimaryKey = new[] { chunk.Columns[SYNTH_PK_COLUMN] };	6✔
94
95	return chunk;	10✔
96	}
97
98	public override void Check(ICheckNotifier notifier)
99	{
100	base.Check(notifier);	×
101	if (Request == null \|\|	×
102	Request == ExtractDatasetCommand	×
103	.EmptyCommand) // it is the globals, and there is no PK involved in there... although there should be...	×
104	return;	×
105
106	var cataloguePrimaryKeys = GetCatalogueItemPrimaryKeys().ToArray();	×
107	if (!cataloguePrimaryKeys.Any())	×
108	{
109	notifier.OnCheckPerformed(new CheckEventArgs(	×
110	$"PKSynthesizer:No CatalogueItems marked IsPrimaryKey in '{Request.SelectedDataSets}'",	×
111	CheckResult.Warning));	×
112
113	var columnInfoPrimaryKeys = GetColumnInfoPrimaryKeys().ToArray();	×
114
115	if (columnInfoPrimaryKeys.Any())	×
116	notifier.OnCheckPerformed(new CheckEventArgs(	×
117	$"PKSynthesizer:Found ColumnInfo(s) marked IsPrimaryKey in '{Request.SelectedDataSets}'{string.Join(",", columnInfoPrimaryKeys.Select(c => c.Name))}",	×
118	CheckResult.Success));	×
119	else
120	notifier.OnCheckPerformed(new CheckEventArgs(	×
121	$"PKSynthesizer:No ColumnInfo marked IsPrimaryKey in '{Request.SelectedDataSets}'",	×
122	CheckResult.Fail));	×
123	}
124	else
125	{
126	notifier.OnCheckPerformed(new CheckEventArgs(	×
127	$"PKSynthesizer:Found CatalogueItem(s) marked IsPrimaryKey in '{Request.SelectedDataSets}'{string.Join(",", cataloguePrimaryKeys.Select(c => c.GetRuntimeName()))}",	×
128	CheckResult.Success));	×
129	}
130	}	×
131
132	private IEnumerable<IColumn> GetCatalogueItemPrimaryKeys()
133	{
134	foreach (var column in Request.ColumnsToExtract.Union(Request.ReleaseIdentifierSubstitutions))	172✔
135	switch (column)	68✔
136	{
137	case ReleaseIdentifierSubstitution ri when ri.IsPrimaryKey \|\| ri.OriginalDatasetColumn.IsPrimaryKey:	16✔
138	yield return ri;	2✔
139
140	break;	2✔
141	case ExtractableColumn { IsPrimaryKey: true } ec:
142	yield return ec;	8✔
143
144	break;
145	}
146	}	16✔
147
148	private IEnumerable<ColumnInfo> GetColumnInfoPrimaryKeys()
149	{
150	return GetProperTables().SelectMany(static t => t.ColumnInfos).Where(static column => column.IsPrimaryKey);	36✔
151	}
152	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous