6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.64

/Rdmp.Core/DataExport/DataExtraction/Commands/ExtractDatasetCommand.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FAnsi.Discovery;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.DataExport.DataExtraction.UserPicks;
using Rdmp.Core.QueryBuilding;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode.DataAccess;

namespace Rdmp.Core.DataExport.DataExtraction.Commands;

/// <summary>
/// Command representing a desire to extract a given dataset in an ExtractionConfiguration through an extraction pipeline.  This includes bundled content
/// (Lookup tables, SupportingDocuments etc).  Also includes optional settings (e.g. IncludeValidation) etc.  You can realise the request by running the
/// QueryBuilder SQL.
/// </summary>
public class ExtractDatasetCommand : ExtractCommand, IExtractDatasetCommand
{
    public ISelectedDataSets SelectedDataSets { get; set; }

    private IExtractableDatasetBundle _datasetBundle;
    private List<IColumn> _origColumnsToExtract;

    public IExtractableCohort ExtractableCohort { get; set; }

    public IExtractableDatasetBundle DatasetBundle
    {
        get => _datasetBundle;
        set
        {
            _datasetBundle = value;

            Catalogue = value == null
                ? null
                : DataExportRepository.CatalogueRepository.GetObjectByID<Catalogue>(value.DataSet.Catalogue_ID);
        }
    }

    public IDataExportRepository DataExportRepository { get; set; }

    public List<IColumn> ColumnsToExtract { get; set; }
    public IHICProjectSalt Salt { get; set; }
    public bool IncludeValidation { get; set; }

    public IExtractionDirectory Directory { get; set; }
    public ICatalogue Catalogue { get; private set; }

    public ISqlQueryBuilder QueryBuilder { get; set; }
    public ICumulativeExtractionResults CumulativeExtractionResults { get; set; }
    public List<ReleaseIdentifierSubstitution> ReleaseIdentifierSubstitutions { get; private set; }
    public List<IExtractionResults> ExtractionResults { get; private set; }
    public int TopX { get; set; }

    /// <inheritdoc/>
    public DateTime? BatchStart { get; set; }

    /// <inheritdoc/>
    public DateTime? BatchEnd { get; set; }

    public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableCohort extractableCohort,
        IExtractableDatasetBundle datasetBundle, List<IColumn> columnsToExtract, IHICProjectSalt salt,
        IExtractionDirectory directory, bool includeValidation = false, bool includeLookups = false) : this(
        configuration, datasetBundle.DataSet)
    {
        DataExportRepository = configuration.DataExportRepository;
        ExtractableCohort = extractableCohort;
        DatasetBundle = datasetBundle;
        ColumnsToExtract = columnsToExtract;

        // create a copy of the columns so we can support Reset()
        _origColumnsToExtract = ColumnsToExtract.ToList();

        Salt = salt;
        Directory = directory;
        IncludeValidation = includeValidation;
        TopX = -1;
    }


    /// <summary>
    /// This version has less arguments because it goes back to the database and queries the configuration and explores who the cohort is etc, it will result in more database
    /// queries than the more explicit constructor
    /// </summary>
    /// <param name="configuration"></param>
    /// <param name="datasetBundle"></param>
    /// <param name="includeValidation"></param>
    /// <param name="includeLookups"></param>
    public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableDatasetBundle datasetBundle,
        bool includeValidation = false, bool includeLookups = false) : this(configuration, datasetBundle.DataSet)
    {
        DataExportRepository = configuration.DataExportRepository;
        //ExtractableCohort = ExtractableCohort.GetExtractableCohortByID((int) configuration.Cohort_ID);
        ExtractableCohort = configuration.GetExtractableCohort();
        DatasetBundle = datasetBundle;
        ColumnsToExtract = new List<IColumn>(Configuration.GetAllExtractableColumnsFor(datasetBundle.DataSet));

        // create a copy of the columns so we can support Reset()
        _origColumnsToExtract = ColumnsToExtract.ToList();

        Salt = new HICProjectSalt(Project);
        Directory = new ExtractionDirectory(Project.ExtractionDirectory, configuration);
        IncludeValidation = includeValidation;
        TopX = -1;
    }

    public static readonly ExtractDatasetCommand EmptyCommand = new();


    private ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableDataSet dataset) : base(
        configuration)
    {
        var selectedDataSets = configuration.SelectedDataSets.Where(ds => ds.ExtractableDataSet_ID == dataset.ID)
            .ToArray();

        if (selectedDataSets.Length != 1)
            throw new Exception(
                $"Could not find 1 ISelectedDataSets for ExtractionConfiguration '{configuration}' | Dataset '{dataset}'");

        SelectedDataSets = selectedDataSets[0];

        ExtractionResults = new List<IExtractionResults>();
    }

    private ExtractDatasetCommand() : base(null)
    {
    }

    /// <summary>
    /// Resets the state of the command to when it was first constructed
    /// </summary>
    public void Reset()
    {
        ColumnsToExtract = _origColumnsToExtract.ToList();
        QueryBuilder = null;
    }

    public void GenerateQueryBuilder()
    {
        var host = new ExtractionQueryBuilder(DataExportRepository);
        QueryBuilder = host.GetSQLCommandForFullExtractionSet(this, out var substitutions);
        ReleaseIdentifierSubstitutions = substitutions;
    }

    public override string ToString() => this == EmptyCommand ? "EmptyCommand" : DatasetBundle.DataSet.ToString();

    public override DirectoryInfo GetExtractionDirectory() => this == EmptyCommand
        ? new DirectoryInfo(Path.GetTempPath())
        : Directory.GetDirectoryForDataset(DatasetBundle.DataSet);

    public override string DescribeExtractionImplementation() => QueryBuilder.SQL;

    /// <inheritdoc/>
    public DiscoveredServer GetDistinctLiveDatabaseServer()
    {
        IDataAccessPoint[] points = QueryBuilder?.TablesUsedInQuery != null
            ? QueryBuilder.TablesUsedInQuery.ToArray()
            : //get it from the request if it has been built
            Catalogue.GetTableInfoList(false); //or from the Catalogue directly if the query hasn't been built

        var singleServer = new DataAccessPointCollection(true, DataAccessContext.DataExport);
        singleServer.AddRange(points);

        return singleServer.GetDistinctServer();
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using FAnsi.Discovery;
12	using Rdmp.Core.Curation.Data;
13	using Rdmp.Core.DataExport.Data;
14	using Rdmp.Core.DataExport.DataExtraction.UserPicks;
15	using Rdmp.Core.QueryBuilding;
16	using Rdmp.Core.Repositories;
17	using Rdmp.Core.ReusableLibraryCode.DataAccess;
18
19	namespace Rdmp.Core.DataExport.DataExtraction.Commands;
20
21	/// <summary>
22	/// Command representing a desire to extract a given dataset in an ExtractionConfiguration through an extraction pipeline. This includes bundled content
23	/// (Lookup tables, SupportingDocuments etc). Also includes optional settings (e.g. IncludeValidation) etc. You can realise the request by running the
24	/// QueryBuilder SQL.
25	/// </summary>
26	public class ExtractDatasetCommand : ExtractCommand, IExtractDatasetCommand
27	{
28	public ISelectedDataSets SelectedDataSets { get; set; }	486✔
29
30	private IExtractableDatasetBundle _datasetBundle;
31	private List<IColumn> _origColumnsToExtract;
32
33	public IExtractableCohort ExtractableCohort { get; set; }	944✔
34
35	public IExtractableDatasetBundle DatasetBundle
36	{
37	get => _datasetBundle;	930✔
38	set
39	{
40	_datasetBundle = value;	204✔
41
42	Catalogue = value == null	204!
43	? null	204✔
44	: DataExportRepository.CatalogueRepository.GetObjectByID<Catalogue>(value.DataSet.Catalogue_ID);	204✔
45	}	204✔
46	}
47
48	public IDataExportRepository DataExportRepository { get; set; }	582✔
49
50	public List<IColumn> ColumnsToExtract { get; set; }	1,196✔
51	public IHICProjectSalt Salt { get; set; }	316✔
52	public bool IncludeValidation { get; set; }	204✔
53
54	public IExtractionDirectory Directory { get; set; }	352✔
55	public ICatalogue Catalogue { get; private set; }	492✔
56
57	public ISqlQueryBuilder QueryBuilder { get; set; }	942✔
58	public ICumulativeExtractionResults CumulativeExtractionResults { get; set; }	206✔
59	public List<ReleaseIdentifierSubstitution> ReleaseIdentifierSubstitutions { get; private set; }	244✔
60	public List<IExtractionResults> ExtractionResults { get; private set; }	204✔
61	public int TopX { get; set; }	326✔
62
63	/// <inheritdoc/>
64	public DateTime? BatchStart { get; set; }	12✔
65
66	/// <inheritdoc/>
67	public DateTime? BatchEnd { get; set; }	16✔
68
69	public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableCohort extractableCohort,
70	IExtractableDatasetBundle datasetBundle, List<IColumn> columnsToExtract, IHICProjectSalt salt,
71	IExtractionDirectory directory, bool includeValidation = false, bool includeLookups = false) : this(	142✔
72	configuration, datasetBundle.DataSet)	142✔
73	{
74	DataExportRepository = configuration.DataExportRepository;	142✔
75	ExtractableCohort = extractableCohort;	142✔
76	DatasetBundle = datasetBundle;	142✔
77	ColumnsToExtract = columnsToExtract;	142✔
78
79	// create a copy of the columns so we can support Reset()
80	_origColumnsToExtract = ColumnsToExtract.ToList();	142✔
81
82	Salt = salt;	142✔
83	Directory = directory;	142✔
84	IncludeValidation = includeValidation;	142✔
85	TopX = -1;	142✔
86	}	142✔
87
88
89	/// <summary>
90	/// This version has less arguments because it goes back to the database and queries the configuration and explores who the cohort is etc, it will result in more database
91	/// queries than the more explicit constructor
92	/// </summary>
93	/// <param name="configuration"></param>
94	/// <param name="datasetBundle"></param>
95	/// <param name="includeValidation"></param>
96	/// <param name="includeLookups"></param>
97	public ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableDatasetBundle datasetBundle,
98	bool includeValidation = false, bool includeLookups = false) : this(configuration, datasetBundle.DataSet)	62✔
99	{
100	DataExportRepository = configuration.DataExportRepository;	62✔
101	//ExtractableCohort = ExtractableCohort.GetExtractableCohortByID((int) configuration.Cohort_ID);
102	ExtractableCohort = configuration.GetExtractableCohort();	62✔
103	DatasetBundle = datasetBundle;	62✔
104	ColumnsToExtract = new List<IColumn>(Configuration.GetAllExtractableColumnsFor(datasetBundle.DataSet));	62✔
105
106	// create a copy of the columns so we can support Reset()
107	_origColumnsToExtract = ColumnsToExtract.ToList();	62✔
108
109	Salt = new HICProjectSalt(Project);	62✔
110	Directory = new ExtractionDirectory(Project.ExtractionDirectory, configuration);	62✔
111	IncludeValidation = includeValidation;	62✔
112	TopX = -1;	62✔
113	}	62✔
114
115	public static readonly ExtractDatasetCommand EmptyCommand = new();	2✔
116
117
118	private ExtractDatasetCommand(IExtractionConfiguration configuration, IExtractableDataSet dataset) : base(	204✔
119	configuration)	204✔
120	{
121	var selectedDataSets = configuration.SelectedDataSets.Where(ds => ds.ExtractableDataSet_ID == dataset.ID)	530✔
122	.ToArray();	204✔
123
124	if (selectedDataSets.Length != 1)	204!
125	throw new Exception(	×
126	$"Could not find 1 ISelectedDataSets for ExtractionConfiguration '{configuration}' \| Dataset '{dataset}'");	×
127
128	SelectedDataSets = selectedDataSets[0];	204✔
129
130	ExtractionResults = new List<IExtractionResults>();	204✔
131	}	204✔
132
133	private ExtractDatasetCommand() : base(null)	2✔
134	{
135	}	2✔
136
137	/// <summary>
138	/// Resets the state of the command to when it was first constructed
139	/// </summary>
140	public void Reset()
141	{
142	ColumnsToExtract = _origColumnsToExtract.ToList();	×
143	QueryBuilder = null;	×
144	}	×
145
146	public void GenerateQueryBuilder()
147	{
148	var host = new ExtractionQueryBuilder(DataExportRepository);	112✔
149	QueryBuilder = host.GetSQLCommandForFullExtractionSet(this, out var substitutions);	112✔
150	ReleaseIdentifierSubstitutions = substitutions;	112✔
151	}	112✔
152
153	public override string ToString() => this == EmptyCommand ? "EmptyCommand" : DatasetBundle.DataSet.ToString();	208!
154
155	public override DirectoryInfo GetExtractionDirectory() => this == EmptyCommand	148!
156	? new DirectoryInfo(Path.GetTempPath())	148✔
157	: Directory.GetDirectoryForDataset(DatasetBundle.DataSet);	148✔
158
159	public override string DescribeExtractionImplementation() => QueryBuilder.SQL;	48✔
160
161	/// <inheritdoc/>
162	public DiscoveredServer GetDistinctLiveDatabaseServer()
163	{
164	IDataAccessPoint[] points = QueryBuilder?.TablesUsedInQuery != null	76!
165	? QueryBuilder.TablesUsedInQuery.ToArray()	76✔
166	: //get it from the request if it has been built	76✔
167	Catalogue.GetTableInfoList(false); //or from the Catalogue directly if the query hasn't been built	76✔
168
169	var singleServer = new DataAccessPointCollection(true, DataAccessContext.DataExport);	76✔
170	singleServer.AddRange(points);	76✔
171
172	return singleServer.GetDistinctServer();	76✔
173	}
174	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous