• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 6245535001

20 Sep 2023 07:44AM UTC coverage: 57.013%. First build
6245535001

push

github

web-flow
8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.11
/Rdmp.Core/QueryBuilding/ExtractionQueryBuilder.cs
1
// Copyright (c) The University of Dundee 2018-2019
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
using System;
8
using System.Collections.Generic;
9
using System.Linq;
10
using FAnsi.Discovery.QuerySyntax;
11
using Rdmp.Core.DataExport;
12
using Rdmp.Core.DataExport.Data;
13
using Rdmp.Core.DataExport.DataExtraction.Commands;
14
using Rdmp.Core.MapsDirectlyToDatabaseTable;
15
using Rdmp.Core.Repositories;
16
using Rdmp.Core.Repositories.Managers;
17
using TypeGuesser;
18

19
namespace Rdmp.Core.QueryBuilding;
20

21
/// <summary>
22
/// Calculates the Extraction SQL for extracting a given ExtractDatasetCommand.  This is done by creating a normal QueryBuilder and then adding adjustment
23
/// components to it to link against the cohort, drop the private identifier column, add the release identifier column etc.
24
/// </summary>
25
public class ExtractionQueryBuilder
26
{
27
    private readonly IDataExportRepository _repository;
28

29
    public ExtractionQueryBuilder(IDataExportRepository repository)
112✔
30
    {
31
        _repository = repository;
112✔
32
    }
112✔
33

34
    /// <summary>
35
    /// This produces the SQL that would retrieve the specified dataset columns including any JOINS
36
    /// 
37
    /// <para>It uses:
38
    /// QueryBuilder and then it adds some custom lines for linking to the cohort</para>
39
    /// </summary>
40
    /// <returns></returns>
41
    public QueryBuilder GetSQLCommandForFullExtractionSet(ExtractDatasetCommand request,
42
        out List<ReleaseIdentifierSubstitution> substitutions)
43
    {
44
        if (request.QueryBuilder != null)
112!
45
            throw new Exception(
×
46
                "Creation of a QueryBuilder from a request can only happen once, to access the results of the creation use the cached answer in the request.QueryBuilder property");
×
47

48
        if (!request.ColumnsToExtract.Any())
112!
49
            throw new Exception("No columns are marked for extraction in this configuration");
×
50

51
        if (request.ExtractableCohort == null)
112!
52
            throw new NullReferenceException("No Cohort selected");
×
53
            
54
        var databaseType = request.Catalogue.GetDistinctLiveDatabaseServerType() ?? throw new NotSupportedException(
112!
55
            $"Catalogue {request.Catalogue} did not know what DatabaseType it hosted, how can we extract from it! does it have no TableInfos?");
112✔
56
        var syntaxHelper = new QuerySyntaxHelperFactory().Create(databaseType);
112✔
57

58
        substitutions = new List<ReleaseIdentifierSubstitution>();
112✔
59

60
        var memoryRepository = new MemoryRepository();
112✔
61

62
        switch (request.ColumnsToExtract.Count(c => c.IsExtractionIdentifier))
938!
63
        {
64
            //no extraction identifiers
65
            case 0:
66
                throw new Exception(
×
67
                    $"There are no Columns in this dataset ({request}) marked as IsExtractionIdentifier");
×
68

69
            //a single extraction identifier e.g. CHI X died on date Y with conditions a,b and c
70
            case 1:
71
                substitutions.Add(new ReleaseIdentifierSubstitution(memoryRepository,
112✔
72
                    request.ColumnsToExtract.FirstOrDefault(c => c.IsExtractionIdentifier), request.ExtractableCohort,
116✔
73
                    false, syntaxHelper));
112✔
74
                break;
112✔
75

76
            //multiple extraction identifiers e.g. Mother X had Babies A, B, C where A,B and C are all CHIs that must be subbed for ProCHIs
77
            default:
78
                foreach (var columnToSubstituteForReleaseIdentifier in request.ColumnsToExtract.Where(c =>
×
79
                             c.IsExtractionIdentifier))
×
80
                    substitutions.Add(new ReleaseIdentifierSubstitution(memoryRepository,
×
81
                        columnToSubstituteForReleaseIdentifier, request.ExtractableCohort, true, syntaxHelper));
×
82
                break;
83
        }
84

85
        var hashingAlgorithm =
112✔
86
            _repository.DataExportPropertyManager.GetValue(DataExportProperty.HashingAlgorithmPattern);
112✔
87
        if (string.IsNullOrWhiteSpace(hashingAlgorithm))
112✔
88
            hashingAlgorithm = null;
102✔
89

90
        //identify any tables we are supposed to force join to
91
        var forcedJoins = request.SelectedDataSets.SelectedDataSetsForcedJoins;
112✔
92

93
        var queryBuilder =
112✔
94
            new QueryBuilder("DISTINCT ", hashingAlgorithm, forcedJoins.Select(s => s.TableInfo).ToArray())
2✔
95
            {
112✔
96
                TopX = request.TopX
112✔
97
            };
112✔
98

99
        queryBuilder.SetSalt(request.Salt.GetSalt());
112✔
100

101
        //add the constant parameters
102
        foreach (var parameter in GetConstantParameters(syntaxHelper, request.Configuration, request.ExtractableCohort))
672✔
103
            queryBuilder.ParameterManager.AddGlobalParameter(parameter);
224✔
104

105
        //add the global parameters
106
        foreach (var globalExtractionFilterParameter in request.Configuration.GlobalExtractionFilterParameters)
228✔
107
            queryBuilder.ParameterManager.AddGlobalParameter(globalExtractionFilterParameter);
2✔
108

109
        //remove the identification column from the query
110
        request.ColumnsToExtract.RemoveAll(c => c.IsExtractionIdentifier);
938✔
111

112
        //add in the ReleaseIdentifier in place of the identification column
113
        queryBuilder.AddColumnRange(substitutions.ToArray());
112✔
114

115
        //add the rest of the columns to the query
116
        queryBuilder.AddColumnRange(request.ColumnsToExtract.Cast<IColumn>().ToArray());
112✔
117

118
        //add the users selected filters
119
        queryBuilder.RootFilterContainer = request.Configuration.GetFilterContainerFor(request.DatasetBundle.DataSet);
112✔
120

121
        var externalCohortTable =
112✔
122
            _repository.GetObjectByID<ExternalCohortTable>(request.ExtractableCohort.ExternalCohortTable_ID);
112✔
123

124
        if (request.ExtractableCohort != null)
112✔
125
        {
126
            //the JOIN with the cohort table:
127
            var cohortJoin = substitutions.Count == 1
112!
128
                ? $" INNER JOIN {externalCohortTable.TableName} ON {substitutions.Single().JoinSQL}"
112✔
129
                : $" INNER JOIN {externalCohortTable.TableName} ON {string.Join(" OR ", substitutions.Select(s => s.JoinSQL))}";
112✔
130

131
            //add the JOIN in after any other joins
132
            queryBuilder.AddCustomLine(cohortJoin, QueryComponent.JoinInfoJoin);
112✔
133

134
            //add the filter cohortID because our new Cohort system uses ID number and a giant combo table with all the cohorts in it we need to say Select XX from XX join Cohort Where Cohort number = Y
135
            queryBuilder.AddCustomLine(request.ExtractableCohort.WhereSQL(), QueryComponent.WHERE);
112✔
136
        }
137

138
        HandleBatching(request, queryBuilder, syntaxHelper);
112✔
139

140
        request.QueryBuilder = queryBuilder;
112✔
141
        return queryBuilder;
112✔
142
    }
143

144
    private void HandleBatching(ExtractDatasetCommand request, QueryBuilder queryBuilder,
145
        IQuerySyntaxHelper syntaxHelper)
146
    {
147
        var batch = request.SelectedDataSets.ExtractionProgressIfAny;
112✔
148
        if (batch == null)
112✔
149
            // there is no batching going on
150
            return;
100✔
151

152
        // this is a batch resume if we have made some progress already
153
        request.IsBatchResume = batch.ProgressDate.HasValue;
12✔
154

155
        var start = batch.ProgressDate ?? batch.StartDate ?? throw new QueryBuildingException(
12!
156
            $"It was not possible to build a batch extraction query for '{request}' because there is no {nameof(ExtractionProgress.StartDate)} or {nameof(ExtractionProgress.ProgressDate)} set on the {nameof(ExtractionProgress)}");
12✔
157

158
        if (batch.NumberOfDaysPerBatch <= 0)
12!
159
            throw new QueryBuildingException(
×
160
                $"{nameof(ExtractionProgress.NumberOfDaysPerBatch)} was {batch.NumberOfDaysPerBatch} for '{request}'");
×
161

162
        var ei = batch.ExtractionInformation;
12✔
163

164

165
        var end = start.AddDays(batch.NumberOfDaysPerBatch);
12✔
166

167
        // Don't load into the future / past end of dataset
168
        if (end > (batch.EndDate ?? DateTime.Now)) end = batch.EndDate ?? DateTime.Now;
14!
169

170
        request.BatchStart = start;
12✔
171
        request.BatchEnd = end;
12✔
172

173
        var line =
12✔
174
            // if it is a first batch, also pull the null dates
12✔
175
            !request.IsBatchResume
12✔
176
                ? $"(({ei.SelectSQL} >= @batchStart AND {ei.SelectSQL} < @batchEnd) OR {ei.SelectSQL} is null)"
12✔
177
                :
12✔
178
                // it is a subsequent batch
12✔
179
                $"({ei.SelectSQL} >= @batchStart AND {ei.SelectSQL} < @batchEnd)";
12✔
180

181

182
        queryBuilder.AddCustomLine(line, QueryComponent.WHERE);
12✔
183

184
        var batchStartDeclaration =
12✔
185
            syntaxHelper.GetParameterDeclaration("@batchStart", new DatabaseTypeRequest(typeof(DateTime)));
12✔
186
        var batchStartParameter =
12✔
187
            new ConstantParameter(batchStartDeclaration, FormatDateAsParameterValue(start), null, syntaxHelper);
12✔
188
        queryBuilder.ParameterManager.AddGlobalParameter(batchStartParameter);
12✔
189

190
        var batchEndDeclaration =
12✔
191
            syntaxHelper.GetParameterDeclaration("@batchEnd", new DatabaseTypeRequest(typeof(DateTime)));
12✔
192
        var batchEndParameter =
12✔
193
            new ConstantParameter(batchEndDeclaration, FormatDateAsParameterValue(end), null, syntaxHelper);
12✔
194
        queryBuilder.ParameterManager.AddGlobalParameter(batchEndParameter);
12✔
195
    }
12✔
196

197
    private static string FormatDateAsParameterValue(DateTime dt) => $"'{dt.Year:D4}-{dt.Month:D2}-{dt.Day:D2}'";
24✔
198

199
    public static List<ConstantParameter> GetConstantParameters(IQuerySyntaxHelper syntaxHelper,
200
        IExtractionConfiguration configuration, IExtractableCohort extractableCohort)
201
    {
202
        //if the server doesn't support parameters then don't try to add them
203
        if (!syntaxHelper.SupportsEmbeddedParameters())
112!
204
            return new List<ConstantParameter>();
×
205

206
        var toReturn = new List<ConstantParameter>();
112✔
207

208
        if (syntaxHelper.DatabaseType == FAnsi.DatabaseType.Oracle)
112!
209
            return toReturn;
×
210

211
        var project = configuration.Project;
112✔
212

213
        if (project.ProjectNumber == null)
112!
214
            throw new ProjectNumberException("Project number has not been entered, cannot create constant parameters");
×
215

216
        if (extractableCohort == null)
112!
217
            throw new Exception("Cohort has not been selected, cannot create constant parameters");
×
218

219
        var externalCohortTable = extractableCohort.ExternalCohortTable;
112✔
220

221
        var declarationSqlCohortId =
112✔
222
            syntaxHelper.GetParameterDeclaration("@CohortDefinitionID", new DatabaseTypeRequest(typeof(int)));
112✔
223
        var declarationSqlProjectNumber =
112✔
224
            syntaxHelper.GetParameterDeclaration("@ProjectNumber", new DatabaseTypeRequest(typeof(int)));
112✔
225

226
        toReturn.Add(new ConstantParameter(declarationSqlCohortId, extractableCohort.OriginID.ToString(),
112✔
227
            $"The ID of the cohort in {externalCohortTable.TableName}", syntaxHelper));
112✔
228
        toReturn.Add(new ConstantParameter(declarationSqlProjectNumber, project.ProjectNumber.ToString(),
112✔
229
            $"The project number of project {project.Name}", syntaxHelper));
112✔
230

231
        return toReturn;
112✔
232
    }
233
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc