6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.08

/Rdmp.Core/QueryBuilding/CohortQueryBuilderHelper.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Linq;
using FAnsi.Discovery.QuerySyntax;
using FAnsi.Naming;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.Aggregation;
using Rdmp.Core.Curation.Data.Spontaneous;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.QueryBuilding.Parameters;

namespace Rdmp.Core.QueryBuilding;

/// <summary>
/// Helper for CohortQueryBuilder which contains code for building individual cohort identification subqueries.  Subqueries are actually built by
/// AggregateBuilder but this class handles tab indentation, parameter renaming (where there are other subqueries with conflicting sql parameter names),
/// injecting globals etc.
/// </summary>
public class CohortQueryBuilderHelper
{
    /// <summary>
    /// Returns the SQL you need to include in your nested query (in UNION / EXCEPT / INTERSECT).  This does not include parameter declarations (which
    /// would appear at the very top) and includes rename operations dependant on what has been written out before by (tracked by <see cref="ParameterManager"/>).
    ///
    /// <para>Use <paramref name="args"/> for the original un renamed / including parameter declarations e.g. to test for cache hits</para>
    /// </summary>
    /// <param name="aggregate"></param>
    /// <param name="args"></param>
    /// <returns></returns>
    public static CohortQueryBuilderDependencySql GetSQLForAggregate(AggregateConfiguration aggregate,
        QueryBuilderArgs args)
    {
        var isJoinAggregate = aggregate.IsCohortIdentificationAggregate;

        //make sure it is a valid configuration
        if (!aggregate.IsAcceptableAsCohortGenerationSource(out var reason))
            throw new QueryBuildingException(
                $"Cannot generate a cohort using AggregateConfiguration {aggregate} because:{reason}");

        //get the extraction identifier (method IsAcceptableAsCohortGenerationSource will ensure this linq returns 1 so no need to check again)
        var extractionIdentifier = aggregate.AggregateDimensions.Single(d => d.IsExtractionIdentifier);

        //create a builder but do it manually, we care about group bys etc or count(*) even
        AggregateBuilder builder;

        //we are getting SQL for a cohort identification aggregate without a HAVING/count statement so it is actually just 'select patientIdentifier from tableX'
        if (string.IsNullOrWhiteSpace(aggregate.HavingSQL) && string.IsNullOrWhiteSpace(aggregate.CountSQL))
        {
            //select list is the extraction identifier
            string selectList;

            if (!isJoinAggregate)
                selectList = extractionIdentifier.SelectSQL + (extractionIdentifier.Alias != null
                    ? $" {extractionIdentifier.Alias}"
                    : "");
            else
                //unless we are also including other columns because this is a patient index joinable inception query
                selectList = string.Join($",{Environment.NewLine}",
                    aggregate.AggregateDimensions.Select(e =>
                        e.SelectSQL +
                        (e.Alias != null
                            ? $" {e.Alias}"
                            : ""))); //joinable patient index tables have patientIdentifier + 1 or more other columns

            if (args.OverrideSelectList != null)
                selectList = args.OverrideSelectList;

            var limitationSQL = args?.OverrideLimitationSQL ?? "distinct";

            //select list is either [chi] or [chi],[mycolumn],[myexcitingcol] (in the case of a patient index table)
            builder = new AggregateBuilder(limitationSQL, selectList, aggregate, aggregate.ForcedJoins);

            //false makes it skip them in the SQL it generates (it uses them only in determining JOIN requirements etc but since we passed in the select SQL explicitly it should be the equivellent of telling the query builder to generate a regular select
            if (!isJoinAggregate)
                builder.AddColumn(extractionIdentifier, false);
            else
                builder.AddColumnRange(aggregate.AggregateDimensions.ToArray(), false);
        }
        else
        {
            if (args.OverrideSelectList != null)
                throw new NotSupportedException(
                    "Cannot override Select list on aggregates that have HAVING / Count SQL configured in them");

            builder = new AggregateBuilder("distinct", aggregate.CountSQL, aggregate, aggregate.ForcedJoins);

            //add the extraction information and do group by it
            if (!isJoinAggregate)
                builder.AddColumn(extractionIdentifier, true);
            else
                builder.AddColumnRange(aggregate.AggregateDimensions.ToArray(),
                    true); //it's a joinable inception query (See JoinableCohortAggregateConfiguration) - these are allowed additional columns

            builder.DoNotWriteOutOrderBy = true;
        }

        if (args.TopX != -1)
            builder.AggregateTopX = new SpontaneouslyInventedAggregateTopX(new MemoryRepository(), args.TopX,
                AggregateTopXOrderByDirection.Descending, null);

        //make sure builder has globals
        foreach (var global in args.Globals)
            builder.ParameterManager.AddGlobalParameter(global);

        //Add the inception join
        if (args.JoinIfAny != null)
            AddJoinToBuilder(aggregate, extractionIdentifier, builder, args);

        //set the where container
        builder.RootFilterContainer = aggregate.RootFilterContainer;

        //we will be harnessing the parameters via ImportAndElevate so do not add them to the SQL directly
        builder.DoNotWriteOutParameters = true;
        var builderSqlWithoutParameters = builder.SQL;

        //get the SQL from the builder (for the current configuration) - without parameters
        var currentBlock = builderSqlWithoutParameters;

        var toReturn = new CohortQueryBuilderDependencySql(currentBlock, builder.ParameterManager);

        if (args.JoinSql != null) toReturn.ParametersUsed.MergeWithoutRename(args.JoinSql.ParametersUsed);

        //we need to generate the full SQL with parameters (and no rename operations) so we can do cache hit tests
        //renaming is deferred to later
        return toReturn;
    }

    public static void AddJoinToBuilder(AggregateConfiguration user, IColumn usersExtractionIdentifier,
        AggregateBuilder builder, QueryBuilderArgs args)
    {
        var joinableTableAlias = args.JoinIfAny.GetJoinTableAlias();
        var joinDirection = args.JoinIfAny.GetJoinDirectionSQL();

        IHasRuntimeName joinOn = null;

        if (args.JoinedTo.Catalogue.IsApiCall(out var plugin))
        {
            if (plugin == null)
                throw new Exception(
                    $"No IPluginCohortCompiler was found that supports API cohort set '{args.JoinedTo}'");

            joinOn = plugin.GetJoinColumnForPatientIndexTable(args.JoinedTo);
        }
        else
        {
            joinOn = args.JoinedTo.AggregateDimensions.SingleOrDefault(d => d.IsExtractionIdentifier);
        }

        if (joinOn == null)
            throw new QueryBuildingException(
                $"AggregateConfiguration {user} uses a join aggregate (patient index aggregate) of {args.JoinedTo} but that AggregateConfiguration does not have an IsExtractionIdentifier dimension so how are we supposed to join these tables on the patient identifier?");

        // will end up with something like this where 51 is the ID of the joinTable:
        // LEFT Join (***INCEPTION QUERY***)ix51 on ["+TestDatabaseNames.Prefix+@"ScratchArea]..[BulkData].[patientIdentifier] = ix51.patientIdentifier

        builder.AddCustomLine(
            $" {joinDirection} Join ({Environment.NewLine}{TabIn(args.JoinSql.Sql, 1)}{Environment.NewLine}){joinableTableAlias}{Environment.NewLine}on {usersExtractionIdentifier.SelectSQL} = {joinableTableAlias}.{joinOn.GetRuntimeName()}",
            QueryComponent.JoinInfoJoin);
    }

    public static string TabIn(string str, int numberOfTabs)
    {
        if (string.IsNullOrWhiteSpace(str))
            return str;

        var tabs = new string('\t', numberOfTabs);
        return tabs + str.Replace(Environment.NewLine, Environment.NewLine + tabs);
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Linq;
9	using FAnsi.Discovery.QuerySyntax;
10	using FAnsi.Naming;
11	using Rdmp.Core.Curation.Data;
12	using Rdmp.Core.Curation.Data.Aggregation;
13	using Rdmp.Core.Curation.Data.Spontaneous;
14	using Rdmp.Core.MapsDirectlyToDatabaseTable;
15	using Rdmp.Core.QueryBuilding.Parameters;
16
17	namespace Rdmp.Core.QueryBuilding;
18
19	/// <summary>
20	/// Helper for CohortQueryBuilder which contains code for building individual cohort identification subqueries. Subqueries are actually built by
21	/// AggregateBuilder but this class handles tab indentation, parameter renaming (where there are other subqueries with conflicting sql parameter names),
22	/// injecting globals etc.
23	/// </summary>
24	public class CohortQueryBuilderHelper
25	{
26	/// <summary>
27	/// Returns the SQL you need to include in your nested query (in UNION / EXCEPT / INTERSECT). This does not include parameter declarations (which
28	/// would appear at the very top) and includes rename operations dependant on what has been written out before by (tracked by <see cref="ParameterManager"/>).
29	///
30	/// <para>Use <paramref name="args"/> for the original un renamed / including parameter declarations e.g. to test for cache hits</para>
31	/// </summary>
32	/// <param name="aggregate"></param>
33	/// <param name="args"></param>
34	/// <returns></returns>
35	public static CohortQueryBuilderDependencySql GetSQLForAggregate(AggregateConfiguration aggregate,
36	QueryBuilderArgs args)
37	{
38	var isJoinAggregate = aggregate.IsCohortIdentificationAggregate;	528✔
39
40	//make sure it is a valid configuration
41	if (!aggregate.IsAcceptableAsCohortGenerationSource(out var reason))	528!
42	throw new QueryBuildingException(	×
43	$"Cannot generate a cohort using AggregateConfiguration {aggregate} because:{reason}");	×
44
45	//get the extraction identifier (method IsAcceptableAsCohortGenerationSource will ensure this linq returns 1 so no need to check again)
46	var extractionIdentifier = aggregate.AggregateDimensions.Single(d => d.IsExtractionIdentifier);	1,262✔
47
48	//create a builder but do it manually, we care about group bys etc or count(*) even
49	AggregateBuilder builder;
50
51	//we are getting SQL for a cohort identification aggregate without a HAVING/count statement so it is actually just 'select patientIdentifier from tableX'
52	if (string.IsNullOrWhiteSpace(aggregate.HavingSQL) && string.IsNullOrWhiteSpace(aggregate.CountSQL))	528✔
53	{
54	//select list is the extraction identifier
55	string selectList;
56
57	if (!isJoinAggregate)	520✔
58	selectList = extractionIdentifier.SelectSQL + (extractionIdentifier.Alias != null	12!
59	? $" {extractionIdentifier.Alias}"	12✔
60	: "");	12✔
61	else
62	//unless we are also including other columns because this is a patient index joinable inception query
63	selectList = string.Join($",{Environment.NewLine}",	508✔
64	aggregate.AggregateDimensions.Select(e =>	508✔
65	e.SelectSQL +	1,222!
66	(e.Alias != null	1,222✔
67	? $" {e.Alias}"	1,222✔
68	: ""))); //joinable patient index tables have patientIdentifier + 1 or more other columns	1,222✔
69
70	if (args.OverrideSelectList != null)	520✔
71	selectList = args.OverrideSelectList;	4✔
72
73	var limitationSQL = args?.OverrideLimitationSQL ?? "distinct";	520!
74
75	//select list is either [chi] or [chi],[mycolumn],[myexcitingcol] (in the case of a patient index table)
76	builder = new AggregateBuilder(limitationSQL, selectList, aggregate, aggregate.ForcedJoins);	520✔
77
78	//false makes it skip them in the SQL it generates (it uses them only in determining JOIN requirements etc but since we passed in the select SQL explicitly it should be the equivellent of telling the query builder to generate a regular select
79	if (!isJoinAggregate)	520✔
80	builder.AddColumn(extractionIdentifier, false);	12✔
81	else
82	builder.AddColumnRange(aggregate.AggregateDimensions.ToArray(), false);	508✔
83	}
84	else
85	{
86	if (args.OverrideSelectList != null)	8!
87	throw new NotSupportedException(	×
88	"Cannot override Select list on aggregates that have HAVING / Count SQL configured in them");	×
89
90	builder = new AggregateBuilder("distinct", aggregate.CountSQL, aggregate, aggregate.ForcedJoins);	8✔
91
92	//add the extraction information and do group by it
93	if (!isJoinAggregate)	8!
94	builder.AddColumn(extractionIdentifier, true);	×
95	else
96	builder.AddColumnRange(aggregate.AggregateDimensions.ToArray(),	8✔
97	true); //it's a joinable inception query (See JoinableCohortAggregateConfiguration) - these are allowed additional columns	8✔
98
99	builder.DoNotWriteOutOrderBy = true;	8✔
100	}
101
102	if (args.TopX != -1)	528✔
103	builder.AggregateTopX = new SpontaneouslyInventedAggregateTopX(new MemoryRepository(), args.TopX,	6✔
104	AggregateTopXOrderByDirection.Descending, null);	6✔
105
106	//make sure builder has globals
107	foreach (var global in args.Globals)	1,072✔
108	builder.ParameterManager.AddGlobalParameter(global);	8✔
109
110	//Add the inception join
111	if (args.JoinIfAny != null)	528✔
112	AddJoinToBuilder(aggregate, extractionIdentifier, builder, args);	116✔
113
114	//set the where container
115	builder.RootFilterContainer = aggregate.RootFilterContainer;	528✔
116
117	//we will be harnessing the parameters via ImportAndElevate so do not add them to the SQL directly
118	builder.DoNotWriteOutParameters = true;	528✔
119	var builderSqlWithoutParameters = builder.SQL;	528✔
120
121	//get the SQL from the builder (for the current configuration) - without parameters
122	var currentBlock = builderSqlWithoutParameters;	528✔
123
124	var toReturn = new CohortQueryBuilderDependencySql(currentBlock, builder.ParameterManager);	528✔
125
126	if (args.JoinSql != null) toReturn.ParametersUsed.MergeWithoutRename(args.JoinSql.ParametersUsed);	644✔
127
128	//we need to generate the full SQL with parameters (and no rename operations) so we can do cache hit tests
129	//renaming is deferred to later
130	return toReturn;	512✔
131	}
132
133	public static void AddJoinToBuilder(AggregateConfiguration user, IColumn usersExtractionIdentifier,
134	AggregateBuilder builder, QueryBuilderArgs args)
135	{
136	var joinableTableAlias = args.JoinIfAny.GetJoinTableAlias();	116✔
137	var joinDirection = args.JoinIfAny.GetJoinDirectionSQL();	116✔
138
139	IHasRuntimeName joinOn = null;	116✔
140
141	if (args.JoinedTo.Catalogue.IsApiCall(out var plugin))	116✔
142	{
143	if (plugin == null)	8!
144	throw new Exception(	×
145	$"No IPluginCohortCompiler was found that supports API cohort set '{args.JoinedTo}'");	×
146
147	joinOn = plugin.GetJoinColumnForPatientIndexTable(args.JoinedTo);	8✔
148	}
149	else
150	{
151	joinOn = args.JoinedTo.AggregateDimensions.SingleOrDefault(d => d.IsExtractionIdentifier);	418✔
152	}
153
154	if (joinOn == null)	116!
155	throw new QueryBuildingException(	×
156	$"AggregateConfiguration {user} uses a join aggregate (patient index aggregate) of {args.JoinedTo} but that AggregateConfiguration does not have an IsExtractionIdentifier dimension so how are we supposed to join these tables on the patient identifier?");	×
157
158	// will end up with something like this where 51 is the ID of the joinTable:
159	// LEFT Join (*INCEPTION QUERY*)ix51 on ["+TestDatabaseNames.Prefix+@"ScratchArea]..[BulkData].[patientIdentifier] = ix51.patientIdentifier
160
161	builder.AddCustomLine(	116✔
162	$" {joinDirection} Join ({Environment.NewLine}{TabIn(args.JoinSql.Sql, 1)}{Environment.NewLine}){joinableTableAlias}{Environment.NewLine}on {usersExtractionIdentifier.SelectSQL} = {joinableTableAlias}.{joinOn.GetRuntimeName()}",	116✔
163	QueryComponent.JoinInfoJoin);	116✔
164	}	116✔
165
166	public static string TabIn(string str, int numberOfTabs)
167	{
168	if (string.IsNullOrWhiteSpace(str))	116!
169	return str;	×
170
171	var tabs = new string('\t', numberOfTabs);	116✔
172	return tabs + str.Replace(Environment.NewLine, Environment.NewLine + tabs);	116✔
173	}
174	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous