6245535001

Committed 20 Sep 2023 07:44AM UTC coverage: 57.013%. First build

Build # 6245535001

Build Type

push

github

Committed by

web-flow

Commit Message

8.1.0 Release (#1628)

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2.
- [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases)
- [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2)

---
updated-dependencies:
- dependency-name: Newtonsoft.Json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

Bumps [NLog](https://github.com/NLog/NLog) from 5.0.5 to 5.1.0.
- [Release notes](https://github.com/NLog/NLog/releases)
- [Changelog](https://github.com/NLog/NLog/blob/dev/CHANGELOG.md)
- [Commits](https://github.com/NLog/NLog/compare/v5.0.5...v5.1.0)

---
updated-dependencies:
- dependency-name: NLog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NLog from 5.0.5 to 5.1.0

* Fix -r flag - should have been --results-directory all along

* Bump Newtonsoft.Json from 13.0.1 to 13.0.2

* Bump YamlDotNet from 12.0.2 to 12.1.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 12.0.2 to 12.1.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v12.0.2...v12.1.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Moq from 4.18.2 to 4.18.3

Bumps [Moq](https://github.com/moq/moq4) from 4.18.2 to 4.18.3.
- [Release notes](https://github.com/moq/moq4/releases)
- [Changelog](https://github.com/moq/moq4/blob/main/CHANGELOG.md)
- [Commits](https://github.com/moq/moq4/compare/v4.18.2...v4.18.3)

---
updated-dependencies:
- dependency-name: Moq
... (continued)

Run Details

10732 of 20257 branches covered (0.0%)

Branch coverage included in aggregate %.

48141 of 48141 new or added lines in 1086 files covered. (100.0%)

30685 of 52388 relevant lines covered (58.57%)

7387.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.0

/Rdmp.Core/QueryBuilding/CohortQueryBuilderDependency.cs

﻿// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using FAnsi.Naming;
using Rdmp.Core.CohortCreation.Execution;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.Aggregation;
using Rdmp.Core.Curation.Data.Cohort;
using Rdmp.Core.Curation.Data.Cohort.Joinables;
using Rdmp.Core.Providers;
using Rdmp.Core.QueryBuilding.Parameters;
using Rdmp.Core.QueryCaching.Aggregation;

namespace Rdmp.Core.QueryBuilding;

/// <summary>
/// A single cohort set in a <see cref="CohortIdentificationConfiguration"/> which selects specific patients from the database by their unique <see cref="IColumn.IsExtractionIdentifier"/>.
/// Can include a join to a patient index table.  This class stores the cached (if available) uncached and partially Cached SQL for relevant subsections of the query (and the whole query).
/// So that the decision about whether to use the cache can be delayed till later
/// 
/// </summary>
public class CohortQueryBuilderDependency
{
    private readonly ICoreChildProvider _childProvider;
    private readonly IReadOnlyCollection<IPluginCohortCompiler> _pluginCohortCompilers;

    /// <summary>
    /// The primary table being queried
    /// </summary>
    public AggregateConfiguration CohortSet { get; }

    /// <summary>
    /// The relationship object describing the JOIN relationship between <see cref="CohortSet"/> and another optional table
    /// </summary>
    public JoinableCohortAggregateConfigurationUse PatientIndexTableIfAny { get; }

    /// <summary>
    /// The column in the <see cref="CohortSet"/> that is marked <see cref="IColumn.IsExtractionIdentifier"/>
    /// </summary>
    public AggregateDimension ExtractionIdentifierColumn { get; }

    /// <summary>
    /// The aggregate (query) referenced by <see cref="PatientIndexTableIfAny"/>
    /// </summary>
    public AggregateConfiguration JoinedTo { get; }

    /// <summary>
    /// The raw SQL that can be used to join the <see cref="CohortSet"/> and <see cref="PatientIndexTableIfAny"/> (if there is one).  Null if they exist
    /// on different servers (this is allowed only if the <see cref="CohortSet"/> is on the same server as the cache while the <see cref="PatientIndexTableIfAny"/>
    /// is remote).
    ///
    /// <para>This SQL does not include the parameter declaration SQL since it is designed for nesting e.g. in UNION / INTERSECT / EXCEPT hierarchy</para>
    /// </summary>
    public CohortQueryBuilderDependencySql SqlCacheless { get; private set; }

    /// <summary>
    /// The raw SQL for the <see cref="CohortSet"/> with a join against the cached artifact for the <see cref="PatientIndexTableIfAny"/>
    /// </summary>
    public CohortQueryBuilderDependencySql SqlPartiallyCached { get; private set; }

    /// <summary>
    /// Sql for a single cache fetch  that pulls the cached result of the <see cref="CohortSet"/> joined to <see cref="PatientIndexTableIfAny"/> (if there was any)
    /// </summary>
    public CohortQueryBuilderDependencySql SqlFullyCached { get; private set; }

    public CohortQueryBuilderDependencySql SqlJoinableCacheless { get; private set; }

    public CohortQueryBuilderDependencySql SqlJoinableCached { get; private set; }

    /// <summary>
    /// Locks on aggregate by ID
    /// </summary>
    private static readonly ConcurrentDictionary<int, object> AggregateLocks = new();


    public CohortQueryBuilderDependency(AggregateConfiguration cohortSet,
        JoinableCohortAggregateConfigurationUse patientIndexTableIfAny, ICoreChildProvider childProvider,
        IReadOnlyCollection<IPluginCohortCompiler> pluginCohortCompilers)
    {
        _childProvider = childProvider;
        _pluginCohortCompilers = pluginCohortCompilers;
        CohortSet = cohortSet;
        PatientIndexTableIfAny = patientIndexTableIfAny;

        //record the IsExtractionIdentifier column for the log (helps with debugging count issues)
        var eis = cohortSet?.AggregateDimensions?.Where(d => d.IsExtractionIdentifier).ToArray();

        //Multiple IsExtractionIdentifier columns is a big problem but it's handled elsewhere
        if (eis is { Length: 1 })
            ExtractionIdentifierColumn = eis[0];

        if (PatientIndexTableIfAny != null)
        {
            var join = _childProvider.AllJoinables.SingleOrDefault(j =>
                           j.ID == PatientIndexTableIfAny.JoinableCohortAggregateConfiguration_ID) ??
                       throw new Exception("ICoreChildProvider did not know about the provided patient index table");
            JoinedTo = _childProvider.AllAggregateConfigurations.SingleOrDefault(ac =>
                ac.ID == join.AggregateConfiguration_ID);

            if (JoinedTo == null)
                throw new Exception(
                    "ICoreChildProvider did not know about the provided patient index table AggregateConfiguration");
        }
    }

    public override string ToString() =>
        JoinedTo != null
            ? $"{CohortSet.Name}{PatientIndexTableIfAny.JoinType} JOIN {JoinedTo.Name}"
            : CohortSet.Name;

    public void Build(CohortQueryBuilderResult parent, ISqlParameter[] globals, CancellationToken cancellationToken)
    {
        cancellationToken.ThrowIfCancellationRequested();

        var isSolitaryPatientIndexTable = CohortSet.IsJoinablePatientIndexTable();

        // if it is a plugin aggregate we only want to ever serve up the cached SQL
        var pluginCohortCompiler = _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(CohortSet));
        var joinedToPluginCohortCompiler =
            JoinedTo == null ? null : _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(JoinedTo));

        if (pluginCohortCompiler != null)
        {
            if (joinedToPluginCohortCompiler != null)
                throw new Exception($"APIs cannot be joined together ('{CohortSet}' and '{JoinedTo}')");

            if (parent.CacheManager == null)
                throw new Exception(
                    $"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cache is configured on {CohortSet.GetCohortIdentificationConfigurationIfAny()}.  You must enable result caching to use plugin aggregates.");

            // It's a plugin aggregate so only ever run the cached SQL
            SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,
                pluginCohortCompiler, cancellationToken);

            if (SqlFullyCached == null)
                throw new Exception(
                    $"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cached results were found after running.");
            return;
        }

        //Includes the parameter declaration and no rename operations (i.e. couldn't be used for building the tree but can be used for cache hit testing).
        if (JoinedTo != null)
        {
            if (joinedToPluginCohortCompiler == null)
            {
                SqlJoinableCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(JoinedTo,
                    new QueryBuilderArgs(
                        new QueryBuilderCustomArgs(), //don't respect customizations in the inception bit!
                        globals));
                SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true, null,
                    cancellationToken);
            }
            else
            {
                // It is not possible to do a cacheless query because an API is involved
                SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true,
                    joinedToPluginCohortCompiler, cancellationToken);

                if (SqlJoinableCached == null)
                    throw new Exception(
                        $"Unable to build query for '{CohortSet}' because it joins to API cohort '{JoinedTo}' that did not exist in the cache");

                // Since the only way to query the dataset is using the cache we can pretend that it is the cacheless way
                // of querying it too.
                SqlJoinableCacheless = SqlJoinableCached;
            }
        }

        if (isSolitaryPatientIndexTable)
        {
            //explicit execution of a patient index table on its own
            //the full uncached SQL for the query
            SqlCacheless =
                CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet, new QueryBuilderArgs(parent.Customise, globals));

            if (SqlJoinableCached != null)
                throw new QueryBuildingException("Patient index tables can't use other patient index tables!");
        }
        else
        {
            //the full uncached SQL for the query
            SqlCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,
                new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,
                    SqlJoinableCacheless, parent.Customise, globals));


            //if the joined to table is cached we can generate a partial too with full sql for the outer sql block and a cache fetch join
            if (SqlJoinableCached != null)
                SqlPartiallyCached = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,
                    new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,
                        SqlJoinableCached, parent.Customise, globals));
        }

        //We would prefer a cache hit on the exact uncached SQL
        SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,
            pluginCohortCompiler, cancellationToken);

        // if we have a patient index table where the Sql is not fully cached then we should invalidate anyone using it
        if (isSolitaryPatientIndexTable && SqlFullyCached == null && parent.CacheManager != null)
            ClearCacheForUsersOfPatientIndexTable(parent.CacheManager, CohortSet);

        //but if that misses we would take a cache hit of an execution of the SqlPartiallyCached
        if (SqlFullyCached == null && SqlPartiallyCached != null)
            SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlPartiallyCached,
                isSolitaryPatientIndexTable, pluginCohortCompiler, cancellationToken);
    }

    private void ClearCacheForUsersOfPatientIndexTable(CachedAggregateConfigurationResultsManager cacheManager,
        AggregateConfiguration cohortSet)
    {
        if (cacheManager == null)
            return;

        var join = CohortSet.JoinableCohortAggregateConfiguration ?? throw new Exception(
            $"{nameof(AggregateConfiguration.JoinableCohortAggregateConfiguration)} was null for CohortSet {cohortSet} so we were unable to clear the joinable cache users");

        // get each Aggregate Configuration that joins using this patient index table
        foreach (var user in join.Users.Select(j => j.AggregateConfiguration))
            cacheManager.DeleteCacheEntryIfAny(user, AggregateOperation.IndexedExtractionIdentifierList);
    }

    private CohortQueryBuilderDependencySql GetCacheFetchSqlIfPossible(CohortQueryBuilderResult parent,
        AggregateConfiguration aggregate,
        CohortQueryBuilderDependencySql sql, bool isPatientIndexTable, IPluginCohortCompiler pluginCohortCompiler,
        CancellationToken cancellationToken)
    {
        if (parent.CacheManager == null)
            return null;

        var aggregateType = isPatientIndexTable
            ? AggregateOperation.JoinableInceptionQuery
            : AggregateOperation.IndexedExtractionIdentifierList;
        IHasFullyQualifiedNameToo existingTable;

        // since we might make a plugin API run call we had better lock this to
        var oLock = AggregateLocks.GetOrAdd(aggregate.ID, i => new object());
        lock (oLock)
        {
            // unless it is a plugin driven aggregate we need to assemble the SQL to check if the cache is stale
            if (pluginCohortCompiler == null)
            {
                var parameterSql =
                    QueryBuilder.GetParameterDeclarationSQL(sql.ParametersUsed.Clone()
                        .GetFinalResolvedParametersList());
                var hitTestSql = parameterSql + sql.Sql;
                existingTable = parent.CacheManager.GetLatestResultsTable(aggregate, aggregateType, hitTestSql);
            }
            else
            {
                existingTable =
                    parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType, out var oldDescription);

                if (pluginCohortCompiler.IsStale(aggregate, oldDescription)) existingTable = null;
            }

            // if there are no cached results in the destination (and it's a plugin cohort) then we need to run the plugin API call
            if (existingTable == null && pluginCohortCompiler != null)
            {
                // no cached results were there so run the plugin
                pluginCohortCompiler.Run(CohortSet, parent.CacheManager, cancellationToken);

                // try again now
                existingTable = parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType);

                if (existingTable == null)
                    throw new Exception(
                        $"Run method on {pluginCohortCompiler} failed to populate the Query Result Cache for {CohortSet}");
            }
        }

        //if there is a cached entry matching the cacheless SQL then we can just do a select from it (in theory)
        if (existingTable != null)
        {
            var sqlCachFetch =
                $@"{CachedAggregateConfigurationResultsManager.CachingPrefix}{aggregate.Name}*/{Environment.NewLine}select * from {existingTable.GetFullyQualifiedName()}{Environment.NewLine}";

            //Cache fetch does not require any parameters
            return new CohortQueryBuilderDependencySql(sqlCachFetch, new ParameterManager());
        }


        return null;
    }

    public string DescribeCachedState()
    {
        if (SqlFullyCached != null)
            return "Fully Cached";

        if (SqlPartiallyCached != null)
            return "Partially Cached";

        return SqlCacheless != null ? "Not Cached" : "Not Built";
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Concurrent;
9	using System.Collections.Generic;
10	using System.Linq;
11	using System.Threading;
12	using FAnsi.Naming;
13	using Rdmp.Core.CohortCreation.Execution;
14	using Rdmp.Core.Curation.Data;
15	using Rdmp.Core.Curation.Data.Aggregation;
16	using Rdmp.Core.Curation.Data.Cohort;
17	using Rdmp.Core.Curation.Data.Cohort.Joinables;
18	using Rdmp.Core.Providers;
19	using Rdmp.Core.QueryBuilding.Parameters;
20	using Rdmp.Core.QueryCaching.Aggregation;
21
22	namespace Rdmp.Core.QueryBuilding;
23
24	/// <summary>
25	/// A single cohort set in a <see cref="CohortIdentificationConfiguration"/> which selects specific patients from the database by their unique <see cref="IColumn.IsExtractionIdentifier"/>.
26	/// Can include a join to a patient index table. This class stores the cached (if available) uncached and partially Cached SQL for relevant subsections of the query (and the whole query).
27	/// So that the decision about whether to use the cache can be delayed till later
28	///
29	/// </summary>
30	public class CohortQueryBuilderDependency
31	{
32	private readonly ICoreChildProvider _childProvider;
33	private readonly IReadOnlyCollection<IPluginCohortCompiler> _pluginCohortCompilers;
34
35	/// <summary>
36	/// The primary table being queried
37	/// </summary>
38	public AggregateConfiguration CohortSet { get; }	3,898✔
39
40	/// <summary>
41	/// The relationship object describing the JOIN relationship between <see cref="CohortSet"/> and another optional table
42	/// </summary>
43	public JoinableCohortAggregateConfigurationUse PatientIndexTableIfAny { get; }	1,572✔
44
45	/// <summary>
46	/// The column in the <see cref="CohortSet"/> that is marked <see cref="IColumn.IsExtractionIdentifier"/>
47	/// </summary>
48	public AggregateDimension ExtractionIdentifierColumn { get; }	430✔
49
50	/// <summary>
51	/// The aggregate (query) referenced by <see cref="PatientIndexTableIfAny"/>
52	/// </summary>
53	public AggregateConfiguration JoinedTo { get; }	3,272✔
54
55	/// <summary>
56	/// The raw SQL that can be used to join the <see cref="CohortSet"/> and <see cref="PatientIndexTableIfAny"/> (if there is one). Null if they exist
57	/// on different servers (this is allowed only if the <see cref="CohortSet"/> is on the same server as the cache while the <see cref="PatientIndexTableIfAny"/>
58	/// is remote).
59	///
60	/// <para>This SQL does not include the parameter declaration SQL since it is designed for nesting e.g. in UNION / INTERSECT / EXCEPT hierarchy</para>
61	/// </summary>
62	public CohortQueryBuilderDependencySql SqlCacheless { get; private set; }	1,572✔
63
64	/// <summary>
65	/// The raw SQL for the <see cref="CohortSet"/> with a join against the cached artifact for the <see cref="PatientIndexTableIfAny"/>
66	/// </summary>
67	public CohortQueryBuilderDependencySql SqlPartiallyCached { get; private set; }	908✔
68
69	/// <summary>
70	/// Sql for a single cache fetch that pulls the cached result of the <see cref="CohortSet"/> joined to <see cref="PatientIndexTableIfAny"/> (if there was any)
71	/// </summary>
72	public CohortQueryBuilderDependencySql SqlFullyCached { get; private set; }	1,910✔
73
74	public CohortQueryBuilderDependencySql SqlJoinableCacheless { get; private set; }	532✔
75
76	public CohortQueryBuilderDependencySql SqlJoinableCached { get; private set; }	528✔
77
78	/// <summary>
79	/// Locks on aggregate by ID
80	/// </summary>
81	private static readonly ConcurrentDictionary<int, object> AggregateLocks = new();	2✔
82
83
84	public CohortQueryBuilderDependency(AggregateConfiguration cohortSet,	430✔
85	JoinableCohortAggregateConfigurationUse patientIndexTableIfAny, ICoreChildProvider childProvider,	430✔
86	IReadOnlyCollection<IPluginCohortCompiler> pluginCohortCompilers)	430✔
87	{
88	_childProvider = childProvider;	430✔
89	_pluginCohortCompilers = pluginCohortCompilers;	430✔
90	CohortSet = cohortSet;	430✔
91	PatientIndexTableIfAny = patientIndexTableIfAny;	430✔
92
93	//record the IsExtractionIdentifier column for the log (helps with debugging count issues)
94	var eis = cohortSet?.AggregateDimensions?.Where(d => d.IsExtractionIdentifier).ToArray();	916!
95
96	//Multiple IsExtractionIdentifier columns is a big problem but it's handled elsewhere
97	if (eis is { Length: 1 })	430✔
98	ExtractionIdentifierColumn = eis[0];	416✔
99
100	if (PatientIndexTableIfAny != null)	430✔
101	{
102	var join = _childProvider.AllJoinables.SingleOrDefault(j =>	78!
103	j.ID == PatientIndexTableIfAny.JoinableCohortAggregateConfiguration_ID) ??	560✔
104	throw new Exception("ICoreChildProvider did not know about the provided patient index table");	78✔
105	JoinedTo = _childProvider.AllAggregateConfigurations.SingleOrDefault(ac =>	78✔
106	ac.ID == join.AggregateConfiguration_ID);	1,704✔
107
108	if (JoinedTo == null)	78!
109	throw new Exception(	×
110	"ICoreChildProvider did not know about the provided patient index table AggregateConfiguration");	×
111	}
112	}	430✔
113
114	public override string ToString() =>
115	JoinedTo != null	1,030✔
116	? $"{CohortSet.Name}{PatientIndexTableIfAny.JoinType} JOIN {JoinedTo.Name}"	1,030✔
117	: CohortSet.Name;	1,030✔
118
119	public void Build(CohortQueryBuilderResult parent, ISqlParameter[] globals, CancellationToken cancellationToken)
120	{
121	cancellationToken.ThrowIfCancellationRequested();	430✔
122
123	var isSolitaryPatientIndexTable = CohortSet.IsJoinablePatientIndexTable();	430✔
124
125	// if it is a plugin aggregate we only want to ever serve up the cached SQL
126	var pluginCohortCompiler = _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(CohortSet));	592✔
127	var joinedToPluginCohortCompiler =	430✔
128	JoinedTo == null ? null : _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(JoinedTo));	476✔
129
130	if (pluginCohortCompiler != null)	430✔
131	{
132	if (joinedToPluginCohortCompiler != null)	14!
133	throw new Exception($"APIs cannot be joined together ('{CohortSet}' and '{JoinedTo}')");	×
134
135	if (parent.CacheManager == null)	14!
136	throw new Exception(	×
137	$"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cache is configured on {CohortSet.GetCohortIdentificationConfigurationIfAny()}. You must enable result caching to use plugin aggregates.");	×
138
139	// It's a plugin aggregate so only ever run the cached SQL
140	SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,	14✔
141	pluginCohortCompiler, cancellationToken);	14✔
142
143	if (SqlFullyCached == null)	14!
144	throw new Exception(	×
145	$"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cached results were found after running.");	×
146	return;	14✔
147	}
148
149	//Includes the parameter declaration and no rename operations (i.e. couldn't be used for building the tree but can be used for cache hit testing).
150	if (JoinedTo != null)	416✔
151	{
152	if (joinedToPluginCohortCompiler == null)	78✔
153	{
154	SqlJoinableCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(JoinedTo,	74✔
155	new QueryBuilderArgs(	74✔
156	new QueryBuilderCustomArgs(), //don't respect customizations in the inception bit!	74✔
157	globals));	74✔
158	SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true, null,	74✔
159	cancellationToken);	74✔
160	}
161	else
162	{
163	// It is not possible to do a cacheless query because an API is involved
164	SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true,	4✔
165	joinedToPluginCohortCompiler, cancellationToken);	4✔
166
167	if (SqlJoinableCached == null)	4!
168	throw new Exception(	×
169	$"Unable to build query for '{CohortSet}' because it joins to API cohort '{JoinedTo}' that did not exist in the cache");	×
170
171	// Since the only way to query the dataset is using the cache we can pretend that it is the cacheless way
172	// of querying it too.
173	SqlJoinableCacheless = SqlJoinableCached;	4✔
174	}
175	}
176
177	if (isSolitaryPatientIndexTable)	416✔
178	{
179	//explicit execution of a patient index table on its own
180	//the full uncached SQL for the query
181	SqlCacheless =	40✔
182	CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet, new QueryBuilderArgs(parent.Customise, globals));	40✔
183
184	if (SqlJoinableCached != null)	40!
185	throw new QueryBuildingException("Patient index tables can't use other patient index tables!");	×
186	}
187	else
188	{
189	//the full uncached SQL for the query
190	SqlCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,	376✔
191	new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,	376✔
192	SqlJoinableCacheless, parent.Customise, globals));	376✔
193
194
195	//if the joined to table is cached we can generate a partial too with full sql for the outer sql block and a cache fetch join
196	if (SqlJoinableCached != null)	360✔
197	SqlPartiallyCached = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,	38✔
198	new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,	38✔
199	SqlJoinableCached, parent.Customise, globals));	38✔
200	}
201
202	//We would prefer a cache hit on the exact uncached SQL
203	SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,	400✔
204	pluginCohortCompiler, cancellationToken);	400✔
205
206	// if we have a patient index table where the Sql is not fully cached then we should invalidate anyone using it
207	if (isSolitaryPatientIndexTable && SqlFullyCached == null && parent.CacheManager != null)	400✔
208	ClearCacheForUsersOfPatientIndexTable(parent.CacheManager, CohortSet);	22✔
209
210	//but if that misses we would take a cache hit of an execution of the SqlPartiallyCached
211	if (SqlFullyCached == null && SqlPartiallyCached != null)	400✔
212	SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlPartiallyCached,	30✔
213	isSolitaryPatientIndexTable, pluginCohortCompiler, cancellationToken);	30✔
214	}	400✔
215
216	private void ClearCacheForUsersOfPatientIndexTable(CachedAggregateConfigurationResultsManager cacheManager,
217	AggregateConfiguration cohortSet)
218	{
219	if (cacheManager == null)	22!
220	return;	×
221
222	var join = CohortSet.JoinableCohortAggregateConfiguration ?? throw new Exception(	22!
223	$"{nameof(AggregateConfiguration.JoinableCohortAggregateConfiguration)} was null for CohortSet {cohortSet} so we were unable to clear the joinable cache users");	22✔
224
225	// get each Aggregate Configuration that joins using this patient index table
226	foreach (var user in join.Users.Select(j => j.AggregateConfiguration))	104✔
227	cacheManager.DeleteCacheEntryIfAny(user, AggregateOperation.IndexedExtractionIdentifierList);	20✔
228	}	22✔
229
230	private CohortQueryBuilderDependencySql GetCacheFetchSqlIfPossible(CohortQueryBuilderResult parent,
231	AggregateConfiguration aggregate,
232	CohortQueryBuilderDependencySql sql, bool isPatientIndexTable, IPluginCohortCompiler pluginCohortCompiler,
233	CancellationToken cancellationToken)
234	{
235	if (parent.CacheManager == null)	522✔
236	return null;	292✔
237
238	var aggregateType = isPatientIndexTable	230✔
239	? AggregateOperation.JoinableInceptionQuery	230✔
240	: AggregateOperation.IndexedExtractionIdentifierList;	230✔
241	IHasFullyQualifiedNameToo existingTable;
242
243	// since we might make a plugin API run call we had better lock this to
244	var oLock = AggregateLocks.GetOrAdd(aggregate.ID, i => new object());	312✔
245	lock (oLock)	230✔
246	{
247	// unless it is a plugin driven aggregate we need to assemble the SQL to check if the cache is stale
248	if (pluginCohortCompiler == null)	230✔
249	{
250	var parameterSql =	212✔
251	QueryBuilder.GetParameterDeclarationSQL(sql.ParametersUsed.Clone()	212✔
252	.GetFinalResolvedParametersList());	212✔
253	var hitTestSql = parameterSql + sql.Sql;	212✔
254	existingTable = parent.CacheManager.GetLatestResultsTable(aggregate, aggregateType, hitTestSql);	212✔
255	}
256	else
257	{
258	existingTable =	18✔
259	parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType, out var oldDescription);	18✔
260
261	if (pluginCohortCompiler.IsStale(aggregate, oldDescription)) existingTable = null;	24✔
262	}
263
264	// if there are no cached results in the destination (and it's a plugin cohort) then we need to run the plugin API call
265	if (existingTable == null && pluginCohortCompiler != null)	230✔
266	{
267	// no cached results were there so run the plugin
268	pluginCohortCompiler.Run(CohortSet, parent.CacheManager, cancellationToken);	6✔
269
270	// try again now
271	existingTable = parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType);	6✔
272
273	if (existingTable == null)	6!
274	throw new Exception(	×
275	$"Run method on {pluginCohortCompiler} failed to populate the Query Result Cache for {CohortSet}");	×
276	}
277	}	230✔
278
279	//if there is a cached entry matching the cacheless SQL then we can just do a select from it (in theory)
280	if (existingTable != null)	230✔
281	{
282	var sqlCachFetch =	118✔
283	$@"{CachedAggregateConfigurationResultsManager.CachingPrefix}{aggregate.Name}/{Environment.NewLine}select from {existingTable.GetFullyQualifiedName()}{Environment.NewLine}";	118✔
284
285	//Cache fetch does not require any parameters
286	return new CohortQueryBuilderDependencySql(sqlCachFetch, new ParameterManager());	118✔
287	}
288
289
290	return null;	112✔
291	}
292
293	public string DescribeCachedState()
294	{
295	if (SqlFullyCached != null)	430!
296	return "Fully Cached";	×
297
298	if (SqlPartiallyCached != null)	430!
299	return "Partially Cached";	×
300
301	return SqlCacheless != null ? "Not Cached" : "Not Built";	430!
302	}
303	}

HicServices / RDMP / 6245535001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous