13318089130

Committed 13 Feb 2025 10:13PM UTC coverage: 57.398% (+0.004%) from 57.394%

Build # 13318089130

Build Type

Pull #2134

github

Committed by

jas88

Commit Message

Update ChildProviderTests.cs

Fix up TestUpTo method

Pull Request Pull Request #2134: CodeQL fixups

Run Details

11346 of 21308 branches covered (53.25%)

Branch coverage included in aggregate %.

104 of 175 new or added lines in 45 files covered. (59.43%)

362 existing lines in 23 files now uncovered.

32218 of 54590 relevant lines covered (59.02%)

17091.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.86

/Rdmp.Core/QueryBuilding/CohortQueryBuilderDependency.cs

﻿// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using FAnsi.Naming;
using Rdmp.Core.CohortCreation.Execution;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.Aggregation;
using Rdmp.Core.Curation.Data.Cohort;
using Rdmp.Core.Curation.Data.Cohort.Joinables;
using Rdmp.Core.Providers;
using Rdmp.Core.QueryBuilding.Parameters;
using Rdmp.Core.QueryCaching.Aggregation;

namespace Rdmp.Core.QueryBuilding;

/// <summary>
/// A single cohort set in a <see cref="CohortIdentificationConfiguration"/> which selects specific patients from the database by their unique <see cref="IColumn.IsExtractionIdentifier"/>.
/// Can include a join to a patient index table.  This class stores the cached (if available) uncached and partially Cached SQL for relevant subsections of the query (and the whole query).
/// So that the decision about whether to use the cache can be delayed till later
/// 
/// </summary>
public class CohortQueryBuilderDependency
{
    private readonly IReadOnlyCollection<IPluginCohortCompiler> _pluginCohortCompilers;

    /// <summary>
    /// The primary table being queried
    /// </summary>
    public AggregateConfiguration CohortSet { get; }

    /// <summary>
    /// The relationship object describing the JOIN relationship between <see cref="CohortSet"/> and another optional table
    /// </summary>
    public JoinableCohortAggregateConfigurationUse PatientIndexTableIfAny { get; }

    /// <summary>
    /// The column in the <see cref="CohortSet"/> that is marked <see cref="IColumn.IsExtractionIdentifier"/>
    /// </summary>
    public AggregateDimension ExtractionIdentifierColumn { get; }

    /// <summary>
    /// The aggregate (query) referenced by <see cref="PatientIndexTableIfAny"/>
    /// </summary>
    public AggregateConfiguration JoinedTo { get; }

    /// <summary>
    /// The raw SQL that can be used to join the <see cref="CohortSet"/> and <see cref="PatientIndexTableIfAny"/> (if there is one).  Null if they exist
    /// on different servers (this is allowed only if the <see cref="CohortSet"/> is on the same server as the cache while the <see cref="PatientIndexTableIfAny"/>
    /// is remote).
    ///
    /// <para>This SQL does not include the parameter declaration SQL since it is designed for nesting e.g. in UNION / INTERSECT / EXCEPT hierarchy</para>
    /// </summary>
    public CohortQueryBuilderDependencySql SqlCacheless { get; private set; }

    /// <summary>
    /// The raw SQL for the <see cref="CohortSet"/> with a join against the cached artifact for the <see cref="PatientIndexTableIfAny"/>
    /// </summary>
    public CohortQueryBuilderDependencySql SqlPartiallyCached { get; private set; }

    /// <summary>
    /// Sql for a single cache fetch  that pulls the cached result of the <see cref="CohortSet"/> joined to <see cref="PatientIndexTableIfAny"/> (if there was any)
    /// </summary>
    public CohortQueryBuilderDependencySql SqlFullyCached { get; private set; }

    public CohortQueryBuilderDependencySql SqlJoinableCacheless { get; private set; }

    public CohortQueryBuilderDependencySql SqlJoinableCached { get; private set; }

    /// <summary>
    /// Locks on aggregate by ID
    /// </summary>
    private static readonly ConcurrentDictionary<int, object> AggregateLocks = new();


    public CohortQueryBuilderDependency(AggregateConfiguration cohortSet,
        JoinableCohortAggregateConfigurationUse patientIndexTableIfAny, ICoreChildProvider childProvider,
        IReadOnlyCollection<IPluginCohortCompiler> pluginCohortCompilers)
    {
        var childProvider1 = childProvider;
        _pluginCohortCompilers = pluginCohortCompilers;
        CohortSet = cohortSet;
        PatientIndexTableIfAny = patientIndexTableIfAny;

        //record the IsExtractionIdentifier column for the log (helps with debugging count issues)
        var eis = cohortSet?.AggregateDimensions?.Where(d => d.IsExtractionIdentifier).ToArray();

        //Multiple IsExtractionIdentifier columns is a big problem but it's handled elsewhere
        if (eis is { Length: 1 })
            ExtractionIdentifierColumn = eis[0];

        if (PatientIndexTableIfAny != null)
        {
            var join = childProvider1.AllJoinables.SingleOrDefault(j =>
                           j.ID == PatientIndexTableIfAny.JoinableCohortAggregateConfiguration_ID) ??
                       throw new Exception("ICoreChildProvider did not know about the provided patient index table");
            JoinedTo = childProvider1.AllAggregateConfigurations.SingleOrDefault(ac =>
                ac.ID == join.AggregateConfiguration_ID);

            if (JoinedTo == null)
                throw new Exception(
                    "ICoreChildProvider did not know about the provided patient index table AggregateConfiguration");
        }
    }

    public override string ToString() =>
        JoinedTo != null
            ? $"{CohortSet.Name}{PatientIndexTableIfAny.JoinType} JOIN {JoinedTo.Name}"
            : CohortSet.Name;

    public void Build(CohortQueryBuilderResult parent, ISqlParameter[] globals, CancellationToken cancellationToken)
    {
        cancellationToken.ThrowIfCancellationRequested();

        var isSolitaryPatientIndexTable = CohortSet.IsJoinablePatientIndexTable();

        // if it is a plugin aggregate we only want to ever serve up the cached SQL
        var pluginCohortCompiler = _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(CohortSet));
        var joinedToPluginCohortCompiler =
            JoinedTo == null ? null : _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(JoinedTo));

        if (pluginCohortCompiler != null)
        {
            if (joinedToPluginCohortCompiler != null)
                throw new Exception($"APIs cannot be joined together ('{CohortSet}' and '{JoinedTo}')");

            if (parent.CacheManager == null)
                throw new Exception(
                    $"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cache is configured on {CohortSet.GetCohortIdentificationConfigurationIfAny()}.  You must enable result caching to use plugin aggregates.");

            // It's a plugin aggregate so only ever run the cached SQL
            SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,
                pluginCohortCompiler, cancellationToken);

            if (SqlFullyCached == null)
                throw new Exception(
                    $"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cached results were found after running.");
            return;
        }

        //Includes the parameter declaration and no rename operations (i.e. couldn't be used for building the tree but can be used for cache hit testing).
        if (JoinedTo != null)
        {
            if (joinedToPluginCohortCompiler == null)
            {
                SqlJoinableCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(JoinedTo,
                    new QueryBuilderArgs(
                        new QueryBuilderCustomArgs(), //don't respect customizations in the inception bit!
                        globals));
                SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true, null,
                    cancellationToken);
            }
            else
            {
                // It is not possible to do a cacheless query because an API is involved
                SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true,
                    joinedToPluginCohortCompiler, cancellationToken);

                // Since the only way to query the dataset is using the cache we can pretend that it is the cacheless way
                // of querying it too.
                SqlJoinableCacheless = SqlJoinableCached ?? throw new Exception(
                    $"Unable to build query for '{CohortSet}' because it joins to API cohort '{JoinedTo}' that did not exist in the cache");
            }
        }

        if (isSolitaryPatientIndexTable)
        {
            //explicit execution of a patient index table on its own
            //the full uncached SQL for the query
            SqlCacheless =
                CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet, new QueryBuilderArgs(parent.Customise, globals));

            if (SqlJoinableCached != null)
                throw new QueryBuildingException("Patient index tables can't use other patient index tables!");
        }
        else
        {
            //the full uncached SQL for the query
            SqlCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,
                new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,
                    SqlJoinableCacheless, parent.Customise, globals));


            //if the joined to table is cached we can generate a partial too with full sql for the outer sql block and a cache fetch join
            if (SqlJoinableCached != null)
                SqlPartiallyCached = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,
                    new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,
                        SqlJoinableCached, parent.Customise, globals));
        }

        //We would prefer a cache hit on the exact uncached SQL
        SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,
            pluginCohortCompiler, cancellationToken);

        // if we have a patient index table where the Sql is not fully cached then we should invalidate anyone using it
        if (isSolitaryPatientIndexTable && SqlFullyCached == null && parent.CacheManager != null)
            ClearCacheForUsersOfPatientIndexTable(parent.CacheManager, CohortSet);

        //but if that misses we would take a cache hit of an execution of the SqlPartiallyCached
        if (SqlFullyCached == null && SqlPartiallyCached != null)
            SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlPartiallyCached,
                isSolitaryPatientIndexTable, pluginCohortCompiler, cancellationToken);
    }

    private void ClearCacheForUsersOfPatientIndexTable(CachedAggregateConfigurationResultsManager cacheManager,
        AggregateConfiguration cohortSet)
    {
        if (cacheManager == null)
            return;

        var join = CohortSet.JoinableCohortAggregateConfiguration ?? throw new Exception(
            $"{nameof(AggregateConfiguration.JoinableCohortAggregateConfiguration)} was null for CohortSet {cohortSet} so we were unable to clear the joinable cache users");

        // get each Aggregate Configuration that joins using this patient index table
        foreach (var user in join.Users.Select(j => j.AggregateConfiguration))
            cacheManager.DeleteCacheEntryIfAny(user, AggregateOperation.IndexedExtractionIdentifierList);
    }

    private CohortQueryBuilderDependencySql GetCacheFetchSqlIfPossible(CohortQueryBuilderResult parent,
        AggregateConfiguration aggregate,
        CohortQueryBuilderDependencySql sql, bool isPatientIndexTable, IPluginCohortCompiler pluginCohortCompiler,
        CancellationToken cancellationToken)
    {
        if (parent.CacheManager == null)
            return null;

        var aggregateType = isPatientIndexTable
            ? AggregateOperation.JoinableInceptionQuery
            : AggregateOperation.IndexedExtractionIdentifierList;
        IHasFullyQualifiedNameToo existingTable;

        // since we might make a plugin API run call we had better lock this to
        var oLock = AggregateLocks.GetOrAdd(aggregate.ID, i => new object());
        lock (oLock)
        {
            // unless it is a plugin driven aggregate we need to assemble the SQL to check if the cache is stale
            if (pluginCohortCompiler == null)
            {
                var parameterSql =
                    QueryBuilder.GetParameterDeclarationSQL(sql.ParametersUsed.Clone()
                        .GetFinalResolvedParametersList());
                var hitTestSql = parameterSql + sql.Sql;
                existingTable = parent.CacheManager.GetLatestResultsTable(aggregate, aggregateType, hitTestSql);
            }
            else
            {
                existingTable =
                    parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType, out var oldDescription);

                if (pluginCohortCompiler.IsStale(aggregate, oldDescription)) existingTable = null;
            }

            // if there are no cached results in the destination (and it's a plugin cohort) then we need to run the plugin API call
            if (existingTable == null && pluginCohortCompiler != null)
            {
                // no cached results were there so run the plugin
                pluginCohortCompiler.Run(CohortSet, parent.CacheManager, cancellationToken);

                // try again now
                existingTable = parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType);

                if (existingTable == null)
                    throw new Exception(
                        $"Run method on {pluginCohortCompiler} failed to populate the Query Result Cache for {CohortSet}");
            }
        }

        //if there is a cached entry matching the cacheless SQL then we can just do a select from it (in theory)
        if (existingTable != null)
        {
            var sqlCachFetch =
                $@"{CachedAggregateConfigurationResultsManager.CachingPrefix}{aggregate.Name}*/{Environment.NewLine}select * from {existingTable.GetFullyQualifiedName()}{Environment.NewLine}";

            //Cache fetch does not require any parameters
            return new CohortQueryBuilderDependencySql(sqlCachFetch, new ParameterManager());
        }


        return null;
    }

    public string DescribeCachedState()
    {
        if (SqlFullyCached != null)
            return "Fully Cached";

        if (SqlPartiallyCached != null)
            return "Partially Cached";

        return SqlCacheless != null ? "Not Cached" : "Not Built";
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Concurrent;
9	using System.Collections.Generic;
10	using System.Linq;
11	using System.Threading;
12	using FAnsi.Naming;
13	using Rdmp.Core.CohortCreation.Execution;
14	using Rdmp.Core.Curation.Data;
15	using Rdmp.Core.Curation.Data.Aggregation;
16	using Rdmp.Core.Curation.Data.Cohort;
17	using Rdmp.Core.Curation.Data.Cohort.Joinables;
18	using Rdmp.Core.Providers;
19	using Rdmp.Core.QueryBuilding.Parameters;
20	using Rdmp.Core.QueryCaching.Aggregation;
21
22	namespace Rdmp.Core.QueryBuilding;
23
24	/// <summary>
25	/// A single cohort set in a <see cref="CohortIdentificationConfiguration"/> which selects specific patients from the database by their unique <see cref="IColumn.IsExtractionIdentifier"/>.
26	/// Can include a join to a patient index table. This class stores the cached (if available) uncached and partially Cached SQL for relevant subsections of the query (and the whole query).
27	/// So that the decision about whether to use the cache can be delayed till later
28	///
29	/// </summary>
30	public class CohortQueryBuilderDependency
31	{
32	private readonly IReadOnlyCollection<IPluginCohortCompiler> _pluginCohortCompilers;
33
34	/// <summary>
35	/// The primary table being queried
36	/// </summary>
37	public AggregateConfiguration CohortSet { get; }	4,164✔
38
39	/// <summary>
40	/// The relationship object describing the JOIN relationship between <see cref="CohortSet"/> and another optional table
41	/// </summary>
42	public JoinableCohortAggregateConfigurationUse PatientIndexTableIfAny { get; }	1,628✔
43
44	/// <summary>
45	/// The column in the <see cref="CohortSet"/> that is marked <see cref="IColumn.IsExtractionIdentifier"/>
46	/// </summary>
47	public AggregateDimension ExtractionIdentifierColumn { get; }	458✔
48
49	/// <summary>
50	/// The aggregate (query) referenced by <see cref="PatientIndexTableIfAny"/>
51	/// </summary>
52	public AggregateConfiguration JoinedTo { get; }	3,466✔
53
54	/// <summary>
55	/// The raw SQL that can be used to join the <see cref="CohortSet"/> and <see cref="PatientIndexTableIfAny"/> (if there is one). Null if they exist
56	/// on different servers (this is allowed only if the <see cref="CohortSet"/> is on the same server as the cache while the <see cref="PatientIndexTableIfAny"/>
57	/// is remote).
58	///
59	/// <para>This SQL does not include the parameter declaration SQL since it is designed for nesting e.g. in UNION / INTERSECT / EXCEPT hierarchy</para>
60	/// </summary>
61	public CohortQueryBuilderDependencySql SqlCacheless { get; private set; }	1,682✔
62
63	/// <summary>
64	/// The raw SQL for the <see cref="CohortSet"/> with a join against the cached artifact for the <see cref="PatientIndexTableIfAny"/>
65	/// </summary>
66	public CohortQueryBuilderDependencySql SqlPartiallyCached { get; private set; }	964✔
67
68	/// <summary>
69	/// Sql for a single cache fetch that pulls the cached result of the <see cref="CohortSet"/> joined to <see cref="PatientIndexTableIfAny"/> (if there was any)
70	/// </summary>
71	public CohortQueryBuilderDependencySql SqlFullyCached { get; private set; }	2,048✔
72
73	public CohortQueryBuilderDependencySql SqlJoinableCacheless { get; private set; }	560✔
74
75	public CohortQueryBuilderDependencySql SqlJoinableCached { get; private set; }	552✔
76
77	/// <summary>
78	/// Locks on aggregate by ID
79	/// </summary>
80	private static readonly ConcurrentDictionary<int, object> AggregateLocks = new();	2✔
81
82
83	public CohortQueryBuilderDependency(AggregateConfiguration cohortSet,	458✔
84	JoinableCohortAggregateConfigurationUse patientIndexTableIfAny, ICoreChildProvider childProvider,	458✔
85	IReadOnlyCollection<IPluginCohortCompiler> pluginCohortCompilers)	458✔
86	{
87	var childProvider1 = childProvider;	458✔
88	_pluginCohortCompilers = pluginCohortCompilers;	458✔
89	CohortSet = cohortSet;	458✔
90	PatientIndexTableIfAny = patientIndexTableIfAny;	458✔
91
92	//record the IsExtractionIdentifier column for the log (helps with debugging count issues)
93	var eis = cohortSet?.AggregateDimensions?.Where(d => d.IsExtractionIdentifier).ToArray();	972!
94
95	//Multiple IsExtractionIdentifier columns is a big problem but it's handled elsewhere
96	if (eis is { Length: 1 })	458✔
97	ExtractionIdentifierColumn = eis[0];	444✔
98
99	if (PatientIndexTableIfAny != null)	458✔
100	{
101	var join = childProvider1.AllJoinables.SingleOrDefault(j =>	78!
102	j.ID == PatientIndexTableIfAny.JoinableCohortAggregateConfiguration_ID) ??	560✔
103	throw new Exception("ICoreChildProvider did not know about the provided patient index table");	78✔
104	JoinedTo = childProvider1.AllAggregateConfigurations.SingleOrDefault(ac =>	78✔
105	ac.ID == join.AggregateConfiguration_ID);	1,704✔
106
107	if (JoinedTo == null)	78!
NEW 108	throw new Exception(	×
UNCOV 109	"ICoreChildProvider did not know about the provided patient index table AggregateConfiguration");	×
110	}
111	}	458✔
112
113	public override string ToString() =>
114	JoinedTo != null	1,112✔
115	? $"{CohortSet.Name}{PatientIndexTableIfAny.JoinType} JOIN {JoinedTo.Name}"	1,112✔
116	: CohortSet.Name;	1,112✔
117
118	public void Build(CohortQueryBuilderResult parent, ISqlParameter[] globals, CancellationToken cancellationToken)
119	{
120	cancellationToken.ThrowIfCancellationRequested();	458✔
121
122	var isSolitaryPatientIndexTable = CohortSet.IsJoinablePatientIndexTable();	458✔
123
124	// if it is a plugin aggregate we only want to ever serve up the cached SQL
125	var pluginCohortCompiler = _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(CohortSet));	624✔
126	var joinedToPluginCohortCompiler =	458✔
127	JoinedTo == null ? null : _pluginCohortCompilers.FirstOrDefault(c => c.ShouldRun(JoinedTo));	504✔
128
129	if (pluginCohortCompiler != null)	458✔
130	{
131	if (joinedToPluginCohortCompiler != null)	14!
UNCOV 132	throw new Exception($"APIs cannot be joined together ('{CohortSet}' and '{JoinedTo}')");	×
133
134	if (parent.CacheManager == null)	14!
UNCOV 135	throw new Exception(	×
UNCOV 136	$"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cache is configured on {CohortSet.GetCohortIdentificationConfigurationIfAny()}. You must enable result caching to use plugin aggregates.");	×
137
138	// It's a plugin aggregate so only ever run the cached SQL
139	SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,	14✔
140	pluginCohortCompiler, cancellationToken);	14✔
141
142	if (SqlFullyCached == null)	14!
UNCOV 143	throw new Exception(	×
UNCOV 144	$"Aggregate '{CohortSet}' is a plugin aggregate (According to '{pluginCohortCompiler}') but no cached results were found after running.");	×
145	return;	14✔
146	}
147
148	//Includes the parameter declaration and no rename operations (i.e. couldn't be used for building the tree but can be used for cache hit testing).
149	if (JoinedTo != null)	444✔
150	{
151	if (joinedToPluginCohortCompiler == null)	78✔
152	{
153	SqlJoinableCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(JoinedTo,	74✔
154	new QueryBuilderArgs(	74✔
155	new QueryBuilderCustomArgs(), //don't respect customizations in the inception bit!	74✔
156	globals));	74✔
157	SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true, null,	74✔
158	cancellationToken);	74✔
159	}
160	else
161	{
162	// It is not possible to do a cacheless query because an API is involved
163	SqlJoinableCached = GetCacheFetchSqlIfPossible(parent, JoinedTo, SqlJoinableCacheless, true,	4✔
164	joinedToPluginCohortCompiler, cancellationToken);	4✔
165
166	// Since the only way to query the dataset is using the cache we can pretend that it is the cacheless way
167	// of querying it too.
168	SqlJoinableCacheless = SqlJoinableCached ?? throw new Exception(	4!
169	$"Unable to build query for '{CohortSet}' because it joins to API cohort '{JoinedTo}' that did not exist in the cache");	4✔
170	}
171	}
172
173	if (isSolitaryPatientIndexTable)	444✔
174	{
175	//explicit execution of a patient index table on its own
176	//the full uncached SQL for the query
177	SqlCacheless =	40✔
178	CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet, new QueryBuilderArgs(parent.Customise, globals));	40✔
179
180	if (SqlJoinableCached != null)	40!
UNCOV 181	throw new QueryBuildingException("Patient index tables can't use other patient index tables!");	×
182	}
183	else
184	{
185	//the full uncached SQL for the query
186	SqlCacheless = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,	404✔
187	new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,	404✔
188	SqlJoinableCacheless, parent.Customise, globals));	404✔
189
190
191	//if the joined to table is cached we can generate a partial too with full sql for the outer sql block and a cache fetch join
192	if (SqlJoinableCached != null)	388✔
193	SqlPartiallyCached = CohortQueryBuilderHelper.GetSQLForAggregate(CohortSet,	38✔
194	new QueryBuilderArgs(PatientIndexTableIfAny, JoinedTo,	38✔
195	SqlJoinableCached, parent.Customise, globals));	38✔
196	}
197
198	//We would prefer a cache hit on the exact uncached SQL
199	SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlCacheless, isSolitaryPatientIndexTable,	428✔
200	pluginCohortCompiler, cancellationToken);	428✔
201
202	// if we have a patient index table where the Sql is not fully cached then we should invalidate anyone using it
203	if (isSolitaryPatientIndexTable && SqlFullyCached == null && parent.CacheManager != null)	428✔
204	ClearCacheForUsersOfPatientIndexTable(parent.CacheManager, CohortSet);	22✔
205
206	//but if that misses we would take a cache hit of an execution of the SqlPartiallyCached
207	if (SqlFullyCached == null && SqlPartiallyCached != null)	428✔
208	SqlFullyCached = GetCacheFetchSqlIfPossible(parent, CohortSet, SqlPartiallyCached,	30✔
209	isSolitaryPatientIndexTable, pluginCohortCompiler, cancellationToken);	30✔
210	}	428✔
211
212	private void ClearCacheForUsersOfPatientIndexTable(CachedAggregateConfigurationResultsManager cacheManager,
213	AggregateConfiguration cohortSet)
214	{
215	if (cacheManager == null)	22!
UNCOV 216	return;	×
217
218	var join = CohortSet.JoinableCohortAggregateConfiguration ?? throw new Exception(	22!
219	$"{nameof(AggregateConfiguration.JoinableCohortAggregateConfiguration)} was null for CohortSet {cohortSet} so we were unable to clear the joinable cache users");	22✔
220
221	// get each Aggregate Configuration that joins using this patient index table
222	foreach (var user in join.Users.Select(j => j.AggregateConfiguration))	104✔
223	cacheManager.DeleteCacheEntryIfAny(user, AggregateOperation.IndexedExtractionIdentifierList);	20✔
224	}	22✔
225
226	private CohortQueryBuilderDependencySql GetCacheFetchSqlIfPossible(CohortQueryBuilderResult parent,
227	AggregateConfiguration aggregate,
228	CohortQueryBuilderDependencySql sql, bool isPatientIndexTable, IPluginCohortCompiler pluginCohortCompiler,
229	CancellationToken cancellationToken)
230	{
231	if (parent.CacheManager == null)	550✔
232	return null;	316✔
233
234	var aggregateType = isPatientIndexTable	234✔
235	? AggregateOperation.JoinableInceptionQuery	234✔
236	: AggregateOperation.IndexedExtractionIdentifierList;	234✔
237	IHasFullyQualifiedNameToo existingTable;
238
239	// since we might make a plugin API run call we had better lock this to
240	var oLock = AggregateLocks.GetOrAdd(aggregate.ID, i => new object());	318✔
241	lock (oLock)	234✔
242	{
243	// unless it is a plugin driven aggregate we need to assemble the SQL to check if the cache is stale
244	if (pluginCohortCompiler == null)	234✔
245	{
246	var parameterSql =	216✔
247	QueryBuilder.GetParameterDeclarationSQL(sql.ParametersUsed.Clone()	216✔
248	.GetFinalResolvedParametersList());	216✔
249	var hitTestSql = parameterSql + sql.Sql;	216✔
250	existingTable = parent.CacheManager.GetLatestResultsTable(aggregate, aggregateType, hitTestSql);	216✔
251	}
252	else
253	{
254	existingTable =	18✔
255	parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType, out var oldDescription);	18✔
256
257	if (pluginCohortCompiler.IsStale(aggregate, oldDescription)) existingTable = null;	24✔
258	}
259
260	// if there are no cached results in the destination (and it's a plugin cohort) then we need to run the plugin API call
261	if (existingTable == null && pluginCohortCompiler != null)	234✔
262	{
263	// no cached results were there so run the plugin
264	pluginCohortCompiler.Run(CohortSet, parent.CacheManager, cancellationToken);	6✔
265
266	// try again now
267	existingTable = parent.CacheManager.GetLatestResultsTableUnsafe(aggregate, aggregateType);	6✔
268
269	if (existingTable == null)	6!
UNCOV 270	throw new Exception(	×
UNCOV 271	$"Run method on {pluginCohortCompiler} failed to populate the Query Result Cache for {CohortSet}");	×
272	}
273	}	234✔
274
275	//if there is a cached entry matching the cacheless SQL then we can just do a select from it (in theory)
276	if (existingTable != null)	234✔
277	{
278	var sqlCachFetch =	120✔
279	$@"{CachedAggregateConfigurationResultsManager.CachingPrefix}{aggregate.Name}/{Environment.NewLine}select from {existingTable.GetFullyQualifiedName()}{Environment.NewLine}";	120✔
280
281	//Cache fetch does not require any parameters
282	return new CohortQueryBuilderDependencySql(sqlCachFetch, new ParameterManager());	120✔
283	}
284
285
286	return null;	114✔
287	}
288
289	public string DescribeCachedState()
290	{
291	if (SqlFullyCached != null)	458!
UNCOV 292	return "Fully Cached";	×
293
294	if (SqlPartiallyCached != null)	458!
UNCOV 295	return "Partially Cached";	×
296
297	return SqlCacheless != null ? "Not Cached" : "Not Built";	458!
298	}
299	}

HicServices / RDMP / 13318089130

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous