19852309549

Committed 02 Dec 2025 08:38AM UTC coverage: 55.8% (-1.4%) from 57.197%

Build # 19852309549

Build Type

Pull #2279

github

Committed by

JFriel

Commit Message

fix build

Pull Request Pull Request #2279: [UI Overhaul] Spike/refresh

Run Details

11249 of 21727 branches covered (51.77%)

Branch coverage included in aggregate %.

807 of 1108 new or added lines in 29 files covered. (72.83%)

724 existing lines in 67 files now uncovered.

31893 of 55588 relevant lines covered (57.37%)

16683.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.78

/Rdmp.Core/Curation/Data/DataLoad/PreLoadDiscardedColumn.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.Linq;
using FAnsi.Implementations.MicrosoftSQL;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.MapsDirectlyToDatabaseTable.Injection;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode.Checks;

namespace Rdmp.Core.Curation.Data.DataLoad;

/// <summary>
/// Describes where a PreLoadDiscardedColumn will ultimately end up.
/// </summary>
public enum DiscardedColumnDestination
{
    /// <summary>
    /// Column appears in RAW and might be used in AdjustRaw but is dropped completely prior to migration to Staging
    /// </summary>
    Oblivion = 1,

    /// <summary>
    /// Column appears in RAW but is separated off and stored in an IdentifierDump (See IdentifierDumper) and not passed through to Staging
    /// </summary>
    StoreInIdentifiersDump = 2,

    /// <summary>
    /// Column appears in RAW but is Diluted during AdjustStaging prior to joining the live dataset e.g. by rounding dates to the nearest quarter.  The undilted value may be stored in the IdentifierDump (See IdentifierDumper).
    /// </summary>
    Dilute = 3
}

/// <summary>
/// Describes a column that is provided to your institution by a data provider but which is not loaded into your LIVE database table.  This column might be very sensitive,
/// irrelevant to you etc.  Each discarded column has a destination (DiscardedColumnDestination)  e.g. it might be dropped completely or routed into an identifier dump for
/// when you still want to store information such as Who an MRI was for but do not want it sitting in your live dataset for governance/anonymisation reasons.
/// 
/// <para>Each instance is tied to a specific TableInfo and when a data load occurs from an unstructured format (e.g. CSV) which RequestsExternalDatabaseCreation then not only are the
/// LIVE columns created in the RAW bubble but also the dropped columns described in PreLoadDiscardedColumn instances.  This allows the live system state to drive required formats/fields
/// for data load resulting in a stricter/more maintainable data load model.</para>
/// </summary>
public class PreLoadDiscardedColumn : DatabaseEntity, IPreLoadDiscardedColumn, ICheckable, IInjectKnown<ITableInfo>
{
    #region Database Properties

    private int _tableInfoID;
    private DiscardedColumnDestination _destination;
    private string _runtimeColumnName;
    private string _sqlDataType;
    private int? _duplicateRecordResolutionOrder;
    private bool _duplicateRecordResolutionIsAscending;
    private Lazy<ITableInfo> _knownTableInfo;

    /// <inheritdoc cref="IPreLoadDiscardedColumn.TableInfo"/>
    public int TableInfo_ID
    {
        get => _tableInfoID;
        set => SetField(ref _tableInfoID, value);
    }

    /// <inheritdoc/>
    public DiscardedColumnDestination Destination
    {
        get => _destination;
        set => SetField(ref _destination, value);
    }

    /// <inheritdoc/>
    public string RuntimeColumnName
    {
        get => _runtimeColumnName;
        set => SetField(ref _runtimeColumnName, value);
    }

    /// <inheritdoc/>
    public string SqlDataType
    {
        get => _sqlDataType;
        set => SetField(ref _sqlDataType, value);
    }

    /// <inheritdoc/>
    public int? DuplicateRecordResolutionOrder
    {
        get => _duplicateRecordResolutionOrder;
        set => SetField(ref _duplicateRecordResolutionOrder, value);
    }

    /// <inheritdoc/>
    public bool DuplicateRecordResolutionIsAscending
    {
        get => _duplicateRecordResolutionIsAscending;
        set => SetField(ref _duplicateRecordResolutionIsAscending, value);
    }

    #endregion

    #region Relationships

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ITableInfo TableInfo => _knownTableInfo.Value;

    #endregion

    //required for IResolveDuplication
    /// <summary>
    /// For setting, use SqlDataType instead, it is an exact alias to allow for IResolveDuplication interface definition (see the fact that ColumnInfo also uses that interface and is also IMapsDirectlyToDatabaseTable)
    /// </summary>
    [NoMappingToDatabase]
    public string Data_type => SqlDataType;

    public PreLoadDiscardedColumn()
    {
        ClearAllInjections();
    }

    /// <summary>
    /// Creates a new virtual column that will be created in RAW during data loads but does not appear in the LIVE table schema.  This allows
    /// identifiable data to be loaded and processed in a data load without ever hitting the live database.
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="parent"></param>
    /// <param name="name"></param>
    public PreLoadDiscardedColumn(ICatalogueRepository repository, ITableInfo parent, string name = null)
    {
        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "TableInfo_ID", parent.ID },
            { "Destination", DiscardedColumnDestination.Oblivion },
            { "RuntimeColumnName", name ?? $"NewColumn{Guid.NewGuid()}" }
        });

        ClearAllInjections();
    }

    internal PreLoadDiscardedColumn(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
    {
        TableInfo_ID = int.Parse(r["TableInfo_ID"].ToString());
        Destination = (DiscardedColumnDestination)r["Destination"];
        RuntimeColumnName = r["RuntimeColumnName"] as string;
        SqlDataType = r["SqlDataType"] as string;

        if (r["DuplicateRecordResolutionOrder"] != DBNull.Value)
            DuplicateRecordResolutionOrder = int.Parse(r["DuplicateRecordResolutionOrder"].ToString());
        else
            DuplicateRecordResolutionOrder = null;

        DuplicateRecordResolutionIsAscending = Convert.ToBoolean(r["DuplicateRecordResolutionIsAscending"]);

        ClearAllInjections();
    }

    /// <inheritdoc/>
    public override string ToString() => $"{RuntimeColumnName} ({Destination})";


    public void Check(ICheckNotifier notifier)
    {
        //if it goes into the identifier dump then the table had better have one
        if (GoesIntoIdentifierDump() && TableInfo.IdentifierDumpServer_ID == null)
            notifier.OnCheckPerformed(
                new CheckEventArgs(
                    $"Column is set to {Destination}  which means its value should be stored in the IdentifierDump but the parent table '{TableInfo}'  doesn't have a dump server configured",
                    CheckResult.Fail));
        else
            notifier.OnCheckPerformed(new CheckEventArgs("Destination is ok", CheckResult.Success));

        if (
            //if column is not diluted (i.e. oblivion or dumped) then there shouldn't be any other columns with the same name
            Destination != DiscardedColumnDestination.Dilute &&

            //are there duplicate named columns?
            TableInfo.GetColumnsAtStage(LoadStage.AdjustRaw)
                .Except(new[] { this })
                .Any(c => c.GetRuntimeName(LoadStage.AdjustRaw).Equals(GetRuntimeName(LoadStage.AdjustRaw),
                    StringComparison.CurrentCultureIgnoreCase)))
            notifier.OnCheckPerformed(
                new CheckEventArgs($"There are 2+ columns called '{GetRuntimeName(LoadStage.AdjustRaw)}' in this table",
                    CheckResult.Fail));
        else
            notifier.OnCheckPerformed(new CheckEventArgs("Name is unique", CheckResult.Success));
    }

    /// <inheritdoc/>
    public string GetRuntimeName() =>
        //belt and bracers, the user could be typing something mental into this field in his database
        MicrosoftQuerySyntaxHelper.Instance.GetRuntimeName(RuntimeColumnName);

    /// <inheritdoc/>
    public string GetRuntimeName(LoadStage stage) => GetRuntimeName();

    /// <summary>
    /// true if destination for column is to store in identifier dump including undiluted versions of dilutes
    /// (Dilution involves making clean values dirty for purposes of anonymisation and storing the clean values in
    /// the Identifier Dump).
    /// </summary>
    /// <returns></returns>
    public bool GoesIntoIdentifierDump() =>
        Destination == DiscardedColumnDestination.StoreInIdentifiersDump
        ||
        Destination == DiscardedColumnDestination.Dilute;

    public void InjectKnown(ITableInfo instance)
    {
        _knownTableInfo = new Lazy<ITableInfo>(instance);
    }

    public void ClearAllInjections()
    {
        _knownTableInfo = new Lazy<ITableInfo>(() => Repository.GetObjectByID<TableInfo>(TableInfo_ID));
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.Linq;
11	using FAnsi.Implementations.MicrosoftSQL;
12	using Rdmp.Core.MapsDirectlyToDatabaseTable;
13	using Rdmp.Core.MapsDirectlyToDatabaseTable.Injection;
14	using Rdmp.Core.Repositories;
15	using Rdmp.Core.ReusableLibraryCode.Checks;
16
17	namespace Rdmp.Core.Curation.Data.DataLoad;
18
19	/// <summary>
20	/// Describes where a PreLoadDiscardedColumn will ultimately end up.
21	/// </summary>
22	public enum DiscardedColumnDestination
23	{
24	/// <summary>
25	/// Column appears in RAW and might be used in AdjustRaw but is dropped completely prior to migration to Staging
26	/// </summary>
27	Oblivion = 1,
28
29	/// <summary>
30	/// Column appears in RAW but is separated off and stored in an IdentifierDump (See IdentifierDumper) and not passed through to Staging
31	/// </summary>
32	StoreInIdentifiersDump = 2,
33
34	/// <summary>
35	/// Column appears in RAW but is Diluted during AdjustStaging prior to joining the live dataset e.g. by rounding dates to the nearest quarter. The undilted value may be stored in the IdentifierDump (See IdentifierDumper).
36	/// </summary>
37	Dilute = 3
38	}
39
40	/// <summary>
41	/// Describes a column that is provided to your institution by a data provider but which is not loaded into your LIVE database table. This column might be very sensitive,
42	/// irrelevant to you etc. Each discarded column has a destination (DiscardedColumnDestination) e.g. it might be dropped completely or routed into an identifier dump for
43	/// when you still want to store information such as Who an MRI was for but do not want it sitting in your live dataset for governance/anonymisation reasons.
44	///
45	/// <para>Each instance is tied to a specific TableInfo and when a data load occurs from an unstructured format (e.g. CSV) which RequestsExternalDatabaseCreation then not only are the
46	/// LIVE columns created in the RAW bubble but also the dropped columns described in PreLoadDiscardedColumn instances. This allows the live system state to drive required formats/fields
47	/// for data load resulting in a stricter/more maintainable data load model.</para>
48	/// </summary>
49	public class PreLoadDiscardedColumn : DatabaseEntity, IPreLoadDiscardedColumn, ICheckable, IInjectKnown<ITableInfo>
50	{
51	#region Database Properties
52
53	private int _tableInfoID;
54	private DiscardedColumnDestination _destination;
55	private string _runtimeColumnName;
56	private string _sqlDataType;
57	private int? _duplicateRecordResolutionOrder;
58	private bool _duplicateRecordResolutionIsAscending;
59	private Lazy<ITableInfo> _knownTableInfo;
60
61	/// <inheritdoc cref="IPreLoadDiscardedColumn.TableInfo"/>
62	public int TableInfo_ID
63	{
64	get => _tableInfoID;	186✔
65	set => SetField(ref _tableInfoID, value);	350✔
66	}
67
68	/// <inheritdoc/>
69	public DiscardedColumnDestination Destination
70	{
71	get => _destination;	316✔
72	set => SetField(ref _destination, value);	378✔
73	}
74
75	/// <inheritdoc/>
76	public string RuntimeColumnName
77	{
78	get => _runtimeColumnName;	1,114✔
79	set => SetField(ref _runtimeColumnName, value);	350✔
80	}
81
82	/// <inheritdoc/>
83	public string SqlDataType
84	{
85	get => _sqlDataType;	232✔
86	set => SetField(ref _sqlDataType, value);	372✔
87	}
88
89	/// <inheritdoc/>
90	public int? DuplicateRecordResolutionOrder
91	{
92	get => _duplicateRecordResolutionOrder;	174✔
93	set => SetField(ref _duplicateRecordResolutionOrder, value);	342✔
94	}
95
96	/// <inheritdoc/>
97	public bool DuplicateRecordResolutionIsAscending
98	{
99	get => _duplicateRecordResolutionIsAscending;	174✔
100	set => SetField(ref _duplicateRecordResolutionIsAscending, value);	342✔
101	}
102
103	#endregion
104
105	#region Relationships
106
107	/// <inheritdoc/>
108	[NoMappingToDatabase]
109	public ITableInfo TableInfo => _knownTableInfo.Value;	8✔
110
111	#endregion
112
113	//required for IResolveDuplication
114	/// <summary>
115	/// For setting, use SqlDataType instead, it is an exact alias to allow for IResolveDuplication interface definition (see the fact that ColumnInfo also uses that interface and is also IMapsDirectlyToDatabaseTable)
116	/// </summary>
117	[NoMappingToDatabase]
118	public string Data_type => SqlDataType;	34✔
119
120	public PreLoadDiscardedColumn()	×
121	{
122	ClearAllInjections();	×
123	}	×
124
125	/// <summary>
126	/// Creates a new virtual column that will be created in RAW during data loads but does not appear in the LIVE table schema. This allows
127	/// identifiable data to be loaded and processed in a data load without ever hitting the live database.
128	/// </summary>
129	/// <param name="repository"></param>
130	/// <param name="parent"></param>
131	/// <param name="name"></param>
132	public PreLoadDiscardedColumn(ICatalogueRepository repository, ITableInfo parent, string name = null)	90✔
133	{
134	repository.InsertAndHydrate(this, new Dictionary<string, object>	90!
135	{	90✔
136	{ "TableInfo_ID", parent.ID },	90✔
137	{ "Destination", DiscardedColumnDestination.Oblivion },	90✔
138	{ "RuntimeColumnName", name ?? $"NewColumn{Guid.NewGuid()}" }	90✔
139	});	90✔
140
141	ClearAllInjections();	90✔
142	}	90✔
143
144	internal PreLoadDiscardedColumn(ICatalogueRepository repository, DbDataReader r)
145	: base(repository, r)	260✔
146	{
147	TableInfo_ID = int.Parse(r["TableInfo_ID"].ToString());	260✔
148	Destination = (DiscardedColumnDestination)r["Destination"];	260✔
149	RuntimeColumnName = r["RuntimeColumnName"] as string;	260✔
150	SqlDataType = r["SqlDataType"] as string;	260✔
151
152	if (r["DuplicateRecordResolutionOrder"] != DBNull.Value)	260!
153	DuplicateRecordResolutionOrder = int.Parse(r["DuplicateRecordResolutionOrder"].ToString());	×
154	else
155	DuplicateRecordResolutionOrder = null;	260✔
156
157	DuplicateRecordResolutionIsAscending = Convert.ToBoolean(r["DuplicateRecordResolutionIsAscending"]);	260✔
158
159	ClearAllInjections();	260✔
160	}	260✔
161
162	/// <inheritdoc/>
163	public override string ToString() => $"{RuntimeColumnName} ({Destination})";	24✔
164
165
166	public void Check(ICheckNotifier notifier)
167	{
168	//if it goes into the identifier dump then the table had better have one
169	if (GoesIntoIdentifierDump() && TableInfo.IdentifierDumpServer_ID == null)	2!
170	notifier.OnCheckPerformed(	×
171	new CheckEventArgs(	×
172	$"Column is set to {Destination} which means its value should be stored in the IdentifierDump but the parent table '{TableInfo}' doesn't have a dump server configured",	×
173	CheckResult.Fail));	×
174	else
175	notifier.OnCheckPerformed(new CheckEventArgs("Destination is ok", CheckResult.Success));	2✔
176
177	if (	2!
178	//if column is not diluted (i.e. oblivion or dumped) then there shouldn't be any other columns with the same name	2✔
179	Destination != DiscardedColumnDestination.Dilute &&	2✔
180		2✔
181	//are there duplicate named columns?	2✔
182	TableInfo.GetColumnsAtStage(LoadStage.AdjustRaw)	2✔
183	.Except(new[] { this })	2✔
184	.Any(c => c.GetRuntimeName(LoadStage.AdjustRaw).Equals(GetRuntimeName(LoadStage.AdjustRaw),	2✔
185	StringComparison.CurrentCultureIgnoreCase)))	2✔
186	notifier.OnCheckPerformed(	×
187	new CheckEventArgs($"There are 2+ columns called '{GetRuntimeName(LoadStage.AdjustRaw)}' in this table",	×
188	CheckResult.Fail));	×
189	else
190	notifier.OnCheckPerformed(new CheckEventArgs("Name is unique", CheckResult.Success));	2✔
191	}	2✔
192
193	/// <inheritdoc/>
194	public string GetRuntimeName() =>
195	//belt and bracers, the user could be typing something mental into this field in his database
196	MicrosoftQuerySyntaxHelper.Instance.GetRuntimeName(RuntimeColumnName);	808✔
197
198	/// <inheritdoc/>
199	public string GetRuntimeName(LoadStage stage) => GetRuntimeName();	26✔
200
201	/// <summary>
202	/// true if destination for column is to store in identifier dump including undiluted versions of dilutes
203	/// (Dilution involves making clean values dirty for purposes of anonymisation and storing the clean values in
204	/// the Identifier Dump).
205	/// </summary>
206	/// <returns></returns>
207	public bool GoesIntoIdentifierDump() =>
208	Destination == DiscardedColumnDestination.StoreInIdentifiersDump	64✔
209	\|\|	64✔
210	Destination == DiscardedColumnDestination.Dilute;	64✔
211
212	public void InjectKnown(ITableInfo instance)
213	{
UNCOV 214	_knownTableInfo = new Lazy<ITableInfo>(instance);	×
UNCOV 215	}	×
216
217	public void ClearAllInjections()
218	{
219	_knownTableInfo = new Lazy<ITableInfo>(() => Repository.GetObjectByID<TableInfo>(TableInfo_ID));	358✔
220	}	352✔
221	}

HicServices / RDMP / 19852309549

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous