9953564349

Committed 16 Jul 2024 08:40AM UTC coverage: 56.679% (+0.001%) from 56.678%

Build # 9953564349

Build Type

Pull #1884

github

Committed by

JFriel

Commit Message

update imports

Pull Request Pull Request #1884: Bugfix/rdmp 210: Improve LoadMetadata directory checking

Run Details

10934 of 20780 branches covered (52.62%)

Branch coverage included in aggregate %.

13 of 24 new or added lines in 3 files covered. (54.17%)

1 existing line in 1 file now uncovered.

30981 of 53172 relevant lines covered (58.27%)

7844.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

38.69

/Rdmp.Core/DataLoad/Engine/Checks/Checkers/CatalogueLoadChecks.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FAnsi.Discovery;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
using Rdmp.Core.DataLoad.Engine.LoadProcess;
using Rdmp.Core.DataLoad.Engine.Migration;
using Rdmp.Core.DataLoad.Triggers;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.DataAccess;

namespace Rdmp.Core.DataLoad.Engine.Checks.Checkers;

internal class CatalogueLoadChecks : ICheckable
{
    private readonly ILoadMetadata _loadMetadata;
    private readonly HICLoadConfigurationFlags _loadConfigurationFlags;
    private readonly HICDatabaseConfiguration _databaseConfiguration;

    public CatalogueLoadChecks(ILoadMetadata loadMetadata, HICLoadConfigurationFlags loadConfigurationFlags,
        HICDatabaseConfiguration databaseConfiguration)
    {
        _loadMetadata = loadMetadata;
        _loadConfigurationFlags = loadConfigurationFlags;
        _databaseConfiguration = databaseConfiguration;
    }


    private bool ValidateFilePath(string directoryPath)
    {
        return Path.Exists(directoryPath);
    }

    public void Check(ICheckNotifier notifier)
    {
        Catalogue[] catalogueMetadatas;

        try
        {
            //check there are catalogues and we can retrieve them
            catalogueMetadatas = _loadMetadata.GetAllCatalogues().Cast<Catalogue>().ToArray();
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs("Crashed trying to fetch Catalogues for metadata",
                CheckResult.Fail, e));
            return;
        }

        //there no catalogues
        if (catalogueMetadatas.Length == 0)
            notifier.OnCheckPerformed(new CheckEventArgs("There are no Catalogues associated with this metadata",
                CheckResult.Fail, null));

        var tablesFound = new List<ITableInfo>();

        //check each catalogue is sufficiently configured to perform a migration
        foreach (var catalogue in catalogueMetadatas)
        {
            notifier.OnCheckPerformed(new CheckEventArgs($"Found Catalogue:{catalogue}", CheckResult.Success, null));

            var tableInfos = catalogue.GetTableInfoList(true).Distinct().ToArray();

            if (tableInfos.Length == 0)
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"Catalogue {catalogue.Name} does not have any TableInfos", CheckResult.Fail, null));

            tablesFound.AddRange(tableInfos.Where(tableInfo => !tablesFound.Contains(tableInfo)));
            if(_loadMetadata.LocationOfForLoadingDirectory != null && !ValidateFilePath(_loadMetadata.LocationOfForLoadingDirectory))
                 notifier.OnCheckPerformed(new CheckEventArgs($"The ForLoading directory for this load ({_loadMetadata.LocationOfForLoadingDirectory}) does not exist.",
                CheckResult.Fail, null));
            if (_loadMetadata.LocationOfForArchivingDirectory!= null && !ValidateFilePath(_loadMetadata.LocationOfForArchivingDirectory))
                notifier.OnCheckPerformed(new CheckEventArgs($"The ForArchiving directory for this load ({_loadMetadata.LocationOfForArchivingDirectory}) does not exist.",
               CheckResult.Fail, null));
            if (_loadMetadata.LocationOfCacheDirectory != null && _loadMetadata.LocationOfForLoadingDirectory != null && !ValidateFilePath(_loadMetadata.LocationOfCacheDirectory))
                notifier.OnCheckPerformed(new CheckEventArgs($"The Cache directory for this load ({_loadMetadata.LocationOfCacheDirectory}) does not exist.",
               CheckResult.Fail, null));
            if (_loadMetadata.LocationOfExecutablesDirectory != null && !ValidateFilePath(_loadMetadata.LocationOfExecutablesDirectory))
                notifier.OnCheckPerformed(new CheckEventArgs($"The Executables directory for this load ({_loadMetadata.LocationOfExecutablesDirectory}) does not exist.",
               CheckResult.Fail, null));
        }


        //check regular tables
        foreach (TableInfo regularTable in tablesFound)
        {
            notifier.OnCheckPerformed(new CheckEventArgs($"About To check configuration of TableInfo:{regularTable}",
                CheckResult.Success, null));
            CheckTableInfo(regularTable, notifier);

            //check anonymisation
            var anonymisationChecks = new AnonymisationChecks(regularTable);
            anonymisationChecks.Check(notifier);
        }
    }


    #region These methods check TableInfos (Lookup including lookups)

    private void CheckTableInfo(TableInfo tableInfo, ICheckNotifier notifier)
    {
        //get all columns

        //check whether the live database and staging databases have appropriate columns and triggers etc on them
        var staging = _databaseConfiguration.DeployInfo[LoadBubble.Staging];
        var live = DataAccessPortal.ExpectDatabase(tableInfo, DataAccessContext.DataLoad);

        var liveTable =
            live.ExpectTable(tableInfo.GetRuntimeName(LoadBubble.Live, _databaseConfiguration.DatabaseNamer),
                tableInfo.Schema);
        var liveCols = liveTable.DiscoverColumns();

        CheckTableInfoSynchronization(tableInfo, notifier);

        CheckTableHasColumnInfosAndPrimaryKeys(live, tableInfo, out _, out _, notifier);

        //check for trying to ignore primary keys
        foreach (var col in tableInfo.ColumnInfos)
            if (col.IgnoreInLoads && col.IsPrimaryKey)
                notifier.OnCheckPerformed(
                    new CheckEventArgs($"ColumnInfo {col} is marked both IgnoreInLoads and IsPrimaryKey",
                        CheckResult.Fail));

        try
        {
            if (!_loadMetadata.IgnoreTrigger)
            {
                //if trigger is created as part of this check then it is likely to have resulted in changes to the underlying table (e.g. added hic_validFrom field) in which case we should resynch the TableInfo to pickup these new columns
                CheckTriggerIntact(liveTable, notifier, out var runSynchronizationAgain);

                if (runSynchronizationAgain)
                    CheckTableInfoSynchronization(tableInfo, notifier);
            }

            if (!_databaseConfiguration.RequiresStagingTableCreation)
            {
                //Important:
                //Regarding this if block: None of the current loading systems in RDMP have RequiresStagingTableCreation as false but in theory you could build some kind of multi threaded horror
                //which had multiple simultaneous loads all populating a single STAGING bubble therefore this code is left in the codebase, it probably works ok but you will need a
                //fair bit of work if you want to realise such a monstrosity (for one cleanup on each run probably cleans up STAGING on exit)

                //live can have additional hic_ columns which do not appear in staging (lookups cannot)
                var stagingTable =
                    staging.ExpectTable(tableInfo.GetRuntimeName(LoadBubble.Staging,
                        _databaseConfiguration.DatabaseNamer));

                if (!stagingTable.Exists())
                    notifier.OnCheckPerformed(new CheckEventArgs(
                        "RequiresStagingTableCreation is false but staging does not exist, this flag should indicate that you anticipate STAGING to be already setup before you kick off the load",
                        CheckResult.Fail));

                var stagingCols = stagingTable.DiscoverColumns();

                ConfirmStagingAndLiveHaveSameColumns(tableInfo.GetRuntimeName(), stagingCols, liveCols, false,
                    notifier);

                CheckStagingToLiveMigrationForTable(stagingTable, stagingCols, liveTable, liveCols, notifier);
            }
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs($"Load Checks Crashed {tableInfo}", CheckResult.Fail, e));
        }
    }

    private static void CheckTableInfoSynchronization(TableInfo tableInfo, ICheckNotifier notifier)
    {
        //live is the current data load's (possibly overridden server/database)
        var tableInfoSynchronizer = new TableInfoSynchronizer(tableInfo);

        var problemList = "";

        //synchronize but refuse to apply all fixes, problems are instead added to problemList
        if (tableInfoSynchronizer.Synchronize(notifier))
        {
            notifier.OnCheckPerformed(new CheckEventArgs($"TableInfo {tableInfo} passed Synchronization check",
                CheckResult.Success, null)); //passed synchronization
        }
        else
        {
            var launchSyncFixer = notifier.OnCheckPerformed(new CheckEventArgs(
                $"TableInfo {tableInfo} failed Synchronization check with following problems:{problemList}",
                CheckResult.Fail,
                null, "Launch Synchronization Fixing")); //failed synchronization

            if (launchSyncFixer)
            {
                var userFixed =
                    //if silent running accept all changes
                    tableInfoSynchronizer.Synchronize(notifier);

                if (!userFixed)
                    notifier.OnCheckPerformed(new CheckEventArgs(
                        $"TableInfo {tableInfo} still failed Synchronization check", CheckResult.Fail,
                        null)); //passed synchronization
            }
        }
    }

    private static void CheckTableHasColumnInfosAndPrimaryKeys(DiscoveredDatabase live, TableInfo tableInfo,
        out ColumnInfo[] columnInfos, out ColumnInfo[] columnInfosWhichArePrimaryKeys, ICheckNotifier notifier)
    {
        columnInfos = tableInfo.ColumnInfos.ToArray();
        columnInfosWhichArePrimaryKeys = columnInfos.Where(col => col.IsPrimaryKey).ToArray();


        //confirm there are at least 1
        if (!columnInfos.Any())
            notifier.OnCheckPerformed(new CheckEventArgs($"TableInfo {tableInfo.Name} has no columninfos",
                CheckResult.Fail, null));


        if (!columnInfosWhichArePrimaryKeys.Any())
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"TableInfo {tableInfo.Name} has no IsPrimaryKey columns", CheckResult.Fail, null));

        var primaryKeys = live.ExpectTable(tableInfo.GetRuntimeName(), tableInfo.Schema).DiscoverColumns()
            .Where(c => c.IsPrimaryKey).ToArray();

        if (primaryKeys.Any(k => k.IsAutoIncrement))
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"AutoIncrement columns {string.Join(",", primaryKeys.Where(k => k.IsAutoIncrement))} are not allowed as Primary Keys for your table because there is no way to differentiate new data being loaded from duplicate old data being loaded (the entire purpose of the RDMP DLE)",
                CheckResult.Fail));

        //confirm primary keys match underlying table
        //sort pks alphabetically and confirm they match the underlying live system table
        var actualPks = primaryKeys.Select(c => c.GetRuntimeName()).OrderBy(s => s).ToArray();
        var pksWeExpect = columnInfosWhichArePrimaryKeys.Select(c => c.GetRuntimeName()).OrderBy(s => s).ToArray();


        if (actualPks.Length != pksWeExpect.Length)
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"Primary keys in Catalogue for database table {tableInfo.GetRuntimeName()} does not match Catalogue entry (difference in number of keys)",
                CheckResult.Fail, null));
        else
            for (var i = 0; i < pksWeExpect.Length; i++)
                if (!pksWeExpect[i].Equals(actualPks[i]))
                    notifier.OnCheckPerformed(new CheckEventArgs(
                        $"Mismatch between primary key defined in Catalogue {pksWeExpect[i]} and one found in live table {tableInfo.GetRuntimeName()}",
                        CheckResult.Fail, null));
                else
                    notifier.OnCheckPerformed(new CheckEventArgs(
                        $"Found primary key {pksWeExpect[i]} in LIVE table {tableInfo.GetRuntimeName()}",
                        CheckResult.Success, null));
    }

    private static void CheckTriggerIntact(DiscoveredTable table, ICheckNotifier notifier,
        out bool runSynchronizationAgain)
    {
        var checker = new TriggerChecks(table);
        checker.Check(notifier);

        runSynchronizationAgain = checker.TriggerCreated;
    }


    private static void ConfirmStagingAndLiveHaveSameColumns(string tableName, DiscoveredColumn[] stagingCols,
        DiscoveredColumn[] liveCols, bool requireSameNumberAndOrder, ICheckNotifier notifier)
    {
        //in LIVE but not STAGING
        foreach (var missingColumn in liveCols.Select(c => c.GetRuntimeName())
                     .Except(stagingCols.Select(c => c.GetRuntimeName())))
            //column is in live but not in staging, but it is hic_
            if (SpecialFieldNames.IsHicPrefixed(missingColumn)) //this is permitted
                continue;
            else
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"Column {missingColumn} is missing from STAGING", CheckResult.Fail, null));

        //in STAGING but not LIVE
        foreach (var missingColumn in stagingCols.Except(liveCols))
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"Column {missingColumn} is in STAGING but not LIVE", CheckResult.Fail, null));


        if (requireSameNumberAndOrder)
        {
            var passedColumnOrderCheck = true;

            if (stagingCols.Length != liveCols.Length)
            {
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"Column count mismatch between staging and live in table {tableName}", CheckResult.Fail, null));
                passedColumnOrderCheck = false;
            }
            else
            //check they are in the same order
            {
                for (var i = 0; i < stagingCols.Length; i++)
                    if (!stagingCols[i].Equals(liveCols[i]))
                    {
                        notifier.OnCheckPerformed(new CheckEventArgs(
                            $"Column name/order mismatch between staging and live in table {tableName}, column {i} is {stagingCols[i]} in staging but is {liveCols[i]} in live.",
                            CheckResult.Fail, null));
                        passedColumnOrderCheck = false;
                        break;
                    }
            }

            if (passedColumnOrderCheck)
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"Column order match confirmed between staging and live on table {tableName}", CheckResult.Success,
                    null));
        }
    }

    private void CheckStagingToLiveMigrationForTable(DiscoveredTable stagingTable, DiscoveredColumn[] stagingCols,
        DiscoveredTable liveTable, DiscoveredColumn[] liveCols, ICheckNotifier notifier)
    {
        try
        {
            new MigrationColumnSet(stagingTable, liveTable, new StagingToLiveMigrationFieldProcessor
            {
                NoBackupTrigger = _loadMetadata.IgnoreTrigger
            });
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"TableInfo {liveTable} passed {nameof(MigrationColumnSet)} check ", CheckResult.Success, null));
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs(
                $"{nameof(MigrationColumnSet)} reports a problem with the configuration of columns on STAGING/LIVE or in the ColumnInfos for TableInfo {liveTable}",
                CheckResult.Fail, e));
        }

        //live columns
        foreach (var col in liveCols)
            if (!SpecialFieldNames.IsHicPrefixed(col) && col.IsAutoIncrement) //must start hic_ if they are identities
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"Column {col} is an identity column in the LIVE database but does not start with hic_",
                    CheckResult.Fail, null)); //this one does not

        //staging columns
        foreach (var col in stagingCols) //staging columns
            if (col.IsAutoIncrement) //if there are any auto increments
                notifier.OnCheckPerformed(new CheckEventArgs(
                    $"Column {col} is an identity column and is in STAGING, the identity flag must be removed from the STAGING table",
                    CheckResult.Fail,
                    null)); //complain since don't want a mismatch between IDs in staging and live or complaints about identity insert from SQL server

        //staging must allow null dataloadrunids and validfroms
        ConfirmNullability(stagingTable.DiscoverColumn(SpecialFieldNames.DataLoadRunID), true, notifier);
        ConfirmNullability(stagingTable.DiscoverColumn(SpecialFieldNames.ValidFrom), true, notifier);

        //live must allow nulls in validFrom
        ConfirmNullability(liveTable.DiscoverColumn(SpecialFieldNames.ValidFrom), true, notifier);
    }

    private static void ConfirmNullability(DiscoveredColumn column, bool expectedNullability, ICheckNotifier notifier)
    {
        var nullability = column.AllowNulls;
        notifier.OnCheckPerformed(new CheckEventArgs(
            $"Nullability of {column} is AllowNulls={nullability}, (expected {expectedNullability})",
            nullability == expectedNullability ? CheckResult.Success : CheckResult.Fail, null));
    }

    #endregion
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.IO;
10	using System.Linq;
11	using FAnsi.Discovery;
12	using Rdmp.Core.Curation;
13	using Rdmp.Core.Curation.Data;
14	using Rdmp.Core.Curation.Data.DataLoad;
15	using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;
16	using Rdmp.Core.DataLoad.Engine.LoadProcess;
17	using Rdmp.Core.DataLoad.Engine.Migration;
18	using Rdmp.Core.DataLoad.Triggers;
19	using Rdmp.Core.ReusableLibraryCode.Checks;
20	using Rdmp.Core.ReusableLibraryCode.DataAccess;
21
22	namespace Rdmp.Core.DataLoad.Engine.Checks.Checkers;
23
24	internal class CatalogueLoadChecks : ICheckable
25	{
26	private readonly ILoadMetadata _loadMetadata;
27	private readonly HICLoadConfigurationFlags _loadConfigurationFlags;
28	private readonly HICDatabaseConfiguration _databaseConfiguration;
29
30	public CatalogueLoadChecks(ILoadMetadata loadMetadata, HICLoadConfigurationFlags loadConfigurationFlags,	70✔
31	HICDatabaseConfiguration databaseConfiguration)	70✔
32	{
33	_loadMetadata = loadMetadata;	70✔
34	_loadConfigurationFlags = loadConfigurationFlags;	70✔
35	_databaseConfiguration = databaseConfiguration;	70✔
36	}	70✔
37
38
39	private bool ValidateFilePath(string directoryPath)
40	{
41	return Path.Exists(directoryPath);	272✔
42	}
43
44	public void Check(ICheckNotifier notifier)
45	{
46	Catalogue[] catalogueMetadatas;
47
48	try
49	{
50	//check there are catalogues and we can retrieve them
51	catalogueMetadatas = _loadMetadata.GetAllCatalogues().Cast<Catalogue>().ToArray();	64✔
52	}	64✔
53	catch (Exception e)	×
54	{
55	notifier.OnCheckPerformed(new CheckEventArgs("Crashed trying to fetch Catalogues for metadata",	×
56	CheckResult.Fail, e));	×
57	return;	×
58	}
59
60	//there no catalogues
61	if (catalogueMetadatas.Length == 0)	64!
62	notifier.OnCheckPerformed(new CheckEventArgs("There are no Catalogues associated with this metadata",	×
63	CheckResult.Fail, null));	×
64
65	var tablesFound = new List<ITableInfo>();	64✔
66
67	//check each catalogue is sufficiently configured to perform a migration
68	foreach (var catalogue in catalogueMetadatas)	264✔
69	{
70	notifier.OnCheckPerformed(new CheckEventArgs($"Found Catalogue:{catalogue}", CheckResult.Success, null));	68✔
71
72	var tableInfos = catalogue.GetTableInfoList(true).Distinct().ToArray();	68✔
73
74	if (tableInfos.Length == 0)	68!
75	notifier.OnCheckPerformed(new CheckEventArgs(	×
76	$"Catalogue {catalogue.Name} does not have any TableInfos", CheckResult.Fail, null));	×
77
78	tablesFound.AddRange(tableInfos.Where(tableInfo => !tablesFound.Contains(tableInfo)));	136✔
79	if(_loadMetadata.LocationOfForLoadingDirectory != null && !ValidateFilePath(_loadMetadata.LocationOfForLoadingDirectory))	68!
NEW 80	notifier.OnCheckPerformed(new CheckEventArgs($"The ForLoading directory for this load ({_loadMetadata.LocationOfForLoadingDirectory}) does not exist.",	×
NEW 81	CheckResult.Fail, null));	×
82	if (_loadMetadata.LocationOfForArchivingDirectory!= null && !ValidateFilePath(_loadMetadata.LocationOfForArchivingDirectory))	68!
NEW 83	notifier.OnCheckPerformed(new CheckEventArgs($"The ForArchiving directory for this load ({_loadMetadata.LocationOfForArchivingDirectory}) does not exist.",	×
NEW 84	CheckResult.Fail, null));	×
85	if (_loadMetadata.LocationOfCacheDirectory != null && _loadMetadata.LocationOfForLoadingDirectory != null && !ValidateFilePath(_loadMetadata.LocationOfCacheDirectory))	68!
NEW 86	notifier.OnCheckPerformed(new CheckEventArgs($"The Cache directory for this load ({_loadMetadata.LocationOfCacheDirectory}) does not exist.",	×
NEW 87	CheckResult.Fail, null));	×
88	if (_loadMetadata.LocationOfExecutablesDirectory != null && !ValidateFilePath(_loadMetadata.LocationOfExecutablesDirectory))	68!
NEW 89	notifier.OnCheckPerformed(new CheckEventArgs($"The Executables directory for this load ({_loadMetadata.LocationOfExecutablesDirectory}) does not exist.",	×
NEW 90	CheckResult.Fail, null));	×
91	}
92
93
94	//check regular tables
95	foreach (TableInfo regularTable in tablesFound)	264✔
96	{
97	notifier.OnCheckPerformed(new CheckEventArgs($"About To check configuration of TableInfo:{regularTable}",	68✔
98	CheckResult.Success, null));	68✔
99	CheckTableInfo(regularTable, notifier);	68✔
100
101	//check anonymisation
102	var anonymisationChecks = new AnonymisationChecks(regularTable);	68✔
103	anonymisationChecks.Check(notifier);	68✔
104	}
105	}	64✔
106
107
108	#region These methods check TableInfos (Lookup including lookups)
109
110	private void CheckTableInfo(TableInfo tableInfo, ICheckNotifier notifier)
111	{
112	//get all columns
113
114	//check whether the live database and staging databases have appropriate columns and triggers etc on them
115	var staging = _databaseConfiguration.DeployInfo[LoadBubble.Staging];	68✔
116	var live = DataAccessPortal.ExpectDatabase(tableInfo, DataAccessContext.DataLoad);	68✔
117
118	var liveTable =	68✔
119	live.ExpectTable(tableInfo.GetRuntimeName(LoadBubble.Live, _databaseConfiguration.DatabaseNamer),	68✔
120	tableInfo.Schema);	68✔
121	var liveCols = liveTable.DiscoverColumns();	68✔
122
123	CheckTableInfoSynchronization(tableInfo, notifier);	68✔
124
125	CheckTableHasColumnInfosAndPrimaryKeys(live, tableInfo, out _, out _, notifier);	68✔
126
127	//check for trying to ignore primary keys
128	foreach (var col in tableInfo.ColumnInfos)	1,792✔
129	if (col.IgnoreInLoads && col.IsPrimaryKey)	828!
130	notifier.OnCheckPerformed(	×
131	new CheckEventArgs($"ColumnInfo {col} is marked both IgnoreInLoads and IsPrimaryKey",	×
132	CheckResult.Fail));	×
133
134	try
135	{
136	if (!_loadMetadata.IgnoreTrigger)	68✔
137	{
138	//if trigger is created as part of this check then it is likely to have resulted in changes to the underlying table (e.g. added hic_validFrom field) in which case we should resynch the TableInfo to pickup these new columns
139	CheckTriggerIntact(liveTable, notifier, out var runSynchronizationAgain);	62✔
140
141	if (runSynchronizationAgain)	62!
142	CheckTableInfoSynchronization(tableInfo, notifier);	×
143	}
144
145	if (!_databaseConfiguration.RequiresStagingTableCreation)	68!
146	{
147	//Important:
148	//Regarding this if block: None of the current loading systems in RDMP have RequiresStagingTableCreation as false but in theory you could build some kind of multi threaded horror
149	//which had multiple simultaneous loads all populating a single STAGING bubble therefore this code is left in the codebase, it probably works ok but you will need a
150	//fair bit of work if you want to realise such a monstrosity (for one cleanup on each run probably cleans up STAGING on exit)
151
152	//live can have additional hic_ columns which do not appear in staging (lookups cannot)
153	var stagingTable =	×
154	staging.ExpectTable(tableInfo.GetRuntimeName(LoadBubble.Staging,	×
155	_databaseConfiguration.DatabaseNamer));	×
156
157	if (!stagingTable.Exists())	×
158	notifier.OnCheckPerformed(new CheckEventArgs(	×
159	"RequiresStagingTableCreation is false but staging does not exist, this flag should indicate that you anticipate STAGING to be already setup before you kick off the load",	×
160	CheckResult.Fail));	×
161
162	var stagingCols = stagingTable.DiscoverColumns();	×
163
164	ConfirmStagingAndLiveHaveSameColumns(tableInfo.GetRuntimeName(), stagingCols, liveCols, false,	×
165	notifier);	×
166
167	CheckStagingToLiveMigrationForTable(stagingTable, stagingCols, liveTable, liveCols, notifier);	×
168	}
169	}	68✔
170	catch (Exception e)	×
171	{
172	notifier.OnCheckPerformed(new CheckEventArgs($"Load Checks Crashed {tableInfo}", CheckResult.Fail, e));	×
173	}	×
174	}	68✔
175
176	private static void CheckTableInfoSynchronization(TableInfo tableInfo, ICheckNotifier notifier)
177	{
178	//live is the current data load's (possibly overridden server/database)
179	var tableInfoSynchronizer = new TableInfoSynchronizer(tableInfo);	68✔
180
181	var problemList = "";	68✔
182
183	//synchronize but refuse to apply all fixes, problems are instead added to problemList
184	if (tableInfoSynchronizer.Synchronize(notifier))	68!
185	{
186	notifier.OnCheckPerformed(new CheckEventArgs($"TableInfo {tableInfo} passed Synchronization check",	68✔
187	CheckResult.Success, null)); //passed synchronization	68✔
188	}
189	else
190	{
191	var launchSyncFixer = notifier.OnCheckPerformed(new CheckEventArgs(	×
192	$"TableInfo {tableInfo} failed Synchronization check with following problems:{problemList}",	×
193	CheckResult.Fail,	×
194	null, "Launch Synchronization Fixing")); //failed synchronization	×
195
196	if (launchSyncFixer)	×
197	{
198	var userFixed =	×
199	//if silent running accept all changes	×
200	tableInfoSynchronizer.Synchronize(notifier);	×
201
202	if (!userFixed)	×
203	notifier.OnCheckPerformed(new CheckEventArgs(	×
204	$"TableInfo {tableInfo} still failed Synchronization check", CheckResult.Fail,	×
205	null)); //passed synchronization	×
206	}
207	}
208	}	×
209
210	private static void CheckTableHasColumnInfosAndPrimaryKeys(DiscoveredDatabase live, TableInfo tableInfo,
211	out ColumnInfo[] columnInfos, out ColumnInfo[] columnInfosWhichArePrimaryKeys, ICheckNotifier notifier)
212	{
213	columnInfos = tableInfo.ColumnInfos.ToArray();	68✔
214	columnInfosWhichArePrimaryKeys = columnInfos.Where(col => col.IsPrimaryKey).ToArray();	896✔
215
216
217	//confirm there are at least 1
218	if (!columnInfos.Any())	68!
219	notifier.OnCheckPerformed(new CheckEventArgs($"TableInfo {tableInfo.Name} has no columninfos",	×
220	CheckResult.Fail, null));	×
221
222
223	if (!columnInfosWhichArePrimaryKeys.Any())	68!
224	notifier.OnCheckPerformed(new CheckEventArgs(	×
225	$"TableInfo {tableInfo.Name} has no IsPrimaryKey columns", CheckResult.Fail, null));	×
226
227	var primaryKeys = live.ExpectTable(tableInfo.GetRuntimeName(), tableInfo.Schema).DiscoverColumns()	68✔
228	.Where(c => c.IsPrimaryKey).ToArray();	896✔
229
230	if (primaryKeys.Any(k => k.IsAutoIncrement))	198!
231	notifier.OnCheckPerformed(new CheckEventArgs(	×
232	$"AutoIncrement columns {string.Join(",", primaryKeys.Where(k => k.IsAutoIncrement))} are not allowed as Primary Keys for your table because there is no way to differentiate new data being loaded from duplicate old data being loaded (the entire purpose of the RDMP DLE)",	×
233	CheckResult.Fail));	×
234
235	//confirm primary keys match underlying table
236	//sort pks alphabetically and confirm they match the underlying live system table
237	var actualPks = primaryKeys.Select(c => c.GetRuntimeName()).OrderBy(s => s).ToArray();	328✔
238	var pksWeExpect = columnInfosWhichArePrimaryKeys.Select(c => c.GetRuntimeName()).OrderBy(s => s).ToArray();	328✔
239
240
241	if (actualPks.Length != pksWeExpect.Length)	68!
242	notifier.OnCheckPerformed(new CheckEventArgs(	×
243	$"Primary keys in Catalogue for database table {tableInfo.GetRuntimeName()} does not match Catalogue entry (difference in number of keys)",	×
244	CheckResult.Fail, null));	×
245	else
246	for (var i = 0; i < pksWeExpect.Length; i++)	396✔
247	if (!pksWeExpect[i].Equals(actualPks[i]))	130!
248	notifier.OnCheckPerformed(new CheckEventArgs(	×
249	$"Mismatch between primary key defined in Catalogue {pksWeExpect[i]} and one found in live table {tableInfo.GetRuntimeName()}",	×
250	CheckResult.Fail, null));	×
251	else
252	notifier.OnCheckPerformed(new CheckEventArgs(	130✔
253	$"Found primary key {pksWeExpect[i]} in LIVE table {tableInfo.GetRuntimeName()}",	130✔
254	CheckResult.Success, null));	130✔
255	}	68✔
256
257	private static void CheckTriggerIntact(DiscoveredTable table, ICheckNotifier notifier,
258	out bool runSynchronizationAgain)
259	{
260	var checker = new TriggerChecks(table);	62✔
261	checker.Check(notifier);	62✔
262
263	runSynchronizationAgain = checker.TriggerCreated;	62✔
264	}	62✔
265
266
267	private static void ConfirmStagingAndLiveHaveSameColumns(string tableName, DiscoveredColumn[] stagingCols,
268	DiscoveredColumn[] liveCols, bool requireSameNumberAndOrder, ICheckNotifier notifier)
269	{
270	//in LIVE but not STAGING
271	foreach (var missingColumn in liveCols.Select(c => c.GetRuntimeName())	×
272	.Except(stagingCols.Select(c => c.GetRuntimeName())))	×
273	//column is in live but not in staging, but it is hic_
274	if (SpecialFieldNames.IsHicPrefixed(missingColumn)) //this is permitted	×
275	continue;
276	else
277	notifier.OnCheckPerformed(new CheckEventArgs(	×
278	$"Column {missingColumn} is missing from STAGING", CheckResult.Fail, null));	×
279
280	//in STAGING but not LIVE
281	foreach (var missingColumn in stagingCols.Except(liveCols))	×
282	notifier.OnCheckPerformed(new CheckEventArgs(	×
283	$"Column {missingColumn} is in STAGING but not LIVE", CheckResult.Fail, null));	×
284
285
286	if (requireSameNumberAndOrder)	×
287	{
288	var passedColumnOrderCheck = true;	×
289
290	if (stagingCols.Length != liveCols.Length)	×
291	{
292	notifier.OnCheckPerformed(new CheckEventArgs(	×
293	$"Column count mismatch between staging and live in table {tableName}", CheckResult.Fail, null));	×
294	passedColumnOrderCheck = false;	×
295	}
296	else
297	//check they are in the same order
298	{
299	for (var i = 0; i < stagingCols.Length; i++)	×
300	if (!stagingCols[i].Equals(liveCols[i]))	×
301	{
302	notifier.OnCheckPerformed(new CheckEventArgs(	×
303	$"Column name/order mismatch between staging and live in table {tableName}, column {i} is {stagingCols[i]} in staging but is {liveCols[i]} in live.",	×
304	CheckResult.Fail, null));	×
305	passedColumnOrderCheck = false;	×
306	break;	×
307	}
308	}
309
310	if (passedColumnOrderCheck)	×
311	notifier.OnCheckPerformed(new CheckEventArgs(	×
312	$"Column order match confirmed between staging and live on table {tableName}", CheckResult.Success,	×
313	null));	×
314	}
315	}	×
316
317	private void CheckStagingToLiveMigrationForTable(DiscoveredTable stagingTable, DiscoveredColumn[] stagingCols,
318	DiscoveredTable liveTable, DiscoveredColumn[] liveCols, ICheckNotifier notifier)
319	{
320	try
321	{
322	new MigrationColumnSet(stagingTable, liveTable, new StagingToLiveMigrationFieldProcessor	×
323	{	×
324	NoBackupTrigger = _loadMetadata.IgnoreTrigger	×
325	});	×
326	notifier.OnCheckPerformed(new CheckEventArgs(	×
327	$"TableInfo {liveTable} passed {nameof(MigrationColumnSet)} check ", CheckResult.Success, null));	×
328	}	×
329	catch (Exception e)	×
330	{
331	notifier.OnCheckPerformed(new CheckEventArgs(	×
332	$"{nameof(MigrationColumnSet)} reports a problem with the configuration of columns on STAGING/LIVE or in the ColumnInfos for TableInfo {liveTable}",	×
333	CheckResult.Fail, e));	×
334	}	×
335
336	//live columns
337	foreach (var col in liveCols)	×
338	if (!SpecialFieldNames.IsHicPrefixed(col) && col.IsAutoIncrement) //must start hic_ if they are identities	×
339	notifier.OnCheckPerformed(new CheckEventArgs(	×
340	$"Column {col} is an identity column in the LIVE database but does not start with hic_",	×
341	CheckResult.Fail, null)); //this one does not	×
342
343	//staging columns
344	foreach (var col in stagingCols) //staging columns	×
345	if (col.IsAutoIncrement) //if there are any auto increments	×
346	notifier.OnCheckPerformed(new CheckEventArgs(	×
347	$"Column {col} is an identity column and is in STAGING, the identity flag must be removed from the STAGING table",	×
348	CheckResult.Fail,	×
349	null)); //complain since don't want a mismatch between IDs in staging and live or complaints about identity insert from SQL server	×
350
351	//staging must allow null dataloadrunids and validfroms
352	ConfirmNullability(stagingTable.DiscoverColumn(SpecialFieldNames.DataLoadRunID), true, notifier);	×
353	ConfirmNullability(stagingTable.DiscoverColumn(SpecialFieldNames.ValidFrom), true, notifier);	×
354
355	//live must allow nulls in validFrom
356	ConfirmNullability(liveTable.DiscoverColumn(SpecialFieldNames.ValidFrom), true, notifier);	×
357	}	×
358
359	private static void ConfirmNullability(DiscoveredColumn column, bool expectedNullability, ICheckNotifier notifier)
360	{
361	var nullability = column.AllowNulls;	×
362	notifier.OnCheckPerformed(new CheckEventArgs(	×
363	$"Nullability of {column} is AllowNulls={nullability}, (expected {expectedNullability})",	×
364	nullability == expectedNullability ? CheckResult.Success : CheckResult.Fail, null));	×
365	}	×
366
367	#endregion
368	}

HicServices / RDMP / 9953564349

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous