11457925069

pending completion

Build # 11457925069

Build Type

push

github

Committed by

JFriel

Commit Message

Merge branch 'develop' of https://github.com/HicServices/RDMP

Run Details

11207 of 21044 branches covered (53.26%)

Branch coverage included in aggregate %.

25 of 51 new or added lines in 3 files covered. (49.02%)

705 existing lines in 27 files now uncovered.

31728 of 53779 relevant lines covered (59.0%)

4115.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.06

/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.IO;
using System.Linq;
using FAnsi.Discovery;
using FAnsi.Discovery.QuerySyntax;
using Rdmp.Core.Curation.Data.Cache;
using Rdmp.Core.Curation.Data.Defaults;
using Rdmp.Core.Curation.Data.ImportExport;
using Rdmp.Core.Curation.Data.Serialization;
using Rdmp.Core.Logging;
using Rdmp.Core.Logging.PastEvents;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Annotations;
using Rdmp.Core.ReusableLibraryCode.DataAccess;

namespace Rdmp.Core.Curation.Data.DataLoad;

/// <summary>
/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
/// </summary>
public enum CacheArchiveType
{
    /// <summary>
    /// Cached files are in a directory uncompressed
    /// </summary>
    None = 0,

    /// <summary>
    /// Cached files are contained in a zip file
    /// </summary>
    Zip = 1
}

/// <inheritdoc cref="ILoadMetadata"/>
public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
    ILoggedActivityRootObject, IHasFolder
{
    #region Database Properties

    private string _locationOfForLoadingDirectory;
    private string _locationOfForArchivingDirectory;
    private string _locationOfExecutablesDirectory;
    private string _locationOfCacheDirectory;
    private string _anonymisationEngineClass;
    private string _name;
    private string _description;
    private CacheArchiveType _cacheArchiveType;
    private int? _overrideRawServerID;
    private bool _ignoreTrigger;
    private string _folder;
    private DateTime? _lastLoadTime;
    private bool _allowReservedPrefix;

    public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");
    public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");
    public string DefaultExecutablesPath = "Executables";
    public string DefaultCachePath = Path.Combine("Data", "Cache");

    public DirectoryInfo GetRootDirectory()
    {
        if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))
        {
            var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");
            var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");
            var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");
            var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");
            if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)
            {
                return new DirectoryInfo(forLoadingRoot);
            }
        }
        return null;
    }


    ///  <inheritdoc/>
    public bool AllowReservedPrefix
    {
        get => _allowReservedPrefix;
        set => SetField(ref _allowReservedPrefix, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForLoadingDirectory
    {
        get => _locationOfForLoadingDirectory;
        set => SetField(ref _locationOfForLoadingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForArchivingDirectory
    {
        get => _locationOfForArchivingDirectory;
        set => SetField(ref _locationOfForArchivingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfExecutablesDirectory
    {
        get => _locationOfExecutablesDirectory;
        set => SetField(ref _locationOfExecutablesDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfCacheDirectory
    {
        get => _locationOfCacheDirectory;
        set => SetField(ref _locationOfCacheDirectory, value);
    }

    /// <summary>
    /// Not used
    /// </summary>
    public string AnonymisationEngineClass
    {
        get => _anonymisationEngineClass;
        set => SetField(ref _anonymisationEngineClass, value);
    }

    /// <inheritdoc/>
    [Unique]
    [NotNull]
    public string Name
    {
        get => _name;
        set => SetField(ref _name, value);
    }

    /// <summary>
    /// Human readable description of the load, what it does etc
    /// </summary>
    public string Description
    {
        get => _description;
        set => SetField(ref _description, value);
    }

    /// <summary>
    /// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>.  This may not be respected
    /// depending on the implementation of the sepecific ICacheLayout
    /// </summary>
    public CacheArchiveType CacheArchiveType
    {
        get => _cacheArchiveType;
        set => SetField(ref _cacheArchiveType, value);
    }

    /// <summary>
    /// Optional.  Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
    /// the system default - see <see cref="ServerDefaults"/>).
    /// </summary>
    public int? OverrideRAWServer_ID
    {
        get => _overrideRawServerID;
        set => SetField(ref _overrideRawServerID, value);
    }


    /// <iheritdoc/>
    public bool IgnoreTrigger
    {
        get => _ignoreTrigger;
        set => SetField(ref _ignoreTrigger, value);
    }

    /// <inheritdoc/>
    [UsefulProperty]
    public string Folder
    {
        get => _folder;
        set => SetField(ref _folder, FolderHelper.Adjust(value));
    }


    /// <summary>
    /// Stores the last time the load was ran.
    /// </summary>
    public DateTime? LastLoadTime
    {
        get => _lastLoadTime;
        set => SetField(ref _lastLoadTime, value);
    }

    #endregion


    #region Relationships

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue
        ? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)
        : null;

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public IOrderedEnumerable<IProcessTask> ProcessTasks
    {
        get
        {
            return
                Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);
        }
    }

    #endregion

    public LoadMetadata()
    {
    }

    /// <summary>
    /// Create a new DLE load.  This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
    /// 
    /// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="name"></param>
    public LoadMetadata(ICatalogueRepository repository, string name = null)
    {
        name ??= $"NewLoadMetadata{Guid.NewGuid()}";

        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "Name", name },
            { "IgnoreTrigger", false /*todo could be system global default here*/ },
            { "Folder", FolderHelper.Root },
            {"LastLoadTime", null }
        });
    }

    internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
    {
        LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();
        LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();
        LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();
        LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();
        Name = r["Name"] as string;
        AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();
        Name = r["Name"].ToString();
        Description = r["Description"] as string; //allows for nulls
        CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];
        OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);
        IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;
        Folder = r["Folder"] as string ?? FolderHelper.Root;
        LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());
        AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;
    }

    internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()
    {
        shareManager.UpsertAndHydrate(this, shareDefinition);
    }

    public void LinkToCatalogue(ICatalogue catalogue)
    {
        var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);
        linkage.SaveToDatabase();
    }

    public void UnlinkFromCatalogue(ICatalogue catalogue)
    {
        foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))
        {
            l.DeleteInDatabase();
        }
    }

    /// <inheritdoc/>
    public override void DeleteInDatabase()
    {
        var firstOrDefault = GetAllCatalogues().FirstOrDefault();

        if (firstOrDefault != null)
            throw new Exception(
                $"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");

        base.DeleteInDatabase();
    }

    /// <inheritdoc/>
    public override string ToString() => Name;

    /// <inheritdoc/>
    public IEnumerable<ICatalogue> GetAllCatalogues()
    {
        var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);
        return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));
    }

    /// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
    public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
    {
        var loggingServers = GetLoggingServers();

        var loggingServer = loggingServers.FirstOrDefault();

        //get distinct connection
        var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);

        serverChosen = (IExternalDatabaseServer)loggingServer;
        return toReturn;
    }

    /// <summary>
    /// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
    /// </summary>
    /// <returns></returns>
    public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);

    private IDataAccessPoint[] GetLoggingServers()
    {
        var catalogue = GetAllCatalogues().ToArray();

        return !catalogue.Any()
            ? throw new NotSupportedException(
                $"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")
            : (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();
    }

    /// <summary>
    /// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
    /// </summary>
    /// <returns></returns>
    public string GetDistinctLoggingTask()
    {
        var catalogueMetadatas = GetAllCatalogues().ToArray();

        if (!catalogueMetadatas.Any())
            throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");

        var cataloguesWithoutLoggingTasks =
            catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();

        if (cataloguesWithoutLoggingTasks.Any())
            throw new Exception(
                $"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");

        var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();
        return distinctLoggingTasks.Length >= 2
            ? throw new Exception(
                $"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")
            : distinctLoggingTasks[0];
    }

    /// <summary>
    /// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
    /// </summary>
    /// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
    /// <returns></returns>
    public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
    {
        var toReturn = new List<TableInfo>();

        foreach (var catalogueMetadata in GetAllCatalogues())
            foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))
                if (!toReturn.Contains(tableInfo))
                    toReturn.Add(tableInfo);

        return toReturn;
    }

    /// <inheritdoc/>
    public DiscoveredServer GetDistinctLiveDatabaseServer()
    {
        var normalTables = new HashSet<ITableInfo>();
        var lookupTables = new HashSet<ITableInfo>();

        foreach (var catalogue in GetAllCatalogues())
        {
            catalogue.GetTableInfos(out var normal, out var lookup);

            foreach (var n in normal)
                normalTables.Add(n);
            foreach (var l in lookup)
                lookupTables.Add(l);
        }

        if (normalTables.Any())
            return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);

        return lookupTables.Any()
            ? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)
            : throw new Exception(
                $"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");
    }

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsThisDependsOn() => null;

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();

    /// <summary>
    /// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
    /// </summary>
    /// <param name="catalogue"></param>
    public void EnsureLoggingWorksFor(ICatalogue catalogue)
    {
        //if there's no logging task / logging server set them up with the same name as the lmd
        IExternalDatabaseServer loggingServer;

        if (catalogue.LiveLoggingServer_ID == null)
        {
            loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);

            if (loggingServer != null)
                catalogue.LiveLoggingServer_ID = loggingServer.ID;
            else
                throw new NotSupportedException(
                    "You do not yet have any logging servers configured so cannot create data loads");
        }
        else
        {
            loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);
        }

        //if there's no logging task yet and there's a logging server
        if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))
        {
            var lm = new LogManager(loggingServer);
            var loggingTaskName = Name;

            lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);
            catalogue.LoggingDataTask = loggingTaskName;
            catalogue.SaveToDatabase();
        }
    }

    /// <inheritdoc/>
    public IQuerySyntaxHelper GetQuerySyntaxHelper()
    {
        var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();
        return syntax.Length > 1
            ? throw new Exception(
                $"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")
            : syntax.SingleOrDefault();
    }

    /// <summary>
    /// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
    /// </summary>
    /// <param name="runs"></param>
    /// <returns></returns>
    public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;

    public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
    {
        return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,
            loadMetadata).Any(p => p.Value == "true");
    }
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.IO;
11	using System.Linq;
12	using FAnsi.Discovery;
13	using FAnsi.Discovery.QuerySyntax;
14	using Rdmp.Core.Curation.Data.Cache;
15	using Rdmp.Core.Curation.Data.Defaults;
16	using Rdmp.Core.Curation.Data.ImportExport;
17	using Rdmp.Core.Curation.Data.Serialization;
18	using Rdmp.Core.Logging;
19	using Rdmp.Core.Logging.PastEvents;
20	using Rdmp.Core.MapsDirectlyToDatabaseTable;
21	using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
22	using Rdmp.Core.Repositories;
23	using Rdmp.Core.ReusableLibraryCode;
24	using Rdmp.Core.ReusableLibraryCode.Annotations;
25	using Rdmp.Core.ReusableLibraryCode.DataAccess;
26
27	namespace Rdmp.Core.Curation.Data.DataLoad;
28
29	/// <summary>
30	/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
31	/// </summary>
32	public enum CacheArchiveType
33	{
34	/// <summary>
35	/// Cached files are in a directory uncompressed
36	/// </summary>
37	None = 0,
38
39	/// <summary>
40	/// Cached files are contained in a zip file
41	/// </summary>
42	Zip = 1
43	}
44
45	/// <inheritdoc cref="ILoadMetadata"/>
46	public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
47	ILoggedActivityRootObject, IHasFolder
48	{
49	#region Database Properties
50
51	private string _locationOfForLoadingDirectory;
52	private string _locationOfForArchivingDirectory;
53	private string _locationOfExecutablesDirectory;
54	private string _locationOfCacheDirectory;
55	private string _anonymisationEngineClass;
56	private string _name;
57	private string _description;
58	private CacheArchiveType _cacheArchiveType;
59	private int? _overrideRawServerID;
60	private bool _ignoreTrigger;
61	private string _folder;
62	private DateTime? _lastLoadTime;
63	private bool _allowReservedPrefix;
64
65	public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");	1,028✔
66	public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");	1,028✔
67	public string DefaultExecutablesPath = "Executables";	1,028✔
68	public string DefaultCachePath = Path.Combine("Data", "Cache");	1,028✔
69
70	public DirectoryInfo GetRootDirectory()
71	{
72	if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))	×
73	{
74	var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");	×
75	var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");	×
76	var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");	×
77	var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");	×
78	if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)	×
79	{
80	return new DirectoryInfo(forLoadingRoot);	×
81	}
82	}
83	return null;	×
84	}
85
86
87	/// <inheritdoc/>
88	public bool AllowReservedPrefix
89	{
90	get => _allowReservedPrefix;	772✔
91	set => SetField(ref _allowReservedPrefix, value);	876✔
92	}
93
94	/// <inheritdoc/>
95	public string LocationOfForLoadingDirectory
96	{
97	get => _locationOfForLoadingDirectory;	1,970✔
98	set => SetField(ref _locationOfForLoadingDirectory, value);	990✔
99	}
100
101	/// <inheritdoc/>
102	public string LocationOfForArchivingDirectory
103	{
104	get => _locationOfForArchivingDirectory;	1,732✔
105	set => SetField(ref _locationOfForArchivingDirectory, value);	990✔
106	}
107
108	/// <inheritdoc/>
109	public string LocationOfExecutablesDirectory
110	{
111	get => _locationOfExecutablesDirectory;	1,732✔
112	set => SetField(ref _locationOfExecutablesDirectory, value);	990✔
113	}
114
115	/// <inheritdoc/>
116	public string LocationOfCacheDirectory
117	{
118	get => _locationOfCacheDirectory;	1,732✔
119	set => SetField(ref _locationOfCacheDirectory, value);	990✔
120	}
121
122	/// <summary>
123	/// Not used
124	/// </summary>
125	public string AnonymisationEngineClass
126	{
127	get => _anonymisationEngineClass;	730✔
128	set => SetField(ref _anonymisationEngineClass, value);	866✔
129	}
130
131	/// <inheritdoc/>
132	[Unique]
133	[NotNull]
134	public string Name
135	{
136	get => _name;	1,020✔
137	set => SetField(ref _name, value);	1,656✔
138	}
139
140	/// <summary>
141	/// Human readable description of the load, what it does etc
142	/// </summary>
143	public string Description
144	{
145	get => _description;	732✔
146	set => SetField(ref _description, value);	866✔
147	}
148
149	/// <summary>
150	/// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>. This may not be respected
151	/// depending on the implementation of the sepecific ICacheLayout
152	/// </summary>
153	public CacheArchiveType CacheArchiveType
154	{
155	get => _cacheArchiveType;	730✔
156	set => SetField(ref _cacheArchiveType, value);	876✔
157	}
158
159	/// <summary>
160	/// Optional. Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
161	/// the system default - see <see cref="ServerDefaults"/>).
162	/// </summary>
163	public int? OverrideRAWServer_ID
164	{
165	get => _overrideRawServerID;	1,608✔
166	set => SetField(ref _overrideRawServerID, value);	866✔
167	}
168
169
170	/// <iheritdoc/>
171	public bool IgnoreTrigger
172	{
173	get => _ignoreTrigger;	1,066✔
174	set => SetField(ref _ignoreTrigger, value);	1,010✔
175	}
176
177	/// <inheritdoc/>
178	[UsefulProperty]
179	public string Folder
180	{
181	get => _folder;	1,378✔
182	set => SetField(ref _folder, FolderHelper.Adjust(value));	1,000✔
183	}
184
185
186	/// <summary>
187	/// Stores the last time the load was ran.
188	/// </summary>
189	public DateTime? LastLoadTime
190	{
191	get => _lastLoadTime;	738✔
192	set => SetField(ref _lastLoadTime, value);	1,018✔
193	}
194
195	#endregion
196
197
198	#region Relationships
199
200	/// <inheritdoc/>
201	[NoMappingToDatabase]
202	public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue	336!
203	? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)	336✔
204	: null;	336✔
205
206	/// <inheritdoc/>
207	[NoMappingToDatabase]
208	public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);	202✔
209
210	/// <inheritdoc/>
211	[NoMappingToDatabase]
212	public IOrderedEnumerable<IProcessTask> ProcessTasks
213	{
214	get
215	{
216	return	280✔
217	Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);	578✔
218	}
219	}
220
221	#endregion
222
223	public LoadMetadata()	30✔
224	{
225	}	30✔
226
227	/// <summary>
228	/// Create a new DLE load. This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
229	///
230	/// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
231	/// </summary>
232	/// <param name="repository"></param>
233	/// <param name="name"></param>
234	public LoadMetadata(ICatalogueRepository repository, string name = null)	332✔
235	{
236	name ??= $"NewLoadMetadata{Guid.NewGuid()}";	332✔
237
238	repository.InsertAndHydrate(this, new Dictionary<string, object>	332✔
239	{	332✔
240	{ "Name", name },	332✔
241	{ "IgnoreTrigger", false /todo could be system global default here/ },	332✔
242	{ "Folder", FolderHelper.Root },	332✔
243	{"LastLoadTime", null }	332✔
244	});	332✔
245	}	332✔
246
247	internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
248	: base(repository, r)	656✔
249	{
250	LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();	656✔
251	LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();	656✔
252	LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();	656✔
253	LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();	656✔
254	Name = r["Name"] as string;	656✔
255	AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();	656✔
256	Name = r["Name"].ToString();	656✔
257	Description = r["Description"] as string; //allows for nulls	656✔
258	CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];	656✔
259	OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);	656✔
260	IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;	656✔
261	Folder = r["Folder"] as string ?? FolderHelper.Root;	656!
262	LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());	656✔
263	AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;	656✔
264	}	656✔
265
266	internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()	10✔
267	{
268	shareManager.UpsertAndHydrate(this, shareDefinition);	10✔
269	}	10✔
270
271	public void LinkToCatalogue(ICatalogue catalogue)
272	{
273	var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);	282✔
274	linkage.SaveToDatabase();	282✔
275	}	282✔
276
277	public void UnlinkFromCatalogue(ICatalogue catalogue)
278	{
279	foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))	56✔
280	{
281	l.DeleteInDatabase();	8✔
282	}
283	}	8✔
284
285	/// <inheritdoc/>
286	public override void DeleteInDatabase()
287	{
288	var firstOrDefault = GetAllCatalogues().FirstOrDefault();	74✔
289
290	if (firstOrDefault != null)	74!
UNCOV 291	throw new Exception(	×
UNCOV 292	$"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");	×
293
294	base.DeleteInDatabase();	74✔
295	}	74✔
296
297	/// <inheritdoc/>
298	public override string ToString() => Name;	98✔
299
300	/// <inheritdoc/>
301	public IEnumerable<ICatalogue> GetAllCatalogues()
302	{
303	var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);	3,756✔
304	return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));	3,636✔
305	}
306
307	/// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
308	public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
309	{
310	var loggingServers = GetLoggingServers();	114✔
311
312	var loggingServer = loggingServers.FirstOrDefault();	112✔
313
314	//get distinct connection
315	var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);	112✔
316
317	serverChosen = (IExternalDatabaseServer)loggingServer;	108✔
318	return toReturn;	108✔
319	}
320
321	/// <summary>
322	/// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
323	/// </summary>
324	/// <returns></returns>
325	public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);	114✔
326
327	private IDataAccessPoint[] GetLoggingServers()
328	{
329	var catalogue = GetAllCatalogues().ToArray();	114✔
330
331	return !catalogue.Any()	114!
332	? throw new NotSupportedException(	114✔
333	$"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")	114✔
334	: (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();	236✔
335	}
336
337	/// <summary>
338	/// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
339	/// </summary>
340	/// <returns></returns>
341	public string GetDistinctLoggingTask()
342	{
343	var catalogueMetadatas = GetAllCatalogues().ToArray();	88✔
344
345	if (!catalogueMetadatas.Any())	88!
UNCOV 346	throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");	×
347
348	var cataloguesWithoutLoggingTasks =	88✔
349	catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();	186✔
350
351	if (cataloguesWithoutLoggingTasks.Any())	88✔
352	throw new Exception(	2✔
353	$"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");	4✔
354
355	var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();	180✔
356	return distinctLoggingTasks.Length >= 2	86!
357	? throw new Exception(	86✔
358	$"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")	86✔
359	: distinctLoggingTasks[0];	86✔
360	}
361
362	/// <summary>
363	/// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
364	/// </summary>
365	/// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
366	/// <returns></returns>
367	public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
368	{
369	var toReturn = new List<TableInfo>();	8✔
370
371	foreach (var catalogueMetadata in GetAllCatalogues())	32✔
372	foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))	32✔
373	if (!toReturn.Contains(tableInfo))	8✔
374	toReturn.Add(tableInfo);	8✔
375
376	return toReturn;	8✔
377	}
378
379	/// <inheritdoc/>
380	public DiscoveredServer GetDistinctLiveDatabaseServer()
381	{
382	var normalTables = new HashSet<ITableInfo>();	230✔
383	var lookupTables = new HashSet<ITableInfo>();	230✔
384
385	foreach (var catalogue in GetAllCatalogues())	944✔
386	{
387	catalogue.GetTableInfos(out var normal, out var lookup);	242✔
388
389	foreach (var n in normal)	968✔
390	normalTables.Add(n);	242✔
391	foreach (var l in lookup)	484!
392	lookupTables.Add(l);	×
393	}
394
395	if (normalTables.Any())	230!
396	return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);	230✔
397
UNCOV 398	return lookupTables.Any()	×
UNCOV 399	? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)	×
UNCOV 400	: throw new Exception(	×
UNCOV 401	$"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");	×
402	}
403
404	/// <inheritdoc/>
UNCOV 405	public IHasDependencies[] GetObjectsThisDependsOn() => null;	×
406
407	/// <inheritdoc/>
408	public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();	74✔
409
410	/// <summary>
411	/// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
412	/// </summary>
413	/// <param name="catalogue"></param>
414	public void EnsureLoggingWorksFor(ICatalogue catalogue)
415	{
416	//if there's no logging task / logging server set them up with the same name as the lmd
417	IExternalDatabaseServer loggingServer;
418
419	if (catalogue.LiveLoggingServer_ID == null)	14!
420	{
UNCOV 421	loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);	×
422
UNCOV 423	if (loggingServer != null)	×
UNCOV 424	catalogue.LiveLoggingServer_ID = loggingServer.ID;	×
425	else
UNCOV 426	throw new NotSupportedException(	×
UNCOV 427	"You do not yet have any logging servers configured so cannot create data loads");	×
428	}
429	else
430	{
431	loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);	14✔
432	}
433
434	//if there's no logging task yet and there's a logging server
435	if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))	14✔
436	{
437	var lm = new LogManager(loggingServer);	14✔
438	var loggingTaskName = Name;	14✔
439
440	lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);	14✔
441	catalogue.LoggingDataTask = loggingTaskName;	14✔
442	catalogue.SaveToDatabase();	14✔
443	}
444	}	14✔
445
446	/// <inheritdoc/>
447	public IQuerySyntaxHelper GetQuerySyntaxHelper()
448	{
449	var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();	4✔
450	return syntax.Length > 1	2!
451	? throw new Exception(	2✔
452	$"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")	2✔
453	: syntax.SingleOrDefault();	2✔
454	}
455
456	/// <summary>
457	/// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
458	/// </summary>
459	/// <param name="runs"></param>
460	/// <returns></returns>
461	public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;	12✔
462
463	public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
464	{
465	return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,	90✔
466	loadMetadata).Any(p => p.Value == "true");	90✔
467	}
468	}

HicServices / RDMP / 11457925069

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous