11978888584

Committed 22 Nov 2024 07:27PM UTC coverage: 57.383% (-0.002%) from 57.385%

Build # 11978888584

Build Type

push

github

Committed by

jas88

Commit Message

Fix up redundant type inheritance

Run Details

11206 of 21050 branches covered (53.24%)

Branch coverage included in aggregate %.

65 of 249 new or added lines in 42 files covered. (26.1%)

17 existing lines in 14 files now uncovered.

31718 of 53752 relevant lines covered (59.01%)

8290.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.59

/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.IO;
using System.Linq;
using FAnsi.Discovery;
using FAnsi.Discovery.QuerySyntax;
using Rdmp.Core.Curation.Data.Cache;
using Rdmp.Core.Curation.Data.Defaults;
using Rdmp.Core.Curation.Data.ImportExport;
using Rdmp.Core.Curation.Data.Serialization;
using Rdmp.Core.Logging;
using Rdmp.Core.Logging.PastEvents;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Annotations;
using Rdmp.Core.ReusableLibraryCode.DataAccess;

namespace Rdmp.Core.Curation.Data.DataLoad;

/// <summary>
/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
/// </summary>
public enum CacheArchiveType
{
    /// <summary>
    /// Cached files are in a directory uncompressed
    /// </summary>
    None = 0,

    /// <summary>
    /// Cached files are contained in a zip file
    /// </summary>
    Zip = 1
}

/// <inheritdoc cref="ILoadMetadata"/>
public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper, IHasFolder
{
    #region Database Properties

    private string _locationOfForLoadingDirectory;
    private string _locationOfForArchivingDirectory;
    private string _locationOfExecutablesDirectory;
    private string _locationOfCacheDirectory;
    private string _anonymisationEngineClass;
    private string _name;
    private string _description;
    private CacheArchiveType _cacheArchiveType;
    private int? _overrideRawServerID;
    private bool _ignoreTrigger;
    private string _folder;
    private DateTime? _lastLoadTime;
    private bool _allowReservedPrefix;

    public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");
    public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");
    public string DefaultExecutablesPath = "Executables";
    public string DefaultCachePath = Path.Combine("Data", "Cache");

    public DirectoryInfo GetRootDirectory()
    {
        if (string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) ||
            string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) ||
            string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) ||
            string.IsNullOrWhiteSpace(_locationOfCacheDirectory)) return null;

        var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");
        var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");
        var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");
        var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");
        if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot) return new DirectoryInfo(forLoadingRoot);

        return null;
    }


    ///  <inheritdoc/>
    public bool AllowReservedPrefix
    {
        get => _allowReservedPrefix;
        set => SetField(ref _allowReservedPrefix, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForLoadingDirectory
    {
        get => _locationOfForLoadingDirectory;
        set => SetField(ref _locationOfForLoadingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForArchivingDirectory
    {
        get => _locationOfForArchivingDirectory;
        set => SetField(ref _locationOfForArchivingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfExecutablesDirectory
    {
        get => _locationOfExecutablesDirectory;
        set => SetField(ref _locationOfExecutablesDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfCacheDirectory
    {
        get => _locationOfCacheDirectory;
        set => SetField(ref _locationOfCacheDirectory, value);
    }

    /// <summary>
    /// Not used
    /// </summary>
    public string AnonymisationEngineClass
    {
        get => _anonymisationEngineClass;
        set => SetField(ref _anonymisationEngineClass, value);
    }

    /// <inheritdoc/>
    [Unique]
    [NotNull]
    public string Name
    {
        get => _name;
        set => SetField(ref _name, value);
    }

    /// <summary>
    /// Human readable description of the load, what it does etc
    /// </summary>
    public string Description
    {
        get => _description;
        set => SetField(ref _description, value);
    }

    /// <summary>
    /// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>.  This may not be respected
    /// depending on the implementation of the sepecific ICacheLayout
    /// </summary>
    public CacheArchiveType CacheArchiveType
    {
        get => _cacheArchiveType;
        set => SetField(ref _cacheArchiveType, value);
    }

    /// <summary>
    /// Optional.  Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
    /// the system default - see <see cref="ServerDefaults"/>).
    /// </summary>
    public int? OverrideRAWServer_ID
    {
        get => _overrideRawServerID;
        set => SetField(ref _overrideRawServerID, value);
    }


    /// <iheritdoc/>
    public bool IgnoreTrigger
    {
        get => _ignoreTrigger;
        set => SetField(ref _ignoreTrigger, value);
    }

    /// <inheritdoc/>
    [UsefulProperty]
    public string Folder
    {
        get => _folder;
        set => SetField(ref _folder, FolderHelper.Adjust(value));
    }


    /// <summary>
    /// Stores the last time the load was ran.
    /// </summary>
    public DateTime? LastLoadTime
    {
        get => _lastLoadTime;
        set => SetField(ref _lastLoadTime, value);
    }

    #endregion


    #region Relationships

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue
        ? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)
        : null;

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public IOrderedEnumerable<IProcessTask> ProcessTasks
    {
        get
        {
            return
                Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(static pt => pt.Order);
        }
    }

    #endregion

    public LoadMetadata()
    {
    }

    /// <summary>
    /// Create a new DLE load.  This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
    /// 
    /// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="name"></param>
    public LoadMetadata(ICatalogueRepository repository, string name = null)
    {
        name ??= $"NewLoadMetadata{Guid.NewGuid()}";

        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "Name", name },
            { "IgnoreTrigger", false /*todo could be system global default here*/ },
            { "Folder", FolderHelper.Root },
            {"LastLoadTime", null }
        });
    }

    internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
    {
        LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();
        LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();
        LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();
        LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();
        Name = r["Name"] as string;
        AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();
        Name = r["Name"].ToString();
        Description = r["Description"] as string; //allows for nulls
        CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];
        OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);
        IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;
        Folder = r["Folder"] as string ?? FolderHelper.Root;
        LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());
        AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;
    }

    internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition)
    {
        shareManager.UpsertAndHydrate(this, shareDefinition);
    }

    public void LinkToCatalogue(ICatalogue catalogue)
    {
        var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);
        linkage.SaveToDatabase();
    }

    public void UnlinkFromCatalogue(ICatalogue catalogue)
    {
        foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>()
                     .Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))
            l.DeleteInDatabase();
    }

    /// <inheritdoc/>
    public override void DeleteInDatabase()
    {
        var firstOrDefault = GetAllCatalogues().FirstOrDefault();

        if (firstOrDefault != null)
            throw new Exception(
                $"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");

        base.DeleteInDatabase();
    }

    /// <inheritdoc/>
    public override string ToString() => Name;

    /// <inheritdoc/>
    public IEnumerable<ICatalogue> GetAllCatalogues()
    {
        var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(static l => l.CatalogueID);
        return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));
    }

    /// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
    public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
    {
        var loggingServers = GetLoggingServers();

        var loggingServer = loggingServers.FirstOrDefault();

        //get distinct connection
        var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);

        serverChosen = (IExternalDatabaseServer)loggingServer;
        return toReturn;
    }

    /// <summary>
    /// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
    /// </summary>
    /// <returns></returns>
    public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);

    private IDataAccessPoint[] GetLoggingServers()
    {
        var catalogue = GetAllCatalogues().ToArray();

        return !catalogue.Any()
            ? throw new NotSupportedException(
                $"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")
            : (IDataAccessPoint[])catalogue.Select(static c => c.LiveLoggingServer).ToArray();
    }

    /// <summary>
    /// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
    /// </summary>
    /// <returns></returns>
    public string GetDistinctLoggingTask()
    {
        var catalogueMetadatas = GetAllCatalogues().ToArray();

        if (catalogueMetadatas.Length == 0)
            throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");

        var cataloguesWithoutLoggingTasks =
            catalogueMetadatas.Where(static c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();

        if (cataloguesWithoutLoggingTasks.Length != 0)
            throw new Exception(
                $"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", static (s, n) => $"{s}{n}(ID={n.ID}),")}");

        var distinctLoggingTasks = catalogueMetadatas.Select(static c => c.LoggingDataTask).Distinct().ToArray();
        return distinctLoggingTasks.Length >= 2
            ? throw new Exception(
                $"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")
            : distinctLoggingTasks[0];
    }

    /// <summary>
    /// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
    /// </summary>
    /// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
    /// <returns></returns>
    public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
    {
        return GetAllCatalogues().SelectMany(cat => cat.GetTableInfoList(includeLookups)).Distinct().OfType<TableInfo>()
            .ToList();
    }

    /// <inheritdoc/>
    public DiscoveredServer GetDistinctLiveDatabaseServer()
    {
        var normalTables = new HashSet<ITableInfo>();
        var lookupTables = new HashSet<ITableInfo>();

        foreach (var catalogue in GetAllCatalogues())
        {
            catalogue.GetTableInfos(out var normal, out var lookup);

            foreach (var n in normal)
                normalTables.Add(n);
            foreach (var l in lookup)
                lookupTables.Add(l);
        }

        if (normalTables.Count != 0)
            return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);

        return lookupTables.Count != 0
            ? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)
            : throw new Exception(
                $"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");
    }

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsThisDependsOn() => null;

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();

    /// <summary>
    /// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
    /// </summary>
    /// <param name="catalogue"></param>
    public void EnsureLoggingWorksFor(ICatalogue catalogue)
    {
        //if there's no logging task / logging server set them up with the same name as the lmd
        IExternalDatabaseServer loggingServer;

        if (catalogue.LiveLoggingServer_ID == null)
        {
            loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);

            if (loggingServer != null)
                catalogue.LiveLoggingServer_ID = loggingServer.ID;
            else
                throw new NotSupportedException(
                    "You do not yet have any logging servers configured so cannot create data loads");
        }
        else
        {
            loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);
        }

        //if there's no logging task yet and there's a logging server
        if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))
        {
            var lm = new LogManager(loggingServer);
            var loggingTaskName = Name;

            lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);
            catalogue.LoggingDataTask = loggingTaskName;
            catalogue.SaveToDatabase();
        }
    }

    /// <inheritdoc/>
    public IQuerySyntaxHelper GetQuerySyntaxHelper()
    {
        var syntax = GetAllCatalogues().Select(static c => c.GetQuerySyntaxHelper()).Distinct().ToArray();
        return syntax.Length > 1
            ? throw new Exception(
                $"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")
            : syntax.SingleOrDefault();
    }

    /// <summary>
    /// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
    /// </summary>
    /// <param name="runs"></param>
    /// <returns></returns>
    public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;

    public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
    {
        return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,
            loadMetadata).Any(static p => p.Value == "true");
    }
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.IO;
11	using System.Linq;
12	using FAnsi.Discovery;
13	using FAnsi.Discovery.QuerySyntax;
14	using Rdmp.Core.Curation.Data.Cache;
15	using Rdmp.Core.Curation.Data.Defaults;
16	using Rdmp.Core.Curation.Data.ImportExport;
17	using Rdmp.Core.Curation.Data.Serialization;
18	using Rdmp.Core.Logging;
19	using Rdmp.Core.Logging.PastEvents;
20	using Rdmp.Core.MapsDirectlyToDatabaseTable;
21	using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
22	using Rdmp.Core.Repositories;
23	using Rdmp.Core.ReusableLibraryCode;
24	using Rdmp.Core.ReusableLibraryCode.Annotations;
25	using Rdmp.Core.ReusableLibraryCode.DataAccess;
26
27	namespace Rdmp.Core.Curation.Data.DataLoad;
28
29	/// <summary>
30	/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
31	/// </summary>
32	public enum CacheArchiveType
33	{
34	/// <summary>
35	/// Cached files are in a directory uncompressed
36	/// </summary>
37	None = 0,
38
39	/// <summary>
40	/// Cached files are contained in a zip file
41	/// </summary>
42	Zip = 1
43	}
44
45	/// <inheritdoc cref="ILoadMetadata"/>
46	public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper, IHasFolder
47	{
48	#region Database Properties
49
50	private string _locationOfForLoadingDirectory;
51	private string _locationOfForArchivingDirectory;
52	private string _locationOfExecutablesDirectory;
53	private string _locationOfCacheDirectory;
54	private string _anonymisationEngineClass;
55	private string _name;
56	private string _description;
57	private CacheArchiveType _cacheArchiveType;
58	private int? _overrideRawServerID;
59	private bool _ignoreTrigger;
60	private string _folder;
61	private DateTime? _lastLoadTime;
62	private bool _allowReservedPrefix;
63
64	public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");	1,028✔
65	public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");	1,028✔
66	public string DefaultExecutablesPath = "Executables";	1,028✔
67	public string DefaultCachePath = Path.Combine("Data", "Cache");	1,028✔
68
69	public DirectoryInfo GetRootDirectory()
70	{
NEW 71	if (string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) \|\|	×
NEW 72	string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) \|\|	×
NEW 73	string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) \|\|	×
NEW 74	string.IsNullOrWhiteSpace(_locationOfCacheDirectory)) return null;	×
75
NEW 76	var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");	×
NEW 77	var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");	×
NEW 78	var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");	×
NEW 79	var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");	×
NEW 80	if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot) return new DirectoryInfo(forLoadingRoot);	×
81
UNCOV 82	return null;	×
83	}
84
85
86	/// <inheritdoc/>
87	public bool AllowReservedPrefix
88	{
89	get => _allowReservedPrefix;	772✔
90	set => SetField(ref _allowReservedPrefix, value);	876✔
91	}
92
93	/// <inheritdoc/>
94	public string LocationOfForLoadingDirectory
95	{
96	get => _locationOfForLoadingDirectory;	1,970✔
97	set => SetField(ref _locationOfForLoadingDirectory, value);	990✔
98	}
99
100	/// <inheritdoc/>
101	public string LocationOfForArchivingDirectory
102	{
103	get => _locationOfForArchivingDirectory;	1,732✔
104	set => SetField(ref _locationOfForArchivingDirectory, value);	990✔
105	}
106
107	/// <inheritdoc/>
108	public string LocationOfExecutablesDirectory
109	{
110	get => _locationOfExecutablesDirectory;	1,732✔
111	set => SetField(ref _locationOfExecutablesDirectory, value);	990✔
112	}
113
114	/// <inheritdoc/>
115	public string LocationOfCacheDirectory
116	{
117	get => _locationOfCacheDirectory;	1,732✔
118	set => SetField(ref _locationOfCacheDirectory, value);	990✔
119	}
120
121	/// <summary>
122	/// Not used
123	/// </summary>
124	public string AnonymisationEngineClass
125	{
126	get => _anonymisationEngineClass;	730✔
127	set => SetField(ref _anonymisationEngineClass, value);	866✔
128	}
129
130	/// <inheritdoc/>
131	[Unique]
132	[NotNull]
133	public string Name
134	{
135	get => _name;	1,020✔
136	set => SetField(ref _name, value);	1,656✔
137	}
138
139	/// <summary>
140	/// Human readable description of the load, what it does etc
141	/// </summary>
142	public string Description
143	{
144	get => _description;	732✔
145	set => SetField(ref _description, value);	866✔
146	}
147
148	/// <summary>
149	/// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>. This may not be respected
150	/// depending on the implementation of the sepecific ICacheLayout
151	/// </summary>
152	public CacheArchiveType CacheArchiveType
153	{
154	get => _cacheArchiveType;	730✔
155	set => SetField(ref _cacheArchiveType, value);	876✔
156	}
157
158	/// <summary>
159	/// Optional. Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
160	/// the system default - see <see cref="ServerDefaults"/>).
161	/// </summary>
162	public int? OverrideRAWServer_ID
163	{
164	get => _overrideRawServerID;	1,608✔
165	set => SetField(ref _overrideRawServerID, value);	866✔
166	}
167
168
169	/// <iheritdoc/>
170	public bool IgnoreTrigger
171	{
172	get => _ignoreTrigger;	1,066✔
173	set => SetField(ref _ignoreTrigger, value);	1,010✔
174	}
175
176	/// <inheritdoc/>
177	[UsefulProperty]
178	public string Folder
179	{
180	get => _folder;	1,378✔
181	set => SetField(ref _folder, FolderHelper.Adjust(value));	1,000✔
182	}
183
184
185	/// <summary>
186	/// Stores the last time the load was ran.
187	/// </summary>
188	public DateTime? LastLoadTime
189	{
190	get => _lastLoadTime;	738✔
191	set => SetField(ref _lastLoadTime, value);	1,018✔
192	}
193
194	#endregion
195
196
197	#region Relationships
198
199	/// <inheritdoc/>
200	[NoMappingToDatabase]
201	public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue	336!
202	? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)	336✔
203	: null;	336✔
204
205	/// <inheritdoc/>
206	[NoMappingToDatabase]
207	public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);	202✔
208
209	/// <inheritdoc/>
210	[NoMappingToDatabase]
211	public IOrderedEnumerable<IProcessTask> ProcessTasks
212	{
213	get
214	{
215	return	280✔
216	Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(static pt => pt.Order);	578✔
217	}
218	}
219
220	#endregion
221
222	public LoadMetadata()	30✔
223	{
224	}	30✔
225
226	/// <summary>
227	/// Create a new DLE load. This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
228	///
229	/// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
230	/// </summary>
231	/// <param name="repository"></param>
232	/// <param name="name"></param>
233	public LoadMetadata(ICatalogueRepository repository, string name = null)	332✔
234	{
235	name ??= $"NewLoadMetadata{Guid.NewGuid()}";	332✔
236
237	repository.InsertAndHydrate(this, new Dictionary<string, object>	332✔
238	{	332✔
239	{ "Name", name },	332✔
240	{ "IgnoreTrigger", false /todo could be system global default here/ },	332✔
241	{ "Folder", FolderHelper.Root },	332✔
242	{"LastLoadTime", null }	332✔
243	});	332✔
244	}	332✔
245
246	internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
247	: base(repository, r)	656✔
248	{
249	LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();	656✔
250	LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();	656✔
251	LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();	656✔
252	LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();	656✔
253	Name = r["Name"] as string;	656✔
254	AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();	656✔
255	Name = r["Name"].ToString();	656✔
256	Description = r["Description"] as string; //allows for nulls	656✔
257	CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];	656✔
258	OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);	656✔
259	IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;	656✔
260	Folder = r["Folder"] as string ?? FolderHelper.Root;	656!
261	LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());	656✔
262	AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;	656✔
263	}	656✔
264
265	internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition)	10✔
266	{
267	shareManager.UpsertAndHydrate(this, shareDefinition);	10✔
268	}	10✔
269
270	public void LinkToCatalogue(ICatalogue catalogue)
271	{
272	var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);	282✔
273	linkage.SaveToDatabase();	282✔
274	}	282✔
275
276	public void UnlinkFromCatalogue(ICatalogue catalogue)
277	{
278	foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>()	32✔
279	.Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))	32✔
280	l.DeleteInDatabase();	8✔
281	}	8✔
282
283	/// <inheritdoc/>
284	public override void DeleteInDatabase()
285	{
286	var firstOrDefault = GetAllCatalogues().FirstOrDefault();	74✔
287
288	if (firstOrDefault != null)	74!
289	throw new Exception(	×
290	$"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");	×
291
292	base.DeleteInDatabase();	74✔
293	}	74✔
294
295	/// <inheritdoc/>
296	public override string ToString() => Name;	98✔
297
298	/// <inheritdoc/>
299	public IEnumerable<ICatalogue> GetAllCatalogues()
300	{
301	var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(static l => l.CatalogueID);	3,756✔
302	return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));	3,636✔
303	}
304
305	/// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
306	public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
307	{
308	var loggingServers = GetLoggingServers();	114✔
309
310	var loggingServer = loggingServers.FirstOrDefault();	112✔
311
312	//get distinct connection
313	var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);	112✔
314
315	serverChosen = (IExternalDatabaseServer)loggingServer;	108✔
316	return toReturn;	108✔
317	}
318
319	/// <summary>
320	/// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
321	/// </summary>
322	/// <returns></returns>
323	public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);	114✔
324
325	private IDataAccessPoint[] GetLoggingServers()
326	{
327	var catalogue = GetAllCatalogues().ToArray();	114✔
328
329	return !catalogue.Any()	114!
330	? throw new NotSupportedException(	114✔
331	$"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")	114✔
332	: (IDataAccessPoint[])catalogue.Select(static c => c.LiveLoggingServer).ToArray();	236✔
333	}
334
335	/// <summary>
336	/// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
337	/// </summary>
338	/// <returns></returns>
339	public string GetDistinctLoggingTask()
340	{
341	var catalogueMetadatas = GetAllCatalogues().ToArray();	88✔
342
343	if (catalogueMetadatas.Length == 0)	88!
344	throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");	×
345
346	var cataloguesWithoutLoggingTasks =	88✔
347	catalogueMetadatas.Where(static c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();	186✔
348
349	if (cataloguesWithoutLoggingTasks.Length != 0)	88✔
350	throw new Exception(	2✔
351	$"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", static (s, n) => $"{s}{n}(ID={n.ID}),")}");	4✔
352
353	var distinctLoggingTasks = catalogueMetadatas.Select(static c => c.LoggingDataTask).Distinct().ToArray();	180✔
354	return distinctLoggingTasks.Length >= 2	86!
355	? throw new Exception(	86✔
356	$"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")	86✔
357	: distinctLoggingTasks[0];	86✔
358	}
359
360	/// <summary>
361	/// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
362	/// </summary>
363	/// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
364	/// <returns></returns>
365	public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
366	{
367	return GetAllCatalogues().SelectMany(cat => cat.GetTableInfoList(includeLookups)).Distinct().OfType<TableInfo>()	16✔
368	.ToList();	8✔
369	}
370
371	/// <inheritdoc/>
372	public DiscoveredServer GetDistinctLiveDatabaseServer()
373	{
374	var normalTables = new HashSet<ITableInfo>();	230✔
375	var lookupTables = new HashSet<ITableInfo>();	230✔
376
377	foreach (var catalogue in GetAllCatalogues())	944✔
378	{
379	catalogue.GetTableInfos(out var normal, out var lookup);	242✔
380
381	foreach (var n in normal)	968✔
382	normalTables.Add(n);	242✔
383	foreach (var l in lookup)	484!
384	lookupTables.Add(l);	×
385	}
386
387	if (normalTables.Count != 0)	230!
388	return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);	230✔
389
NEW 390	return lookupTables.Count != 0	×
391	? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)	×
392	: throw new Exception(	×
393	$"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");	×
394	}
395
396	/// <inheritdoc/>
397	public IHasDependencies[] GetObjectsThisDependsOn() => null;	×
398
399	/// <inheritdoc/>
400	public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();	74✔
401
402	/// <summary>
403	/// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
404	/// </summary>
405	/// <param name="catalogue"></param>
406	public void EnsureLoggingWorksFor(ICatalogue catalogue)
407	{
408	//if there's no logging task / logging server set them up with the same name as the lmd
409	IExternalDatabaseServer loggingServer;
410
411	if (catalogue.LiveLoggingServer_ID == null)	14!
412	{
413	loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);	×
414
415	if (loggingServer != null)	×
416	catalogue.LiveLoggingServer_ID = loggingServer.ID;	×
417	else
418	throw new NotSupportedException(	×
419	"You do not yet have any logging servers configured so cannot create data loads");	×
420	}
421	else
422	{
423	loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);	14✔
424	}
425
426	//if there's no logging task yet and there's a logging server
427	if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))	14✔
428	{
429	var lm = new LogManager(loggingServer);	14✔
430	var loggingTaskName = Name;	14✔
431
432	lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);	14✔
433	catalogue.LoggingDataTask = loggingTaskName;	14✔
434	catalogue.SaveToDatabase();	14✔
435	}
436	}	14✔
437
438	/// <inheritdoc/>
439	public IQuerySyntaxHelper GetQuerySyntaxHelper()
440	{
441	var syntax = GetAllCatalogues().Select(static c => c.GetQuerySyntaxHelper()).Distinct().ToArray();	4✔
442	return syntax.Length > 1	2!
443	? throw new Exception(	2✔
444	$"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")	2✔
445	: syntax.SingleOrDefault();	2✔
446	}
447
448	/// <summary>
449	/// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
450	/// </summary>
451	/// <param name="runs"></param>
452	/// <returns></returns>
453	public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;	12✔
454
455	public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
456	{
457	return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,	90✔
458	loadMetadata).Any(static p => p.Value == "true");	90✔
459	}
460	}

HicServices / RDMP / 11978888584

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous