9757025998

Committed 02 Jul 2024 07:49AM UTC coverage: 56.677% (-0.2%) from 56.914%

Build # 9757025998

Build Type

Pull #1867

github

Committed by

JFriel

Commit Message

remove with values

Pull Request Pull Request #1867: Release/8.2.0

Run Details

10911 of 20750 branches covered (52.58%)

Branch coverage included in aggregate %.

369 of 831 new or added lines in 38 files covered. (44.4%)

375 existing lines in 25 files now uncovered.

30965 of 53135 relevant lines covered (58.28%)

7844.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.28

/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.IO;
using System.Linq;
using FAnsi.Discovery;
using FAnsi.Discovery.QuerySyntax;
using Rdmp.Core.Curation.Data.Cache;
using Rdmp.Core.Curation.Data.Defaults;
using Rdmp.Core.Curation.Data.ImportExport;
using Rdmp.Core.Curation.Data.Serialization;
using Rdmp.Core.Logging;
using Rdmp.Core.Logging.PastEvents;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Annotations;
using Rdmp.Core.ReusableLibraryCode.DataAccess;

namespace Rdmp.Core.Curation.Data.DataLoad;

/// <summary>
/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
/// </summary>
public enum CacheArchiveType
{
    /// <summary>
    /// Cached files are in a directory uncompressed
    /// </summary>
    None = 0,

    /// <summary>
    /// Cached files are contained in a zip file
    /// </summary>
    Zip = 1
}

/// <inheritdoc cref="ILoadMetadata"/>
public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
    ILoggedActivityRootObject, IHasFolder
{
    #region Database Properties

    private string _locationOfForLoadingDirectory;
    private string _locationOfForArchivingDirectory;
    private string _locationOfExecutablesDirectory;
    private string _locationOfCacheDirectory;
    private string _anonymisationEngineClass;
    private string _name;
    private string _description;
    private CacheArchiveType _cacheArchiveType;
    private int? _overrideRawServerID;
    private bool _ignoreTrigger;
    private string _folder;
    private DateTime? _lastLoadTime;


    public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");
    public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");
    public string DefaultExecutablesPath = "Executables";
    public string DefaultCachePath = Path.Combine("Data", "Cache");

    public DirectoryInfo GetRootDirectory()
    {
        if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))
        {
            var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");
            var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");
            var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");
            var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");
            if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)
            {
                return new DirectoryInfo(forLoadingRoot);
            }
        }
        return null;
    }

    ///  <inheritdoc/>
    public string LocationOfForLoadingDirectory
    {
        get => _locationOfForLoadingDirectory;
        set => SetField(ref _locationOfForLoadingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForArchivingDirectory
    {
        get => _locationOfForArchivingDirectory;
        set => SetField(ref _locationOfForArchivingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfExecutablesDirectory
    {
        get => _locationOfExecutablesDirectory;
        set => SetField(ref _locationOfExecutablesDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfCacheDirectory
    {
        get => _locationOfCacheDirectory;
        set => SetField(ref _locationOfCacheDirectory, value);
    }

    /// <summary>
    /// Not used
    /// </summary>
    public string AnonymisationEngineClass
    {
        get => _anonymisationEngineClass;
        set => SetField(ref _anonymisationEngineClass, value);
    }

    /// <inheritdoc/>
    [Unique]
    [NotNull]
    public string Name
    {
        get => _name;
        set => SetField(ref _name, value);
    }

    /// <summary>
    /// Human readable description of the load, what it does etc
    /// </summary>
    public string Description
    {
        get => _description;
        set => SetField(ref _description, value);
    }

    /// <summary>
    /// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>.  This may not be respected
    /// depending on the implementation of the sepecific ICacheLayout
    /// </summary>
    public CacheArchiveType CacheArchiveType
    {
        get => _cacheArchiveType;
        set => SetField(ref _cacheArchiveType, value);
    }

    /// <summary>
    /// Optional.  Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
    /// the system default - see <see cref="ServerDefaults"/>).
    /// </summary>
    public int? OverrideRAWServer_ID
    {
        get => _overrideRawServerID;
        set => SetField(ref _overrideRawServerID, value);
    }


    /// <iheritdoc/>
    public bool IgnoreTrigger
    {
        get => _ignoreTrigger;
        set => SetField(ref _ignoreTrigger, value);
    }

    /// <inheritdoc/>
    [UsefulProperty]
    public string Folder
    {
        get => _folder;
        set => SetField(ref _folder, FolderHelper.Adjust(value));
    }


    /// <summary>
    /// Stores the last time the load was ran.
    /// </summary>
    public DateTime? LastLoadTime
    {
        get => _lastLoadTime;
        set => SetField(ref _lastLoadTime, value);
    }

    #endregion


    #region Relationships

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue
        ? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)
        : null;

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public IOrderedEnumerable<IProcessTask> ProcessTasks
    {
        get
        {
            return
                Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);
        }
    }

    #endregion

    public LoadMetadata()
    {
    }

    /// <summary>
    /// Create a new DLE load.  This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
    /// 
    /// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="name"></param>
    public LoadMetadata(ICatalogueRepository repository, string name = null)
    {
        name ??= $"NewLoadMetadata{Guid.NewGuid()}";

        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "Name", name },
            { "IgnoreTrigger", false /*todo could be system global default here*/ },
            { "Folder", FolderHelper.Root },
            {"LastLoadTime", null }
        });
    }

    internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
    {
        LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();
        LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();
        LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();
        LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();
        Name = r["Name"] as string;
        AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();
        Name = r["Name"].ToString();
        Description = r["Description"] as string; //allows for nulls
        CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];
        OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);
        IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;
        Folder = r["Folder"] as string ?? FolderHelper.Root;
        LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());
    }

    internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()
    {
        shareManager.UpsertAndHydrate(this, shareDefinition);
    }

    public void LinkToCatalogue(ICatalogue catalogue)
    {
        var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);
        linkage.SaveToDatabase();
    }

    public void UnlinkFromCatalogue(ICatalogue catalogue)
    {
        foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))
        {
            l.DeleteInDatabase();
        }
    }

    /// <inheritdoc/>
    public override void DeleteInDatabase()
    {
        var firstOrDefault = GetAllCatalogues().FirstOrDefault();

        if (firstOrDefault != null)
            throw new Exception(
                $"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");

        base.DeleteInDatabase();
    }

    /// <inheritdoc/>
    public override string ToString() => Name;

    /// <inheritdoc/>
    public IEnumerable<ICatalogue> GetAllCatalogues()
    {
        var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);
        return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));
    }

    /// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
    public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
    {
        var loggingServers = GetLoggingServers();

        var loggingServer = loggingServers.FirstOrDefault();

        //get distinct connection
        var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);

        serverChosen = (IExternalDatabaseServer)loggingServer;
        return toReturn;
    }

    /// <summary>
    /// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
    /// </summary>
    /// <returns></returns>
    public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);

    private IDataAccessPoint[] GetLoggingServers()
    {
        var catalogue = GetAllCatalogues().ToArray();

        return !catalogue.Any()
            ? throw new NotSupportedException(
                $"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")
            : (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();
    }

    /// <summary>
    /// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
    /// </summary>
    /// <returns></returns>
    public string GetDistinctLoggingTask()
    {
        var catalogueMetadatas = GetAllCatalogues().ToArray();

        if (!catalogueMetadatas.Any())
            throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");

        var cataloguesWithoutLoggingTasks =
            catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();

        if (cataloguesWithoutLoggingTasks.Any())
            throw new Exception(
                $"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");

        var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();
        return distinctLoggingTasks.Length >= 2
            ? throw new Exception(
                $"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")
            : distinctLoggingTasks[0];
    }

    /// <summary>
    /// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
    /// </summary>
    /// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
    /// <returns></returns>
    public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
    {
        var toReturn = new List<TableInfo>();

        foreach (var catalogueMetadata in GetAllCatalogues())
            foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))
                if (!toReturn.Contains(tableInfo))
                    toReturn.Add(tableInfo);

        return toReturn;
    }

    /// <inheritdoc/>
    public DiscoveredServer GetDistinctLiveDatabaseServer()
    {
        var normalTables = new HashSet<ITableInfo>();
        var lookupTables = new HashSet<ITableInfo>();

        foreach (var catalogue in GetAllCatalogues())
        {
            catalogue.GetTableInfos(out var normal, out var lookup);

            foreach (var n in normal)
                normalTables.Add(n);
            foreach (var l in lookup)
                lookupTables.Add(l);
        }

        if (normalTables.Any())
            return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);

        return lookupTables.Any()
            ? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)
            : throw new Exception(
                $"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");
    }

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsThisDependsOn() => null;

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();

    /// <summary>
    /// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
    /// </summary>
    /// <param name="catalogue"></param>
    public void EnsureLoggingWorksFor(ICatalogue catalogue)
    {
        //if there's no logging task / logging server set them up with the same name as the lmd
        IExternalDatabaseServer loggingServer;

        if (catalogue.LiveLoggingServer_ID == null)
        {
            loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);

            if (loggingServer != null)
                catalogue.LiveLoggingServer_ID = loggingServer.ID;
            else
                throw new NotSupportedException(
                    "You do not yet have any logging servers configured so cannot create data loads");
        }
        else
        {
            loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);
        }

        //if there's no logging task yet and there's a logging server
        if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))
        {
            var lm = new LogManager(loggingServer);
            var loggingTaskName = Name;

            lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);
            catalogue.LoggingDataTask = loggingTaskName;
            catalogue.SaveToDatabase();
        }
    }

    /// <inheritdoc/>
    public IQuerySyntaxHelper GetQuerySyntaxHelper()
    {
        var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();
        return syntax.Length > 1
            ? throw new Exception(
                $"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")
            : syntax.SingleOrDefault();
    }

    /// <summary>
    /// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
    /// </summary>
    /// <param name="runs"></param>
    /// <returns></returns>
    public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;

    public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
    {
        return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,
            loadMetadata).Any(p => p.Value == "true");
    }
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.IO;
11	using System.Linq;
12	using FAnsi.Discovery;
13	using FAnsi.Discovery.QuerySyntax;
14	using Rdmp.Core.Curation.Data.Cache;
15	using Rdmp.Core.Curation.Data.Defaults;
16	using Rdmp.Core.Curation.Data.ImportExport;
17	using Rdmp.Core.Curation.Data.Serialization;
18	using Rdmp.Core.Logging;
19	using Rdmp.Core.Logging.PastEvents;
20	using Rdmp.Core.MapsDirectlyToDatabaseTable;
21	using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
22	using Rdmp.Core.Repositories;
23	using Rdmp.Core.ReusableLibraryCode;
24	using Rdmp.Core.ReusableLibraryCode.Annotations;
25	using Rdmp.Core.ReusableLibraryCode.DataAccess;
26
27	namespace Rdmp.Core.Curation.Data.DataLoad;
28
29	/// <summary>
30	/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
31	/// </summary>
32	public enum CacheArchiveType
33	{
34	/// <summary>
35	/// Cached files are in a directory uncompressed
36	/// </summary>
37	None = 0,
38
39	/// <summary>
40	/// Cached files are contained in a zip file
41	/// </summary>
42	Zip = 1
43	}
44
45	/// <inheritdoc cref="ILoadMetadata"/>
46	public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
47	ILoggedActivityRootObject, IHasFolder
48	{
49	#region Database Properties
50
51	private string _locationOfForLoadingDirectory;
52	private string _locationOfForArchivingDirectory;
53	private string _locationOfExecutablesDirectory;
54	private string _locationOfCacheDirectory;
55	private string _anonymisationEngineClass;
56	private string _name;
57	private string _description;
58	private CacheArchiveType _cacheArchiveType;
59	private int? _overrideRawServerID;
60	private bool _ignoreTrigger;
61	private string _folder;
62	private DateTime? _lastLoadTime;
63
64
65	public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");	902✔
66	public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");	902✔
67	public string DefaultExecutablesPath = "Executables";	902✔
68	public string DefaultCachePath = Path.Combine("Data", "Cache");	902✔
69
70	public DirectoryInfo GetRootDirectory()
71	{
NEW 72	if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))	×
73	{
NEW 74	var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");	×
NEW 75	var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");	×
NEW 76	var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");	×
NEW 77	var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");	×
NEW 78	if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)	×
79	{
NEW 80	return new DirectoryInfo(forLoadingRoot);	×
81	}
82	}
NEW 83	return null;	×
84	}
85
86	/// <inheritdoc/>
87	public string LocationOfForLoadingDirectory
88	{
89	get => _locationOfForLoadingDirectory;	1,732✔
90	set => SetField(ref _locationOfForLoadingDirectory, value);	862✔
91	}
92
93	/// <inheritdoc/>
94	public string LocationOfForArchivingDirectory
95	{
96	get => _locationOfForArchivingDirectory;	1,566✔
97	set => SetField(ref _locationOfForArchivingDirectory, value);	862✔
98	}
99
100	/// <inheritdoc/>
101	public string LocationOfExecutablesDirectory
102	{
103	get => _locationOfExecutablesDirectory;	1,566✔
104	set => SetField(ref _locationOfExecutablesDirectory, value);	862✔
105	}
106
107	/// <inheritdoc/>
108	public string LocationOfCacheDirectory
109	{
110	get => _locationOfCacheDirectory;	1,566✔
111	set => SetField(ref _locationOfCacheDirectory, value);	862✔
112	}
113
114	/// <summary>
115	/// Not used
116	/// </summary>
117	public string AnonymisationEngineClass
118	{
119	get => _anonymisationEngineClass;	714✔
120	set => SetField(ref _anonymisationEngineClass, value);	742✔
121	}
122
123	/// <inheritdoc/>
124	[Unique]
125	[NotNull]
126	public string Name
127	{
128	get => _name;	1,000✔
129	set => SetField(ref _name, value);	1,412✔
130	}
131
132	/// <summary>
133	/// Human readable description of the load, what it does etc
134	/// </summary>
135	public string Description
136	{
137	get => _description;	716✔
138	set => SetField(ref _description, value);	742✔
139	}
140
141	/// <summary>
142	/// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>. This may not be respected
143	/// depending on the implementation of the sepecific ICacheLayout
144	/// </summary>
145	public CacheArchiveType CacheArchiveType
146	{
147	get => _cacheArchiveType;	714✔
148	set => SetField(ref _cacheArchiveType, value);	752✔
149	}
150
151	/// <summary>
152	/// Optional. Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
153	/// the system default - see <see cref="ServerDefaults"/>).
154	/// </summary>
155	public int? OverrideRAWServer_ID
156	{
157	get => _overrideRawServerID;	1,160✔
158	set => SetField(ref _overrideRawServerID, value);	742✔
159	}
160
161
162	/// <iheritdoc/>
163	public bool IgnoreTrigger
164	{
165	get => _ignoreTrigger;	1,052✔
166	set => SetField(ref _ignoreTrigger, value);	886✔
167	}
168
169	/// <inheritdoc/>
170	[UsefulProperty]
171	public string Folder
172	{
173	get => _folder;	934✔
174	set => SetField(ref _folder, FolderHelper.Adjust(value));	876✔
175	}
176
177
178	/// <summary>
179	/// Stores the last time the load was ran.
180	/// </summary>
181	public DateTime? LastLoadTime
182	{
183	get => _lastLoadTime;	722✔
184	set => SetField(ref _lastLoadTime, value);	894✔
185	}
186
187	#endregion
188
189
190	#region Relationships
191
192	/// <inheritdoc/>
193	[NoMappingToDatabase]
194	public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue	332!
195	? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)	332✔
196	: null;	332✔
197
198	/// <inheritdoc/>
199	[NoMappingToDatabase]
200	public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);	200✔
201
202	/// <inheritdoc/>
203	[NoMappingToDatabase]
204	public IOrderedEnumerable<IProcessTask> ProcessTasks
205	{
206	get
207	{
208	return	276✔
209	Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);	570✔
210	}
211	}
212
213	#endregion
214
215	public LoadMetadata()	28✔
216	{
217	}	28✔
218
219	/// <summary>
220	/// Create a new DLE load. This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
221	///
222	/// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
223	/// </summary>
224	/// <param name="repository"></param>
225	/// <param name="name"></param>
226	public LoadMetadata(ICatalogueRepository repository, string name = null)	328✔
227	{
228	name ??= $"NewLoadMetadata{Guid.NewGuid()}";	328✔
229
230	repository.InsertAndHydrate(this, new Dictionary<string, object>	328✔
231	{	328✔
232	{ "Name", name },	328✔
233	{ "IgnoreTrigger", false /todo could be system global default here/ },	328✔
234	{ "Folder", FolderHelper.Root },	328✔
235	{"LastLoadTime", null }	328✔
236	});	328✔
237	}	328✔
238
239	internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
240	: base(repository, r)	536✔
241	{
242	LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();	536✔
243	LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();	536✔
244	LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();	536✔
245	LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();	536✔
246	Name = r["Name"] as string;	536✔
247	AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();	536✔
248	Name = r["Name"].ToString();	536✔
249	Description = r["Description"] as string; //allows for nulls	536✔
250	CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];	536✔
251	OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);	536✔
252	IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;	536✔
253	Folder = r["Folder"] as string ?? FolderHelper.Root;	536!
254	LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());	536!
255	}	536✔
256
257	internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()	10✔
258	{
259	shareManager.UpsertAndHydrate(this, shareDefinition);	10✔
260	}	10✔
261
262	public void LinkToCatalogue(ICatalogue catalogue)
263	{
264	var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);	278✔
265	linkage.SaveToDatabase();	278✔
266	}	278✔
267
268	public void UnlinkFromCatalogue(ICatalogue catalogue)
269	{
270	foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))	56✔
271	{
272	l.DeleteInDatabase();	8✔
273	}
274	}	8✔
275
276	/// <inheritdoc/>
277	public override void DeleteInDatabase()
278	{
279	var firstOrDefault = GetAllCatalogues().FirstOrDefault();	74✔
280
281	if (firstOrDefault != null)	74!
282	throw new Exception(	×
283	$"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");	×
284
285	base.DeleteInDatabase();	74✔
286	}	74✔
287
288	/// <inheritdoc/>
289	public override string ToString() => Name;	98✔
290
291	/// <inheritdoc/>
292	public IEnumerable<ICatalogue> GetAllCatalogues()
293	{
294	var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);	3,724✔
295	return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));	3,604✔
296	}
297
298	/// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
299	public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
300	{
301	var loggingServers = GetLoggingServers();	112✔
302
303	var loggingServer = loggingServers.FirstOrDefault();	110✔
304
305	//get distinct connection
306	var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);	110✔
307
308	serverChosen = (IExternalDatabaseServer)loggingServer;	106✔
309	return toReturn;	106✔
310	}
311
312	/// <summary>
313	/// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
314	/// </summary>
315	/// <returns></returns>
316	public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);	112✔
317
318	private IDataAccessPoint[] GetLoggingServers()
319	{
320	var catalogue = GetAllCatalogues().ToArray();	112✔
321
322	return !catalogue.Any()	112!
323	? throw new NotSupportedException(	112✔
324	$"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")	112✔
325	: (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();	232✔
326	}
327
328	/// <summary>
329	/// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
330	/// </summary>
331	/// <returns></returns>
332	public string GetDistinctLoggingTask()
333	{
334	var catalogueMetadatas = GetAllCatalogues().ToArray();	86✔
335
336	if (!catalogueMetadatas.Any())	86!
337	throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");	×
338
339	var cataloguesWithoutLoggingTasks =	86✔
340	catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();	182✔
341
342	if (cataloguesWithoutLoggingTasks.Any())	86✔
343	throw new Exception(	2✔
344	$"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");	4✔
345
346	var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();	176✔
347	return distinctLoggingTasks.Length >= 2	84!
348	? throw new Exception(	84✔
349	$"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")	84✔
350	: distinctLoggingTasks[0];	84✔
351	}
352
353	/// <summary>
354	/// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
355	/// </summary>
356	/// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
357	/// <returns></returns>
358	public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
359	{
360	var toReturn = new List<TableInfo>();	8✔
361
362	foreach (var catalogueMetadata in GetAllCatalogues())	32✔
363	foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))	32✔
364	if (!toReturn.Contains(tableInfo))	8✔
365	toReturn.Add(tableInfo);	8✔
366
367	return toReturn;	8✔
368	}
369
370	/// <inheritdoc/>
371	public DiscoveredServer GetDistinctLiveDatabaseServer()
372	{
373	var normalTables = new HashSet<ITableInfo>();	226✔
374	var lookupTables = new HashSet<ITableInfo>();	226✔
375
376	foreach (var catalogue in GetAllCatalogues())	928✔
377	{
378	catalogue.GetTableInfos(out var normal, out var lookup);	238✔
379
380	foreach (var n in normal)	952✔
381	normalTables.Add(n);	238✔
382	foreach (var l in lookup)	476!
383	lookupTables.Add(l);	×
384	}
385
386	if (normalTables.Any())	226!
387	return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);	226✔
388
389	return lookupTables.Any()	×
390	? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)	×
391	: throw new Exception(	×
392	$"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");	×
393	}
394
395	/// <inheritdoc/>
396	public IHasDependencies[] GetObjectsThisDependsOn() => null;	×
397
398	/// <inheritdoc/>
399	public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();	74✔
400
401	/// <summary>
402	/// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
403	/// </summary>
404	/// <param name="catalogue"></param>
405	public void EnsureLoggingWorksFor(ICatalogue catalogue)
406	{
407	//if there's no logging task / logging server set them up with the same name as the lmd
408	IExternalDatabaseServer loggingServer;
409
410	if (catalogue.LiveLoggingServer_ID == null)	14!
411	{
412	loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);	×
413
414	if (loggingServer != null)	×
415	catalogue.LiveLoggingServer_ID = loggingServer.ID;	×
416	else
417	throw new NotSupportedException(	×
418	"You do not yet have any logging servers configured so cannot create data loads");	×
419	}
420	else
421	{
422	loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);	14✔
423	}
424
425	//if there's no logging task yet and there's a logging server
426	if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))	14✔
427	{
428	var lm = new LogManager(loggingServer);	14✔
429	var loggingTaskName = Name;	14✔
430
431	lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);	14✔
432	catalogue.LoggingDataTask = loggingTaskName;	14✔
433	catalogue.SaveToDatabase();	14✔
434	}
435	}	14✔
436
437	/// <inheritdoc/>
438	public IQuerySyntaxHelper GetQuerySyntaxHelper()
439	{
440	var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();	4✔
441	return syntax.Length > 1	2!
442	? throw new Exception(	2✔
443	$"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")	2✔
444	: syntax.SingleOrDefault();	2✔
445	}
446
447	/// <summary>
448	/// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
449	/// </summary>
450	/// <param name="runs"></param>
451	/// <returns></returns>
452	public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;	12✔
453
454	public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
455	{
456	return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,	88✔
457	loadMetadata).Any(p => p.Value == "true");	88✔
458	}
459	}

HicServices / RDMP / 9757025998

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous