13139403805

Committed 04 Feb 2025 03:51PM UTC coverage: 57.447% (+0.01%) from 57.437%

Build # 13139403805

Build Type

Pull #2125

github

Committed by

JFriel

Commit Message

fiz clone

Pull Request Pull Request #2125: Task/rdmp 265 version data loads

Run Details

11322 of 21267 branches covered (53.24%)

Branch coverage included in aggregate %.

87 of 158 new or added lines in 8 files covered. (55.06%)

3 existing lines in 3 files now uncovered.

32222 of 54532 relevant lines covered (59.09%)

17087.34 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.02

/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.IO;
using System.Linq;
using FAnsi.Discovery;
using FAnsi.Discovery.QuerySyntax;
using MongoDB.Driver;
using Rdmp.Core.Curation.Data.Cache;
using Rdmp.Core.Curation.Data.Defaults;
using Rdmp.Core.Curation.Data.ImportExport;
using Rdmp.Core.Curation.Data.Serialization;
using Rdmp.Core.Logging;
using Rdmp.Core.Logging.PastEvents;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Annotations;
using Rdmp.Core.ReusableLibraryCode.DataAccess;

namespace Rdmp.Core.Curation.Data.DataLoad;

/// <summary>
/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
/// </summary>
public enum CacheArchiveType
{
    /// <summary>
    /// Cached files are in a directory uncompressed
    /// </summary>
    None = 0,

    /// <summary>
    /// Cached files are contained in a zip file
    /// </summary>
    Zip = 1
}

/// <inheritdoc cref="ILoadMetadata"/>
public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
    ILoggedActivityRootObject, IHasFolder, IVersionable
{
    #region Database Properties

    private string _locationOfForLoadingDirectory;
    private string _locationOfForArchivingDirectory;
    private string _locationOfExecutablesDirectory;
    private string _locationOfCacheDirectory;
    private string _anonymisationEngineClass;
    private string _name;
    private string _description;
    private CacheArchiveType _cacheArchiveType;
    private int? _overrideRawServerID;
    private bool _ignoreTrigger;
    private string _folder;
    private DateTime? _lastLoadTime;
    private bool _allowReservedPrefix;
    private int? _rootLoadMetadata_ID;

    public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");
    public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");
    public string DefaultExecutablesPath = "Executables";
    public string DefaultCachePath = Path.Combine("Data", "Cache");

    public DirectoryInfo GetRootDirectory()
    {
        if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))
        {
            var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");
            var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");
            var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");
            var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");
            if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)
            {
                return new DirectoryInfo(forLoadingRoot);
            }
        }
        return null;
    }


    ///  <inheritdoc/>
    public bool AllowReservedPrefix
    {
        get => _allowReservedPrefix;
        set => SetField(ref _allowReservedPrefix, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForLoadingDirectory
    {
        get => _locationOfForLoadingDirectory;
        set => SetField(ref _locationOfForLoadingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForArchivingDirectory
    {
        get => _locationOfForArchivingDirectory;
        set => SetField(ref _locationOfForArchivingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfExecutablesDirectory
    {
        get => _locationOfExecutablesDirectory;
        set => SetField(ref _locationOfExecutablesDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfCacheDirectory
    {
        get => _locationOfCacheDirectory;
        set => SetField(ref _locationOfCacheDirectory, value);
    }

    /// <summary>
    /// Not used
    /// </summary>
    public string AnonymisationEngineClass
    {
        get => _anonymisationEngineClass;
        set => SetField(ref _anonymisationEngineClass, value);
    }

    /// <inheritdoc/>
    [Unique]
    [NotNull]
    public string Name
    {
        get => _name;
        set => SetField(ref _name, value);
    }

    /// <summary>
    /// Human readable description of the load, what it does etc
    /// </summary>
    public string Description
    {
        get => _description;
        set => SetField(ref _description, value);
    }

    /// <summary>
    /// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>.  This may not be respected
    /// depending on the implementation of the sepecific ICacheLayout
    /// </summary>
    public CacheArchiveType CacheArchiveType
    {
        get => _cacheArchiveType;
        set => SetField(ref _cacheArchiveType, value);
    }

    /// <summary>
    /// Optional.  Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
    /// the system default - see <see cref="ServerDefaults"/>).
    /// </summary>
    public int? OverrideRAWServer_ID
    {
        get => _overrideRawServerID;
        set => SetField(ref _overrideRawServerID, value);
    }


    /// <iheritdoc/>
    public bool IgnoreTrigger
    {
        get => _ignoreTrigger;
        set => SetField(ref _ignoreTrigger, value);
    }

    /// <inheritdoc/>
    [UsefulProperty]
    public string Folder
    {
        get => _folder;
        set => SetField(ref _folder, FolderHelper.Adjust(value));
    }


    /// <summary>
    /// Stores the last time the load was ran.
    /// </summary>
    public DateTime? LastLoadTime
    {
        get => _lastLoadTime;
        set => SetField(ref _lastLoadTime, value);
    }

    /// <inheritdoc/>
    public int? RootLoadMetadata_ID { get => _rootLoadMetadata_ID; private set => SetField(ref _rootLoadMetadata_ID, value); }

    #endregion


    #region Relationships

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue
        ? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)
        : null;

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public IOrderedEnumerable<IProcessTask> ProcessTasks
    {
        get
        {
            return
                Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);
        }
    }

    #endregion

    public LoadMetadata()
    {
    }

    /// <summary>
    /// Create a new DLE load.  This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
    /// 
    /// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="name"></param>
    /// <param name="rootLoadMetadata"></param>
    public LoadMetadata(ICatalogueRepository repository, string name = null, LoadMetadata rootLoadMetadata = null)
    {
        name ??= $"NewLoadMetadata{Guid.NewGuid()}";
        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "Name", name },
            { "IgnoreTrigger", false /*todo could be system global default here*/ },
            { "Folder", FolderHelper.Root },
            {"LastLoadTime", null },
            {"RootLoadMetadata_ID", rootLoadMetadata is null? null: rootLoadMetadata.ID }
        });
    }

    internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
    {
        LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();
        LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();
        LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();
        LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();
        Name = r["Name"] as string;
        AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();
        Name = r["Name"].ToString();
        Description = r["Description"] as string; //allows for nulls
        CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];
        OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);
        IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;
        Folder = r["Folder"] as string ?? FolderHelper.Root;
        LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());
        AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;
        RootLoadMetadata_ID = ObjectToNullableInt(r["RootLoadMetadata_ID"]);
    }


    public LoadMetadata Clone()
    {
        var lmd = new LoadMetadata(CatalogueRepository, this.Name)
        {
            LocationOfForLoadingDirectory = LocationOfForLoadingDirectory,
            LocationOfForArchivingDirectory = LocationOfForArchivingDirectory,
            LocationOfCacheDirectory = LocationOfCacheDirectory,
            LocationOfExecutablesDirectory = LocationOfExecutablesDirectory,
            AnonymisationEngineClass = AnonymisationEngineClass,
            Name = Name,
            Description = Description,
            CacheArchiveType = CacheArchiveType,
            AllowReservedPrefix = false,
            LastLoadTime = LastLoadTime,
            OverrideRAWServer_ID = OverrideRAWServer_ID,
            IgnoreTrigger = IgnoreTrigger,
            Folder = Folder,
            //RootLoadMetadata_ID = this.ID
        };
        lmd.SaveToDatabase();
        //link to catalogue
        foreach (var catalogue in this.GetAllCatalogues())
        {
            lmd.LinkToCatalogue(catalogue);
        }
        //process task
        var pts = CatalogueRepository.GetAllObjectsWhere<ProcessTask>("LoadMetadata_ID", this.ID);
        foreach (ProcessTask pt in pts)
        {
            pt.Clone(lmd);
        }
        return lmd;
    }

    internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()
    {
        shareManager.UpsertAndHydrate(this, shareDefinition);
    }

    public void LinkToCatalogue(ICatalogue catalogue)
    {
        var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);
        linkage.SaveToDatabase();
    }

    public void UnlinkFromCatalogue(ICatalogue catalogue)
    {
        foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))
        {
            l.DeleteInDatabase();
        }
    }

    /// <inheritdoc/>
    public override void DeleteInDatabase()
    {
        var firstOrDefault = GetAllCatalogues().FirstOrDefault();

        if (firstOrDefault != null && this.RootLoadMetadata_ID == null)
            throw new Exception(
                $"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");

        var versions = Repository.GetAllObjectsWhere<LoadMetadata>("RootLoadMetadata_ID", ID);
        foreach (var version in versions)
        {
            version.DeleteInDatabase();
        }
        if (this.RootLoadMetadata_ID != null)
        {
            var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID);
            foreach (var link in catalogueLinkIDs)
            {
                link.DeleteInDatabase();
            }
        }


        base.DeleteInDatabase();
    }

    /// <inheritdoc/>
    public override string ToString() => Name;

    /// <inheritdoc/>
    public IEnumerable<ICatalogue> GetAllCatalogues()
    {
        var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);
        return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));
    }

    /// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
    public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
    {
        var loggingServers = GetLoggingServers();

        var loggingServer = loggingServers.FirstOrDefault();

        //get distinct connection
        var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);

        serverChosen = (IExternalDatabaseServer)loggingServer;
        return toReturn;
    }

    /// <summary>
    /// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
    /// </summary>
    /// <returns></returns>
    public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);

    private IDataAccessPoint[] GetLoggingServers()
    {
        var catalogue = GetAllCatalogues().ToArray();

        return !catalogue.Any()
            ? throw new NotSupportedException(
                $"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")
            : (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();
    }

    /// <summary>
    /// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
    /// </summary>
    /// <returns></returns>
    public string GetDistinctLoggingTask()
    {
        var catalogueMetadatas = GetAllCatalogues().ToArray();

        if (!catalogueMetadatas.Any())
            throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");

        var cataloguesWithoutLoggingTasks =
            catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();

        if (cataloguesWithoutLoggingTasks.Any())
            throw new Exception(
                $"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");

        var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();
        return distinctLoggingTasks.Length >= 2
            ? throw new Exception(
                $"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")
            : distinctLoggingTasks[0];
    }

    /// <summary>
    /// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
    /// </summary>
    /// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
    /// <returns></returns>
    public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
    {
        var toReturn = new List<TableInfo>();

        foreach (var catalogueMetadata in GetAllCatalogues())
            foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))
                if (!toReturn.Contains(tableInfo))
                    toReturn.Add(tableInfo);

        return toReturn;
    }

    /// <inheritdoc/>
    public DiscoveredServer GetDistinctLiveDatabaseServer()
    {
        var normalTables = new HashSet<ITableInfo>();
        var lookupTables = new HashSet<ITableInfo>();

        foreach (var catalogue in GetAllCatalogues())
        {
            catalogue.GetTableInfos(out var normal, out var lookup);

            foreach (var n in normal)
                normalTables.Add(n);
            foreach (var l in lookup)
                lookupTables.Add(l);
        }

        if (normalTables.Any())
            return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);

        return lookupTables.Any()
            ? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)
            : throw new Exception(
                $"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");
    }

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsThisDependsOn() => null;

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();

    /// <summary>
    /// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
    /// </summary>
    /// <param name="catalogue"></param>
    public void EnsureLoggingWorksFor(ICatalogue catalogue)
    {
        //if there's no logging task / logging server set them up with the same name as the lmd
        IExternalDatabaseServer loggingServer;

        if (catalogue.LiveLoggingServer_ID == null)
        {
            loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);

            if (loggingServer != null)
                catalogue.LiveLoggingServer_ID = loggingServer.ID;
            else
                throw new NotSupportedException(
                    "You do not yet have any logging servers configured so cannot create data loads");
        }
        else
        {
            loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);
        }

        //if there's no logging task yet and there's a logging server
        if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))
        {
            var lm = new LogManager(loggingServer);
            var loggingTaskName = Name;

            lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);
            catalogue.LoggingDataTask = loggingTaskName;
            catalogue.SaveToDatabase();
        }
    }

    /// <inheritdoc/>
    public IQuerySyntaxHelper GetQuerySyntaxHelper()
    {
        var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();
        return syntax.Length > 1
            ? throw new Exception(
                $"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")
            : syntax.SingleOrDefault();
    }

    /// <summary>
    /// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
    /// </summary>
    /// <param name="runs"></param>
    /// <returns></returns>
    public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;

    public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
    {
        return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,
            loadMetadata).Any(p => p.Value == "true");
    }


    /// <inheritdoc/>
    public DatabaseEntity SaveNewVersion()
    {
        var lmd = this.Clone();
        lmd.RootLoadMetadata_ID = this.RootLoadMetadata_ID != null ? this.RootLoadMetadata_ID : this.ID;
        lmd.Name = $"{this.Name} - {DateTime.Now}";
        lmd.SaveToDatabase();
        return lmd;
    }
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.IO;
11	using System.Linq;
12	using FAnsi.Discovery;
13	using FAnsi.Discovery.QuerySyntax;
14	using MongoDB.Driver;
15	using Rdmp.Core.Curation.Data.Cache;
16	using Rdmp.Core.Curation.Data.Defaults;
17	using Rdmp.Core.Curation.Data.ImportExport;
18	using Rdmp.Core.Curation.Data.Serialization;
19	using Rdmp.Core.Logging;
20	using Rdmp.Core.Logging.PastEvents;
21	using Rdmp.Core.MapsDirectlyToDatabaseTable;
22	using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
23	using Rdmp.Core.Repositories;
24	using Rdmp.Core.ReusableLibraryCode;
25	using Rdmp.Core.ReusableLibraryCode.Annotations;
26	using Rdmp.Core.ReusableLibraryCode.DataAccess;
27
28	namespace Rdmp.Core.Curation.Data.DataLoad;
29
30	/// <summary>
31	/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
32	/// </summary>
33	public enum CacheArchiveType
34	{
35	/// <summary>
36	/// Cached files are in a directory uncompressed
37	/// </summary>
38	None = 0,
39
40	/// <summary>
41	/// Cached files are contained in a zip file
42	/// </summary>
43	Zip = 1
44	}
45
46	/// <inheritdoc cref="ILoadMetadata"/>
47	public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
48	ILoggedActivityRootObject, IHasFolder, IVersionable
49	{
50	#region Database Properties
51
52	private string _locationOfForLoadingDirectory;
53	private string _locationOfForArchivingDirectory;
54	private string _locationOfExecutablesDirectory;
55	private string _locationOfCacheDirectory;
56	private string _anonymisationEngineClass;
57	private string _name;
58	private string _description;
59	private CacheArchiveType _cacheArchiveType;
60	private int? _overrideRawServerID;
61	private bool _ignoreTrigger;
62	private string _folder;
63	private DateTime? _lastLoadTime;
64	private bool _allowReservedPrefix;
65	private int? _rootLoadMetadata_ID;
66
67	public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");	1,130✔
68	public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");	1,130✔
69	public string DefaultExecutablesPath = "Executables";	1,130✔
70	public string DefaultCachePath = Path.Combine("Data", "Cache");	1,130✔
71
72	public DirectoryInfo GetRootDirectory()
73	{
74	if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))	×
75	{
76	var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");	×
77	var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");	×
78	var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");	×
79	var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");	×
80	if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)	×
81	{
82	return new DirectoryInfo(forLoadingRoot);	×
83	}
84	}
85	return null;	×
86	}
87
88
89	/// <inheritdoc/>
90	public bool AllowReservedPrefix
91	{
92	get => _allowReservedPrefix;	930✔
93	set => SetField(ref _allowReservedPrefix, value);	984✔
94	}
95
96	/// <inheritdoc/>
97	public string LocationOfForLoadingDirectory
98	{
99	get => _locationOfForLoadingDirectory;	2,204✔
100	set => SetField(ref _locationOfForLoadingDirectory, value);	1,112✔
101	}
102
103	/// <inheritdoc/>
104	public string LocationOfForArchivingDirectory
105	{
106	get => _locationOfForArchivingDirectory;	1,952✔
107	set => SetField(ref _locationOfForArchivingDirectory, value);	1,112✔
108	}
109
110	/// <inheritdoc/>
111	public string LocationOfExecutablesDirectory
112	{
113	get => _locationOfExecutablesDirectory;	1,952✔
114	set => SetField(ref _locationOfExecutablesDirectory, value);	1,112✔
115	}
116
117	/// <inheritdoc/>
118	public string LocationOfCacheDirectory
119	{
120	get => _locationOfCacheDirectory;	1,952✔
121	set => SetField(ref _locationOfCacheDirectory, value);	1,112✔
122	}
123
124	/// <summary>
125	/// Not used
126	/// </summary>
127	public string AnonymisationEngineClass
128	{
129	get => _anonymisationEngineClass;	880✔
130	set => SetField(ref _anonymisationEngineClass, value);	974✔
131	}
132
133	/// <inheritdoc/>
134	[Unique]
135	[NotNull]
136	public string Name
137	{
138	get => _name;	1,222✔
139	set => SetField(ref _name, value);	1,844✔
140	}
141
142	/// <summary>
143	/// Human readable description of the load, what it does etc
144	/// </summary>
145	public string Description
146	{
147	get => _description;	894✔
148	set => SetField(ref _description, value);	980✔
149	}
150
151	/// <summary>
152	/// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>. This may not be respected
153	/// depending on the implementation of the sepecific ICacheLayout
154	/// </summary>
155	public CacheArchiveType CacheArchiveType
156	{
157	get => _cacheArchiveType;	880✔
158	set => SetField(ref _cacheArchiveType, value);	984✔
159	}
160
161	/// <summary>
162	/// Optional. Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
163	/// the system default - see <see cref="ServerDefaults"/>).
164	/// </summary>
165	public int? OverrideRAWServer_ID
166	{
167	get => _overrideRawServerID;	1,776✔
168	set => SetField(ref _overrideRawServerID, value);	974✔
169	}
170
171
172	/// <iheritdoc/>
173	public bool IgnoreTrigger
174	{
175	get => _ignoreTrigger;	1,272✔
176	set => SetField(ref _ignoreTrigger, value);	1,132✔
177	}
178
179	/// <inheritdoc/>
180	[UsefulProperty]
181	public string Folder
182	{
183	get => _folder;	1,532✔
184	set => SetField(ref _folder, FolderHelper.Adjust(value));	1,108✔
185	}
186
187
188	/// <summary>
189	/// Stores the last time the load was ran.
190	/// </summary>
191	public DateTime? LastLoadTime
192	{
193	get => _lastLoadTime;	888✔
194	set => SetField(ref _lastLoadTime, value);	1,126✔
195	}
196
197	/// <inheritdoc/>
198	public int? RootLoadMetadata_ID { get => _rootLoadMetadata_ID; private set => SetField(ref _rootLoadMetadata_ID, value); }	8,922✔
199
200	#endregion
201
202
203	#region Relationships
204
205	/// <inheritdoc/>
206	[NoMappingToDatabase]
207	public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue	350!
208	? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)	350✔
209	: null;	350✔
210
211	/// <inheritdoc/>
212	[NoMappingToDatabase]
213	public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);	202✔
214
215	/// <inheritdoc/>
216	[NoMappingToDatabase]
217	public IOrderedEnumerable<IProcessTask> ProcessTasks
218	{
219	get
220	{
221	return	320✔
222	Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);	620✔
223	}
224	}
225
226	#endregion
227
228	public LoadMetadata()	30✔
229	{
230	}	30✔
231
232	/// <summary>
233	/// Create a new DLE load. This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
234	///
235	/// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
236	/// </summary>
237	/// <param name="repository"></param>
238	/// <param name="name"></param>
239	/// <param name="rootLoadMetadata"></param>
240	public LoadMetadata(ICatalogueRepository repository, string name = null, LoadMetadata rootLoadMetadata = null)	358✔
241	{
242	name ??= $"NewLoadMetadata{Guid.NewGuid()}";	358✔
243	repository.InsertAndHydrate(this, new Dictionary<string, object>	358!
244	{	358✔
245	{ "Name", name },	358✔
246	{ "IgnoreTrigger", false /todo could be system global default here/ },	358✔
247	{ "Folder", FolderHelper.Root },	358✔
248	{"LastLoadTime", null },	358✔
249	{"RootLoadMetadata_ID", rootLoadMetadata is null? null: rootLoadMetadata.ID }	358✔
250	});	358✔
251	}	358✔
252
253	internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
254	: base(repository, r)	732✔
255	{
256	LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();	732✔
257	LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();	732✔
258	LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();	732✔
259	LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();	732✔
260	Name = r["Name"] as string;	732✔
261	AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();	732✔
262	Name = r["Name"].ToString();	732✔
263	Description = r["Description"] as string; //allows for nulls	732✔
264	CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];	732✔
265	OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);	732✔
266	IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;	732✔
267	Folder = r["Folder"] as string ?? FolderHelper.Root;	732!
268	LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());	732✔
269	AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;	732✔
270	RootLoadMetadata_ID = ObjectToNullableInt(r["RootLoadMetadata_ID"]);	732✔
271	}	732✔
272
273
274	public LoadMetadata Clone()
275	{
276	var lmd = new LoadMetadata(CatalogueRepository, this.Name)	6✔
277	{	6✔
278	LocationOfForLoadingDirectory = LocationOfForLoadingDirectory,	6✔
279	LocationOfForArchivingDirectory = LocationOfForArchivingDirectory,	6✔
280	LocationOfCacheDirectory = LocationOfCacheDirectory,	6✔
281	LocationOfExecutablesDirectory = LocationOfExecutablesDirectory,	6✔
282	AnonymisationEngineClass = AnonymisationEngineClass,	6✔
283	Name = Name,	6✔
284	Description = Description,	6✔
285	CacheArchiveType = CacheArchiveType,	6✔
286	AllowReservedPrefix = false,	6✔
287	LastLoadTime = LastLoadTime,	6✔
288	OverrideRAWServer_ID = OverrideRAWServer_ID,	6✔
289	IgnoreTrigger = IgnoreTrigger,	6✔
290	Folder = Folder,	6✔
291	//RootLoadMetadata_ID = this.ID	6✔
292	};	6✔
293	lmd.SaveToDatabase();	6✔
294	//link to catalogue
295	foreach (var catalogue in this.GetAllCatalogues())	24✔
296	{
297	lmd.LinkToCatalogue(catalogue);	6✔
298	}
299	//process task
300	var pts = CatalogueRepository.GetAllObjectsWhere<ProcessTask>("LoadMetadata_ID", this.ID);	6✔
301	foreach (ProcessTask pt in pts)	24✔
302	{
303	pt.Clone(lmd);	6✔
304	}
305	return lmd;	6✔
306	}
307
308	internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()	10✔
309	{
310	shareManager.UpsertAndHydrate(this, shareDefinition);	10✔
311	}	10✔
312
313	public void LinkToCatalogue(ICatalogue catalogue)
314	{
315	var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);	308✔
316	linkage.SaveToDatabase();	308✔
317	}	308✔
318
319	public void UnlinkFromCatalogue(ICatalogue catalogue)
320	{
321	foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == this.ID))	56✔
322	{
323	l.DeleteInDatabase();	8✔
324	}
325	}	8✔
326
327	/// <inheritdoc/>
328	public override void DeleteInDatabase()
329	{
330	var firstOrDefault = GetAllCatalogues().FirstOrDefault();	74✔
331
332	if (firstOrDefault != null && this.RootLoadMetadata_ID == null)	74!
333	throw new Exception(	×
334	$"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");	×
335
336	var versions = Repository.GetAllObjectsWhere<LoadMetadata>("RootLoadMetadata_ID", ID);	74✔
337	foreach (var version in versions)	148!
338	{
NEW 339	version.DeleteInDatabase();	×
340	}
341	if (this.RootLoadMetadata_ID != null)	74!
342	{
NEW 343	var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID);	×
NEW 344	foreach (var link in catalogueLinkIDs)	×
345	{
NEW 346	link.DeleteInDatabase();	×
347	}
348	}
349
350
351	base.DeleteInDatabase();	74✔
352	}	74✔
353
354	/// <inheritdoc/>
355	public override string ToString() => Name;	98✔
356
357	/// <inheritdoc/>
358	public IEnumerable<ICatalogue> GetAllCatalogues()
359	{
360	var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);	3,978✔
361	return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));	3,858✔
362	}
363
364	/// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
365	public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
366	{
367	var loggingServers = GetLoggingServers();	114✔
368
369	var loggingServer = loggingServers.FirstOrDefault();	112✔
370
371	//get distinct connection
372	var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);	112✔
373
374	serverChosen = (IExternalDatabaseServer)loggingServer;	108✔
375	return toReturn;	108✔
376	}
377
378	/// <summary>
379	/// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
380	/// </summary>
381	/// <returns></returns>
382	public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);	114✔
383
384	private IDataAccessPoint[] GetLoggingServers()
385	{
386	var catalogue = GetAllCatalogues().ToArray();	114✔
387
388	return !catalogue.Any()	114!
389	? throw new NotSupportedException(	114✔
390	$"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")	114✔
391	: (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();	236✔
392	}
393
394	/// <summary>
395	/// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
396	/// </summary>
397	/// <returns></returns>
398	public string GetDistinctLoggingTask()
399	{
400	var catalogueMetadatas = GetAllCatalogues().ToArray();	88✔
401
402	if (!catalogueMetadatas.Any())	88!
403	throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");	×
404
405	var cataloguesWithoutLoggingTasks =	88✔
406	catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();	186✔
407
408	if (cataloguesWithoutLoggingTasks.Any())	88✔
409	throw new Exception(	2✔
410	$"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");	4✔
411
412	var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();	180✔
413	return distinctLoggingTasks.Length >= 2	86!
414	? throw new Exception(	86✔
415	$"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")	86✔
416	: distinctLoggingTasks[0];	86✔
417	}
418
419	/// <summary>
420	/// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
421	/// </summary>
422	/// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
423	/// <returns></returns>
424	public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
425	{
426	var toReturn = new List<TableInfo>();	8✔
427
428	foreach (var catalogueMetadata in GetAllCatalogues())	32✔
429	foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))	32✔
430	if (!toReturn.Contains(tableInfo))	8✔
431	toReturn.Add(tableInfo);	8✔
432
433	return toReturn;	8✔
434	}
435
436	/// <inheritdoc/>
437	public DiscoveredServer GetDistinctLiveDatabaseServer()
438	{
439	var normalTables = new HashSet<ITableInfo>();	244✔
440	var lookupTables = new HashSet<ITableInfo>();	244✔
441
442	foreach (var catalogue in GetAllCatalogues())	1,000✔
443	{
444	catalogue.GetTableInfos(out var normal, out var lookup);	256✔
445
446	foreach (var n in normal)	1,024✔
447	normalTables.Add(n);	256✔
448	foreach (var l in lookup)	512!
449	lookupTables.Add(l);	×
450	}
451
452	if (normalTables.Any())	244!
453	return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);	244✔
454
455	return lookupTables.Any()	×
456	? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)	×
457	: throw new Exception(	×
458	$"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");	×
459	}
460
461	/// <inheritdoc/>
462	public IHasDependencies[] GetObjectsThisDependsOn() => null;	×
463
464	/// <inheritdoc/>
465	public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();	74✔
466
467	/// <summary>
468	/// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
469	/// </summary>
470	/// <param name="catalogue"></param>
471	public void EnsureLoggingWorksFor(ICatalogue catalogue)
472	{
473	//if there's no logging task / logging server set them up with the same name as the lmd
474	IExternalDatabaseServer loggingServer;
475
476	if (catalogue.LiveLoggingServer_ID == null)	14!
477	{
478	loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);	×
479
480	if (loggingServer != null)	×
481	catalogue.LiveLoggingServer_ID = loggingServer.ID;	×
482	else
483	throw new NotSupportedException(	×
484	"You do not yet have any logging servers configured so cannot create data loads");	×
485	}
486	else
487	{
488	loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);	14✔
489	}
490
491	//if there's no logging task yet and there's a logging server
492	if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))	14✔
493	{
494	var lm = new LogManager(loggingServer);	14✔
495	var loggingTaskName = Name;	14✔
496
497	lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);	14✔
498	catalogue.LoggingDataTask = loggingTaskName;	14✔
499	catalogue.SaveToDatabase();	14✔
500	}
501	}	14✔
502
503	/// <inheritdoc/>
504	public IQuerySyntaxHelper GetQuerySyntaxHelper()
505	{
506	var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();	4✔
507	return syntax.Length > 1	2!
508	? throw new Exception(	2✔
509	$"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")	2✔
510	: syntax.SingleOrDefault();	2✔
511	}
512
513	/// <summary>
514	/// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
515	/// </summary>
516	/// <param name="runs"></param>
517	/// <returns></returns>
518	public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;	12✔
519
520	public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
521	{
522	return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,	90✔
523	loadMetadata).Any(p => p.Value == "true");	90✔
524	}
525
526
527	/// <inheritdoc/>
528	public DatabaseEntity SaveNewVersion()
529	{
530	var lmd = this.Clone();	4✔
531	lmd.RootLoadMetadata_ID = this.RootLoadMetadata_ID != null ? this.RootLoadMetadata_ID : this.ID;	4!
532	lmd.Name = $"{this.Name} - {DateTime.Now}";	4✔
533	lmd.SaveToDatabase();	4✔
534	return lmd;	4✔
535	}
536	}

HicServices / RDMP / 13139403805

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous