13493861722

Committed 24 Feb 2025 08:35AM UTC coverage: 57.414% (-0.03%) from 57.446%

Build # 13493861722

Build Type

push

github

Committed by

web-flow

Commit Message

marge v8.4.3 into main (#2145)

* simple db update

* add changelog

* revert csproj

* Fix up some codeql/inspection code issues (#2087)

* Update OverviewModel.cs

Fix up some .Dispose/using issues, make finding most recent load ID more efficient

* LINQ tidying

* CodeQL fixups

* Update Catalogue.cs

Add Hashcode, Equals methods.

* Update Catalogue.cs

Tweak equality semantics for RDMP DB oddities

* Bump actions/setup-dotnet from 4.1.0 to 4.2.0

Bumps [actions/setup-dotnet](https://github.com/actions/setup-dotnet) from 4.1.0 to 4.2.0.
- [Release notes](https://github.com/actions/setup-dotnet/releases)
- [Commits](https://github.com/actions/setup-dotnet/compare/v4.1.0...v4.2.0)

---
updated-dependencies:
- dependency-name: actions/setup-dotnet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NUnit.Analyzers from 4.4.0 to 4.5.0

Bumps [NUnit.Analyzers](https://github.com/nunit/nunit.analyzers) from 4.4.0 to 4.5.0.
- [Release notes](https://github.com/nunit/nunit.analyzers/releases)
- [Changelog](https://github.com/nunit/nunit.analyzers/blob/master/CHANGES.md)
- [Commits](https://github.com/nunit/nunit.analyzers/compare/4.4.0...4.5.0)

---
updated-dependencies:
- dependency-name: NUnit.Analyzers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Minio from 6.0.3 to 6.0.4

Bumps [Minio](https://github.com/minio/minio-dotnet) from 6.0.3 to 6.0.4.
- [Release notes](https://github.com/minio/minio-dotnet/releases)
- [Commits](https://github.com/minio/minio-dotnet/compare/6.0.3...6.0.4)

---
updated-dependencies:
- dependency-name: Minio
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump the aws-sdk group across 1 directory with 4 updates

Bumps the aws-sdk grou... (continued)

Run Details

11358 of 21338 branches covered (53.23%)

Branch coverage included in aggregate %.

1116 of 1482 new or added lines in 50 files covered. (75.3%)

19 existing lines in 10 files now uncovered.

32283 of 54673 relevant lines covered (59.05%)

17450.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.94

/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs

// Copyright (c) The University of Dundee 2018-2024
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.IO;
using System.Linq;
using FAnsi.Discovery;
using FAnsi.Discovery.QuerySyntax;
using Rdmp.Core.Curation.Data.Cache;
using Rdmp.Core.Curation.Data.Defaults;
using Rdmp.Core.Curation.Data.ImportExport;
using Rdmp.Core.Curation.Data.Serialization;
using Rdmp.Core.Logging;
using Rdmp.Core.Logging.PastEvents;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode;
using Rdmp.Core.ReusableLibraryCode.Annotations;
using Rdmp.Core.ReusableLibraryCode.DataAccess;

namespace Rdmp.Core.Curation.Data.DataLoad;

/// <summary>
/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
/// </summary>
public enum CacheArchiveType
{
    /// <summary>
    /// Cached files are in a directory uncompressed
    /// </summary>
    None = 0,

    /// <summary>
    /// Cached files are contained in a zip file
    /// </summary>
    Zip = 1
}

/// <inheritdoc cref="ILoadMetadata"/>
public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
    ILoggedActivityRootObject, IHasFolder, IVersionable
{
    #region Database Properties

    private string _locationOfForLoadingDirectory;
    private string _locationOfForArchivingDirectory;
    private string _locationOfExecutablesDirectory;
    private string _locationOfCacheDirectory;
    private string _anonymisationEngineClass;
    private string _name;
    private string _description;
    private CacheArchiveType _cacheArchiveType;
    private int? _overrideRawServerID;
    private bool _ignoreTrigger;
    private string _folder;
    private DateTime? _lastLoadTime;
    private bool _allowReservedPrefix;
    private int? _rootLoadMetadata_ID;

    public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");
    public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");
    public string DefaultExecutablesPath = "Executables";
    public string DefaultCachePath = Path.Combine("Data", "Cache");

    public DirectoryInfo GetRootDirectory()
    {
        if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))
        {
            var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");
            var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");
            var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");
            var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");
            if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)
            {
                return new DirectoryInfo(forLoadingRoot);
            }
        }
        return null;
    }


    ///  <inheritdoc/>
    public bool AllowReservedPrefix
    {
        get => _allowReservedPrefix;
        set => SetField(ref _allowReservedPrefix, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForLoadingDirectory
    {
        get => _locationOfForLoadingDirectory;
        set => SetField(ref _locationOfForLoadingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfForArchivingDirectory
    {
        get => _locationOfForArchivingDirectory;
        set => SetField(ref _locationOfForArchivingDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfExecutablesDirectory
    {
        get => _locationOfExecutablesDirectory;
        set => SetField(ref _locationOfExecutablesDirectory, value);
    }

    ///  <inheritdoc/>
    public string LocationOfCacheDirectory
    {
        get => _locationOfCacheDirectory;
        set => SetField(ref _locationOfCacheDirectory, value);
    }

    /// <summary>
    /// Not used
    /// </summary>
    public string AnonymisationEngineClass
    {
        get => _anonymisationEngineClass;
        set => SetField(ref _anonymisationEngineClass, value);
    }

    /// <inheritdoc/>
    [Unique]
    [NotNull]
    public string Name
    {
        get => _name;
        set => SetField(ref _name, value);
    }

    /// <summary>
    /// Human readable description of the load, what it does etc
    /// </summary>
    public string Description
    {
        get => _description;
        set => SetField(ref _description, value);
    }

    /// <summary>
    /// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>.  This may not be respected
    /// depending on the implementation of the specific ICacheLayout
    /// </summary>
    public CacheArchiveType CacheArchiveType
    {
        get => _cacheArchiveType;
        set => SetField(ref _cacheArchiveType, value);
    }

    /// <summary>
    /// Optional.  Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
    /// the system default - see <see cref="ServerDefaults"/>).
    /// </summary>
    public int? OverrideRAWServer_ID
    {
        get => _overrideRawServerID;
        set => SetField(ref _overrideRawServerID, value);
    }


    /// <iheritdoc/>
    public bool IgnoreTrigger
    {
        get => _ignoreTrigger;
        set => SetField(ref _ignoreTrigger, value);
    }

    /// <inheritdoc/>
    [UsefulProperty]
    public string Folder
    {
        get => _folder;
        set => SetField(ref _folder, FolderHelper.Adjust(value));
    }


    /// <summary>
    /// Stores the last time the load was ran.
    /// </summary>
    public DateTime? LastLoadTime
    {
        get => _lastLoadTime;
        set => SetField(ref _lastLoadTime, value);
    }

    /// <inheritdoc/>
    public int? RootLoadMetadata_ID { get => _rootLoadMetadata_ID; private set => SetField(ref _rootLoadMetadata_ID, value); }

    #endregion


    #region Relationships

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue
        ? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)
        : null;

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public IOrderedEnumerable<IProcessTask> ProcessTasks
    {
        get
        {
            return
                Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);
        }
    }

    #endregion

    public LoadMetadata()
    {
    }

    /// <summary>
    /// Create a new DLE load.  This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
    /// 
    /// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="name"></param>
    /// <param name="rootLoadMetadata"></param>
    public LoadMetadata(ICatalogueRepository repository, string name = null, LoadMetadata rootLoadMetadata = null)
    {
        name ??= $"NewLoadMetadata{Guid.NewGuid()}";
        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "Name", name },
            { "IgnoreTrigger", false /*todo could be system global default here*/ },
            { "Folder", FolderHelper.Root },
            {"LastLoadTime", null },
            {"RootLoadMetadata_ID", rootLoadMetadata is null? null: rootLoadMetadata.ID }
        });
    }

    internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
    {
        LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();
        LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();
        LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();
        LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();
        Name = r["Name"] as string;
        AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();
        Name = r["Name"].ToString();
        Description = r["Description"] as string; //allows for nulls
        CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];
        OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);
        IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;
        Folder = r["Folder"] as string ?? FolderHelper.Root;
        LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());
        AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;
        RootLoadMetadata_ID = ObjectToNullableInt(r["RootLoadMetadata_ID"]);
    }


    public LoadMetadata Clone()
    {
        var lmd = new LoadMetadata(CatalogueRepository, Name)
        {
            LocationOfForLoadingDirectory = LocationOfForLoadingDirectory,
            LocationOfForArchivingDirectory = LocationOfForArchivingDirectory,
            LocationOfCacheDirectory = LocationOfCacheDirectory,
            LocationOfExecutablesDirectory = LocationOfExecutablesDirectory,
            AnonymisationEngineClass = AnonymisationEngineClass,
            Name = Name,
            Description = Description,
            CacheArchiveType = CacheArchiveType,
            AllowReservedPrefix = false,
            LastLoadTime = LastLoadTime,
            OverrideRAWServer_ID = OverrideRAWServer_ID,
            IgnoreTrigger = IgnoreTrigger,
            Folder = Folder,
        };
        lmd.SaveToDatabase();
        //link to catalogue
        foreach (var catalogue in GetAllCatalogues())
        {
            lmd.LinkToCatalogue(catalogue);
        }
        //process task
        var pts = CatalogueRepository.GetAllObjectsWhere<ProcessTask>("LoadMetadata_ID", ID);
        foreach (ProcessTask pt in pts)
        {
            pt.Clone(lmd);
        }
        return lmd;
    }

    internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()
    {
        shareManager.UpsertAndHydrate(this, shareDefinition);
    }

    public void LinkToCatalogue(ICatalogue catalogue)
    {
        var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);
        linkage.SaveToDatabase();
    }

    public void UnlinkFromCatalogue(ICatalogue catalogue)
    {
        foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == ID))
        {
            l.DeleteInDatabase();
        }
    }

    /// <inheritdoc/>
    public override void DeleteInDatabase()
    {
        var firstOrDefault = GetAllCatalogues().FirstOrDefault();

        if (firstOrDefault != null && RootLoadMetadata_ID == null)
            throw new Exception(
                $"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");

        var versions = Repository.GetAllObjectsWhere<LoadMetadata>("RootLoadMetadata_ID", ID);
        foreach (var version in versions)
        {
            version.DeleteInDatabase();
        }
        if (RootLoadMetadata_ID != null)
        {
            var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID);
            foreach (var link in catalogueLinkIDs)
            {
                link.DeleteInDatabase();
            }
        }


        base.DeleteInDatabase();
    }

    /// <inheritdoc/>
    public override string ToString() => Name;

    /// <inheritdoc/>
    public IEnumerable<ICatalogue> GetAllCatalogues()
    {
        var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);
        return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));
    }

    /// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
    public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
    {
        var loggingServers = GetLoggingServers();

        var loggingServer = loggingServers.FirstOrDefault();

        //get distinct connection
        var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);

        serverChosen = (IExternalDatabaseServer)loggingServer;
        return toReturn;
    }

    /// <summary>
    /// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
    /// </summary>
    /// <returns></returns>
    public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);

    private IDataAccessPoint[] GetLoggingServers()
    {
        var catalogue = GetAllCatalogues().ToArray();

        return !catalogue.Any()
            ? throw new NotSupportedException(
                $"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")
            : (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();
    }

    /// <summary>
    /// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
    /// </summary>
    /// <returns></returns>
    public string GetDistinctLoggingTask()
    {
        var catalogueMetadatas = GetAllCatalogues().ToArray();

        if (!catalogueMetadatas.Any())
            throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");

        var cataloguesWithoutLoggingTasks =
            catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();

        if (cataloguesWithoutLoggingTasks.Any())
            throw new Exception(
                $"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");

        var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();
        return distinctLoggingTasks.Length >= 2
            ? throw new Exception(
                $"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")
            : distinctLoggingTasks[0];
    }

    /// <summary>
    /// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
    /// </summary>
    /// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
    /// <returns></returns>
    public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
    {
        var toReturn = new List<TableInfo>();

        foreach (var catalogueMetadata in GetAllCatalogues())
            foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))
                if (!toReturn.Contains(tableInfo))
                    toReturn.Add(tableInfo);

        return toReturn;
    }

    /// <inheritdoc/>
    public DiscoveredServer GetDistinctLiveDatabaseServer()
    {
        var normalTables = new HashSet<ITableInfo>();
        var lookupTables = new HashSet<ITableInfo>();

        foreach (var catalogue in GetAllCatalogues())
        {
            catalogue.GetTableInfos(out var normal, out var lookup);

            foreach (var n in normal)
                normalTables.Add(n);
            foreach (var l in lookup)
                lookupTables.Add(l);
        }

        if (normalTables.Any())
            return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);

        return lookupTables.Any()
            ? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)
            : throw new Exception(
                $"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");
    }

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsThisDependsOn() => null;

    /// <inheritdoc/>
    public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();

    /// <summary>
    /// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
    /// </summary>
    /// <param name="catalogue"></param>
    public void EnsureLoggingWorksFor(ICatalogue catalogue)
    {
        //if there's no logging task / logging server set them up with the same name as the lmd
        IExternalDatabaseServer loggingServer;

        if (catalogue.LiveLoggingServer_ID == null)
        {
            loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);

            if (loggingServer != null)
                catalogue.LiveLoggingServer_ID = loggingServer.ID;
            else
                throw new NotSupportedException(
                    "You do not yet have any logging servers configured so cannot create data loads");
        }
        else
        {
            loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);
        }

        //if there's no logging task yet and there's a logging server
        if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))
        {
            var lm = new LogManager(loggingServer);
            var loggingTaskName = Name;

            lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);
            catalogue.LoggingDataTask = loggingTaskName;
            catalogue.SaveToDatabase();
        }
    }

    /// <inheritdoc/>
    public IQuerySyntaxHelper GetQuerySyntaxHelper()
    {
        var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();
        return syntax.Length > 1
            ? throw new Exception(
                $"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")
            : syntax.SingleOrDefault();
    }

    /// <summary>
    /// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
    /// </summary>
    /// <param name="runs"></param>
    /// <returns></returns>
    public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;

    public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
    {
        return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,
            loadMetadata).Any(p => p.Value == "true");
    }


    /// <inheritdoc/>
    public DatabaseEntity SaveNewVersion()
    {
        var lmd = Clone();
        lmd.RootLoadMetadata_ID = RootLoadMetadata_ID != null ? RootLoadMetadata_ID : ID;
        lmd.Name = $"{Name} - {DateTime.Now}";
        lmd.SaveToDatabase();
        return lmd;
    }
}

1	// Copyright (c) The University of Dundee 2018-2024
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.IO;
11	using System.Linq;
12	using FAnsi.Discovery;
13	using FAnsi.Discovery.QuerySyntax;
14	using Rdmp.Core.Curation.Data.Cache;
15	using Rdmp.Core.Curation.Data.Defaults;
16	using Rdmp.Core.Curation.Data.ImportExport;
17	using Rdmp.Core.Curation.Data.Serialization;
18	using Rdmp.Core.Logging;
19	using Rdmp.Core.Logging.PastEvents;
20	using Rdmp.Core.MapsDirectlyToDatabaseTable;
21	using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
22	using Rdmp.Core.Repositories;
23	using Rdmp.Core.ReusableLibraryCode;
24	using Rdmp.Core.ReusableLibraryCode.Annotations;
25	using Rdmp.Core.ReusableLibraryCode.DataAccess;
26
27	namespace Rdmp.Core.Curation.Data.DataLoad;
28
29	/// <summary>
30	/// How are files cached within the cache (e.g. within a zip? tar? just uncompressed in a directory).
31	/// </summary>
32	public enum CacheArchiveType
33	{
34	/// <summary>
35	/// Cached files are in a directory uncompressed
36	/// </summary>
37	None = 0,
38
39	/// <summary>
40	/// Cached files are contained in a zip file
41	/// </summary>
42	Zip = 1
43	}
44
45	/// <inheritdoc cref="ILoadMetadata"/>
46	public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHasQuerySyntaxHelper,
47	ILoggedActivityRootObject, IHasFolder, IVersionable
48	{
49	#region Database Properties
50
51	private string _locationOfForLoadingDirectory;
52	private string _locationOfForArchivingDirectory;
53	private string _locationOfExecutablesDirectory;
54	private string _locationOfCacheDirectory;
55	private string _anonymisationEngineClass;
56	private string _name;
57	private string _description;
58	private CacheArchiveType _cacheArchiveType;
59	private int? _overrideRawServerID;
60	private bool _ignoreTrigger;
61	private string _folder;
62	private DateTime? _lastLoadTime;
63	private bool _allowReservedPrefix;
64	private int? _rootLoadMetadata_ID;
65
66	public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");	1,130✔
67	public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");	1,130✔
68	public string DefaultExecutablesPath = "Executables";	1,130✔
69	public string DefaultCachePath = Path.Combine("Data", "Cache");	1,130✔
70
71	public DirectoryInfo GetRootDirectory()
72	{
73	if (!string.IsNullOrWhiteSpace(_locationOfForLoadingDirectory) && !string.IsNullOrWhiteSpace(_locationOfForArchivingDirectory) && !string.IsNullOrWhiteSpace(_locationOfExecutablesDirectory) && !string.IsNullOrWhiteSpace(_locationOfCacheDirectory))	×
74	{
75	var forLoadingRoot = _locationOfForLoadingDirectory.Replace(DefaultForLoadingPath, "");	×
76	var forArchivingRoot = _locationOfForArchivingDirectory.Replace(DefaultForArchivingPath, "");	×
77	var forExecutablesRoot = _locationOfExecutablesDirectory.Replace(DefaultExecutablesPath, "");	×
78	var forCacheRoot = _locationOfCacheDirectory.Replace(DefaultCachePath, "");	×
79	if (forLoadingRoot == forArchivingRoot && forExecutablesRoot == forCacheRoot && forArchivingRoot == forExecutablesRoot)	×
80	{
81	return new DirectoryInfo(forLoadingRoot);	×
82	}
83	}
84	return null;	×
85	}
86
87
88	/// <inheritdoc/>
89	public bool AllowReservedPrefix
90	{
91	get => _allowReservedPrefix;	930✔
92	set => SetField(ref _allowReservedPrefix, value);	984✔
93	}
94
95	/// <inheritdoc/>
96	public string LocationOfForLoadingDirectory
97	{
98	get => _locationOfForLoadingDirectory;	2,204✔
99	set => SetField(ref _locationOfForLoadingDirectory, value);	1,112✔
100	}
101
102	/// <inheritdoc/>
103	public string LocationOfForArchivingDirectory
104	{
105	get => _locationOfForArchivingDirectory;	1,952✔
106	set => SetField(ref _locationOfForArchivingDirectory, value);	1,112✔
107	}
108
109	/// <inheritdoc/>
110	public string LocationOfExecutablesDirectory
111	{
112	get => _locationOfExecutablesDirectory;	1,952✔
113	set => SetField(ref _locationOfExecutablesDirectory, value);	1,112✔
114	}
115
116	/// <inheritdoc/>
117	public string LocationOfCacheDirectory
118	{
119	get => _locationOfCacheDirectory;	1,952✔
120	set => SetField(ref _locationOfCacheDirectory, value);	1,112✔
121	}
122
123	/// <summary>
124	/// Not used
125	/// </summary>
126	public string AnonymisationEngineClass
127	{
128	get => _anonymisationEngineClass;	880✔
129	set => SetField(ref _anonymisationEngineClass, value);	974✔
130	}
131
132	/// <inheritdoc/>
133	[Unique]
134	[NotNull]
135	public string Name
136	{
137	get => _name;	1,222✔
138	set => SetField(ref _name, value);	1,844✔
139	}
140
141	/// <summary>
142	/// Human readable description of the load, what it does etc
143	/// </summary>
144	public string Description
145	{
146	get => _description;	894✔
147	set => SetField(ref _description, value);	980✔
148	}
149
150	/// <summary>
151	/// The format for storing files in when reading/writing to a cache with a <see cref="CacheProgress"/>. This may not be respected
152	/// depending on the implementation of the specific ICacheLayout
153	/// </summary>
154	public CacheArchiveType CacheArchiveType
155	{
156	get => _cacheArchiveType;	880✔
157	set => SetField(ref _cacheArchiveType, value);	984✔
158	}
159
160	/// <summary>
161	/// Optional. Indicates that when running the Data Load Engine, the specific <see cref="ExternalDatabaseServer"/> should be used for the RAW server (instead of
162	/// the system default - see <see cref="ServerDefaults"/>).
163	/// </summary>
164	public int? OverrideRAWServer_ID
165	{
166	get => _overrideRawServerID;	1,780✔
167	set => SetField(ref _overrideRawServerID, value);	974✔
168	}
169
170
171	/// <iheritdoc/>
172	public bool IgnoreTrigger
173	{
174	get => _ignoreTrigger;	1,272✔
175	set => SetField(ref _ignoreTrigger, value);	1,132✔
176	}
177
178	/// <inheritdoc/>
179	[UsefulProperty]
180	public string Folder
181	{
182	get => _folder;	1,532✔
183	set => SetField(ref _folder, FolderHelper.Adjust(value));	1,108✔
184	}
185
186
187	/// <summary>
188	/// Stores the last time the load was ran.
189	/// </summary>
190	public DateTime? LastLoadTime
191	{
192	get => _lastLoadTime;	888✔
193	set => SetField(ref _lastLoadTime, value);	1,126✔
194	}
195
196	/// <inheritdoc/>
197	public int? RootLoadMetadata_ID { get => _rootLoadMetadata_ID; private set => SetField(ref _rootLoadMetadata_ID, value); }	8,922✔
198
199	#endregion
200
201
202	#region Relationships
203
204	/// <inheritdoc/>
205	[NoMappingToDatabase]
206	public ExternalDatabaseServer OverrideRAWServer => OverrideRAWServer_ID.HasValue	350!
207	? Repository.GetObjectByID<ExternalDatabaseServer>(OverrideRAWServer_ID.Value)	350✔
208	: null;	350✔
209
210	/// <inheritdoc/>
211	[NoMappingToDatabase]
212	public ILoadProgress[] LoadProgresses => Repository.GetAllObjectsWithParent<LoadProgress>(this);	202✔
213
214	/// <inheritdoc/>
215	[NoMappingToDatabase]
216	public IOrderedEnumerable<IProcessTask> ProcessTasks
217	{
218	get
219	{
220	return	320✔
221	Repository.GetAllObjectsWithParent<ProcessTask>(this).Cast<IProcessTask>().OrderBy(pt => pt.Order);	620✔
222	}
223	}
224
225	#endregion
226
227	public LoadMetadata()	30✔
228	{
229	}	30✔
230
231	/// <summary>
232	/// Create a new DLE load. This load will not have any <see cref="ProcessTask"/> and will not load any <see cref="TableInfo"/> yet.
233	///
234	/// <para>To set the loaded tables, set <see cref="Catalogue.LoadMetadatas"/> on some of your datasets</para>
235	/// </summary>
236	/// <param name="repository"></param>
237	/// <param name="name"></param>
238	/// <param name="rootLoadMetadata"></param>
239	public LoadMetadata(ICatalogueRepository repository, string name = null, LoadMetadata rootLoadMetadata = null)	358✔
240	{
241	name ??= $"NewLoadMetadata{Guid.NewGuid()}";	358✔
242	repository.InsertAndHydrate(this, new Dictionary<string, object>	358!
243	{	358✔
244	{ "Name", name },	358✔
245	{ "IgnoreTrigger", false /todo could be system global default here/ },	358✔
246	{ "Folder", FolderHelper.Root },	358✔
247	{"LastLoadTime", null },	358✔
248	{"RootLoadMetadata_ID", rootLoadMetadata is null? null: rootLoadMetadata.ID }	358✔
249	});	358✔
250	}	358✔
251
252	internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
253	: base(repository, r)	732✔
254	{
255	LocationOfForLoadingDirectory = r["LocationOfForLoadingDirectory"].ToString();	732✔
256	LocationOfForArchivingDirectory = r["LocationOfForArchivingDirectory"].ToString();	732✔
257	LocationOfExecutablesDirectory = r["LocationOfExecutablesDirectory"].ToString();	732✔
258	LocationOfCacheDirectory = r["LocationOfCacheDirectory"].ToString();	732✔
259	Name = r["Name"] as string;	732✔
260	AnonymisationEngineClass = r["AnonymisationEngineClass"].ToString();	732✔
261	Name = r["Name"].ToString();	732✔
262	Description = r["Description"] as string; //allows for nulls	732✔
263	CacheArchiveType = (CacheArchiveType)r["CacheArchiveType"];	732✔
264	OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]);	732✔
265	IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;	732✔
266	Folder = r["Folder"] as string ?? FolderHelper.Root;	732!
267	LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());	732✔
268	AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;	732✔
269	RootLoadMetadata_ID = ObjectToNullableInt(r["RootLoadMetadata_ID"]);	732✔
270	}	732✔
271
272
273	public LoadMetadata Clone()
274	{
275	var lmd = new LoadMetadata(CatalogueRepository, Name)	6✔
276	{	6✔
277	LocationOfForLoadingDirectory = LocationOfForLoadingDirectory,	6✔
278	LocationOfForArchivingDirectory = LocationOfForArchivingDirectory,	6✔
279	LocationOfCacheDirectory = LocationOfCacheDirectory,	6✔
280	LocationOfExecutablesDirectory = LocationOfExecutablesDirectory,	6✔
281	AnonymisationEngineClass = AnonymisationEngineClass,	6✔
282	Name = Name,	6✔
283	Description = Description,	6✔
284	CacheArchiveType = CacheArchiveType,	6✔
285	AllowReservedPrefix = false,	6✔
286	LastLoadTime = LastLoadTime,	6✔
287	OverrideRAWServer_ID = OverrideRAWServer_ID,	6✔
288	IgnoreTrigger = IgnoreTrigger,	6✔
289	Folder = Folder,	6✔
290	};	6✔
291	lmd.SaveToDatabase();	6✔
292	//link to catalogue
293	foreach (var catalogue in GetAllCatalogues())	24✔
294	{
295	lmd.LinkToCatalogue(catalogue);	6✔
296	}
297	//process task
298	var pts = CatalogueRepository.GetAllObjectsWhere<ProcessTask>("LoadMetadata_ID", ID);	6✔
299	foreach (ProcessTask pt in pts)	24✔
300	{
301	pt.Clone(lmd);	6✔
302	}
303	return lmd;	6✔
304	}
305
306	internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()	10✔
307	{
308	shareManager.UpsertAndHydrate(this, shareDefinition);	10✔
309	}	10✔
310
311	public void LinkToCatalogue(ICatalogue catalogue)
312	{
313	var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, this, catalogue);	308✔
314	linkage.SaveToDatabase();	308✔
315	}	308✔
316
317	public void UnlinkFromCatalogue(ICatalogue catalogue)
318	{
319	foreach (var l in CatalogueRepository.GetAllObjects<LoadMetadataCatalogueLinkage>().Where(link => link.CatalogueID == catalogue.ID && link.LoadMetadataID == ID))	56✔
320	{
321	l.DeleteInDatabase();	8✔
322	}
323	}	8✔
324
325	/// <inheritdoc/>
326	public override void DeleteInDatabase()
327	{
328	var firstOrDefault = GetAllCatalogues().FirstOrDefault();	74✔
329
330	if (firstOrDefault != null && RootLoadMetadata_ID == null)	74!
331	throw new Exception(	×
332	$"This load is used by {firstOrDefault.Name} so cannot be deleted (Disassociate it first)");	×
333
334	var versions = Repository.GetAllObjectsWhere<LoadMetadata>("RootLoadMetadata_ID", ID);	74✔
335	foreach (var version in versions)	148!
336	{
NEW 337	version.DeleteInDatabase();	×
338	}
339	if (RootLoadMetadata_ID != null)	74!
340	{
NEW 341	var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID);	×
NEW 342	foreach (var link in catalogueLinkIDs)	×
343	{
NEW 344	link.DeleteInDatabase();	×
345	}
346	}
347
348
349	base.DeleteInDatabase();	74✔
350	}	74✔
351
352	/// <inheritdoc/>
353	public override string ToString() => Name;	98✔
354
355	/// <inheritdoc/>
356	public IEnumerable<ICatalogue> GetAllCatalogues()
357	{
358	var catalogueLinkIDs = Repository.GetAllObjectsWhere<LoadMetadataCatalogueLinkage>("LoadMetadataID", ID).Select(l => l.CatalogueID);	3,978✔
359	return Repository.GetAllObjects<Catalogue>().Where(cat => catalogueLinkIDs.Contains(cat.ID));	3,858✔
360	}
361
362	/// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
363	public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
364	{
365	var loggingServers = GetLoggingServers();	114✔
366
367	var loggingServer = loggingServers.FirstOrDefault();	112✔
368
369	//get distinct connection
370	var toReturn = DataAccessPortal.ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);	112✔
371
372	serverChosen = (IExternalDatabaseServer)loggingServer;	108✔
373	return toReturn;	108✔
374	}
375
376	/// <summary>
377	/// The unique logging server for auditing the load (found by querying <see cref="Catalogue.LiveLoggingServer"/>)
378	/// </summary>
379	/// <returns></returns>
380	public DiscoveredServer GetDistinctLoggingDatabase() => GetDistinctLoggingDatabase(out _);	114✔
381
382	private IDataAccessPoint[] GetLoggingServers()
383	{
384	var catalogue = GetAllCatalogues().ToArray();	114✔
385
386	return !catalogue.Any()	114!
387	? throw new NotSupportedException(	114✔
388	$"LoadMetaData '{ToString()} (ID={ID}) does not have any Catalogues associated with it so it is not possible to fetch its LoggingDatabaseSettings")	114✔
389	: (IDataAccessPoint[])catalogue.Select(c => c.LiveLoggingServer).ToArray();	236✔
390	}
391
392	/// <summary>
393	/// Returns the unique value of <see cref="Catalogue.LoggingDataTask"/> amongst all catalogues loaded by the <see cref="LoadMetadata"/>
394	/// </summary>
395	/// <returns></returns>
396	public string GetDistinctLoggingTask()
397	{
398	var catalogueMetadatas = GetAllCatalogues().ToArray();	88✔
399
400	if (!catalogueMetadatas.Any())	88!
401	throw new Exception($"There are no Catalogues associated with load metadata (ID={ID})");	×
402
403	var cataloguesWithoutLoggingTasks =	88✔
404	catalogueMetadatas.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();	186✔
405
406	if (cataloguesWithoutLoggingTasks.Any())	88✔
407	throw new Exception(	2✔
408	$"The following Catalogues do not have a LoggingDataTask specified:{cataloguesWithoutLoggingTasks.Aggregate("", (s, n) => $"{s}{n}(ID={n.ID}),")}");	4✔
409
410	var distinctLoggingTasks = catalogueMetadatas.Select(c => c.LoggingDataTask).Distinct().ToArray();	180✔
411	return distinctLoggingTasks.Length >= 2	86!
412	? throw new Exception(	86✔
413	$"There are {distinctLoggingTasks.Length} logging tasks in Catalogues belonging to this metadata (ID={ID})")	86✔
414	: distinctLoggingTasks[0];	86✔
415	}
416
417	/// <summary>
418	/// Return all <see cref="TableInfo"/> underlying the <see cref="Catalogue"/>(s) which use this load (what tables will be loaded by the DLE).
419	/// </summary>
420	/// <param name="includeLookups">true to include lookup tables (e.g. z_sex etc) configured in the <see cref="Catalogue"/>(s)</param>
421	/// <returns></returns>
422	public List<TableInfo> GetDistinctTableInfoList(bool includeLookups)
423	{
424	var toReturn = new List<TableInfo>();	8✔
425
426	foreach (var catalogueMetadata in GetAllCatalogues())	32✔
427	foreach (TableInfo tableInfo in catalogueMetadata.GetTableInfoList(includeLookups))	32✔
428	if (!toReturn.Contains(tableInfo))	8✔
429	toReturn.Add(tableInfo);	8✔
430
431	return toReturn;	8✔
432	}
433
434	/// <inheritdoc/>
435	public DiscoveredServer GetDistinctLiveDatabaseServer()
436	{
437	var normalTables = new HashSet<ITableInfo>();	244✔
438	var lookupTables = new HashSet<ITableInfo>();	244✔
439
440	foreach (var catalogue in GetAllCatalogues())	1,000✔
441	{
442	catalogue.GetTableInfos(out var normal, out var lookup);	256✔
443
444	foreach (var n in normal)	1,024✔
445	normalTables.Add(n);	256✔
446	foreach (var l in lookup)	512!
447	lookupTables.Add(l);	×
448	}
449
450	if (normalTables.Any())	244!
451	return DataAccessPortal.ExpectDistinctServer(normalTables.ToArray(), DataAccessContext.DataLoad, true);	244✔
452
453	return lookupTables.Any()	×
454	? DataAccessPortal.ExpectDistinctServer(lookupTables.ToArray(), DataAccessContext.DataLoad, true)	×
455	: throw new Exception(	×
456	$"LoadMetadata {this} has no TableInfos configured (or possibly the tables have been deleted resulting in MISSING ColumnInfos?)");	×
457	}
458
459	/// <inheritdoc/>
460	public IHasDependencies[] GetObjectsThisDependsOn() => null;	×
461
462	/// <inheritdoc/>
463	public IHasDependencies[] GetObjectsDependingOnThis() => GetAllCatalogues().ToArray();	74✔
464
465	/// <summary>
466	/// Tests that the logging database for the load is reachable and that it has an appropriate logging task for the load (if not a new task will be created 'Loading X')
467	/// </summary>
468	/// <param name="catalogue"></param>
469	public void EnsureLoggingWorksFor(ICatalogue catalogue)
470	{
471	//if there's no logging task / logging server set them up with the same name as the lmd
472	IExternalDatabaseServer loggingServer;
473
474	if (catalogue.LiveLoggingServer_ID == null)	14!
475	{
476	loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);	×
477
478	if (loggingServer != null)	×
479	catalogue.LiveLoggingServer_ID = loggingServer.ID;	×
480	else
481	throw new NotSupportedException(	×
482	"You do not yet have any logging servers configured so cannot create data loads");	×
483	}
484	else
485	{
486	loggingServer = Repository.GetObjectByID<ExternalDatabaseServer>(catalogue.LiveLoggingServer_ID.Value);	14✔
487	}
488
489	//if there's no logging task yet and there's a logging server
490	if (string.IsNullOrWhiteSpace(catalogue.LoggingDataTask))	14✔
491	{
492	var lm = new LogManager(loggingServer);	14✔
493	var loggingTaskName = Name;	14✔
494
495	lm.CreateNewLoggingTaskIfNotExists(loggingTaskName);	14✔
496	catalogue.LoggingDataTask = loggingTaskName;	14✔
497	catalogue.SaveToDatabase();	14✔
498	}
499	}	14✔
500
501	/// <inheritdoc/>
502	public IQuerySyntaxHelper GetQuerySyntaxHelper()
503	{
504	var syntax = GetAllCatalogues().Select(c => c.GetQuerySyntaxHelper()).Distinct().ToArray();	4✔
505	return syntax.Length > 1	2!
506	? throw new Exception(	2✔
507	$"LoadMetadata '{this}' has multiple underlying Catalogue Live Database Type(s) - not allowed")	2✔
508	: syntax.SingleOrDefault();	2✔
509	}
510
511	/// <summary>
512	/// Returns all runs since each LoadMetadata has its own task and all runs apply to that task and hence this object
513	/// </summary>
514	/// <param name="runs"></param>
515	/// <returns></returns>
516	public IEnumerable<ArchivalDataLoadInfo> FilterRuns(IEnumerable<ArchivalDataLoadInfo> runs) => runs;	12✔
517
518	public static bool UsesPersistentRaw(ILoadMetadata loadMetadata)
519	{
520	return loadMetadata.CatalogueRepository.GetExtendedProperties(ExtendedProperty.PersistentRaw,	90✔
521	loadMetadata).Any(p => p.Value == "true");	90✔
522	}
523
524
525	/// <inheritdoc/>
526	public DatabaseEntity SaveNewVersion()
527	{
528	var lmd = Clone();	4✔
529	lmd.RootLoadMetadata_ID = RootLoadMetadata_ID != null ? RootLoadMetadata_ID : ID;	4!
530	lmd.Name = $"{Name} - {DateTime.Now}";	4✔
531	lmd.SaveToDatabase();	4✔
532	return lmd;	4✔
533	}
534	}

HicServices / RDMP / 13493861722

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous