13493861722

Committed 24 Feb 2025 08:35AM UTC coverage: 57.414% (-0.03%) from 57.446%

Build # 13493861722

Build Type

push

github

Committed by

web-flow

Commit Message

marge v8.4.3 into main (#2145)

* simple db update

* add changelog

* revert csproj

* Fix up some codeql/inspection code issues (#2087)

* Update OverviewModel.cs

Fix up some .Dispose/using issues, make finding most recent load ID more efficient

* LINQ tidying

* CodeQL fixups

* Update Catalogue.cs

Add Hashcode, Equals methods.

* Update Catalogue.cs

Tweak equality semantics for RDMP DB oddities

* Bump actions/setup-dotnet from 4.1.0 to 4.2.0

Bumps [actions/setup-dotnet](https://github.com/actions/setup-dotnet) from 4.1.0 to 4.2.0.
- [Release notes](https://github.com/actions/setup-dotnet/releases)
- [Commits](https://github.com/actions/setup-dotnet/compare/v4.1.0...v4.2.0)

---
updated-dependencies:
- dependency-name: actions/setup-dotnet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump NUnit.Analyzers from 4.4.0 to 4.5.0

Bumps [NUnit.Analyzers](https://github.com/nunit/nunit.analyzers) from 4.4.0 to 4.5.0.
- [Release notes](https://github.com/nunit/nunit.analyzers/releases)
- [Changelog](https://github.com/nunit/nunit.analyzers/blob/master/CHANGES.md)
- [Commits](https://github.com/nunit/nunit.analyzers/compare/4.4.0...4.5.0)

---
updated-dependencies:
- dependency-name: NUnit.Analyzers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump Minio from 6.0.3 to 6.0.4

Bumps [Minio](https://github.com/minio/minio-dotnet) from 6.0.3 to 6.0.4.
- [Release notes](https://github.com/minio/minio-dotnet/releases)
- [Commits](https://github.com/minio/minio-dotnet/compare/6.0.3...6.0.4)

---
updated-dependencies:
- dependency-name: Minio
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump the aws-sdk group across 1 directory with 4 updates

Bumps the aws-sdk grou... (continued)

Run Details

11358 of 21338 branches covered (53.23%)

Branch coverage included in aggregate %.

1116 of 1482 new or added lines in 50 files covered. (75.3%)

19 existing lines in 10 files now uncovered.

32283 of 54673 relevant lines covered (59.05%)

17450.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.52

/Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Rdmp.Core.Curation.Data.Cohort;
using Rdmp.Core.Curation.Data.ImportExport;
using Rdmp.Core.Curation.Data.Serialization;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
using Rdmp.Core.Repositories;
using Rdmp.Core.ReusableLibraryCode.Annotations;
using Rdmp.Core.ReusableLibraryCode.Checks;

namespace Rdmp.Core.Curation.Data.DataLoad;

/// <summary>
/// Describes a specific operation carried out at a specific step of a LoadMetadata.  This could be 'unzip all files called *.zip in for loading' or
/// 'after loading the data to live, call sp_clean_table1' or 'Connect to webservice X and download 1,000,000 records which will be serialized into XML'
/// 
/// <para>The class achieves this wide ranging functionality through the interaction of ProcessTaskType and Path.  e.g. when ProcessTaskType is Attacher then
/// Path functions as the Type name of a class that implements IAttacher e.g. 'LoadModules.Generic.Attachers.AnySeparatorFileAttacher'.  </para>
/// 
/// <para>Each ProcessTask can have one or more strongly typed arguments (see entity ProcessTaskArgument), these are discovered at design time by using
/// reflection to query the Path e.g. 'AnySeparatorFileAttacher' for all properties marked with [DemandsInitialization] attribute.  This allows for 3rd party developers
/// to write plugin classes to easily handle proprietary/bespoke source file types or complex data load requirements.</para>
/// </summary>
public class ProcessTask : DatabaseEntity, IProcessTask, IOrderable, INamed, ICheckable
{
    #region Database Properties

    private int _loadMetadataID;
    private int? _relatesSolelyToCatalogueID;
    private int _order;
    private string _path;
    private string _name;
    private LoadStage _loadStage;
    private ProcessTaskType _processTaskType;
    private bool _isDisabled;
#nullable enable
    private string? _SerialisableConfiguration;
#nullable disable

    /// <summary>
    /// The load the process task exists as part of
    /// </summary>
    [Relationship(typeof(LoadMetadata), RelationshipType.SharedObject)]
    public int LoadMetadata_ID
    {
        get => _loadMetadataID;
        set => SetField(ref _loadMetadataID, value);
    }

    /// <inheritdoc/>
    [Obsolete(
        "Since you can't change which Catalogues are loaded by a LoadMetadata at runtime, this property is now obsolete")]
    public int? RelatesSolelyToCatalogue_ID
    {
        get => _relatesSolelyToCatalogueID;
        set => SetField(ref _relatesSolelyToCatalogueID, value);
    }

    /// <inheritdoc/>
    public int Order
    {
        get => _order;
        set => SetField(ref _order, value);
    }

    /// <inheritdoc/>
    [AdjustableLocation]
    public string Path
    {
        get => _path;
        set => SetField(ref _path, value);
    }

    /// <inheritdoc cref="IProcessTask.Name"/>
    [NotNull]
    public string Name
    {
        get => _name;
        set => SetField(ref _name, value);
    }

    /// <inheritdoc/>
    public LoadStage LoadStage
    {
        get => _loadStage;
        set => SetField(ref _loadStage, value);
    }

    /// <inheritdoc/>
    public ProcessTaskType ProcessTaskType
    {
        get => _processTaskType;
        set => SetField(ref _processTaskType, value);
    }

    /// <inheritdoc/>
    public bool IsDisabled
    {
        get => _isDisabled;
        set => SetField(ref _isDisabled, value);
    }


    /// <inheritdoc/>
#nullable enable
    public string? SerialisableConfiguration
    {
        get => _SerialisableConfiguration;
        set => SetField(ref _SerialisableConfiguration, value);
    }


#nullable disable

    #endregion

    #region Relationships

    /// <inheritdoc cref="LoadMetadata_ID"/>
    [NoMappingToDatabase]
    public LoadMetadata LoadMetadata => Repository.GetObjectByID<LoadMetadata>(LoadMetadata_ID);

    /// <inheritdoc/>
    [NoMappingToDatabase]
    public IEnumerable<ProcessTaskArgument> ProcessTaskArguments =>
        Repository.GetAllObjectsWithParent<ProcessTaskArgument>(this);

    /// <summary>
    /// All <see cref="ILoadProgress"/> (if any) that can be advanced by executing this load.  This allows batch execution of large loads
    /// </summary>
    [NoMappingToDatabase]
    public ILoadProgress[] LoadProgresses => LoadMetadata.LoadProgresses;

    #endregion

    public ProcessTask()
    {
    }

    /// <summary>
    /// Creates a new operation in the data load (e.g. copy files from A to B, load all CSV files to RAW table B etc)
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="parent"></param>
    /// <param name="stage"></param>
    public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadStage stage)
    {
        var order =
            repository.GetAllObjectsWithParent<ProcessTask>(parent).Select(t => t.Order).DefaultIfEmpty().Max() + 1;

        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
            { "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID },
            { "ProcessTaskType", ProcessTaskType.Executable.ToString() },
            { "LoadStage", stage },
            { "Name", $"New Process{Guid.NewGuid()}" },
            { "Order", order },
        });
    }

    /// <summary>
    /// Creates a new operation in the data load (e.g. copy files from A to B, load all CSV files to RAW table B etc)
    /// </summary>
    /// <param name="repository"></param>
    /// <param name="parent"></param>
    /// <param name="stage"></param>
    /// <param name="serialisableConfiguration"></param>
    public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadStage stage, string serialisableConfiguration = null)
    {
        var order =
            repository.GetAllObjectsWithParent<ProcessTask>(parent).Select(t => t.Order).DefaultIfEmpty().Max() + 1;

        repository.InsertAndHydrate(this, new Dictionary<string, object>
        {
             { "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID },
            { "ProcessTaskType", ProcessTaskType.Executable.ToString() },
            { "LoadStage", stage },
            { "Name", $"New Process{Guid.NewGuid()}" },
            { "Order", order },
            {"SerialisableConfiguration", serialisableConfiguration}
        });
    }

    internal ProcessTask(ICatalogueRepository repository, DbDataReader r)
        : base(repository, r)
    {
        LoadMetadata_ID = int.Parse(r["LoadMetaData_ID"].ToString());

        if (r["RelatesSolelyToCatalogue_ID"] != DBNull.Value)
            _relatesSolelyToCatalogueID = int.Parse(r["RelatesSolelyToCatalogue_ID"].ToString());

        Path = r["Path"] as string;
        Name = r["Name"] as string;
        Order = int.Parse(r["Order"].ToString());
        if (Enum.TryParse(r["ProcessTaskType"] as string, out ProcessTaskType processTaskType))
            ProcessTaskType = processTaskType;
        else
            throw new Exception($"Could not parse ProcessTaskType:{r["ProcessTaskType"]}");

        if (Enum.TryParse(r["LoadStage"] as string, out LoadStage loadStage))
            LoadStage = loadStage;
        else
            throw new Exception($"Could not parse LoadStage:{r["LoadStage"]}");

        IsDisabled = Convert.ToBoolean(r["IsDisabled"]);
        if (r["SerialisableConfiguration"] is not null)
            SerialisableConfiguration = r["SerialisableConfiguration"].ToString();
    }

    internal ProcessTask(ShareManager shareManager, ShareDefinition shareDefinition)
    {
        shareManager.UpsertAndHydrate(this, shareDefinition);
    }

    /// <inheritdoc/>
    public override string ToString() => Name;

    /// <inheritdoc/>
    public void Check(ICheckNotifier notifier)
    {
        switch (ProcessTaskType)
        {
            case ProcessTaskType.Executable:
                CheckFileExistenceAndUniqueness(notifier);
                break;
            case ProcessTaskType.SQLFile:
                CheckFileExistenceAndUniqueness(notifier);
                CheckForProblemsInSQLFile(notifier);
                break;
            case ProcessTaskType.SQLBakFile:
                CheckFileExistenceAndUniqueness(notifier);
                break;
            case ProcessTaskType.Attacher:
                break;
            case ProcessTaskType.DataProvider:
                break;
            case ProcessTaskType.MutilateDataTable:
                break;
            default:
                throw new ArgumentOutOfRangeException();
        }
    }

    private void CheckForProblemsInSQLFile(ICheckNotifier notifier)
    {
        try
        {
            var sql = File.ReadAllText(Path);

            //let's check for any SQL that indicates user is trying to modify a STAGING table in a RAW script (for example)
            foreach (var tableInfo in LoadMetadata.GetDistinctTableInfoList(false))
                //for each stage get all the object names that are in that stage
                foreach (var stage in new[] { LoadStage.AdjustRaw, LoadStage.AdjustStaging, LoadStage.PostLoad })
                {
                    //process task belongs in that stage anyway so nothing is prohibited
                    if (stage == (LoadStage == LoadStage.Mounting ? LoadStage.AdjustRaw : LoadStage))
                        continue;

                    //figure out what is prohibited
                    var prohibitedSql = tableInfo.GetQuerySyntaxHelper()
                        .EnsureFullyQualified(tableInfo.GetDatabaseRuntimeName(stage), null,
                            tableInfo.GetRuntimeName(stage));

                    //if we reference it, complain
                    if (sql.Contains(prohibitedSql))
                        notifier.OnCheckPerformed(
                            new CheckEventArgs(
                                $"Sql in file '{Path}' contains a reference to '{prohibitedSql}' which is prohibited since the ProcessTask ('{Name}') runs in LoadStage {LoadStage}",
                                CheckResult.Warning));
                }
        }
        catch (Exception e)
        {
            notifier.OnCheckPerformed(new CheckEventArgs($"Failed to check the contents of the SQL file '{Path}'",
                CheckResult.Fail, e));
        }
    }

    private void CheckFileExistenceAndUniqueness(ICheckNotifier notifier)
    {
        if (string.IsNullOrWhiteSpace(Path))
        {
            notifier.OnCheckPerformed(new CheckEventArgs($"No Path specified for ProcessTask '{Name}'",
                CheckResult.Fail));
            return;
        }

        notifier.OnCheckPerformed(!File.Exists(Path)
            ? new CheckEventArgs($"Could not find File '{Path}' for ProcessTask '{Name}'", CheckResult.Fail)
            : new CheckEventArgs($"Found File '{Path}'", CheckResult.Success));


        var matchingPaths = Repository.GetAllObjects<ProcessTask>().Where(pt => pt.Path.Equals(Path));
        foreach (var duplicate in matchingPaths.Except(new[] { this }))
            notifier.OnCheckPerformed(
                new CheckEventArgs(
                    $"ProcessTask '{duplicate}' (ID={duplicate.ID}) also uses file '{System.IO.Path.GetFileName(Path)}'",
                    CheckResult.Warning));

        //conflicting tokens in Name string
        foreach (Match match in Regex.Matches(Name, @"'.*((\.exe')|(\.sql'))"))
            if (match.Success)
            {
                var referencedFile = System.IO.Path.GetFileName(match.Value.Trim('\''));
                var actualFile = System.IO.Path.GetFileName(Path);

                if (referencedFile != actualFile)
                    notifier.OnCheckPerformed(
                        new CheckEventArgs(
                            $"Name of ProcessTask '{Name}' (ID={ID}) references file '{match.Value}' but the Path of the ProcessTask is '{Path}'",
                            CheckResult.Fail));
            }
    }

    /// <summary>
    /// Returns all tables loaded by the parent <see cref="LoadMetadata"/>
    /// </summary>
    /// <returns></returns>
    public IEnumerable<TableInfo> GetTableInfos() => LoadMetadata.GetDistinctTableInfoList(true);

    /// <inheritdoc/>
    public IEnumerable<IArgument> GetAllArguments() => ProcessTaskArguments;

    /// <inheritdoc/>
    public IArgument CreateNewArgument() => new ProcessTaskArgument((ICatalogueRepository)Repository, this);

    /// <inheritdoc/>
    public string GetClassNameWhoArgumentsAreFor() => Path;

    /// <summary>
    /// Creates a new copy of the processTask and all its arguments in the database, this clone is then hooked up to the
    /// new LoadMetadata at the specified stage
    /// </summary>
    /// <param name="loadMetadata">The new LoadMetadata parent for the clone</param>
    /// <param name="loadStage">The new load stage to put the clone in </param>
    /// <returns>the new ProcessTask (the clone has a different ID to the parent)</returns>
    public ProcessTask CloneToNewLoadMetadataStage(LoadMetadata loadMetadata, LoadStage loadStage)
    {
        var cataRepository = (ICatalogueRepository)Repository;

        //clone only accepts sql connections so make sure we aren't in mysql land or something
        using (cataRepository.BeginNewTransaction())
        {
            try
            {
                //get list of arguments to also clone (will happen outside of transaction
                var toCloneArguments = ProcessTaskArguments.ToArray();

                //create a new transaction for all the cloning - note that once all objects are cloned the transaction is committed then all the objects are adjusted outside the transaction
                var clone = new ProcessTask(CatalogueRepository, LoadMetadata, loadStage);
                CopyShallowValuesTo(clone);

                //foreach of our child arguments
                foreach (var argument in toCloneArguments)
                    //clone it but rewire it to the proper ProcessTask parent (the clone)
                    argument.ShallowClone(clone);

                //the values passed into parameter
                clone.LoadMetadata_ID = loadMetadata.ID;
                clone.LoadStage = loadStage;
                clone.SaveToDatabase();

                //it worked
                cataRepository.EndTransaction(true);

                //return the clone
                return clone;
            }
            catch (Exception)
            {
                cataRepository.EndTransaction(false);
                throw;
            }
        }
    }

    /// <inheritdoc/>
    public IArgument[] CreateArgumentsForClassIfNotExists(Type t) =>
        ArgumentFactory.CreateArgumentsForClassIfNotExistsGeneric(
                t,

                //tell it how to create new instances of us related to parent
                this,

                //what arguments already exist
                GetAllArguments().ToArray())

            //convert the result back from generic to specific (us)
            .ToArray();

    /// <inheritdoc/>
    public IArgument[] CreateArgumentsForClassIfNotExists<T>() => CreateArgumentsForClassIfNotExists(typeof(T));

    /// <summary>
    /// Returns true if the <see cref="ProcessTaskType"/> is allowed to happen during the given <see cref="LoadStage"/>  (e.g. you can't use an IAttacher to
    /// load data into STAGING/LIVE - only RAW).
    /// </summary>
    /// <param name="type"></param>
    /// <param name="stage"></param>
    /// <returns></returns>
    public static bool IsCompatibleStage(ProcessTaskType type, LoadStage stage)
    {
        return type switch
        {
            ProcessTaskType.Executable => true,
            ProcessTaskType.SQLFile => stage != LoadStage.GetFiles,
            ProcessTaskType.SQLBakFile => stage != LoadStage.GetFiles,
            ProcessTaskType.Attacher => stage == LoadStage.Mounting,
            ProcessTaskType.DataProvider => true,
            ProcessTaskType.MutilateDataTable => stage != LoadStage.GetFiles,
            _ => throw new ArgumentOutOfRangeException(nameof(type))
        };
    }

    /// <summary>
    /// True if <see cref="Path"/> is the name of a C# class (as opposed to the path to an executable or SQL file etc)
    /// </summary>
    /// <returns></returns>
    public bool IsPluginType() => ProcessTaskType == ProcessTaskType.Attacher ||
                                  ProcessTaskType == ProcessTaskType.MutilateDataTable ||
                                  ProcessTaskType == ProcessTaskType.DataProvider;

    /// <summary>
    /// Sets the value of the corresponding <see cref="IArgument"/> (which must already exist) to the given value.  If your argument doesn't exist yet you
    /// can call <see cref="CreateArgumentsForClassIfNotExists"/>
    /// </summary>
    /// <param name="parameterName"></param>
    /// <param name="o"></param>
    public void SetArgumentValue(string parameterName, object o)
    {
        var matchingArgument = ProcessTaskArguments.SingleOrDefault(p => p.Name.Equals(parameterName)) ??
                               throw new Exception(
                                   $"Could not find a ProcessTaskArgument called '{parameterName}', have you called CreateArgumentsForClassIfNotExists<T> yet?");
        matchingArgument.SetValue(o);
        matchingArgument.SaveToDatabase();
    }


    public ProcessTask Clone(LoadMetadata lmd)
    {
        var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage) {
            ProcessTaskType = ProcessTaskType,
            Order = Order,
            IsDisabled = IsDisabled,
            SerialisableConfiguration = SerialisableConfiguration,
            Path = Path,
            Name= Name,
        };
        pt.LoadMetadata_ID = lmd.ID;
        pt.SaveToDatabase();
        foreach(var pta in ProcessTaskArguments)
        {
            pta.ShallowClone(pt);
        }
        return pt;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Data.Common;
10	using System.IO;
11	using System.Linq;
12	using System.Text.RegularExpressions;
13	using Rdmp.Core.Curation.Data.Cohort;
14	using Rdmp.Core.Curation.Data.ImportExport;
15	using Rdmp.Core.Curation.Data.Serialization;
16	using Rdmp.Core.MapsDirectlyToDatabaseTable;
17	using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes;
18	using Rdmp.Core.Repositories;
19	using Rdmp.Core.ReusableLibraryCode.Annotations;
20	using Rdmp.Core.ReusableLibraryCode.Checks;
21
22	namespace Rdmp.Core.Curation.Data.DataLoad;
23
24	/// <summary>
25	/// Describes a specific operation carried out at a specific step of a LoadMetadata. This could be 'unzip all files called *.zip in for loading' or
26	/// 'after loading the data to live, call sp_clean_table1' or 'Connect to webservice X and download 1,000,000 records which will be serialized into XML'
27	///
28	/// <para>The class achieves this wide ranging functionality through the interaction of ProcessTaskType and Path. e.g. when ProcessTaskType is Attacher then
29	/// Path functions as the Type name of a class that implements IAttacher e.g. 'LoadModules.Generic.Attachers.AnySeparatorFileAttacher'. </para>
30	///
31	/// <para>Each ProcessTask can have one or more strongly typed arguments (see entity ProcessTaskArgument), these are discovered at design time by using
32	/// reflection to query the Path e.g. 'AnySeparatorFileAttacher' for all properties marked with [DemandsInitialization] attribute. This allows for 3rd party developers
33	/// to write plugin classes to easily handle proprietary/bespoke source file types or complex data load requirements.</para>
34	/// </summary>
35	public class ProcessTask : DatabaseEntity, IProcessTask, IOrderable, INamed, ICheckable
36	{
37	#region Database Properties
38
39	private int _loadMetadataID;
40	private int? _relatesSolelyToCatalogueID;
41	private int _order;
42	private string _path;
43	private string _name;
44	private LoadStage _loadStage;
45	private ProcessTaskType _processTaskType;
46	private bool _isDisabled;
47	#nullable enable
48	private string? _SerialisableConfiguration;
49	#nullable disable
50
51	/// <summary>
52	/// The load the process task exists as part of
53	/// </summary>
54	[Relationship(typeof(LoadMetadata), RelationshipType.SharedObject)]
55	public int LoadMetadata_ID
56	{
57	get => _loadMetadataID;	25,446✔
58	set => SetField(ref _loadMetadataID, value);	1,144✔
59	}
60
61	/// <inheritdoc/>
62	[Obsolete(
63	"Since you can't change which Catalogues are loaded by a LoadMetadata at runtime, this property is now obsolete")]
64	public int? RelatesSolelyToCatalogue_ID
65	{
66	get => _relatesSolelyToCatalogueID;	720✔
67	set => SetField(ref _relatesSolelyToCatalogueID, value);	192✔
68	}
69
70	/// <inheritdoc/>
71	public int Order
72	{
73	get => _order;	1,164✔
74	set => SetField(ref _order, value);	1,154✔
75	}
76
77	/// <inheritdoc/>
78	[AdjustableLocation]
79	public string Path
80	{
81	get => _path;	1,316✔
82	set => SetField(ref _path, value);	1,260✔
83	}
84
85	/// <inheritdoc cref="IProcessTask.Name"/>
86	[NotNull]
87	public string Name
88	{
89	get => _name;	1,004✔
90	set => SetField(ref _name, value);	1,246✔
91	}
92
93	/// <inheritdoc/>
94	public LoadStage LoadStage
95	{
96	get => _loadStage;	4,232✔
97	set => SetField(ref _loadStage, value);	1,168✔
98	}
99
100	/// <inheritdoc/>
101	public ProcessTaskType ProcessTaskType
102	{
103	get => _processTaskType;	1,148✔
104	set => SetField(ref _processTaskType, value);	1,302✔
105	}
106
107	/// <inheritdoc/>
108	public bool IsDisabled
109	{
110	get => _isDisabled;	1,520✔
111	set => SetField(ref _isDisabled, value);	1,100✔
112	}
113
114
115	/// <inheritdoc/>
116	#nullable enable
117	public string? SerialisableConfiguration
118	{
119	get => _SerialisableConfiguration;	824✔
120	set => SetField(ref _SerialisableConfiguration, value);	1,090✔
121	}
122
123
124	#nullable disable
125
126	#endregion
127
128	#region Relationships
129
130	/// <inheritdoc cref="LoadMetadata_ID"/>
131	[NoMappingToDatabase]
132	public LoadMetadata LoadMetadata => Repository.GetObjectByID<LoadMetadata>(LoadMetadata_ID);	168✔
133
134	/// <inheritdoc/>
135	[NoMappingToDatabase]
136	public IEnumerable<ProcessTaskArgument> ProcessTaskArguments =>
137	Repository.GetAllObjectsWithParent<ProcessTaskArgument>(this);	602✔
138
139	/// <summary>
140	/// All <see cref="ILoadProgress"/> (if any) that can be advanced by executing this load. This allows batch execution of large loads
141	/// </summary>
142	[NoMappingToDatabase]
143	public ILoadProgress[] LoadProgresses => LoadMetadata.LoadProgresses;	2✔
144
145	#endregion
146
147	public ProcessTask()	×
148	{
149	}	×
150
151	/// <summary>
152	/// Creates a new operation in the data load (e.g. copy files from A to B, load all CSV files to RAW table B etc)
153	/// </summary>
154	/// <param name="repository"></param>
155	/// <param name="parent"></param>
156	/// <param name="stage"></param>
157	public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadStage stage)	236✔
158	{
159	var order =	236✔
160	repository.GetAllObjectsWithParent<ProcessTask>(parent).Select(t => t.Order).DefaultIfEmpty().Max() + 1;	262✔
161
162	repository.InsertAndHydrate(this, new Dictionary<string, object>	236!
163	{	236✔
164	{ "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID },	236✔
165	{ "ProcessTaskType", ProcessTaskType.Executable.ToString() },	236✔
166	{ "LoadStage", stage },	236✔
167	{ "Name", $"New Process{Guid.NewGuid()}" },	236✔
168	{ "Order", order },	236✔
169	});	236✔
170	}	236✔
171
172	/// <summary>
173	/// Creates a new operation in the data load (e.g. copy files from A to B, load all CSV files to RAW table B etc)
174	/// </summary>
175	/// <param name="repository"></param>
176	/// <param name="parent"></param>
177	/// <param name="stage"></param>
178	/// <param name="serialisableConfiguration"></param>
179	public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadStage stage, string serialisableConfiguration = null)	×
180	{
181	var order =	×
182	repository.GetAllObjectsWithParent<ProcessTask>(parent).Select(t => t.Order).DefaultIfEmpty().Max() + 1;	×
183
184	repository.InsertAndHydrate(this, new Dictionary<string, object>	×
185	{	×
NEW 186	{ "LoadMetadata_ID", parent.RootLoadMetadata_ID??parent.ID },	×
187	{ "ProcessTaskType", ProcessTaskType.Executable.ToString() },	×
188	{ "LoadStage", stage },	×
189	{ "Name", $"New Process{Guid.NewGuid()}" },	×
190	{ "Order", order },	×
191	{"SerialisableConfiguration", serialisableConfiguration}	×
192	});	×
193	}	×
194
195	internal ProcessTask(ICatalogueRepository repository, DbDataReader r)
196	: base(repository, r)	890✔
197	{
198	LoadMetadata_ID = int.Parse(r["LoadMetaData_ID"].ToString());	890✔
199
200	if (r["RelatesSolelyToCatalogue_ID"] != DBNull.Value)	890!
201	_relatesSolelyToCatalogueID = int.Parse(r["RelatesSolelyToCatalogue_ID"].ToString());	×
202
203	Path = r["Path"] as string;	890✔
204	Name = r["Name"] as string;	890✔
205	Order = int.Parse(r["Order"].ToString());	890✔
206	if (Enum.TryParse(r["ProcessTaskType"] as string, out ProcessTaskType processTaskType))	890!
207	ProcessTaskType = processTaskType;	890✔
208	else
209	throw new Exception($"Could not parse ProcessTaskType:{r["ProcessTaskType"]}");	×
210
211	if (Enum.TryParse(r["LoadStage"] as string, out LoadStage loadStage))	890!
212	LoadStage = loadStage;	890✔
213	else
214	throw new Exception($"Could not parse LoadStage:{r["LoadStage"]}");	×
215
216	IsDisabled = Convert.ToBoolean(r["IsDisabled"]);	890✔
217	if (r["SerialisableConfiguration"] is not null)	890✔
218	SerialisableConfiguration = r["SerialisableConfiguration"].ToString();	890✔
219	}	890✔
220
221	internal ProcessTask(ShareManager shareManager, ShareDefinition shareDefinition)	6✔
222	{
223	shareManager.UpsertAndHydrate(this, shareDefinition);	6✔
224	}	6✔
225
226	/// <inheritdoc/>
227	public override string ToString() => Name;	72✔
228
229	/// <inheritdoc/>
230	public void Check(ICheckNotifier notifier)
231	{
232	switch (ProcessTaskType)	66!
233	{
234	case ProcessTaskType.Executable:
235	CheckFileExistenceAndUniqueness(notifier);	×
236	break;	×
237	case ProcessTaskType.SQLFile:
238	CheckFileExistenceAndUniqueness(notifier);	×
239	CheckForProblemsInSQLFile(notifier);	×
240	break;	×
241	case ProcessTaskType.SQLBakFile:
242	CheckFileExistenceAndUniqueness(notifier);	×
243	break;	×
244	case ProcessTaskType.Attacher:
245	break;
246	case ProcessTaskType.DataProvider:
247	break;
248	case ProcessTaskType.MutilateDataTable:
249	break;
250	default:
251	throw new ArgumentOutOfRangeException();	×
252	}
253	}	66✔
254
255	private void CheckForProblemsInSQLFile(ICheckNotifier notifier)
256	{
257	try
258	{
259	var sql = File.ReadAllText(Path);	×
260
261	//let's check for any SQL that indicates user is trying to modify a STAGING table in a RAW script (for example)
262	foreach (var tableInfo in LoadMetadata.GetDistinctTableInfoList(false))	×
263	//for each stage get all the object names that are in that stage
264	foreach (var stage in new[] { LoadStage.AdjustRaw, LoadStage.AdjustStaging, LoadStage.PostLoad })	×
265	{
266	//process task belongs in that stage anyway so nothing is prohibited
267	if (stage == (LoadStage == LoadStage.Mounting ? LoadStage.AdjustRaw : LoadStage))	×
268	continue;
269
270	//figure out what is prohibited
271	var prohibitedSql = tableInfo.GetQuerySyntaxHelper()	×
272	.EnsureFullyQualified(tableInfo.GetDatabaseRuntimeName(stage), null,	×
273	tableInfo.GetRuntimeName(stage));	×
274
275	//if we reference it, complain
276	if (sql.Contains(prohibitedSql))	×
277	notifier.OnCheckPerformed(	×
278	new CheckEventArgs(	×
279	$"Sql in file '{Path}' contains a reference to '{prohibitedSql}' which is prohibited since the ProcessTask ('{Name}') runs in LoadStage {LoadStage}",	×
280	CheckResult.Warning));	×
281	}
282	}	×
283	catch (Exception e)	×
284	{
285	notifier.OnCheckPerformed(new CheckEventArgs($"Failed to check the contents of the SQL file '{Path}'",	×
286	CheckResult.Fail, e));	×
287	}	×
288	}	×
289
290	private void CheckFileExistenceAndUniqueness(ICheckNotifier notifier)
291	{
292	if (string.IsNullOrWhiteSpace(Path))	×
293	{
294	notifier.OnCheckPerformed(new CheckEventArgs($"No Path specified for ProcessTask '{Name}'",	×
295	CheckResult.Fail));	×
296	return;	×
297	}
298
299	notifier.OnCheckPerformed(!File.Exists(Path)	×
300	? new CheckEventArgs($"Could not find File '{Path}' for ProcessTask '{Name}'", CheckResult.Fail)	×
301	: new CheckEventArgs($"Found File '{Path}'", CheckResult.Success));	×
302
303
304	var matchingPaths = Repository.GetAllObjects<ProcessTask>().Where(pt => pt.Path.Equals(Path));	×
305	foreach (var duplicate in matchingPaths.Except(new[] { this }))	×
306	notifier.OnCheckPerformed(	×
307	new CheckEventArgs(	×
308	$"ProcessTask '{duplicate}' (ID={duplicate.ID}) also uses file '{System.IO.Path.GetFileName(Path)}'",	×
309	CheckResult.Warning));	×
310
311	//conflicting tokens in Name string
312	foreach (Match match in Regex.Matches(Name, @"'.*((\.exe')\|(\.sql'))"))	×
313	if (match.Success)	×
314	{
315	var referencedFile = System.IO.Path.GetFileName(match.Value.Trim('\''));	×
316	var actualFile = System.IO.Path.GetFileName(Path);	×
317
318	if (referencedFile != actualFile)	×
319	notifier.OnCheckPerformed(	×
320	new CheckEventArgs(	×
321	$"Name of ProcessTask '{Name}' (ID={ID}) references file '{match.Value}' but the Path of the ProcessTask is '{Path}'",	×
322	CheckResult.Fail));	×
323	}
324	}	×
325
326	/// <summary>
327	/// Returns all tables loaded by the parent <see cref="LoadMetadata"/>
328	/// </summary>
329	/// <returns></returns>
330	public IEnumerable<TableInfo> GetTableInfos() => LoadMetadata.GetDistinctTableInfoList(true);	2✔
331
332	/// <inheritdoc/>
333	public IEnumerable<IArgument> GetAllArguments() => ProcessTaskArguments;	354✔
334
335	/// <inheritdoc/>
336	public IArgument CreateNewArgument() => new ProcessTaskArgument((ICatalogueRepository)Repository, this);	1,662✔
337
338	/// <inheritdoc/>
339	public string GetClassNameWhoArgumentsAreFor() => Path;	18✔
340
341	/// <summary>
342	/// Creates a new copy of the processTask and all its arguments in the database, this clone is then hooked up to the
343	/// new LoadMetadata at the specified stage
344	/// </summary>
345	/// <param name="loadMetadata">The new LoadMetadata parent for the clone</param>
346	/// <param name="loadStage">The new load stage to put the clone in </param>
347	/// <returns>the new ProcessTask (the clone has a different ID to the parent)</returns>
348	public ProcessTask CloneToNewLoadMetadataStage(LoadMetadata loadMetadata, LoadStage loadStage)
349	{
350	var cataRepository = (ICatalogueRepository)Repository;	4✔
351
352	//clone only accepts sql connections so make sure we aren't in mysql land or something
353	using (cataRepository.BeginNewTransaction())	4✔
354	{
355	try
356	{
357	//get list of arguments to also clone (will happen outside of transaction
358	var toCloneArguments = ProcessTaskArguments.ToArray();	4✔
359
360	//create a new transaction for all the cloning - note that once all objects are cloned the transaction is committed then all the objects are adjusted outside the transaction
361	var clone = new ProcessTask(CatalogueRepository, LoadMetadata, loadStage);	4✔
362	CopyShallowValuesTo(clone);	4✔
363
364	//foreach of our child arguments
365	foreach (var argument in toCloneArguments)	12✔
366	//clone it but rewire it to the proper ProcessTask parent (the clone)
367	argument.ShallowClone(clone);	2✔
368
369	//the values passed into parameter
370	clone.LoadMetadata_ID = loadMetadata.ID;	4✔
371	clone.LoadStage = loadStage;	4✔
372	clone.SaveToDatabase();	4✔
373
374	//it worked
375	cataRepository.EndTransaction(true);	4✔
376
377	//return the clone
378	return clone;	4✔
379	}
380	catch (Exception)	×
381	{
382	cataRepository.EndTransaction(false);	×
383	throw;	×
384	}
385	}
386	}	4✔
387
388	/// <inheritdoc/>
389	public IArgument[] CreateArgumentsForClassIfNotExists(Type t) =>
390	ArgumentFactory.CreateArgumentsForClassIfNotExistsGeneric(	104✔
391	t,	104✔
392		104✔
393	//tell it how to create new instances of us related to parent	104✔
394	this,	104✔
395		104✔
396	//what arguments already exist	104✔
397	GetAllArguments().ToArray())	104✔
398		104✔
399	//convert the result back from generic to specific (us)	104✔
400	.ToArray();	104✔
401
402	/// <inheritdoc/>
403	public IArgument[] CreateArgumentsForClassIfNotExists<T>() => CreateArgumentsForClassIfNotExists(typeof(T));	94✔
404
405	/// <summary>
406	/// Returns true if the <see cref="ProcessTaskType"/> is allowed to happen during the given <see cref="LoadStage"/> (e.g. you can't use an IAttacher to
407	/// load data into STAGING/LIVE - only RAW).
408	/// </summary>
409	/// <param name="type"></param>
410	/// <param name="stage"></param>
411	/// <returns></returns>
412	public static bool IsCompatibleStage(ProcessTaskType type, LoadStage stage)
413	{
414	return type switch	×
415	{	×
416	ProcessTaskType.Executable => true,	×
417	ProcessTaskType.SQLFile => stage != LoadStage.GetFiles,	×
418	ProcessTaskType.SQLBakFile => stage != LoadStage.GetFiles,	×
419	ProcessTaskType.Attacher => stage == LoadStage.Mounting,	×
420	ProcessTaskType.DataProvider => true,	×
421	ProcessTaskType.MutilateDataTable => stage != LoadStage.GetFiles,	×
422	_ => throw new ArgumentOutOfRangeException(nameof(type))	×
423	};	×
424	}
425
426	/// <summary>
427	/// True if <see cref="Path"/> is the name of a C# class (as opposed to the path to an executable or SQL file etc)
428	/// </summary>
429	/// <returns></returns>
430	public bool IsPluginType() => ProcessTaskType == ProcessTaskType.Attacher \|\|	×
431	ProcessTaskType == ProcessTaskType.MutilateDataTable \|\|	×
432	ProcessTaskType == ProcessTaskType.DataProvider;	×
433
434	/// <summary>
435	/// Sets the value of the corresponding <see cref="IArgument"/> (which must already exist) to the given value. If your argument doesn't exist yet you
436	/// can call <see cref="CreateArgumentsForClassIfNotExists"/>
437	/// </summary>
438	/// <param name="parameterName"></param>
439	/// <param name="o"></param>
440	public void SetArgumentValue(string parameterName, object o)
441	{
442	var matchingArgument = ProcessTaskArguments.SingleOrDefault(p => p.Name.Equals(parameterName)) ??	3,414!
443	throw new Exception(	198✔
444	$"Could not find a ProcessTaskArgument called '{parameterName}', have you called CreateArgumentsForClassIfNotExists<T> yet?");	198✔
445	matchingArgument.SetValue(o);	198✔
446	matchingArgument.SaveToDatabase();	198✔
447	}	198✔
448
449
450	public ProcessTask Clone(LoadMetadata lmd)
451	{
452	var pt = new ProcessTask(CatalogueRepository, lmd, LoadStage) {	8✔
453	ProcessTaskType = ProcessTaskType,	8✔
454	Order = Order,	8✔
455	IsDisabled = IsDisabled,	8✔
456	SerialisableConfiguration = SerialisableConfiguration,	8✔
457	Path = Path,	8✔
458	Name= Name,	8✔
459	};	8✔
460	pt.LoadMetadata_ID = lmd.ID;	8✔
461	pt.SaveToDatabase();	8✔
462	foreach(var pta in ProcessTaskArguments)	320✔
463	{
464	pta.ShallowClone(pt);	152✔
465	}
466	return pt;	8✔
467	}
468	}

HicServices / RDMP / 13493861722

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous