14792559889

Committed 02 May 2025 09:25AM UTC coverage: 57.332% (-0.05%) from 57.38%

Build # 14792559889

Build Type

Pull #2191

github

Committed by

JFriel

Commit Message

fix build

Pull Request Pull Request #2191: [Datasets] Task/rdmp 33 dataset update post load

Run Details

11367 of 21386 branches covered (53.15%)

Branch coverage included in aggregate %.

75 of 202 new or added lines in 20 files covered. (37.13%)

281 existing lines in 2 files now uncovered.

32344 of 54856 relevant lines covered (58.96%)

17460.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Rdmp.Core/DataLoad/Engine/Mutilators/PostLoad/UpdateCatalogueMetadata.cs

﻿using FAnsi.Discovery;
using MongoDB.Driver;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Triggers;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
using Rdmp.Core.QueryBuilding;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Rdmp.Core.DataLoad.Engine.Mutilators.PostLoad
{
    class UpdateCatalogueMetadata : IMutilateDataTables
    {

        [DemandsInitialization("Update the time period associated with the dataset", DefaultValue = true)]
        public bool UpdateTimePeriods { get; set; }

        [DemandsInitialization("Update the description of the dataset", DefaultValue = true)]
        public bool UpdateDescription { get; set; }

        [DemandsInitialization("The Column within the data to use for temporal updates")]
        public string DateColumn { get; set; }


        [DemandsInitialization("Catalogues in Data Load to not update")]
        public List<Catalogue> CataloguestoIgnore { get; set; }

        [DemandsInitialization(@"The Text added to the description to denote the update.
Some variables are available:
%d - Todays date
%c - The number of new records
%s - The earliest date in the new records
%l - The latest date in the new records
", DefaultValue = "Update %d: Added %c Records with dates between %s and %l.")]
        public string DescriptionUpdateText { get; set; }

        public void Check(ICheckNotifier notifier)
        {
        }

        public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
        {
        }

        public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
        {
        }

        public ExitCodeType Mutilate(IDataLoadJob job)
        {
            if (!UpdateTimePeriods && !UpdateDescription) return ExitCodeType.OperationNotRequired;
            var catalogues = job.LoadMetadata.GetAllCatalogues().Where(c => !CataloguestoIgnore.Contains(c));
            foreach(var catalogue in catalogues)
            {
                var column = catalogue.CatalogueItems.Where(ci => ci.ColumnInfo.GetRuntimeName() == DateColumn).First();
                var dataLoadID = job.JobID;
                var qb = new QueryBuilder(null, null);
                qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), column.ColumnInfo));
                qb.AddCustomLine($"{SpecialFieldNames.DataLoadRunID} = {dataLoadID}", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
                var sql = qb.SQL;

                var dt = new DataTable();
                dt.BeginLoadData();
                var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad, false);
                var con = server.GetConnection();
                con.Open();
                using (var cmd = server.GetCommand(sql, con))
                {
                    using var da = server.GetDataAdapter(cmd);
                    da.Fill(dt);
                }
                dt.EndLoadData();
                if (dt.Rows.Count == 0) return ExitCodeType.OperationNotRequired;
                var latestDate = dt.AsEnumerable().Max(row => DateTime.Parse(row[0].ToString()));
                var earliestDate = dt.AsEnumerable().Min(row => DateTime.Parse(row[0].ToString()));
                var newRecordsCount = dt.Rows.Count;
                if (UpdateTimePeriods)
                {
                    catalogue.StartDate = earliestDate;
                    catalogue.EndDate = latestDate;
                }
                if (UpdateDescription)
                {
                    var description = catalogue.Description;
                    description = description + "\n\r" + DescriptionUpdateText.Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")).Replace("%c", $"{newRecordsCount}").Replace("%s", $"{earliestDate.Day}/{earliestDate.Month}/{earliestDate.Year}").Replace("%l", $"{latestDate.Day}/{latestDate.Month}/{latestDate.Year}");
                }
                catalogue.SaveToDatabase();
            }

            return ExitCodeType.Success;
        }
    }
}

1	using FAnsi.Discovery;
2	using MongoDB.Driver;
3	using Rdmp.Core.Curation.Data;
4	using Rdmp.Core.Curation.Data.DataLoad;
5	using Rdmp.Core.DataLoad.Engine.Job;
6	using Rdmp.Core.DataLoad.Triggers;
7	using Rdmp.Core.MapsDirectlyToDatabaseTable;
8	using Rdmp.Core.QueryBuilding;
9	using Rdmp.Core.ReusableLibraryCode.Checks;
10	using Rdmp.Core.ReusableLibraryCode.Progress;
11	using System;
12	using System.Collections.Generic;
13	using System.Data;
14	using System.Linq;
15	using System.Text;
16	using System.Threading.Tasks;
17
18	namespace Rdmp.Core.DataLoad.Engine.Mutilators.PostLoad
19	{
20	class UpdateCatalogueMetadata : IMutilateDataTables
21	{
22
23	[DemandsInitialization("Update the time period associated with the dataset", DefaultValue = true)]
NEW 24	public bool UpdateTimePeriods { get; set; }	×
25
26	[DemandsInitialization("Update the description of the dataset", DefaultValue = true)]
NEW 27	public bool UpdateDescription { get; set; }	×
28
29	[DemandsInitialization("The Column within the data to use for temporal updates")]
NEW 30	public string DateColumn { get; set; }	×
31
32
33	[DemandsInitialization("Catalogues in Data Load to not update")]
NEW 34	public List<Catalogue> CataloguestoIgnore { get; set; }	×
35
36	[DemandsInitialization(@"The Text added to the description to denote the update.
37	Some variables are available:
38	%d - Todays date
39	%c - The number of new records
40	%s - The earliest date in the new records
41	%l - The latest date in the new records
42	", DefaultValue = "Update %d: Added %c Records with dates between %s and %l.")]
NEW 43	public string DescriptionUpdateText { get; set; }	×
44
45	public void Check(ICheckNotifier notifier)
46	{
NEW 47	}	×
48
49	public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
50	{
NEW 51	}	×
52
53	public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
54	{
NEW 55	}	×
56
57	public ExitCodeType Mutilate(IDataLoadJob job)
58	{
NEW 59	if (!UpdateTimePeriods && !UpdateDescription) return ExitCodeType.OperationNotRequired;	×
NEW 60	var catalogues = job.LoadMetadata.GetAllCatalogues().Where(c => !CataloguestoIgnore.Contains(c));	×
NEW 61	foreach(var catalogue in catalogues)	×
62	{
NEW 63	var column = catalogue.CatalogueItems.Where(ci => ci.ColumnInfo.GetRuntimeName() == DateColumn).First();	×
NEW 64	var dataLoadID = job.JobID;	×
NEW 65	var qb = new QueryBuilder(null, null);	×
NEW 66	qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), column.ColumnInfo));	×
NEW 67	qb.AddCustomLine($"{SpecialFieldNames.DataLoadRunID} = {dataLoadID}", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);	×
NEW 68	var sql = qb.SQL;	×
69
NEW 70	var dt = new DataTable();	×
NEW 71	dt.BeginLoadData();	×
NEW 72	var server = catalogue.GetDistinctLiveDatabaseServer(ReusableLibraryCode.DataAccess.DataAccessContext.DataLoad, false);	×
NEW 73	var con = server.GetConnection();	×
NEW 74	con.Open();	×
NEW 75	using (var cmd = server.GetCommand(sql, con))	×
76	{
NEW 77	using var da = server.GetDataAdapter(cmd);	×
NEW 78	da.Fill(dt);	×
79	}
NEW 80	dt.EndLoadData();	×
NEW 81	if (dt.Rows.Count == 0) return ExitCodeType.OperationNotRequired;	×
NEW 82	var latestDate = dt.AsEnumerable().Max(row => DateTime.Parse(row[0].ToString()));	×
NEW 83	var earliestDate = dt.AsEnumerable().Min(row => DateTime.Parse(row[0].ToString()));	×
NEW 84	var newRecordsCount = dt.Rows.Count;	×
NEW 85	if (UpdateTimePeriods)	×
86	{
NEW 87	catalogue.StartDate = earliestDate;	×
NEW 88	catalogue.EndDate = latestDate;	×
89	}
NEW 90	if (UpdateDescription)	×
91	{
NEW 92	var description = catalogue.Description;	×
NEW 93	description = description + "\n\r" + DescriptionUpdateText.Replace("%d", DateTime.Now.ToString("dd/MM/yyyy")).Replace("%c", $"{newRecordsCount}").Replace("%s", $"{earliestDate.Day}/{earliestDate.Month}/{earliestDate.Year}").Replace("%l", $"{latestDate.Day}/{latestDate.Month}/{latestDate.Year}");	×
94	}
NEW 95	catalogue.SaveToDatabase();	×
96	}
97
NEW 98	return ExitCodeType.Success;	×
NEW 99	}	×
100	}
101	}

HicServices / RDMP / 14792559889

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous