12032849274

Committed 26 Nov 2024 02:32PM UTC coverage: 57.242% (-0.1%) from 57.386%

Build # 12032849274

Build Type

push

github

Committed by

JBaird00183

Commit Message

Bump the aws-sdk group with 4 updates (#2074)

Bumps the aws-sdk group with 4 updates: [AWSSDK.S3](https://github.com/aws/aws-sdk-net), [AWSSDK.SecurityToken](https://github.com/aws/aws-sdk-net), [AWSSDK.SSO](https://github.com/aws/aws-sdk-net) and [AWSSDK.SSOOIDC](https://github.com/aws/aws-sdk-net).


Updates `AWSSDK.S3` from 3.7.407 to 3.7.407.1
- [Release notes](https://github.com/aws/aws-sdk-net/releases)
- [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD)
- [Commits](https://github.com/aws/aws-sdk-net/commits)

Updates `AWSSDK.SecurityToken` from 3.7.401.5 to 3.7.401.6
- [Release notes](https://github.com/aws/aws-sdk-net/releases)
- [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD)
- [Commits](https://github.com/aws/aws-sdk-net/commits)

Updates `AWSSDK.SSO` from 3.7.400.56 to 3.7.400.57
- [Release notes](https://github.com/aws/aws-sdk-net/releases)
- [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD)
- [Commits](https://github.com/aws/aws-sdk-net/commits)

Updates `AWSSDK.SSOOIDC` from 3.7.400.56 to 3.7.400.57
- [Release notes](https://github.com/aws/aws-sdk-net/releases)
- [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD)
- [Commits](https://github.com/aws/aws-sdk-net/commits)

---
updated-dependencies:
- dependency-name: AWSSDK.S3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: aws-sdk
- dependency-name: AWSSDK.SecurityToken
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: aws-sdk
- dependency-name: AWSSDK.SSO
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: aws-sdk
- dependency-name: AWSSDK.SSOOIDC
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: aws-sdk
...

Signed-off-by: dependabot[bot] <support@gi... (continued)

Run Details

11215 of 21100 branches covered (53.15%)

Branch coverage included in aggregate %.

31759 of 53974 relevant lines covered (58.84%)

8252.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.22

/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using FAnsi;
using FAnsi.Connections;
using FAnsi.Discovery;
using FAnsi.Discovery.QuerySyntax;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Triggers;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.Migration.QueryBuilding;

/// <summary>
///     Migrates from STAGING to LIVE a single table (with a MigrationColumnSet).  This is an UPSERT (new replaces old)
///     operation achieved (in SQL) with MERGE and
///     UPDATE (based on primary key).  Both tables must be on the same server.  A MERGE sql statement will be created
///     using LiveMigrationQueryHelper and executed
///     within a transaction.
/// </summary>
public class OverwriteMigrationStrategy : DatabaseMigrationStrategy
{
    public OverwriteMigrationStrategy(IManagedConnection managedConnection)
        : base(managedConnection)
    {
    }

    public override void MigrateTable(IDataLoadJob job, MigrationColumnSet columnsToMigrate, int dataLoadInfoID,
        GracefulCancellationToken cancellationToken, ref int inserts, ref int updates)
    {
        var server = columnsToMigrate.DestinationTable.Database.Server;

        //see CrossDatabaseMergeCommandTest

        /*          ------------MIGRATE NEW RECORDS (novel by primary key)--------
         *

INSERT INTO CrossDatabaseMergeCommandTo..ToTable (Name,Age,Postcode,hic_dataLoadRunID)
SELECT
[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Name,
[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Age,
[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Postcode,
1
FROM
[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING
left join
CrossDatabaseMergeCommandTo..ToTable
on
[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Age = CrossDatabaseMergeCommandTo..ToTable.Age
AND
[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Name = CrossDatabaseMergeCommandTo..ToTable.Name
WHERE
CrossDatabaseMergeCommandTo..ToTable.Age is null
*/

        var sbInsert = new StringBuilder();
        var syntax = server.GetQuerySyntaxHelper();


        sbInsert.AppendLine(
            $"INSERT INTO {columnsToMigrate.DestinationTable.GetFullyQualifiedName()} ({string.Join(",", columnsToMigrate.FieldsToUpdate.Select(c => syntax.EnsureWrapped(c.GetRuntimeName())))}");

        //if we are not ignoring the trigger then we should record the data load run ID
        if (!job.LoadMetadata.IgnoreTrigger)
            sbInsert.AppendLine($",{syntax.EnsureWrapped(SpecialFieldNames.DataLoadRunID)}");

        sbInsert.AppendLine(")");

        sbInsert.AppendLine("SELECT");

        // Add the columns we are migrating
        sbInsert.AppendLine(string.Join($",{Environment.NewLine}",
            columnsToMigrate.FieldsToUpdate.Select(c => c.GetFullyQualifiedName())));

        // If we are using trigger also add the run ID e.g. ",50"
        if (!job.LoadMetadata.IgnoreTrigger)
            sbInsert.AppendLine($",{dataLoadInfoID}");

        sbInsert.AppendLine("FROM");
        sbInsert.AppendLine(columnsToMigrate.SourceTable.GetFullyQualifiedName());
        sbInsert.AppendLine("LEFT JOIN");
        sbInsert.AppendLine(columnsToMigrate.DestinationTable.GetFullyQualifiedName());
        sbInsert.AppendLine("ON");

        sbInsert.AppendLine(
            string.Join($" AND {Environment.NewLine}",
                columnsToMigrate.PrimaryKeys.Select(
                    pk =>
                        string.Format("{0}.{1}={2}.{1}", columnsToMigrate.SourceTable.GetFullyQualifiedName(),
                            syntax.EnsureWrapped(pk.GetRuntimeName()),
                            columnsToMigrate.DestinationTable.GetFullyQualifiedName()))));

        sbInsert.AppendLine("WHERE");
        sbInsert.AppendLine(string.Join(" AND ",
            columnsToMigrate.PrimaryKeys.Select(pk =>
                $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(pk.GetRuntimeName())} IS NULL")));

        //right at the end of the SELECT
        if (columnsToMigrate.DestinationTable.Database.Server.DatabaseType == DatabaseType.MySql)
            sbInsert.Append(" FOR UPDATE");

        var insertSql = sbInsert.ToString();

        var cmd = server.GetCommand(insertSql, _managedConnection);
        cmd.CommandTimeout = Timeout;

        job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
            $"INSERT query: {Environment.NewLine}{insertSql}"));

        cancellationToken.ThrowIfCancellationRequested();


        try
        {
            inserts = cmd.ExecuteNonQuery();

            var sqlLines = new List<CustomLine>();

            var toSet = columnsToMigrate.FieldsToUpdate.Where(c => !c.IsPrimaryKey).Select(c =>
                string.Format("t1.{0} = t2.{0}", syntax.EnsureWrapped(c.GetRuntimeName()))).ToArray();

            if (!toSet.Any())
            {
                job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                    $"Table {columnsToMigrate.DestinationTable} is entirely composed of PrimaryKey columns or hic_ columns so UPDATE will NOT take place"));
                return;
            }

            var toDiff = columnsToMigrate.FieldsToDiff.Where(c => !c.IsPrimaryKey).ToArray();

            if (!toDiff.Any())
            {
                job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
                    $"Table {columnsToMigrate.DestinationTable} is entirely composed of PrimaryKey columns or hic_ columns/ other non DIFF columns that will not result in an UPDATE will NOT take place"));
                return;
            }

            //t1.Name = t2.Name, t1.Age=T2.Age etc
            sqlLines.Add(new CustomLine(string.Join(",", toSet), QueryComponent.SET));

            //also update the hic_dataLoadRunID field
            if (!job.LoadMetadata.IgnoreTrigger)
                sqlLines.Add(new CustomLine(
                    $"t1.{syntax.EnsureWrapped(SpecialFieldNames.DataLoadRunID)}={dataLoadInfoID}",
                    QueryComponent.SET));

            //t1.Name <> t2.Name AND t1.Age <> t2.Age etc
            sqlLines.Add(new CustomLine(string.Join(" OR ", toDiff.Select(c => GetORLine(c, syntax))),
                QueryComponent.WHERE));

            //the join
            sqlLines.AddRange(columnsToMigrate.PrimaryKeys.Select(p =>
                new CustomLine(string.Format("t1.{0} = t2.{0}", syntax.EnsureWrapped(p.GetRuntimeName())),
                    QueryComponent.JoinInfoJoin)));

            var updateHelper = columnsToMigrate.DestinationTable.Database.Server.GetQuerySyntaxHelper().UpdateHelper;

            var updateQuery = updateHelper.BuildUpdate(
                columnsToMigrate.DestinationTable,
                columnsToMigrate.SourceTable,
                sqlLines);

            job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
                $"Update query:{Environment.NewLine}{updateQuery}"));

            var updateCmd = server.GetCommand(updateQuery, _managedConnection);
            updateCmd.CommandTimeout = Timeout;
            cancellationToken.ThrowIfCancellationRequested();

            try
            {
                updates = updateCmd.ExecuteNonQuery();
            }
            catch (Exception e)
            {
                job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,
                    $"Did not successfully perform the update queries: {updateQuery}", e));
                throw new Exception($"Did not successfully perform the update queries: {updateQuery} - {e}");
            }
        }
        catch (OperationCanceledException)
        {
            throw; // have to catch and rethrow this because of the catch-all below
        }
        catch (Exception e)
        {
            job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,
                $"Failed to migrate {columnsToMigrate.SourceTable} to {columnsToMigrate.DestinationTable}", e));
            throw new Exception(
                $"Failed to migrate {columnsToMigrate.SourceTable} to {columnsToMigrate.DestinationTable}: {e}");
        }
    }

    private static string GetORLine(DiscoveredColumn c, IQuerySyntaxHelper syntax)
    {
        return string.Format(
            "(t1.{0} <> t2.{0} OR (t1.{0} is null AND t2.{0} is not null) OR (t2.{0} is null AND t1.{0} is not null))",
            syntax.EnsureWrapped(c.GetRuntimeName()));
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Collections.Generic;
9	using System.Linq;
10	using System.Text;
11	using FAnsi;
12	using FAnsi.Connections;
13	using FAnsi.Discovery;
14	using FAnsi.Discovery.QuerySyntax;
15	using Rdmp.Core.DataFlowPipeline;
16	using Rdmp.Core.DataLoad.Engine.Job;
17	using Rdmp.Core.DataLoad.Triggers;
18	using Rdmp.Core.ReusableLibraryCode.Progress;
19
20	namespace Rdmp.Core.DataLoad.Engine.Migration.QueryBuilding;
21
22	/// <summary>
23	/// Migrates from STAGING to LIVE a single table (with a MigrationColumnSet). This is an UPSERT (new replaces old)
24	/// operation achieved (in SQL) with MERGE and
25	/// UPDATE (based on primary key). Both tables must be on the same server. A MERGE sql statement will be created
26	/// using LiveMigrationQueryHelper and executed
27	/// within a transaction.
28	/// </summary>
29	public class OverwriteMigrationStrategy : DatabaseMigrationStrategy
30	{
31	public OverwriteMigrationStrategy(IManagedConnection managedConnection)
32	: base(managedConnection)	48✔
33	{
34	}	48✔
35
36	public override void MigrateTable(IDataLoadJob job, MigrationColumnSet columnsToMigrate, int dataLoadInfoID,
37	GracefulCancellationToken cancellationToken, ref int inserts, ref int updates)
38	{
39	var server = columnsToMigrate.DestinationTable.Database.Server;	50✔
40
41	//see CrossDatabaseMergeCommandTest
42
43	/* ------------MIGRATE NEW RECORDS (novel by primary key)--------
44	*
45
46	INSERT INTO CrossDatabaseMergeCommandTo..ToTable (Name,Age,Postcode,hic_dataLoadRunID)
47	SELECT
48	[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Name,
49	[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Age,
50	[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Postcode,
51	1
52	FROM
53	[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING
54	left join
55	CrossDatabaseMergeCommandTo..ToTable
56	on
57	[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Age = CrossDatabaseMergeCommandTo..ToTable.Age
58	AND
59	[CrossDatabaseMergeCommandFrom]..CrossDatabaseMergeCommandTo_ToTable_STAGING.Name = CrossDatabaseMergeCommandTo..ToTable.Name
60	WHERE
61	CrossDatabaseMergeCommandTo..ToTable.Age is null
62	*/
63
64	var sbInsert = new StringBuilder();	50✔
65	var syntax = server.GetQuerySyntaxHelper();	50✔
66
67
68	sbInsert.AppendLine(	50✔
69	$"INSERT INTO {columnsToMigrate.DestinationTable.GetFullyQualifiedName()} ({string.Join(",", columnsToMigrate.FieldsToUpdate.Select(c => syntax.EnsureWrapped(c.GetRuntimeName())))}");	352✔
70
71	//if we are not ignoring the trigger then we should record the data load run ID
72	if (!job.LoadMetadata.IgnoreTrigger)	50✔
73	sbInsert.AppendLine($",{syntax.EnsureWrapped(SpecialFieldNames.DataLoadRunID)}");	44✔
74
75	sbInsert.AppendLine(")");	50✔
76
77	sbInsert.AppendLine("SELECT");	50✔
78
79	// Add the columns we are migrating
80	sbInsert.AppendLine(string.Join($",{Environment.NewLine}",	50✔
81	columnsToMigrate.FieldsToUpdate.Select(c => c.GetFullyQualifiedName())));	352✔
82
83	// If we are using trigger also add the run ID e.g. ",50"
84	if (!job.LoadMetadata.IgnoreTrigger)	50✔
85	sbInsert.AppendLine($",{dataLoadInfoID}");	44✔
86
87	sbInsert.AppendLine("FROM");	50✔
88	sbInsert.AppendLine(columnsToMigrate.SourceTable.GetFullyQualifiedName());	50✔
89	sbInsert.AppendLine("LEFT JOIN");	50✔
90	sbInsert.AppendLine(columnsToMigrate.DestinationTable.GetFullyQualifiedName());	50✔
91	sbInsert.AppendLine("ON");	50✔
92
93	sbInsert.AppendLine(	50✔
94	string.Join($" AND {Environment.NewLine}",	50✔
95	columnsToMigrate.PrimaryKeys.Select(	50✔
96	pk =>	50✔
97	string.Format("{0}.{1}={2}.{1}", columnsToMigrate.SourceTable.GetFullyQualifiedName(),	142✔
98	syntax.EnsureWrapped(pk.GetRuntimeName()),	142✔
99	columnsToMigrate.DestinationTable.GetFullyQualifiedName()))));	142✔
100
101	sbInsert.AppendLine("WHERE");	50✔
102	sbInsert.AppendLine(string.Join(" AND ",	50✔
103	columnsToMigrate.PrimaryKeys.Select(pk =>	50✔
104	$"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(pk.GetRuntimeName())} IS NULL")));	142✔
105
106	//right at the end of the SELECT
107	if (columnsToMigrate.DestinationTable.Database.Server.DatabaseType == DatabaseType.MySql)	50✔
108	sbInsert.Append(" FOR UPDATE");	18✔
109
110	var insertSql = sbInsert.ToString();	50✔
111
112	var cmd = server.GetCommand(insertSql, _managedConnection);	50✔
113	cmd.CommandTimeout = Timeout;	50✔
114
115	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	50✔
116	$"INSERT query: {Environment.NewLine}{insertSql}"));	50✔
117
118	cancellationToken.ThrowIfCancellationRequested();	50✔
119
120
121	try
122	{
123	inserts = cmd.ExecuteNonQuery();	50✔
124
125	var sqlLines = new List<CustomLine>();	50✔
126
127	var toSet = columnsToMigrate.FieldsToUpdate.Where(c => !c.IsPrimaryKey).Select(c =>	352✔
128	string.Format("t1.{0} = t2.{0}", syntax.EnsureWrapped(c.GetRuntimeName()))).ToArray();	260✔
129
130	if (!toSet.Any())	50✔
131	{
132	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	8✔
133	$"Table {columnsToMigrate.DestinationTable} is entirely composed of PrimaryKey columns or hic_ columns so UPDATE will NOT take place"));	8✔
134	return;	8✔
135	}
136
137	var toDiff = columnsToMigrate.FieldsToDiff.Where(c => !c.IsPrimaryKey).ToArray();	326✔
138
139	if (!toDiff.Any())	42✔
140	{
141	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,	2✔
142	$"Table {columnsToMigrate.DestinationTable} is entirely composed of PrimaryKey columns or hic_ columns/ other non DIFF columns that will not result in an UPDATE will NOT take place"));	2✔
143	return;	2✔
144	}
145
146	//t1.Name = t2.Name, t1.Age=T2.Age etc
147	sqlLines.Add(new CustomLine(string.Join(",", toSet), QueryComponent.SET));	40✔
148
149	//also update the hic_dataLoadRunID field
150	if (!job.LoadMetadata.IgnoreTrigger)	40✔
151	sqlLines.Add(new CustomLine(	34✔
152	$"t1.{syntax.EnsureWrapped(SpecialFieldNames.DataLoadRunID)}={dataLoadInfoID}",	34✔
153	QueryComponent.SET));	34✔
154
155	//t1.Name <> t2.Name AND t1.Age <> t2.Age etc
156	sqlLines.Add(new CustomLine(string.Join(" OR ", toDiff.Select(c => GetORLine(c, syntax))),	248✔
157	QueryComponent.WHERE));	40✔
158
159	//the join
160	sqlLines.AddRange(columnsToMigrate.PrimaryKeys.Select(p =>	40✔
161	new CustomLine(string.Format("t1.{0} = t2.{0}", syntax.EnsureWrapped(p.GetRuntimeName())),	112✔
162	QueryComponent.JoinInfoJoin)));	112✔
163
164	var updateHelper = columnsToMigrate.DestinationTable.Database.Server.GetQuerySyntaxHelper().UpdateHelper;	40✔
165
166	var updateQuery = updateHelper.BuildUpdate(	40✔
167	columnsToMigrate.DestinationTable,	40✔
168	columnsToMigrate.SourceTable,	40✔
169	sqlLines);	40✔
170
171	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,	40✔
172	$"Update query:{Environment.NewLine}{updateQuery}"));	40✔
173
174	var updateCmd = server.GetCommand(updateQuery, _managedConnection);	40✔
175	updateCmd.CommandTimeout = Timeout;	40✔
176	cancellationToken.ThrowIfCancellationRequested();	40✔
177
178	try
179	{
180	updates = updateCmd.ExecuteNonQuery();	40✔
181	}	40✔
182	catch (Exception e)	×
183	{
184	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,	×
185	$"Did not successfully perform the update queries: {updateQuery}", e));	×
186	throw new Exception($"Did not successfully perform the update queries: {updateQuery} - {e}");	×
187	}
188	}	40✔
189	catch (OperationCanceledException)	×
190	{
191	throw; // have to catch and rethrow this because of the catch-all below	×
192	}
193	catch (Exception e)	×
194	{
195	job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error,	×
196	$"Failed to migrate {columnsToMigrate.SourceTable} to {columnsToMigrate.DestinationTable}", e));	×
197	throw new Exception(	×
198	$"Failed to migrate {columnsToMigrate.SourceTable} to {columnsToMigrate.DestinationTable}: {e}");	×
199	}
200	}	50✔
201
202	private static string GetORLine(DiscoveredColumn c, IQuerySyntaxHelper syntax)
203	{
204	return string.Format(	208✔
205	"(t1.{0} <> t2.{0} OR (t1.{0} is null AND t2.{0} is not null) OR (t2.{0} is null AND t1.{0} is not null))",	208✔
206	syntax.EnsureWrapped(c.GetRuntimeName()));	208✔
207	}
208	}

HicServices / RDMP / 12032849274

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous