• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 12427673626

20 Dec 2024 07:55AM UTC coverage: 57.448%. Remained the same
12427673626

push

github

web-flow
Fix up some codeql/inspection code issues (#2087)

* Update OverviewModel.cs

Fix up some .Dispose/using issues, make finding most recent load ID more efficient

* LINQ tidying

* CodeQL fixups

* Update Catalogue.cs

Add Hashcode, Equals methods.

* Update Catalogue.cs

Tweak equality semantics for RDMP DB oddities

11301 of 21223 branches covered (53.25%)

Branch coverage included in aggregate %.

35 of 95 new or added lines in 13 files covered. (36.84%)

5 existing lines in 2 files now uncovered.

32153 of 54417 relevant lines covered (59.09%)

8580.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

16.18
/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs
1
// Copyright (c) The University of Dundee 2024-2024
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
using Rdmp.Core.CommandExecution;
8
using Rdmp.Core.DataExport.Data;
9
using Rdmp.Core.DataLoad.Triggers;
10
using Rdmp.Core.DataViewing;
11
using Rdmp.Core.Logging;
12
using Rdmp.Core.QueryBuilding;
13
using Rdmp.Core.Repositories;
14
using Rdmp.Core.ReusableLibraryCode.DataAccess;
15
using System;
16
using System.Collections.Generic;
17
using System.Data;
18
using System.Linq;
19

20
namespace Rdmp.Core.Curation.Data.Overview;
21

22
/// <summary>
23
/// Used to populate information about a catalogue for use in the overview UI
24
/// </summary>
25
public class OverviewModel
26
{
27
    private readonly ICatalogue _catalogue;
28
    private readonly IBasicActivateItems _activator;
29

30
    private DataTable _dataLoads;
31

32
    private int _numberOfPeople;
33
    private int _numberOfRecords;
34

35
    public OverviewModel(IBasicActivateItems activator, ICatalogue catalogue)
1✔
36
    {
37
        _activator = activator;
1✔
38
        _catalogue = catalogue;
1✔
39
        if (catalogue != null)
1✔
40
        {
41
            Regen("");
1✔
42
        }
43
    }
1✔
44

45
    public void Regen(string whereClause)
46
    {
47
        using var dt = new DataTable();
1✔
48
        var hasExtractionIdentifier = true;
1✔
49
        var column = _catalogue.CatalogueItems.FirstOrDefault(static ci => ci.ExtractionInformation.IsExtractionIdentifier);
1✔
50
        if (column is null)
1✔
51
        {
52
            column = _catalogue.CatalogueItems.FirstOrDefault();
1✔
53
            hasExtractionIdentifier = false;
1✔
54
        }
55

56
        if (column is null) return;
2!
57

58
        var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
×
59
        var server = discoveredColumn.Table.Database.Server;
×
60
        using var con = server.GetConnection();
×
61
        con.Open();
×
NEW
62
        var populatedWhere = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : "";
×
63
        var sql = $"SELECT {column.ColumnInfo.GetRuntimeName()} FROM {discoveredColumn.Table.GetRuntimeName()} {populatedWhere}";
×
64
        using var cmd = server.GetCommand(sql, con);
×
65
        cmd.CommandTimeout = 30000;
×
66
        using var da = server.GetDataAdapter(cmd);
×
67
        dt.BeginLoadData();
×
68
        da.Fill(dt);
×
69
        dt.EndLoadData();
×
70
        _numberOfRecords = dt.Rows.Count;
×
NEW
71
        _numberOfPeople = hasExtractionIdentifier
×
NEW
72
            ? dt.AsEnumerable().Select(r => r[column.ColumnInfo.GetRuntimeName()]).Distinct().Count()
×
NEW
73
            : 0;
×
74
        GetDataLoads();
×
75
    }
1✔
76

77
    public int GetNumberOfRecords()
78
    {
79
        return _numberOfRecords;
×
80
    }
81

82
    public int GetNumberOfPeople()
83
    {
84
        return _numberOfPeople;
×
85
    }
86

87
    public Tuple<DateTime, DateTime> GetStartEndDates(ColumnInfo dateColumn, string whereClause)
88
    {
NEW
89
        using var dt = new DataTable();
×
90

91
        var discoveredColumn = _catalogue.CatalogueItems.First().ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
×
92
        var server = discoveredColumn.Table.Database.Server;
×
93
        var populatedWhereClause = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : "";
×
94
        using var con = server.GetConnection();
×
95
        con.Open();
×
96
        if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer)
×
97
        {
98
            var sql = $@"
×
99
        select min({dateColumn.GetRuntimeName()}) as min, max({dateColumn.GetRuntimeName()}) as max
×
100
        from
×
101
        (select {dateColumn.GetRuntimeName()},
×
102
        count(1) over (partition by year({dateColumn.GetRuntimeName()})) as occurs 
×
103
        from {discoveredColumn.Table.GetRuntimeName()} {populatedWhereClause}) as t
×
104
        where occurs >1
×
105
        ";
×
106

107
            using var cmd = server.GetCommand(sql, con);
×
108
            cmd.CommandTimeout = 30000;
×
109
            using var da = server.GetDataAdapter(cmd);
×
110
            dt.BeginLoadData();
×
111
            da.Fill(dt);
×
112
            dt.EndLoadData();
×
113
        }
114
        else
115
        {
116
            var repo = new MemoryCatalogueRepository();
×
117
            var qb = new QueryBuilder(null, null);
×
118
            qb.AddColumn(new ColumnInfoToIColumn(repo, dateColumn));
×
119
            qb.AddCustomLine($"{dateColumn.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
×
120
            var cmd = server.GetCommand(qb.SQL, con);
×
121
            using var da = server.GetDataAdapter(cmd);
×
122
            dt.BeginLoadData();
×
123
            da.Fill(dt);
×
124
            var latest = dt.AsEnumerable()
×
NEW
125
                .Max(r => r.Field<DateTime>(dateColumn.Name));
×
126
            var earliest = dt.AsEnumerable()
×
NEW
127
                .Min(r => r.Field<DateTime>(dateColumn.Name));
×
NEW
128
            dt.Rows.Clear();
×
NEW
129
            dt.Rows.Add(earliest, latest);
×
130
        }
131

132
        return new Tuple<DateTime, DateTime>(DateTime.Parse(dt.Rows[0].ItemArray[0].ToString()), DateTime.Parse(dt.Rows[0].ItemArray[1].ToString()));
×
133
    }
×
134

135

136
    public static DataTable GetCountsByDatePeriod(ColumnInfo dateColumn, string datePeriod, string optionalWhere = "")
137
    {
NEW
138
        var dt = new DataTable();
×
139
        if (!(new[] { "Day", "Month", "Year" }).Contains(datePeriod))
×
140
        {
141
            throw new Exception("Invalid Date period");
×
142
        }
143

144
        var discoveredColumn = dateColumn.Discover(DataAccessContext.InternalDataProcessing);
×
145
        var server = discoveredColumn.Table.Database.Server;
×
146
        using var con = server.GetConnection();
×
147
        con.Open();
×
NEW
148
        var dateString = datePeriod switch
×
UNCOV
149
        {
×
NEW
150
            "Day" => "yyyy-MM-dd",
×
NEW
151
            "Month" => "yyyy-MM",
×
NEW
152
            "Year" => "yyyy",
×
NEW
153
            _ => "yyyy-MM"
×
NEW
154
        };
×
155

UNCOV
156
        if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer)
×
157
        {
158
            var sql = @$"
×
159
        SELECT format({dateColumn.GetRuntimeName()}, '{dateString}') as YearMonth, count(*) as '# Records'
×
160
        FROM {discoveredColumn.Table.GetRuntimeName()}
×
161
        WHERE {dateColumn.GetRuntimeName()} IS NOT NULL
×
162
        {(optionalWhere != "" ? "AND" : "")} {optionalWhere.Replace('"', '\'')}
×
163
        GROUP BY format({dateColumn.GetRuntimeName()}, '{dateString}')
×
164
        ORDER BY 1
×
165
        ";
×
166

167
            using var cmd = server.GetCommand(sql, con);
×
168
            cmd.CommandTimeout = 30000;
×
169
            using var da = server.GetDataAdapter(cmd);
×
170
            dt.BeginLoadData();
×
171
            da.Fill(dt);
×
172
            dt.EndLoadData();
×
173
        }
174
        else
175
        {
176
            var repo = new MemoryCatalogueRepository();
×
177
            var qb = new QueryBuilder(null, null);
×
178
            qb.AddColumn(new ColumnInfoToIColumn(repo, dateColumn));
×
179
            qb.AddCustomLine($"{dateColumn.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
×
180
            var cmd = server.GetCommand(qb.SQL, con);
×
181
            using var da = server.GetDataAdapter(cmd);
×
182
            dt.BeginLoadData();
×
183
            da.Fill(dt);
×
184
            Dictionary<string, int> counts = [];
×
185
            foreach (var key in dt.AsEnumerable().Select(row => DateTime.Parse(row.ItemArray[0].ToString()).ToString(dateString)))
×
186
            {
187
                counts[key]++;
×
188
            }
189

190
            dt = new DataTable();
×
191
            foreach (var item in counts)
×
192
            {
NEW
193
                var dr = dt.NewRow();
×
194
                dr["YearMonth"] = item.Key;
×
195
                dr["# Records"] = item.Value;
×
196
                dt.Rows.Add(dr);
×
197
            }
198

NEW
199
            dt.EndLoadData();
×
200
        }
201

202
        return dt;
×
203
    }
×
204

205
    private void GetDataLoads()
206
    {
207
        _dataLoads = new DataTable();
1✔
208
        var repo = new MemoryCatalogueRepository();
1✔
209
        var qb = new QueryBuilder(null, null);
1✔
210
        var columnInfo = _catalogue.CatalogueItems.Where(static c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).FirstOrDefault();
1✔
211
        if (columnInfo == null) return;
2!
212

NEW
213
        qb.AddColumn(new ColumnInfoToIColumn(repo, columnInfo));
×
NEW
214
        qb.AddCustomLine($"{columnInfo.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
×
NEW
215
        var sql = qb.SQL;
×
NEW
216
        var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server;
×
NEW
217
        using var con = server.GetConnection();
×
NEW
218
        con.Open();
×
219

NEW
220
        using var cmd = server.GetCommand(sql, con);
×
NEW
221
        cmd.CommandTimeout = 30000;
×
NEW
222
        using var da = server.GetDataAdapter(cmd);
×
NEW
223
        _dataLoads.BeginLoadData();
×
NEW
224
        da.Fill(_dataLoads);
×
NEW
225
        _dataLoads.EndLoadData();
×
UNCOV
226
    }
×
227

228
    public DataTable GetMostRecentDataLoad()
229
    {
230
        if (_dataLoads == null) GetDataLoads();
3✔
231
        if (_dataLoads.Rows.Count == 0) return null;
4!
232

NEW
233
        var maxDataLoadId = _dataLoads.AsEnumerable().Select(static r => int.Parse(r[0].ToString())).Max();
×
234
        var loggingServers = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<ExternalDatabaseServer>("CreatedByAssembly", "Rdmp.Core/Databases.LoggingDatabase");
×
235
        var columnInfo = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).First();
×
236
        var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server;
×
237

238
        DataTable dt = new();
×
239
        foreach (var loggingServer in loggingServers)
×
240
        {
241
            var logCollection = new ViewLogsCollection(loggingServer, new LogViewerFilter(LoggingTables.DataLoadRun));
×
242
            var dataLoadRunSql = $"{logCollection.GetSql()} WHERE ID={maxDataLoadId}";
×
243
            var logServer = loggingServer.Discover(DataAccessContext.InternalDataProcessing).Server;
×
244
            using var loggingCon = logServer.GetConnection();
×
245
            loggingCon.Open();
×
246
            using var loggingCmd = logServer.GetCommand(dataLoadRunSql, loggingCon);
×
247
            loggingCmd.CommandTimeout = 30000;
×
248
            using var loggingDa = server.GetDataAdapter(loggingCmd);
×
249
            dt.BeginLoadData();
×
250
            loggingDa.Fill(dt);
×
251
            dt.EndLoadData();
×
252
            if (dt.Rows.Count > 0)
×
253
            {
254
                break;
×
255
            }
256
        }
257

UNCOV
258
        return dt;
×
259
    }
260

261
    public List<CumulativeExtractionResults> GetExtractions()
262
    {
263
        var datasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", _catalogue.ID).Select(d => d.ID);
2✔
264
        var results = _activator.RepositoryLocator.DataExportRepository.GetAllObjects<CumulativeExtractionResults>().Where(result => datasets.Contains(result.ExtractableDataSet_ID)).ToList();
8✔
265
        return results;
2✔
266
    }
267
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc