• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

HicServices / RDMP / 12136273647

03 Dec 2024 09:03AM UTC coverage: 57.448% (+0.07%) from 57.382%
12136273647

push

github

JFriel
Merge branch 'develop' of https://github.com/HicServices/RDMP

11297 of 21215 branches covered (53.25%)

Branch coverage included in aggregate %.

439 of 673 new or added lines in 30 files covered. (65.23%)

5 existing lines in 4 files now uncovered.

32146 of 54407 relevant lines covered (59.08%)

17095.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

16.02
/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs
1
// Copyright (c) The University of Dundee 2024-2024
2
// This file is part of the Research Data Management Platform (RDMP).
3
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6

7
using Rdmp.Core.CommandExecution;
8
using Rdmp.Core.DataExport.Data;
9
using Rdmp.Core.DataLoad.Triggers;
10
using Rdmp.Core.DataViewing;
11
using Rdmp.Core.Logging;
12
using Rdmp.Core.QueryBuilding;
13
using Rdmp.Core.Repositories;
14
using Rdmp.Core.ReusableLibraryCode.DataAccess;
15
using System;
16
using System.Collections;
17
using System.Collections.Generic;
18
using System.Data;
19
using System.Linq;
20

21
namespace Rdmp.Core.Curation.Data.Overview;
22

23
/// <summary>
24
/// Used to populate information about a catalogue for use in the overview UI
25
/// </summary>
26
public class OverviewModel
27
{
28

29
    private readonly ICatalogue _catalogue;
30
    private readonly IBasicActivateItems _activator;
31

32
    private DataTable _dataLoads;
33

34
    private int _numberOfPeople;
35
    private int _numberOfRecords;
36

37
    public OverviewModel(IBasicActivateItems activator, ICatalogue catalogue)
2✔
38
    {
39
        _activator = activator;
2✔
40
        _catalogue = catalogue;
2✔
41
        if (catalogue != null)
2✔
42
        {
43
            Regen("");
2✔
44
        }
45
    }
2✔
46

47
    public void Regen(string whereClause)
48
    {
49
        DataTable dt = new();
2✔
50
        bool hasExtractionIdentifier = true;
2✔
51
        var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault();
2✔
52
        if (column is null)
2✔
53
        {
54
            column = _catalogue.CatalogueItems.FirstOrDefault();
2✔
55
            hasExtractionIdentifier = false;
2✔
56
        }
57
        if (column is null) return;
4!
NEW
58
        var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
×
NEW
59
        var server = discoveredColumn.Table.Database.Server;
×
NEW
60
        using var con = server.GetConnection();
×
NEW
61
        con.Open();
×
NEW
62
        string populatedWhere = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : "";
×
NEW
63
        var sql = $"SELECT {column.ColumnInfo.GetRuntimeName()} FROM {discoveredColumn.Table.GetRuntimeName()} {populatedWhere}";
×
NEW
64
        using var cmd = server.GetCommand(sql, con);
×
NEW
65
        cmd.CommandTimeout = 30000;
×
NEW
66
        using var da = server.GetDataAdapter(cmd);
×
NEW
67
        dt.BeginLoadData();
×
NEW
68
        da.Fill(dt);
×
NEW
69
        dt.EndLoadData();
×
NEW
70
        con.Dispose();
×
NEW
71
        _numberOfRecords = dt.Rows.Count;
×
NEW
72
        _numberOfPeople = hasExtractionIdentifier ? dt.DefaultView.ToTable(true, column.ColumnInfo.GetRuntimeName()).Rows.Count : 0;
×
NEW
73
        GetDataLoads();
×
NEW
74
        dt.Dispose();
×
NEW
75
    }
×
76

77
    public int GetNumberOfRecords()
78
    {
NEW
79
        return _numberOfRecords;
×
80
    }
81

82
    public int GetNumberOfPeople()
83
    {
NEW
84
        return _numberOfPeople;
×
85
    }
86

87
    public Tuple<DateTime, DateTime> GetStartEndDates(ColumnInfo dateColumn, string whereClause)
88
    {
NEW
89
        DataTable dt = new();
×
90

NEW
91
        var discoveredColumn = _catalogue.CatalogueItems.First().ColumnInfo.Discover(DataAccessContext.InternalDataProcessing);
×
NEW
92
        var server = discoveredColumn.Table.Database.Server;
×
NEW
93
        var populatedWhereClause = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : "";
×
NEW
94
        using var con = server.GetConnection();
×
NEW
95
        con.Open();
×
NEW
96
        if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer)
×
97
        {
NEW
98
            var sql = $@"
×
NEW
99
        select min({dateColumn.GetRuntimeName()}) as min, max({dateColumn.GetRuntimeName()}) as max
×
NEW
100
        from
×
NEW
101
        (select {dateColumn.GetRuntimeName()},
×
NEW
102
        count(1) over (partition by year({dateColumn.GetRuntimeName()})) as occurs 
×
NEW
103
        from {discoveredColumn.Table.GetRuntimeName()} {populatedWhereClause}) as t
×
NEW
104
        where occurs >1
×
NEW
105
        ";
×
106

NEW
107
            using var cmd = server.GetCommand(sql, con);
×
NEW
108
            cmd.CommandTimeout = 30000;
×
NEW
109
            using var da = server.GetDataAdapter(cmd);
×
NEW
110
            dt.BeginLoadData();
×
NEW
111
            da.Fill(dt);
×
NEW
112
            dt.EndLoadData();
×
113
        }
114
        else
115
        {
NEW
116
            var repo = new MemoryCatalogueRepository();
×
NEW
117
            var qb = new QueryBuilder(null, null);
×
NEW
118
            qb.AddColumn(new ColumnInfoToIColumn(repo, dateColumn));
×
NEW
119
            qb.AddCustomLine($"{dateColumn.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
×
NEW
120
            var cmd = server.GetCommand(qb.SQL, con);
×
NEW
121
            using var da = server.GetDataAdapter(cmd);
×
NEW
122
            dt.BeginLoadData();
×
NEW
123
            da.Fill(dt);
×
NEW
124
            var latest = dt.AsEnumerable()
×
NEW
125
               .Max(r => r.Field<DateTime>(dateColumn.Name));
×
NEW
126
            var earliest = dt.AsEnumerable()
×
NEW
127
              .Min(r => r.Field<DateTime>(dateColumn.Name));
×
NEW
128
            dt = new();
×
NEW
129
            dt.Rows.Add([earliest, latest]);
×
130
        }
NEW
131
        con.Dispose();
×
NEW
132
        return new Tuple<DateTime, DateTime>(DateTime.Parse(dt.Rows[0].ItemArray[0].ToString()), DateTime.Parse(dt.Rows[0].ItemArray[1].ToString()));
×
NEW
133
    }
×
134

135

136
    public static DataTable GetCountsByDatePeriod(ColumnInfo dateColumn, string datePeriod, string optionalWhere = "")
137
    {
NEW
138
        DataTable dt = new();
×
NEW
139
        if (!(new[] { "Day", "Month", "Year" }).Contains(datePeriod))
×
140
        {
NEW
141
            throw new Exception("Invalid Date period");
×
142
        }
NEW
143
        var discoveredColumn = dateColumn.Discover(DataAccessContext.InternalDataProcessing);
×
NEW
144
        var server = discoveredColumn.Table.Database.Server;
×
NEW
145
        using var con = server.GetConnection();
×
NEW
146
        con.Open();
×
NEW
147
        var dateString = "yyyy-MM";
×
148
        switch (datePeriod)
149
        {
150
            case "Day":
NEW
151
                dateString = "yyyy-MM-dd";
×
NEW
152
                break;
×
153
            case "Month":
NEW
154
                dateString = "yyyy-MM";
×
NEW
155
                break;
×
156
            case "Year":
NEW
157
                dateString = "yyyy";
×
158
                break;
159
        }
NEW
160
        if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer)
×
161
        {
NEW
162
            var sql = @$"
×
NEW
163
        SELECT format({dateColumn.GetRuntimeName()}, '{dateString}') as YearMonth, count(*) as '# Records'
×
NEW
164
        FROM {discoveredColumn.Table.GetRuntimeName()}
×
NEW
165
        WHERE {dateColumn.GetRuntimeName()} IS NOT NULL
×
NEW
166
        {(optionalWhere != "" ? "AND" : "")} {optionalWhere.Replace('"', '\'')}
×
NEW
167
        GROUP BY format({dateColumn.GetRuntimeName()}, '{dateString}')
×
NEW
168
        ORDER BY 1
×
NEW
169
        ";
×
170

NEW
171
            using var cmd = server.GetCommand(sql, con);
×
NEW
172
            cmd.CommandTimeout = 30000;
×
NEW
173
            using var da = server.GetDataAdapter(cmd);
×
NEW
174
            dt.BeginLoadData();
×
NEW
175
            da.Fill(dt);
×
NEW
176
            dt.EndLoadData();
×
177
        }
178
        else
179
        {
NEW
180
            var repo = new MemoryCatalogueRepository();
×
NEW
181
            var qb = new QueryBuilder(null, null);
×
NEW
182
            qb.AddColumn(new ColumnInfoToIColumn(repo, dateColumn));
×
NEW
183
            qb.AddCustomLine($"{dateColumn.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
×
NEW
184
            var cmd = server.GetCommand(qb.SQL, con);
×
NEW
185
            using var da = server.GetDataAdapter(cmd);
×
NEW
186
            dt.BeginLoadData();
×
NEW
187
            da.Fill(dt);
×
NEW
188
            Dictionary<string, int> counts = [];
×
NEW
189
            foreach (var key in dt.AsEnumerable().Select(row => DateTime.Parse(row.ItemArray[0].ToString()).ToString(dateString)))
×
190
            {
NEW
191
                counts[key]++;
×
192
            }
NEW
193
            dt = new DataTable();
×
NEW
194
            foreach (var item in counts)
×
195
            {
NEW
196
                DataRow dr = dt.NewRow();
×
NEW
197
                dr["YearMonth"] = item.Key;
×
NEW
198
                dr["# Records"] = item.Value;
×
NEW
199
                dt.Rows.Add(dr);
×
200
            }
NEW
201
            dt.EndLoadData();
×
202

203
        }
NEW
204
        con.Dispose();
×
NEW
205
        return dt;
×
NEW
206
    }
×
207

208
    private void GetDataLoads()
209
    {
210
        _dataLoads = new();
2✔
211
        var repo = new MemoryCatalogueRepository();
2✔
212
        var qb = new QueryBuilder(null, null);
2✔
213
        var columnInfo = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).FirstOrDefault();
2✔
214
        if (columnInfo != null)
2!
215
        {
NEW
216
            qb.AddColumn(new ColumnInfoToIColumn(repo, columnInfo));
×
NEW
217
            qb.AddCustomLine($"{columnInfo.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE);
×
NEW
218
            var sql = qb.SQL;
×
NEW
219
            var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server;
×
NEW
220
            using var con = server.GetConnection();
×
NEW
221
            con.Open();
×
222

NEW
223
            using var cmd = server.GetCommand(sql, con);
×
NEW
224
            cmd.CommandTimeout = 30000;
×
NEW
225
            using var da = server.GetDataAdapter(cmd);
×
NEW
226
            _dataLoads.BeginLoadData();
×
NEW
227
            da.Fill(_dataLoads);
×
NEW
228
            _dataLoads.EndLoadData();
×
229
        }
230

231
    }
2✔
232

233
    public DataTable GetMostRecentDataLoad()
234
    {
235
        if (_dataLoads == null) GetDataLoads();
6✔
236
        if (_dataLoads.Rows.Count == 0) return null;
8!
NEW
237
        var maxDataLoadId = _dataLoads.AsEnumerable().Select(r => int.Parse(r[0].ToString())).Distinct().Max();
×
NEW
238
        var loggingServers = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere<ExternalDatabaseServer>("CreatedByAssembly", "Rdmp.Core/Databases.LoggingDatabase");
×
NEW
239
        var columnInfo = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).First();
×
NEW
240
        var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server;
×
241

NEW
242
        DataTable dt = new();
×
NEW
243
        foreach (var loggingServer in loggingServers)
×
244
        {
NEW
245
            var logCollection = new ViewLogsCollection(loggingServer, new LogViewerFilter(LoggingTables.DataLoadRun));
×
NEW
246
            var dataLoadRunSql = $"{logCollection.GetSql()} WHERE ID={maxDataLoadId}";
×
NEW
247
            var logServer = loggingServer.Discover(DataAccessContext.InternalDataProcessing).Server;
×
NEW
248
            using var loggingCon = logServer.GetConnection();
×
NEW
249
            loggingCon.Open();
×
NEW
250
            using var loggingCmd = logServer.GetCommand(dataLoadRunSql, loggingCon);
×
NEW
251
            loggingCmd.CommandTimeout = 30000;
×
NEW
252
            using var loggingDa = server.GetDataAdapter(loggingCmd);
×
NEW
253
            dt.BeginLoadData();
×
NEW
254
            loggingDa.Fill(dt);
×
NEW
255
            dt.EndLoadData();
×
NEW
256
            loggingCon.Dispose();
×
NEW
257
            if (dt.Rows.Count > 0)
×
258
            {
NEW
259
                break;
×
260
            }
261
        }
NEW
262
        return dt;
×
263
    }
264

265
    public List<CumulativeExtractionResults> GetExtractions()
266
    {
267
        var datasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere<ExtractableDataSet>("Catalogue_ID", _catalogue.ID).Select(d => d.ID);
4✔
268
        var results = _activator.RepositoryLocator.DataExportRepository.GetAllObjects<CumulativeExtractionResults>().Where(result => datasets.Contains(result.ExtractableDataSet_ID)).ToList();
16✔
269
        return results;
4✔
270

271
    }
272

273
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc