7194961165

Committed 13 Dec 2023 12:07PM UTC coverage: 56.776% (-0.2%) from 57.013%

Build # 7194961165

Build Type

push

github

Committed by

web-flow

Commit Message

Merge Latest Release into main (#1702)

* Bump YamlDotNet from 13.3.1 to 13.4.0

Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 13.3.1 to 13.4.0.
- [Release notes](https://github.com/aaubry/YamlDotNet/releases)
- [Commits](https://github.com/aaubry/YamlDotNet/compare/v13.3.1...v13.4.0)

---
updated-dependencies:
- dependency-name: YamlDotNet
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Bump shogo82148/actions-setup-perl from 1.23.1 to 1.24.1

Bumps [shogo82148/actions-setup-perl](https://github.com/shogo82148/actions-setup-perl) from 1.23.1 to 1.24.1.
- [Release notes](https://github.com/shogo82148/actions-setup-perl/releases)
- [Commits](https://github.com/shogo82148/actions-setup-perl/compare/v1.23.1...v1.24.1)

---
updated-dependencies:
- dependency-name: shogo82148/actions-setup-perl
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix checkbox issue

* improve confirmation text (#1639)

* improve confirmation text
* Loop tidyup, use var where possible

---------

Co-authored-by: jas88 <j.a.sutherland@dundee.ac.uk>

* correct typo in create logging sql (#1640)

* Feature/ci codescan (#1641)

* Move SecurityCodescan.VS2019 to run on Github CI alone, integrate results with CodeQL
* Remove SecurityCodescan from Packages.md, no longer used via Nuget

---------

Co-authored-by: James A Sutherland <j@sutherland.pw>

* hide source control when not available

* Remove old Plugin object bits, tidy up (#1636)

* Remove old Plugin object bits, tidy up

* Purge remaining bits of AllExpiredPluginsNode

* Fix plugin display name in tree

* Update CreateNewDataExtractionProjectUI.cs

Casting fix

* Feature/rdmp42 delete plugins (#1642)

* add ui plugin delete functionality

* Warning and inherita... (continued)

Run Details

10722 of 20351 branches covered (0.0%)

Branch coverage included in aggregate %.

215 of 789 new or added lines in 63 files covered. (27.25%)

39 existing lines in 16 files now uncovered.

30650 of 52518 relevant lines covered (58.36%)

7294.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.15

/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Globalization;
using System.IO;
using System.Linq;
using NPOI.XWPF.UserModel;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.Repositories;
using Rdmp.Core.Repositories.Managers;
using Rdmp.Core.ReusableLibraryCode.Annotations;

namespace Rdmp.Core.Reports.ExtractionTime;

/// <summary>
/// Generates a Microsoft Word DocX file containing information about all the datasets extracted (and released) as part of a Data Release.  This includes
/// row counts and unique patient counts as well as the number of patients in the original cohort (not all patients will appear in all datasets).  Also
/// included are the tickets for the project, the cohort ID number etc
/// </summary>
public class WordDataReleaseFileGenerator : DocXHelper
{
    private readonly IDataExportRepository _repository;
    public IExtractionConfiguration Configuration { get; set; }
    protected ICumulativeExtractionResults[] ExtractionResults { get; set; }
    protected IExtractableCohort Cohort { get; set; }
    protected IProject Project { get; set; }

    private const int CohortCountTimeoutInSeconds = 600; // 10 minutes

    public WordDataReleaseFileGenerator(IExtractionConfiguration configuration, IDataExportRepository repository)
    {
        _repository = repository;
        Configuration = configuration;
        Project = configuration.Project;

        if (Configuration.Cohort_ID == null)
            throw new NullReferenceException("Configuration has no Cohort");

        Cohort = _repository.GetObjectByID<ExtractableCohort>((int)Configuration.Cohort_ID);

        ExtractionResults =
            Configuration.CumulativeExtractionResults
                .OrderBy(
                    c => _repository.GetObjectByID<ExtractableDataSet>(c.ExtractableDataSet_ID).ToString()
                ).ToArray();
    }

    public void GenerateWordFile(string saveAsFilename)
    {
        var f = string.IsNullOrWhiteSpace(saveAsFilename)
            ? GetUniqueFilenameInWorkArea("ReleaseDocument")
            : new FileInfo(saveAsFilename);

        // Create an instance of Word  and make it visible.=
        using var document = GetNewDocFile(f);
        //actually changes it to landscape :)
        SetLandscape(document);

        InsertHeader(document, $"Project:{Project.Name}", 1);
        InsertHeader(document, Configuration.Name, 2);

        var disclaimer = _repository.DataExportPropertyManager.GetValue(DataExportProperty.ReleaseDocumentDisclaimer);

        if (disclaimer != null)
            InsertParagraph(document, disclaimer);

        CreateTopTable1(document);

        InsertParagraph(document, Environment.NewLine);

        CreateCohortDetailsTable(document);

        InsertParagraph(document, Environment.NewLine);

        CreateFileSummary(document);

        //interactive mode, user didn't ask us to save to a specific location so we created it in temp and so we can now show them where that file is
        if (string.IsNullOrWhiteSpace(saveAsFilename))
            ShowFile(f);
    }

    private void CreateTopTable1(XWPFDocument document)
    {
        var hasTicket = !string.IsNullOrWhiteSpace(Project.MasterTicket);
        var hasProchi = Cohort.GetReleaseIdentifier().ToLower().Contains("prochi");

        var currentRow = 0;
        var requiredRows = 1;

        if (hasProchi)
            requiredRows++;
        if (hasTicket)
            requiredRows++;

        var table = InsertTable(document, requiredRows, 2);

        if (hasTicket)
        {
            SetTableCell(table, currentRow, 0, "Master Issue");
            SetTableCell(table, currentRow, 1, Project.MasterTicket);
            currentRow++;
        }

        SetTableCell(table, currentRow, 0, "ReleaseIdentifier");
        SetTableCell(table, currentRow, 1, Cohort.GetReleaseIdentifier(true));
        currentRow++;

        if (hasProchi)
        {
            SetTableCell(table, currentRow, 0, "Prefix");
            SetTableCell(table, currentRow, 1, GetFirstProCHIPrefix());
        }
    }

    /// <summary>
    /// Returns the first 3 digits of the first release identifier in the cohort (this is very hic specific).
    /// </summary>
    /// <returns></returns>
    private string GetFirstProCHIPrefix()
    {
        var ect = Cohort.ExternalCohortTable;

        var db = ect.Discover();
        using var con = db.Server.GetConnection();
        con.Open();

        var sql =
            $"SELECT  TOP 1 LEFT({Cohort.GetReleaseIdentifier()},3) FROM {ect.TableName} WHERE {Cohort.WhereSQL()}";

        using var cmd = db.Server.GetCommand(sql, con);
        cmd.CommandTimeout = CohortCountTimeoutInSeconds;
        return (string)cmd.ExecuteScalar();
    }

    private void CreateCohortDetailsTable(XWPFDocument document)
    {
        var table = InsertTable(document, 2, 4);

        var tableLine = 0;

        SetTableCell(table, tableLine, 0, "Version");
        SetTableCell(table, tableLine, 1, "Description");
        SetTableCell(table, tableLine, 2, "Date Extracted");
        SetTableCell(table, tableLine, 3, "Unique Individuals");
        tableLine++;

        SetTableCell(table, tableLine, 0,
            Cohort.GetExternalData(CohortCountTimeoutInSeconds).ExternalVersion.ToString());
        SetTableCell(table, tableLine, 1,
            $"{Cohort} (ID={Cohort.ID}, OriginID={Cohort.OriginID})"); //description fetched from remote table

        var lastExtracted = ExtractionResults.Any()
            ? ExtractionResults.Max(r => r.DateOfExtraction).ToString(CultureInfo.CurrentCulture)
            : "Never";
        SetTableCell(table, tableLine, 2, lastExtracted);
        SetTableCell(table, tableLine, 3,
            Cohort.GetCountDistinctFromDatabase(CohortCountTimeoutInSeconds).ToString("N0"));
    }

    [NotNull]
    private string getDOI([NotNull] Curation.Data.Dataset ds)
    {
        return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : "";
    }

    private void CreateFileSummary(XWPFDocument document)
    {
        var table = InsertTable(document, ExtractionResults.Length + 1, 6);

        var tableLine = 0;

        SetTableCell(table, tableLine, 0, "Data Requirement");
        SetTableCell(table, tableLine, 1, "Notes");
        SetTableCell(table, tableLine, 2, "Filename");
        SetTableCell(table, tableLine, 3, "Records");
        SetTableCell(table, tableLine, 4, "Unique Individuals");
        SetTableCell(table, tableLine, 5, "Datasets");
        tableLine++;

        foreach (var result in ExtractionResults)
        {
            var filename = GetFileName(result);
            var extractableDataset = _repository.GetObjectByID<ExtractableDataSet>(result.ExtractableDataSet_ID);
            SetTableCell(table, tableLine, 0,
              extractableDataset.ToString());
            var linkedDatasets = extractableDataset.Catalogue.CatalogueItems.Select(static c => c.ColumnInfo).Where(ci => ci.Dataset_ID != null).Distinct().Select(ci => ci.Dataset_ID);
            var datasets = _repository.CatalogueRepository.GetAllObjects<Curation.Data.Dataset>().Where(d => linkedDatasets.Contains(d.ID)).ToList();
            var datasetString = string.Join("",datasets.Select(ds=> $"{ds.Name} {getDOI(ds)}, {Environment.NewLine}"));
            SetTableCell(table, tableLine, 1, result.FiltersUsed);
            SetTableCell(table, tableLine, 2, filename);
            SetTableCell(table, tableLine, 3, result.RecordsExtracted.ToString("N0"));
            SetTableCell(table, tableLine, 4, result.DistinctReleaseIdentifiersEncountered.ToString("N0"));
            SetTableCell(table, tableLine, 5, datasetString);
            tableLine++;
        }
    }

    private string GetFileName(ICumulativeExtractionResults result)
    {
        if (string.IsNullOrWhiteSpace(result?.DestinationDescription))
            return "";

        if (string.IsNullOrWhiteSpace(Project.ExtractionDirectory))
            return result.DestinationDescription;

        //can we express it relative
        if (result.DestinationDescription.StartsWith(Project.ExtractionDirectory,
                StringComparison.CurrentCultureIgnoreCase))
        {
            var relative = result.DestinationDescription[Project.ExtractionDirectory.Length..].Replace('\\', '/');

            return $"./{relative.Trim('/')}";
        }

        return result.DestinationDescription;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Globalization;
9	using System.IO;
10	using System.Linq;
11	using NPOI.XWPF.UserModel;
12	using Rdmp.Core.DataExport.Data;
13	using Rdmp.Core.Repositories;
14	using Rdmp.Core.Repositories.Managers;
15	using Rdmp.Core.ReusableLibraryCode.Annotations;
16
17	namespace Rdmp.Core.Reports.ExtractionTime;
18
19	/// <summary>
20	/// Generates a Microsoft Word DocX file containing information about all the datasets extracted (and released) as part of a Data Release. This includes
21	/// row counts and unique patient counts as well as the number of patients in the original cohort (not all patients will appear in all datasets). Also
22	/// included are the tickets for the project, the cohort ID number etc
23	/// </summary>
24	public class WordDataReleaseFileGenerator : DocXHelper
25	{
26	private readonly IDataExportRepository _repository;
27	public IExtractionConfiguration Configuration { get; set; }	40✔
28	protected ICumulativeExtractionResults[] ExtractionResults { get; set; }	38✔
29	protected IExtractableCohort Cohort { get; set; }	64✔
30	protected IProject Project { get; set; }	90✔
31
32	private const int CohortCountTimeoutInSeconds = 600; // 10 minutes
33
34	public WordDataReleaseFileGenerator(IExtractionConfiguration configuration, IDataExportRepository repository)	8✔
35	{
36	_repository = repository;	8✔
37	Configuration = configuration;	8✔
38	Project = configuration.Project;	8✔
39
40	if (Configuration.Cohort_ID == null)	8!
41	throw new NullReferenceException("Configuration has no Cohort");	×
42
43	Cohort = _repository.GetObjectByID<ExtractableCohort>((int)Configuration.Cohort_ID);	8✔
44
45	ExtractionResults =	8✔
46	Configuration.CumulativeExtractionResults	8✔
47	.OrderBy(	8✔
48	c => _repository.GetObjectByID<ExtractableDataSet>(c.ExtractableDataSet_ID).ToString()	22✔
49	).ToArray();	8✔
50	}	8✔
51
52	public void GenerateWordFile(string saveAsFilename)
53	{
54	var f = string.IsNullOrWhiteSpace(saveAsFilename)	8!
55	? GetUniqueFilenameInWorkArea("ReleaseDocument")	8✔
56	: new FileInfo(saveAsFilename);	8✔
57
58	// Create an instance of Word and make it visible.=
59	using var document = GetNewDocFile(f);	8✔
60	//actually changes it to landscape :)
61	SetLandscape(document);	8✔
62
63	InsertHeader(document, $"Project:{Project.Name}", 1);	8✔
64	InsertHeader(document, Configuration.Name, 2);	8✔
65
66	var disclaimer = _repository.DataExportPropertyManager.GetValue(DataExportProperty.ReleaseDocumentDisclaimer);	8✔
67
68	if (disclaimer != null)	8!
69	InsertParagraph(document, disclaimer);	×
70
71	CreateTopTable1(document);	8✔
72
73	InsertParagraph(document, Environment.NewLine);	8✔
74
75	CreateCohortDetailsTable(document);	8✔
76
77	InsertParagraph(document, Environment.NewLine);	8✔
78
79	CreateFileSummary(document);	8✔
80
81	//interactive mode, user didn't ask us to save to a specific location so we created it in temp and so we can now show them where that file is
82	if (string.IsNullOrWhiteSpace(saveAsFilename))	8!
83	ShowFile(f);	×
84	}	16✔
85
86	private void CreateTopTable1(XWPFDocument document)
87	{
88	var hasTicket = !string.IsNullOrWhiteSpace(Project.MasterTicket);	8✔
89	var hasProchi = Cohort.GetReleaseIdentifier().ToLower().Contains("prochi");	8✔
90
91	var currentRow = 0;	8✔
92	var requiredRows = 1;	8✔
93
94	if (hasProchi)	8!
95	requiredRows++;	×
96	if (hasTicket)	8!
97	requiredRows++;	×
98
99	var table = InsertTable(document, requiredRows, 2);	8✔
100
101	if (hasTicket)	8!
102	{
103	SetTableCell(table, currentRow, 0, "Master Issue");	×
104	SetTableCell(table, currentRow, 1, Project.MasterTicket);	×
105	currentRow++;	×
106	}
107
108	SetTableCell(table, currentRow, 0, "ReleaseIdentifier");	8✔
109	SetTableCell(table, currentRow, 1, Cohort.GetReleaseIdentifier(true));	8✔
110	currentRow++;	8✔
111
112	if (hasProchi)	8!
113	{
114	SetTableCell(table, currentRow, 0, "Prefix");	×
115	SetTableCell(table, currentRow, 1, GetFirstProCHIPrefix());	×
116	}
117	}	8✔
118
119	/// <summary>
120	/// Returns the first 3 digits of the first release identifier in the cohort (this is very hic specific).
121	/// </summary>
122	/// <returns></returns>
123	private string GetFirstProCHIPrefix()
124	{
125	var ect = Cohort.ExternalCohortTable;	×
126
127	var db = ect.Discover();	×
128	using var con = db.Server.GetConnection();	×
129	con.Open();	×
130
131	var sql =	×
132	$"SELECT TOP 1 LEFT({Cohort.GetReleaseIdentifier()},3) FROM {ect.TableName} WHERE {Cohort.WhereSQL()}";	×
133
134	using var cmd = db.Server.GetCommand(sql, con);	×
135	cmd.CommandTimeout = CohortCountTimeoutInSeconds;	×
136	return (string)cmd.ExecuteScalar();	×
137	}	×
138
139	private void CreateCohortDetailsTable(XWPFDocument document)
140	{
141	var table = InsertTable(document, 2, 4);	8✔
142
143	var tableLine = 0;	8✔
144
145	SetTableCell(table, tableLine, 0, "Version");	8✔
146	SetTableCell(table, tableLine, 1, "Description");	8✔
147	SetTableCell(table, tableLine, 2, "Date Extracted");	8✔
148	SetTableCell(table, tableLine, 3, "Unique Individuals");	8✔
149	tableLine++;	8✔
150
151	SetTableCell(table, tableLine, 0,	8✔
152	Cohort.GetExternalData(CohortCountTimeoutInSeconds).ExternalVersion.ToString());	8✔
153	SetTableCell(table, tableLine, 1,	8✔
154	$"{Cohort} (ID={Cohort.ID}, OriginID={Cohort.OriginID})"); //description fetched from remote table	8✔
155
156	var lastExtracted = ExtractionResults.Any()	8✔
157	? ExtractionResults.Max(r => r.DateOfExtraction).ToString(CultureInfo.CurrentCulture)	22✔
158	: "Never";	8✔
159	SetTableCell(table, tableLine, 2, lastExtracted);	8✔
160	SetTableCell(table, tableLine, 3,	8✔
161	Cohort.GetCountDistinctFromDatabase(CohortCountTimeoutInSeconds).ToString("N0"));	8✔
162	}	8✔
163
164	[NotNull]
165	private string getDOI([NotNull] Curation.Data.Dataset ds)
166	{
NEW 167	return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : "";	×
168	}
169
170	private void CreateFileSummary(XWPFDocument document)
171	{
172	var table = InsertTable(document, ExtractionResults.Length + 1, 6);	8✔
173
174	var tableLine = 0;	8✔
175
176	SetTableCell(table, tableLine, 0, "Data Requirement");	8✔
177	SetTableCell(table, tableLine, 1, "Notes");	8✔
178	SetTableCell(table, tableLine, 2, "Filename");	8✔
179	SetTableCell(table, tableLine, 3, "Records");	8✔
180	SetTableCell(table, tableLine, 4, "Unique Individuals");	8✔
181	SetTableCell(table, tableLine, 5, "Datasets");	8✔
182	tableLine++;	8✔
183
184	foreach (var result in ExtractionResults)	60✔
185	{
186	var filename = GetFileName(result);	22✔
187	var extractableDataset = _repository.GetObjectByID<ExtractableDataSet>(result.ExtractableDataSet_ID);	22✔
188	SetTableCell(table, tableLine, 0,	22✔
189	extractableDataset.ToString());	22✔
190	var linkedDatasets = extractableDataset.Catalogue.CatalogueItems.Select(static c => c.ColumnInfo).Where(ci => ci.Dataset_ID != null).Distinct().Select(ci => ci.Dataset_ID);	22✔
191	var datasets = _repository.CatalogueRepository.GetAllObjects<Curation.Data.Dataset>().Where(d => linkedDatasets.Contains(d.ID)).ToList();	22✔
192	var datasetString = string.Join("",datasets.Select(ds=> $"{ds.Name} {getDOI(ds)}, {Environment.NewLine}"));	22✔
193	SetTableCell(table, tableLine, 1, result.FiltersUsed);	22✔
194	SetTableCell(table, tableLine, 2, filename);	22✔
195	SetTableCell(table, tableLine, 3, result.RecordsExtracted.ToString("N0"));	22✔
196	SetTableCell(table, tableLine, 4, result.DistinctReleaseIdentifiersEncountered.ToString("N0"));	22✔
197	SetTableCell(table, tableLine, 5, datasetString);	22✔
198	tableLine++;	22✔
199	}
200	}	8✔
201
202	private string GetFileName(ICumulativeExtractionResults result)
203	{
204	if (string.IsNullOrWhiteSpace(result?.DestinationDescription))	22!
205	return "";	×
206
207	if (string.IsNullOrWhiteSpace(Project.ExtractionDirectory))	22!
208	return result.DestinationDescription;	×
209
210	//can we express it relative
211	if (result.DestinationDescription.StartsWith(Project.ExtractionDirectory,	22!
212	StringComparison.CurrentCultureIgnoreCase))	22✔
213	{
214	var relative = result.DestinationDescription[Project.ExtractionDirectory.Length..].Replace('\\', '/');	22✔
215
216	return $"./{relative.Trim('/')}";	22✔
217	}
218
219	return result.DestinationDescription;	×
220	}
221	}

HicServices / RDMP / 7194961165

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous