9757494962

Committed 02 Jul 2024 08:23AM UTC coverage: 56.679% (-0.2%) from 56.914%

Build # 9757494962

Build Type

Pull #1867

github

Committed by

web-flow

Commit Message

Merge branch 'develop' into release/8.2.0

Pull Request Pull Request #1867: Release/8.2.0

Run Details

10912 of 20750 branches covered (52.59%)

Branch coverage included in aggregate %.

369 of 831 new or added lines in 38 files covered. (44.4%)

375 existing lines in 25 files now uncovered.

30965 of 53135 relevant lines covered (58.28%)

7847.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

55.78

/Rdmp.Core/Reports/ExtractionTime/WordDataReleaseFileGenerator.cs

// Copyright (c) The University of Dundee 2018-2019
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Globalization;
using System.IO;
using System.Linq;
using NPOI.XWPF.UserModel;
using Rdmp.Core.DataExport.Data;
using Rdmp.Core.Repositories;
using Rdmp.Core.Repositories.Managers;
using Rdmp.Core.ReusableLibraryCode.Annotations;

namespace Rdmp.Core.Reports.ExtractionTime;

/// <summary>
/// Generates a Microsoft Word DocX file containing information about all the datasets extracted (and released) as part of a Data Release.  This includes
/// row counts and unique patient counts as well as the number of patients in the original cohort (not all patients will appear in all datasets).  Also
/// included are the tickets for the project, the cohort ID number etc
/// </summary>
public class WordDataReleaseFileGenerator : DocXHelper
{
    private readonly IDataExportRepository _repository;
    public IExtractionConfiguration Configuration { get; set; }
    protected ICumulativeExtractionResults[] ExtractionResults { get; set; }
    protected IExtractableCohort Cohort { get; set; }
    protected IProject Project { get; set; }

    private const int CohortCountTimeoutInSeconds = 600; // 10 minutes

    public WordDataReleaseFileGenerator(IExtractionConfiguration configuration, IDataExportRepository repository)
    {
        _repository = repository;
        Configuration = configuration;
        Project = configuration.Project;

        if (Configuration.Cohort_ID == null)
            throw new NullReferenceException("Configuration has no Cohort");

        Cohort = _repository.GetObjectByID<ExtractableCohort>((int)Configuration.Cohort_ID);

        ExtractionResults =
            Configuration.CumulativeExtractionResults
                .OrderBy(
                    c => _repository.GetObjectByID<ExtractableDataSet>(c.ExtractableDataSet_ID).ToString()
                ).ToArray();
    }

    public void GenerateWordFile(string saveAsFilename)
    {
        var f = string.IsNullOrWhiteSpace(saveAsFilename)
            ? GetUniqueFilenameInWorkArea("ReleaseDocument")
            : new FileInfo(saveAsFilename);

        // Create an instance of Word  and make it visible.=
        using var document = GetNewDocFile(f);
        //actually changes it to landscape :)
        SetLandscape(document);

        InsertHeader(document, $"Project:{Project.Name}", 1);
        InsertHeader(document, Configuration.Name, 2);

        var disclaimer = _repository.DataExportPropertyManager.GetValue(DataExportProperty.ReleaseDocumentDisclaimer);

        if (disclaimer != null)
            InsertParagraph(document, disclaimer);

        CreateTopTable1(document);

        InsertParagraph(document, Environment.NewLine);

        CreateCohortDetailsTable(document);

        InsertParagraph(document, Environment.NewLine);

        CreateFileSummary(document);

        //interactive mode, user didn't ask us to save to a specific location so we created it in temp and so we can now show them where that file is
        if (string.IsNullOrWhiteSpace(saveAsFilename))
            ShowFile(f);
    }

    private void CreateTopTable1(XWPFDocument document)
    {
        var hasTicket = !string.IsNullOrWhiteSpace(Project.MasterTicket);
        var hasProchi = Cohort.GetReleaseIdentifier().ToLower().Contains("prochi");

        var currentRow = 0;
        var requiredRows = 1;

        if (hasProchi)
            requiredRows++;
        if (hasTicket)
            requiredRows++;

        var table = InsertTable(document, requiredRows, 2);

        if (hasTicket)
        {
            SetTableCell(table, currentRow, 0, "Master Issue");
            SetTableCell(table, currentRow, 1, Project.MasterTicket);
            currentRow++;
        }

        SetTableCell(table, currentRow, 0, "ReleaseIdentifier");
        SetTableCell(table, currentRow, 1, Cohort.GetReleaseIdentifier(true));
        currentRow++;

        if (hasProchi)
        {
            SetTableCell(table, currentRow, 0, "Prefix");
            SetTableCell(table, currentRow, 1, GetFirstProCHIPrefix());
        }
    }

    /// <summary>
    /// Returns the first 3 digits of the first release identifier in the cohort (this is very hic specific).
    /// </summary>
    /// <returns></returns>
    private string GetFirstProCHIPrefix()
    {
        var ect = Cohort.ExternalCohortTable;

        var db = ect.Discover();
        using var con = db.Server.GetConnection();
        con.Open();

        var sql =
            $"SELECT  TOP 1 LEFT({Cohort.GetReleaseIdentifier()},3) FROM {ect.TableName} WHERE {Cohort.WhereSQL()}";

        using var cmd = db.Server.GetCommand(sql, con);
        cmd.CommandTimeout = CohortCountTimeoutInSeconds;
        return (string)cmd.ExecuteScalar();
    }

    private void CreateCohortDetailsTable(XWPFDocument document)
    {
        var table = InsertTable(document, 2, 4);

        var tableLine = 0;

        SetTableCell(table, tableLine, 0, "Version");
        SetTableCell(table, tableLine, 1, "Description");
        SetTableCell(table, tableLine, 2, "Date Extracted");
        SetTableCell(table, tableLine, 3, "Unique Individuals");
        tableLine++;

        SetTableCell(table, tableLine, 0,
            Cohort.GetExternalData(CohortCountTimeoutInSeconds).ExternalVersion.ToString());
        SetTableCell(table, tableLine, 1,
            $"{Cohort} (ID={Cohort.ID}, OriginID={Cohort.OriginID})"); //description fetched from remote table

        var lastExtracted = ExtractionResults.Any()
            ? ExtractionResults.Max(r => r.DateOfExtraction).ToString(CultureInfo.CurrentCulture)
            : "Never";
        SetTableCell(table, tableLine, 2, lastExtracted);
        SetTableCell(table, tableLine, 3,
            Cohort.GetCountDistinctFromDatabase(CohortCountTimeoutInSeconds).ToString("N0"));
    }

    [NotNull]
    private string getDOI([NotNull] Curation.Data.Dataset ds)
    {
        return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : "";
    }

    private void CreateFileSummary(XWPFDocument document)
    {
        var table = InsertTable(document, ExtractionResults.Length + 1, 6);

        var tableLine = 0;

        SetTableCell(table, tableLine, 0, "Data Requirement");
        SetTableCell(table, tableLine, 1, "Notes");
        SetTableCell(table, tableLine, 2, "Filename");
        SetTableCell(table, tableLine, 3, "Records");
        SetTableCell(table, tableLine, 4, "Unique Individuals");
        SetTableCell(table, tableLine, 5, "Datasets");
        tableLine++;

        foreach (var result in ExtractionResults)
        {
            var filename = GetFileName(result);
            var extractableDataset = _repository.GetObjectByID<ExtractableDataSet>(result.ExtractableDataSet_ID);
            SetTableCell(table, tableLine, 0,
              extractableDataset.ToString());
            var linkedDatasets = extractableDataset.Catalogue.CatalogueItems.Select(static c => c.ColumnInfo).Where(ci => ci.Dataset_ID != null).Distinct().Select(ci => ci.Dataset_ID);
            var datasets = _repository.CatalogueRepository.GetAllObjects<Curation.Data.Dataset>().Where(d => linkedDatasets.Contains(d.ID)).ToList();
            var datasetString = string.Join("",datasets.Select(ds=> $"{ds.Name} {getDOI(ds)}, {Environment.NewLine}"));
            SetTableCell(table, tableLine, 1, result.FiltersUsed);
            SetTableCell(table, tableLine, 2, filename);
            SetTableCell(table, tableLine, 3, result.RecordsExtracted.ToString("N0"));
            SetTableCell(table, tableLine, 4, result.DistinctReleaseIdentifiersEncountered.ToString("N0"));
            SetTableCell(table, tableLine, 5, datasetString);
            tableLine++;
        }
    }

    private string GetFileName(ICumulativeExtractionResults result)
    {
        if (string.IsNullOrWhiteSpace(result?.DestinationDescription))
            return "";

        if (string.IsNullOrWhiteSpace(Project.ExtractionDirectory))
            return result.DestinationDescription;

        //can we express it relative
        if (result.DestinationDescription.StartsWith(Project.ExtractionDirectory,
                StringComparison.CurrentCultureIgnoreCase))
        {
            var relative = result.DestinationDescription[Project.ExtractionDirectory.Length..].Replace('\\', '/');

            return $"./{relative.Trim('/')}";
        }

        return result.DestinationDescription;
    }
}

1	// Copyright (c) The University of Dundee 2018-2019
2	// This file is part of the Research Data Management Platform (RDMP).
3	// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
4	// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
5	// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
6
7	using System;
8	using System.Globalization;
9	using System.IO;
10	using System.Linq;
11	using NPOI.XWPF.UserModel;
12	using Rdmp.Core.DataExport.Data;
13	using Rdmp.Core.Repositories;
14	using Rdmp.Core.Repositories.Managers;
15	using Rdmp.Core.ReusableLibraryCode.Annotations;
16
17	namespace Rdmp.Core.Reports.ExtractionTime;
18
19	/// <summary>
20	/// Generates a Microsoft Word DocX file containing information about all the datasets extracted (and released) as part of a Data Release. This includes
21	/// row counts and unique patient counts as well as the number of patients in the original cohort (not all patients will appear in all datasets). Also
22	/// included are the tickets for the project, the cohort ID number etc
23	/// </summary>
24	public class WordDataReleaseFileGenerator : DocXHelper
25	{
26	private readonly IDataExportRepository _repository;
27	public IExtractionConfiguration Configuration { get; set; }	10✔
28	protected ICumulativeExtractionResults[] ExtractionResults { get; set; }	8✔
29	protected IExtractableCohort Cohort { get; set; }	16✔
30	protected IProject Project { get; set; }	6✔
31
32	private const int CohortCountTimeoutInSeconds = 600; // 10 minutes
33
34	public WordDataReleaseFileGenerator(IExtractionConfiguration configuration, IDataExportRepository repository)	2✔
35	{
36	_repository = repository;	2✔
37	Configuration = configuration;	2✔
38	Project = configuration.Project;	2✔
39
40	if (Configuration.Cohort_ID == null)	2!
41	throw new NullReferenceException("Configuration has no Cohort");	×
42
43	Cohort = _repository.GetObjectByID<ExtractableCohort>((int)Configuration.Cohort_ID);	2✔
44
45	ExtractionResults =	2✔
46	Configuration.CumulativeExtractionResults	2✔
47	.OrderBy(	2✔
UNCOV 48	c => _repository.GetObjectByID<ExtractableDataSet>(c.ExtractableDataSet_ID).ToString()	×
49	).ToArray();	2✔
50	}	2✔
51
52	public void GenerateWordFile(string saveAsFilename)
53	{
54	var f = string.IsNullOrWhiteSpace(saveAsFilename)	2!
55	? GetUniqueFilenameInWorkArea("ReleaseDocument")	2✔
56	: new FileInfo(saveAsFilename);	2✔
57
58	// Create an instance of Word and make it visible.=
59	using var document = GetNewDocFile(f);	2✔
60	//actually changes it to landscape :)
61	SetLandscape(document);	2✔
62
63	InsertHeader(document, $"Project:{Project.Name}", 1);	2✔
64	InsertHeader(document, Configuration.Name, 2);	2✔
65
66	var disclaimer = _repository.DataExportPropertyManager.GetValue(DataExportProperty.ReleaseDocumentDisclaimer);	2✔
67
68	if (disclaimer != null)	2!
69	InsertParagraph(document, disclaimer);	×
70
71	CreateTopTable1(document);	2✔
72
73	InsertParagraph(document, Environment.NewLine);	2✔
74
75	CreateCohortDetailsTable(document);	2✔
76
77	InsertParagraph(document, Environment.NewLine);	2✔
78
79	CreateFileSummary(document);	2✔
80
81	//interactive mode, user didn't ask us to save to a specific location so we created it in temp and so we can now show them where that file is
82	if (string.IsNullOrWhiteSpace(saveAsFilename))	2!
83	ShowFile(f);	×
84	}	4✔
85
86	private void CreateTopTable1(XWPFDocument document)
87	{
88	var hasTicket = !string.IsNullOrWhiteSpace(Project.MasterTicket);	2✔
89	var hasProchi = Cohort.GetReleaseIdentifier().ToLower().Contains("prochi");	2✔
90
91	var currentRow = 0;	2✔
92	var requiredRows = 1;	2✔
93
94	if (hasProchi)	2!
95	requiredRows++;	×
96	if (hasTicket)	2!
97	requiredRows++;	×
98
99	var table = InsertTable(document, requiredRows, 2);	2✔
100
101	if (hasTicket)	2!
102	{
103	SetTableCell(table, currentRow, 0, "Master Issue");	×
104	SetTableCell(table, currentRow, 1, Project.MasterTicket);	×
105	currentRow++;	×
106	}
107
108	SetTableCell(table, currentRow, 0, "ReleaseIdentifier");	2✔
109	SetTableCell(table, currentRow, 1, Cohort.GetReleaseIdentifier(true));	2✔
110	currentRow++;	2✔
111
112	if (hasProchi)	2!
113	{
114	SetTableCell(table, currentRow, 0, "Prefix");	×
115	SetTableCell(table, currentRow, 1, GetFirstProCHIPrefix());	×
116	}
117	}	2✔
118
119	/// <summary>
120	/// Returns the first 3 digits of the first release identifier in the cohort (this is very hic specific).
121	/// </summary>
122	/// <returns></returns>
123	private string GetFirstProCHIPrefix()
124	{
125	var ect = Cohort.ExternalCohortTable;	×
126
127	var db = ect.Discover();	×
128	using var con = db.Server.GetConnection();	×
129	con.Open();	×
130
131	var sql =	×
132	$"SELECT TOP 1 LEFT({Cohort.GetReleaseIdentifier()},3) FROM {ect.TableName} WHERE {Cohort.WhereSQL()}";	×
133
134	using var cmd = db.Server.GetCommand(sql, con);	×
135	cmd.CommandTimeout = CohortCountTimeoutInSeconds;	×
136	return (string)cmd.ExecuteScalar();	×
137	}	×
138
139	private void CreateCohortDetailsTable(XWPFDocument document)
140	{
141	var table = InsertTable(document, 2, 4);	2✔
142
143	var tableLine = 0;	2✔
144
145	SetTableCell(table, tableLine, 0, "Version");	2✔
146	SetTableCell(table, tableLine, 1, "Description");	2✔
147	SetTableCell(table, tableLine, 2, "Date Extracted");	2✔
148	SetTableCell(table, tableLine, 3, "Unique Individuals");	2✔
149	tableLine++;	2✔
150
151	SetTableCell(table, tableLine, 0,	2✔
152	Cohort.GetExternalData(CohortCountTimeoutInSeconds).ExternalVersion.ToString());	2✔
153	SetTableCell(table, tableLine, 1,	2✔
154	$"{Cohort} (ID={Cohort.ID}, OriginID={Cohort.OriginID})"); //description fetched from remote table	2✔
155
156	var lastExtracted = ExtractionResults.Any()	2!
UNCOV 157	? ExtractionResults.Max(r => r.DateOfExtraction).ToString(CultureInfo.CurrentCulture)	×
158	: "Never";	2✔
159	SetTableCell(table, tableLine, 2, lastExtracted);	2✔
160	SetTableCell(table, tableLine, 3,	2✔
161	Cohort.GetCountDistinctFromDatabase(CohortCountTimeoutInSeconds).ToString("N0"));	2✔
162	}	2✔
163
164	[NotNull]
165	private string getDOI([NotNull] Curation.Data.Dataset ds)
166	{
167	return !string.IsNullOrWhiteSpace(ds.DigitalObjectIdentifier) ? $" (DOI: {ds.DigitalObjectIdentifier})" : "";	×
168	}
169
170	private void CreateFileSummary(XWPFDocument document)
171	{
172	var table = InsertTable(document, ExtractionResults.Length + 1, 6);	2✔
173
174	var tableLine = 0;	2✔
175
176	SetTableCell(table, tableLine, 0, "Data Requirement");	2✔
177	SetTableCell(table, tableLine, 1, "Notes");	2✔
178	SetTableCell(table, tableLine, 2, "Filename");	2✔
179	SetTableCell(table, tableLine, 3, "Records");	2✔
180	SetTableCell(table, tableLine, 4, "Unique Individuals");	2✔
181	SetTableCell(table, tableLine, 5, "Datasets");	2✔
182	tableLine++;	2✔
183
184	foreach (var result in ExtractionResults)	4!
185	{
UNCOV 186	var filename = GetFileName(result);	×
UNCOV 187	var extractableDataset = _repository.GetObjectByID<ExtractableDataSet>(result.ExtractableDataSet_ID);	×
UNCOV 188	SetTableCell(table, tableLine, 0,	×
UNCOV 189	extractableDataset.ToString());	×
UNCOV 190	var linkedDatasets = extractableDataset.Catalogue.CatalogueItems.Select(static c => c.ColumnInfo).Where(ci => ci.Dataset_ID != null).Distinct().Select(ci => ci.Dataset_ID);	×
UNCOV 191	var datasets = _repository.CatalogueRepository.GetAllObjects<Curation.Data.Dataset>().Where(d => linkedDatasets.Contains(d.ID)).ToList();	×
UNCOV 192	var datasetString = string.Join("",datasets.Select(ds=> $"{ds.Name} {getDOI(ds)}, {Environment.NewLine}"));	×
UNCOV 193	SetTableCell(table, tableLine, 1, result.FiltersUsed);	×
UNCOV 194	SetTableCell(table, tableLine, 2, filename);	×
UNCOV 195	SetTableCell(table, tableLine, 3, result.RecordsExtracted.ToString("N0"));	×
UNCOV 196	SetTableCell(table, tableLine, 4, result.DistinctReleaseIdentifiersEncountered.ToString("N0"));	×
UNCOV 197	SetTableCell(table, tableLine, 5, datasetString);	×
UNCOV 198	tableLine++;	×
199	}
200	}	2✔
201
202	private string GetFileName(ICumulativeExtractionResults result)
203	{
UNCOV 204	if (string.IsNullOrWhiteSpace(result?.DestinationDescription))	×
205	return "";	×
206
UNCOV 207	if (string.IsNullOrWhiteSpace(Project.ExtractionDirectory))	×
208	return result.DestinationDescription;	×
209
210	//can we express it relative
UNCOV 211	if (result.DestinationDescription.StartsWith(Project.ExtractionDirectory,	×
UNCOV 212	StringComparison.CurrentCultureIgnoreCase))	×
213	{
UNCOV 214	var relative = result.DestinationDescription[Project.ExtractionDirectory.Length..].Replace('\\', '/');	×
215
UNCOV 216	return $"./{relative.Trim('/')}";	×
217	}
218
219	return result.DestinationDescription;	×
220	}
221	}

HicServices / RDMP / 9757494962

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous