#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.7

/src/main/java/de/gwdg/metadataqa/marc/cli/Completeness.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.CsvUtils;
import de.gwdg.metadataqa.marc.Utils;
import de.gwdg.metadataqa.marc.analysis.GroupSelector;
import de.gwdg.metadataqa.marc.analysis.completeness.CompletenessDAO;
import de.gwdg.metadataqa.marc.analysis.completeness.RecordCompleteness;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.CompletenessParameters;
import de.gwdg.metadataqa.marc.cli.plugin.CompletenessFactory;
import de.gwdg.metadataqa.marc.cli.plugin.CompletenessPlugin;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordIgnorator;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.definition.tags.TagCategory;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import de.gwdg.metadataqa.marc.model.validation.ValidationErrorFormat;
import de.gwdg.metadataqa.marc.utils.BasicStatistics;
import de.gwdg.metadataqa.marc.utils.TagHierarchy;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.marc4j.marc.Record;

import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;

public class Completeness extends QACli<CompletenessParameters> implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(Completeness.class.getCanonicalName());
  private static final Pattern dataFieldPattern = Pattern.compile("^(\\d\\d\\d)\\$(.*)$");
  public static final String ALL_TYPE = "all";
  private CompletenessDAO completenessDAO = new CompletenessDAO();

  private boolean readyToProcess;
  private CompletenessPlugin plugin;
  private RecordFilter recordFilter;
  private RecordIgnorator recordIgnorator;

  public Completeness(String[] args) throws ParseException {
    parameters = new CompletenessParameters(args);
    plugin = CompletenessFactory.create(parameters);
    recordFilter = parameters.getRecordFilter();
    recordIgnorator = parameters.getRecordIgnorator();
    readyToProcess = true;
    initializeGroups(parameters.getGroupBy(), parameters.isPica());
    if (doGroups()) {
      initializeMeta(parameters);
      if (doSaveGroupIds) {
        idCollectorFile = prepareReportFile(parameters.getOutputDir(), "id-groupid.csv");
        printToFile(idCollectorFile, CsvUtils.createCsv("id", "groupId"));
      }
    }
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new Completeness(args);
    } catch (ParseException e) {
      System.err.println("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
    if (processor.getParameters().getArgs().length < 1) {
      System.err.println("Please provide a MARC file name!");
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(marcRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    if (!recordFilter.isAllowable(bibliographicRecord))
      return;

    if (recordIgnorator.isIgnorable(bibliographicRecord))
      return;

    RecordCompleteness recordCompleteness = new RecordCompleteness(bibliographicRecord, parameters, completenessDAO, plugin, groupBy);
    recordCompleteness.process();

    if (doSaveGroupIds)
      saveGroupIds(bibliographicRecord.getId(true), recordCompleteness.getGroupIds());
    if (doGroups())
      for (String id : recordCompleteness.getGroupIds())
        Utils.count(id, completenessDAO.getGroupCounter());

    for (String path : recordCompleteness.getRecordFrequency().keySet()) {
      if (groupBy != null) {
        for (String groupId : recordCompleteness.getGroupIds()) {
          completenessDAO.getGroupedElementFrequency().computeIfAbsent(groupId, s -> new HashMap<>());
          completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());
          completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());
          Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(recordCompleteness.getDocumentType()));
          Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(ALL_TYPE));

          completenessDAO.getGroupedFieldHistogram().computeIfAbsent(groupId, s -> new HashMap<>());
          completenessDAO.getGroupedFieldHistogram().get(groupId).computeIfAbsent(path, s -> new HashMap<>());
          Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getGroupedFieldHistogram().get(groupId).get(path));
        }
      } else {
        Utils.count(path, completenessDAO.getElementFrequency().get(recordCompleteness.getDocumentType()));
        Utils.count(path, completenessDAO.getElementFrequency().get(ALL_TYPE));
        completenessDAO.getFieldHistogram().computeIfAbsent(path, s -> new HashMap<>());
        Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getFieldHistogram().get(path));
      }
    }

    for (String key : recordCompleteness.getRecordPackageCounter().keySet()) {
      if (groupBy != null) {
        for (String groupId : recordCompleteness.getGroupIds()) {
          completenessDAO.getGroupedPackageCounter().computeIfAbsent(groupId, s -> new HashMap<>());
          completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());
          completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());
          Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(recordCompleteness.getDocumentType()));
          Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(ALL_TYPE));
        }
      } else {
        completenessDAO.getPackageCounter().computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());
        completenessDAO.getPackageCounter().computeIfAbsent(ALL_TYPE, s -> new HashMap<>());
        Utils.count(key, completenessDAO.getPackageCounter().get(recordCompleteness.getDocumentType()));
        Utils.count(key, completenessDAO.getPackageCounter().get(ALL_TYPE));
      }
    }
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters.formatParameters());
    completenessDAO.initialize();
  }

  @Override
  public void fileOpened(Path file) {
    // do nothing
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    String fileExtension = ".csv";
    final char separator = getSeparator(parameters.getFormat());
    if (parameters.getFormat().equals(ValidationErrorFormat.TAB_SEPARATED)) {
      fileExtension = ".tsv";
    }

    saveLibraries003(fileExtension, separator);
    saveLibraries(fileExtension, separator);
    if (groupBy != null) {
      saveGroups(fileExtension, separator);
      saveGroupedPackages(fileExtension, separator);
      saveGroupedMarcElements(fileExtension, separator);
    } else {
      savePackages(fileExtension, separator);
      saveMarcElements(fileExtension, separator);
    }
    saveParameters("completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  private void saveLibraries003(String fileExtension, char separator) {
    logger.info("Saving libraries003...");
    var path = Paths.get(parameters.getOutputDir(), "libraries003" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("library", "count"));
      completenessDAO.getLibrary003Counter().forEach((key, value) -> {
        try {
          writer.write(CsvUtils.createCsv(key, value));
        } catch (IOException e) {
          logger.log(Level.SEVERE, "saveLibraries003", e);
        }
      });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveLibraries003", e);
    }
  }

  private void saveMarcElements(String fileExtension, char separator) {
    Path path = Paths.get(parameters.getOutputDir(), "marc-elements" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv(
        "groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
        "number-of-record", "number-of-instances",
        "min", "max", "mean", "stddev", "histogram"
      ));
      completenessDAO.getElementCardinality().forEach((documentType, cardinalities) ->
        cardinalities.forEach((marcPath, cardinality) -> {
          try {
            writer.write(formatCardinality(marcPath, cardinality, documentType, null));
          } catch (IOException e) {
            logger.log(Level.SEVERE, "saveMarcElements", e);
          }
        })
      );
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveMarcElements", e);
    }
  }

  private void saveGroupedMarcElements(String fileExtension, char separator) {
    logger.info("saving grouped MARC elements...");
    Path path = Paths.get(parameters.getOutputDir(), "completeness-grouped-marc-elements" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv(
        "groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
        "number-of-record", "number-of-instances",
        "min", "max", "mean", "stddev", "histogram"
      ));
      completenessDAO.getGroupedElementCardinality().forEach((groupId, documentTypes) ->
        documentTypes.forEach((documentType, cardinalities) ->
          cardinalities.forEach((marcPath, cardinality) -> {
            try {
              writer.write(formatCardinality(marcPath, cardinality, documentType, groupId));
            } catch (IOException e) {
              logger.log(Level.SEVERE, "saveMarcElements", e);
            }
          })
        )
      );
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveMarcElements", e);
    }
  }

  private void savePackages(String fileExtension, char separator) {
    logger.info("saving packages...");
    var path = Paths.get(parameters.getOutputDir(), "packages" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("documenttype", "packageid", "name", "label", "iscoretag", "count"));
      completenessDAO.getPackageCounter().forEach((documentType, packages) ->
        packages.forEach((packageName, count) -> {
          try {
            TagCategory tagCategory = TagCategory.getPackage(packageName);
            String range = packageName;
            String label = "";
            int id = 100;
            boolean isPartOfMarcScore = false;
            if (tagCategory != null) {
              id = tagCategory.getId();
              range = tagCategory.getRange();
              label = tagCategory.getLabel();
              isPartOfMarcScore = tagCategory.isPartOfMarcCore();
            } else {
              logger.severe(packageName + " has not been found in TagCategory");
            }
            writer.write(CsvUtils.createCsv(documentType, id, range, label, isPartOfMarcScore, count));
          } catch (IOException e) {
            logger.log(Level.SEVERE, "savePackages", e);
          }
        })
      );
    } catch (IOException e) {
      logger.log(Level.SEVERE, "savePackages", e);
    }
  }

  private void saveGroupedPackages(String fileExtension, char separator) {
    logger.info("saving grouped packages...");
    var path = Paths.get(parameters.getOutputDir(), "completeness-grouped-packages" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("group", "documenttype", "packageid", "name", "label", "iscoretag", "count"));
      completenessDAO.getGroupedPackageCounter()
        .entrySet()
        .stream()
        .sorted(Map.Entry.comparingByKey())
        .forEach(groupData -> {
          String groupId = groupData.getKey();
          Map<String, Map<String, Integer>> documentTypes = groupData.getValue();
          documentTypes
            .entrySet()
            .stream()
            .sorted(Map.Entry.comparingByKey())
            .forEach(doctypeData -> {
              String documentType = doctypeData.getKey();
              Map<String, Integer> packages = doctypeData.getValue();
              packages
                .forEach((packageName, count) -> {
                try {
                  TagCategory tagCategory = TagCategory.getPackage(packageName);
                  String range = packageName;
                  String label = "";
                  int id = 100;
                  boolean isPartOfMarcScore = false;
                  if (tagCategory != null) {
                    id = tagCategory.getId();
                    range = tagCategory.getRange();
                    label = tagCategory.getLabel();
                    isPartOfMarcScore = tagCategory.isPartOfMarcCore();
                  } else {
                    logger.severe(packageName + " has not been found in TagCategory");
                  }
                  writer.write(CsvUtils.createCsv(groupId, documentType, id, range, label, isPartOfMarcScore, count));
                } catch (IOException e) {
                  logger.log(Level.SEVERE, "savePackages", e);
                }
              });
            });
      });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "savePackages", e);
    }
  }

  private void saveLibraries(String fileExtension, char separator) {
    logger.info("Saving libraries...");
    var path = Paths.get(parameters.getOutputDir(), "libraries" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("library", "count"));
      completenessDAO.getLibraryCounter().forEach((key, value) -> {
        try {
          writer.write(CsvUtils.createCsv(key, value));
        } catch (IOException e) {
          logger.log(Level.SEVERE, "saveLibraries", e);
        }
      });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveLibraries", e);
    }
  }

  private void saveGroups(String fileExtension, char separator) {
    logger.info("Saving groups...");
    GroupSelector groupSelector = new GroupSelector(parameters.getGroupListFile());
    var path = Paths.get(parameters.getOutputDir(), "completeness-groups" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("id", "group", "count"));
      completenessDAO.getGroupCounter()
        .entrySet()
        .stream()
        .sorted((a,b) -> a.getKey().compareTo(b.getKey()))
        .forEach(item -> {
          try {
            writer.write(CsvUtils.createCsv(item.getKey(), groupSelector.getOrgName(item.getKey()), item.getValue()));
          } catch (IOException e) {
            logger.log(Level.SEVERE, "saveLibraries", e);
          }
        });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveLibraries", e);
    }
  }

  private String formatCardinality(String marcPath,
                                   int cardinality,
                                   String documentType,
                                   String groupId) {
    if (marcPath.equals("")) {
      logger.severe("Empty key from " + marcPath);
    }

    String marcPathLabel = marcPath.replace("!ind", "ind").replaceAll("\\|(\\d)$", "$1");
    String sortkey = marcPath.replaceAll("^leader", "000");
    int packageId = TagCategory.OTHER.getId();
    String packageLabel = TagCategory.OTHER.getLabel();
    String tagLabel = "";
    String subfieldLabel = "";
    TagHierarchy tagHierarchy = plugin.getTagHierarchy(marcPathLabel);
    if (tagHierarchy != null) {
      packageId = tagHierarchy.getPackageId();
      packageLabel = tagHierarchy.getPackageLabel();
      tagLabel = tagHierarchy.getTagLabel();
      subfieldLabel = tagHierarchy.getSubfieldLabel();
    } else {
      logger.severe(() -> "Key can not be found in the TagHierarchy: " + marcPathLabel);
    }

    // Integer cardinality = entry.getValue();
    Integer frequency = (groupId != null)
      ? completenessDAO.getGroupedElementFrequency().get(groupId).get(documentType).get(marcPath)
      : completenessDAO.getElementFrequency().get(documentType).get(marcPath);

    Map<Integer, Integer> histogram = null;
    if (groupId != null) {
      histogram = completenessDAO.getGroupedFieldHistogram().get(groupId).get(marcPath);
      if (!completenessDAO.getGroupedFieldHistogram().get(groupId).containsKey(marcPath)) {
        logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);
      }
    } else {
      histogram = completenessDAO.getFieldHistogram().get(marcPath);
      if (!completenessDAO.getFieldHistogram().containsKey(marcPath)) {
        logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);
      }
    }
    BasicStatistics statistics = new BasicStatistics(histogram);

    List<Object> values = Arrays.asList(
      (groupId != null ? groupId : 0),
      documentType, marcPathLabel, sortkey, packageId, packageLabel, tagLabel, subfieldLabel,
      frequency,   // = number-of-record
      cardinality, // = number-of-instances
      statistics.getMin(), statistics.getMax(),
      statistics.getMean(), statistics.getStdDev(),
      statistics.formatHistogram()
    );

    return CsvUtils.createCsvFromObjects(values);
  }

  private char getSeparator(ValidationErrorFormat format) {
    if (format.equals(ValidationErrorFormat.TAB_SEPARATED)) {
      return '\t';
    } else {
      return ',';
    }
  }

  @Override
  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());
    formatter.printHelp(message, options);
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.CsvUtils;
4	import de.gwdg.metadataqa.marc.Utils;
5	import de.gwdg.metadataqa.marc.analysis.GroupSelector;
6	import de.gwdg.metadataqa.marc.analysis.completeness.CompletenessDAO;
7	import de.gwdg.metadataqa.marc.analysis.completeness.RecordCompleteness;
8	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
9	import de.gwdg.metadataqa.marc.cli.parameters.CompletenessParameters;
10	import de.gwdg.metadataqa.marc.cli.plugin.CompletenessFactory;
11	import de.gwdg.metadataqa.marc.cli.plugin.CompletenessPlugin;
12	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
13	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
14	import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
15	import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordIgnorator;
16	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
17	import de.gwdg.metadataqa.marc.definition.tags.TagCategory;
18	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
19	import de.gwdg.metadataqa.marc.model.validation.ValidationErrorFormat;
20	import de.gwdg.metadataqa.marc.utils.BasicStatistics;
21	import de.gwdg.metadataqa.marc.utils.TagHierarchy;
22	import org.apache.commons.cli.HelpFormatter;
23	import org.apache.commons.cli.Options;
24	import org.apache.commons.cli.ParseException;
25	import org.marc4j.marc.Record;
26
27	import java.io.IOException;
28	import java.io.Serializable;
29	import java.nio.file.Files;
30	import java.nio.file.Path;
31	import java.nio.file.Paths;
32	import java.util.Arrays;
33	import java.util.HashMap;
34	import java.util.List;
35	import java.util.Map;
36	import java.util.logging.Level;
37	import java.util.logging.Logger;
38	import java.util.regex.Pattern;
39
40	public class Completeness extends QACli<CompletenessParameters> implements BibliographicInputProcessor, Serializable {
41
42	private static final Logger logger = Logger.getLogger(Completeness.class.getCanonicalName());	1✔
43	private static final Pattern dataFieldPattern = Pattern.compile("^(\\d\\d\\d)\\$(.*)$");	1✔
44	public static final String ALL_TYPE = "all";
45	private CompletenessDAO completenessDAO = new CompletenessDAO();	1✔
46
47	private boolean readyToProcess;
48	private CompletenessPlugin plugin;
49	private RecordFilter recordFilter;
50	private RecordIgnorator recordIgnorator;
51
52	public Completeness(String[] args) throws ParseException {	1✔
53	parameters = new CompletenessParameters(args);	1✔
54	plugin = CompletenessFactory.create(parameters);	1✔
55	recordFilter = parameters.getRecordFilter();	1✔
56	recordIgnorator = parameters.getRecordIgnorator();	1✔
57	readyToProcess = true;	1✔
58	initializeGroups(parameters.getGroupBy(), parameters.isPica());	1✔
59	if (doGroups()) {	1✔
60	initializeMeta(parameters);	1✔
61	if (doSaveGroupIds) {	1✔
62	idCollectorFile = prepareReportFile(parameters.getOutputDir(), "id-groupid.csv");	1✔
63	printToFile(idCollectorFile, CsvUtils.createCsv("id", "groupId"));	1✔
64	}
65	}
66	}	1✔
67
68	public static void main(String[] args) {
69	BibliographicInputProcessor processor = null;	×
70	try {
71	processor = new Completeness(args);	×
72	} catch (ParseException e) {	×
73	System.err.println("ERROR. " + e.getLocalizedMessage());	×
74	System.exit(1);	×
75	}	×
76	if (processor.getParameters().getArgs().length < 1) {	×
77	System.err.println("Please provide a MARC file name!");	×
78	processor.printHelp(processor.getParameters().getOptions());	×
79	System.exit(0);	×
80	}
81	if (processor.getParameters().doHelp()) {	×
82	processor.printHelp(processor.getParameters().getOptions());	×
83	System.exit(0);	×
84	}
85	RecordIterator iterator = new RecordIterator(processor);	×
NEW 86	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	×
87	iterator.start();	×
88	}	×
89
90	@Override
91	public CommonParameters getParameters() {
92	return parameters;	1✔
93	}
94
95	@Override
96	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
97	// do nothing
98	}	1✔
99
100	@Override
101	public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 102	processRecord(marcRecord, recordNumber);	×
103	}	×
104
105	@Override
106	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
107	if (!recordFilter.isAllowable(bibliographicRecord))	1✔
UNCOV 108	return;	×
109
110	if (recordIgnorator.isIgnorable(bibliographicRecord))	1✔
UNCOV 111	return;	×
112
113	RecordCompleteness recordCompleteness = new RecordCompleteness(bibliographicRecord, parameters, completenessDAO, plugin, groupBy);	1✔
114	recordCompleteness.process();	1✔
115
116	if (doSaveGroupIds)	1✔
117	saveGroupIds(bibliographicRecord.getId(true), recordCompleteness.getGroupIds());	1✔
118	if (doGroups())	1✔
119	for (String id : recordCompleteness.getGroupIds())	1✔
120	Utils.count(id, completenessDAO.getGroupCounter());	1✔
121
122	for (String path : recordCompleteness.getRecordFrequency().keySet()) {	1✔
123	if (groupBy != null) {	1✔
124	for (String groupId : recordCompleteness.getGroupIds()) {	1✔
125	completenessDAO.getGroupedElementFrequency().computeIfAbsent(groupId, s -> new HashMap<>());	1✔
126	completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());	1✔
127	completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());	1✔
128	Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(recordCompleteness.getDocumentType()));	1✔
129	Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(ALL_TYPE));	1✔
130
131	completenessDAO.getGroupedFieldHistogram().computeIfAbsent(groupId, s -> new HashMap<>());	1✔
132	completenessDAO.getGroupedFieldHistogram().get(groupId).computeIfAbsent(path, s -> new HashMap<>());	1✔
133	Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getGroupedFieldHistogram().get(groupId).get(path));	1✔
134	}	1✔
135	} else {
136	Utils.count(path, completenessDAO.getElementFrequency().get(recordCompleteness.getDocumentType()));	1✔
137	Utils.count(path, completenessDAO.getElementFrequency().get(ALL_TYPE));	1✔
138	completenessDAO.getFieldHistogram().computeIfAbsent(path, s -> new HashMap<>());	1✔
139	Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getFieldHistogram().get(path));	1✔
140	}
141	}	1✔
142
143	for (String key : recordCompleteness.getRecordPackageCounter().keySet()) {	1✔
144	if (groupBy != null) {	1✔
145	for (String groupId : recordCompleteness.getGroupIds()) {	1✔
146	completenessDAO.getGroupedPackageCounter().computeIfAbsent(groupId, s -> new HashMap<>());	1✔
147	completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());	1✔
148	completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());	1✔
149	Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(recordCompleteness.getDocumentType()));	1✔
150	Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(ALL_TYPE));	1✔
151	}	1✔
152	} else {
153	completenessDAO.getPackageCounter().computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());	1✔
154	completenessDAO.getPackageCounter().computeIfAbsent(ALL_TYPE, s -> new HashMap<>());	1✔
155	Utils.count(key, completenessDAO.getPackageCounter().get(recordCompleteness.getDocumentType()));	1✔
156	Utils.count(key, completenessDAO.getPackageCounter().get(ALL_TYPE));	1✔
157	}
158	}	1✔
159	}	1✔
160
161	@Override
162	public void beforeIteration() {
163	logger.info(parameters.formatParameters());	1✔
164	completenessDAO.initialize();	1✔
165	}	1✔
166
167	@Override
168	public void fileOpened(Path file) {
169	// do nothing
170	}	1✔
171
172	@Override
173	public void fileProcessed() {
174	// do nothing
UNCOV 175	}	×
176
177	@Override
178	public void afterIteration(int numberOfprocessedRecords, long duration) {
179	String fileExtension = ".csv";	1✔
180	final char separator = getSeparator(parameters.getFormat());	1✔
181	if (parameters.getFormat().equals(ValidationErrorFormat.TAB_SEPARATED)) {	1✔
UNCOV 182	fileExtension = ".tsv";	×
183	}
184
185	saveLibraries003(fileExtension, separator);	1✔
186	saveLibraries(fileExtension, separator);	1✔
187	if (groupBy != null) {	1✔
188	saveGroups(fileExtension, separator);	1✔
189	saveGroupedPackages(fileExtension, separator);	1✔
190	saveGroupedMarcElements(fileExtension, separator);	1✔
191	} else {
192	savePackages(fileExtension, separator);	1✔
193	saveMarcElements(fileExtension, separator);	1✔
194	}
195	saveParameters("completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
196	}	1✔
197
198	private void saveLibraries003(String fileExtension, char separator) {
199	logger.info("Saving libraries003...");	1✔
200	var path = Paths.get(parameters.getOutputDir(), "libraries003" + fileExtension);	1✔
201	try (var writer = Files.newBufferedWriter(path)) {	1✔
202	writer.write(CsvUtils.createCsv("library", "count"));	1✔
203	completenessDAO.getLibrary003Counter().forEach((key, value) -> {	1✔
204	try {
205	writer.write(CsvUtils.createCsv(key, value));	1✔
UNCOV 206	} catch (IOException e) {	×
207	logger.log(Level.SEVERE, "saveLibraries003", e);	×
208	}	1✔
209	});	1✔
UNCOV 210	} catch (IOException e) {	×
211	logger.log(Level.SEVERE, "saveLibraries003", e);	×
212	}	1✔
213	}	1✔
214
215	private void saveMarcElements(String fileExtension, char separator) {
216	Path path = Paths.get(parameters.getOutputDir(), "marc-elements" + fileExtension);	1✔
217	try (var writer = Files.newBufferedWriter(path)) {	1✔
218	writer.write(CsvUtils.createCsv(	1✔
219	"groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
220	"number-of-record", "number-of-instances",
221	"min", "max", "mean", "stddev", "histogram"
222	));
223	completenessDAO.getElementCardinality().forEach((documentType, cardinalities) ->	1✔
224	cardinalities.forEach((marcPath, cardinality) -> {	1✔
225	try {
226	writer.write(formatCardinality(marcPath, cardinality, documentType, null));	1✔
UNCOV 227	} catch (IOException e) {	×
228	logger.log(Level.SEVERE, "saveMarcElements", e);	×
229	}	1✔
230	})	1✔
231	);
UNCOV 232	} catch (IOException e) {	×
233	logger.log(Level.SEVERE, "saveMarcElements", e);	×
234	}	1✔
235	}	1✔
236
237	private void saveGroupedMarcElements(String fileExtension, char separator) {
238	logger.info("saving grouped MARC elements...");	1✔
239	Path path = Paths.get(parameters.getOutputDir(), "completeness-grouped-marc-elements" + fileExtension);	1✔
240	try (var writer = Files.newBufferedWriter(path)) {	1✔
241	writer.write(CsvUtils.createCsv(	1✔
242	"groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
243	"number-of-record", "number-of-instances",
244	"min", "max", "mean", "stddev", "histogram"
245	));
246	completenessDAO.getGroupedElementCardinality().forEach((groupId, documentTypes) ->	1✔
247	documentTypes.forEach((documentType, cardinalities) ->	1✔
248	cardinalities.forEach((marcPath, cardinality) -> {	1✔
249	try {
250	writer.write(formatCardinality(marcPath, cardinality, documentType, groupId));	1✔
UNCOV 251	} catch (IOException e) {	×
252	logger.log(Level.SEVERE, "saveMarcElements", e);	×
253	}	1✔
254	})	1✔
255	)
256	);
UNCOV 257	} catch (IOException e) {	×
258	logger.log(Level.SEVERE, "saveMarcElements", e);	×
259	}	1✔
260	}	1✔
261
262	private void savePackages(String fileExtension, char separator) {
263	logger.info("saving packages...");	1✔
264	var path = Paths.get(parameters.getOutputDir(), "packages" + fileExtension);	1✔
265	try (var writer = Files.newBufferedWriter(path)) {	1✔
266	writer.write(CsvUtils.createCsv("documenttype", "packageid", "name", "label", "iscoretag", "count"));	1✔
267	completenessDAO.getPackageCounter().forEach((documentType, packages) ->	1✔
268	packages.forEach((packageName, count) -> {	1✔
269	try {
270	TagCategory tagCategory = TagCategory.getPackage(packageName);	1✔
271	String range = packageName;	1✔
272	String label = "";	1✔
273	int id = 100;	1✔
274	boolean isPartOfMarcScore = false;	1✔
275	if (tagCategory != null) {	1✔
276	id = tagCategory.getId();	1✔
277	range = tagCategory.getRange();	1✔
278	label = tagCategory.getLabel();	1✔
279	isPartOfMarcScore = tagCategory.isPartOfMarcCore();	1✔
280	} else {
UNCOV 281	logger.severe(packageName + " has not been found in TagCategory");	×
282	}
283	writer.write(CsvUtils.createCsv(documentType, id, range, label, isPartOfMarcScore, count));	1✔
UNCOV 284	} catch (IOException e) {	×
285	logger.log(Level.SEVERE, "savePackages", e);	×
286	}	1✔
287	})	1✔
288	);
UNCOV 289	} catch (IOException e) {	×
290	logger.log(Level.SEVERE, "savePackages", e);	×
291	}	1✔
292	}	1✔
293
294	private void saveGroupedPackages(String fileExtension, char separator) {
295	logger.info("saving grouped packages...");	1✔
296	var path = Paths.get(parameters.getOutputDir(), "completeness-grouped-packages" + fileExtension);	1✔
297	try (var writer = Files.newBufferedWriter(path)) {	1✔
298	writer.write(CsvUtils.createCsv("group", "documenttype", "packageid", "name", "label", "iscoretag", "count"));	1✔
299	completenessDAO.getGroupedPackageCounter()	1✔
300	.entrySet()	1✔
301	.stream()	1✔
302	.sorted(Map.Entry.comparingByKey())	1✔
303	.forEach(groupData -> {	1✔
304	String groupId = groupData.getKey();	1✔
305	Map<String, Map<String, Integer>> documentTypes = groupData.getValue();	1✔
306	documentTypes	1✔
307	.entrySet()	1✔
308	.stream()	1✔
309	.sorted(Map.Entry.comparingByKey())	1✔
310	.forEach(doctypeData -> {	1✔
311	String documentType = doctypeData.getKey();	1✔
312	Map<String, Integer> packages = doctypeData.getValue();	1✔
313	packages	1✔
314	.forEach((packageName, count) -> {	1✔
315	try {
316	TagCategory tagCategory = TagCategory.getPackage(packageName);	1✔
317	String range = packageName;	1✔
318	String label = "";	1✔
319	int id = 100;	1✔
320	boolean isPartOfMarcScore = false;	1✔
321	if (tagCategory != null) {	1✔
322	id = tagCategory.getId();	1✔
323	range = tagCategory.getRange();	1✔
324	label = tagCategory.getLabel();	1✔
325	isPartOfMarcScore = tagCategory.isPartOfMarcCore();	1✔
326	} else {
UNCOV 327	logger.severe(packageName + " has not been found in TagCategory");	×
328	}
329	writer.write(CsvUtils.createCsv(groupId, documentType, id, range, label, isPartOfMarcScore, count));	1✔
UNCOV 330	} catch (IOException e) {	×
331	logger.log(Level.SEVERE, "savePackages", e);	×
332	}	1✔
333	});	1✔
334	});	1✔
335	});	1✔
UNCOV 336	} catch (IOException e) {	×
337	logger.log(Level.SEVERE, "savePackages", e);	×
338	}	1✔
339	}	1✔
340
341	private void saveLibraries(String fileExtension, char separator) {
342	logger.info("Saving libraries...");	1✔
343	var path = Paths.get(parameters.getOutputDir(), "libraries" + fileExtension);	1✔
344	try (var writer = Files.newBufferedWriter(path)) {	1✔
345	writer.write(CsvUtils.createCsv("library", "count"));	1✔
346	completenessDAO.getLibraryCounter().forEach((key, value) -> {	1✔
347	try {
UNCOV 348	writer.write(CsvUtils.createCsv(key, value));	×
349	} catch (IOException e) {	×
350	logger.log(Level.SEVERE, "saveLibraries", e);	×
351	}	×
352	});	×
353	} catch (IOException e) {	×
354	logger.log(Level.SEVERE, "saveLibraries", e);	×
355	}	1✔
356	}	1✔
357
358	private void saveGroups(String fileExtension, char separator) {
359	logger.info("Saving groups...");	1✔
360	GroupSelector groupSelector = new GroupSelector(parameters.getGroupListFile());	1✔
361	var path = Paths.get(parameters.getOutputDir(), "completeness-groups" + fileExtension);	1✔
362	try (var writer = Files.newBufferedWriter(path)) {	1✔
363	writer.write(CsvUtils.createCsv("id", "group", "count"));	1✔
364	completenessDAO.getGroupCounter()	1✔
365	.entrySet()	1✔
366	.stream()	1✔
367	.sorted((a,b) -> a.getKey().compareTo(b.getKey()))	1✔
368	.forEach(item -> {	1✔
369	try {
370	writer.write(CsvUtils.createCsv(item.getKey(), groupSelector.getOrgName(item.getKey()), item.getValue()));	1✔
UNCOV 371	} catch (IOException e) {	×
372	logger.log(Level.SEVERE, "saveLibraries", e);	×
373	}	1✔
374	});	1✔
UNCOV 375	} catch (IOException e) {	×
376	logger.log(Level.SEVERE, "saveLibraries", e);	×
377	}	1✔
378	}	1✔
379
380	private String formatCardinality(String marcPath,
381	int cardinality,
382	String documentType,
383	String groupId) {
384	if (marcPath.equals("")) {	1✔
UNCOV 385	logger.severe("Empty key from " + marcPath);	×
386	}
387
388	String marcPathLabel = marcPath.replace("!ind", "ind").replaceAll("\\\|(\\d)$", "$1");	1✔
389	String sortkey = marcPath.replaceAll("^leader", "000");	1✔
390	int packageId = TagCategory.OTHER.getId();	1✔
391	String packageLabel = TagCategory.OTHER.getLabel();	1✔
392	String tagLabel = "";	1✔
393	String subfieldLabel = "";	1✔
394	TagHierarchy tagHierarchy = plugin.getTagHierarchy(marcPathLabel);	1✔
395	if (tagHierarchy != null) {	1✔
396	packageId = tagHierarchy.getPackageId();	1✔
397	packageLabel = tagHierarchy.getPackageLabel();	1✔
398	tagLabel = tagHierarchy.getTagLabel();	1✔
399	subfieldLabel = tagHierarchy.getSubfieldLabel();	1✔
400	} else {
401	logger.severe(() -> "Key can not be found in the TagHierarchy: " + marcPathLabel);	1✔
402	}
403
404	// Integer cardinality = entry.getValue();
405	Integer frequency = (groupId != null)	1✔
406	? completenessDAO.getGroupedElementFrequency().get(groupId).get(documentType).get(marcPath)	1✔
407	: completenessDAO.getElementFrequency().get(documentType).get(marcPath);	1✔
408
409	Map<Integer, Integer> histogram = null;	1✔
410	if (groupId != null) {	1✔
411	histogram = completenessDAO.getGroupedFieldHistogram().get(groupId).get(marcPath);	1✔
412	if (!completenessDAO.getGroupedFieldHistogram().get(groupId).containsKey(marcPath)) {	1✔
UNCOV 413	logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);	×
414	}
415	} else {
416	histogram = completenessDAO.getFieldHistogram().get(marcPath);	1✔
417	if (!completenessDAO.getFieldHistogram().containsKey(marcPath)) {	1✔
UNCOV 418	logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);	×
419	}
420	}
421	BasicStatistics statistics = new BasicStatistics(histogram);	1✔
422
423	List<Object> values = Arrays.asList(	1✔
424	(groupId != null ? groupId : 0),	1✔
425	documentType, marcPathLabel, sortkey, packageId, packageLabel, tagLabel, subfieldLabel,	1✔
426	frequency, // = number-of-record
427	cardinality, // = number-of-instances	1✔
428	statistics.getMin(), statistics.getMax(),	1✔
429	statistics.getMean(), statistics.getStdDev(),	1✔
430	statistics.formatHistogram()	1✔
431	);
432
433	return CsvUtils.createCsvFromObjects(values);	1✔
434	}
435
436	private char getSeparator(ValidationErrorFormat format) {
437	if (format.equals(ValidationErrorFormat.TAB_SEPARATED)) {	1✔
UNCOV 438	return '\t';	×
439	} else {
440	return ',';	1✔
441	}
442	}
443
444	@Override
445	public void printHelp(Options options) {
UNCOV 446	HelpFormatter formatter = new HelpFormatter();	×
447	String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());	×
448	formatter.printHelp(message, options);	×
449	}	×
450
451	@Override
452	public boolean readyToProcess() {
453	return readyToProcess;	1✔
454	}
455	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous