#1447

Committed 11 Apr 2025 03:29PM UTC coverage: 90.49% (-0.1%) from 90.59%

Build # #1447

Build Type

push

Committed by

Commit Message

Inject record ID if missing #607

Coverage Stats

25 of 60 new or added lines in 17 files covered. (41.67%)

70 existing lines in 15 files now uncovered.

36528 of 40367 relevant lines covered (90.49%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.78

/src/main/java/de/gwdg/metadataqa/marc/cli/Completeness.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.CsvUtils;
import de.gwdg.metadataqa.marc.Utils;
import de.gwdg.metadataqa.marc.analysis.GroupSelector;
import de.gwdg.metadataqa.marc.analysis.completeness.CompletenessDAO;
import de.gwdg.metadataqa.marc.analysis.completeness.RecordCompleteness;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.CompletenessParameters;
import de.gwdg.metadataqa.marc.cli.plugin.CompletenessFactory;
import de.gwdg.metadataqa.marc.cli.plugin.CompletenessPlugin;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordIgnorator;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.definition.tags.TagCategory;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import de.gwdg.metadataqa.marc.model.validation.ValidationErrorFormat;
import de.gwdg.metadataqa.marc.utils.BasicStatistics;
import de.gwdg.metadataqa.marc.utils.TagHierarchy;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.marc4j.marc.Record;

import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;

public class Completeness extends QACli<CompletenessParameters> implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(Completeness.class.getCanonicalName());
  private static final Pattern dataFieldPattern = Pattern.compile("^(\\d\\d\\d)\\$(.*)$");
  public static final String ALL_TYPE = "all";
  private CompletenessDAO completenessDAO = new CompletenessDAO();

  private boolean readyToProcess;
  private CompletenessPlugin plugin;
  private RecordFilter recordFilter;
  private RecordIgnorator recordIgnorator;

  public Completeness(String[] args) throws ParseException {
    parameters = new CompletenessParameters(args);
    plugin = CompletenessFactory.create(parameters);
    recordFilter = parameters.getRecordFilter();
    recordIgnorator = parameters.getRecordIgnorator();
    readyToProcess = true;
    initializeGroups(parameters.getGroupBy(), parameters.isPica());
    if (doGroups()) {
      initializeMeta(parameters);
      if (doSaveGroupIds) {
        idCollectorFile = prepareReportFile(parameters.getOutputDir(), "id-groupid.csv");
        printToFile(idCollectorFile, CsvUtils.createCsv("id", "groupId"));
      }
    }
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new Completeness(args);
    } catch (ParseException e) {
      System.err.println("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
    if (processor.getParameters().getArgs().length < 1) {
      System.err.println("Please provide a MARC file name!");
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(marcRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    logger.info("processRecord");
    if (!recordFilter.isAllowable(bibliographicRecord))
      return;

    if (recordIgnorator.isIgnorable(bibliographicRecord))
      return;

    RecordCompleteness recordCompleteness = new RecordCompleteness(bibliographicRecord, parameters, completenessDAO, plugin, groupBy);
    recordCompleteness.process();

    if (doSaveGroupIds)
      saveGroupIds(bibliographicRecord.getId(true), recordCompleteness.getGroupIds());
    if (doGroups())
      for (String id : recordCompleteness.getGroupIds())
        Utils.count(id, completenessDAO.getGroupCounter());

    for (String path : recordCompleteness.getRecordFrequency().keySet()) {
      if (groupBy != null) {
        for (String groupId : recordCompleteness.getGroupIds()) {
          completenessDAO.getGroupedElementFrequency().computeIfAbsent(groupId, s -> new HashMap<>());
          completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());
          completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());
          Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(recordCompleteness.getDocumentType()));
          Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(ALL_TYPE));

          completenessDAO.getGroupedFieldHistogram().computeIfAbsent(groupId, s -> new HashMap<>());
          completenessDAO.getGroupedFieldHistogram().get(groupId).computeIfAbsent(path, s -> new HashMap<>());
          Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getGroupedFieldHistogram().get(groupId).get(path));
        }
      } else {
        Utils.count(path, completenessDAO.getElementFrequency().get(recordCompleteness.getDocumentType()));
        Utils.count(path, completenessDAO.getElementFrequency().get(ALL_TYPE));
        completenessDAO.getFieldHistogram().computeIfAbsent(path, s -> new HashMap<>());
        Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getFieldHistogram().get(path));
      }
    }

    for (String key : recordCompleteness.getRecordPackageCounter().keySet()) {
      if (groupBy != null) {
        for (String groupId : recordCompleteness.getGroupIds()) {
          completenessDAO.getGroupedPackageCounter().computeIfAbsent(groupId, s -> new HashMap<>());
          completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());
          completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());
          Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(recordCompleteness.getDocumentType()));
          Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(ALL_TYPE));
        }
      } else {
        completenessDAO.getPackageCounter().computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());
        completenessDAO.getPackageCounter().computeIfAbsent(ALL_TYPE, s -> new HashMap<>());
        Utils.count(key, completenessDAO.getPackageCounter().get(recordCompleteness.getDocumentType()));
        Utils.count(key, completenessDAO.getPackageCounter().get(ALL_TYPE));
      }
    }
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters.formatParameters());
    completenessDAO.initialize();
  }

  @Override
  public void fileOpened(Path file) {
    // do nothing
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    String fileExtension = ".csv";
    final char separator = getSeparator(parameters.getFormat());
    if (parameters.getFormat().equals(ValidationErrorFormat.TAB_SEPARATED)) {
      fileExtension = ".tsv";
    }

    saveLibraries003(fileExtension, separator);
    saveLibraries(fileExtension, separator);
    if (groupBy != null) {
      saveGroups(fileExtension, separator);
      saveGroupedPackages(fileExtension, separator);
      saveGroupedMarcElements(fileExtension, separator);
    } else {
      savePackages(fileExtension, separator);
      saveMarcElements(fileExtension, separator);
    }
    saveParameters("completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  private void saveLibraries003(String fileExtension, char separator) {
    logger.info("Saving libraries003...");
    var path = Paths.get(parameters.getOutputDir(), "libraries003" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("library", "count"));
      completenessDAO.getLibrary003Counter().forEach((key, value) -> {
        try {
          writer.write(CsvUtils.createCsv(key, value));
        } catch (IOException e) {
          logger.log(Level.SEVERE, "saveLibraries003", e);
        }
      });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveLibraries003", e);
    }
  }

  private void saveMarcElements(String fileExtension, char separator) {
    Path path = Paths.get(parameters.getOutputDir(), "marc-elements" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv(
        "groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
        "number-of-record", "number-of-instances",
        "min", "max", "mean", "stddev", "histogram"
      ));
      completenessDAO.getElementCardinality().forEach((documentType, cardinalities) ->
        cardinalities.forEach((marcPath, cardinality) -> {
          try {
            writer.write(formatCardinality(marcPath, cardinality, documentType, null));
          } catch (IOException e) {
            logger.log(Level.SEVERE, "saveMarcElements", e);
          }
        })
      );
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveMarcElements", e);
    }
  }

  private void saveGroupedMarcElements(String fileExtension, char separator) {
    logger.info("saving grouped MARC elements...");
    Path path = Paths.get(parameters.getOutputDir(), "completeness-grouped-marc-elements" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv(
        "groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
        "number-of-record", "number-of-instances",
        "min", "max", "mean", "stddev", "histogram"
      ));
      completenessDAO.getGroupedElementCardinality().forEach((groupId, documentTypes) ->
        documentTypes.forEach((documentType, cardinalities) ->
          cardinalities.forEach((marcPath, cardinality) -> {
            try {
              writer.write(formatCardinality(marcPath, cardinality, documentType, groupId));
            } catch (IOException e) {
              logger.log(Level.SEVERE, "saveMarcElements", e);
            }
          })
        )
      );
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveMarcElements", e);
    }
  }

  private void savePackages(String fileExtension, char separator) {
    logger.info("saving packages...");
    var path = Paths.get(parameters.getOutputDir(), "packages" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("documenttype", "packageid", "name", "label", "iscoretag", "count"));
      completenessDAO.getPackageCounter().forEach((documentType, packages) ->
        packages.forEach((packageName, count) -> {
          try {
            TagCategory tagCategory = TagCategory.getPackage(packageName);
            String range = packageName;
            String label = "";
            int id = 100;
            boolean isPartOfMarcScore = false;
            if (tagCategory != null) {
              id = tagCategory.getId();
              range = tagCategory.getRange();
              label = tagCategory.getLabel();
              isPartOfMarcScore = tagCategory.isPartOfMarcCore();
            } else {
              logger.severe(packageName + " has not been found in TagCategory");
            }
            writer.write(CsvUtils.createCsv(documentType, id, range, label, isPartOfMarcScore, count));
          } catch (IOException e) {
            logger.log(Level.SEVERE, "savePackages", e);
          }
        })
      );
    } catch (IOException e) {
      logger.log(Level.SEVERE, "savePackages", e);
    }
  }

  private void saveGroupedPackages(String fileExtension, char separator) {
    logger.info("saving grouped packages...");
    var path = Paths.get(parameters.getOutputDir(), "completeness-grouped-packages" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("group", "documenttype", "packageid", "name", "label", "iscoretag", "count"));
      completenessDAO.getGroupedPackageCounter()
        .entrySet()
        .stream()
        .sorted(Map.Entry.comparingByKey())
        .forEach(groupData -> {
          String groupId = groupData.getKey();
          Map<String, Map<String, Integer>> documentTypes = groupData.getValue();
          documentTypes
            .entrySet()
            .stream()
            .sorted(Map.Entry.comparingByKey())
            .forEach(doctypeData -> {
              String documentType = doctypeData.getKey();
              Map<String, Integer> packages = doctypeData.getValue();
              packages
                .forEach((packageName, count) -> {
                try {
                  TagCategory tagCategory = TagCategory.getPackage(packageName);
                  String range = packageName;
                  String label = "";
                  int id = 100;
                  boolean isPartOfMarcScore = false;
                  if (tagCategory != null) {
                    id = tagCategory.getId();
                    range = tagCategory.getRange();
                    label = tagCategory.getLabel();
                    isPartOfMarcScore = tagCategory.isPartOfMarcCore();
                  } else {
                    logger.severe(packageName + " has not been found in TagCategory");
                  }
                  writer.write(CsvUtils.createCsv(groupId, documentType, id, range, label, isPartOfMarcScore, count));
                } catch (IOException e) {
                  logger.log(Level.SEVERE, "savePackages", e);
                }
              });
            });
      });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "savePackages", e);
    }
  }

  private void saveLibraries(String fileExtension, char separator) {
    logger.info("Saving libraries...");
    var path = Paths.get(parameters.getOutputDir(), "libraries" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("library", "count"));
      completenessDAO.getLibraryCounter().forEach((key, value) -> {
        try {
          writer.write(CsvUtils.createCsv(key, value));
        } catch (IOException e) {
          logger.log(Level.SEVERE, "saveLibraries", e);
        }
      });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveLibraries", e);
    }
  }

  private void saveGroups(String fileExtension, char separator) {
    logger.info("Saving groups...");
    GroupSelector groupSelector = new GroupSelector(parameters.getGroupListFile());
    var path = Paths.get(parameters.getOutputDir(), "completeness-groups" + fileExtension);
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(CsvUtils.createCsv("id", "group", "count"));
      completenessDAO.getGroupCounter()
        .entrySet()
        .stream()
        .sorted((a,b) -> a.getKey().compareTo(b.getKey()))
        .forEach(item -> {
          try {
            writer.write(CsvUtils.createCsv(item.getKey(), groupSelector.getOrgName(item.getKey()), item.getValue()));
          } catch (IOException e) {
            logger.log(Level.SEVERE, "saveLibraries", e);
          }
        });
    } catch (IOException e) {
      logger.log(Level.SEVERE, "saveLibraries", e);
    }
  }

  private String formatCardinality(String marcPath,
                                   int cardinality,
                                   String documentType,
                                   String groupId) {
    if (marcPath.equals("")) {
      logger.severe("Empty key from " + marcPath);
    }

    String marcPathLabel = marcPath.replace("!ind", "ind").replaceAll("\\|(\\d)$", "$1");
    String sortkey = marcPath.replaceAll("^leader", "000");
    int packageId = TagCategory.OTHER.getId();
    String packageLabel = TagCategory.OTHER.getLabel();
    String tagLabel = "";
    String subfieldLabel = "";
    TagHierarchy tagHierarchy = plugin.getTagHierarchy(marcPathLabel);
    if (tagHierarchy != null) {
      packageId = tagHierarchy.getPackageId();
      packageLabel = tagHierarchy.getPackageLabel();
      tagLabel = tagHierarchy.getTagLabel();
      subfieldLabel = tagHierarchy.getSubfieldLabel();
    } else {
      logger.severe(() -> "Key can not be found in the TagHierarchy: " + marcPathLabel);
    }

    // Integer cardinality = entry.getValue();
    Integer frequency = (groupId != null)
      ? completenessDAO.getGroupedElementFrequency().get(groupId).get(documentType).get(marcPath)
      : completenessDAO.getElementFrequency().get(documentType).get(marcPath);

    Map<Integer, Integer> histogram = null;
    if (groupId != null) {
      histogram = completenessDAO.getGroupedFieldHistogram().get(groupId).get(marcPath);
      if (!completenessDAO.getGroupedFieldHistogram().get(groupId).containsKey(marcPath)) {
        logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);
      }
    } else {
      histogram = completenessDAO.getFieldHistogram().get(marcPath);
      if (!completenessDAO.getFieldHistogram().containsKey(marcPath)) {
        logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);
      }
    }
    BasicStatistics statistics = new BasicStatistics(histogram);

    List<Object> values = Arrays.asList(
      (groupId != null ? groupId : 0),
      documentType, marcPathLabel, sortkey, packageId, packageLabel, tagLabel, subfieldLabel,
      frequency,   // = number-of-record
      cardinality, // = number-of-instances
      statistics.getMin(), statistics.getMax(),
      statistics.getMean(), statistics.getStdDev(),
      statistics.formatHistogram()
    );

    return CsvUtils.createCsvFromObjects(values);
  }

  private char getSeparator(ValidationErrorFormat format) {
    if (format.equals(ValidationErrorFormat.TAB_SEPARATED)) {
      return '\t';
    } else {
      return ',';
    }
  }

  @Override
  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());
    formatter.printHelp(message, options);
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.CsvUtils;
4	import de.gwdg.metadataqa.marc.Utils;
5	import de.gwdg.metadataqa.marc.analysis.GroupSelector;
6	import de.gwdg.metadataqa.marc.analysis.completeness.CompletenessDAO;
7	import de.gwdg.metadataqa.marc.analysis.completeness.RecordCompleteness;
8	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
9	import de.gwdg.metadataqa.marc.cli.parameters.CompletenessParameters;
10	import de.gwdg.metadataqa.marc.cli.plugin.CompletenessFactory;
11	import de.gwdg.metadataqa.marc.cli.plugin.CompletenessPlugin;
12	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
13	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
14	import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
15	import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordIgnorator;
16	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
17	import de.gwdg.metadataqa.marc.definition.tags.TagCategory;
18	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
19	import de.gwdg.metadataqa.marc.model.validation.ValidationErrorFormat;
20	import de.gwdg.metadataqa.marc.utils.BasicStatistics;
21	import de.gwdg.metadataqa.marc.utils.TagHierarchy;
22	import org.apache.commons.cli.HelpFormatter;
23	import org.apache.commons.cli.Options;
24	import org.apache.commons.cli.ParseException;
25	import org.marc4j.marc.Record;
26
27	import java.io.IOException;
28	import java.io.Serializable;
29	import java.nio.file.Files;
30	import java.nio.file.Path;
31	import java.nio.file.Paths;
32	import java.util.Arrays;
33	import java.util.HashMap;
34	import java.util.List;
35	import java.util.Map;
36	import java.util.logging.Level;
37	import java.util.logging.Logger;
38	import java.util.regex.Pattern;
39
40	public class Completeness extends QACli<CompletenessParameters> implements BibliographicInputProcessor, Serializable {
41
42	private static final Logger logger = Logger.getLogger(Completeness.class.getCanonicalName());	1✔
43	private static final Pattern dataFieldPattern = Pattern.compile("^(\\d\\d\\d)\\$(.*)$");	1✔
44	public static final String ALL_TYPE = "all";
45	private CompletenessDAO completenessDAO = new CompletenessDAO();	1✔
46
47	private boolean readyToProcess;
48	private CompletenessPlugin plugin;
49	private RecordFilter recordFilter;
50	private RecordIgnorator recordIgnorator;
51
52	public Completeness(String[] args) throws ParseException {	1✔
53	parameters = new CompletenessParameters(args);	1✔
54	plugin = CompletenessFactory.create(parameters);	1✔
55	recordFilter = parameters.getRecordFilter();	1✔
56	recordIgnorator = parameters.getRecordIgnorator();	1✔
57	readyToProcess = true;	1✔
58	initializeGroups(parameters.getGroupBy(), parameters.isPica());	1✔
59	if (doGroups()) {	1✔
60	initializeMeta(parameters);	1✔
61	if (doSaveGroupIds) {	1✔
62	idCollectorFile = prepareReportFile(parameters.getOutputDir(), "id-groupid.csv");	1✔
63	printToFile(idCollectorFile, CsvUtils.createCsv("id", "groupId"));	1✔
64	}
65	}
66	}	1✔
67
68	public static void main(String[] args) {
69	BibliographicInputProcessor processor = null;	×
70	try {
71	processor = new Completeness(args);	×
72	} catch (ParseException e) {	×
73	System.err.println("ERROR. " + e.getLocalizedMessage());	×
74	System.exit(1);	×
75	}	×
76	if (processor.getParameters().getArgs().length < 1) {	×
77	System.err.println("Please provide a MARC file name!");	×
78	processor.printHelp(processor.getParameters().getOptions());	×
79	System.exit(0);	×
80	}
81	if (processor.getParameters().doHelp()) {	×
82	processor.printHelp(processor.getParameters().getOptions());	×
83	System.exit(0);	×
84	}
85	RecordIterator iterator = new RecordIterator(processor);	×
NEW 86	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	×
87	iterator.start();	×
88	}	×
89
90	@Override
91	public CommonParameters getParameters() {
92	return parameters;	1✔
93	}
94
95	@Override
96	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
97	// do nothing
98	}	1✔
99
100	@Override
101	public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 102	processRecord(marcRecord, recordNumber);	×
103	}	×
104
105	@Override
106	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
107	logger.info("processRecord");	1✔
108	if (!recordFilter.isAllowable(bibliographicRecord))	1✔
109	return;	×
110
111	if (recordIgnorator.isIgnorable(bibliographicRecord))	1✔
112	return;	×
113
114	RecordCompleteness recordCompleteness = new RecordCompleteness(bibliographicRecord, parameters, completenessDAO, plugin, groupBy);	1✔
115	recordCompleteness.process();	1✔
116
117	if (doSaveGroupIds)	1✔
118	saveGroupIds(bibliographicRecord.getId(true), recordCompleteness.getGroupIds());	1✔
119	if (doGroups())	1✔
120	for (String id : recordCompleteness.getGroupIds())	1✔
121	Utils.count(id, completenessDAO.getGroupCounter());	1✔
122
123	for (String path : recordCompleteness.getRecordFrequency().keySet()) {	1✔
124	if (groupBy != null) {	1✔
125	for (String groupId : recordCompleteness.getGroupIds()) {	1✔
126	completenessDAO.getGroupedElementFrequency().computeIfAbsent(groupId, s -> new HashMap<>());	1✔
127	completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());	1✔
128	completenessDAO.getGroupedElementFrequency().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());	1✔
129	Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(recordCompleteness.getDocumentType()));	1✔
130	Utils.count(path, completenessDAO.getGroupedElementFrequency().get(groupId).get(ALL_TYPE));	1✔
131
132	completenessDAO.getGroupedFieldHistogram().computeIfAbsent(groupId, s -> new HashMap<>());	1✔
133	completenessDAO.getGroupedFieldHistogram().get(groupId).computeIfAbsent(path, s -> new HashMap<>());	1✔
134	Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getGroupedFieldHistogram().get(groupId).get(path));	1✔
135	}	1✔
136	} else {
137	Utils.count(path, completenessDAO.getElementFrequency().get(recordCompleteness.getDocumentType()));	1✔
138	Utils.count(path, completenessDAO.getElementFrequency().get(ALL_TYPE));	1✔
139	completenessDAO.getFieldHistogram().computeIfAbsent(path, s -> new HashMap<>());	1✔
140	Utils.count(recordCompleteness.getRecordFrequency().get(path), completenessDAO.getFieldHistogram().get(path));	1✔
141	}
142	}	1✔
143
144	for (String key : recordCompleteness.getRecordPackageCounter().keySet()) {	1✔
145	if (groupBy != null) {	1✔
146	for (String groupId : recordCompleteness.getGroupIds()) {	1✔
147	completenessDAO.getGroupedPackageCounter().computeIfAbsent(groupId, s -> new HashMap<>());	1✔
148	completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());	1✔
149	completenessDAO.getGroupedPackageCounter().get(groupId).computeIfAbsent(ALL_TYPE, s -> new HashMap<>());	1✔
150	Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(recordCompleteness.getDocumentType()));	1✔
151	Utils.count(key, completenessDAO.getGroupedPackageCounter().get(groupId).get(ALL_TYPE));	1✔
152	}	1✔
153	} else {
154	completenessDAO.getPackageCounter().computeIfAbsent(recordCompleteness.getDocumentType(), s -> new HashMap<>());	1✔
155	completenessDAO.getPackageCounter().computeIfAbsent(ALL_TYPE, s -> new HashMap<>());	1✔
156	Utils.count(key, completenessDAO.getPackageCounter().get(recordCompleteness.getDocumentType()));	1✔
157	Utils.count(key, completenessDAO.getPackageCounter().get(ALL_TYPE));	1✔
158	}
159	}	1✔
160	}	1✔
161
162	@Override
163	public void beforeIteration() {
164	logger.info(parameters.formatParameters());	1✔
165	completenessDAO.initialize();	1✔
166	}	1✔
167
168	@Override
169	public void fileOpened(Path file) {
170	// do nothing
171	}	1✔
172
173	@Override
174	public void fileProcessed() {
175	// do nothing
176	}	×
177
178	@Override
179	public void afterIteration(int numberOfprocessedRecords, long duration) {
180	String fileExtension = ".csv";	1✔
181	final char separator = getSeparator(parameters.getFormat());	1✔
182	if (parameters.getFormat().equals(ValidationErrorFormat.TAB_SEPARATED)) {	1✔
183	fileExtension = ".tsv";	×
184	}
185
186	saveLibraries003(fileExtension, separator);	1✔
187	saveLibraries(fileExtension, separator);	1✔
188	if (groupBy != null) {	1✔
189	saveGroups(fileExtension, separator);	1✔
190	saveGroupedPackages(fileExtension, separator);	1✔
191	saveGroupedMarcElements(fileExtension, separator);	1✔
192	} else {
193	savePackages(fileExtension, separator);	1✔
194	saveMarcElements(fileExtension, separator);	1✔
195	}
196	saveParameters("completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
197	}	1✔
198
199	private void saveLibraries003(String fileExtension, char separator) {
200	logger.info("Saving libraries003...");	1✔
201	var path = Paths.get(parameters.getOutputDir(), "libraries003" + fileExtension);	1✔
202	try (var writer = Files.newBufferedWriter(path)) {	1✔
203	writer.write(CsvUtils.createCsv("library", "count"));	1✔
204	completenessDAO.getLibrary003Counter().forEach((key, value) -> {	1✔
205	try {
206	writer.write(CsvUtils.createCsv(key, value));	1✔
207	} catch (IOException e) {	×
208	logger.log(Level.SEVERE, "saveLibraries003", e);	×
209	}	1✔
210	});	1✔
211	} catch (IOException e) {	×
212	logger.log(Level.SEVERE, "saveLibraries003", e);	×
213	}	1✔
214	}	1✔
215
216	private void saveMarcElements(String fileExtension, char separator) {
217	Path path = Paths.get(parameters.getOutputDir(), "marc-elements" + fileExtension);	1✔
218	try (var writer = Files.newBufferedWriter(path)) {	1✔
219	writer.write(CsvUtils.createCsv(	1✔
220	"groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
221	"number-of-record", "number-of-instances",
222	"min", "max", "mean", "stddev", "histogram"
223	));
224	completenessDAO.getElementCardinality().forEach((documentType, cardinalities) ->	1✔
225	cardinalities.forEach((marcPath, cardinality) -> {	1✔
226	try {
227	writer.write(formatCardinality(marcPath, cardinality, documentType, null));	1✔
228	} catch (IOException e) {	×
229	logger.log(Level.SEVERE, "saveMarcElements", e);	×
230	}	1✔
231	})	1✔
232	);
233	} catch (IOException e) {	×
234	logger.log(Level.SEVERE, "saveMarcElements", e);	×
235	}	1✔
236	}	1✔
237
238	private void saveGroupedMarcElements(String fileExtension, char separator) {
239	logger.info("saving grouped MARC elements...");	1✔
240	Path path = Paths.get(parameters.getOutputDir(), "completeness-grouped-marc-elements" + fileExtension);	1✔
241	try (var writer = Files.newBufferedWriter(path)) {	1✔
242	writer.write(CsvUtils.createCsv(	1✔
243	"groupId", "documenttype", "path", "sortkey", "packageid", "package", "tag", "subfield",
244	"number-of-record", "number-of-instances",
245	"min", "max", "mean", "stddev", "histogram"
246	));
247	completenessDAO.getGroupedElementCardinality().forEach((groupId, documentTypes) ->	1✔
248	documentTypes.forEach((documentType, cardinalities) ->	1✔
249	cardinalities.forEach((marcPath, cardinality) -> {	1✔
250	try {
251	writer.write(formatCardinality(marcPath, cardinality, documentType, groupId));	1✔
252	} catch (IOException e) {	×
253	logger.log(Level.SEVERE, "saveMarcElements", e);	×
254	}	1✔
255	})	1✔
256	)
257	);
258	} catch (IOException e) {	×
259	logger.log(Level.SEVERE, "saveMarcElements", e);	×
260	}	1✔
261	}	1✔
262
263	private void savePackages(String fileExtension, char separator) {
264	logger.info("saving packages...");	1✔
265	var path = Paths.get(parameters.getOutputDir(), "packages" + fileExtension);	1✔
266	try (var writer = Files.newBufferedWriter(path)) {	1✔
267	writer.write(CsvUtils.createCsv("documenttype", "packageid", "name", "label", "iscoretag", "count"));	1✔
268	completenessDAO.getPackageCounter().forEach((documentType, packages) ->	1✔
269	packages.forEach((packageName, count) -> {	1✔
270	try {
271	TagCategory tagCategory = TagCategory.getPackage(packageName);	1✔
272	String range = packageName;	1✔
273	String label = "";	1✔
274	int id = 100;	1✔
275	boolean isPartOfMarcScore = false;	1✔
276	if (tagCategory != null) {	1✔
277	id = tagCategory.getId();	1✔
278	range = tagCategory.getRange();	1✔
279	label = tagCategory.getLabel();	1✔
280	isPartOfMarcScore = tagCategory.isPartOfMarcCore();	1✔
281	} else {
282	logger.severe(packageName + " has not been found in TagCategory");	×
283	}
284	writer.write(CsvUtils.createCsv(documentType, id, range, label, isPartOfMarcScore, count));	1✔
285	} catch (IOException e) {	×
286	logger.log(Level.SEVERE, "savePackages", e);	×
287	}	1✔
288	})	1✔
289	);
290	} catch (IOException e) {	×
291	logger.log(Level.SEVERE, "savePackages", e);	×
292	}	1✔
293	}	1✔
294
295	private void saveGroupedPackages(String fileExtension, char separator) {
296	logger.info("saving grouped packages...");	1✔
297	var path = Paths.get(parameters.getOutputDir(), "completeness-grouped-packages" + fileExtension);	1✔
298	try (var writer = Files.newBufferedWriter(path)) {	1✔
299	writer.write(CsvUtils.createCsv("group", "documenttype", "packageid", "name", "label", "iscoretag", "count"));	1✔
300	completenessDAO.getGroupedPackageCounter()	1✔
301	.entrySet()	1✔
302	.stream()	1✔
303	.sorted(Map.Entry.comparingByKey())	1✔
304	.forEach(groupData -> {	1✔
305	String groupId = groupData.getKey();	1✔
306	Map<String, Map<String, Integer>> documentTypes = groupData.getValue();	1✔
307	documentTypes	1✔
308	.entrySet()	1✔
309	.stream()	1✔
310	.sorted(Map.Entry.comparingByKey())	1✔
311	.forEach(doctypeData -> {	1✔
312	String documentType = doctypeData.getKey();	1✔
313	Map<String, Integer> packages = doctypeData.getValue();	1✔
314	packages	1✔
315	.forEach((packageName, count) -> {	1✔
316	try {
317	TagCategory tagCategory = TagCategory.getPackage(packageName);	1✔
318	String range = packageName;	1✔
319	String label = "";	1✔
320	int id = 100;	1✔
321	boolean isPartOfMarcScore = false;	1✔
322	if (tagCategory != null) {	1✔
323	id = tagCategory.getId();	1✔
324	range = tagCategory.getRange();	1✔
325	label = tagCategory.getLabel();	1✔
326	isPartOfMarcScore = tagCategory.isPartOfMarcCore();	1✔
327	} else {
328	logger.severe(packageName + " has not been found in TagCategory");	×
329	}
330	writer.write(CsvUtils.createCsv(groupId, documentType, id, range, label, isPartOfMarcScore, count));	1✔
331	} catch (IOException e) {	×
332	logger.log(Level.SEVERE, "savePackages", e);	×
333	}	1✔
334	});	1✔
335	});	1✔
336	});	1✔
337	} catch (IOException e) {	×
338	logger.log(Level.SEVERE, "savePackages", e);	×
339	}	1✔
340	}	1✔
341
342	private void saveLibraries(String fileExtension, char separator) {
343	logger.info("Saving libraries...");	1✔
344	var path = Paths.get(parameters.getOutputDir(), "libraries" + fileExtension);	1✔
345	try (var writer = Files.newBufferedWriter(path)) {	1✔
346	writer.write(CsvUtils.createCsv("library", "count"));	1✔
347	completenessDAO.getLibraryCounter().forEach((key, value) -> {	1✔
348	try {
349	writer.write(CsvUtils.createCsv(key, value));	×
350	} catch (IOException e) {	×
351	logger.log(Level.SEVERE, "saveLibraries", e);	×
352	}	×
353	});	×
354	} catch (IOException e) {	×
355	logger.log(Level.SEVERE, "saveLibraries", e);	×
356	}	1✔
357	}	1✔
358
359	private void saveGroups(String fileExtension, char separator) {
360	logger.info("Saving groups...");	1✔
361	GroupSelector groupSelector = new GroupSelector(parameters.getGroupListFile());	1✔
362	var path = Paths.get(parameters.getOutputDir(), "completeness-groups" + fileExtension);	1✔
363	try (var writer = Files.newBufferedWriter(path)) {	1✔
364	writer.write(CsvUtils.createCsv("id", "group", "count"));	1✔
365	completenessDAO.getGroupCounter()	1✔
366	.entrySet()	1✔
367	.stream()	1✔
368	.sorted((a,b) -> a.getKey().compareTo(b.getKey()))	1✔
369	.forEach(item -> {	1✔
370	try {
371	writer.write(CsvUtils.createCsv(item.getKey(), groupSelector.getOrgName(item.getKey()), item.getValue()));	1✔
372	} catch (IOException e) {	×
373	logger.log(Level.SEVERE, "saveLibraries", e);	×
374	}	1✔
375	});	1✔
376	} catch (IOException e) {	×
377	logger.log(Level.SEVERE, "saveLibraries", e);	×
378	}	1✔
379	}	1✔
380
381	private String formatCardinality(String marcPath,
382	int cardinality,
383	String documentType,
384	String groupId) {
385	if (marcPath.equals("")) {	1✔
386	logger.severe("Empty key from " + marcPath);	×
387	}
388
389	String marcPathLabel = marcPath.replace("!ind", "ind").replaceAll("\\\|(\\d)$", "$1");	1✔
390	String sortkey = marcPath.replaceAll("^leader", "000");	1✔
391	int packageId = TagCategory.OTHER.getId();	1✔
392	String packageLabel = TagCategory.OTHER.getLabel();	1✔
393	String tagLabel = "";	1✔
394	String subfieldLabel = "";	1✔
395	TagHierarchy tagHierarchy = plugin.getTagHierarchy(marcPathLabel);	1✔
396	if (tagHierarchy != null) {	1✔
397	packageId = tagHierarchy.getPackageId();	1✔
398	packageLabel = tagHierarchy.getPackageLabel();	1✔
399	tagLabel = tagHierarchy.getTagLabel();	1✔
400	subfieldLabel = tagHierarchy.getSubfieldLabel();	1✔
401	} else {
402	logger.severe(() -> "Key can not be found in the TagHierarchy: " + marcPathLabel);	1✔
403	}
404
405	// Integer cardinality = entry.getValue();
406	Integer frequency = (groupId != null)	1✔
407	? completenessDAO.getGroupedElementFrequency().get(groupId).get(documentType).get(marcPath)	1✔
408	: completenessDAO.getElementFrequency().get(documentType).get(marcPath);	1✔
409
410	Map<Integer, Integer> histogram = null;	1✔
411	if (groupId != null) {	1✔
412	histogram = completenessDAO.getGroupedFieldHistogram().get(groupId).get(marcPath);	1✔
413	if (!completenessDAO.getGroupedFieldHistogram().get(groupId).containsKey(marcPath)) {	1✔
414	logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);	×
415	}
416	} else {
417	histogram = completenessDAO.getFieldHistogram().get(marcPath);	1✔
418	if (!completenessDAO.getFieldHistogram().containsKey(marcPath)) {	1✔
419	logger.log(Level.WARNING,"Field {0} is not registered in histogram", marcPath);	×
420	}
421	}
422	BasicStatistics statistics = new BasicStatistics(histogram);	1✔
423
424	List<Object> values = Arrays.asList(	1✔
425	(groupId != null ? groupId : 0),	1✔
426	documentType, marcPathLabel, sortkey, packageId, packageLabel, tagLabel, subfieldLabel,	1✔
427	frequency, // = number-of-record
428	cardinality, // = number-of-instances	1✔
429	statistics.getMin(), statistics.getMax(),	1✔
430	statistics.getMean(), statistics.getStdDev(),	1✔
431	statistics.formatHistogram()	1✔
432	);
433
434	return CsvUtils.createCsvFromObjects(values);	1✔
435	}
436
437	private char getSeparator(ValidationErrorFormat format) {
438	if (format.equals(ValidationErrorFormat.TAB_SEPARATED)) {	1✔
439	return '\t';	×
440	} else {
441	return ',';	1✔
442	}
443	}
444
445	@Override
446	public void printHelp(Options options) {
447	HelpFormatter formatter = new HelpFormatter();	×
448	String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());	×
449	formatter.printHelp(message, options);	×
450	}	×
451
452	@Override
453	public boolean readyToProcess() {
454	return readyToProcess;	1✔
455	}
456	}

pkiraly / metadata-qa-marc / #1447

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous