#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/de/gwdg/metadataqa/marc/cli/DataElements.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.analysis.DataElementCounter;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.CompletenessParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import de.gwdg.metadataqa.marc.model.validation.ValidationErrorFormat;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.marc4j.marc.Record;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;

public class DataElements implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(DataElements.class.getCanonicalName());
  private static final Pattern dataFieldPattern = Pattern.compile("^(\\d\\d\\d)\\$(.*)$");

  private final Options options;
  private CompletenessParameters parameters;
  private Map<String, Integer> library003Counter = new TreeMap<>();
  private Map<String, Integer> libraryCounter = new TreeMap<>();
  private Map<String, Map<String, Integer>> packageCounter = new TreeMap<>();
  private Map<String, Map<String, Integer>> elementCardinality = new TreeMap<>();
  private Map<String, Map<String, Integer>> elementFrequency = new TreeMap<>();
  private Map<String, Map<Integer, Integer>> fieldHistogram = new HashMap<>();
  private boolean readyToProcess;
  private DataElementCounter dataElementCounter;
  private File outputFile;

  public DataElements(String[] args) throws ParseException {
    parameters = new CompletenessParameters(args);
    options = parameters.getOptions();
    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new DataElements(args);
    } catch (ParseException e) {
      System.err.println("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
    if (processor.getParameters().getArgs().length < 1) {
      System.err.println("Please provide a MARC file name!");
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    if (parameters.getRecordIgnorator().isIgnorable(bibliographicRecord))
      return;

    printToFile(outputFile, StringUtils.join(dataElementCounter.count(bibliographicRecord), ",") + "\n");
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters.formatParameters());
    elementCardinality.put("all", new TreeMap<>());
    elementFrequency.put("all", new TreeMap<>());
    packageCounter.put("all", new TreeMap<>());
    dataElementCounter = new DataElementCounter(parameters.getOutputDir(), "top-fields.txt", DataElementCounter.Basis.EXISTENCE);
    outputFile = new File(parameters.getOutputDir(), "record-patterns.csv");
    if (outputFile.exists()) {
      try {
        Files.delete(outputFile.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());
      }
    }
    printToFile(outputFile, dataElementCounter.getHeader() + "\n");
  }

  @Override
  public void fileOpened(Path file) {
    // do nothing
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    // do nothing
  }

  private void printToFile(File file, String message) {
    try {
      FileUtils.writeStringToFile(file, message, StandardCharsets.UTF_8, true);
    } catch (IOException e) {
      if (parameters.doLog())
        logger.log(Level.SEVERE, "printToFile", e);
    }
  }

  private char getSeparator(ValidationErrorFormat format) {
    if (format.equals(ValidationErrorFormat.TAB_SEPARATED)) {
      return '\t';
    } else {
      return ',';
    }
  }

  @Override
  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());
    formatter.printHelp(message, options);
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.analysis.DataElementCounter;
4	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
5	import de.gwdg.metadataqa.marc.cli.parameters.CompletenessParameters;
6	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
7	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
8	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
9	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
10	import de.gwdg.metadataqa.marc.model.validation.ValidationErrorFormat;
11	import org.apache.commons.cli.HelpFormatter;
12	import org.apache.commons.cli.Options;
13	import org.apache.commons.cli.ParseException;
14	import org.apache.commons.io.FileUtils;
15	import org.apache.commons.lang3.StringUtils;
16	import org.marc4j.marc.Record;
17
18	import java.io.File;
19	import java.io.IOException;
20	import java.io.Serializable;
21	import java.nio.charset.StandardCharsets;
22	import java.nio.file.Files;
23	import java.nio.file.Path;
24	import java.util.HashMap;
25	import java.util.List;
26	import java.util.Map;
27	import java.util.TreeMap;
28	import java.util.logging.Level;
29	import java.util.logging.Logger;
30	import java.util.regex.Pattern;
31
32	public class DataElements implements BibliographicInputProcessor, Serializable {
33
34	private static final Logger logger = Logger.getLogger(DataElements.class.getCanonicalName());	×
35	private static final Pattern dataFieldPattern = Pattern.compile("^(\\d\\d\\d)\\$(.*)$");	×
36
37	private final Options options;
38	private CompletenessParameters parameters;
39	private Map<String, Integer> library003Counter = new TreeMap<>();	×
40	private Map<String, Integer> libraryCounter = new TreeMap<>();	×
41	private Map<String, Map<String, Integer>> packageCounter = new TreeMap<>();	×
42	private Map<String, Map<String, Integer>> elementCardinality = new TreeMap<>();	×
43	private Map<String, Map<String, Integer>> elementFrequency = new TreeMap<>();	×
44	private Map<String, Map<Integer, Integer>> fieldHistogram = new HashMap<>();	×
45	private boolean readyToProcess;
46	private DataElementCounter dataElementCounter;
47	private File outputFile;
48
49	public DataElements(String[] args) throws ParseException {	×
50	parameters = new CompletenessParameters(args);	×
51	options = parameters.getOptions();	×
52	readyToProcess = true;	×
53	}	×
54
55	public static void main(String[] args) {
56	BibliographicInputProcessor processor = null;	×
57	try {
58	processor = new DataElements(args);	×
59	} catch (ParseException e) {	×
60	System.err.println("ERROR. " + e.getLocalizedMessage());	×
61	System.exit(1);	×
62	}	×
63	if (processor.getParameters().getArgs().length < 1) {	×
64	System.err.println("Please provide a MARC file name!");	×
65	processor.printHelp(processor.getParameters().getOptions());	×
66	System.exit(0);	×
67	}
68	if (processor.getParameters().doHelp()) {	×
69	processor.printHelp(processor.getParameters().getOptions());	×
70	System.exit(0);	×
71	}
72	RecordIterator iterator = new RecordIterator(processor);	×
NEW 73	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	×
74	iterator.start();	×
75	}	×
76
77	@Override
78	public CommonParameters getParameters() {
UNCOV 79	return parameters;	×
80	}
81
82	@Override
83	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
84	// do nothing
UNCOV 85	}	×
86
87	@Override
88	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 89	processRecord(bibliographicRecord, recordNumber);	×
90	}	×
91
92	@Override
93	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
NEW 94	if (parameters.getRecordIgnorator().isIgnorable(bibliographicRecord))	×
95	return;	×
96
NEW 97	printToFile(outputFile, StringUtils.join(dataElementCounter.count(bibliographicRecord), ",") + "\n");	×
98	}	×
99
100	@Override
101	public void beforeIteration() {
UNCOV 102	logger.info(parameters.formatParameters());	×
103	elementCardinality.put("all", new TreeMap<>());	×
104	elementFrequency.put("all", new TreeMap<>());	×
105	packageCounter.put("all", new TreeMap<>());	×
106	dataElementCounter = new DataElementCounter(parameters.getOutputDir(), "top-fields.txt", DataElementCounter.Basis.EXISTENCE);	×
107	outputFile = new File(parameters.getOutputDir(), "record-patterns.csv");	×
NEW 108	if (outputFile.exists()) {	×
109	try {
NEW 110	Files.delete(outputFile.toPath());	×
NEW 111	} catch (IOException e) {	×
NEW 112	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());	×
NEW 113	}	×
114	}
115	printToFile(outputFile, dataElementCounter.getHeader() + "\n");	×
116	}	×
117
118	@Override
119	public void fileOpened(Path file) {
120	// do nothing
121	}	×
122
123	@Override
124	public void fileProcessed() {
125	// do nothing
UNCOV 126	}	×
127
128	@Override
129	public void afterIteration(int numberOfprocessedRecords, long duration) {
130	// do nothing
UNCOV 131	}	×
132
133	private void printToFile(File file, String message) {
134	try {
UNCOV 135	FileUtils.writeStringToFile(file, message, StandardCharsets.UTF_8, true);	×
UNCOV 136	} catch (IOException e) {	×
137	if (parameters.doLog())	×
UNCOV 138	logger.log(Level.SEVERE, "printToFile", e);	×
UNCOV 139	}	×
UNCOV 140	}	×
141
142	private char getSeparator(ValidationErrorFormat format) {
143	if (format.equals(ValidationErrorFormat.TAB_SEPARATED)) {	×
144	return '\t';	×
145	} else {
146	return ',';	×
147	}
148	}
149
150	@Override
151	public void printHelp(Options options) {
152	HelpFormatter formatter = new HelpFormatter();	×
UNCOV 153	String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());	×
UNCOV 154	formatter.printHelp(message, options);	×
UNCOV 155	}	×
156
157	@Override
158	public boolean readyToProcess() {
159	return readyToProcess;	×
160	}
161	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous