#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.67

/src/main/java/de/gwdg/metadataqa/marc/cli/SerialScore.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.analysis.serial.Marc21Serial;
import de.gwdg.metadataqa.marc.analysis.serial.MarcSerial;
import de.gwdg.metadataqa.marc.analysis.serial.SerialFields;
import de.gwdg.metadataqa.marc.analysis.serial.UnimarcSerial;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.SerialScoreParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.MarcLeader;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.dao.record.Marc21Record;
import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
import de.gwdg.metadataqa.marc.dao.record.UnimarcRecord;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.marc4j.marc.Record;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static de.gwdg.metadataqa.marc.Utils.createRow;
import static de.gwdg.metadataqa.marc.Utils.quote;

/**
 * usage:
 * java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar \
 * de.gwdg.metadataqa.marc.cli.SerialScore [MARC21 file]
 *
 * @author Péter Király <peter.kiraly at gwdg.de>
 */
public class SerialScore extends QACli<SerialScoreParameters> implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(
    SerialScore.class.getCanonicalName()
  );
  private final Options options;
  private final boolean readyToProcess;
  private File output = null;
  private Map<Integer, Integer> histogram = new HashMap<>();

  public SerialScore(String[] args) throws ParseException {
    parameters = new SerialScoreParameters(args);
    options = parameters.getOptions();
    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new SerialScore(args);
    } catch (ParseException e) {
      logger.severe("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }

    if (processor.getParameters().getArgs().length < 1) {
      logger.severe("Please provide a MARC file name!");
      System.exit(0);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }

    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(() -> parameters.formatParameters());
    printFields();

    output = new File(parameters.getOutputDir(), parameters.getFileName());
    if (output.exists()) {
      try {
        Files.delete(output.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", output.getAbsolutePath());
      }
    }

    print(createRow(MarcSerial.getHeaders()));
  }

  @Override
  public void fileOpened(Path path) {
    // do nothing
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) {
    if (!(bibliographicRecord instanceof MarcRecord)) {
      return;
    }

    MarcRecord marcRecord = (MarcRecord) bibliographicRecord;
    if (!marcRecord.getType().equals(MarcLeader.Type.CONTINUING_RESOURCES)) {
      return;
    }

    if (parameters.getRecordIgnorator().isIgnorable(marcRecord)) {
      return;
    }

    MarcSerial serial;
    if (marcRecord instanceof Marc21Record) {
      serial = new Marc21Serial((Marc21Record) marcRecord);
    } else {
      serial = new UnimarcSerial((UnimarcRecord) marcRecord);
    }

    // Count the scores
    List<Integer> scores = serial.determineRecordQualityScore();

    // Create the histogram message
    String message = createRow(
      quote(marcRecord.getId().trim()), StringUtils.join(scores, ",")
    );
    print(message);
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    printHistogram();
    saveParameters("serials.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  private void printHistogram() {
    // FIXME histogram is never updated
    Path path;
    path = Paths.get(parameters.getOutputDir(), "serial-histogram.csv");
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(createRow("score", "frequency"));
      histogram
        .entrySet()
        .stream()
        .sorted(Map.Entry.comparingByKey())
        .forEach(
          entry -> {
            try {
              writer.write(createRow(entry.getKey(), entry.getValue()));
            } catch (IOException e) {
              logger.log(Level.SEVERE, "printHistogram", e);
            }
          }
        );
    } catch (IOException e) {
      logger.log(Level.SEVERE, "printHistogram", e);
    }
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }

  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format(
      "java -cp qa-catalogue.jar %s [options] [file]",
      this.getClass().getCanonicalName()
    );
    formatter.printHelp(message, options);
  }

  private void print(String message) {
    try {
      FileUtils.writeStringToFile(output, message, StandardCharsets.UTF_8, true);
    } catch (IOException e) {
      logger.log(Level.SEVERE, "print", e);
    }
  }

  private void printFields() {
    var path = Paths.get(parameters.getOutputDir(), "serial-score-fields.csv");
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(createRow("name", "transformed"));
      for (SerialFields field : SerialFields.values()) {
        try {
          writer.write(createRow(quote(field.getLabel()), field.getMachine()));
        } catch (IOException e) {
          logger.log(Level.SEVERE, "printFields", e);
        }
      }
    } catch (IOException e) {
      logger.log(Level.SEVERE, "printFields", e);
    }
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.analysis.serial.Marc21Serial;
4	import de.gwdg.metadataqa.marc.analysis.serial.MarcSerial;
5	import de.gwdg.metadataqa.marc.analysis.serial.SerialFields;
6	import de.gwdg.metadataqa.marc.analysis.serial.UnimarcSerial;
7	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
8	import de.gwdg.metadataqa.marc.cli.parameters.SerialScoreParameters;
9	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
10	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
11	import de.gwdg.metadataqa.marc.dao.MarcLeader;
12	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
13	import de.gwdg.metadataqa.marc.dao.record.Marc21Record;
14	import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
15	import de.gwdg.metadataqa.marc.dao.record.UnimarcRecord;
16	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
17	import org.apache.commons.cli.HelpFormatter;
18	import org.apache.commons.cli.Options;
19	import org.apache.commons.cli.ParseException;
20	import org.apache.commons.io.FileUtils;
21	import org.apache.commons.lang3.StringUtils;
22	import org.marc4j.marc.Record;
23
24	import java.io.File;
25	import java.io.IOException;
26	import java.io.Serializable;
27	import java.nio.charset.StandardCharsets;
28	import java.nio.file.Files;
29	import java.nio.file.Path;
30	import java.nio.file.Paths;
31	import java.util.HashMap;
32	import java.util.List;
33	import java.util.Map;
34	import java.util.logging.Level;
35	import java.util.logging.Logger;
36
37	import static de.gwdg.metadataqa.marc.Utils.createRow;
38	import static de.gwdg.metadataqa.marc.Utils.quote;
39
40	/**
41	* usage:
42	* java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar \
43	* de.gwdg.metadataqa.marc.cli.SerialScore [MARC21 file]
44	*
45	* @author Péter Király <peter.kiraly at gwdg.de>
46	*/
47	public class SerialScore extends QACli<SerialScoreParameters> implements BibliographicInputProcessor, Serializable {
48
49	private static final Logger logger = Logger.getLogger(	1✔
50	SerialScore.class.getCanonicalName()	1✔
51	);
52	private final Options options;
53	private final boolean readyToProcess;
54	private File output = null;	1✔
55	private Map<Integer, Integer> histogram = new HashMap<>();	1✔
56
57	public SerialScore(String[] args) throws ParseException {	1✔
58	parameters = new SerialScoreParameters(args);	1✔
59	options = parameters.getOptions();	1✔
60	readyToProcess = true;	1✔
61	}	1✔
62
63	public static void main(String[] args) {
64	BibliographicInputProcessor processor = null;	1✔
65	try {
66	processor = new SerialScore(args);	1✔
UNCOV 67	} catch (ParseException e) {	×
NEW 68	logger.severe("ERROR. " + e.getLocalizedMessage());	×
UNCOV 69	System.exit(1);	×
70	}	1✔
71
72	if (processor.getParameters().getArgs().length < 1) {	1✔
NEW 73	logger.severe("Please provide a MARC file name!");	×
UNCOV 74	System.exit(0);	×
75	}
76	if (processor.getParameters().doHelp()) {	1✔
77	processor.printHelp(processor.getParameters().getOptions());	×
78	System.exit(0);	×
79	}
80
81	RecordIterator iterator = new RecordIterator(processor);	1✔
82	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	1✔
83	iterator.start();	1✔
84	}	1✔
85
86	@Override
87	public CommonParameters getParameters() {
88	return parameters;	1✔
89	}
90
91	@Override
92	public void beforeIteration() {
93	logger.info(() -> parameters.formatParameters());	1✔
94	printFields();	1✔
95
96	output = new File(parameters.getOutputDir(), parameters.getFileName());	1✔
97	if (output.exists()) {	1✔
98	try {
99	Files.delete(output.toPath());	1✔
NEW 100	} catch (IOException e) {	×
NEW 101	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", output.getAbsolutePath());	×
102	}	1✔
103	}
104
105	print(createRow(MarcSerial.getHeaders()));	1✔
106	}	1✔
107
108	@Override
109	public void fileOpened(Path path) {
110	// do nothing
111	}	1✔
112
113	@Override
114	public void processRecord(Record marc4jRecord, int recordNumber) {
115	// do nothing
116	}	1✔
117
118	@Override
119	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 120	processRecord(bibliographicRecord, recordNumber);	×
UNCOV 121	}	×
122
123	@Override
124	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) {
125	if (!(bibliographicRecord instanceof MarcRecord)) {	1✔
NEW 126	return;	×
127	}
128
129	MarcRecord marcRecord = (MarcRecord) bibliographicRecord;	1✔
130	if (!marcRecord.getType().equals(MarcLeader.Type.CONTINUING_RESOURCES)) {	1✔
NEW 131	return;	×
132	}
133
134	if (parameters.getRecordIgnorator().isIgnorable(marcRecord)) {	1✔
NEW 135	return;	×
136	}
137
138	MarcSerial serial;
139	if (marcRecord instanceof Marc21Record) {	1✔
140	serial = new Marc21Serial((Marc21Record) marcRecord);	1✔
141	} else {
142	serial = new UnimarcSerial((UnimarcRecord) marcRecord);	1✔
143	}
144
145	// Count the scores
146	List<Integer> scores = serial.determineRecordQualityScore();	1✔
147
148	// Create the histogram message
149	String message = createRow(	1✔
150	quote(marcRecord.getId().trim()), StringUtils.join(scores, ",")	1✔
151	);
152	print(message);	1✔
153	}	1✔
154
155	@Override
156	public void fileProcessed() {
157	// do nothing
UNCOV 158	}	×
159
160	@Override
161	public void afterIteration(int numberOfprocessedRecords, long duration) {
162	printHistogram();	1✔
163	saveParameters("serials.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
164	}	1✔
165
166	private void printHistogram() {
167	// FIXME histogram is never updated
168	Path path;
169	path = Paths.get(parameters.getOutputDir(), "serial-histogram.csv");	1✔
170	try (var writer = Files.newBufferedWriter(path)) {	1✔
171	writer.write(createRow("score", "frequency"));	1✔
172	histogram	1✔
173	.entrySet()	1✔
174	.stream()	1✔
175	.sorted(Map.Entry.comparingByKey())	1✔
176	.forEach(	1✔
177	entry -> {
178	try {
UNCOV 179	writer.write(createRow(entry.getKey(), entry.getValue()));	×
UNCOV 180	} catch (IOException e) {	×
UNCOV 181	logger.log(Level.SEVERE, "printHistogram", e);	×
UNCOV 182	}	×
UNCOV 183	}	×
184	);
UNCOV 185	} catch (IOException e) {	×
UNCOV 186	logger.log(Level.SEVERE, "printHistogram", e);	×
187	}	1✔
188	}	1✔
189
190	@Override
191	public boolean readyToProcess() {
192	return readyToProcess;	1✔
193	}
194
195	public void printHelp(Options options) {
UNCOV 196	HelpFormatter formatter = new HelpFormatter();	×
UNCOV 197	String message = String.format(	×
198	"java -cp qa-catalogue.jar %s [options] [file]",
UNCOV 199	this.getClass().getCanonicalName()	×
200	);
UNCOV 201	formatter.printHelp(message, options);	×
UNCOV 202	}	×
203
204	private void print(String message) {
205	try {
206	FileUtils.writeStringToFile(output, message, StandardCharsets.UTF_8, true);	1✔
UNCOV 207	} catch (IOException e) {	×
208	logger.log(Level.SEVERE, "print", e);	×
209	}	1✔
210	}	1✔
211
212	private void printFields() {
213	var path = Paths.get(parameters.getOutputDir(), "serial-score-fields.csv");	1✔
214	try (var writer = Files.newBufferedWriter(path)) {	1✔
215	writer.write(createRow("name", "transformed"));	1✔
216	for (SerialFields field : SerialFields.values()) {	1✔
217	try {
218	writer.write(createRow(quote(field.getLabel()), field.getMachine()));	1✔
UNCOV 219	} catch (IOException e) {	×
UNCOV 220	logger.log(Level.SEVERE, "printFields", e);	×
221	}	1✔
222	}
UNCOV 223	} catch (IOException e) {	×
UNCOV 224	logger.log(Level.SEVERE, "printFields", e);	×
225	}	1✔
226	}	1✔
227	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous