#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.21

/src/main/java/de/gwdg/metadataqa/marc/cli/Formatter.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.cli.parameters.FormatterParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.DataField;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import de.gwdg.metadataqa.marc.utils.SchemaSpec;
import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import org.marc4j.MarcException;
import org.marc4j.MarcXmlWriter;
import org.marc4j.marc.Record;

import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * usage:
 * java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar de.gwdg.metadataqa.marc.cli.Validator [MARC21 file]
 *
 * @author Péter Király <peter.kiraly at gwdg.de>
 */
public class Formatter implements BibliographicInputProcessor {

  private static final Logger logger = Logger.getLogger(Formatter.class.getCanonicalName());

  private final FormatterParameters parameters;
  private final boolean readyToProcess;
  private BufferedWriter writer;
  private MarcXmlWriter marcXmlWriter;

  public Formatter(String[] args) throws ParseException {
    parameters = new FormatterParameters(args);
    readyToProcess = true;
  }

  public static void main(String[] args) {
    try {
      BibliographicInputProcessor processor = new Formatter(args);
      if (processor.getParameters().getArgs().length < 1) {
        logger.severe("Please provide a MARC file name!");
        System.exit(1);
      }
      if (processor.getParameters().doHelp()) {
        processor.printHelp(processor.getParameters().getOptions());
        System.exit(0);
      }
      RecordIterator iterator = new RecordIterator(processor);
      logger.info(() -> processor.getParameters().formatParameters());
      iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
      iterator.start();
    } catch(Exception e) {
      logger.severe(() -> "ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
  }

  @Override
  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());
    formatter.printHelp(message, options);
  }

  @Override
  public FormatterParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters::formatParameters);

    // print headers
    if (parameters.hasSelector()) {
      logger.info("has selector");
      var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());
      try {
        writer = Files.newBufferedWriter(path);
      } catch (IOException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
      List<String> values = new ArrayList<>();
      if (parameters.withId())
        values.add("id");
      for (SchemaSpec spec : parameters.getSelector()) {
        values.add(spec.encode());
      }
      try {
        writer.write(StringUtils.join(values, parameters.getSeparator()) + "\n");
      } catch (IOException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
    }
    if (parameters.getIds() != null && !parameters.getIds().isEmpty()
        && parameters.getFormat().equals("xml")) {
      var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());
      logger.info("path: " + path.toAbsolutePath());
      try {
        // outputStream = new FileOutputStream(path.toFile());
        marcXmlWriter = new MarcXmlWriter(new FileOutputStream(path.toFile()));
        marcXmlWriter.setIndent(true);
      } catch (FileNotFoundException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
    }
  }

  @Override
  public void fileOpened(Path file) {
    // do nothing
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    String id = marc4jRecord.getControlNumber() != null ?
      marc4jRecord.getControlNumber().trim() : null;

    boolean hasSpecifiedId = parameters.hasId() && id != null && id.equals(parameters.getId());

    if (!hasSpecifiedId)
      hasSpecifiedId = id != null
                       && parameters.getIds() != null
                       && !parameters.getIds().isEmpty()
                       && parameters.getIds().contains(id);

    boolean hasSpecifiedRecordNumber = parameters.getCountNr() > -1
      && parameters.getCountNr() == recordNumber;

    if (hasSpecifiedId || hasSpecifiedRecordNumber) {
      if (parameters.getFormat().equals("xml")) {
        marcXmlWriter.write(marc4jRecord);
        // MarcXmlWriter.writeSingleRecord(marc4jRecord, System.out, true);
        // MarcXmlWriter.writeSingleRecord(marc4jRecord, outputStream, true);
      } else {
        logger.info(marc4jRecord::toString);
      }
    }
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    if (parameters.hasId() && bibliographicRecord.getId().trim().equals(parameters.getId())) {
      for (DataField field : bibliographicRecord.getDatafields()) {
       logger.info(field.getTag());
      }
    }

    if (parameters.hasSearch()) {
      List<String> results = bibliographicRecord.search(parameters.getPath(), parameters.getQuery());
      if (!results.isEmpty()) {
        logger.info(bibliographicRecord::toString);
      }
    }
    if (!parameters.hasSelector()) {
      return;
    }

    List<String> selectionResults = new ArrayList<>();
    if (parameters.withId()) {
      selectionResults.add(bibliographicRecord.getId());
    }
    if (parameters.getSchemaType().equals(SchemaType.PICA)) {
      List<String> selectedPicaResults = selectPicaResults((PicaRecord) bibliographicRecord);
      selectionResults.addAll(selectedPicaResults);
    } else {
      List<String> selectedMarcResults = selectMarcResults((MarcRecord) bibliographicRecord);
      selectionResults.addAll(selectedMarcResults);
    }

    try {
      writer.write(StringUtils.join(selectionResults, parameters.getSeparator()) + "\n");
    } catch (IOException e) {
      logger.log(Level.SEVERE, "processRecord", e);
    }
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {

    if (writer != null)
      try {
        writer.close();
      } catch (IOException e) {
        logger.log(Level.SEVERE, "afterIteration", e);
      }

    if (marcXmlWriter != null)
      try {
        marcXmlWriter.close();
      } catch (MarcException e) {
        logger.log(Level.SEVERE, "afterIteration", e);
      }
  }

  private List<String> selectPicaResults(PicaRecord picaRecord) {
    List<String> selectionResults = new ArrayList<>();
    for (SchemaSpec marcSpec : parameters.getSelector()) {
      PicaSpec spec = (PicaSpec) marcSpec;
      List<String> results = picaRecord.select(spec);

      if (results.isEmpty() || spec.getFunction() == null) {
        selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
        continue;
      }

      List<String> candidates = new ArrayList<>();
      for (String result : results) {
        if (spec.getFunction().equals("extractPicaDate")) {
          candidates.add(extractPicaDate(result));
        }
      }
      results = candidates;
      selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
    }
    return selectionResults;
  }

  private List<String> selectMarcResults(MarcRecord marcRecord) {
    List<String> selectionResults = new ArrayList<>();
    for (SchemaSpec marcSpec : parameters.getSelector()) {
      List<String> results = marcRecord.select(marcSpec);
      selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
    }
    return selectionResults;
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }

  public static String extractPicaDate(String dateInString) {
    String[] parts1 = dateInString.split(":", 2);
    String[] dateParts = parts1[1].split("-");
    return dateParts[2] + dateParts[1] + dateParts[0];
    // DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd-yy", Locale.ENGLISH);
    // LocalDate dateTime = LocalDate.parse(dateInString, formatter);
    // return dateTime;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.cli.parameters.FormatterParameters;
4	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
5	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
6	import de.gwdg.metadataqa.marc.dao.DataField;
7	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
8	import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
9	import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
10	import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
11	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
12	import de.gwdg.metadataqa.marc.utils.SchemaSpec;
13	import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
14	import org.apache.commons.cli.HelpFormatter;
15	import org.apache.commons.cli.Options;
16	import org.apache.commons.cli.ParseException;
17	import org.apache.commons.lang3.StringUtils;
18	import org.marc4j.MarcException;
19	import org.marc4j.MarcXmlWriter;
20	import org.marc4j.marc.Record;
21
22	import java.io.BufferedWriter;
23	import java.io.FileNotFoundException;
24	import java.io.FileOutputStream;
25	import java.io.IOException;
26	import java.io.OutputStream;
27	import java.nio.file.Files;
28	import java.nio.file.Path;
29	import java.nio.file.Paths;
30	import java.util.ArrayList;
31	import java.util.List;
32	import java.util.logging.Level;
33	import java.util.logging.Logger;
34
35	/**
36	* usage:
37	* java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar de.gwdg.metadataqa.marc.cli.Validator [MARC21 file]
38	*
39	* @author Péter Király <peter.kiraly at gwdg.de>
40	*/
41	public class Formatter implements BibliographicInputProcessor {
42
43	private static final Logger logger = Logger.getLogger(Formatter.class.getCanonicalName());	1✔
44
45	private final FormatterParameters parameters;
46	private final boolean readyToProcess;
47	private BufferedWriter writer;
48	private MarcXmlWriter marcXmlWriter;
49
50	public Formatter(String[] args) throws ParseException {	1✔
51	parameters = new FormatterParameters(args);	1✔
52	readyToProcess = true;	1✔
53	}	1✔
54
55	public static void main(String[] args) {
56	try {
UNCOV 57	BibliographicInputProcessor processor = new Formatter(args);	×
UNCOV 58	if (processor.getParameters().getArgs().length < 1) {	×
NEW 59	logger.severe("Please provide a MARC file name!");	×
UNCOV 60	System.exit(1);	×
61	}
UNCOV 62	if (processor.getParameters().doHelp()) {	×
63	processor.printHelp(processor.getParameters().getOptions());	×
64	System.exit(0);	×
65	}
66	RecordIterator iterator = new RecordIterator(processor);	×
NEW 67	logger.info(() -> processor.getParameters().formatParameters());	×
NEW 68	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	×
69	iterator.start();	×
70	} catch(Exception e) {	×
NEW 71	logger.severe(() -> "ERROR. " + e.getLocalizedMessage());	×
UNCOV 72	System.exit(1);	×
73	}	×
74	}	×
75
76	@Override
77	public void printHelp(Options options) {
78	HelpFormatter formatter = new HelpFormatter();	×
79	String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());	×
80	formatter.printHelp(message, options);	×
81	}	×
82
83	@Override
84	public FormatterParameters getParameters() {
85	return parameters;	1✔
86	}
87
88	@Override
89	public void beforeIteration() {
90	logger.info(parameters::formatParameters);	1✔
91
92	// print headers
93	if (parameters.hasSelector()) {	1✔
94	logger.info("has selector");	1✔
95	var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());	1✔
96	try {
97	writer = Files.newBufferedWriter(path);	1✔
UNCOV 98	} catch (IOException e) {	×
UNCOV 99	logger.log(Level.WARNING, "beforeIteration", e);	×
100	}	1✔
101	List<String> values = new ArrayList<>();	1✔
102	if (parameters.withId())	1✔
UNCOV 103	values.add("id");	×
104	for (SchemaSpec spec : parameters.getSelector()) {	1✔
105	values.add(spec.encode());	1✔
106	}	1✔
107	try {
108	writer.write(StringUtils.join(values, parameters.getSeparator()) + "\n");	1✔
UNCOV 109	} catch (IOException e) {	×
110	logger.log(Level.WARNING, "beforeIteration", e);	×
111	}	1✔
112	}
113	if (parameters.getIds() != null && !parameters.getIds().isEmpty()	1✔
NEW 114	&& parameters.getFormat().equals("xml")) {	×
NEW 115	var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());	×
NEW 116	logger.info("path: " + path.toAbsolutePath());	×
117	try {
118	// outputStream = new FileOutputStream(path.toFile());
NEW 119	marcXmlWriter = new MarcXmlWriter(new FileOutputStream(path.toFile()));	×
NEW 120	marcXmlWriter.setIndent(true);	×
NEW 121	} catch (FileNotFoundException e) {	×
NEW 122	logger.log(Level.WARNING, "beforeIteration", e);	×
NEW 123	}	×
124	}
125	}	1✔
126
127	@Override
128	public void fileOpened(Path file) {
129	// do nothing
130	}	1✔
131
132	@Override
133	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
134	String id = marc4jRecord.getControlNumber() != null ?	1✔
135	marc4jRecord.getControlNumber().trim() : null;	1✔
136
137	boolean hasSpecifiedId = parameters.hasId() && id != null && id.equals(parameters.getId());	1✔
138
139	if (!hasSpecifiedId)	1✔
140	hasSpecifiedId = id != null	1✔
141	&& parameters.getIds() != null	1✔
NEW 142	&& !parameters.getIds().isEmpty()	×
143	&& parameters.getIds().contains(id);	1✔
144
145	boolean hasSpecifiedRecordNumber = parameters.getCountNr() > -1	1✔
146	&& parameters.getCountNr() == recordNumber;	1✔
147
148	if (hasSpecifiedId \|\| hasSpecifiedRecordNumber) {	1✔
NEW 149	if (parameters.getFormat().equals("xml")) {	×
NEW 150	marcXmlWriter.write(marc4jRecord);	×
151	// MarcXmlWriter.writeSingleRecord(marc4jRecord, System.out, true);
152	// MarcXmlWriter.writeSingleRecord(marc4jRecord, outputStream, true);
153	} else {
NEW 154	logger.info(marc4jRecord::toString);	×
155	}
156	}
157	}	1✔
158
159	@Override
160	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 161	processRecord(bibliographicRecord, recordNumber);	×
UNCOV 162	}	×
163
164	@Override
165	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
166	if (parameters.hasId() && bibliographicRecord.getId().trim().equals(parameters.getId())) {	1✔
NEW 167	for (DataField field : bibliographicRecord.getDatafields()) {	×
NEW 168	logger.info(field.getTag());	×
UNCOV 169	}	×
170	}
171
172	if (parameters.hasSearch()) {	1✔
NEW 173	List<String> results = bibliographicRecord.search(parameters.getPath(), parameters.getQuery());	×
UNCOV 174	if (!results.isEmpty()) {	×
NEW 175	logger.info(bibliographicRecord::toString);	×
176	}
177	}
178	if (!parameters.hasSelector()) {	1✔
NEW 179	return;	×
180	}
181
182	List<String> selectionResults = new ArrayList<>();	1✔
183	if (parameters.withId()) {	1✔
NEW 184	selectionResults.add(bibliographicRecord.getId());	×
185	}
186	if (parameters.getSchemaType().equals(SchemaType.PICA)) {	1✔
187	List<String> selectedPicaResults = selectPicaResults((PicaRecord) bibliographicRecord);	1✔
188	selectionResults.addAll(selectedPicaResults);	1✔
189	} else {	1✔
190	List<String> selectedMarcResults = selectMarcResults((MarcRecord) bibliographicRecord);	1✔
191	selectionResults.addAll(selectedMarcResults);	1✔
192	}
193
194	try {
195	writer.write(StringUtils.join(selectionResults, parameters.getSeparator()) + "\n");	1✔
NEW 196	} catch (IOException e) {	×
NEW 197	logger.log(Level.SEVERE, "processRecord", e);	×
198	}	1✔
199	}	1✔
200
201	@Override
202	public void fileProcessed() {
203	// do nothing
UNCOV 204	}	×
205
206	@Override
207	public void afterIteration(int numberOfprocessedRecords, long duration) {
208
209	if (writer != null)	1✔
210	try {
211	writer.close();	1✔
UNCOV 212	} catch (IOException e) {	×
UNCOV 213	logger.log(Level.SEVERE, "afterIteration", e);	×
214	}	1✔
215
216	if (marcXmlWriter != null)	1✔
217	try {
NEW 218	marcXmlWriter.close();	×
NEW 219	} catch (MarcException e) {	×
NEW 220	logger.log(Level.SEVERE, "afterIteration", e);	×
NEW 221	}	×
222	}	1✔
223
224	private List<String> selectPicaResults(PicaRecord picaRecord) {
225	List<String> selectionResults = new ArrayList<>();	1✔
226	for (SchemaSpec marcSpec : parameters.getSelector()) {	1✔
227	PicaSpec spec = (PicaSpec) marcSpec;	1✔
228	List<String> results = picaRecord.select(spec);	1✔
229
230	if (results.isEmpty() \|\| spec.getFunction() == null) {	1✔
231	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
232	continue;	1✔
233	}
234
235	List<String> candidates = new ArrayList<>();	1✔
236	for (String result : results) {	1✔
237	if (spec.getFunction().equals("extractPicaDate")) {	1✔
238	candidates.add(extractPicaDate(result));	1✔
239	}
240	}	1✔
241	results = candidates;	1✔
242	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
243	}	1✔
244	return selectionResults;	1✔
245	}
246
247	private List<String> selectMarcResults(MarcRecord marcRecord) {
248	List<String> selectionResults = new ArrayList<>();	1✔
249	for (SchemaSpec marcSpec : parameters.getSelector()) {	1✔
250	List<String> results = marcRecord.select(marcSpec);	1✔
251	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
252	}	1✔
253	return selectionResults;	1✔
254	}
255
256	@Override
257	public boolean readyToProcess() {
258	return readyToProcess;	1✔
259	}
260
261	public static String extractPicaDate(String dateInString) {
262	String[] parts1 = dateInString.split(":", 2);	1✔
263	String[] dateParts = parts1[1].split("-");	1✔
264	return dateParts[2] + dateParts[1] + dateParts[0];	1✔
265	// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd-yy", Locale.ENGLISH);
266	// LocalDate dateTime = LocalDate.parse(dateInString, formatter);
267	// return dateTime;
268	}
269	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous