#1647

Committed 06 May 2026 01:13PM UTC coverage: 90.202% (+0.006%) from 90.196%

Build # #1647

Build Type

push

Committed by

Commit Message

Merge branch 'main' of github.com:pkiraly/qa-catalogue

Coverage Stats

36743 of 40734 relevant lines covered (90.2%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.23

/src/main/java/de/gwdg/metadataqa/marc/cli/Formatter.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.cli.parameters.FormatterParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.DataField;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import de.gwdg.metadataqa.marc.utils.SchemaSpec;
import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import org.marc4j.MarcException;
import org.marc4j.MarcXmlWriter;
import org.marc4j.marc.Record;

import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * usage:
 * java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar de.gwdg.metadataqa.marc.cli.Validator [MARC21 file]
 */
public class Formatter implements BibliographicInputProcessor {

  private static final Logger logger = Logger.getLogger(Formatter.class.getCanonicalName());

  private final FormatterParameters parameters;
  private final boolean readyToProcess;
  private BufferedWriter writer;
  private MarcXmlWriter marcXmlWriter;

  public Formatter(String[] args) throws ParseException {
    parameters = new FormatterParameters(args);
    readyToProcess = true;
  }

  public static void main(String[] args) {
    try {
      BibliographicInputProcessor processor = new Formatter(args);
      if (processor.getParameters().getArgs().length < 1) {
        logger.severe("Please provide a MARC file name!");
        System.exit(1);
      }
      if (processor.getParameters().doHelp()) {
        processor.printHelp(processor.getParameters().getOptions());
        System.exit(0);
      }
      RecordIterator iterator = new RecordIterator(processor);
      logger.info(() -> processor.getParameters().formatParameters());
      iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
      iterator.start();
    } catch(Exception e) {
      logger.severe(() -> "ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
  }

  @Override
  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());
    formatter.printHelp(message, options);
  }

  @Override
  public FormatterParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters::formatParameters);

    // print headers
    if (parameters.hasSelector()) {
      logger.info("has selector");
      var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());
      try {
        writer = Files.newBufferedWriter(path);
      } catch (IOException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
      List<String> values = new ArrayList<>();
      if (parameters.withId())
        values.add("id");
      for (SchemaSpec spec : parameters.getSelector()) {
        values.add(spec.encode());
      }
      try {
        writer.write(StringUtils.join(values, parameters.getSeparator()) + "\n");
      } catch (IOException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
    }
    if (parameters.getFormat() != null && parameters.getFormat().equals("xml") &&
        (parameters.hasId() || (parameters.getIds() != null && !parameters.getIds().isEmpty()))) {
      var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());
      logger.info("path: " + path.toAbsolutePath());
      try {
        // outputStream = new FileOutputStream(path.toFile());
        marcXmlWriter = new MarcXmlWriter(new FileOutputStream(path.toFile()));
        marcXmlWriter.setIndent(true);
      } catch (FileNotFoundException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
    }
  }

  @Override
  public void fileOpened(Path file) {
    // do nothing
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    String id = marc4jRecord.getControlNumber() != null ?
      marc4jRecord.getControlNumber().trim() : null;

    boolean hasSpecifiedId = parameters.hasId() && id != null && parameters.getId().contains(id);

    if (!hasSpecifiedId)
      hasSpecifiedId = id != null
                       && parameters.getIds() != null
                       && !parameters.getIds().isEmpty()
                       && parameters.getIds().contains(id);

    boolean hasSpecifiedRecordNumber = parameters.getCountNr() > -1
      && parameters.getCountNr() == recordNumber;

    if (hasSpecifiedId || hasSpecifiedRecordNumber) {
      if (parameters.getFormat() != null && parameters.getFormat().equals("xml")) {
        marcXmlWriter.write(marc4jRecord);
        // MarcXmlWriter.writeSingleRecord(marc4jRecord, System.out, true);
        // MarcXmlWriter.writeSingleRecord(marc4jRecord, outputStream, true);
      } else {
        logger.info(marc4jRecord::toString);
      }
    }
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    if (parameters.hasSearch()) {
      List<String> results = bibliographicRecord.search(parameters.getPath(), parameters.getQuery());
      if (!results.isEmpty()) {
        logger.info(bibliographicRecord::toString);
      }
    }
    if (!parameters.hasSelector()) {
      return;
    }

    List<String> selectionResults = new ArrayList<>();
    if (parameters.withId()) {
      selectionResults.add(bibliographicRecord.getId());
    }
    if (parameters.getSchemaType().equals(SchemaType.PICA)) {
      List<String> selectedPicaResults = selectPicaResults((PicaRecord) bibliographicRecord);
      selectionResults.addAll(selectedPicaResults);
    } else {
      List<String> selectedMarcResults = selectMarcResults((MarcRecord) bibliographicRecord);
      selectionResults.addAll(selectedMarcResults);
    }

    try {
      writer.write(StringUtils.join(selectionResults, parameters.getSeparator()) + "\n");
    } catch (IOException e) {
      logger.log(Level.SEVERE, "processRecord", e);
    }
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {

    if (writer != null)
      try {
        writer.close();
      } catch (IOException e) {
        logger.log(Level.SEVERE, "afterIteration", e);
      }

    if (marcXmlWriter != null)
      try {
        marcXmlWriter.close();
      } catch (MarcException e) {
        logger.log(Level.SEVERE, "afterIteration", e);
      }
  }

  private List<String> selectPicaResults(PicaRecord picaRecord) {
    List<String> selectionResults = new ArrayList<>();
    for (SchemaSpec marcSpec : parameters.getSelector()) {
      PicaSpec spec = (PicaSpec) marcSpec;
      List<String> results = picaRecord.select(spec);

      if (results.isEmpty() || spec.getFunction() == null) {
        selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
        continue;
      }

      List<String> candidates = new ArrayList<>();
      for (String result : results) {
        if (spec.getFunction().equals("extractPicaDate")) {
          candidates.add(extractPicaDate(result));
        }
      }
      results = candidates;
      selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
    }
    return selectionResults;
  }

  private List<String> selectMarcResults(MarcRecord marcRecord) {
    List<String> selectionResults = new ArrayList<>();
    for (SchemaSpec marcSpec : parameters.getSelector()) {
      List<String> results = marcRecord.select(marcSpec);
      selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
    }
    return selectionResults;
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }

  public static String extractPicaDate(String dateInString) {
    String[] parts1 = dateInString.split(":", 2);
    String[] dateParts = parts1[1].split("-");
    return dateParts[2] + dateParts[1] + dateParts[0];
    // DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd-yy", Locale.ENGLISH);
    // LocalDate dateTime = LocalDate.parse(dateInString, formatter);
    // return dateTime;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.cli.parameters.FormatterParameters;
4	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
5	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
6	import de.gwdg.metadataqa.marc.dao.DataField;
7	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
8	import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
9	import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
10	import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
11	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
12	import de.gwdg.metadataqa.marc.utils.SchemaSpec;
13	import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
14	import org.apache.commons.cli.HelpFormatter;
15	import org.apache.commons.cli.Options;
16	import org.apache.commons.cli.ParseException;
17	import org.apache.commons.lang3.StringUtils;
18	import org.marc4j.MarcException;
19	import org.marc4j.MarcXmlWriter;
20	import org.marc4j.marc.Record;
21
22	import java.io.BufferedWriter;
23	import java.io.FileNotFoundException;
24	import java.io.FileOutputStream;
25	import java.io.IOException;
26	import java.io.OutputStream;
27	import java.nio.file.Files;
28	import java.nio.file.Path;
29	import java.nio.file.Paths;
30	import java.util.ArrayList;
31	import java.util.List;
32	import java.util.logging.Level;
33	import java.util.logging.Logger;
34
35	/**
36	* usage:
37	* java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar de.gwdg.metadataqa.marc.cli.Validator [MARC21 file]
38	*/
39	public class Formatter implements BibliographicInputProcessor {
40
41	private static final Logger logger = Logger.getLogger(Formatter.class.getCanonicalName());	1✔
42
43	private final FormatterParameters parameters;
44	private final boolean readyToProcess;
45	private BufferedWriter writer;
46	private MarcXmlWriter marcXmlWriter;
47
48	public Formatter(String[] args) throws ParseException {	1✔
49	parameters = new FormatterParameters(args);	1✔
50	readyToProcess = true;	1✔
51	}	1✔
52
53	public static void main(String[] args) {
54	try {
55	BibliographicInputProcessor processor = new Formatter(args);	×
56	if (processor.getParameters().getArgs().length < 1) {	×
57	logger.severe("Please provide a MARC file name!");	×
58	System.exit(1);	×
59	}
60	if (processor.getParameters().doHelp()) {	×
61	processor.printHelp(processor.getParameters().getOptions());	×
62	System.exit(0);	×
63	}
64	RecordIterator iterator = new RecordIterator(processor);	×
65	logger.info(() -> processor.getParameters().formatParameters());	×
66	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	×
67	iterator.start();	×
68	} catch(Exception e) {	×
69	logger.severe(() -> "ERROR. " + e.getLocalizedMessage());	×
70	System.exit(1);	×
71	}	×
72	}	×
73
74	@Override
75	public void printHelp(Options options) {
76	HelpFormatter formatter = new HelpFormatter();	×
77	String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());	×
78	formatter.printHelp(message, options);	×
79	}	×
80
81	@Override
82	public FormatterParameters getParameters() {
83	return parameters;	1✔
84	}
85
86	@Override
87	public void beforeIteration() {
88	logger.info(parameters::formatParameters);	1✔
89
90	// print headers
91	if (parameters.hasSelector()) {	1✔
92	logger.info("has selector");	1✔
93	var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());	1✔
94	try {
95	writer = Files.newBufferedWriter(path);	1✔
96	} catch (IOException e) {	×
97	logger.log(Level.WARNING, "beforeIteration", e);	×
98	}	1✔
99	List<String> values = new ArrayList<>();	1✔
100	if (parameters.withId())	1✔
101	values.add("id");	×
102	for (SchemaSpec spec : parameters.getSelector()) {	1✔
103	values.add(spec.encode());	1✔
104	}	1✔
105	try {
106	writer.write(StringUtils.join(values, parameters.getSeparator()) + "\n");	1✔
107	} catch (IOException e) {	×
108	logger.log(Level.WARNING, "beforeIteration", e);	×
109	}	1✔
110	}
111	if (parameters.getFormat() != null && parameters.getFormat().equals("xml") &&	1✔
112	(parameters.hasId() \|\| (parameters.getIds() != null && !parameters.getIds().isEmpty()))) {	×
113	var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());	×
114	logger.info("path: " + path.toAbsolutePath());	×
115	try {
116	// outputStream = new FileOutputStream(path.toFile());
117	marcXmlWriter = new MarcXmlWriter(new FileOutputStream(path.toFile()));	×
118	marcXmlWriter.setIndent(true);	×
119	} catch (FileNotFoundException e) {	×
120	logger.log(Level.WARNING, "beforeIteration", e);	×
121	}	×
122	}
123	}	1✔
124
125	@Override
126	public void fileOpened(Path file) {
127	// do nothing
128	}	1✔
129
130	@Override
131	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
132	String id = marc4jRecord.getControlNumber() != null ?	1✔
133	marc4jRecord.getControlNumber().trim() : null;	1✔
134
135	boolean hasSpecifiedId = parameters.hasId() && id != null && parameters.getId().contains(id);	1✔
136
137	if (!hasSpecifiedId)	1✔
138	hasSpecifiedId = id != null	1✔
139	&& parameters.getIds() != null	1✔
140	&& !parameters.getIds().isEmpty()	×
141	&& parameters.getIds().contains(id);	1✔
142
143	boolean hasSpecifiedRecordNumber = parameters.getCountNr() > -1	1✔
144	&& parameters.getCountNr() == recordNumber;	1✔
145
146	if (hasSpecifiedId \|\| hasSpecifiedRecordNumber) {	1✔
147	if (parameters.getFormat() != null && parameters.getFormat().equals("xml")) {	×
148	marcXmlWriter.write(marc4jRecord);	×
149	// MarcXmlWriter.writeSingleRecord(marc4jRecord, System.out, true);
150	// MarcXmlWriter.writeSingleRecord(marc4jRecord, outputStream, true);
151	} else {
152	logger.info(marc4jRecord::toString);	×
153	}
154	}
155	}	1✔
156
157	@Override
158	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
159	processRecord(bibliographicRecord, recordNumber);	×
160	}	×
161
162	@Override
163	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
164	if (parameters.hasSearch()) {	1✔
165	List<String> results = bibliographicRecord.search(parameters.getPath(), parameters.getQuery());	×
166	if (!results.isEmpty()) {	×
167	logger.info(bibliographicRecord::toString);	×
168	}
169	}
170	if (!parameters.hasSelector()) {	1✔
171	return;	×
172	}
173
174	List<String> selectionResults = new ArrayList<>();	1✔
175	if (parameters.withId()) {	1✔
176	selectionResults.add(bibliographicRecord.getId());	×
177	}
178	if (parameters.getSchemaType().equals(SchemaType.PICA)) {	1✔
179	List<String> selectedPicaResults = selectPicaResults((PicaRecord) bibliographicRecord);	1✔
180	selectionResults.addAll(selectedPicaResults);	1✔
181	} else {	1✔
182	List<String> selectedMarcResults = selectMarcResults((MarcRecord) bibliographicRecord);	1✔
183	selectionResults.addAll(selectedMarcResults);	1✔
184	}
185
186	try {
187	writer.write(StringUtils.join(selectionResults, parameters.getSeparator()) + "\n");	1✔
188	} catch (IOException e) {	×
189	logger.log(Level.SEVERE, "processRecord", e);	×
190	}	1✔
191	}	1✔
192
193	@Override
194	public void fileProcessed() {
195	// do nothing
196	}	×
197
198	@Override
199	public void afterIteration(int numberOfprocessedRecords, long duration) {
200
201	if (writer != null)	1✔
202	try {
203	writer.close();	1✔
204	} catch (IOException e) {	×
205	logger.log(Level.SEVERE, "afterIteration", e);	×
206	}	1✔
207
208	if (marcXmlWriter != null)	1✔
209	try {
210	marcXmlWriter.close();	×
211	} catch (MarcException e) {	×
212	logger.log(Level.SEVERE, "afterIteration", e);	×
213	}	×
214	}	1✔
215
216	private List<String> selectPicaResults(PicaRecord picaRecord) {
217	List<String> selectionResults = new ArrayList<>();	1✔
218	for (SchemaSpec marcSpec : parameters.getSelector()) {	1✔
219	PicaSpec spec = (PicaSpec) marcSpec;	1✔
220	List<String> results = picaRecord.select(spec);	1✔
221
222	if (results.isEmpty() \|\| spec.getFunction() == null) {	1✔
223	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
224	continue;	1✔
225	}
226
227	List<String> candidates = new ArrayList<>();	1✔
228	for (String result : results) {	1✔
229	if (spec.getFunction().equals("extractPicaDate")) {	1✔
230	candidates.add(extractPicaDate(result));	1✔
231	}
232	}	1✔
233	results = candidates;	1✔
234	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
235	}	1✔
236	return selectionResults;	1✔
237	}
238
239	private List<String> selectMarcResults(MarcRecord marcRecord) {
240	List<String> selectionResults = new ArrayList<>();	1✔
241	for (SchemaSpec marcSpec : parameters.getSelector()) {	1✔
242	List<String> results = marcRecord.select(marcSpec);	1✔
243	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
244	}	1✔
245	return selectionResults;	1✔
246	}
247
248	@Override
249	public boolean readyToProcess() {
250	return readyToProcess;	1✔
251	}
252
253	public static String extractPicaDate(String dateInString) {
254	String[] parts1 = dateInString.split(":", 2);	1✔
255	String[] dateParts = parts1[1].split("-");	1✔
256	return dateParts[2] + dateParts[1] + dateParts[0];	1✔
257	// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd-yy", Locale.ENGLISH);
258	// LocalDate dateTime = LocalDate.parse(dateInString, formatter);
259	// return dateTime;
260	}
261	}

pkiraly / metadata-qa-marc / #1647

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous