#1401

Committed 03 Feb 2025 09:56PM UTC coverage: 90.262%. Remained the same

Build # #1401

Build Type

push

Committed by

Commit Message

Merge branch 'main' of github.com:pkiraly/qa-catalogue

Coverage Stats

39 of 99 new or added lines in 11 files covered. (39.39%)

94 existing lines in 7 files now uncovered.

36438 of 40369 relevant lines covered (90.26%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.33

/src/main/java/de/gwdg/metadataqa/marc/cli/Formatter.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.cli.parameters.FormatterParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.DataField;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import de.gwdg.metadataqa.marc.utils.SchemaSpec;
import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import org.marc4j.MarcException;
import org.marc4j.MarcXmlWriter;
import org.marc4j.marc.Record;

import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * usage:
 * java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar de.gwdg.metadataqa.marc.cli.Validator [MARC21 file]
 *
 * @author Péter Király <peter.kiraly at gwdg.de>
 */
public class Formatter implements BibliographicInputProcessor {

  private static final Logger logger = Logger.getLogger(Formatter.class.getCanonicalName());

  private final FormatterParameters parameters;
  private final boolean readyToProcess;
  private BufferedWriter writer;
  private MarcXmlWriter marcXmlWriter;

  public Formatter(String[] args) throws ParseException {
    parameters = new FormatterParameters(args);
    readyToProcess = true;
  }

  public static void main(String[] args) {
    logger.severe(() -> "'" + StringUtils.join(args, "', '") + "'");
    try {
      BibliographicInputProcessor processor = new Formatter(args);
      if (processor.getParameters().getArgs().length < 1) {
        logger.severe("Please provide a MARC file name!");
        System.exit(1);
      }
      if (processor.getParameters().doHelp()) {
        processor.printHelp(processor.getParameters().getOptions());
        System.exit(0);
      }
      RecordIterator iterator = new RecordIterator(processor);
      logger.info(() -> processor.getParameters().formatParameters());
      iterator.start();
    } catch(Exception e) {
      logger.severe(() -> "ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
  }

  @Override
  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());
    formatter.printHelp(message, options);
  }

  @Override
  public FormatterParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters::formatParameters);

    // print headers
    if (parameters.hasSelector()) {
      var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());
      try {
        writer = Files.newBufferedWriter(path);
      } catch (IOException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
      List<String> values = new ArrayList<>();
      if (parameters.withId())
        values.add("id");
      for (SchemaSpec spec : parameters.getSelector()) {
        values.add(spec.encode());
      }
      try {
        writer.write(StringUtils.join(values, parameters.getSeparator()) + "\n");
      } catch (IOException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
    }
    if (parameters.getIds() != null && !parameters.getIds().isEmpty()
        && parameters.getFormat().equals("xml")) {
      var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());
      logger.info("path: " + path.toAbsolutePath());
      try {
        // outputStream = new FileOutputStream(path.toFile());
        marcXmlWriter = new MarcXmlWriter(new FileOutputStream(path.toFile()));
        marcXmlWriter.setIndent(true);
      } catch (FileNotFoundException e) {
        logger.log(Level.WARNING, "beforeIteration", e);
      }
    }
  }

  @Override
  public void fileOpened(Path file) {
    // do nothing
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    String id = marc4jRecord.getControlNumber() != null ?
      marc4jRecord.getControlNumber().trim() : null;

    boolean hasSpecifiedId = parameters.hasId() && id != null && id.equals(parameters.getId());

    if (!hasSpecifiedId)
      hasSpecifiedId = id != null
                       && parameters.getIds() != null
                       && !parameters.getIds().isEmpty()
                       && parameters.getIds().contains(id);

    boolean hasSpecifiedRecordNumber = parameters.getCountNr() > -1
      && parameters.getCountNr() == recordNumber;

    if (hasSpecifiedId || hasSpecifiedRecordNumber) {
      if (parameters.getFormat().equals("xml")) {
        marcXmlWriter.write(marc4jRecord);
        // MarcXmlWriter.writeSingleRecord(marc4jRecord, System.out, true);
        // MarcXmlWriter.writeSingleRecord(marc4jRecord, outputStream, true);
      } else {
        logger.info(marc4jRecord::toString);
      }
    }
  }

  @Override
  public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord marcRecord, int recordNumber) throws IOException {
    if (parameters.hasId() && marcRecord.getId().trim().equals(parameters.getId())) {
      for (DataField field : marcRecord.getDatafields()) {
       logger.info(field.getTag());
      }
    }

    if (parameters.hasSearch()) {
      List<String> results = marcRecord.search(parameters.getPath(), parameters.getQuery());
      if (!results.isEmpty()) {
        logger.info(marcRecord::toString);
      }
    }
    if (!parameters.hasSelector()) {
      return;
    }

    List<String> selectionResults = new ArrayList<>();
    if (parameters.withId()) {
      selectionResults.add(marcRecord.getId());
    }
    if (parameters.getSchemaType().equals(SchemaType.PICA)) {
      List<String> selectedPicaResults = selectPicaResults((PicaRecord) marcRecord);
      selectionResults.addAll(selectedPicaResults);
    } else {
      List<String> selectedMarcResults = selectMarcResults((MarcRecord) marcRecord);
      selectionResults.addAll(selectedMarcResults);
    }

    try {
      writer.write(StringUtils.join(selectionResults, parameters.getSeparator()) + "\n");
    } catch (IOException e) {
      logger.log(Level.SEVERE, "processRecord", e);
    }
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {

    if (writer != null)
      try {
        writer.close();
      } catch (IOException e) {
        logger.log(Level.SEVERE, "afterIteration", e);
      }

    if (marcXmlWriter != null)
      try {
        marcXmlWriter.close();
      } catch (MarcException e) {
        logger.log(Level.SEVERE, "afterIteration", e);
      }
  }

  private List<String> selectPicaResults(PicaRecord picaRecord) {
    List<String> selectionResults = new ArrayList<>();
    for (SchemaSpec marcSpec : parameters.getSelector()) {
      PicaSpec spec = (PicaSpec) marcSpec;
      List<String> results = picaRecord.select(spec);

      if (results.isEmpty() || spec.getFunction() == null) {
        selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
        continue;
      }

      List<String> candidates = new ArrayList<>();
      for (String result : results) {
        if (spec.getFunction().equals("extractPicaDate")) {
          candidates.add(extractPicaDate(result));
        }
      }
      results = candidates;
      selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
    }
    return selectionResults;
  }

  private List<String> selectMarcResults(MarcRecord marcRecord) {
    List<String> selectionResults = new ArrayList<>();
    for (SchemaSpec marcSpec : parameters.getSelector()) {
      List<String> results = marcRecord.select(marcSpec);
      selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "||"));
    }
    return selectionResults;
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }

  public static String extractPicaDate(String dateInString) {
    String[] parts1 = dateInString.split(":", 2);
    String[] dateParts = parts1[1].split("-");
    return dateParts[2] + dateParts[1] + dateParts[0];
    // DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd-yy", Locale.ENGLISH);
    // LocalDate dateTime = LocalDate.parse(dateInString, formatter);
    // return dateTime;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.cli.parameters.FormatterParameters;
4	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
5	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
6	import de.gwdg.metadataqa.marc.dao.DataField;
7	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
8	import de.gwdg.metadataqa.marc.dao.record.MarcRecord;
9	import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
10	import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
11	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
12	import de.gwdg.metadataqa.marc.utils.SchemaSpec;
13	import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
14	import org.apache.commons.cli.HelpFormatter;
15	import org.apache.commons.cli.Options;
16	import org.apache.commons.cli.ParseException;
17	import org.apache.commons.lang3.StringUtils;
18	import org.marc4j.MarcException;
19	import org.marc4j.MarcXmlWriter;
20	import org.marc4j.marc.Record;
21
22	import java.io.BufferedWriter;
23	import java.io.FileNotFoundException;
24	import java.io.FileOutputStream;
25	import java.io.IOException;
26	import java.io.OutputStream;
27	import java.nio.file.Files;
28	import java.nio.file.Path;
29	import java.nio.file.Paths;
30	import java.util.ArrayList;
31	import java.util.List;
32	import java.util.logging.Level;
33	import java.util.logging.Logger;
34
35	/**
36	* usage:
37	* java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar de.gwdg.metadataqa.marc.cli.Validator [MARC21 file]
38	*
39	* @author Péter Király <peter.kiraly at gwdg.de>
40	*/
41	public class Formatter implements BibliographicInputProcessor {
42
43	private static final Logger logger = Logger.getLogger(Formatter.class.getCanonicalName());	1✔
44
45	private final FormatterParameters parameters;
46	private final boolean readyToProcess;
47	private BufferedWriter writer;
48	private MarcXmlWriter marcXmlWriter;
49
50	public Formatter(String[] args) throws ParseException {	1✔
51	parameters = new FormatterParameters(args);	1✔
52	readyToProcess = true;	1✔
53	}	1✔
54
55	public static void main(String[] args) {
UNCOV 56	logger.severe(() -> "'" + StringUtils.join(args, "', '") + "'");	×
57	try {
UNCOV 58	BibliographicInputProcessor processor = new Formatter(args);	×
UNCOV 59	if (processor.getParameters().getArgs().length < 1) {	×
UNCOV 60	logger.severe("Please provide a MARC file name!");	×
UNCOV 61	System.exit(1);	×
62	}
UNCOV 63	if (processor.getParameters().doHelp()) {	×
64	processor.printHelp(processor.getParameters().getOptions());	×
65	System.exit(0);	×
66	}
67	RecordIterator iterator = new RecordIterator(processor);	×
UNCOV 68	logger.info(() -> processor.getParameters().formatParameters());	×
69	iterator.start();	×
70	} catch(Exception e) {	×
71	logger.severe(() -> "ERROR. " + e.getLocalizedMessage());	×
UNCOV 72	System.exit(1);	×
73	}	×
74	}	×
75
76	@Override
77	public void printHelp(Options options) {
78	HelpFormatter formatter = new HelpFormatter();	×
79	String message = String.format("java -cp qa-catalogue.jar %s [options] [file]", this.getClass().getCanonicalName());	×
80	formatter.printHelp(message, options);	×
UNCOV 81	}	×
82
83	@Override
84	public FormatterParameters getParameters() {
85	return parameters;	1✔
86	}
87
88	@Override
89	public void beforeIteration() {
90	logger.info(parameters::formatParameters);	1✔
91
92	// print headers
93	if (parameters.hasSelector()) {	1✔
94	var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());	1✔
95	try {
96	writer = Files.newBufferedWriter(path);	1✔
UNCOV 97	} catch (IOException e) {	×
UNCOV 98	logger.log(Level.WARNING, "beforeIteration", e);	×
99	}	1✔
100	List<String> values = new ArrayList<>();	1✔
101	if (parameters.withId())	1✔
UNCOV 102	values.add("id");	×
103	for (SchemaSpec spec : parameters.getSelector()) {	1✔
104	values.add(spec.encode());	1✔
105	}	1✔
106	try {
107	writer.write(StringUtils.join(values, parameters.getSeparator()) + "\n");	1✔
108	} catch (IOException e) {	×
UNCOV 109	logger.log(Level.WARNING, "beforeIteration", e);	×
110	}	1✔
111	}
112	if (parameters.getIds() != null && !parameters.getIds().isEmpty()	1✔
NEW 113	&& parameters.getFormat().equals("xml")) {	×
NEW 114	var path = Paths.get(parameters.getOutputDir(), parameters.getFileName());	×
NEW 115	logger.info("path: " + path.toAbsolutePath());	×
116	try {
117	// outputStream = new FileOutputStream(path.toFile());
NEW 118	marcXmlWriter = new MarcXmlWriter(new FileOutputStream(path.toFile()));	×
NEW 119	marcXmlWriter.setIndent(true);	×
NEW 120	} catch (FileNotFoundException e) {	×
NEW 121	logger.log(Level.WARNING, "beforeIteration", e);	×
NEW 122	}	×
123	}
124	}	1✔
125
126	@Override
127	public void fileOpened(Path file) {
128	// do nothing
129	}	1✔
130
131	@Override
132	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
133	String id = marc4jRecord.getControlNumber() != null ?	1✔
134	marc4jRecord.getControlNumber().trim() : null;	1✔
135
136	boolean hasSpecifiedId = parameters.hasId() && id != null && id.equals(parameters.getId());	1✔
137
138	if (!hasSpecifiedId)	1✔
139	hasSpecifiedId = id != null	1✔
140	&& parameters.getIds() != null	1✔
NEW 141	&& !parameters.getIds().isEmpty()	×
142	&& parameters.getIds().contains(id);	1✔
143
144	boolean hasSpecifiedRecordNumber = parameters.getCountNr() > -1	1✔
145	&& parameters.getCountNr() == recordNumber;	1✔
146
147	if (hasSpecifiedId \|\| hasSpecifiedRecordNumber) {	1✔
NEW 148	if (parameters.getFormat().equals("xml")) {	×
NEW 149	marcXmlWriter.write(marc4jRecord);	×
150	// MarcXmlWriter.writeSingleRecord(marc4jRecord, System.out, true);
151	// MarcXmlWriter.writeSingleRecord(marc4jRecord, outputStream, true);
152	} else {
NEW 153	logger.info(marc4jRecord::toString);	×
154	}
155	}
156	}	1✔
157
158	@Override
159	public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
160	// do nothing
UNCOV 161	}	×
162
163	@Override
164	public void processRecord(BibliographicRecord marcRecord, int recordNumber) throws IOException {
165	if (parameters.hasId() && marcRecord.getId().trim().equals(parameters.getId())) {	1✔
UNCOV 166	for (DataField field : marcRecord.getDatafields()) {	×
UNCOV 167	logger.info(field.getTag());	×
UNCOV 168	}	×
169	}
170
171	if (parameters.hasSearch()) {	1✔
UNCOV 172	List<String> results = marcRecord.search(parameters.getPath(), parameters.getQuery());	×
UNCOV 173	if (!results.isEmpty()) {	×
UNCOV 174	logger.info(marcRecord::toString);	×
175	}
176	}
177	if (!parameters.hasSelector()) {	1✔
178	return;	×
179	}
180
181	List<String> selectionResults = new ArrayList<>();	1✔
182	if (parameters.withId()) {	1✔
183	selectionResults.add(marcRecord.getId());	×
184	}
185	if (parameters.getSchemaType().equals(SchemaType.PICA)) {	1✔
186	List<String> selectedPicaResults = selectPicaResults((PicaRecord) marcRecord);	1✔
187	selectionResults.addAll(selectedPicaResults);	1✔
188	} else {	1✔
189	List<String> selectedMarcResults = selectMarcResults((MarcRecord) marcRecord);	1✔
190	selectionResults.addAll(selectedMarcResults);	1✔
191	}
192
193	try {
194	writer.write(StringUtils.join(selectionResults, parameters.getSeparator()) + "\n");	1✔
UNCOV 195	} catch (IOException e) {	×
196	logger.log(Level.SEVERE, "processRecord", e);	×
197	}	1✔
198	}	1✔
199
200	@Override
201	public void fileProcessed() {
202	// do nothing
203	}	×
204
205	@Override
206	public void afterIteration(int numberOfprocessedRecords, long duration) {
207
208	if (writer != null)	1✔
209	try {
210	writer.close();	1✔
NEW 211	} catch (IOException e) {	×
NEW 212	logger.log(Level.SEVERE, "afterIteration", e);	×
213	}	1✔
214
215	if (marcXmlWriter != null)	1✔
216	try {
NEW 217	marcXmlWriter.close();	×
NEW 218	} catch (MarcException e) {	×
NEW 219	logger.log(Level.SEVERE, "afterIteration", e);	×
NEW 220	}	×
221	}	1✔
222
223	private List<String> selectPicaResults(PicaRecord picaRecord) {
224	List<String> selectionResults = new ArrayList<>();	1✔
225	for (SchemaSpec marcSpec : parameters.getSelector()) {	1✔
226	PicaSpec spec = (PicaSpec) marcSpec;	1✔
227	List<String> results = picaRecord.select(spec);	1✔
228
229	if (results.isEmpty() \|\| spec.getFunction() == null) {	1✔
230	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
231	continue;	1✔
232	}
233
234	List<String> candidates = new ArrayList<>();	1✔
235	for (String result : results) {	1✔
236	if (spec.getFunction().equals("extractPicaDate")) {	1✔
237	candidates.add(extractPicaDate(result));	1✔
238	}
239	}	1✔
240	results = candidates;	1✔
241	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
242	}	1✔
243	return selectionResults;	1✔
244	}
245
246	private List<String> selectMarcResults(MarcRecord marcRecord) {
247	List<String> selectionResults = new ArrayList<>();	1✔
248	for (SchemaSpec marcSpec : parameters.getSelector()) {	1✔
249	List<String> results = marcRecord.select(marcSpec);	1✔
250	selectionResults.add(results.isEmpty() ? "" : StringUtils.join(results, "\|\|"));	1✔
251	}	1✔
252	return selectionResults;	1✔
253	}
254
255	@Override
256	public boolean readyToProcess() {
257	return readyToProcess;	1✔
258	}
259
260	public static String extractPicaDate(String dateInString) {
261	String[] parts1 = dateInString.split(":", 2);	1✔
262	String[] dateParts = parts1[1].split("-");	1✔
263	return dateParts[2] + dateParts[1] + dateParts[0];	1✔
264	// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd-yy", Locale.ENGLISH);
265	// LocalDate dateTime = LocalDate.parse(dateInString, formatter);
266	// return dateTime;
267	}
268	}

pkiraly / metadata-qa-marc / #1401

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous