#1447

Committed 11 Apr 2025 03:29PM UTC coverage: 90.49% (-0.1%) from 90.59%

Build # #1447

Build Type

push

Committed by

Commit Message

Inject record ID if missing #607

Coverage Stats

25 of 60 new or added lines in 17 files covered. (41.67%)

70 existing lines in 15 files now uncovered.

36528 of 40367 relevant lines covered (90.49%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.42

/src/main/java/de/gwdg/metadataqa/marc/cli/Shacl4bib.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.api.configuration.SchemaConfiguration;
import de.gwdg.metadataqa.api.rule.RuleCatalog;
import de.gwdg.metadataqa.marc.CsvUtils;
import de.gwdg.metadataqa.marc.RuleCatalogUtils;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.Shacl4bibParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.BibSelector;
import de.gwdg.metadataqa.marc.cli.utils.BibSelectorFactory;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.cli.utils.ShaclUtils;
import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.marc4j.marc.Record;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

public class Shacl4bib extends QACli<Shacl4bibParameters>
                       implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(Shacl4bib.class.getCanonicalName());

  private final boolean readyToProcess;
  private File outputFile;
  private RuleCatalog ruleCatalog;
  private SchemaConfiguration schema;
  private RecordFilter recordFilter;

  public Shacl4bib(String[] args) throws ParseException {
    parameters = new Shacl4bibParameters(args);
    recordFilter = parameters.getRecordFilter();
    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new Shacl4bib(args);
    } catch (ParseException e) {
      System.err.println("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
    if (processor.getParameters().getArgs().length < 1) {
      System.err.println("Please provide a MARC file name!");
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters.formatParameters());
    outputFile = new File(parameters.getOutputDir(), parameters.getShaclOutputFile());

    schema = ShaclUtils.setupSchema(parameters);
    ruleCatalog = ShaclUtils.setupRuleCatalog(schema, parameters);

    if (outputFile.exists()) {
      try {
        Files.delete(outputFile.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());
      }
    }
    List<String> header = ruleCatalog.getHeader();
    header.add(0, "id");
    printToFile(outputFile, CsvUtils.createCsv(header));
  }

  @Override
  public void fileOpened(Path path) {
    logger.log(Level.INFO, "file opened: {0}", new Object[]{path});
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    if (!recordFilter.isAllowable(bibliographicRecord)) {
      logger.info("ignoring " + bibliographicRecord.getId());
      return;
    }

    BibSelector selector = BibSelectorFactory.create(schema.getFormat(), bibliographicRecord);

    if (selector != null) {
      List<Object> values = RuleCatalogUtils.extract(ruleCatalog, ruleCatalog.measure(selector));
      values.add(0, bibliographicRecord.getId(true));
      printToFile(outputFile, CsvUtils.createCsvFromObjects(values));
    }
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    copyFileToOutputDir(parameters.getShaclConfigurationFile());
    saveParameters("shacl4bib.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  @Override
  public void printHelp(Options options) {

  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.api.configuration.SchemaConfiguration;
4	import de.gwdg.metadataqa.api.rule.RuleCatalog;
5	import de.gwdg.metadataqa.marc.CsvUtils;
6	import de.gwdg.metadataqa.marc.RuleCatalogUtils;
7	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
8	import de.gwdg.metadataqa.marc.cli.parameters.Shacl4bibParameters;
9	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
10	import de.gwdg.metadataqa.marc.cli.utils.BibSelector;
11	import de.gwdg.metadataqa.marc.cli.utils.BibSelectorFactory;
12	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
13	import de.gwdg.metadataqa.marc.cli.utils.ShaclUtils;
14	import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
15	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
16	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
17	import org.apache.commons.cli.Options;
18	import org.apache.commons.cli.ParseException;
19	import org.marc4j.marc.Record;
20
21	import java.io.File;
22	import java.io.IOException;
23	import java.io.Serializable;
24	import java.nio.file.Files;
25	import java.nio.file.Path;
26	import java.util.List;
27	import java.util.Map;
28	import java.util.logging.Level;
29	import java.util.logging.Logger;
30
31	public class Shacl4bib extends QACli<Shacl4bibParameters>
32	implements BibliographicInputProcessor, Serializable {
33
34	private static final Logger logger = Logger.getLogger(Shacl4bib.class.getCanonicalName());	1✔
35
36	private final boolean readyToProcess;
37	private File outputFile;
38	private RuleCatalog ruleCatalog;
39	private SchemaConfiguration schema;
40	private RecordFilter recordFilter;
41
42	public Shacl4bib(String[] args) throws ParseException {	1✔
43	parameters = new Shacl4bibParameters(args);	1✔
44	recordFilter = parameters.getRecordFilter();	1✔
45	readyToProcess = true;	1✔
46	}	1✔
47
48	public static void main(String[] args) {
49	BibliographicInputProcessor processor = null;	1✔
50	try {
51	processor = new Shacl4bib(args);	1✔
52	} catch (ParseException e) {	×
53	System.err.println("ERROR. " + e.getLocalizedMessage());	×
54	System.exit(1);	×
55	}	1✔
56	if (processor.getParameters().getArgs().length < 1) {	1✔
57	System.err.println("Please provide a MARC file name!");	×
58	processor.printHelp(processor.getParameters().getOptions());	×
59	System.exit(0);	×
60	}
61	if (processor.getParameters().doHelp()) {	1✔
62	processor.printHelp(processor.getParameters().getOptions());	×
63	System.exit(0);	×
64	}
65	RecordIterator iterator = new RecordIterator(processor);	1✔
66	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	1✔
67	iterator.start();	1✔
68	}	1✔
69
70	@Override
71	public CommonParameters getParameters() {
72	return parameters;	1✔
73	}
74
75	@Override
76	public void beforeIteration() {
77	logger.info(parameters.formatParameters());	1✔
78	outputFile = new File(parameters.getOutputDir(), parameters.getShaclOutputFile());	1✔
79
80	schema = ShaclUtils.setupSchema(parameters);	1✔
81	ruleCatalog = ShaclUtils.setupRuleCatalog(schema, parameters);	1✔
82
83	if (outputFile.exists()) {	1✔
84	try {
85	Files.delete(outputFile.toPath());	×
86	} catch (IOException e) {	×
87	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());	×
88	}	×
89	}
90	List<String> header = ruleCatalog.getHeader();	1✔
91	header.add(0, "id");	1✔
92	printToFile(outputFile, CsvUtils.createCsv(header));	1✔
93	}	1✔
94
95	@Override
96	public void fileOpened(Path path) {
97	logger.log(Level.INFO, "file opened: {0}", new Object[]{path});	1✔
98	}	1✔
99
100	@Override
101	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
102	// do nothing
103	}	1✔
104
105	@Override
106	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 107	processRecord(bibliographicRecord, recordNumber);	×
UNCOV 108	}	×
109
110	@Override
111	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
112	if (!recordFilter.isAllowable(bibliographicRecord)) {	1✔
113	logger.info("ignoring " + bibliographicRecord.getId());	×
114	return;	×
115	}
116
117	BibSelector selector = BibSelectorFactory.create(schema.getFormat(), bibliographicRecord);	1✔
118
119	if (selector != null) {	1✔
120	List<Object> values = RuleCatalogUtils.extract(ruleCatalog, ruleCatalog.measure(selector));	1✔
121	values.add(0, bibliographicRecord.getId(true));	1✔
122	printToFile(outputFile, CsvUtils.createCsvFromObjects(values));	1✔
123	}
124	}	1✔
125
126	@Override
127	public void fileProcessed() {
128	// do nothing
129	}	×
130
131	@Override
132	public void afterIteration(int numberOfprocessedRecords, long duration) {
133	copyFileToOutputDir(parameters.getShaclConfigurationFile());	1✔
134	saveParameters("shacl4bib.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
135	}	1✔
136
137	@Override
138	public void printHelp(Options options) {
139
140	}	×
141
142	@Override
143	public boolean readyToProcess() {
144	return readyToProcess;	1✔
145	}
146	}

pkiraly / metadata-qa-marc / #1447

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous