#1430

Committed 12 Mar 2025 12:23PM UTC coverage: 90.395% (-0.06%) from 90.457%

Build # #1430

Build Type

push

Committed by

Commit Message

Merge branch 'main' of github.com:pkiraly/qa-catalogue

Coverage Stats

36517 of 40397 relevant lines covered (90.4%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.57

/src/main/java/de/gwdg/metadataqa/marc/cli/Shacl4bib.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.api.configuration.ConfigurationReader;
import de.gwdg.metadataqa.api.configuration.SchemaConfiguration;
import de.gwdg.metadataqa.api.rule.RuleCatalog;
import de.gwdg.metadataqa.marc.CsvUtils;
import de.gwdg.metadataqa.marc.RuleCatalogUtils;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.Shacl4bibParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.BibSelector;
import de.gwdg.metadataqa.marc.cli.utils.BibSelectorFactory;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.marc4j.marc.Record;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

public class Shacl4bib extends QACli<Shacl4bibParameters> implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(Shacl4bib.class.getCanonicalName());
  private final boolean readyToProcess;
  private File outputFile;
  private RuleCatalog ruleCatalog;
  private SchemaConfiguration schema;
  private RecordFilter recordFilter;

  public Shacl4bib(String[] args) throws ParseException {
    parameters = new Shacl4bibParameters(args);
    recordFilter = parameters.getRecordFilter();
    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new Shacl4bib(args);
    } catch (ParseException e) {
      System.err.println("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
    if (processor.getParameters().getArgs().length < 1) {
      System.err.println("Please provide a MARC file name!");
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    RecordIterator iterator = new RecordIterator(processor);
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters.formatParameters());
    outputFile = new File(parameters.getOutputDir(), parameters.getShaclOutputFile());

    String shaclConfigurationFile = parameters.getShaclConfigurationFile();
    try {
      if (shaclConfigurationFile.endsWith(".json"))
        schema = ConfigurationReader.readSchemaJson(shaclConfigurationFile);
      else
        schema = ConfigurationReader.readSchemaYaml(shaclConfigurationFile);
    } catch (IOException exception) {
      logger.severe("Error when the SHACL schema is initialized. " + exception.getLocalizedMessage());
      System.exit(0);
    }

    this.ruleCatalog = new RuleCatalog(schema.asSchema())
      .setOnlyIdInHeader(true)
      .setOutputType(parameters.getShaclOutputType());

    if (outputFile.exists()) {
      try {
        Files.delete(outputFile.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());
      }
    }
    List<String> header = ruleCatalog.getHeader();
    header.add(0, "id");
    printToFile(outputFile, CsvUtils.createCsv(header));
  }

  @Override
  public void fileOpened(Path path) {
    logger.log(Level.INFO, "file opened: {0}", new Object[]{path});
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    if (!recordFilter.isAllowable(bibliographicRecord)) {
      logger.info("ignoring " + bibliographicRecord.getId());
      return;
    }

    BibSelector selector = BibSelectorFactory.create(schema.getFormat(), bibliographicRecord);

    if (selector != null) {
      List<Object> values = RuleCatalogUtils.extract(ruleCatalog, ruleCatalog.measure(selector));
      values.add(0, bibliographicRecord.getId(true));
      printToFile(outputFile, CsvUtils.createCsvFromObjects(values));
    }
  }

  @Override
  public void fileProcessed() {

  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    copySchaclFileToOutputDir();
    saveParameters("shacl4bib.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  private void copySchaclFileToOutputDir() {
    File source = new File(parameters.getShaclConfigurationFile());
    try {
      FileUtils.copyFileToDirectory(source, new File(parameters.getOutputDir()));
    } catch (IOException e) {
      logger.warning(e.getLocalizedMessage());
    }
  }

  @Override
  public void printHelp(Options options) {

  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.api.configuration.ConfigurationReader;
4	import de.gwdg.metadataqa.api.configuration.SchemaConfiguration;
5	import de.gwdg.metadataqa.api.rule.RuleCatalog;
6	import de.gwdg.metadataqa.marc.CsvUtils;
7	import de.gwdg.metadataqa.marc.RuleCatalogUtils;
8	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
9	import de.gwdg.metadataqa.marc.cli.parameters.Shacl4bibParameters;
10	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
11	import de.gwdg.metadataqa.marc.cli.utils.BibSelector;
12	import de.gwdg.metadataqa.marc.cli.utils.BibSelectorFactory;
13	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
14	import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
15	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
16	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
17	import org.apache.commons.cli.Options;
18	import org.apache.commons.cli.ParseException;
19	import org.apache.commons.io.FileUtils;
20	import org.marc4j.marc.Record;
21
22	import java.io.File;
23	import java.io.IOException;
24	import java.io.Serializable;
25	import java.nio.file.Files;
26	import java.nio.file.Path;
27	import java.util.List;
28	import java.util.Map;
29	import java.util.logging.Level;
30	import java.util.logging.Logger;
31
32	public class Shacl4bib extends QACli<Shacl4bibParameters> implements BibliographicInputProcessor, Serializable {
33
34	private static final Logger logger = Logger.getLogger(Shacl4bib.class.getCanonicalName());	1✔
35	private final boolean readyToProcess;
36	private File outputFile;
37	private RuleCatalog ruleCatalog;
38	private SchemaConfiguration schema;
39	private RecordFilter recordFilter;
40
41	public Shacl4bib(String[] args) throws ParseException {	1✔
42	parameters = new Shacl4bibParameters(args);	1✔
43	recordFilter = parameters.getRecordFilter();	1✔
44	readyToProcess = true;	1✔
45	}	1✔
46
47	public static void main(String[] args) {
48	BibliographicInputProcessor processor = null;	1✔
49	try {
50	processor = new Shacl4bib(args);	1✔
51	} catch (ParseException e) {	×
52	System.err.println("ERROR. " + e.getLocalizedMessage());	×
53	System.exit(1);	×
54	}	1✔
55	if (processor.getParameters().getArgs().length < 1) {	1✔
56	System.err.println("Please provide a MARC file name!");	×
57	processor.printHelp(processor.getParameters().getOptions());	×
58	System.exit(0);	×
59	}
60	if (processor.getParameters().doHelp()) {	1✔
61	processor.printHelp(processor.getParameters().getOptions());	×
62	System.exit(0);	×
63	}
64	RecordIterator iterator = new RecordIterator(processor);	1✔
65	iterator.start();	1✔
66	}	1✔
67
68	@Override
69	public CommonParameters getParameters() {
70	return parameters;	1✔
71	}
72
73	@Override
74	public void beforeIteration() {
75	logger.info(parameters.formatParameters());	1✔
76	outputFile = new File(parameters.getOutputDir(), parameters.getShaclOutputFile());	1✔
77
78	String shaclConfigurationFile = parameters.getShaclConfigurationFile();	1✔
79	try {
80	if (shaclConfigurationFile.endsWith(".json"))	1✔
81	schema = ConfigurationReader.readSchemaJson(shaclConfigurationFile);	1✔
82	else
83	schema = ConfigurationReader.readSchemaYaml(shaclConfigurationFile);	1✔
84	} catch (IOException exception) {	×
85	logger.severe("Error when the SHACL schema is initialized. " + exception.getLocalizedMessage());	×
86	System.exit(0);	×
87	}	1✔
88
89	this.ruleCatalog = new RuleCatalog(schema.asSchema())	1✔
90	.setOnlyIdInHeader(true)	1✔
91	.setOutputType(parameters.getShaclOutputType());	1✔
92
93	if (outputFile.exists()) {	1✔
94	try {
95	Files.delete(outputFile.toPath());	×
96	} catch (IOException e) {	×
97	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());	×
98	}	×
99	}
100	List<String> header = ruleCatalog.getHeader();	1✔
101	header.add(0, "id");	1✔
102	printToFile(outputFile, CsvUtils.createCsv(header));	1✔
103	}	1✔
104
105	@Override
106	public void fileOpened(Path path) {
107	logger.log(Level.INFO, "file opened: {0}", new Object[]{path});	1✔
108	}	1✔
109
110	@Override
111	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
112	// do nothing
113	}	1✔
114
115	@Override
116	public void processRecord(BibliographicRecord marcRecord, int recordNumber, List<ValidationError> errors) throws IOException {
117	// do nothing
118	}	×
119
120	@Override
121	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
122	if (!recordFilter.isAllowable(bibliographicRecord)) {	1✔
123	logger.info("ignoring " + bibliographicRecord.getId());	×
124	return;	×
125	}
126
127	BibSelector selector = BibSelectorFactory.create(schema.getFormat(), bibliographicRecord);	1✔
128
129	if (selector != null) {	1✔
130	List<Object> values = RuleCatalogUtils.extract(ruleCatalog, ruleCatalog.measure(selector));	1✔
131	values.add(0, bibliographicRecord.getId(true));	1✔
132	printToFile(outputFile, CsvUtils.createCsvFromObjects(values));	1✔
133	}
134	}	1✔
135
136	@Override
137	public void fileProcessed() {
138
139	}	×
140
141	@Override
142	public void afterIteration(int numberOfprocessedRecords, long duration) {
143	copySchaclFileToOutputDir();	1✔
144	saveParameters("shacl4bib.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
145	}	1✔
146
147	private void copySchaclFileToOutputDir() {
148	File source = new File(parameters.getShaclConfigurationFile());	1✔
149	try {
150	FileUtils.copyFileToDirectory(source, new File(parameters.getOutputDir()));	1✔
151	} catch (IOException e) {	×
152	logger.warning(e.getLocalizedMessage());	×
153	}	1✔
154	}	1✔
155
156	@Override
157	public void printHelp(Options options) {
158
159	}	×
160
161	@Override
162	public boolean readyToProcess() {
163	return readyToProcess;	1✔
164	}
165	}

pkiraly / metadata-qa-marc / #1430

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous