#1451

Committed 14 Apr 2025 09:26AM UTC coverage: 90.493% (+0.003%) from 90.49%

Build # #1451

Build Type

push

Committed by

Commit Message

Add Danish Royal Library's dataset

Coverage Stats

23 of 23 new or added lines in 2 files covered. (100.0%)

16 existing lines in 2 files now uncovered.

36579 of 40422 relevant lines covered (90.49%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.0

/src/main/java/de/gwdg/metadataqa/marc/cli/TranslationAnalysis.java

package de.gwdg.metadataqa.marc.cli;

import com.fasterxml.jackson.databind.ObjectMapper;
import de.gwdg.metadataqa.api.configuration.SchemaConfiguration;
import de.gwdg.metadataqa.api.interfaces.MetricResult;
import de.gwdg.metadataqa.api.model.XmlFieldInstance;
import de.gwdg.metadataqa.api.rule.RuleCatalog;
import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
import de.gwdg.metadataqa.marc.CsvUtils;
import de.gwdg.metadataqa.marc.RuleCatalogUtils;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.TranslationParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.BibSelector;
import de.gwdg.metadataqa.marc.cli.utils.BibSelectorFactory;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.cli.utils.ShaclUtils;
import de.gwdg.metadataqa.marc.cli.utils.TranslationModel;
import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;
import de.gwdg.metadataqa.marc.cli.utils.translation.PublicationYearNormaliser;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.marc4j.marc.Record;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;

public class TranslationAnalysis extends QACli<TranslationParameters>
                                 implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(TranslationAnalysis.class.getCanonicalName());

  private boolean readyToProcess;
  private File outputFile;
  private RuleCatalog ruleCatalog;
  private SchemaConfiguration schema;
  private final RecordFilter recordFilter;
  private Map<String, Long> failed;
  private Map<String, String> rulePathMap;
  private Map<String, File> debugFiles;
  private PlaceNameNormaliser placeNameNormaliser;
  private ObjectMapper mapper;
  private File exportFile;
  private PublicationYearNormaliser publicationYearNormaliser;

  public TranslationAnalysis(String[] args) throws ParseException {
    parameters = new TranslationParameters(args);
    recordFilter = parameters.getRecordFilter();
    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new TranslationAnalysis(args);
    } catch (ParseException e) {
      System.err.println("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }
    if (processor.getParameters().getArgs().length < 1) {
      System.err.println("Please provide a MARC file name!");
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }
    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters.formatParameters());
    outputFile = new File(parameters.getOutputDir(), parameters.getShaclOutputFile());

    schema = ShaclUtils.setupSchema(parameters);
    ruleCatalog = ShaclUtils.setupRuleCatalog(schema, parameters);
    rulePathMap = ShaclUtils.createRulePathMap(schema);
    failed = new HashMap<>();
    if (parameters.getDebugFailedRules() != null && !parameters.getDebugFailedRules().isEmpty()) {
      debugFiles = new HashMap<>();
      for (String ruleId : parameters.getDebugFailedRules()) {
        debugFiles.put(ruleId, new File(parameters.getOutputDir(), String.format("translations-deubg-%s.txt", ruleId)));
      }
    }

    if (outputFile.exists()) {
      try {
        Files.delete(outputFile.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());
      }
    }
    List<String> header = ruleCatalog.getHeader();
    header.add(0, "id");
    header.addAll(TranslationModel.header());
    printToFile(outputFile, CsvUtils.createCsv(header));

    if (parameters.getTranslationPlaceNameDictionaryDir() != null)
      placeNameNormaliser = new PlaceNameNormaliser(parameters.getTranslationPlaceNameDictionaryDir(), parameters.getOutputDir());

    if (parameters.getTranslationExport() != null) {
      exportFile = new File(parameters.getOutputDir(), parameters.getTranslationExport());
      if (exportFile.exists())
        exportFile.delete();
      mapper = new ObjectMapper();
    }

    publicationYearNormaliser = new PublicationYearNormaliser(parameters.getOutputDir());
  }

  @Override
  public void fileOpened(Path path) {
    logger.log(Level.INFO, "file opened: {0}", new Object[]{path});
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord,
                            int recordNumber,
                            List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord,
                            int recordNumber)
      throws IOException {
    if (!recordFilter.isAllowable(bibliographicRecord)) {
      // logger.info("ignoring " + bibliographicRecord.getId());
      return;
    }

    BibSelector selector = BibSelectorFactory.create(schema.getFormat(), bibliographicRecord);

    if (selector != null) {
      List<MetricResult> results = ruleCatalog.measure(selector);
      Map<String, RuleCheckerOutput> resultMap = (Map<String, RuleCheckerOutput>) results.get(0).getResultMap();
      TranslationModel model = new TranslationModel(resultMap, selector, placeNameNormaliser, publicationYearNormaliser, parameters.getMarcVersion());

      List<String> debugIds = parameters.getDebugFailedRules();
      if (debugIds != null && !debugIds.isEmpty()) {
        for (String debugId : debugIds) {
          if (resultMap.get(debugId).getStatus().equals(RuleCheckingOutputStatus.FAILED)) {
            failed.computeIfAbsent(debugId, k -> 0L);
            failed.put(debugId, failed.get(debugId) + 1);
            List<XmlFieldInstance> values = selector.get(rulePathMap.get(debugId));
            printToFile(debugFiles.get(debugId), values.get(0).getValue() + "\n");
          }
        }
      }

      List<Object> values = RuleCatalogUtils.extract(ruleCatalog, results);
      values.add(0, bibliographicRecord.getId(true));
      values.addAll(model.values());

      printToFile(outputFile, CsvUtils.createCsvFromObjects(values));

      if (model.isTranslation()) {
        Map<String, Object> extracted = model.extract();
        extracted.put("id", bibliographicRecord.getId());
        if (exportFile != null && mapper != null)
          printToFile(exportFile, mapper.writeValueAsString(extracted) + "\n");
      }
    }
  }

  @Override
  public void fileProcessed() {
    // do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    String report = failed.entrySet().stream()
        .map(e ->
          String.format(
            "%s: %d failures (%.2f%% of records)",
            e.getKey(), e.getValue(), e.getValue() * 100.0 / numberOfprocessedRecords
          ))
          .collect(Collectors.joining(", "));
    if (!report.isEmpty())
      logger.log(Level.WARNING, "failed rules: {0}", report);
    copyFileToOutputDir(parameters.getShaclConfigurationFile());
    saveParameters("translation-analysis.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));

    placeNameNormaliser.reportUnresolvedPlaceNames();
    publicationYearNormaliser.reportUnresolvedYears();
    publicationYearNormaliser.reportPatterns();
  }

  @Override
  public void printHelp(Options options) {

  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import com.fasterxml.jackson.databind.ObjectMapper;
4	import de.gwdg.metadataqa.api.configuration.SchemaConfiguration;
5	import de.gwdg.metadataqa.api.interfaces.MetricResult;
6	import de.gwdg.metadataqa.api.model.XmlFieldInstance;
7	import de.gwdg.metadataqa.api.rule.RuleCatalog;
8	import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
9	import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
10	import de.gwdg.metadataqa.marc.CsvUtils;
11	import de.gwdg.metadataqa.marc.RuleCatalogUtils;
12	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
13	import de.gwdg.metadataqa.marc.cli.parameters.TranslationParameters;
14	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
15	import de.gwdg.metadataqa.marc.cli.utils.BibSelector;
16	import de.gwdg.metadataqa.marc.cli.utils.BibSelectorFactory;
17	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
18	import de.gwdg.metadataqa.marc.cli.utils.ShaclUtils;
19	import de.gwdg.metadataqa.marc.cli.utils.TranslationModel;
20	import de.gwdg.metadataqa.marc.cli.utils.ignorablerecords.RecordFilter;
21	import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;
22	import de.gwdg.metadataqa.marc.cli.utils.translation.PublicationYearNormaliser;
23	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
24	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
25	import org.apache.commons.cli.Options;
26	import org.apache.commons.cli.ParseException;
27	import org.marc4j.marc.Record;
28
29	import java.io.File;
30	import java.io.IOException;
31	import java.io.Serializable;
32	import java.nio.file.Files;
33	import java.nio.file.Path;
34	import java.util.HashMap;
35	import java.util.List;
36	import java.util.Map;
37	import java.util.logging.Level;
38	import java.util.logging.Logger;
39	import java.util.stream.Collectors;
40
41	public class TranslationAnalysis extends QACli<TranslationParameters>
42	implements BibliographicInputProcessor, Serializable {
43
44	private static final Logger logger = Logger.getLogger(TranslationAnalysis.class.getCanonicalName());	1✔
45
46	private boolean readyToProcess;
47	private File outputFile;
48	private RuleCatalog ruleCatalog;
49	private SchemaConfiguration schema;
50	private final RecordFilter recordFilter;
51	private Map<String, Long> failed;
52	private Map<String, String> rulePathMap;
53	private Map<String, File> debugFiles;
54	private PlaceNameNormaliser placeNameNormaliser;
55	private ObjectMapper mapper;
56	private File exportFile;
57	private PublicationYearNormaliser publicationYearNormaliser;
58
59	public TranslationAnalysis(String[] args) throws ParseException {	1✔
60	parameters = new TranslationParameters(args);	1✔
61	recordFilter = parameters.getRecordFilter();	1✔
62	readyToProcess = true;	1✔
63	}	1✔
64
65	public static void main(String[] args) {
66	BibliographicInputProcessor processor = null;	×
67	try {
68	processor = new TranslationAnalysis(args);	×
69	} catch (ParseException e) {	×
70	System.err.println("ERROR. " + e.getLocalizedMessage());	×
71	System.exit(1);	×
72	}	×
73	if (processor.getParameters().getArgs().length < 1) {	×
74	System.err.println("Please provide a MARC file name!");	×
75	processor.printHelp(processor.getParameters().getOptions());	×
76	System.exit(0);	×
77	}
78	if (processor.getParameters().doHelp()) {	×
79	processor.printHelp(processor.getParameters().getOptions());	×
80	System.exit(0);	×
81	}
82	RecordIterator iterator = new RecordIterator(processor);	×
83	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	×
84	iterator.start();	×
85	}	×
86
87	@Override
88	public CommonParameters getParameters() {
89	return parameters;	1✔
90	}
91
92	@Override
93	public void beforeIteration() {
94	logger.info(parameters.formatParameters());	1✔
95	outputFile = new File(parameters.getOutputDir(), parameters.getShaclOutputFile());	1✔
96
97	schema = ShaclUtils.setupSchema(parameters);	1✔
98	ruleCatalog = ShaclUtils.setupRuleCatalog(schema, parameters);	1✔
99	rulePathMap = ShaclUtils.createRulePathMap(schema);	1✔
100	failed = new HashMap<>();	1✔
101	if (parameters.getDebugFailedRules() != null && !parameters.getDebugFailedRules().isEmpty()) {	1✔
102	debugFiles = new HashMap<>();	1✔
103	for (String ruleId : parameters.getDebugFailedRules()) {	1✔
104	debugFiles.put(ruleId, new File(parameters.getOutputDir(), String.format("translations-deubg-%s.txt", ruleId)));	1✔
105	}	1✔
106	}
107
108	if (outputFile.exists()) {	1✔
109	try {
110	Files.delete(outputFile.toPath());	×
111	} catch (IOException e) {	×
112	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", outputFile.getAbsolutePath());	×
113	}	×
114	}
115	List<String> header = ruleCatalog.getHeader();	1✔
116	header.add(0, "id");	1✔
117	header.addAll(TranslationModel.header());	1✔
118	printToFile(outputFile, CsvUtils.createCsv(header));	1✔
119
120	if (parameters.getTranslationPlaceNameDictionaryDir() != null)	1✔
121	placeNameNormaliser = new PlaceNameNormaliser(parameters.getTranslationPlaceNameDictionaryDir(), parameters.getOutputDir());	1✔
122
123	if (parameters.getTranslationExport() != null) {	1✔
124	exportFile = new File(parameters.getOutputDir(), parameters.getTranslationExport());	1✔
125	if (exportFile.exists())	1✔
126	exportFile.delete();	1✔
127	mapper = new ObjectMapper();	1✔
128	}
129
130	publicationYearNormaliser = new PublicationYearNormaliser(parameters.getOutputDir());	1✔
131	}	1✔
132
133	@Override
134	public void fileOpened(Path path) {
135	logger.log(Level.INFO, "file opened: {0}", new Object[]{path});	1✔
136	}	1✔
137
138	@Override
139	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
140	// do nothing
141	}	1✔
142
143	@Override
144	public void processRecord(BibliographicRecord bibliographicRecord,
145	int recordNumber,
146	List<ValidationError> errors) throws IOException {
147	processRecord(bibliographicRecord, recordNumber);	×
148	}	×
149
150	@Override
151	public void processRecord(BibliographicRecord bibliographicRecord,
152	int recordNumber)
153	throws IOException {
154	if (!recordFilter.isAllowable(bibliographicRecord)) {	1✔
155	// logger.info("ignoring " + bibliographicRecord.getId());
156	return;	×
157	}
158
159	BibSelector selector = BibSelectorFactory.create(schema.getFormat(), bibliographicRecord);	1✔
160
161	if (selector != null) {	1✔
162	List<MetricResult> results = ruleCatalog.measure(selector);	1✔
163	Map<String, RuleCheckerOutput> resultMap = (Map<String, RuleCheckerOutput>) results.get(0).getResultMap();	1✔
164	TranslationModel model = new TranslationModel(resultMap, selector, placeNameNormaliser, publicationYearNormaliser, parameters.getMarcVersion());	1✔
165
166	List<String> debugIds = parameters.getDebugFailedRules();	1✔
167	if (debugIds != null && !debugIds.isEmpty()) {	1✔
168	for (String debugId : debugIds) {	1✔
169	if (resultMap.get(debugId).getStatus().equals(RuleCheckingOutputStatus.FAILED)) {	1✔
170	failed.computeIfAbsent(debugId, k -> 0L);	1✔
171	failed.put(debugId, failed.get(debugId) + 1);	1✔
172	List<XmlFieldInstance> values = selector.get(rulePathMap.get(debugId));	1✔
173	printToFile(debugFiles.get(debugId), values.get(0).getValue() + "\n");	1✔
174	}
175	}	1✔
176	}
177
178	List<Object> values = RuleCatalogUtils.extract(ruleCatalog, results);	1✔
179	values.add(0, bibliographicRecord.getId(true));	1✔
180	values.addAll(model.values());	1✔
181
182	printToFile(outputFile, CsvUtils.createCsvFromObjects(values));	1✔
183
184	if (model.isTranslation()) {	1✔
185	Map<String, Object> extracted = model.extract();	1✔
186	extracted.put("id", bibliographicRecord.getId());	1✔
187	if (exportFile != null && mapper != null)	1✔
188	printToFile(exportFile, mapper.writeValueAsString(extracted) + "\n");	1✔
189	}
190	}
191	}	1✔
192
193	@Override
194	public void fileProcessed() {
195	// do nothing
196	}	×
197
198	@Override
199	public void afterIteration(int numberOfprocessedRecords, long duration) {
200	String report = failed.entrySet().stream()	1✔
201	.map(e ->	1✔
202	String.format(	1✔
203	"%s: %d failures (%.2f%% of records)",
204	e.getKey(), e.getValue(), e.getValue() * 100.0 / numberOfprocessedRecords	1✔
205	))
206	.collect(Collectors.joining(", "));	1✔
207	if (!report.isEmpty())	1✔
208	logger.log(Level.WARNING, "failed rules: {0}", report);	1✔
209	copyFileToOutputDir(parameters.getShaclConfigurationFile());	1✔
210	saveParameters("translation-analysis.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
211
212	placeNameNormaliser.reportUnresolvedPlaceNames();	1✔
213	publicationYearNormaliser.reportUnresolvedYears();	1✔
214	publicationYearNormaliser.reportPatterns();	1✔
215	}	1✔
216
217	@Override
218	public void printHelp(Options options) {
219
UNCOV 220	}	×
221
222	@Override
223	public boolean readyToProcess() {
224	return readyToProcess;	1✔
225	}
226	}

pkiraly / metadata-qa-marc / #1451

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous