#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.97

/src/main/java/de/gwdg/metadataqa/marc/cli/ThompsonTraillCompleteness.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.analysis.thompsontraill.Marc21ThompsonTraillAnalysis;
import de.gwdg.metadataqa.marc.analysis.thompsontraill.PicaThompsonTraillAnalysis;
import de.gwdg.metadataqa.marc.analysis.thompsontraill.ThompsonTraillAnalysis;
import de.gwdg.metadataqa.marc.analysis.thompsontraill.ThompsonTraillFields;
import de.gwdg.metadataqa.marc.analysis.thompsontraill.UnimarcThompsonTraillAnalysis;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.ThompsonTraillCompletenessParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.marc4j.marc.Record;

import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static de.gwdg.metadataqa.marc.Utils.createRow;
import static de.gwdg.metadataqa.marc.Utils.quote;

/**
 * usage:
 * java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar \
 * de.gwdg.metadataqa.marc.cli.ThompsonTraillCompleteness [MARC21 file]
 *
 * @author Péter Király <peter.kiraly at gwdg.de>
 */
public class ThompsonTraillCompleteness extends QACli<ThompsonTraillCompletenessParameters> implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(
    ThompsonTraillCompleteness.class.getCanonicalName()
  );
  private final Options options;
  private final boolean readyToProcess;
  private File output = null;
  private ThompsonTraillAnalysis thompsonTraillAnalysis;

  public ThompsonTraillCompleteness(String[] args) throws ParseException {
    parameters = new ThompsonTraillCompletenessParameters(args);
    logger.info("tt().marcxml: " + parameters.isMarcxml());
    options = parameters.getOptions();
    // Create a ThompsonTraillAnalysis object based on the schema type
    setThompsonTraillAnalysis();

    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new ThompsonTraillCompleteness(args);
    } catch (ParseException e) {
      logger.severe("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }

    if (processor.getParameters().getArgs().length < 1) {
      logger.severe("Please provide a MARC file name!");
      System.exit(1);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }

    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(() -> parameters.formatParameters());
    printFields();

    output = new File(parameters.getOutputDir(), parameters.getFileName());
    if (output.exists()) {
      try {
        Files.delete(output.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", output.getAbsolutePath());
      }
    }
    print(createRow(thompsonTraillAnalysis.getHeader()));
  }

  @Override
  public void fileOpened(Path path) {
    // do nothing
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord marcRecord, int recordNumber) {
    if (parameters.getRecordIgnorator().isIgnorable(marcRecord))
      return;

    List<Integer> scores = thompsonTraillAnalysis.getScores(marcRecord);
    String id = parameters.getTrimId()
              ? marcRecord.getId().trim()
              : marcRecord.getId();

    String message = String.format(
      "\"%s\",%s%n",
      id, StringUtils.join(scores, ",")
    );
    print(message);
  }

  @Override
  public void fileProcessed() {
    // The file processed method is not implemented.
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    saveParameters("tt-completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }

  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format(
      "java -cp qa-catalogue.jar %s [options] [file]",
      this.getClass().getCanonicalName()
    );
    formatter.printHelp(message, options);
  }

  private void print(String message) {
    try {
      FileUtils.writeStringToFile(output, message, StandardCharsets.UTF_8, true);
    } catch (IOException e) {
      logger.log(Level.SEVERE, "print", e);
    }
  }

  private void printFields() {
    var path = Paths.get(parameters.getOutputDir(), "tt-completeness-fields.csv");
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(createRow("name", "transformed", "fields"));
      Map<ThompsonTraillFields, List<String>> map = thompsonTraillAnalysis.getThompsonTraillTagsMap();
      for (ThompsonTraillFields field : ThompsonTraillFields.values()) {
        writeFieldRow(writer, field, map);
      }
    } catch (IOException e) {
      logger.log(Level.SEVERE, "printFields", e);
    }
  }

  private void writeFieldRow(BufferedWriter writer, ThompsonTraillFields field, Map<ThompsonTraillFields, List<String>> map) {
    try {
      writer.write(createRow(field.getLabel(), field.getMachine(), quote(StringUtils.join(map.getOrDefault(field, List.of()), ","))));
    } catch (IOException e) {
      logger.log(Level.SEVERE, "printFields", e);
    }
  }

  private void setThompsonTraillAnalysis() {
    switch (parameters.getSchemaType()) {
      case PICA:
        thompsonTraillAnalysis = new PicaThompsonTraillAnalysis();
        break;
      case UNIMARC:
        thompsonTraillAnalysis = new UnimarcThompsonTraillAnalysis();
        break;
      case MARC21:
      default:
        thompsonTraillAnalysis = new Marc21ThompsonTraillAnalysis();
        break;
    }
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.analysis.thompsontraill.Marc21ThompsonTraillAnalysis;
4	import de.gwdg.metadataqa.marc.analysis.thompsontraill.PicaThompsonTraillAnalysis;
5	import de.gwdg.metadataqa.marc.analysis.thompsontraill.ThompsonTraillAnalysis;
6	import de.gwdg.metadataqa.marc.analysis.thompsontraill.ThompsonTraillFields;
7	import de.gwdg.metadataqa.marc.analysis.thompsontraill.UnimarcThompsonTraillAnalysis;
8	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
9	import de.gwdg.metadataqa.marc.cli.parameters.ThompsonTraillCompletenessParameters;
10	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
11	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
12	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
13	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
14	import org.apache.commons.cli.HelpFormatter;
15	import org.apache.commons.cli.Options;
16	import org.apache.commons.cli.ParseException;
17	import org.apache.commons.io.FileUtils;
18	import org.apache.commons.lang3.StringUtils;
19	import org.marc4j.marc.Record;
20
21	import java.io.BufferedWriter;
22	import java.io.File;
23	import java.io.IOException;
24	import java.io.Serializable;
25	import java.nio.charset.StandardCharsets;
26	import java.nio.file.Files;
27	import java.nio.file.Path;
28	import java.nio.file.Paths;
29	import java.util.List;
30	import java.util.Map;
31	import java.util.logging.Level;
32	import java.util.logging.Logger;
33
34	import static de.gwdg.metadataqa.marc.Utils.createRow;
35	import static de.gwdg.metadataqa.marc.Utils.quote;
36
37	/**
38	* usage:
39	* java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar \
40	* de.gwdg.metadataqa.marc.cli.ThompsonTraillCompleteness [MARC21 file]
41	*
42	* @author Péter Király <peter.kiraly at gwdg.de>
43	*/
44	public class ThompsonTraillCompleteness extends QACli<ThompsonTraillCompletenessParameters> implements BibliographicInputProcessor, Serializable {
45
46	private static final Logger logger = Logger.getLogger(	1✔
47	ThompsonTraillCompleteness.class.getCanonicalName()	1✔
48	);
49	private final Options options;
50	private final boolean readyToProcess;
51	private File output = null;	1✔
52	private ThompsonTraillAnalysis thompsonTraillAnalysis;
53
54	public ThompsonTraillCompleteness(String[] args) throws ParseException {	1✔
55	parameters = new ThompsonTraillCompletenessParameters(args);	1✔
56	logger.info("tt().marcxml: " + parameters.isMarcxml());	1✔
57	options = parameters.getOptions();	1✔
58	// Create a ThompsonTraillAnalysis object based on the schema type
59	setThompsonTraillAnalysis();	1✔
60
61	readyToProcess = true;	1✔
62	}	1✔
63
64	public static void main(String[] args) {
65	BibliographicInputProcessor processor = null;	1✔
66	try {
67	processor = new ThompsonTraillCompleteness(args);	1✔
UNCOV 68	} catch (ParseException e) {	×
NEW 69	logger.severe("ERROR. " + e.getLocalizedMessage());	×
UNCOV 70	System.exit(1);	×
71	}	1✔
72
73	if (processor.getParameters().getArgs().length < 1) {	1✔
NEW 74	logger.severe("Please provide a MARC file name!");	×
UNCOV 75	System.exit(1);	×
76	}
77	if (processor.getParameters().doHelp()) {	1✔
78	processor.printHelp(processor.getParameters().getOptions());	×
79	System.exit(0);	×
80	}
81
82	RecordIterator iterator = new RecordIterator(processor);	1✔
83	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	1✔
84	iterator.start();	1✔
85	}	1✔
86
87	@Override
88	public CommonParameters getParameters() {
89	return parameters;	1✔
90	}
91
92	@Override
93	public void beforeIteration() {
94	logger.info(() -> parameters.formatParameters());	1✔
95	printFields();	1✔
96
97	output = new File(parameters.getOutputDir(), parameters.getFileName());	1✔
98	if (output.exists()) {	1✔
99	try {
NEW 100	Files.delete(output.toPath());	×
NEW 101	} catch (IOException e) {	×
NEW 102	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", output.getAbsolutePath());	×
NEW 103	}	×
104	}
105	print(createRow(thompsonTraillAnalysis.getHeader()));	1✔
106	}	1✔
107
108	@Override
109	public void fileOpened(Path path) {
110	// do nothing
111	}	1✔
112
113	@Override
114	public void processRecord(Record marc4jRecord, int recordNumber) {
115	// do nothing
116	}	1✔
117
118	@Override
119	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 120	processRecord(bibliographicRecord, recordNumber);	×
UNCOV 121	}	×
122
123	@Override
124	public void processRecord(BibliographicRecord marcRecord, int recordNumber) {
125	if (parameters.getRecordIgnorator().isIgnorable(marcRecord))	1✔
UNCOV 126	return;	×
127
128	List<Integer> scores = thompsonTraillAnalysis.getScores(marcRecord);	1✔
129	String id = parameters.getTrimId()	1✔
130	? marcRecord.getId().trim()	×
131	: marcRecord.getId();	1✔
132
133	String message = String.format(	1✔
134	"\"%s\",%s%n",
135	id, StringUtils.join(scores, ",")	1✔
136	);
137	print(message);	1✔
138	}	1✔
139
140	@Override
141	public void fileProcessed() {
142	// The file processed method is not implemented.
UNCOV 143	}	×
144
145	@Override
146	public void afterIteration(int numberOfprocessedRecords, long duration) {
147	saveParameters("tt-completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
148	}	1✔
149
150	@Override
151	public boolean readyToProcess() {
152	return readyToProcess;	1✔
153	}
154
155	public void printHelp(Options options) {
UNCOV 156	HelpFormatter formatter = new HelpFormatter();	×
UNCOV 157	String message = String.format(	×
158	"java -cp qa-catalogue.jar %s [options] [file]",
UNCOV 159	this.getClass().getCanonicalName()	×
160	);
UNCOV 161	formatter.printHelp(message, options);	×
UNCOV 162	}	×
163
164	private void print(String message) {
165	try {
166	FileUtils.writeStringToFile(output, message, StandardCharsets.UTF_8, true);	1✔
167	} catch (IOException e) {	×
168	logger.log(Level.SEVERE, "print", e);	×
169	}	1✔
170	}	1✔
171
172	private void printFields() {
173	var path = Paths.get(parameters.getOutputDir(), "tt-completeness-fields.csv");	1✔
174	try (var writer = Files.newBufferedWriter(path)) {	1✔
175	writer.write(createRow("name", "transformed", "fields"));	1✔
176	Map<ThompsonTraillFields, List<String>> map = thompsonTraillAnalysis.getThompsonTraillTagsMap();	1✔
177	for (ThompsonTraillFields field : ThompsonTraillFields.values()) {	1✔
178	writeFieldRow(writer, field, map);	1✔
179	}
UNCOV 180	} catch (IOException e) {	×
UNCOV 181	logger.log(Level.SEVERE, "printFields", e);	×
182	}	1✔
183	}	1✔
184
185	private void writeFieldRow(BufferedWriter writer, ThompsonTraillFields field, Map<ThompsonTraillFields, List<String>> map) {
186	try {
187	writer.write(createRow(field.getLabel(), field.getMachine(), quote(StringUtils.join(map.getOrDefault(field, List.of()), ","))));	1✔
NEW 188	} catch (IOException e) {	×
NEW 189	logger.log(Level.SEVERE, "printFields", e);	×
190	}	1✔
191	}	1✔
192
193	private void setThompsonTraillAnalysis() {
194	switch (parameters.getSchemaType()) {	1✔
195	case PICA:
196	thompsonTraillAnalysis = new PicaThompsonTraillAnalysis();	1✔
197	break;	1✔
198	case UNIMARC:
199	thompsonTraillAnalysis = new UnimarcThompsonTraillAnalysis();	1✔
200	break;	1✔
201	case MARC21:
202	default:
203	thompsonTraillAnalysis = new Marc21ThompsonTraillAnalysis();	1✔
204	break;
205	}
206	}	1✔
207	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous