#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.76

/src/main/java/de/gwdg/metadataqa/marc/cli/ShelfReadyCompleteness.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.analysis.shelfready.ShelfReadyAnalysis;
import de.gwdg.metadataqa.marc.analysis.shelfready.ShelfReadyFieldsBooks;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.ShelfReadyCompletenessParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.dao.record.Marc21BibliographicRecord;
import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
import de.gwdg.metadataqa.marc.dao.record.UnimarcRecord;
import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.marc4j.marc.Record;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static de.gwdg.metadataqa.marc.Utils.createRow;
import static de.gwdg.metadataqa.marc.Utils.quote;

/**
 * usage:
 * java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar \
 * de.gwdg.metadataqa.marc.cli.ShelfReadyCompleteness [MARC21 file]
 *
 * @author Péter Király <peter.kiraly at gwdg.de>
 */
public class ShelfReadyCompleteness extends QACli<ShelfReadyCompletenessParameters> implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(
    ShelfReadyCompleteness.class.getCanonicalName()
  );
  private final Options options;
  private final boolean readyToProcess;
  private File output = null;

  public ShelfReadyCompleteness(String[] args) throws ParseException {
    parameters = new ShelfReadyCompletenessParameters(args);
    options = parameters.getOptions();
    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new ShelfReadyCompleteness(args);
    } catch (ParseException e) {
      System.err.println("ERROR. " + e.getLocalizedMessage());
      System.exit(1);
    }

    if (processor.getParameters().getArgs().length < 1) {
      System.err.println("Please provide a MARC file name!");
      System.exit(1);
    }
    if (processor.getParameters().doHelp()) {
      processor.printHelp(processor.getParameters().getOptions());
      System.exit(0);
    }

    RecordIterator iterator = new RecordIterator(processor);
    iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
    iterator.start();
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void beforeIteration() {
    logger.info(parameters.formatParameters());
    printFields();

    output = new File(parameters.getOutputDir(), parameters.getFileName());
    if (output.exists()) {
      try {
        Files.delete(output.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", output.getAbsolutePath());
      }
    }
    print(createRow(createHeaders()));
  }

  private List<String> createHeaders() {
    List<String> headers = new ArrayList<>();
    headers.add("id");
    headers.addAll(ShelfReadyAnalysis.getHeaders());
    headers.add("total");
    return headers;
  }

  @Override
  public void fileOpened(Path path) {
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) {
    if (parameters.getRecordIgnorator().isIgnorable(bibliographicRecord))
      return;

    List<Double> scores = ShelfReadyAnalysis.getScores(bibliographicRecord);
    String id = parameters.getTrimId()
              ? bibliographicRecord.getId().trim()
              : bibliographicRecord.getId();

    List<String> scoresToString = new ArrayList<>();
    for (Double score : scores)
      scoresToString.add(String.format(Locale.ENGLISH, "%.2f", score));

    String message = String.format(
      "\"%s\",%s%n",
      id, StringUtils.join(scoresToString, ",")
    );
    print(message);
  }

  @Override
  public void fileProcessed() {

  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    saveParameters("shelf-ready-completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }

  public void printHelp(Options options) {
    HelpFormatter formatter = new HelpFormatter();
    String message = String.format(
      "java -cp qa-catalogue.jar %s [options] [file]",
      this.getClass().getCanonicalName()
    );
    formatter.printHelp(message, options);
  }

  private void print(String message) {
    try {
      FileUtils.writeStringToFile(output, message, StandardCharsets.UTF_8, true);
    } catch (IOException e) {
      logger.log(Level.WARNING, "print", e);
    }
  }

  private void printFields() {
    var path = Paths.get(parameters.getOutputDir(), "shelf-ready-completeness-fields.csv");
    try (var writer = Files.newBufferedWriter(path)) {
      writer.write(createRow("name", "label", "marcpath", "score"));
      Map<ShelfReadyFieldsBooks, Map<String, List<String>>> map;

      if (parameters.getSchemaType() == SchemaType.MARC21) {
        map = (new Marc21BibliographicRecord()).getShelfReadyMap();
      } else if (parameters.getSchemaType() == SchemaType.PICA) {
        map = (new PicaRecord()).getShelfReadyMap();
      } else if (parameters.getSchemaType() == SchemaType.UNIMARC) {
        map = (new UnimarcRecord()).getShelfReadyMap();
      } else {
        throw new IllegalArgumentException("Unknown schema type: " + parameters.getSchemaType());
      }
      for (Map.Entry<ShelfReadyFieldsBooks, Map<String, List<String>>> field : map.entrySet()) {
        ShelfReadyFieldsBooks category = field.getKey();
        String paths = transformPaths(field.getValue());
        try {
          writer.write(createRow(category.name(), quote(category.getLabel()), quote(paths), category.getScore()));
        } catch (IOException e) {
          logger.log(Level.WARNING, "printFields", e);
        }
      }
    } catch (IOException e) {
      logger.log(Level.WARNING, "printFields", e);
    }
  }

  private String transformPaths(Map<String, List<String>> value) {
    List<String> paths = new ArrayList<>();
    for (Map.Entry<String, List<String>> field : value.entrySet()) {
      if (field.getValue() == null || field.getValue().isEmpty()) {
        paths.add(field.getKey());
        continue;
      }

      for (String code : field.getValue()) {
        paths.add(field.getKey() + "$" + code);
      }
    }
    return StringUtils.join(paths, ",");
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.analysis.shelfready.ShelfReadyAnalysis;
4	import de.gwdg.metadataqa.marc.analysis.shelfready.ShelfReadyFieldsBooks;
5	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
6	import de.gwdg.metadataqa.marc.cli.parameters.ShelfReadyCompletenessParameters;
7	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
8	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
9	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
10	import de.gwdg.metadataqa.marc.dao.record.Marc21BibliographicRecord;
11	import de.gwdg.metadataqa.marc.dao.record.PicaRecord;
12	import de.gwdg.metadataqa.marc.dao.record.UnimarcRecord;
13	import de.gwdg.metadataqa.marc.definition.bibliographic.SchemaType;
14	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
15	import org.apache.commons.cli.HelpFormatter;
16	import org.apache.commons.cli.Options;
17	import org.apache.commons.cli.ParseException;
18	import org.apache.commons.io.FileUtils;
19	import org.apache.commons.lang3.StringUtils;
20	import org.marc4j.marc.Record;
21
22	import java.io.File;
23	import java.io.IOException;
24	import java.io.Serializable;
25	import java.nio.charset.StandardCharsets;
26	import java.nio.file.Files;
27	import java.nio.file.Path;
28	import java.nio.file.Paths;
29	import java.util.ArrayList;
30	import java.util.List;
31	import java.util.Locale;
32	import java.util.Map;
33	import java.util.logging.Level;
34	import java.util.logging.Logger;
35
36	import static de.gwdg.metadataqa.marc.Utils.createRow;
37	import static de.gwdg.metadataqa.marc.Utils.quote;
38
39	/**
40	* usage:
41	* java -cp target/qa-catalogue-0.1-SNAPSHOT-jar-with-dependencies.jar \
42	* de.gwdg.metadataqa.marc.cli.ShelfReadyCompleteness [MARC21 file]
43	*
44	* @author Péter Király <peter.kiraly at gwdg.de>
45	*/
46	public class ShelfReadyCompleteness extends QACli<ShelfReadyCompletenessParameters> implements BibliographicInputProcessor, Serializable {
47
48	private static final Logger logger = Logger.getLogger(	1✔
49	ShelfReadyCompleteness.class.getCanonicalName()	1✔
50	);
51	private final Options options;
52	private final boolean readyToProcess;
53	private File output = null;	1✔
54
55	public ShelfReadyCompleteness(String[] args) throws ParseException {	1✔
56	parameters = new ShelfReadyCompletenessParameters(args);	1✔
57	options = parameters.getOptions();	1✔
58	readyToProcess = true;	1✔
59	}	1✔
60
61	public static void main(String[] args) {
62	BibliographicInputProcessor processor = null;	1✔
63	try {
64	processor = new ShelfReadyCompleteness(args);	1✔
UNCOV 65	} catch (ParseException e) {	×
66	System.err.println("ERROR. " + e.getLocalizedMessage());	×
67	System.exit(1);	×
68	}	1✔
69
70	if (processor.getParameters().getArgs().length < 1) {	1✔
UNCOV 71	System.err.println("Please provide a MARC file name!");	×
72	System.exit(1);	×
73	}
74	if (processor.getParameters().doHelp()) {	1✔
UNCOV 75	processor.printHelp(processor.getParameters().getOptions());	×
76	System.exit(0);	×
77	}
78
79	RecordIterator iterator = new RecordIterator(processor);	1✔
80	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	1✔
81	iterator.start();	1✔
82	}	1✔
83
84	@Override
85	public CommonParameters getParameters() {
86	return parameters;	1✔
87	}
88
89	@Override
90	public void beforeIteration() {
91	logger.info(parameters.formatParameters());	1✔
92	printFields();	1✔
93
94	output = new File(parameters.getOutputDir(), parameters.getFileName());	1✔
95	if (output.exists()) {	1✔
96	try {
97	Files.delete(output.toPath());	1✔
NEW 98	} catch (IOException e) {	×
NEW 99	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", output.getAbsolutePath());	×
100	}	1✔
101	}
102	print(createRow(createHeaders()));	1✔
103	}	1✔
104
105	private List<String> createHeaders() {
106	List<String> headers = new ArrayList<>();	1✔
107	headers.add("id");	1✔
108	headers.addAll(ShelfReadyAnalysis.getHeaders());	1✔
109	headers.add("total");	1✔
110	return headers;	1✔
111	}
112
113	@Override
114	public void fileOpened(Path path) {
115	}	1✔
116
117	@Override
118	public void processRecord(Record marc4jRecord, int recordNumber) {
119	// do nothing
120	}	1✔
121
122	@Override
123	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 124	processRecord(bibliographicRecord, recordNumber);	×
UNCOV 125	}	×
126
127	@Override
128	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) {
129	if (parameters.getRecordIgnorator().isIgnorable(bibliographicRecord))	1✔
130	return;	×
131
132	List<Double> scores = ShelfReadyAnalysis.getScores(bibliographicRecord);	1✔
133	String id = parameters.getTrimId()	1✔
NEW 134	? bibliographicRecord.getId().trim()	×
135	: bibliographicRecord.getId();	1✔
136
137	List<String> scoresToString = new ArrayList<>();	1✔
138	for (Double score : scores)	1✔
139	scoresToString.add(String.format(Locale.ENGLISH, "%.2f", score));	1✔
140
141	String message = String.format(	1✔
142	"\"%s\",%s%n",
143	id, StringUtils.join(scoresToString, ",")	1✔
144	);
145	print(message);	1✔
146	}	1✔
147
148	@Override
149	public void fileProcessed() {
150
UNCOV 151	}	×
152
153	@Override
154	public void afterIteration(int numberOfprocessedRecords, long duration) {
155	saveParameters("shelf-ready-completeness.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	1✔
156	}	1✔
157
158	@Override
159	public boolean readyToProcess() {
160	return readyToProcess;	1✔
161	}
162
163	public void printHelp(Options options) {
UNCOV 164	HelpFormatter formatter = new HelpFormatter();	×
UNCOV 165	String message = String.format(	×
166	"java -cp qa-catalogue.jar %s [options] [file]",
UNCOV 167	this.getClass().getCanonicalName()	×
168	);
UNCOV 169	formatter.printHelp(message, options);	×
170	}	×
171
172	private void print(String message) {
173	try {
174	FileUtils.writeStringToFile(output, message, StandardCharsets.UTF_8, true);	1✔
175	} catch (IOException e) {	×
176	logger.log(Level.WARNING, "print", e);	×
177	}	1✔
178	}	1✔
179
180	private void printFields() {
181	var path = Paths.get(parameters.getOutputDir(), "shelf-ready-completeness-fields.csv");	1✔
182	try (var writer = Files.newBufferedWriter(path)) {	1✔
183	writer.write(createRow("name", "label", "marcpath", "score"));	1✔
184	Map<ShelfReadyFieldsBooks, Map<String, List<String>>> map;
185
186	if (parameters.getSchemaType() == SchemaType.MARC21) {	1✔
187	map = (new Marc21BibliographicRecord()).getShelfReadyMap();	1✔
188	} else if (parameters.getSchemaType() == SchemaType.PICA) {	1✔
189	map = (new PicaRecord()).getShelfReadyMap();	1✔
190	} else if (parameters.getSchemaType() == SchemaType.UNIMARC) {	1✔
191	map = (new UnimarcRecord()).getShelfReadyMap();	1✔
192	} else {
NEW 193	throw new IllegalArgumentException("Unknown schema type: " + parameters.getSchemaType());	×
194	}
195	for (Map.Entry<ShelfReadyFieldsBooks, Map<String, List<String>>> field : map.entrySet()) {	1✔
196	ShelfReadyFieldsBooks category = field.getKey();	1✔
197	String paths = transformPaths(field.getValue());	1✔
198	try {
199	writer.write(createRow(category.name(), quote(category.getLabel()), quote(paths), category.getScore()));	1✔
UNCOV 200	} catch (IOException e) {	×
UNCOV 201	logger.log(Level.WARNING, "printFields", e);	×
202	}	1✔
203	}	1✔
UNCOV 204	} catch (IOException e) {	×
UNCOV 205	logger.log(Level.WARNING, "printFields", e);	×
206	}	1✔
207	}	1✔
208
209	private String transformPaths(Map<String, List<String>> value) {
210	List<String> paths = new ArrayList<>();	1✔
211	for (Map.Entry<String, List<String>> field : value.entrySet()) {	1✔
212	if (field.getValue() == null \|\| field.getValue().isEmpty()) {	1✔
213	paths.add(field.getKey());	1✔
214	continue;	1✔
215	}
216
217	for (String code : field.getValue()) {	1✔
218	paths.add(field.getKey() + "$" + code);	1✔
219	}	1✔
220	}	1✔
221	return StringUtils.join(paths, ",");	1✔
222	}
223	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous