#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/de/gwdg/metadataqa/marc/cli/NetworkAnalysis.java

package de.gwdg.metadataqa.marc.cli;

import de.gwdg.metadataqa.marc.analysis.NetworkAnalyzer;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.cli.parameters.NetworkAction;
import de.gwdg.metadataqa.marc.cli.parameters.NetworkParameters;
import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
import de.gwdg.metadataqa.marc.cli.utils.PairGenerator;
import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
import de.gwdg.metadataqa.marc.dao.DataField;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.model.validation.ValidationError;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.marc4j.marc.Record;

import java.io.BufferedWriter;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

import static de.gwdg.metadataqa.marc.Utils.createRow;

public class NetworkAnalysis extends QACli<NetworkParameters> implements BibliographicInputProcessor, Serializable {

  private static final Logger logger = Logger.getLogger(NetworkAnalysis.class.getCanonicalName());

  private final boolean readyToProcess;
  private final List<String> orphans = new ArrayList<>();
  private BufferedWriter networkWriter;

  public NetworkAnalysis(String[] args) throws ParseException {
    parameters = new NetworkParameters(args);
    readyToProcess = true;
  }

  public static void main(String[] args) {
    BibliographicInputProcessor processor = null;
    try {
      processor = new NetworkAnalysis(args);
    } catch (ParseException e) {
      logger.severe(createRow("ERROR. ", e.getLocalizedMessage()));
      System.exit(1);
    }
    NetworkParameters params = (NetworkParameters)processor.getParameters();
    NetworkAction action = params.getAction();
    logger.info("Action: " + ((NetworkParameters)processor.getParameters()).getAction());
    if (action.equals(NetworkAction.PAIRING)) {
      PairGenerator generator = new PairGenerator(params.getOutputDir(), params.getGroupLimit(),false);
      generator.generatePairs();
    } else {
      if (params.getArgs().length < 1) {
        logger.severe("Please provide a MARC file name!");
        processor.printHelp(params.getOptions());
        System.exit(0);
      }
      if (params.doHelp()) {
        processor.printHelp(params.getOptions());
        System.exit(0);
      }
      RecordIterator iterator = new RecordIterator(processor);
      iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());
      iterator.start();
    }
  }

  @Override
  public CommonParameters getParameters() {
    return parameters;
  }

  @Override
  public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
    // do nothing
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
    processRecord(bibliographicRecord, recordNumber);
  }

  @Override
  public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
    if (parameters.getRecordIgnorator().isIgnorable(bibliographicRecord))
      return;

    NetworkAnalyzer analyzer = new NetworkAnalyzer(bibliographicRecord);
    Set<DataField> collector = analyzer.process();
    orphans.add(bibliographicRecord.getId(true));

    if (collector.isEmpty()) {
      return;
    }

    for (DataField field : collector) {
      networkWriter.write(createRow(
        field.toString().hashCode(),
        field.getDefinition().getTag(),
        recordNumber
      ));
    }
  }

  @Override
  public void beforeIteration() {
    var path = Paths.get(parameters.getOutputDir(), "network.csv");
    logger.info(() -> parameters.formatParameters());
    try {
      networkWriter = Files.newBufferedWriter(path);
      networkWriter.write(createRow("concept", "tag", "id"));
    } catch (IOException e) {
      logger.log(Level.WARNING, "document", e);
    }
  }

  @Override
  public void fileOpened(Path path) {
    // Do nothing
  }

  @Override
  public void fileProcessed() {
    // Do nothing
  }

  @Override
  public void afterIteration(int numberOfprocessedRecords, long duration) {
    try {
      networkWriter.close();
    } catch (IOException e) {
      logger.severe("Failed to close networkWriter. " + e.getLocalizedMessage());
      logger.log(Level.SEVERE, "afterIteration", e);
    }
    saveParameters("network.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));
  }

  @Override
  public void printHelp(Options options) {
    // Do nothing
  }

  @Override
  public boolean readyToProcess() {
    return readyToProcess;
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import de.gwdg.metadataqa.marc.analysis.NetworkAnalyzer;
4	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
5	import de.gwdg.metadataqa.marc.cli.parameters.NetworkAction;
6	import de.gwdg.metadataqa.marc.cli.parameters.NetworkParameters;
7	import de.gwdg.metadataqa.marc.cli.processor.BibliographicInputProcessor;
8	import de.gwdg.metadataqa.marc.cli.utils.PairGenerator;
9	import de.gwdg.metadataqa.marc.cli.utils.RecordIterator;
10	import de.gwdg.metadataqa.marc.dao.DataField;
11	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
12	import de.gwdg.metadataqa.marc.model.validation.ValidationError;
13	import org.apache.commons.cli.Options;
14	import org.apache.commons.cli.ParseException;
15	import org.marc4j.marc.Record;
16
17	import java.io.BufferedWriter;
18	import java.io.IOException;
19	import java.io.Serializable;
20	import java.nio.file.Files;
21	import java.nio.file.Path;
22	import java.nio.file.Paths;
23	import java.util.ArrayList;
24	import java.util.List;
25	import java.util.Map;
26	import java.util.Set;
27	import java.util.logging.Level;
28	import java.util.logging.Logger;
29
30	import static de.gwdg.metadataqa.marc.Utils.createRow;
31
32	public class NetworkAnalysis extends QACli<NetworkParameters> implements BibliographicInputProcessor, Serializable {
33
34	private static final Logger logger = Logger.getLogger(NetworkAnalysis.class.getCanonicalName());	×
35
36	private final boolean readyToProcess;
37	private final List<String> orphans = new ArrayList<>();	×
38	private BufferedWriter networkWriter;
39
40	public NetworkAnalysis(String[] args) throws ParseException {	×
41	parameters = new NetworkParameters(args);	×
42	readyToProcess = true;	×
43	}	×
44
45	public static void main(String[] args) {
46	BibliographicInputProcessor processor = null;	×
47	try {
48	processor = new NetworkAnalysis(args);	×
49	} catch (ParseException e) {	×
NEW 50	logger.severe(createRow("ERROR. ", e.getLocalizedMessage()));	×
51	System.exit(1);	×
52	}	×
53	NetworkParameters params = (NetworkParameters)processor.getParameters();	×
54	NetworkAction action = params.getAction();	×
55	logger.info("Action: " + ((NetworkParameters)processor.getParameters()).getAction());	×
56	if (action.equals(NetworkAction.PAIRING)) {	×
57	PairGenerator generator = new PairGenerator(params.getOutputDir(), params.getGroupLimit(),false);	×
58	generator.generatePairs();	×
59	} else {	×
60	if (params.getArgs().length < 1) {	×
NEW 61	logger.severe("Please provide a MARC file name!");	×
62	processor.printHelp(params.getOptions());	×
63	System.exit(0);	×
64	}
65	if (params.doHelp()) {	×
66	processor.printHelp(params.getOptions());	×
67	System.exit(0);	×
68	}
69	RecordIterator iterator = new RecordIterator(processor);	×
NEW 70	iterator.setProcessWithErrors(processor.getParameters().getProcessRecordsWithoutId());	×
71	iterator.start();	×
72	}
UNCOV 73	}	×
74
75	@Override
76	public CommonParameters getParameters() {
UNCOV 77	return parameters;	×
78	}
79
80	@Override
81	public void processRecord(Record marc4jRecord, int recordNumber) throws IOException {
82	// do nothing
UNCOV 83	}	×
84
85	@Override
86	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber, List<ValidationError> errors) throws IOException {
NEW 87	processRecord(bibliographicRecord, recordNumber);	×
88	}	×
89
90	@Override
91	public void processRecord(BibliographicRecord bibliographicRecord, int recordNumber) throws IOException {
NEW 92	if (parameters.getRecordIgnorator().isIgnorable(bibliographicRecord))	×
93	return;	×
94
NEW 95	NetworkAnalyzer analyzer = new NetworkAnalyzer(bibliographicRecord);	×
NEW 96	Set<DataField> collector = analyzer.process();	×
NEW 97	orphans.add(bibliographicRecord.getId(true));	×
98
NEW 99	if (collector.isEmpty()) {	×
NEW 100	return;	×
101	}
102
NEW 103	for (DataField field : collector) {	×
NEW 104	networkWriter.write(createRow(	×
NEW 105	field.toString().hashCode(),	×
NEW 106	field.getDefinition().getTag(),	×
NEW 107	recordNumber	×
108	));
109	}	×
110	}	×
111
112	@Override
113	public void beforeIteration() {
114	var path = Paths.get(parameters.getOutputDir(), "network.csv");	×
NEW 115	logger.info(() -> parameters.formatParameters());	×
116	try {
UNCOV 117	networkWriter = Files.newBufferedWriter(path);	×
118	networkWriter.write(createRow("concept", "tag", "id"));	×
119	} catch (IOException e) {	×
UNCOV 120	logger.log(Level.WARNING, "document", e);	×
121	}	×
122	}	×
123
124	@Override
125	public void fileOpened(Path path) {
126	// Do nothing
UNCOV 127	}	×
128
129	@Override
130	public void fileProcessed() {
131	// Do nothing
UNCOV 132	}	×
133
134	@Override
135	public void afterIteration(int numberOfprocessedRecords, long duration) {
136	try {
UNCOV 137	networkWriter.close();	×
UNCOV 138	} catch (IOException e) {	×
UNCOV 139	logger.severe("Failed to close networkWriter. " + e.getLocalizedMessage());	×
UNCOV 140	logger.log(Level.SEVERE, "afterIteration", e);	×
141	}	×
142	saveParameters("network.params.json", parameters, Map.of("numberOfprocessedRecords", numberOfprocessedRecords, "duration", duration));	×
143	}	×
144
145	@Override
146	public void printHelp(Options options) {
147	// Do nothing
UNCOV 148	}	×
149
150	@Override
151	public boolean readyToProcess() {
152	return readyToProcess;	×
153	}
154	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous