#1527

Committed 22 Aug 2025 02:21PM UTC coverage: 90.345%. Remained the same

Build # #1527

Build Type

push

Committed by

Commit Message

Improve timeline handling

Run Details

5191 of 6416 new or added lines in 219 files covered. (80.91%)

886 existing lines in 78 files now uncovered.

36717 of 40641 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.05

/src/main/java/de/gwdg/metadataqa/marc/cli/QACli.java

package de.gwdg.metadataqa.marc.cli;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.gwdg.metadataqa.marc.CsvUtils;
import de.gwdg.metadataqa.marc.Utils;
import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
import de.gwdg.metadataqa.marc.utils.BibiographicPath;
import de.gwdg.metadataqa.marc.utils.pica.path.PicaPath;
import de.gwdg.metadataqa.marc.utils.pica.path.PicaPathParser;
import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

public abstract class QACli<T extends CommonParameters> {
  private static final Logger logger = Logger.getLogger(QACli.class.getCanonicalName());
  protected T parameters;

  public static final String ALL = "0";
  protected BibiographicPath groupBy = null;
  protected File idCollectorFile;
  private FileTime jarModifiedTime;
  private boolean isJarModifiedTimeDetected = false;
  protected boolean doSaveGroupIds = true;

  protected void initializeGroups(String groupBy, boolean isPica) {
    if (groupBy != null) {
      this.groupBy = isPica
        ? PicaPathParser.parse(groupBy)
        : null; // TODO: create it for MARC21
    }
  }

  protected void saveParameters(String fileName, T parameters) {
    saveParameters(fileName, parameters, null);
  }

  protected void saveParameters(String fileName, T parameters, Map<String, Object> results) {
    // Map<String, Object> responseDao = Map.of("parameters", parameters, "results", results);
    ObjectMapper mapper = new ObjectMapper();
    try {
      String json = mapper.writeValueAsString(parameters);
      logger.info("json: " + json);
      Map<String, Object> configuration = mapper.readValue(json, new TypeReference<>(){});
      configuration.put("mqaf.version", de.gwdg.metadataqa.api.cli.Version.getVersion());
      configuration.put("qa-catalogue.version", de.gwdg.metadataqa.marc.cli.Version.getVersion());
      if (results != null)
        for (Map.Entry<String, Object> entry : results.entrySet()) {
          Object value = entry.getValue();
          if (entry.getKey().equals("duration")) {
            value = Utils.formatDuration((long) value);
          }
          configuration.put(entry.getKey(), value);
        }

      File configFile = Paths.get(parameters.getOutputDir(), fileName).toFile();
      logger.log(Level.INFO, "Saving configuration to {0}.", configFile.getAbsolutePath());
      mapper.writeValue(configFile, configuration);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  protected Set<String> getGroupIds(CommonParameters parameters, BibliographicRecord bibliographicRecord) {
    if (this.groupBy != null && parameters.isPica()) {
      // TODO: MARC21
      PicaSpec picaSpec = new PicaSpec((PicaPath) groupBy);
      List<String> idLists = bibliographicRecord.select(picaSpec);
      return QACli.extractGroupIds(idLists);
    }
    return new HashSet<>();
  }

  protected void saveGroupIds(String recordId, Set<String> groupIds) {
    if (doGroups() && !groupIds.isEmpty())
      for (String groupId : groupIds)
        printToFile(idCollectorFile, CsvUtils.createCsv(recordId, groupId));
  }

  public static Set<String> extractGroupIds(List<String> idLists) {
    Set<String> groupIds = new HashSet<>();
    groupIds.add(ALL);
    if (idLists != null) {
      for (String idList : idLists) {
        String[] ids = idList.split(",");
        groupIds.addAll(Arrays.asList(ids));
      }
    }
    return groupIds;
  }

  public boolean doGroups() {
    return groupBy != null;
  }

  protected boolean isJarNewerThan(String outputDir, String fileName) {
    try {
      initializeJarModifiedTime();
      File reportFile = new File(outputDir, fileName);
      if (!reportFile.exists())
        return true;
      else {
        FileTime groupModifiedTime = Files.readAttributes(reportFile.toPath(), BasicFileAttributes.class).lastModifiedTime();
        return (jarModifiedTime == null || jarModifiedTime.compareTo(groupModifiedTime) > 0);
      }
    } catch (IOException e) {
      logger.severe("Error during prepareReportFile: " + e);
    }
    return false;
  }

  protected void initializeMeta(CommonParameters parameters) {
    File idCollectorMeta = new File(parameters.getOutputDir(), "id-groupid.meta.txt");
    String currentFileList = getFilesWithDate(parameters.getArgs());
    if (isJarNewerThan(parameters.getOutputDir(), "id-groupid.csv")) {
      try {
        Files.delete(idCollectorMeta.toPath());
      } catch (IOException e) {
        logger.severe("Deletion of " + idCollectorMeta.getAbsolutePath() + " was unsuccessful!");
      }
    } else {
      if (idCollectorMeta.exists()) {
        try {
          String storedFileList = FileUtils.readFileToString(idCollectorMeta, StandardCharsets.UTF_8).trim();
          doSaveGroupIds = ! currentFileList.equals(storedFileList);
          try {
            Files.delete(idCollectorMeta.toPath());
          } catch (IOException e) {
            logger.log(Level.SEVERE, "The output file ({}) has not been deleted", idCollectorMeta.getAbsolutePath());
          }
        } catch (IOException e) {
          logger.severe(e.getLocalizedMessage());
        }
      }
    }
    printToFile(idCollectorMeta, currentFileList);
  }

  private String getFilesWithDate(String[] fileNames) {
    List<String> filesWithDate = new ArrayList<>();
    for (String fileName : fileNames) {
      try {
        FileTime modifiedTime = Files.readAttributes(new File(fileName).toPath(), BasicFileAttributes.class).lastModifiedTime();
        filesWithDate.add(fileName + ":" + modifiedTime.toString());
      } catch (IOException e) {
        logger.warning(e.getLocalizedMessage());
      }
    }
    return StringUtils.join(filesWithDate, ",");
  }

  protected File prepareReportFile(String outputDir, String fileName) {
    File reportFile = new File(outputDir, fileName);
    if (reportFile.exists()) {
      try {
        Files.delete(reportFile.toPath());
      } catch (IOException e) {
        logger.log(Level.SEVERE, "The output file ({}) has not been deleted", reportFile.getAbsolutePath());
      }
    }
    return reportFile;
  }

  /**
   * Print to file
   * @param file The output file
   * @param content The content
   */
  protected void printToFile(File file, String content) {
    try {
      FileUtils.writeStringToFile(file, content, StandardCharsets.UTF_8, true);
    } catch (IOException e) {
      if (parameters.doLog())
        logger.log(Level.SEVERE, "printToFile", e);
    }
  }

  private void initializeJarModifiedTime() {
    if (!isJarModifiedTimeDetected) {
      try {
        File currentJar = new File(this.getClass().getProtectionDomain().getCodeSource().getLocation().toURI());
        if (currentJar.isFile()) {
          jarModifiedTime = Files.readAttributes(currentJar.toPath(), BasicFileAttributes.class).lastModifiedTime();
        }
      } catch (URISyntaxException | IOException e) {
        throw new RuntimeException(e);
      }
      isJarModifiedTimeDetected = true;
    }
  }

  protected void copyFileToOutputDir(String fileName) {
    File source = new File(fileName);
    try {
      FileUtils.copyFileToDirectory(source, new File(parameters.getOutputDir()));
    } catch (IOException e) {
      logger.warning(e.getLocalizedMessage());
    }
  }
}

1	package de.gwdg.metadataqa.marc.cli;
2
3	import com.fasterxml.jackson.core.type.TypeReference;
4	import com.fasterxml.jackson.databind.ObjectMapper;
5	import de.gwdg.metadataqa.marc.CsvUtils;
6	import de.gwdg.metadataqa.marc.Utils;
7	import de.gwdg.metadataqa.marc.cli.parameters.CommonParameters;
8	import de.gwdg.metadataqa.marc.dao.record.BibliographicRecord;
9	import de.gwdg.metadataqa.marc.utils.BibiographicPath;
10	import de.gwdg.metadataqa.marc.utils.pica.path.PicaPath;
11	import de.gwdg.metadataqa.marc.utils.pica.path.PicaPathParser;
12	import de.gwdg.metadataqa.marc.utils.pica.path.PicaSpec;
13	import org.apache.commons.io.FileUtils;
14	import org.apache.commons.lang3.StringUtils;
15
16	import java.io.File;
17	import java.io.IOException;
18	import java.net.URISyntaxException;
19	import java.nio.charset.StandardCharsets;
20	import java.nio.file.Files;
21	import java.nio.file.Paths;
22	import java.nio.file.attribute.BasicFileAttributes;
23	import java.nio.file.attribute.FileTime;
24	import java.util.ArrayList;
25	import java.util.Arrays;
26	import java.util.HashSet;
27	import java.util.List;
28	import java.util.Map;
29	import java.util.Set;
30	import java.util.logging.Level;
31	import java.util.logging.Logger;
32
33	public abstract class QACli<T extends CommonParameters> {	1✔
34	private static final Logger logger = Logger.getLogger(QACli.class.getCanonicalName());	1✔
35	protected T parameters;
36
37	public static final String ALL = "0";
38	protected BibiographicPath groupBy = null;	1✔
39	protected File idCollectorFile;
40	private FileTime jarModifiedTime;
41	private boolean isJarModifiedTimeDetected = false;	1✔
42	protected boolean doSaveGroupIds = true;	1✔
43
44	protected void initializeGroups(String groupBy, boolean isPica) {
45	if (groupBy != null) {	1✔
46	this.groupBy = isPica	1✔
47	? PicaPathParser.parse(groupBy)	1✔
48	: null; // TODO: create it for MARC21	1✔
49	}
50	}	1✔
51
52	protected void saveParameters(String fileName, T parameters) {
UNCOV 53	saveParameters(fileName, parameters, null);	×
UNCOV 54	}	×
55
56	protected void saveParameters(String fileName, T parameters, Map<String, Object> results) {
57	// Map<String, Object> responseDao = Map.of("parameters", parameters, "results", results);
58	ObjectMapper mapper = new ObjectMapper();	1✔
59	try {
60	String json = mapper.writeValueAsString(parameters);	1✔
61	logger.info("json: " + json);	1✔
62	Map<String, Object> configuration = mapper.readValue(json, new TypeReference<>(){});	1✔
63	configuration.put("mqaf.version", de.gwdg.metadataqa.api.cli.Version.getVersion());	1✔
64	configuration.put("qa-catalogue.version", de.gwdg.metadataqa.marc.cli.Version.getVersion());	1✔
65	if (results != null)	1✔
66	for (Map.Entry<String, Object> entry : results.entrySet()) {	1✔
67	Object value = entry.getValue();	1✔
68	if (entry.getKey().equals("duration")) {	1✔
69	value = Utils.formatDuration((long) value);	1✔
70	}
71	configuration.put(entry.getKey(), value);	1✔
72	}	1✔
73
74	File configFile = Paths.get(parameters.getOutputDir(), fileName).toFile();	1✔
75	logger.log(Level.INFO, "Saving configuration to {0}.", configFile.getAbsolutePath());	1✔
76	mapper.writeValue(configFile, configuration);	1✔
UNCOV 77	} catch (IOException e) {	×
UNCOV 78	throw new RuntimeException(e);	×
79	}	1✔
80	}	1✔
81
82	protected Set<String> getGroupIds(CommonParameters parameters, BibliographicRecord bibliographicRecord) {
83	if (this.groupBy != null && parameters.isPica()) {	1✔
84	// TODO: MARC21
85	PicaSpec picaSpec = new PicaSpec((PicaPath) groupBy);	1✔
86	List<String> idLists = bibliographicRecord.select(picaSpec);	1✔
87	return QACli.extractGroupIds(idLists);	1✔
88	}
89	return new HashSet<>();	1✔
90	}
91
92	protected void saveGroupIds(String recordId, Set<String> groupIds) {
93	if (doGroups() && !groupIds.isEmpty())	1✔
94	for (String groupId : groupIds)	1✔
95	printToFile(idCollectorFile, CsvUtils.createCsv(recordId, groupId));	1✔
96	}	1✔
97
98	public static Set<String> extractGroupIds(List<String> idLists) {
99	Set<String> groupIds = new HashSet<>();	1✔
100	groupIds.add(ALL);	1✔
101	if (idLists != null) {	1✔
102	for (String idList : idLists) {	1✔
103	String[] ids = idList.split(",");	1✔
104	groupIds.addAll(Arrays.asList(ids));	1✔
105	}	1✔
106	}
107	return groupIds;	1✔
108	}
109
110	public boolean doGroups() {
111	return groupBy != null;	1✔
112	}
113
114	protected boolean isJarNewerThan(String outputDir, String fileName) {
115	try {
116	initializeJarModifiedTime();	1✔
117	File reportFile = new File(outputDir, fileName);	1✔
118	if (!reportFile.exists())	1✔
119	return true;	1✔
120	else {
UNCOV 121	FileTime groupModifiedTime = Files.readAttributes(reportFile.toPath(), BasicFileAttributes.class).lastModifiedTime();	×
122	return (jarModifiedTime == null \|\| jarModifiedTime.compareTo(groupModifiedTime) > 0);	×
123	}
UNCOV 124	} catch (IOException e) {	×
125	logger.severe("Error during prepareReportFile: " + e);	×
126	}
UNCOV 127	return false;	×
128	}
129
130	protected void initializeMeta(CommonParameters parameters) {
131	File idCollectorMeta = new File(parameters.getOutputDir(), "id-groupid.meta.txt");	1✔
132	String currentFileList = getFilesWithDate(parameters.getArgs());	1✔
133	if (isJarNewerThan(parameters.getOutputDir(), "id-groupid.csv")) {	1✔
134	try {
135	Files.delete(idCollectorMeta.toPath());	1✔
136	} catch (IOException e) {	1✔
137	logger.severe("Deletion of " + idCollectorMeta.getAbsolutePath() + " was unsuccessful!");	1✔
138	}	1✔
139	} else {
UNCOV 140	if (idCollectorMeta.exists()) {	×
141	try {
UNCOV 142	String storedFileList = FileUtils.readFileToString(idCollectorMeta, StandardCharsets.UTF_8).trim();	×
143	doSaveGroupIds = ! currentFileList.equals(storedFileList);	×
144	try {
NEW 145	Files.delete(idCollectorMeta.toPath());	×
NEW 146	} catch (IOException e) {	×
NEW 147	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", idCollectorMeta.getAbsolutePath());	×
NEW 148	}	×
149	} catch (IOException e) {	×
150	logger.severe(e.getLocalizedMessage());	×
151	}	×
152	}
153	}
154	printToFile(idCollectorMeta, currentFileList);	1✔
155	}	1✔
156
157	private String getFilesWithDate(String[] fileNames) {
158	List<String> filesWithDate = new ArrayList<>();	1✔
159	for (String fileName : fileNames) {	1✔
160	try {
161	FileTime modifiedTime = Files.readAttributes(new File(fileName).toPath(), BasicFileAttributes.class).lastModifiedTime();	1✔
162	filesWithDate.add(fileName + ":" + modifiedTime.toString());	1✔
UNCOV 163	} catch (IOException e) {	×
UNCOV 164	logger.warning(e.getLocalizedMessage());	×
165	}	1✔
166	}
167	return StringUtils.join(filesWithDate, ",");	1✔
168	}
169
170	protected File prepareReportFile(String outputDir, String fileName) {
171	File reportFile = new File(outputDir, fileName);	1✔
172	if (reportFile.exists()) {	1✔
173	try {
NEW 174	Files.delete(reportFile.toPath());	×
NEW 175	} catch (IOException e) {	×
NEW 176	logger.log(Level.SEVERE, "The output file ({}) has not been deleted", reportFile.getAbsolutePath());	×
NEW 177	}	×
178	}
179	return reportFile;	1✔
180	}
181
182	/**
183	* Print to file
184	* @param file The output file
185	* @param content The content
186	*/
187	protected void printToFile(File file, String content) {
188	try {
189	FileUtils.writeStringToFile(file, content, StandardCharsets.UTF_8, true);	1✔
UNCOV 190	} catch (IOException e) {	×
UNCOV 191	if (parameters.doLog())	×
UNCOV 192	logger.log(Level.SEVERE, "printToFile", e);	×
193	}	1✔
194	}	1✔
195
196	private void initializeJarModifiedTime() {
197	if (!isJarModifiedTimeDetected) {	1✔
198	try {
199	File currentJar = new File(this.getClass().getProtectionDomain().getCodeSource().getLocation().toURI());	1✔
200	if (currentJar.isFile()) {	1✔
201	jarModifiedTime = Files.readAttributes(currentJar.toPath(), BasicFileAttributes.class).lastModifiedTime();	×
202	}
UNCOV 203	} catch (URISyntaxException \| IOException e) {	×
UNCOV 204	throw new RuntimeException(e);	×
205	}	1✔
206	isJarModifiedTimeDetected = true;	1✔
207	}
208	}	1✔
209
210	protected void copyFileToOutputDir(String fileName) {
211	File source = new File(fileName);	1✔
212	try {
213	FileUtils.copyFileToDirectory(source, new File(parameters.getOutputDir()));	1✔
NEW 214	} catch (IOException e) {	×
NEW 215	logger.warning(e.getLocalizedMessage());	×
216	}	1✔
217	}	1✔
218	}

pkiraly / metadata-qa-marc / #1527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous