#1450

Committed 11 Apr 2025 05:13PM UTC coverage: 90.49%. Remained the same

Build # #1450

Build Type

push

Committed by

Commit Message

Normalize years #608

Coverage Stats

45 of 46 new or added lines in 1 file covered. (97.83%)

36529 of 40368 relevant lines covered (90.49%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.33

/src/main/java/de/gwdg/metadataqa/marc/cli/utils/translation/PublicationYearNormaliser.java

package de.gwdg.metadataqa.marc.cli.utils.translation;

import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class PublicationYearNormaliser {
  private static final Logger logger = Logger.getLogger(PublicationYearNormaliser.class.getCanonicalName());

  public static final String UNRESOLVED_YEARS_FILE = "translations-unresolved-years.txt";


  private static final Pattern yearPattern = Pattern.compile("^\\d{4}$");
  private final String outputDir;
  private Map<String, Integer> unresolvedYears;

  public PublicationYearNormaliser(String outputDir) {
    this.outputDir = outputDir;
    this.unresolvedYears = new HashMap<>();
  }

  public List<String> processYear(List<String> extracted) {
    List<String> normalizedYears = new ArrayList<>();
    for (String original : extracted) {
      if (!yearPattern.matcher(original).matches()) {
        String normalized = normalizeYear(original);
        if (Pattern.compile("^(\\d{4})-(\\d{4})$").matcher(normalized).matches()) {
          normalizedYears.addAll(Arrays.asList(normalized.split("-")));
        } else if (Pattern.compile("^(\\d{4}), ?(\\d{4})$").matcher(normalized).matches()) {
          normalizedYears.addAll(Arrays.asList(normalized.split(", ?")));
        } else if (normalized.equals("s.d.") || normalized.equals("n.d.") || normalized.equals("s.a.")) {
          //
        } else if (yearPattern.matcher(normalized).matches()) {
          normalizedYears.add(normalized);
        } else {
          String key = String.format("'%s' -> '%s'", original, normalized);
          unresolvedYears.put(key, unresolvedYears.getOrDefault(key, 0) + 1);
          // System.err.println("Unhandled case: " + original + " -> " + s);
        }
      } else {
        normalizedYears.add(original);
      }
    }
    return normalizedYears;
  }

  public static String normalizeYear(String year) {
    year = year.trim();

    year = year.replaceAll("^(?:sierpień|kwiecień|grudzień|febrer|février|maj|junio|July|septiembre|ottobre|octobre|Oktober|octubre de) (\\d{4})\\.?$", "$1");
    year = year.replaceAll("^dr\\.? ?(\\d{4})\\.?$", "$1"); // druk
    year = year.replaceAll("^(\\d{4}) :$", "$1");
    year = year.replaceAll("^(\\d{4})\\]\\.$", "$1");
    year = year.replaceAll("^\\d{4} \\[(\\d{4})\\]\\.?$", "$1");
    year = year.replaceAll("(\\d)\\.$", "$1");
    year = year.replaceAll("^c(\\d)", "$1");
    year = year.replaceAll("c(\\d{4})", "$1");
    year = year.replaceAll("\\d{4} \\[(\\d{4})\\]$", "$1");
    year = year.replaceAll("^\\[c?(\\d{4})\\]$", "$1");
    year = year.replaceAll("^\\[(.*?)\\]\\.?$", "$1");
    year = year.replaceAll("\\[(.*?)\\]", "$1");
    year = year.replaceAll("\\((.*?)\\)", "$1");
    year = year.replaceAll("\\[", "");
    year = year.replaceAll("\\]", "");
    year = year.replaceAll("^(ca\\.) ?", "");
    year = year.replaceAll("^(copyright|cop\\.|©|czerwiec|listopad|janvier|lipiec|wrzesień|październik|marzec|luty|januari|mars|maart|juin|juli|styczeń|grudzień|druk|styczeń) ", "");
    year = year.replaceAll("(\\d{4})\\?", "$1");
    year = year.replaceAll("^(\\d{3})\\?", "$10");
    year = year.replaceAll("-\\?$", "0");
    year = year.replaceAll("^(\\d{3})-$", "$10");
    year = year.replaceAll("^(\\d{4})-$", "$1");
    year = year.replaceAll("^(\\d{4})-(\\d{2})$", "$1");
    year = year.replaceAll("^.* i\\.e\\. (\\d{4})$", "$1");
    year = year.replaceAll("^(?:Shōwa|Shōwa|Minguo|Taishō) \\d+ (\\d{4})\\.?$", "$1");
    year = year.replaceAll("^(\\d{4}), cop\\. \\d{4}$", "$1");
    year = year.replaceAll("^\\d{4} !(\\d{4})$", "$1");
    year = year.replaceAll("^cop\\. ?(\\d{4})$", "$1");
    year = year.replaceAll("^ca ?(\\d{4})$", "$1");
    year = year.replaceAll("^(\\d{4}) (k\\.|körül)$", "$1");

    return year;
  }

  public void reportUnresolvedYears() {
    if (unresolvedYears.isEmpty())
      return;
    String content = unresolvedYears.entrySet().stream()
      .sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue()))
      .map(e -> e.getKey() + ": " + e.getValue())
      .collect(Collectors.joining("\n"));

    try {
      FileUtils.writeStringToFile(new File(outputDir, UNRESOLVED_YEARS_FILE), content, StandardCharsets.UTF_8, false);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
}

1	package de.gwdg.metadataqa.marc.cli.utils.translation;
2
3	import org.apache.commons.io.FileUtils;
4
5	import java.io.File;
6	import java.io.IOException;
7	import java.nio.charset.StandardCharsets;
8	import java.util.ArrayList;
9	import java.util.Arrays;
10	import java.util.HashMap;
11	import java.util.List;
12	import java.util.Map;
13	import java.util.logging.Logger;
14	import java.util.regex.Pattern;
15	import java.util.stream.Collectors;
16
17	public class PublicationYearNormaliser {
18	private static final Logger logger = Logger.getLogger(PublicationYearNormaliser.class.getCanonicalName());	1✔
19
20	public static final String UNRESOLVED_YEARS_FILE = "translations-unresolved-years.txt";
21
22
23	private static final Pattern yearPattern = Pattern.compile("^\\d{4}$");	1✔
24	private final String outputDir;
25	private Map<String, Integer> unresolvedYears;
26
27	public PublicationYearNormaliser(String outputDir) {	1✔
28	this.outputDir = outputDir;	1✔
29	this.unresolvedYears = new HashMap<>();	1✔
30	}	1✔
31
32	public List<String> processYear(List<String> extracted) {
33	List<String> normalizedYears = new ArrayList<>();	1✔
34	for (String original : extracted) {	1✔
35	if (!yearPattern.matcher(original).matches()) {	1✔
36	String normalized = normalizeYear(original);	1✔
37	if (Pattern.compile("^(\\d{4})-(\\d{4})$").matcher(normalized).matches()) {	1✔
38	normalizedYears.addAll(Arrays.asList(normalized.split("-")));	1✔
39	} else if (Pattern.compile("^(\\d{4}), ?(\\d{4})$").matcher(normalized).matches()) {	1✔
40	normalizedYears.addAll(Arrays.asList(normalized.split(", ?")));	1✔
41	} else if (normalized.equals("s.d.") \|\| normalized.equals("n.d.") \|\| normalized.equals("s.a.")) {	1✔
42	//
43	} else if (yearPattern.matcher(normalized).matches()) {	1✔
44	normalizedYears.add(normalized);	1✔
45	} else {
NEW 46	String key = String.format("'%s' -> '%s'", original, normalized);	×
47	unresolvedYears.put(key, unresolvedYears.getOrDefault(key, 0) + 1);	×
48	// System.err.println("Unhandled case: " + original + " -> " + s);
49	}
50	} else {	1✔
51	normalizedYears.add(original);	1✔
52	}
53	}	1✔
54	return normalizedYears;	1✔
55	}
56
57	public static String normalizeYear(String year) {
58	year = year.trim();	1✔
59
60	year = year.replaceAll("^(?:sierpień\|kwiecień\|grudzień\|febrer\|février\|maj\|junio\|July\|septiembre\|ottobre\|octobre\|Oktober\|octubre de) (\\d{4})\\.?$", "$1");	1✔
61	year = year.replaceAll("^dr\\.? ?(\\d{4})\\.?$", "$1"); // druk	1✔
62	year = year.replaceAll("^(\\d{4}) :$", "$1");	1✔
63	year = year.replaceAll("^(\\d{4})\\]\\.$", "$1");	1✔
64	year = year.replaceAll("^\\d{4} \\[(\\d{4})\\]\\.?$", "$1");	1✔
65	year = year.replaceAll("(\\d)\\.$", "$1");	1✔
66	year = year.replaceAll("^c(\\d)", "$1");	1✔
67	year = year.replaceAll("c(\\d{4})", "$1");	1✔
68	year = year.replaceAll("\\d{4} \\[(\\d{4})\\]$", "$1");	1✔
69	year = year.replaceAll("^\\[c?(\\d{4})\\]$", "$1");	1✔
70	year = year.replaceAll("^\\[(.*?)\\]\\.?$", "$1");	1✔
71	year = year.replaceAll("\\[(.*?)\\]", "$1");	1✔
72	year = year.replaceAll("\\((.*?)\\)", "$1");	1✔
73	year = year.replaceAll("\\[", "");	1✔
74	year = year.replaceAll("\\]", "");	1✔
75	year = year.replaceAll("^(ca\\.) ?", "");	1✔
76	year = year.replaceAll("^(copyright\|cop\\.\|©\|czerwiec\|listopad\|janvier\|lipiec\|wrzesień\|październik\|marzec\|luty\|januari\|mars\|maart\|juin\|juli\|styczeń\|grudzień\|druk\|styczeń) ", "");	1✔
77	year = year.replaceAll("(\\d{4})\\?", "$1");	1✔
78	year = year.replaceAll("^(\\d{3})\\?", "$10");	1✔
79	year = year.replaceAll("-\\?$", "0");	1✔
80	year = year.replaceAll("^(\\d{3})-$", "$10");	1✔
81	year = year.replaceAll("^(\\d{4})-$", "$1");	1✔
82	year = year.replaceAll("^(\\d{4})-(\\d{2})$", "$1");	1✔
83	year = year.replaceAll("^.* i\\.e\\. (\\d{4})$", "$1");	1✔
84	year = year.replaceAll("^(?:Shōwa\|Shōwa\|Minguo\|Taishō) \\d+ (\\d{4})\\.?$", "$1");	1✔
85	year = year.replaceAll("^(\\d{4}), cop\\. \\d{4}$", "$1");	1✔
86	year = year.replaceAll("^\\d{4} !(\\d{4})$", "$1");	1✔
87	year = year.replaceAll("^cop\\. ?(\\d{4})$", "$1");	1✔
88	year = year.replaceAll("^ca ?(\\d{4})$", "$1");	1✔
89	year = year.replaceAll("^(\\d{4}) (k\\.\|körül)$", "$1");	1✔
90
91	return year;	1✔
92	}
93
94	public void reportUnresolvedYears() {
95	if (unresolvedYears.isEmpty())	1✔
96	return;	1✔
97	String content = unresolvedYears.entrySet().stream()	×
98	.sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue()))	×
99	.map(e -> e.getKey() + ": " + e.getValue())	×
100	.collect(Collectors.joining("\n"));	×
101
102	try {
103	FileUtils.writeStringToFile(new File(outputDir, UNRESOLVED_YEARS_FILE), content, StandardCharsets.UTF_8, false);	×
104	} catch (IOException e) {	×
105	throw new RuntimeException(e);	×
106	}	×
107	}	×
108	}

pkiraly / metadata-qa-marc / #1450

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous