#1440

Committed 20 Mar 2025 08:50PM UTC coverage: 90.59% (+0.008%) from 90.582%

Build # #1440

Build Type

push

Committed by

Commit Message

Merge branch 'main' of github.com:pkiraly/qa-catalogue

Coverage Stats

36458 of 40245 relevant lines covered (90.59%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.56

/src/main/java/de/gwdg/metadataqa/marc/cli/utils/TranslationModel.java

package de.gwdg.metadataqa.marc.cli.utils;

import de.gwdg.metadataqa.api.model.XmlFieldInstance;
import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
import de.gwdg.metadataqa.marc.EncodedValue;
import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceName;
import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;
import de.gwdg.metadataqa.marc.definition.general.codelist.LanguageCodes;

import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * To validate the result, and decide if it is a translation
 */
public class TranslationModel {
  private static final Logger logger = Logger.getLogger(TranslationModel.class.getCanonicalName());

  private static final Pattern yearPattern = Pattern.compile("^\\d{4}$");

  private static Map<String, String> languageCodes = TranslationModel.toMap(List.of(
    "jap", "Japanese", "fra", "French", "fr", "French", "hu", "Hungarian", "esp", "Spanish",
    "srb", "Serbian"
  ));


  private final BibSelector selector;
  private final Map<String, RuleCheckerOutput> resultMap;
  private final PlaceNameNormaliser placeNameNormaliser;

  private boolean translation;
  private boolean translator;
  private boolean sourceLanguage;
  private boolean targetLanguage;
  private boolean originalTitle;
  private boolean originalPublication;

  /**
   * @param resultMap           The results of the measurement
   * @param selector            The data object that contains selected data elements of the bib record
   * @param placeNameNormaliser
   */
  public TranslationModel(Map<String, RuleCheckerOutput> resultMap,
                          BibSelector selector,
                          PlaceNameNormaliser placeNameNormaliser) {
    this.resultMap = resultMap;
    this.selector = selector;
    this.placeNameNormaliser = placeNameNormaliser;
    evaluate();
  }

  public static List<String> header() {
    return List.of(
      "translator", "sourceLanguage", "targetLanguage",
      "originalTitle", "originalPublication", "translation"
    );
  }

  public List<Integer> values() {
    return List.of(
      translator, sourceLanguage, targetLanguage,
      originalTitle, originalPublication, translation
    ).stream().map(s -> s.compareTo(false)).collect(Collectors.toList());
  }

  public Map<String, Object> extract() {
    Map<String, Object> extracted = new HashMap<>();
    // sourceLanguage
    // System.err.println(extract("041$h"));
    extracted.put("sourceLanguage", extract("041$h"));
    // targetLanguage
    // System.err.println(extract("041$a"));
    extracted.put("targetLanguage", extract("041$a"));
    // Language of a work
    // System.err.println(extract("240$l"));
    // System.err.println(extract("260$a"));
    extracted.put("author", extract("100$a"));
    extracted.put("publicationPlace", extract("260$a"));
    extracted.put("publicationYear", extract("260$c"));
    return extracted;
  }

  private List<? extends Object> extract(String path) {
    List<XmlFieldInstance> instances = selector.get(path);
    if (path.equals("260$a")) {
      instances.addAll(selector.get("264$a"));
    } else if (path.equals("260$c")) {
      instances.addAll(selector.get("264$c"));
    } else if (path.equals("100$a") && instances.size() == 0) {
      instances.addAll(selector.get("245$c"));
    }
    List<String> extracted = new ArrayList<>();
    if (path == "041$h" || path == "041$a") {
      for (XmlFieldInstance instance : instances) {
        String value = instance.getValue();
        if (value != null) {
          if (value.contains(", "))
            extracted.addAll(Arrays.asList(value.split(", "))
              .stream()
              .map(s -> resultLanguageCode(s.trim().toLowerCase()))
              .collect(Collectors.toList()));
          else if (!value.contains(" ")) {
            if (value.length() > 3) {
              for (int i = 0; i < value.length(); i += 3) {
                String abr = value.substring(i, i + 3).toLowerCase();
                extracted.add(resultLanguageCode(abr));
              }
            } else {
              extracted.add(resultLanguageCode(value.toLowerCase()));
            }
          } else {
            logger.warning(path + " - Unhandled language: " + value);
          }
        }
      }
      extracted = extracted.stream()
        .distinct()
        .filter(s -> !s.equals("und"))
        .collect(Collectors.toList());
    } else {
      extracted = instances.stream()
        .map(XmlFieldInstance::getValue)
        .map(s -> Normalizer.normalize(s, Normalizer.Form.NFKC))
        .collect(Collectors.toList());
      if (path.equals("240$l")) {
        extracted = extracted.stream()
          .map(s -> s.replaceAll("\\.$", ""))
          .collect(Collectors.toList());
      } else if (path.equals("260$a") && placeNameNormaliser != null) {
        return processPlaceName(extracted).stream().map(PlaceName::getCity).collect(Collectors.toList());
      } else if (path.equals("260$c")) {
        return processYear(extracted);
      }
    }
    return extracted;
  }

  private static String resultLanguageCode(String abbreviation) {
    EncodedValue code = LanguageCodes.getInstance().getCode(abbreviation);
    if (code != null) {
      return code.getLabel();
    } else if (languageCodes.containsKey(abbreviation)) {
      return languageCodes.get(abbreviation);
    } else {
      return abbreviation;
    }
  }

  private List<? extends Object> processYear(List<String> extracted) {
    List<String> normalized = new ArrayList<>();
    for (String s : extracted) {
      if (!yearPattern.matcher(s).matches()) {
        String original = s;
        s = s.replaceAll("(\\d)\\.$", "$1");
        s = s.replaceAll("^c(\\d)", "$1");
        s = s.replaceAll("c(\\d{4})", "$1");
        s = s.replaceAll("\\d{4} \\[(\\d{4})\\]$", "$1");
        s = s.replaceAll("^\\[c?(\\d{4})\\]$", "$1");
        s = s.replaceAll("^\\[(.*?)\\]\\.?$", "$1");
        s = s.replaceAll("\\[(.*?)\\]", "$1");
        s = s.replaceAll("\\((.*?)\\)", "$1");
        s = s.replaceAll("\\[", "");
        s = s.replaceAll("\\]", "");
        s = s.replaceAll("^(ca\\.) ?", "");
        s = s.replaceAll("^(copyright|cop\\.|©|czerwiec|listopad|janvier|lipiec|wrzesień|październik|marzec|luty|januari|mars|maart|juin|juli|styczeń|grudzień|druk|styczeń) ", "");
        s = s.replaceAll("(\\d{4})\\?", "$1");
        s = s.replaceAll("^(\\d{3})\\?", "$10");
        s = s.replaceAll("-\\?$", "0");
        s = s.replaceAll("^(\\d{3})-$", "$10");
        s = s.replaceAll("^(\\d{4})-$", "$1");
        s = s.replaceAll("^(\\d{4})-(\\d{2})$", "$1");
        s = s.replaceAll("^.* i\\.e\\. (\\d{4})$", "$1");
        s = s.replaceAll("Shōwa \\d+ (\\d{4})$", "$1");
        if (Pattern.compile("^(\\d{4})-(\\d{4})$").matcher(s).matches()) {
          normalized.addAll(Arrays.asList(s.split("-")));
        } else if (Pattern.compile("^(\\d{4}), ?(\\d{4})$").matcher(s).matches()) {
          normalized.addAll(Arrays.asList(s.split(", ?")));
        } else if (s.equals("s.d.") || s.equals("n.d.")) {
          //
        } else if (yearPattern.matcher(s).matches()) {
          normalized.add(s);
        } else {
          System.err.println("Unhandled case: " + original + " -> " + s);
        }
      } else {
        normalized.add(s);
      }
    }
    return normalized;
  }

  private List<PlaceName> processPlaceName(List<String> input) {
    return placeNameNormaliser.normalise(input);
    /*
    if (!knownMultiwordCities.contains(result) && Pattern.matches("^.*[^a-zA-Zøäșóō].*$", result)) {
      System.err.println(String.format("'%s' -> '%s'",
        StringUtils.join(input, ", "), StringUtils.join(extracted, ", ")));
    }
     */
  }

  private void evaluate() {
    if (passed("041ind1"))
      translation = true;

    if (passed("041h")) {
      translation = true;
      sourceLanguage = true;
    }

    if (passed("041a")) {
      targetLanguage = true;
    }

    if (passed("245c")) {
      translation = true;
      translator = true;
    }

    if (passed("7004")) {
      translation = true;
      translator = true;
    }

    if (passed("700e")) {
      translation = true;
      translator = true;
    }

    if (passed("500a")) {
      translation = true;
      translator = true;
    }

    if (passed("240a")) {
      translation = true;
      originalTitle = true;
    }

    // TODO: maybe the value should check against other language code
    if (passed("240l")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765ind2")) {
      translation = true;
    }

    if (passed("765t")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765s")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765d")) {
      translation = true;
      originalPublication = true;
    }
  }

  /**
   * Check if the rule has been passed the test
   * @param path
   * @return
   */
  public boolean passed(String path) {
    return resultMap.containsKey(path) && resultMap.get(path).getStatus().equals(RuleCheckingOutputStatus.PASSED);
  }

  public boolean isTranslation() {
    return translation;
  }

  private static Map<String, String> toMap(List<String> input) {
    Map<String, String> map = new HashMap<>();
    for (int i = 0; i < input.size(); i+=2) {
      map.put(input.get(i), input.get(i+1));
    }
    return map;
  }
}

1	package de.gwdg.metadataqa.marc.cli.utils;
2
3	import de.gwdg.metadataqa.api.model.XmlFieldInstance;
4	import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
5	import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
6	import de.gwdg.metadataqa.marc.EncodedValue;
7	import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceName;
8	import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;
9	import de.gwdg.metadataqa.marc.definition.general.codelist.LanguageCodes;
10
11	import java.text.Normalizer;
12	import java.util.ArrayList;
13	import java.util.Arrays;
14	import java.util.HashMap;
15	import java.util.List;
16	import java.util.Map;
17	import java.util.logging.Logger;
18	import java.util.regex.Pattern;
19	import java.util.stream.Collectors;
20
21	/**
22	* To validate the result, and decide if it is a translation
23	*/
24	public class TranslationModel {
25	private static final Logger logger = Logger.getLogger(TranslationModel.class.getCanonicalName());	1✔
26
27	private static final Pattern yearPattern = Pattern.compile("^\\d{4}$");	1✔
28
29	private static Map<String, String> languageCodes = TranslationModel.toMap(List.of(	1✔
30	"jap", "Japanese", "fra", "French", "fr", "French", "hu", "Hungarian", "esp", "Spanish",
31	"srb", "Serbian"
32	));
33
34
35	private final BibSelector selector;
36	private final Map<String, RuleCheckerOutput> resultMap;
37	private final PlaceNameNormaliser placeNameNormaliser;
38
39	private boolean translation;
40	private boolean translator;
41	private boolean sourceLanguage;
42	private boolean targetLanguage;
43	private boolean originalTitle;
44	private boolean originalPublication;
45
46	/**
47	* @param resultMap The results of the measurement
48	* @param selector The data object that contains selected data elements of the bib record
49	* @param placeNameNormaliser
50	*/
51	public TranslationModel(Map<String, RuleCheckerOutput> resultMap,
52	BibSelector selector,
53	PlaceNameNormaliser placeNameNormaliser) {	1✔
54	this.resultMap = resultMap;	1✔
55	this.selector = selector;	1✔
56	this.placeNameNormaliser = placeNameNormaliser;	1✔
57	evaluate();	1✔
58	}	1✔
59
60	public static List<String> header() {
61	return List.of(	1✔
62	"translator", "sourceLanguage", "targetLanguage",
63	"originalTitle", "originalPublication", "translation"
64	);
65	}
66
67	public List<Integer> values() {
68	return List.of(	1✔
69	translator, sourceLanguage, targetLanguage,	1✔
70	originalTitle, originalPublication, translation	1✔
71	).stream().map(s -> s.compareTo(false)).collect(Collectors.toList());	1✔
72	}
73
74	public Map<String, Object> extract() {
75	Map<String, Object> extracted = new HashMap<>();	1✔
76	// sourceLanguage
77	// System.err.println(extract("041$h"));
78	extracted.put("sourceLanguage", extract("041$h"));	1✔
79	// targetLanguage
80	// System.err.println(extract("041$a"));
81	extracted.put("targetLanguage", extract("041$a"));	1✔
82	// Language of a work
83	// System.err.println(extract("240$l"));
84	// System.err.println(extract("260$a"));
85	extracted.put("author", extract("100$a"));	1✔
86	extracted.put("publicationPlace", extract("260$a"));	1✔
87	extracted.put("publicationYear", extract("260$c"));	1✔
88	return extracted;	1✔
89	}
90
91	private List<? extends Object> extract(String path) {
92	List<XmlFieldInstance> instances = selector.get(path);	1✔
93	if (path.equals("260$a")) {	1✔
94	instances.addAll(selector.get("264$a"));	1✔
95	} else if (path.equals("260$c")) {	1✔
96	instances.addAll(selector.get("264$c"));	1✔
97	} else if (path.equals("100$a") && instances.size() == 0) {	1✔
98	instances.addAll(selector.get("245$c"));	×
99	}
100	List<String> extracted = new ArrayList<>();	1✔
101	if (path == "041$h" \|\| path == "041$a") {	1✔
102	for (XmlFieldInstance instance : instances) {	1✔
103	String value = instance.getValue();	1✔
104	if (value != null) {	1✔
105	if (value.contains(", "))	1✔
106	extracted.addAll(Arrays.asList(value.split(", "))	×
107	.stream()	×
108	.map(s -> resultLanguageCode(s.trim().toLowerCase()))	×
109	.collect(Collectors.toList()));	×
110	else if (!value.contains(" ")) {	1✔
111	if (value.length() > 3) {	1✔
112	for (int i = 0; i < value.length(); i += 3) {	1✔
113	String abr = value.substring(i, i + 3).toLowerCase();	1✔
114	extracted.add(resultLanguageCode(abr));	1✔
115	}
116	} else {
117	extracted.add(resultLanguageCode(value.toLowerCase()));	1✔
118	}
119	} else {
120	logger.warning(path + " - Unhandled language: " + value);	×
121	}
122	}
123	}	1✔
124	extracted = extracted.stream()	1✔
125	.distinct()	1✔
126	.filter(s -> !s.equals("und"))	1✔
127	.collect(Collectors.toList());	1✔
128	} else {
129	extracted = instances.stream()	1✔
130	.map(XmlFieldInstance::getValue)	1✔
131	.map(s -> Normalizer.normalize(s, Normalizer.Form.NFKC))	1✔
132	.collect(Collectors.toList());	1✔
133	if (path.equals("240$l")) {	1✔
134	extracted = extracted.stream()	×
135	.map(s -> s.replaceAll("\\.$", ""))	×
136	.collect(Collectors.toList());	×
137	} else if (path.equals("260$a") && placeNameNormaliser != null) {	1✔
138	return processPlaceName(extracted).stream().map(PlaceName::getCity).collect(Collectors.toList());	1✔
139	} else if (path.equals("260$c")) {	1✔
140	return processYear(extracted);	1✔
141	}
142	}
143	return extracted;	1✔
144	}
145
146	private static String resultLanguageCode(String abbreviation) {
147	EncodedValue code = LanguageCodes.getInstance().getCode(abbreviation);	1✔
148	if (code != null) {	1✔
149	return code.getLabel();	1✔
150	} else if (languageCodes.containsKey(abbreviation)) {	×
151	return languageCodes.get(abbreviation);	×
152	} else {
153	return abbreviation;	×
154	}
155	}
156
157	private List<? extends Object> processYear(List<String> extracted) {
158	List<String> normalized = new ArrayList<>();	1✔
159	for (String s : extracted) {	1✔
160	if (!yearPattern.matcher(s).matches()) {	1✔
161	String original = s;	1✔
162	s = s.replaceAll("(\\d)\\.$", "$1");	1✔
163	s = s.replaceAll("^c(\\d)", "$1");	1✔
164	s = s.replaceAll("c(\\d{4})", "$1");	1✔
165	s = s.replaceAll("\\d{4} \\[(\\d{4})\\]$", "$1");	1✔
166	s = s.replaceAll("^\\[c?(\\d{4})\\]$", "$1");	1✔
167	s = s.replaceAll("^\\[(.*?)\\]\\.?$", "$1");	1✔
168	s = s.replaceAll("\\[(.*?)\\]", "$1");	1✔
169	s = s.replaceAll("\\((.*?)\\)", "$1");	1✔
170	s = s.replaceAll("\\[", "");	1✔
171	s = s.replaceAll("\\]", "");	1✔
172	s = s.replaceAll("^(ca\\.) ?", "");	1✔
173	s = s.replaceAll("^(copyright\|cop\\.\|©\|czerwiec\|listopad\|janvier\|lipiec\|wrzesień\|październik\|marzec\|luty\|januari\|mars\|maart\|juin\|juli\|styczeń\|grudzień\|druk\|styczeń) ", "");	1✔
174	s = s.replaceAll("(\\d{4})\\?", "$1");	1✔
175	s = s.replaceAll("^(\\d{3})\\?", "$10");	1✔
176	s = s.replaceAll("-\\?$", "0");	1✔
177	s = s.replaceAll("^(\\d{3})-$", "$10");	1✔
178	s = s.replaceAll("^(\\d{4})-$", "$1");	1✔
179	s = s.replaceAll("^(\\d{4})-(\\d{2})$", "$1");	1✔
180	s = s.replaceAll("^.* i\\.e\\. (\\d{4})$", "$1");	1✔
181	s = s.replaceAll("Shōwa \\d+ (\\d{4})$", "$1");	1✔
182	if (Pattern.compile("^(\\d{4})-(\\d{4})$").matcher(s).matches()) {	1✔
183	normalized.addAll(Arrays.asList(s.split("-")));	1✔
184	} else if (Pattern.compile("^(\\d{4}), ?(\\d{4})$").matcher(s).matches()) {	1✔
185	normalized.addAll(Arrays.asList(s.split(", ?")));	1✔
186	} else if (s.equals("s.d.") \|\| s.equals("n.d.")) {	1✔
187	//
188	} else if (yearPattern.matcher(s).matches()) {	1✔
189	normalized.add(s);	1✔
190	} else {
191	System.err.println("Unhandled case: " + original + " -> " + s);	1✔
192	}
193	} else {	1✔
194	normalized.add(s);	1✔
195	}
196	}	1✔
197	return normalized;	1✔
198	}
199
200	private List<PlaceName> processPlaceName(List<String> input) {
201	return placeNameNormaliser.normalise(input);	1✔
202	/*
203	if (!knownMultiwordCities.contains(result) && Pattern.matches("^.[^a-zA-Zøäșóō].$", result)) {
204	System.err.println(String.format("'%s' -> '%s'",
205	StringUtils.join(input, ", "), StringUtils.join(extracted, ", ")));
206	}
207	*/
208	}
209
210	private void evaluate() {
211	if (passed("041ind1"))	1✔
212	translation = true;	1✔
213
214	if (passed("041h")) {	1✔
215	translation = true;	1✔
216	sourceLanguage = true;	1✔
217	}
218
219	if (passed("041a")) {	1✔
220	targetLanguage = true;	1✔
221	}
222
223	if (passed("245c")) {	1✔
224	translation = true;	1✔
225	translator = true;	1✔
226	}
227
228	if (passed("7004")) {	1✔
229	translation = true;	×
230	translator = true;	×
231	}
232
233	if (passed("700e")) {	1✔
234	translation = true;	×
235	translator = true;	×
236	}
237
238	if (passed("500a")) {	1✔
239	translation = true;	1✔
240	translator = true;	1✔
241	}
242
243	if (passed("240a")) {	1✔
244	translation = true;	1✔
245	originalTitle = true;	1✔
246	}
247
248	// TODO: maybe the value should check against other language code
249	if (passed("240l")) {	1✔
250	translation = true;	1✔
251	originalTitle = true;	1✔
252	}
253
254	if (passed("765ind2")) {	1✔
255	translation = true;	×
256	}
257
258	if (passed("765t")) {	1✔
259	translation = true;	×
260	originalTitle = true;	×
261	}
262
263	if (passed("765s")) {	1✔
264	translation = true;	×
265	originalTitle = true;	×
266	}
267
268	if (passed("765d")) {	1✔
269	translation = true;	×
270	originalPublication = true;	×
271	}
272	}	1✔
273
274	/**
275	* Check if the rule has been passed the test
276	* @param path
277	* @return
278	*/
279	public boolean passed(String path) {
280	return resultMap.containsKey(path) && resultMap.get(path).getStatus().equals(RuleCheckingOutputStatus.PASSED);	1✔
281	}
282
283	public boolean isTranslation() {
284	return translation;	1✔
285	}
286
287	private static Map<String, String> toMap(List<String> input) {
288	Map<String, String> map = new HashMap<>();	1✔
289	for (int i = 0; i < input.size(); i+=2) {	1✔
290	map.put(input.get(i), input.get(i+1));	1✔
291	}
292	return map;	1✔
293	}
294	}

pkiraly / metadata-qa-marc / #1440

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous