#1433

Committed 18 Mar 2025 09:04PM UTC coverage: 90.582% (-0.02%) from 90.603%

Build # #1433

Build Type

push

Committed by

Commit Message

Normalise place names with external dictionary file #598

Coverage Stats

145 of 169 new or added lines in 5 files covered. (85.8%)

36406 of 40191 relevant lines covered (90.58%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.72

/src/main/java/de/gwdg/metadataqa/marc/cli/utils/TranslationModel.java

package de.gwdg.metadataqa.marc.cli.utils;

import de.gwdg.metadataqa.api.model.XmlFieldInstance;
import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceName;
import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.stream.Collectors;

/**
 * To validate the result, and decide if it is a translation
 */
public class TranslationModel {
  private static final Logger logger = Logger.getLogger(TranslationModel.class.getCanonicalName());

  private final BibSelector selector;
  private final Map<String, RuleCheckerOutput> resultMap;
  private final PlaceNameNormaliser placeNameNormaliser;

  private boolean translation;
  private boolean translator;
  private boolean sourceLanguage;
  private boolean targetLanguage;
  private boolean originalTitle;
  private boolean originalPublication;

  /**
   * @param resultMap           The results of the measurement
   * @param selector            The data object that contains selected data elements of the bib record
   * @param placeNameNormaliser
   */
  public TranslationModel(Map<String, RuleCheckerOutput> resultMap,
                          BibSelector selector,
                          PlaceNameNormaliser placeNameNormaliser) {
    this.resultMap = resultMap;
    this.selector = selector;
    this.placeNameNormaliser = placeNameNormaliser;
    evaluate();
  }

  public static List<String> header() {
    return List.of(
      "translator", "sourceLanguage", "targetLanguage",
      "originalTitle", "originalPublication", "translation"
    );
  }

  public List<Integer> values() {
    return List.of(
      translator, sourceLanguage, targetLanguage,
      originalTitle, originalPublication, translation
    ).stream().map(s -> s.compareTo(false)).collect(Collectors.toList());
  }

  public Map<String, Object> extract() {
    Map<String, Object> extracted = new HashMap<>();
    // sourceLanguage
    // System.err.println(extract("041$h"));
    extracted.put("sourceLanguage", extract("041$h"));
    // targetLanguage
    // System.err.println(extract("041$a"));
    extracted.put("targetLanguage", extract("041$a"));
    // Language of a work
    // System.err.println(extract("240$l"));
    // System.err.println(extract("260$a"));
    extracted.put("author", extract("100$a"));
    extracted.put("publicationPlace", extract("260$a"));
    return extracted;
  }

  private List<? extends Object> extract(String path) {
    List<XmlFieldInstance> instances = selector.get(path);
    if (path.equals("260$a")) {
      instances.addAll(selector.get("264$a"));
    } else if (path.equals("100$a") && instances.size() == 0) {
      instances.addAll(selector.get("245$c"));
    }
    List<String> extracted = new ArrayList<>();
    if (path == "041$h" || path == "041$a") {
      for (XmlFieldInstance instance : instances) {
        String value = instance.getValue();
        if (value != null) {
          if (value.contains(", "))
            extracted.addAll(Arrays.asList(value.split(", "))
              .stream()
              .map(s -> s.trim().toLowerCase())
              .collect(Collectors.toList()));
          else if (!value.contains(" ")) {
            if (value.length() > 3) {
              for (int i = 0; i < value.length(); i += 3) {
                extracted.add(value.substring(i, i + 3).toLowerCase());
              }
            } else {
              extracted.add(value.toLowerCase());
            }
          } else {
            logger.warning(path + " - Unhandled case: " + value);
          }
        }
      }
      extracted = extracted.stream()
        .distinct()
        .filter(s -> !s.equals("und"))
        .collect(Collectors.toList());
    } else {
      extracted = instances.stream()
        .map(XmlFieldInstance::getValue)
        .collect(Collectors.toList());
      if (path.equals("240$l")) {
        extracted = extracted.stream()
          .map(s -> s.replaceAll("\\.$", ""))
          .collect(Collectors.toList());
      } else if (path.equals("260$a") && placeNameNormaliser != null) {
        return processPlaceName(extracted).stream().map(PlaceName::getCity).collect(Collectors.toList());
      }
    }
    return extracted;
  }

  private List<PlaceName> processPlaceName(List<String> input) {
    return placeNameNormaliser.normalise(input);
    /*
    if (!knownMultiwordCities.contains(result) && Pattern.matches("^.*[^a-zA-Zøäșóō].*$", result)) {
      System.err.println(String.format("'%s' -> '%s'",
        StringUtils.join(input, ", "), StringUtils.join(extracted, ", ")));
    }
     */
  }

  private void evaluate() {
    if (passed("041ind1"))
      translation = true;

    if (passed("041h")) {
      translation = true;
      sourceLanguage = true;
    }

    if (passed("041a")) {
      targetLanguage = true;
    }

    if (passed("245c")) {
      translation = true;
      translator = true;
    }

    if (passed("7004")) {
      translation = true;
      translator = true;
    }

    if (passed("700e")) {
      translation = true;
      translator = true;
    }

    if (passed("500a")) {
      translation = true;
      translator = true;
    }

    if (passed("240a")) {
      translation = true;
      originalTitle = true;
    }

    // TODO: maybe the value should check against other language code
    if (passed("240l")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765ind2")) {
      translation = true;
    }

    if (passed("765t")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765s")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765d")) {
      translation = true;
      originalPublication = true;
    }
  }

  /**
   * Check if the rule has been passed the test
   * @param path
   * @return
   */
  public boolean passed(String path) {
    return resultMap.containsKey(path) && resultMap.get(path).getStatus().equals(RuleCheckingOutputStatus.PASSED);
  }

  public boolean isTranslation() {
    return translation;
  }
}

1	package de.gwdg.metadataqa.marc.cli.utils;
2
3	import de.gwdg.metadataqa.api.model.XmlFieldInstance;
4	import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
5	import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
6	import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceName;
7	import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;
8
9	import java.util.ArrayList;
10	import java.util.Arrays;
11	import java.util.HashMap;
12	import java.util.List;
13	import java.util.Map;
14	import java.util.logging.Logger;
15	import java.util.stream.Collectors;
16
17	/**
18	* To validate the result, and decide if it is a translation
19	*/
20	public class TranslationModel {
21	private static final Logger logger = Logger.getLogger(TranslationModel.class.getCanonicalName());	1✔
22
23	private final BibSelector selector;
24	private final Map<String, RuleCheckerOutput> resultMap;
25	private final PlaceNameNormaliser placeNameNormaliser;
26
27	private boolean translation;
28	private boolean translator;
29	private boolean sourceLanguage;
30	private boolean targetLanguage;
31	private boolean originalTitle;
32	private boolean originalPublication;
33
34	/**
35	* @param resultMap The results of the measurement
36	* @param selector The data object that contains selected data elements of the bib record
37	* @param placeNameNormaliser
38	*/
39	public TranslationModel(Map<String, RuleCheckerOutput> resultMap,
40	BibSelector selector,
41	PlaceNameNormaliser placeNameNormaliser) {	1✔
42	this.resultMap = resultMap;	1✔
43	this.selector = selector;	1✔
44	this.placeNameNormaliser = placeNameNormaliser;	1✔
45	evaluate();	1✔
46	}	1✔
47
48	public static List<String> header() {
49	return List.of(	1✔
50	"translator", "sourceLanguage", "targetLanguage",
51	"originalTitle", "originalPublication", "translation"
52	);
53	}
54
55	public List<Integer> values() {
56	return List.of(	1✔
57	translator, sourceLanguage, targetLanguage,	1✔
58	originalTitle, originalPublication, translation	1✔
59	).stream().map(s -> s.compareTo(false)).collect(Collectors.toList());	1✔
60	}
61
62	public Map<String, Object> extract() {
63	Map<String, Object> extracted = new HashMap<>();	1✔
64	// sourceLanguage
65	// System.err.println(extract("041$h"));
66	extracted.put("sourceLanguage", extract("041$h"));	1✔
67	// targetLanguage
68	// System.err.println(extract("041$a"));
69	extracted.put("targetLanguage", extract("041$a"));	1✔
70	// Language of a work
71	// System.err.println(extract("240$l"));
72	// System.err.println(extract("260$a"));
73	extracted.put("author", extract("100$a"));	1✔
74	extracted.put("publicationPlace", extract("260$a"));	1✔
75	return extracted;	1✔
76	}
77
78	private List<? extends Object> extract(String path) {
79	List<XmlFieldInstance> instances = selector.get(path);	1✔
80	if (path.equals("260$a")) {	1✔
81	instances.addAll(selector.get("264$a"));	1✔
82	} else if (path.equals("100$a") && instances.size() == 0) {	1✔
NEW 83	instances.addAll(selector.get("245$c"));	×
84	}
85	List<String> extracted = new ArrayList<>();	1✔
86	if (path == "041$h" \|\| path == "041$a") {	1✔
87	for (XmlFieldInstance instance : instances) {	1✔
88	String value = instance.getValue();	1✔
89	if (value != null) {	1✔
90	if (value.contains(", "))	1✔
NEW 91	extracted.addAll(Arrays.asList(value.split(", "))	×
NEW 92	.stream()	×
NEW 93	.map(s -> s.trim().toLowerCase())	×
NEW 94	.collect(Collectors.toList()));	×
95	else if (!value.contains(" ")) {	1✔
96	if (value.length() > 3) {	1✔
97	for (int i = 0; i < value.length(); i += 3) {	1✔
98	extracted.add(value.substring(i, i + 3).toLowerCase());	1✔
99	}
100	} else {
101	extracted.add(value.toLowerCase());	1✔
102	}
103	} else {
NEW 104	logger.warning(path + " - Unhandled case: " + value);	×
105	}
106	}
107	}	1✔
108	extracted = extracted.stream()	1✔
109	.distinct()	1✔
110	.filter(s -> !s.equals("und"))	1✔
111	.collect(Collectors.toList());	1✔
112	} else {
113	extracted = instances.stream()	1✔
114	.map(XmlFieldInstance::getValue)	1✔
115	.collect(Collectors.toList());	1✔
116	if (path.equals("240$l")) {	1✔
NEW 117	extracted = extracted.stream()	×
NEW 118	.map(s -> s.replaceAll("\\.$", ""))	×
NEW 119	.collect(Collectors.toList());	×
120	} else if (path.equals("260$a") && placeNameNormaliser != null) {	1✔
121	return processPlaceName(extracted).stream().map(PlaceName::getCity).collect(Collectors.toList());	1✔
122	}
123	}
124	return extracted;	1✔
125	}
126
127	private List<PlaceName> processPlaceName(List<String> input) {
128	return placeNameNormaliser.normalise(input);	1✔
129	/*
130	if (!knownMultiwordCities.contains(result) && Pattern.matches("^.[^a-zA-Zøäșóō].$", result)) {
131	System.err.println(String.format("'%s' -> '%s'",
132	StringUtils.join(input, ", "), StringUtils.join(extracted, ", ")));
133	}
134	*/
135	}
136
137	private void evaluate() {
138	if (passed("041ind1"))	1✔
139	translation = true;	1✔
140
141	if (passed("041h")) {	1✔
142	translation = true;	1✔
143	sourceLanguage = true;	1✔
144	}
145
146	if (passed("041a")) {	1✔
147	targetLanguage = true;	1✔
148	}
149
150	if (passed("245c")) {	1✔
151	translation = true;	1✔
152	translator = true;	1✔
153	}
154
155	if (passed("7004")) {	1✔
156	translation = true;	×
157	translator = true;	×
158	}
159
160	if (passed("700e")) {	1✔
NEW 161	translation = true;	×
NEW 162	translator = true;	×
163	}
164
165	if (passed("500a")) {	1✔
166	translation = true;	1✔
167	translator = true;	1✔
168	}
169
170	if (passed("240a")) {	1✔
171	translation = true;	1✔
172	originalTitle = true;	1✔
173	}
174
175	// TODO: maybe the value should check against other language code
176	if (passed("240l")) {	1✔
177	translation = true;	1✔
178	originalTitle = true;	1✔
179	}
180
181	if (passed("765ind2")) {	1✔
182	translation = true;	×
183	}
184
185	if (passed("765t")) {	1✔
186	translation = true;	×
187	originalTitle = true;	×
188	}
189
190	if (passed("765s")) {	1✔
191	translation = true;	×
192	originalTitle = true;	×
193	}
194
195	if (passed("765d")) {	1✔
196	translation = true;	×
197	originalPublication = true;	×
198	}
199	}	1✔
200
201	/**
202	* Check if the rule has been passed the test
203	* @param path
204	* @return
205	*/
206	public boolean passed(String path) {
207	return resultMap.containsKey(path) && resultMap.get(path).getStatus().equals(RuleCheckingOutputStatus.PASSED);	1✔
208	}
209
210	public boolean isTranslation() {
211	return translation;	1✔
212	}
213	}

pkiraly / metadata-qa-marc / #1433

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous