#1447

Committed 11 Apr 2025 03:29PM UTC coverage: 90.49% (-0.1%) from 90.59%

Build # #1447

Build Type

push

Committed by

Commit Message

Inject record ID if missing #607

Coverage Stats

25 of 60 new or added lines in 17 files covered. (41.67%)

70 existing lines in 15 files now uncovered.

36528 of 40367 relevant lines covered (90.49%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.86

/src/main/java/de/gwdg/metadataqa/marc/cli/utils/TranslationModel.java

package de.gwdg.metadataqa.marc.cli.utils;

import de.gwdg.metadataqa.api.model.XmlFieldInstance;
import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
import de.gwdg.metadataqa.marc.EncodedValue;
import de.gwdg.metadataqa.marc.MarcSubfield;
import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceName;
import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;
import de.gwdg.metadataqa.marc.cli.utils.translation.PublicationYearNormaliser;
import de.gwdg.metadataqa.marc.dao.DataField;
import de.gwdg.metadataqa.marc.definition.MarcVersion;
import de.gwdg.metadataqa.marc.definition.general.codelist.LanguageCodes;
import org.apache.commons.lang3.StringUtils;

import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * To validate the result, and decide if it is a translation
 */
public class TranslationModel {
  private static final Logger logger = Logger.getLogger(TranslationModel.class.getCanonicalName());

  private static final Pattern yearPattern = Pattern.compile("^\\d{4}$");

  private static Map<String, String> languageCodes = TranslationModel.toMap(List.of(
    "jap", "Japanese", "fra", "French", "fr", "French", "hu", "Hungarian", "esp", "Spanish",
    "srb", "Serbian"
  ));


  private final BibSelector selector;
  private final Map<String, RuleCheckerOutput> resultMap;
  private final PlaceNameNormaliser placeNameNormaliser;
  private final PublicationYearNormaliser yearNormaliser;
  private final MarcVersion marcVersion;

  private boolean translation;
  private boolean translator;
  private boolean sourceLanguage;
  private boolean targetLanguage;
  private boolean originalTitle;
  private boolean originalPublication;

  /**
   * @param resultMap           The results of the measurement
   * @param selector            The data object that contains selected data elements of the bib record
   * @param placeNameNormaliser
   * @param marcVersion
   */
  public TranslationModel(Map<String, RuleCheckerOutput> resultMap,
                          BibSelector selector,
                          PlaceNameNormaliser placeNameNormaliser,
                          PublicationYearNormaliser yearNormaliser,
                          MarcVersion marcVersion) {
    this.resultMap = resultMap;
    this.selector = selector;
    this.placeNameNormaliser = placeNameNormaliser;
    this.yearNormaliser = yearNormaliser;
    this.marcVersion = marcVersion;
    evaluate();
  }

  public static List<String> header() {
    return List.of(
      "translator", "sourceLanguage", "targetLanguage",
      "originalTitle", "originalPublication", "translation"
    );
  }

  public List<Integer> values() {
    return List.of(
      translator, sourceLanguage, targetLanguage,
      originalTitle, originalPublication, translation
    ).stream().map(s -> s.compareTo(false)).collect(Collectors.toList());
  }

  public Map<String, Object> extract() {
    Map<String, Object> extracted = new HashMap<>();
    // sourceLanguage
    // System.err.println(extract("041$h"));
    extracted.put("sourceLanguage", extract("041$h"));
    // targetLanguage
    // System.err.println(extract("041$a"));
    extracted.put("targetLanguage", extract("041$a"));
    // Language of a work
    // System.err.println(extract("240$l"));
    // System.err.println(extract("260$a"));
    extracted.put("author", extract("100$a"));
    extracted.put("publicationPlace", extract("260$a"));
    extracted.put("publicationYear", extract("260$c"));
    return extracted;
  }

  private List<? extends Object> extract(String path) {
    List<XmlFieldInstance> instances = new ArrayList<>();
    if (path.equals("100$a") && marcVersion == MarcVersion.HUNMARC) {
      instances = getAuthorsFromHunmarc();
    } else {
      instances = selector.get(path);
    }
    if (path.equals("260$a")) {
      instances.addAll(selector.get("264$a"));
    } else if (path.equals("260$c")) {
      instances.addAll(selector.get("264$c"));
    } else if (path.equals("100$a") && instances.size() == 0) {
      instances.addAll(selector.get("245$c"));
    }
    List<String> extracted = new ArrayList<>();
    if (path == "041$h" || path == "041$a") {
      for (XmlFieldInstance instance : instances) {
        String value = instance.getValue();
        if (value != null) {
          value = value.trim();
          if (value.contains(", "))
            extracted.addAll(Arrays.asList(value.split(", "))
              .stream()
              .map(s -> resultLanguageCode(s.trim().toLowerCase()))
              .collect(Collectors.toList()));
          else if (!value.contains(" ")) {
            if (value.length() > 3) {
              for (int i = 0; i < value.length(); i += 3) {
                int end = i + 3;
                String abr = (value.length() >= end)
                  ? value.substring(i, end).toLowerCase()
                  : value.substring(i).toLowerCase();
                extracted.add(resultLanguageCode(abr));
              }
            } else {
              extracted.add(resultLanguageCode(value.toLowerCase()));
            }
          } else {
            logger.warning(String.format("%s - Unhandled language: '%s'", path, value));
          }
        }
      }
      extracted = extracted.stream()
        .distinct()
        .filter(s -> !s.equals("und"))
        .collect(Collectors.toList());
    } else {
      extracted = instances.stream()
        .map(XmlFieldInstance::getValue)
        .map(s -> Normalizer.normalize(s, Normalizer.Form.NFKC))
        .collect(Collectors.toList());
      if (path.equals("240$l")) {
        extracted = extracted.stream()
          .map(s -> s.replaceAll("\\.$", ""))
          .collect(Collectors.toList());
      } else if (path.equals("260$a") && placeNameNormaliser != null) {
        List<PlaceName> placeNames = processPlaceName(extracted);
        if (placeNames.isEmpty()) {
          return new ArrayList<>();
        } else {
          for (PlaceName p : placeNames) {
            if (p == null || p.getCity() == null) {
              logger.warning(String.format("%s - null in place name: '%s'", path, StringUtils.join(extracted, "' -- '")));
            }
          }
          return placeNames.stream()
            .filter(s -> s.getCity() != null)
            .map(PlaceName::getCity)
            .collect(Collectors.toList());
        }
      } else if (path.equals("260$c") && yearNormaliser != null) {
        return yearNormaliser.processYear(extracted);
      }
    }
    return extracted;
  }

  private List<XmlFieldInstance> getAuthorsFromHunmarc() {
    // System.err.println(selector.getClass());
    List<String> names = new ArrayList<>();
    MarcSpecSelector mSelector = (MarcSpecSelector) selector;
    for (Object f : mSelector.extract("100")) {
      StringBuilder name = new StringBuilder();
      DataField field = (DataField) f;
      List<MarcSubfield> a = field.getSubfield("a");
      if (a != null && !a.isEmpty()) {
        if (a.size() == 1) {
          name.append(a.get(0).getValue());
        } else {
          logger.warning("Multiple 100$a: " + a);
        }
      }
      List<MarcSubfield> j = field.getSubfield("j");
      if (j != null && !j.isEmpty()) {
        name.append(" ").append(j.get(0).getValue());
        if (j.size() > 1) {
          logger.warning("Multiple 100$j: " + j);
        }
      }
      if (name.length() > 0) {
        // logger.info(String.format("a: '%s', j: '%s' -> '%s'", a.toString(), j.toString(), name.toString()));
        names.add(name.toString());
      }
    }
    return new ArrayList<>();
  }

  private static String resultLanguageCode(String abbreviation) {
    EncodedValue code = LanguageCodes.getInstance().getCode(abbreviation);
    if (code != null) {
      return code.getLabel();
    } else if (languageCodes.containsKey(abbreviation)) {
      return languageCodes.get(abbreviation);
    } else {
      return abbreviation;
    }
  }

  private List<PlaceName> processPlaceName(List<String> input) {
    return placeNameNormaliser.normalise(input);
    /*
    if (!knownMultiwordCities.contains(result) && Pattern.matches("^.*[^a-zA-Zøäșóō].*$", result)) {
      System.err.println(String.format("'%s' -> '%s'",
        StringUtils.join(input, ", "), StringUtils.join(extracted, ", ")));
    }
     */
  }

  private void evaluate() {
    if (passed("041ind1"))
      translation = true;

    if (passed("041h")) {
      translation = true;
      sourceLanguage = true;
    }

    if (passed("041a")) {
      targetLanguage = true;
    }

    if (passed("245c")) {
      translation = true;
      translator = true;
    }

    if (passed("7004")) {
      translation = true;
      translator = true;
    }

    if (passed("700e")) {
      translation = true;
      translator = true;
    }

    if (passed("500a")) {
      translation = true;
      translator = true;
    }

    if (passed("240a")) {
      translation = true;
      originalTitle = true;
    }

    // TODO: maybe the value should check against other language code
    if (passed("240l")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765ind2")) {
      translation = true;
    }

    if (passed("765t")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765s")) {
      translation = true;
      originalTitle = true;
    }

    if (passed("765d")) {
      translation = true;
      originalPublication = true;
    }
  }

  /**
   * Check if the rule has been passed the test
   * @param path
   * @return
   */
  public boolean passed(String path) {
    return resultMap.containsKey(path) && resultMap.get(path).getStatus().equals(RuleCheckingOutputStatus.PASSED);
  }

  public boolean isTranslation() {
    return translation;
  }

  private static Map<String, String> toMap(List<String> input) {
    Map<String, String> map = new HashMap<>();
    for (int i = 0; i < input.size(); i+=2) {
      map.put(input.get(i), input.get(i+1));
    }
    return map;
  }
}

1	package de.gwdg.metadataqa.marc.cli.utils;
2
3	import de.gwdg.metadataqa.api.model.XmlFieldInstance;
4	import de.gwdg.metadataqa.api.rule.RuleCheckerOutput;
5	import de.gwdg.metadataqa.api.rule.RuleCheckingOutputStatus;
6	import de.gwdg.metadataqa.marc.EncodedValue;
7	import de.gwdg.metadataqa.marc.MarcSubfield;
8	import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceName;
9	import de.gwdg.metadataqa.marc.cli.utils.placename.PlaceNameNormaliser;
10	import de.gwdg.metadataqa.marc.cli.utils.translation.PublicationYearNormaliser;
11	import de.gwdg.metadataqa.marc.dao.DataField;
12	import de.gwdg.metadataqa.marc.definition.MarcVersion;
13	import de.gwdg.metadataqa.marc.definition.general.codelist.LanguageCodes;
14	import org.apache.commons.lang3.StringUtils;
15
16	import java.text.Normalizer;
17	import java.util.ArrayList;
18	import java.util.Arrays;
19	import java.util.HashMap;
20	import java.util.List;
21	import java.util.Map;
22	import java.util.logging.Logger;
23	import java.util.regex.Pattern;
24	import java.util.stream.Collectors;
25
26	/**
27	* To validate the result, and decide if it is a translation
28	*/
29	public class TranslationModel {
30	private static final Logger logger = Logger.getLogger(TranslationModel.class.getCanonicalName());	1✔
31
32	private static final Pattern yearPattern = Pattern.compile("^\\d{4}$");	1✔
33
34	private static Map<String, String> languageCodes = TranslationModel.toMap(List.of(	1✔
35	"jap", "Japanese", "fra", "French", "fr", "French", "hu", "Hungarian", "esp", "Spanish",
36	"srb", "Serbian"
37	));
38
39
40	private final BibSelector selector;
41	private final Map<String, RuleCheckerOutput> resultMap;
42	private final PlaceNameNormaliser placeNameNormaliser;
43	private final PublicationYearNormaliser yearNormaliser;
44	private final MarcVersion marcVersion;
45
46	private boolean translation;
47	private boolean translator;
48	private boolean sourceLanguage;
49	private boolean targetLanguage;
50	private boolean originalTitle;
51	private boolean originalPublication;
52
53	/**
54	* @param resultMap The results of the measurement
55	* @param selector The data object that contains selected data elements of the bib record
56	* @param placeNameNormaliser
57	* @param marcVersion
58	*/
59	public TranslationModel(Map<String, RuleCheckerOutput> resultMap,
60	BibSelector selector,
61	PlaceNameNormaliser placeNameNormaliser,
62	PublicationYearNormaliser yearNormaliser,
63	MarcVersion marcVersion) {	1✔
64	this.resultMap = resultMap;	1✔
65	this.selector = selector;	1✔
66	this.placeNameNormaliser = placeNameNormaliser;	1✔
67	this.yearNormaliser = yearNormaliser;	1✔
68	this.marcVersion = marcVersion;	1✔
69	evaluate();	1✔
70	}	1✔
71
72	public static List<String> header() {
73	return List.of(	1✔
74	"translator", "sourceLanguage", "targetLanguage",
75	"originalTitle", "originalPublication", "translation"
76	);
77	}
78
79	public List<Integer> values() {
80	return List.of(	1✔
81	translator, sourceLanguage, targetLanguage,	1✔
82	originalTitle, originalPublication, translation	1✔
83	).stream().map(s -> s.compareTo(false)).collect(Collectors.toList());	1✔
84	}
85
86	public Map<String, Object> extract() {
87	Map<String, Object> extracted = new HashMap<>();	1✔
88	// sourceLanguage
89	// System.err.println(extract("041$h"));
90	extracted.put("sourceLanguage", extract("041$h"));	1✔
91	// targetLanguage
92	// System.err.println(extract("041$a"));
93	extracted.put("targetLanguage", extract("041$a"));	1✔
94	// Language of a work
95	// System.err.println(extract("240$l"));
96	// System.err.println(extract("260$a"));
97	extracted.put("author", extract("100$a"));	1✔
98	extracted.put("publicationPlace", extract("260$a"));	1✔
99	extracted.put("publicationYear", extract("260$c"));	1✔
100	return extracted;	1✔
101	}
102
103	private List<? extends Object> extract(String path) {
104	List<XmlFieldInstance> instances = new ArrayList<>();	1✔
105	if (path.equals("100$a") && marcVersion == MarcVersion.HUNMARC) {	1✔
106	instances = getAuthorsFromHunmarc();	×
107	} else {
108	instances = selector.get(path);	1✔
109	}
110	if (path.equals("260$a")) {	1✔
111	instances.addAll(selector.get("264$a"));	1✔
112	} else if (path.equals("260$c")) {	1✔
113	instances.addAll(selector.get("264$c"));	1✔
114	} else if (path.equals("100$a") && instances.size() == 0) {	1✔
UNCOV 115	instances.addAll(selector.get("245$c"));	×
116	}
117	List<String> extracted = new ArrayList<>();	1✔
118	if (path == "041$h" \|\| path == "041$a") {	1✔
119	for (XmlFieldInstance instance : instances) {	1✔
120	String value = instance.getValue();	1✔
121	if (value != null) {	1✔
122	value = value.trim();	1✔
123	if (value.contains(", "))	1✔
UNCOV 124	extracted.addAll(Arrays.asList(value.split(", "))	×
UNCOV 125	.stream()	×
UNCOV 126	.map(s -> resultLanguageCode(s.trim().toLowerCase()))	×
UNCOV 127	.collect(Collectors.toList()));	×
128	else if (!value.contains(" ")) {	1✔
129	if (value.length() > 3) {	1✔
130	for (int i = 0; i < value.length(); i += 3) {	1✔
131	int end = i + 3;	1✔
132	String abr = (value.length() >= end)	1✔
133	? value.substring(i, end).toLowerCase()	1✔
134	: value.substring(i).toLowerCase();	1✔
135	extracted.add(resultLanguageCode(abr));	1✔
136	}
137	} else {
138	extracted.add(resultLanguageCode(value.toLowerCase()));	1✔
139	}
140	} else {
UNCOV 141	logger.warning(String.format("%s - Unhandled language: '%s'", path, value));	×
142	}
143	}
144	}	1✔
145	extracted = extracted.stream()	1✔
146	.distinct()	1✔
147	.filter(s -> !s.equals("und"))	1✔
148	.collect(Collectors.toList());	1✔
149	} else {
150	extracted = instances.stream()	1✔
151	.map(XmlFieldInstance::getValue)	1✔
152	.map(s -> Normalizer.normalize(s, Normalizer.Form.NFKC))	1✔
153	.collect(Collectors.toList());	1✔
154	if (path.equals("240$l")) {	1✔
UNCOV 155	extracted = extracted.stream()	×
UNCOV 156	.map(s -> s.replaceAll("\\.$", ""))	×
UNCOV 157	.collect(Collectors.toList());	×
158	} else if (path.equals("260$a") && placeNameNormaliser != null) {	1✔
159	List<PlaceName> placeNames = processPlaceName(extracted);	1✔
160	if (placeNames.isEmpty()) {	1✔
161	return new ArrayList<>();	1✔
162	} else {
163	for (PlaceName p : placeNames) {	1✔
164	if (p == null \|\| p.getCity() == null) {	1✔
UNCOV 165	logger.warning(String.format("%s - null in place name: '%s'", path, StringUtils.join(extracted, "' -- '")));	×
166	}
167	}	1✔
168	return placeNames.stream()	1✔
169	.filter(s -> s.getCity() != null)	1✔
170	.map(PlaceName::getCity)	1✔
171	.collect(Collectors.toList());	1✔
172	}
173	} else if (path.equals("260$c") && yearNormaliser != null) {	1✔
174	return yearNormaliser.processYear(extracted);	1✔
175	}
176	}
177	return extracted;	1✔
178	}
179
180	private List<XmlFieldInstance> getAuthorsFromHunmarc() {
181	// System.err.println(selector.getClass());
UNCOV 182	List<String> names = new ArrayList<>();	×
UNCOV 183	MarcSpecSelector mSelector = (MarcSpecSelector) selector;	×
UNCOV 184	for (Object f : mSelector.extract("100")) {	×
UNCOV 185	StringBuilder name = new StringBuilder();	×
UNCOV 186	DataField field = (DataField) f;	×
UNCOV 187	List<MarcSubfield> a = field.getSubfield("a");	×
UNCOV 188	if (a != null && !a.isEmpty()) {	×
UNCOV 189	if (a.size() == 1) {	×
UNCOV 190	name.append(a.get(0).getValue());	×
191	} else {
UNCOV 192	logger.warning("Multiple 100$a: " + a);	×
193	}
194	}
UNCOV 195	List<MarcSubfield> j = field.getSubfield("j");	×
UNCOV 196	if (j != null && !j.isEmpty()) {	×
UNCOV 197	name.append(" ").append(j.get(0).getValue());	×
UNCOV 198	if (j.size() > 1) {	×
UNCOV 199	logger.warning("Multiple 100$j: " + j);	×
200	}
201	}
UNCOV 202	if (name.length() > 0) {	×
203	// logger.info(String.format("a: '%s', j: '%s' -> '%s'", a.toString(), j.toString(), name.toString()));
UNCOV 204	names.add(name.toString());	×
205	}
UNCOV 206	}	×
UNCOV 207	return new ArrayList<>();	×
208	}
209
210	private static String resultLanguageCode(String abbreviation) {
211	EncodedValue code = LanguageCodes.getInstance().getCode(abbreviation);	1✔
212	if (code != null) {	1✔
213	return code.getLabel();	1✔
UNCOV 214	} else if (languageCodes.containsKey(abbreviation)) {	×
UNCOV 215	return languageCodes.get(abbreviation);	×
216	} else {
UNCOV 217	return abbreviation;	×
218	}
219	}
220
221	private List<PlaceName> processPlaceName(List<String> input) {
222	return placeNameNormaliser.normalise(input);	1✔
223	/*
224	if (!knownMultiwordCities.contains(result) && Pattern.matches("^.[^a-zA-Zøäșóō].$", result)) {
225	System.err.println(String.format("'%s' -> '%s'",
226	StringUtils.join(input, ", "), StringUtils.join(extracted, ", ")));
227	}
228	*/
229	}
230
231	private void evaluate() {
232	if (passed("041ind1"))	1✔
233	translation = true;	1✔
234
235	if (passed("041h")) {	1✔
236	translation = true;	1✔
237	sourceLanguage = true;	1✔
238	}
239
240	if (passed("041a")) {	1✔
241	targetLanguage = true;	1✔
242	}
243
244	if (passed("245c")) {	1✔
245	translation = true;	1✔
246	translator = true;	1✔
247	}
248
249	if (passed("7004")) {	1✔
UNCOV 250	translation = true;	×
UNCOV 251	translator = true;	×
252	}
253
254	if (passed("700e")) {	1✔
255	translation = true;	×
UNCOV 256	translator = true;	×
257	}
258
259	if (passed("500a")) {	1✔
260	translation = true;	1✔
261	translator = true;	1✔
262	}
263
264	if (passed("240a")) {	1✔
265	translation = true;	1✔
266	originalTitle = true;	1✔
267	}
268
269	// TODO: maybe the value should check against other language code
270	if (passed("240l")) {	1✔
271	translation = true;	1✔
272	originalTitle = true;	1✔
273	}
274
275	if (passed("765ind2")) {	1✔
UNCOV 276	translation = true;	×
277	}
278
279	if (passed("765t")) {	1✔
UNCOV 280	translation = true;	×
UNCOV 281	originalTitle = true;	×
282	}
283
284	if (passed("765s")) {	1✔
UNCOV 285	translation = true;	×
UNCOV 286	originalTitle = true;	×
287	}
288
289	if (passed("765d")) {	1✔
UNCOV 290	translation = true;	×
UNCOV 291	originalPublication = true;	×
292	}
293	}	1✔
294
295	/**
296	* Check if the rule has been passed the test
297	* @param path
298	* @return
299	*/
300	public boolean passed(String path) {
301	return resultMap.containsKey(path) && resultMap.get(path).getStatus().equals(RuleCheckingOutputStatus.PASSED);	1✔
302	}
303
304	public boolean isTranslation() {
305	return translation;	1✔
306	}
307
308	private static Map<String, String> toMap(List<String> input) {
309	Map<String, String> map = new HashMap<>();	1✔
310	for (int i = 0; i < input.size(); i+=2) {	1✔
311	map.put(input.get(i), input.get(i+1));	1✔
312	}
313	return map;	1✔
314	}
315	}

pkiraly / metadata-qa-marc / #1447

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous