#1433

Committed 18 Mar 2025 09:04PM UTC coverage: 90.582% (-0.02%) from 90.603%

Build # #1433

Build Type

push

Committed by

Commit Message

Normalise place names with external dictionary file #598

Coverage Stats

145 of 169 new or added lines in 5 files covered. (85.8%)

36406 of 40191 relevant lines covered (90.58%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.31

/src/main/java/de/gwdg/metadataqa/marc/cli/utils/placename/PlaceNameNormaliser.java

package de.gwdg.metadataqa.marc.cli.utils.placename;

import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvException;
import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.stream.Collectors;

public class PlaceNameNormaliser {
  private static final Logger logger = Logger.getLogger(PlaceNameNormaliser.class.getCanonicalName());
  public static final String UNRESOLVED_PLACE_NAMES_FILE = "translations-unresolved-place-names.txt";

  /*
  private Set<String> knownMultiwordCities = Set.of(
    "New York", "Ithaca, NY", "Stamford, CN", "Tel Aviv", "Buenos Aires", "Middletown, CN",
    "Santiago de Chile", "Princeton, NJ", "Mount Vernon, NY", "Los Angeles", "Pôrto Alegre",
    "Berkeley, CA", "Philadelphia, PA", "'s-Gravenhage", "São Paulo", "Kentfield, CA", "Avon, CN"
  );
   */

  private final String translationPlaceNameDictionaryDir;
  private final String outputDir;

  private Map<String, PlaceName> coords;
  private Map<String, List<String>> synonyms;

  private Map<String, Integer> unresolvedPlaceNames = new HashMap<>();

  public PlaceNameNormaliser(String translationPlaceNameDictionaryDir, String outputDir) {
    this.translationPlaceNameDictionaryDir = translationPlaceNameDictionaryDir;
    this.outputDir = outputDir;
    coords = new HashMap<>();
    synonyms = new HashMap<>();

    File errorFile = new File(outputDir, UNRESOLVED_PLACE_NAMES_FILE);
    if (errorFile.exists())
      errorFile.delete();

    try {
      processCoords(readCsvFile("coord.csv"));
      processSynonyms(readCsvFile("place-synonyms-normalized.csv"));
    } catch (IOException e) {
      throw new RuntimeException(e);
    } catch (CsvException e) {
      throw new RuntimeException(e);
    }
  }

  public Map<String, PlaceName> getCoords() {
    return coords;
  }

  public Map<String, List<String>> getSynonyms() {
    return synonyms;
  }

  private void processSynonyms(List<String[]> rows) {
    for (String[] row : rows) {
      synonyms.computeIfAbsent(row[0], k -> new ArrayList<>()).add(row[1]);
    }
  }

  public void reportUnresolvedPlaceNames() {
    if (unresolvedPlaceNames.isEmpty())
      return;
    String content = unresolvedPlaceNames.entrySet().stream()
      .sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue()))
      .map(e -> e.getKey() + ": " + e.getValue())
      .collect(Collectors.joining("\n"));

    try {
      FileUtils.writeStringToFile(new File(outputDir, UNRESOLVED_PLACE_NAMES_FILE), content, StandardCharsets.UTF_8, true);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private void processCoords(List<String[]> rows) {
    for (String[] row : rows) {
      coords.put(row[0], new PlaceName(row));
    }
  }

  private List<String[]> readCsvFile(String csvFile) throws IOException, CsvException {
    FileReader filereader = new FileReader(new File(translationPlaceNameDictionaryDir, csvFile));
    CSVReader csvReader = new CSVReaderBuilder(filereader).withSkipLines(1).build();
    return csvReader.readAll();
  }

  public List<PlaceName> normalise(String placeName) {
    return resolve(clean(placeName));
  }

  public List<PlaceName> normalise(List<String> placeNames) {
    List<PlaceName> normalised = new ArrayList<>();
    for (String placeName : placeNames)
      normalised.addAll(resolve(clean(placeName)));
    return normalised;
  }

  public String clean(String input) {
    String output = input;
    output = Normalizer.normalize(output, Normalizer.Form.NFKC);
    output = output.replaceAll("\\s*[,:;]\\s*$", "");
    output = output.replaceAll("^\\[(.+)\\]$", "$1");
    output = output.replaceAll("^\\[(.+)$", "$1");
    output = output.replaceAll("^(.+)\\]$", "$1");
    output = output.replaceAll(" \\[etc\\.$", "");
    output = output.replaceAll(", Calif\\.$", ", CA");
    output = output.replaceAll(", California$", ", CA");
    output = output.replaceAll(", (Conn\\.|Connecticut)$", ", CT");
    output = output.replaceAll(", N\\. ?Y\\.$", ", NY");
    output = output.replaceAll(", N\\. ?J\\.$", ", NJ");
    output = output.replaceAll(", Massachusetts$", ", MA");
    output = output.replaceAll(", Arizona$", ", AZ");
    output = output.replaceAll(", Pa\\.$", ", PA");
    output = output.replaceAll(", Ill\\.$", ", IL");
    output = output.replaceAll(", Illinois$", ", IL");
    output = output.replaceAll(", Texas$", ", TX");
    output = output.replaceAll(", Maryland$", ", MD");
    return output;
  }

  public List<PlaceName> resolve(String originalNameForm) {
    if (coords.containsKey(originalNameForm)) {
      return List.of(coords.get(originalNameForm));
    }
    List<PlaceName> placeNames = new ArrayList<>();
    if (synonyms.containsKey(originalNameForm)) {
      for (String synonym : synonyms.get(originalNameForm)) {
        placeNames.add(coords.get(synonym));
      }
    } else {
      unresolvedPlaceNames.computeIfAbsent(originalNameForm, k -> 0);
      unresolvedPlaceNames.put(originalNameForm, unresolvedPlaceNames.get(originalNameForm)+1);
      // logger.info("Unresolved place name: " + originalNameForm);
    }
    return placeNames;
  }
}

1	package de.gwdg.metadataqa.marc.cli.utils.placename;
2
3	import com.opencsv.CSVReader;
4	import com.opencsv.CSVReaderBuilder;
5	import com.opencsv.exceptions.CsvException;
6	import org.apache.commons.io.FileUtils;
7
8	import java.io.File;
9	import java.io.FileReader;
10	import java.io.IOException;
11	import java.nio.charset.StandardCharsets;
12	import java.text.Normalizer;
13	import java.util.ArrayList;
14	import java.util.HashMap;
15	import java.util.List;
16	import java.util.Map;
17	import java.util.logging.Logger;
18	import java.util.stream.Collectors;
19
20	public class PlaceNameNormaliser {
21	private static final Logger logger = Logger.getLogger(PlaceNameNormaliser.class.getCanonicalName());	1✔
22	public static final String UNRESOLVED_PLACE_NAMES_FILE = "translations-unresolved-place-names.txt";
23
24	/*
25	private Set<String> knownMultiwordCities = Set.of(
26	"New York", "Ithaca, NY", "Stamford, CN", "Tel Aviv", "Buenos Aires", "Middletown, CN",
27	"Santiago de Chile", "Princeton, NJ", "Mount Vernon, NY", "Los Angeles", "Pôrto Alegre",
28	"Berkeley, CA", "Philadelphia, PA", "'s-Gravenhage", "São Paulo", "Kentfield, CA", "Avon, CN"
29	);
30	*/
31
32	private final String translationPlaceNameDictionaryDir;
33	private final String outputDir;
34
35	private Map<String, PlaceName> coords;
36	private Map<String, List<String>> synonyms;
37
38	private Map<String, Integer> unresolvedPlaceNames = new HashMap<>();	1✔
39
40	public PlaceNameNormaliser(String translationPlaceNameDictionaryDir, String outputDir) {	1✔
41	this.translationPlaceNameDictionaryDir = translationPlaceNameDictionaryDir;	1✔
42	this.outputDir = outputDir;	1✔
43	coords = new HashMap<>();	1✔
44	synonyms = new HashMap<>();	1✔
45
46	File errorFile = new File(outputDir, UNRESOLVED_PLACE_NAMES_FILE);	1✔
47	if (errorFile.exists())	1✔
NEW 48	errorFile.delete();	×
49
50	try {
51	processCoords(readCsvFile("coord.csv"));	1✔
52	processSynonyms(readCsvFile("place-synonyms-normalized.csv"));	1✔
NEW 53	} catch (IOException e) {	×
NEW 54	throw new RuntimeException(e);	×
NEW 55	} catch (CsvException e) {	×
NEW 56	throw new RuntimeException(e);	×
57	}	1✔
58	}	1✔
59
60	public Map<String, PlaceName> getCoords() {
61	return coords;	1✔
62	}
63
64	public Map<String, List<String>> getSynonyms() {
65	return synonyms;	1✔
66	}
67
68	private void processSynonyms(List<String[]> rows) {
69	for (String[] row : rows) {	1✔
70	synonyms.computeIfAbsent(row[0], k -> new ArrayList<>()).add(row[1]);	1✔
71	}	1✔
72	}	1✔
73
74	public void reportUnresolvedPlaceNames() {
75	if (unresolvedPlaceNames.isEmpty())	1✔
NEW 76	return;	×
77	String content = unresolvedPlaceNames.entrySet().stream()	1✔
78	.sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue()))	1✔
79	.map(e -> e.getKey() + ": " + e.getValue())	1✔
80	.collect(Collectors.joining("\n"));	1✔
81
82	try {
83	FileUtils.writeStringToFile(new File(outputDir, UNRESOLVED_PLACE_NAMES_FILE), content, StandardCharsets.UTF_8, true);	1✔
NEW 84	} catch (IOException e) {	×
NEW 85	throw new RuntimeException(e);	×
86	}	1✔
87	}	1✔
88
89	private void processCoords(List<String[]> rows) {
90	for (String[] row : rows) {	1✔
91	coords.put(row[0], new PlaceName(row));	1✔
92	}	1✔
93	}	1✔
94
95	private List<String[]> readCsvFile(String csvFile) throws IOException, CsvException {
96	FileReader filereader = new FileReader(new File(translationPlaceNameDictionaryDir, csvFile));	1✔
97	CSVReader csvReader = new CSVReaderBuilder(filereader).withSkipLines(1).build();	1✔
98	return csvReader.readAll();	1✔
99	}
100
101	public List<PlaceName> normalise(String placeName) {
NEW 102	return resolve(clean(placeName));	×
103	}
104
105	public List<PlaceName> normalise(List<String> placeNames) {
106	List<PlaceName> normalised = new ArrayList<>();	1✔
107	for (String placeName : placeNames)	1✔
108	normalised.addAll(resolve(clean(placeName)));	1✔
109	return normalised;	1✔
110	}
111
112	public String clean(String input) {
113	String output = input;	1✔
114	output = Normalizer.normalize(output, Normalizer.Form.NFKC);	1✔
115	output = output.replaceAll("\\s[,:;]\\s$", "");	1✔
116	output = output.replaceAll("^\\[(.+)\\]$", "$1");	1✔
117	output = output.replaceAll("^\\[(.+)$", "$1");	1✔
118	output = output.replaceAll("^(.+)\\]$", "$1");	1✔
119	output = output.replaceAll(" \\[etc\\.$", "");	1✔
120	output = output.replaceAll(", Calif\\.$", ", CA");	1✔
121	output = output.replaceAll(", California$", ", CA");	1✔
122	output = output.replaceAll(", (Conn\\.\|Connecticut)$", ", CT");	1✔
123	output = output.replaceAll(", N\\. ?Y\\.$", ", NY");	1✔
124	output = output.replaceAll(", N\\. ?J\\.$", ", NJ");	1✔
125	output = output.replaceAll(", Massachusetts$", ", MA");	1✔
126	output = output.replaceAll(", Arizona$", ", AZ");	1✔
127	output = output.replaceAll(", Pa\\.$", ", PA");	1✔
128	output = output.replaceAll(", Ill\\.$", ", IL");	1✔
129	output = output.replaceAll(", Illinois$", ", IL");	1✔
130	output = output.replaceAll(", Texas$", ", TX");	1✔
131	output = output.replaceAll(", Maryland$", ", MD");	1✔
132	return output;	1✔
133	}
134
135	public List<PlaceName> resolve(String originalNameForm) {
136	if (coords.containsKey(originalNameForm)) {	1✔
137	return List.of(coords.get(originalNameForm));	1✔
138	}
139	List<PlaceName> placeNames = new ArrayList<>();	1✔
140	if (synonyms.containsKey(originalNameForm)) {	1✔
141	for (String synonym : synonyms.get(originalNameForm)) {	1✔
142	placeNames.add(coords.get(synonym));	1✔
143	}	1✔
144	} else {
145	unresolvedPlaceNames.computeIfAbsent(originalNameForm, k -> 0);	1✔
146	unresolvedPlaceNames.put(originalNameForm, unresolvedPlaceNames.get(originalNameForm)+1);	1✔
147	// logger.info("Unresolved place name: " + originalNameForm);
148	}
149	return placeNames;	1✔
150	}
151	}

pkiraly / metadata-qa-marc / #1433

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous