14648908172

Committed 24 Apr 2025 06:25PM UTC coverage: 60.848% (-0.2%) from 61.027%

Build # 14648908172

Build Type

push

github

Committed by

ljacqu

Commit Message

Hunspell: Refactor to streams / support spaces in base words

Run Details

373 of 688 branches covered (54.22%)

22 of 27 new or added lines in 1 file covered. (81.48%)

962 of 1581 relevant lines covered (60.85%)

3.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.36

/src/main/java/ch/jalu/wordeval/dictionary/hunspell/HunspellUnmuncherService.java

package ch.jalu.wordeval.dictionary.hunspell;

import com.google.common.base.Preconditions;
import org.springframework.stereotype.Service;

import java.util.List;
import java.util.function.Function;
import java.util.stream.Stream;

/**
 * Service to "unmunch" words (i.e. expand) based on Hunspell affix rules.
 */
@Service
public class HunspellUnmuncherService {

  /**
   * Processes the given dictionary lines and expands them based on the specified affix rules.
   *
   * @param lines the lines of the dictionary file (including affix information)
   * @param affixDefinition the dictionary's affix definitions
   * @return all words isolated and expanded
   */
  public Stream<String> unmunch(Stream<String> lines, HunspellAffixes affixDefinition) {
    return lines.flatMap(line -> unmunch(line, affixDefinition));
  }

  private Stream<String> unmunch(String line, HunspellAffixes affixDefinition) {
    int indexOfSlash = line.indexOf('/');
    if (indexOfSlash <= 0) {
      // We don't support empty strings as base
      return Stream.of(line);
    }
    // Slashes can be escaped with a backslash apparently (nl.dic), but this currently goes beyond the desired scope,
    // since a word with a slash won't be interesting to wordeval anyway. So it's the job of a sanitizer to skip these
    // words before they're passed to this class.
    Preconditions.checkArgument(line.indexOf('\\') < 0, line);

    String baseWord = line.substring(0, indexOfSlash);
    List<String> affixFlags = extractAffixFlags(line.substring(indexOfSlash + 1), affixDefinition.getFlagType());

    boolean includeBaseWord = affixDefinition.getNeedAffixFlag() == null
        || !affixFlags.contains(affixDefinition.getNeedAffixFlag());
    if (includeBaseWord) {
      return Stream.concat(Stream.of(baseWord), streamThroughAllAffixes(baseWord, affixFlags, affixDefinition));
    }
    return streamThroughAllAffixes(baseWord, affixFlags, affixDefinition);
  }

  /**
   * Returns the affix flags indicated in the "meta part" of the line, i.e. the section after the slash separating
   * the word.
   *
   * @param metaPart the part with the affixes
   * @param flagType flag type to parse the list with
   * @return all affix flags in the given meta part
   */
  private List<String> extractAffixFlags(String metaPart, AffixFlagType flagType) {
    int indexFirstWhitespace = -1;
    for (int i = 0; i < metaPart.length(); ++i) {
      if (Character.isWhitespace(metaPart.charAt(i))) {
        indexFirstWhitespace = i;
        break;
      }
    }

    String affixList = indexFirstWhitespace >= 0 ? metaPart.substring(0, indexFirstWhitespace) : metaPart;
    return flagType.split(affixList);
  }

  private Stream<String> streamThroughAllAffixes(String baseWord, List<String> affixFlags,
                                                 HunspellAffixes affixDefinition) {
    return affixFlags.stream()
        .flatMap(affixFlag -> affixDefinition.streamThroughMatchingRules(baseWord, affixFlag))
        .flatMap(affixRule -> {
          String wordWithAffix = affixRule.applyRule(baseWord);
          if (wordWithAffix == null) {
            return Stream.empty();
          }

          boolean hasContinuationClasses = !affixRule.getContinuationClasses().isEmpty();
          boolean isCrossProductPrefix = affixRule.getType() == AffixType.PFX && affixRule.isCrossProduct();
          if (hasContinuationClasses && isCrossProductPrefix) {
            return Stream.of(
                    Stream.of(wordWithAffix),
                    streamThroughAllAffixes(wordWithAffix, affixRule.getContinuationClasses(), affixDefinition),
                    addSuffixes(wordWithAffix, affixFlags, affixDefinition))
                .flatMap(Function.identity());
          } else if (hasContinuationClasses) {
            return Stream.concat(
                    Stream.of(wordWithAffix),
                    streamThroughAllAffixes(wordWithAffix, affixRule.getContinuationClasses(), affixDefinition));
          } else if (isCrossProductPrefix) {
            return Stream.concat(
                    Stream.of(wordWithAffix),
                    addSuffixes(wordWithAffix, affixFlags, affixDefinition));
          } else {
            return Stream.of(wordWithAffix);
          }
        });
  }

  private static Stream<String> addSuffixes(String word,
                                            List<String> affixFlags,
                                            HunspellAffixes affixDefinition) {
    return affixFlags.stream()
        .flatMap(affixFlag -> affixDefinition.getAffixRulesByFlag().get(affixFlag).stream())
        .filter(rule -> rule.getType() == AffixType.SFX && rule.isCrossProduct() && rule.matches(word))
        .map(rule -> rule.applyRule(word));
  }
}

1	package ch.jalu.wordeval.dictionary.hunspell;
2
3	import com.google.common.base.Preconditions;
4	import org.springframework.stereotype.Service;
5
6	import java.util.List;
7	import java.util.function.Function;
8	import java.util.stream.Stream;
9
10	/**
11	* Service to "unmunch" words (i.e. expand) based on Hunspell affix rules.
12	*/
13	@Service
14	public class HunspellUnmuncherService {	3✔
15
16	/**
17	* Processes the given dictionary lines and expands them based on the specified affix rules.
18	*
19	* @param lines the lines of the dictionary file (including affix information)
20	* @param affixDefinition the dictionary's affix definitions
21	* @return all words isolated and expanded
22	*/
23	public Stream<String> unmunch(Stream<String> lines, HunspellAffixes affixDefinition) {
24	return lines.flatMap(line -> unmunch(line, affixDefinition));	11✔
25	}
26
27	private Stream<String> unmunch(String line, HunspellAffixes affixDefinition) {
28	int indexOfSlash = line.indexOf('/');	4✔
29	if (indexOfSlash <= 0) {	2✔
30	// We don't support empty strings as base
31	return Stream.of(line);	3✔
32	}
33	// Slashes can be escaped with a backslash apparently (nl.dic), but this currently goes beyond the desired scope,
34	// since a word with a slash won't be interesting to wordeval anyway. So it's the job of a sanitizer to skip these
35	// words before they're passed to this class.
36	Preconditions.checkArgument(line.indexOf('\\') < 0, line);	9✔
37
38	String baseWord = line.substring(0, indexOfSlash);	5✔
39	List<String> affixFlags = extractAffixFlags(line.substring(indexOfSlash + 1), affixDefinition.getFlagType());	10✔
40
41	boolean includeBaseWord = affixDefinition.getNeedAffixFlag() == null	5✔
42	\|\| !affixFlags.contains(affixDefinition.getNeedAffixFlag());	7!
43	if (includeBaseWord) {	2✔
44	return Stream.concat(Stream.of(baseWord), streamThroughAllAffixes(baseWord, affixFlags, affixDefinition));	9✔
45	}
46	return streamThroughAllAffixes(baseWord, affixFlags, affixDefinition);	6✔
47	}
48
49	/**
50	* Returns the affix flags indicated in the "meta part" of the line, i.e. the section after the slash separating
51	* the word.
52	*
53	* @param metaPart the part with the affixes
54	* @param flagType flag type to parse the list with
55	* @return all affix flags in the given meta part
56	*/
57	private List<String> extractAffixFlags(String metaPart, AffixFlagType flagType) {
58	int indexFirstWhitespace = -1;	2✔
59	for (int i = 0; i < metaPart.length(); ++i) {	8✔
60	if (Character.isWhitespace(metaPart.charAt(i))) {	5✔
61	indexFirstWhitespace = i;	2✔
62	break;	1✔
63	}
64	}
65
66	String affixList = indexFirstWhitespace >= 0 ? metaPart.substring(0, indexFirstWhitespace) : metaPart;	9✔
67	return flagType.split(affixList);	4✔
68	}
69
70	private Stream<String> streamThroughAllAffixes(String baseWord, List<String> affixFlags,
71	HunspellAffixes affixDefinition) {
72	return affixFlags.stream()	6✔
73	.flatMap(affixFlag -> affixDefinition.streamThroughMatchingRules(baseWord, affixFlag))	11✔
74	.flatMap(affixRule -> {	1✔
75	String wordWithAffix = affixRule.applyRule(baseWord);	4✔
76	if (wordWithAffix == null) {	2✔
77	return Stream.empty();	2✔
78	}
79
80	boolean hasContinuationClasses = !affixRule.getContinuationClasses().isEmpty();	8✔
81	boolean isCrossProductPrefix = affixRule.getType() == AffixType.PFX && affixRule.isCrossProduct();	11!
82	if (hasContinuationClasses && isCrossProductPrefix) {	4!
NEW 83	return Stream.of(	×
NEW 84	Stream.of(wordWithAffix),	×
NEW 85	streamThroughAllAffixes(wordWithAffix, affixRule.getContinuationClasses(), affixDefinition),	×
NEW 86	addSuffixes(wordWithAffix, affixFlags, affixDefinition))	×
NEW 87	.flatMap(Function.identity());	×
88	} else if (hasContinuationClasses) {	2✔
89	return Stream.concat(	3✔
90	Stream.of(wordWithAffix),	4✔
91	streamThroughAllAffixes(wordWithAffix, affixRule.getContinuationClasses(), affixDefinition));	3✔
92	} else if (isCrossProductPrefix) {	2✔
93	return Stream.concat(	3✔
94	Stream.of(wordWithAffix),	4✔
95	addSuffixes(wordWithAffix, affixFlags, affixDefinition));	1✔
96	} else {
97	return Stream.of(wordWithAffix);	3✔
98	}
99	});
100	}
101
102	private static Stream<String> addSuffixes(String word,
103	List<String> affixFlags,
104	HunspellAffixes affixDefinition) {
105	return affixFlags.stream()	5✔
106	.flatMap(affixFlag -> affixDefinition.getAffixRulesByFlag().get(affixFlag).stream())	9✔
107	.filter(rule -> rule.getType() == AffixType.SFX && rule.isCrossProduct() && rule.matches(word))	18✔
108	.map(rule -> rule.applyRule(word));	5✔
109	}
110	}

ljacqu / wordeval / 14648908172

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous