2144239727

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/8855444d35e4b908ac74334a52621429481df45e">8855444d3">&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/</a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/8855444d35e4b908ac74334a52621429481df45e">8855444d3</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/8855444d35e4b908ac74334a52621429481df45e">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/8855444d35e4b908ac74334a52621429481df45e&quot;&gt;8855444d3&lt;/a&gt;&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/8855444d35e4b908ac74334a52621429481df45e&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;/a&amp;gt;&amp;lt... (continued)

Pull Request Pull Request #7234: Release/343 release candidate

Run Details

4002 of 4002 new or added lines in 187 files covered. (100.0%)

8080 of 10941 relevant lines covered (73.85%)

0.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.17

/src/main/scala/com/johnsnowlabs/nlp/annotators/spell/norvig/NorvigSweetingModel.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators.spell.norvig

import com.johnsnowlabs.nlp.annotators.spell.util.Utilities
import com.johnsnowlabs.nlp.serialization.MapFeature
import com.johnsnowlabs.nlp._
import org.apache.spark.ml.util.Identifiable
import org.slf4j.LoggerFactory

import scala.collection.immutable.HashSet

/** This annotator retrieves tokens and makes corrections automatically if not found in an English
  * dictionary. Inspired by Norvig model and [[https://github.com/wolfgarbe/SymSpell SymSpell]].
  *
  * The Symmetric Delete spelling correction algorithm reduces the complexity of edit candidate
  * generation and dictionary lookup for a given Damerau-Levenshtein distance. It is six orders of
  * magnitude faster (than the standard approach with deletes + transposes + replaces + inserts)
  * and language independent.
  *
  * This is the instantiated model of the [[NorvigSweetingApproach]]. For training your own model,
  * please see the documentation of that class.
  *
  * Pretrained models can be loaded with `pretrained` of the companion object:
  * {{{
  * val spellChecker = NorvigSweetingModel.pretrained()
  *   .setInputCols("token")
  *   .setOutputCol("spell")
  *   .setDoubleVariants(true)
  * }}}
  * The default model is `"spellcheck_norvig"`, if no name is provided. For available pretrained
  * models please see the [[https://nlp.johnsnowlabs.com/models?task=Spell+Check Models Hub]].
  *
  * For extended examples of usage, see the
  * [[https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/training/english/vivekn-sentiment/VivekNarayanSentimentApproach.ipynb Spark NLP Workshop]]
  * and the
  * [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/spell/norvig/NorvigSweetingTestSpec.scala NorvigSweetingTestSpec]].
  *
  * ==Example==
  * {{{
  * import spark.implicits._
  * import com.johnsnowlabs.nlp.base.DocumentAssembler
  * import com.johnsnowlabs.nlp.annotators.Tokenizer
  * import com.johnsnowlabs.nlp.annotators.spell.norvig.NorvigSweetingModel
  *
  * import org.apache.spark.ml.Pipeline
  *
  * val documentAssembler = new DocumentAssembler()
  *   .setInputCol("text")
  *   .setOutputCol("document")
  *
  * val tokenizer = new Tokenizer()
  *   .setInputCols("document")
  *   .setOutputCol("token")
  *
  * val spellChecker = NorvigSweetingModel.pretrained()
  *   .setInputCols("token")
  *   .setOutputCol("spell")
  *
  * val pipeline = new Pipeline().setStages(Array(
  *   documentAssembler,
  *   tokenizer,
  *   spellChecker
  * ))
  *
  * val data = Seq("somtimes i wrrite wordz erong.").toDF("text")
  * val result = pipeline.fit(data).transform(data)
  * result.select("spell.result").show(false)
  * +--------------------------------------+
  * |result                                |
  * +--------------------------------------+
  * |[sometimes, i, write, words, wrong, .]|
  * +--------------------------------------+
  * }}}
  *
  * @see
  *   [[com.johnsnowlabs.nlp.annotators.spell.symmetric.SymmetricDeleteModel SymmetricDeleteModel]]
  *   for an alternative approach to spell checking
  * @see
  *   [[com.johnsnowlabs.nlp.annotators.spell.context.ContextSpellCheckerModel ContextSpellCheckerModel]]
  *   for a DL based approach
  * @groupname anno Annotator types
  * @groupdesc anno
  *   Required input and expected output annotator types
  * @groupname Ungrouped Members
  * @groupname param Parameters
  * @groupname setParam Parameter setters
  * @groupname getParam Parameter getters
  * @groupname Ungrouped Members
  * @groupprio param  1
  * @groupprio anno  2
  * @groupprio Ungrouped 3
  * @groupprio setParam  4
  * @groupprio getParam  5
  * @groupdesc param
  *   A list of (hyper-)parameter keys this annotator can take. Users can set and get the
  *   parameter values through setters and getters, respectively.
  */
class NorvigSweetingModel(override val uid: String)
    extends AnnotatorModel[NorvigSweetingModel]
    with HasSimpleAnnotate[NorvigSweetingModel]
    with NorvigSweetingParams {

  import com.johnsnowlabs.nlp.AnnotatorType._

  /** Annotator reference id. Used to identify elements in metadata or to refer to this annotator
    * type
    */
  def this() = this(Identifiable.randomUID("SPELL"))

  private val logger = LoggerFactory.getLogger("NorvigApproach")

  /** Output annotator type : TOKEN
    *
    * @group anno
    */
  override val outputAnnotatorType: AnnotatorType = TOKEN

  /** Input annotator type : TOKEN
    *
    * @group anno
    */
  override val inputAnnotatorTypes: Array[AnnotatorType] = Array(TOKEN)

  /** Number of words in the dictionary
    *
    * @group param
    */
  protected val wordCount: MapFeature[String, Long] = new MapFeature(this, "wordCount")

  /** @group getParam */
  protected def getWordCount: Map[String, Long] = $$(wordCount)

  /** @group setParam */
  def setWordCount(value: Map[String, Long]): this.type = set(wordCount, value)

  private lazy val allWords: HashSet[String] = {
    if ($(caseSensitive)) HashSet($$(wordCount).keys.toSeq: _*)
    else HashSet($$(wordCount).keys.toSeq.map(_.toLowerCase): _*)
  }

  private lazy val frequencyBoundaryValues: (Long, Long) = {
    val min: Long = $$(wordCount).filter(_._1.length > $(wordSizeIgnore)).minBy(_._2)._2
    val max = $$(wordCount).filter(_._1.length > $(wordSizeIgnore)).maxBy(_._2)._2
    (min, max)
  }

  override def annotate(annotations: Seq[Annotation]): Seq[Annotation] = {
    annotations.map { token =>
      val verifiedWord = checkSpellWord(token.result)
      Annotation(
        outputAnnotatorType,
        token.begin,
        token.end,
        verifiedWord._1,
        Map("confidence" -> verifiedWord._2.toString, "sentence" -> token.metadata("sentence")))
    }
  }

  def checkSpellWord(raw: String): (String, Double) = {
    val input = Utilities.limitDuplicates($(dupsLimit), raw)
    logger.debug(s"spell checker target word: $input")
    val possibility = getBestSpellingSuggestion(input)
    if (possibility._1.isDefined) return (possibility._1.get, possibility._2)
    val listedSuggestions = suggestions(input)
    val sortedFrequencies = getSortedWordsByFrequency(listedSuggestions, input)
    val sortedHamming = getSortedWordsByHamming(listedSuggestions, input)
    getResult(sortedFrequencies, sortedHamming, input)
  }

  private def getBestSpellingSuggestion(word: String): (Option[String], Double) = {
    var score: Double = 0
    if ($(shortCircuit)) {
      val suggestedWord = getShortCircuitSuggestion(word).getOrElse(word)
      score = getScoreFrequency(suggestedWord)
      (Some(suggestedWord), score)
    } else {
      val suggestions = getSuggestion(word: String)
      (suggestions._1, suggestions._2)
    }
  }

  private def getShortCircuitSuggestion(word: String): Option[String] = {
    if (Utilities.reductions(word, $(reductLimit)).exists(allWords.contains)) Some(word)
    else if (Utilities.getVowelSwaps(word, $(vowelSwapLimit)).exists(allWords.contains))
      Some(word)
    else if (Utilities.variants(word).exists(allWords.contains)) Some(word)
    else if (both(word).exists(allWords.contains)) Some(word)
    else if ($(doubleVariants) && computeDoubleVariants(word).exists(allWords.contains))
      Some(word)
    else None
  }

  /** variants of variants of a word */
  def computeDoubleVariants(word: String): List[String] =
    Utilities.variants(word).flatMap(variant => Utilities.variants(variant))

  private def getSuggestion(word: String): (Option[String], Double) = {
    if (allWords.contains(word)) {
      logger.debug("Word found in dictionary. No spell change")
      (Some(word), 1)
    } else if (word.length <= $(wordSizeIgnore)) {
      logger.debug("word ignored because length is less than wordSizeIgnore")
      (Some(word), 0)
    } else if (allWords.contains(word.distinct)) {
      logger.debug("Word as distinct found in dictionary")
      val score = getScoreFrequency(word.distinct)
      (Some(word.distinct), score)
    } else (None, -1)
  }

  def getScoreFrequency(word: String): Double = {
    val frequency = Utilities.getFrequency(word, $$(wordCount))
    normalizeFrequencyValue(frequency)
  }

  def normalizeFrequencyValue(value: Long): Double = {
    if (value > frequencyBoundaryValues._2) {
      return 1
    }
    if (value < frequencyBoundaryValues._1) {
      return 0
    }
    val normalizedValue =
      (value - frequencyBoundaryValues._1).toDouble / (frequencyBoundaryValues._2 - frequencyBoundaryValues._1).toDouble
    BigDecimal(normalizedValue).setScale(4, BigDecimal.RoundingMode.HALF_UP).toDouble
  }

  private def suggestions(word: String): List[String] = {
    val intersectedPossibilities = allWords.intersect({
      val base =
        Utilities.reductions(word, $(reductLimit)) ++
          Utilities.getVowelSwaps(word, $(vowelSwapLimit)) ++
          Utilities.variants(word) ++
          both(word)
      if ($(doubleVariants)) base ++ computeDoubleVariants(word) else base
    }.toSet)
    if (intersectedPossibilities.nonEmpty) intersectedPossibilities.toList
    else List.empty[String]
  }

  private def both(word: String): List[String] = {
    Utilities
      .reductions(word, $(reductLimit))
      .flatMap(reduction => Utilities.getVowelSwaps(reduction, $(vowelSwapLimit)))
  }

  def getSortedWordsByFrequency(words: List[String], input: String): List[(String, Long)] = {
    val filteredWords = words.withFilter(_.length >= input.length)
    val sortedWordsByFrequency = filteredWords
      .map(word => (word, compareFrequencies(word)))
      .sortWith(_._2 > _._2)
      .take($(intersections))
    logger.debug(s"recommended by frequency: ${sortedWordsByFrequency.mkString(", ")}")
    sortedWordsByFrequency
  }

  private def compareFrequencies(value: String): Long =
    Utilities.getFrequency(value, $$(wordCount))

  def getSortedWordsByHamming(words: List[String], input: String): List[(String, Long)] = {
    val sortedWordByHamming = words
      .map(word => (word, compareHammers(input)(word)))
      .sortBy(_._2)
      .takeRight($(intersections))
    logger.debug(s"recommended by hamming: ${sortedWordByHamming.mkString(", ")}")
    sortedWordByHamming
  }

  private def compareHammers(input: String)(value: String): Long =
    Utilities.computeHammingDistance(input, value)

  def getResult(
      wordsByFrequency: List[(String, Long)],
      wordsByHamming: List[(String, Long)],
      input: String): (String, Double) = {
    var recommendation: (Option[String], Double) = (None, 0)
    val intersectWords =
      wordsByFrequency.map(word => word._1).intersect(wordsByHamming.map(word => word._1))
    if (wordsByFrequency.isEmpty && wordsByHamming.isEmpty) {
      logger.debug("no intersection or frequent words found")
      recommendation = (Some(input), 0)
    } else if (wordsByFrequency.isEmpty || wordsByHamming.isEmpty) {
      logger.debug("no intersection but one recommendation found")
      recommendation = getRecommendation(wordsByFrequency, wordsByHamming)
    } else if (intersectWords.nonEmpty) {
      logger.debug("hammer and frequency recommendations found")
      val frequencyAndHammingRecommendation =
        getFrequencyAndHammingRecommendation(wordsByFrequency, wordsByHamming, intersectWords)
      recommendation =
        (frequencyAndHammingRecommendation._1, frequencyAndHammingRecommendation._2)
    } else {
      logger.debug("no intersection of hammer and frequency")
      recommendation =
        getFrequencyOrHammingRecommendation(wordsByFrequency, wordsByHamming, input)
    }
    (recommendation._1.getOrElse(input), recommendation._2)
  }

  private def getRecommendation(
      wordsByFrequency: List[(String, Long)],
      wordsByHamming: List[(String, Long)]) = {
    if (wordsByFrequency.nonEmpty) {
      getResultByFrequency(wordsByFrequency)
    } else {
      getResultByHamming(wordsByHamming)
    }
  }

  private def getFrequencyAndHammingRecommendation(
      wordsByFrequency: List[(String, Long)],
      wordsByHamming: List[(String, Long)],
      intersectWords: List[String]): (Option[String], Double) = {
    val wordsByFrequencyAndHamming = intersectWords.map { word =>
      val frequency = wordsByFrequency.find(_._1 == word).get._2
      val hamming = wordsByHamming.find(_._1 == word).get._2
      (word, frequency, hamming)
    }
    val bestFrequencyValue = wordsByFrequencyAndHamming.maxBy(_._2)._2
    val bestHammingValue = wordsByFrequencyAndHamming.minBy(_._3)._3
    val bestRecommendations = wordsByFrequencyAndHamming.filter(word =>
      word._2 == bestFrequencyValue && word._3 == bestHammingValue)
    if (bestRecommendations.nonEmpty) {
      val result = (
        Utilities.getRandomValueFromList(bestRecommendations),
        Utilities.computeConfidenceValue(bestRecommendations))
      (Some(result._1.get._1), result._2)
    } else {
      if ($(frequencyPriority)) {
        (Some(wordsByFrequencyAndHamming.sortBy(_._3).maxBy(_._2)._1), 1.toDouble)
      } else {
        (Some(wordsByFrequencyAndHamming.sortBy(_._2).reverse.minBy(_._3)._1), 1.toDouble)
      }
    }
  }

  def getResultByFrequency(wordsByFrequency: List[(String, Long)]): (Option[String], Double) = {
    val bestFrequencyValue = wordsByFrequency.maxBy(_._2)._2
    val bestRecommendations = wordsByFrequency.filter(_._2 == bestFrequencyValue).map(_._1)
    (
      Utilities.getRandomValueFromList(bestRecommendations),
      Utilities.computeConfidenceValue(bestRecommendations))
  }

  def getResultByHamming(wordsByHamming: List[(String, Long)]): (Option[String], Double) = {
    val bestHammingValue = wordsByHamming.minBy(_._2)._2
    val bestRecommendations = wordsByHamming.filter(_._2 == bestHammingValue).map(_._1)
    (
      Utilities.getRandomValueFromList(bestRecommendations),
      Utilities.computeConfidenceValue(bestRecommendations))
  }

  def getFrequencyOrHammingRecommendation(
      wordsByFrequency: List[(String, Long)],
      wordsByHamming: List[(String, Long)],
      input: String): (Option[String], Double) = {
    val frequencyResult: String = getResultByFrequency(wordsByFrequency)._1.getOrElse(input)
    val hammingResult: String = getResultByHamming(wordsByHamming)._1.getOrElse(input)
    var result = List(frequencyResult, hammingResult)
    if (frequencyResult == input) {
      result = List(hammingResult)
    } else if (hammingResult == input) {
      result = List(frequencyResult)
    }

    (Utilities.getRandomValueFromList(result), Utilities.computeConfidenceValue(result))
  }

}

trait ReadablePretrainedNorvig
    extends ParamsAndFeaturesReadable[NorvigSweetingModel]
    with HasPretrained[NorvigSweetingModel] {
  override val defaultModelName = Some("spellcheck_norvig")

  /** Java compliant-overrides */
  override def pretrained(): NorvigSweetingModel = super.pretrained()

  override def pretrained(name: String): NorvigSweetingModel = super.pretrained(name)

  override def pretrained(name: String, lang: String): NorvigSweetingModel =
    super.pretrained(name, lang)

  override def pretrained(name: String, lang: String, remoteLoc: String): NorvigSweetingModel =
    super.pretrained(name, lang, remoteLoc)
}

/** This is the companion object of [[NorvigSweetingModel]]. Please refer to that class for the
  * documentation.
  */
object NorvigSweetingModel extends ReadablePretrainedNorvig

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.annotators.spell.norvig
18
19	import com.johnsnowlabs.nlp.annotators.spell.util.Utilities
20	import com.johnsnowlabs.nlp.serialization.MapFeature
21	import com.johnsnowlabs.nlp._
22	import org.apache.spark.ml.util.Identifiable
23	import org.slf4j.LoggerFactory
24
25	import scala.collection.immutable.HashSet
26
27	/** This annotator retrieves tokens and makes corrections automatically if not found in an English
28	* dictionary. Inspired by Norvig model and [[https://github.com/wolfgarbe/SymSpell SymSpell]].
29	*
30	* The Symmetric Delete spelling correction algorithm reduces the complexity of edit candidate
31	* generation and dictionary lookup for a given Damerau-Levenshtein distance. It is six orders of
32	* magnitude faster (than the standard approach with deletes + transposes + replaces + inserts)
33	* and language independent.
34	*
35	* This is the instantiated model of the [[NorvigSweetingApproach]]. For training your own model,
36	* please see the documentation of that class.
37	*
38	* Pretrained models can be loaded with `pretrained` of the companion object:
39	* {{{
40	* val spellChecker = NorvigSweetingModel.pretrained()
41	* .setInputCols("token")
42	* .setOutputCol("spell")
43	* .setDoubleVariants(true)
44	* }}}
45	* The default model is `"spellcheck_norvig"`, if no name is provided. For available pretrained
46	* models please see the [[https://nlp.johnsnowlabs.com/models?task=Spell+Check Models Hub]].
47	*
48	* For extended examples of usage, see the
49	* [[https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/training/english/vivekn-sentiment/VivekNarayanSentimentApproach.ipynb Spark NLP Workshop]]
50	* and the
51	* [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/spell/norvig/NorvigSweetingTestSpec.scala NorvigSweetingTestSpec]].
52	*
53	* ==Example==
54	* {{{
55	* import spark.implicits._
56	* import com.johnsnowlabs.nlp.base.DocumentAssembler
57	* import com.johnsnowlabs.nlp.annotators.Tokenizer
58	* import com.johnsnowlabs.nlp.annotators.spell.norvig.NorvigSweetingModel
59	*
60	* import org.apache.spark.ml.Pipeline
61	*
62	* val documentAssembler = new DocumentAssembler()
63	* .setInputCol("text")
64	* .setOutputCol("document")
65	*
66	* val tokenizer = new Tokenizer()
67	* .setInputCols("document")
68	* .setOutputCol("token")
69	*
70	* val spellChecker = NorvigSweetingModel.pretrained()
71	* .setInputCols("token")
72	* .setOutputCol("spell")
73	*
74	* val pipeline = new Pipeline().setStages(Array(
75	* documentAssembler,
76	* tokenizer,
77	* spellChecker
78	* ))
79	*
80	* val data = Seq("somtimes i wrrite wordz erong.").toDF("text")
81	* val result = pipeline.fit(data).transform(data)
82	* result.select("spell.result").show(false)
83	* +--------------------------------------+
84	* \|result \|
85	* +--------------------------------------+
86	* \|[sometimes, i, write, words, wrong, .]\|
87	* +--------------------------------------+
88	* }}}
89	*
90	* @see
91	* [[com.johnsnowlabs.nlp.annotators.spell.symmetric.SymmetricDeleteModel SymmetricDeleteModel]]
92	* for an alternative approach to spell checking
93	* @see
94	* [[com.johnsnowlabs.nlp.annotators.spell.context.ContextSpellCheckerModel ContextSpellCheckerModel]]
95	* for a DL based approach
96	* @groupname anno Annotator types
97	* @groupdesc anno
98	* Required input and expected output annotator types
99	* @groupname Ungrouped Members
100	* @groupname param Parameters
101	* @groupname setParam Parameter setters
102	* @groupname getParam Parameter getters
103	* @groupname Ungrouped Members
104	* @groupprio param 1
105	* @groupprio anno 2
106	* @groupprio Ungrouped 3
107	* @groupprio setParam 4
108	* @groupprio getParam 5
109	* @groupdesc param
110	* A list of (hyper-)parameter keys this annotator can take. Users can set and get the
111	* parameter values through setters and getters, respectively.
112	*/
113	class NorvigSweetingModel(override val uid: String)
114	extends AnnotatorModel[NorvigSweetingModel]
115	with HasSimpleAnnotate[NorvigSweetingModel]
116	with NorvigSweetingParams {
117
118	import com.johnsnowlabs.nlp.AnnotatorType._
119
120	/** Annotator reference id. Used to identify elements in metadata or to refer to this annotator
121	* type
122	*/
123	def this() = this(Identifiable.randomUID("SPELL"))	1✔
124
125	private val logger = LoggerFactory.getLogger("NorvigApproach")	1✔
126
127	/** Output annotator type : TOKEN
128	*
129	* @group anno
130	*/
131	override val outputAnnotatorType: AnnotatorType = TOKEN	1✔
132
133	/** Input annotator type : TOKEN
134	*
135	* @group anno
136	*/
137	override val inputAnnotatorTypes: Array[AnnotatorType] = Array(TOKEN)	1✔
138
139	/** Number of words in the dictionary
140	*
141	* @group param
142	*/
143	protected val wordCount: MapFeature[String, Long] = new MapFeature(this, "wordCount")	1✔
144
145	/** @group getParam */
146	protected def getWordCount: Map[String, Long] = $$(wordCount)	×
147
148	/** @group setParam */
149	def setWordCount(value: Map[String, Long]): this.type = set(wordCount, value)	1✔
150
151	private lazy val allWords: HashSet[String] = {
152	if ($(caseSensitive)) HashSet($$(wordCount).keys.toSeq: _*)
153	else HashSet($$(wordCount).keys.toSeq.map(_.toLowerCase): _*)
154	}
155
156	private lazy val frequencyBoundaryValues: (Long, Long) = {
157	val min: Long = $$(wordCount).filter(_._1.length > $(wordSizeIgnore)).minBy(_._2)._2
158	val max = $$(wordCount).filter(_._1.length > $(wordSizeIgnore)).maxBy(_._2)._2
159	(min, max)
160	}
161
162	override def annotate(annotations: Seq[Annotation]): Seq[Annotation] = {
163	annotations.map { token =>	1✔
164	val verifiedWord = checkSpellWord(token.result)	1✔
165	Annotation(	1✔
166	outputAnnotatorType,	1✔
167	token.begin,	1✔
168	token.end,	1✔
169	verifiedWord._1,	1✔
170	Map("confidence" -> verifiedWord._2.toString, "sentence" -> token.metadata("sentence")))	1✔
171	}
172	}
173
174	def checkSpellWord(raw: String): (String, Double) = {
175	val input = Utilities.limitDuplicates($(dupsLimit), raw)	1✔
176	logger.debug(s"spell checker target word: $input")	1✔
177	val possibility = getBestSpellingSuggestion(input)	1✔
178	if (possibility._1.isDefined) return (possibility._1.get, possibility._2)	1✔
179	val listedSuggestions = suggestions(input)	1✔
180	val sortedFrequencies = getSortedWordsByFrequency(listedSuggestions, input)	1✔
181	val sortedHamming = getSortedWordsByHamming(listedSuggestions, input)	1✔
182	getResult(sortedFrequencies, sortedHamming, input)	1✔
183	}
184
185	private def getBestSpellingSuggestion(word: String): (Option[String], Double) = {
186	var score: Double = 0	1✔
187	if ($(shortCircuit)) {	1✔
188	val suggestedWord = getShortCircuitSuggestion(word).getOrElse(word)	×
189	score = getScoreFrequency(suggestedWord)	×
190	(Some(suggestedWord), score)	×
191	} else {	1✔
192	val suggestions = getSuggestion(word: String)	1✔
193	(suggestions._1, suggestions._2)	1✔
194	}
195	}
196
197	private def getShortCircuitSuggestion(word: String): Option[String] = {
198	if (Utilities.reductions(word, $(reductLimit)).exists(allWords.contains)) Some(word)	×
199	else if (Utilities.getVowelSwaps(word, $(vowelSwapLimit)).exists(allWords.contains))	×
200	Some(word)	×
201	else if (Utilities.variants(word).exists(allWords.contains)) Some(word)	×
202	else if (both(word).exists(allWords.contains)) Some(word)	×
203	else if ($(doubleVariants) && computeDoubleVariants(word).exists(allWords.contains))	×
204	Some(word)	×
205	else None	×
206	}
207
208	/** variants of variants of a word */
209	def computeDoubleVariants(word: String): List[String] =
210	Utilities.variants(word).flatMap(variant => Utilities.variants(variant))	×
211
212	private def getSuggestion(word: String): (Option[String], Double) = {
213	if (allWords.contains(word)) {	1✔
214	logger.debug("Word found in dictionary. No spell change")	1✔
215	(Some(word), 1)	1✔
216	} else if (word.length <= $(wordSizeIgnore)) {	1✔
217	logger.debug("word ignored because length is less than wordSizeIgnore")	1✔
218	(Some(word), 0)	1✔
219	} else if (allWords.contains(word.distinct)) {	1✔
220	logger.debug("Word as distinct found in dictionary")	1✔
221	val score = getScoreFrequency(word.distinct)	1✔
222	(Some(word.distinct), score)	1✔
223	} else (None, -1)	1✔
224	}
225
226	def getScoreFrequency(word: String): Double = {
227	val frequency = Utilities.getFrequency(word, $$(wordCount))	1✔
228	normalizeFrequencyValue(frequency)	1✔
229	}
230
231	def normalizeFrequencyValue(value: Long): Double = {
232	if (value > frequencyBoundaryValues._2) {	1✔
233	return 1	1✔
234	}
235	if (value < frequencyBoundaryValues._1) {	1✔
236	return 0	×
237	}
238	val normalizedValue =
239	(value - frequencyBoundaryValues._1).toDouble / (frequencyBoundaryValues._2 - frequencyBoundaryValues._1).toDouble	1✔
240	BigDecimal(normalizedValue).setScale(4, BigDecimal.RoundingMode.HALF_UP).toDouble	1✔
241	}
242
243	private def suggestions(word: String): List[String] = {
244	val intersectedPossibilities = allWords.intersect({	1✔
245	val base =
246	Utilities.reductions(word, $(reductLimit)) ++	1✔
247	Utilities.getVowelSwaps(word, $(vowelSwapLimit)) ++	1✔
248	Utilities.variants(word) ++	1✔
249	both(word)	1✔
250	if ($(doubleVariants)) base ++ computeDoubleVariants(word) else base	×
251	}.toSet)	1✔
252	if (intersectedPossibilities.nonEmpty) intersectedPossibilities.toList	1✔
253	else List.empty[String]	1✔
254	}
255
256	private def both(word: String): List[String] = {
257	Utilities
258	.reductions(word, $(reductLimit))	1✔
259	.flatMap(reduction => Utilities.getVowelSwaps(reduction, $(vowelSwapLimit)))	1✔
260	}
261
262	def getSortedWordsByFrequency(words: List[String], input: String): List[(String, Long)] = {
263	val filteredWords = words.withFilter(_.length >= input.length)	1✔
264	val sortedWordsByFrequency = filteredWords
265	.map(word => (word, compareFrequencies(word)))	1✔
266	.sortWith(_._2 > _._2)	1✔
267	.take($(intersections))	1✔
268	logger.debug(s"recommended by frequency: ${sortedWordsByFrequency.mkString(", ")}")	1✔
269	sortedWordsByFrequency
270	}
271
272	private def compareFrequencies(value: String): Long =
273	Utilities.getFrequency(value, $$(wordCount))	1✔
274
275	def getSortedWordsByHamming(words: List[String], input: String): List[(String, Long)] = {
276	val sortedWordByHamming = words
277	.map(word => (word, compareHammers(input)(word)))	1✔
278	.sortBy(_._2)	1✔
279	.takeRight($(intersections))	1✔
280	logger.debug(s"recommended by hamming: ${sortedWordByHamming.mkString(", ")}")	1✔
281	sortedWordByHamming
282	}
283
284	private def compareHammers(input: String)(value: String): Long =
285	Utilities.computeHammingDistance(input, value)	1✔
286
287	def getResult(
288	wordsByFrequency: List[(String, Long)],
289	wordsByHamming: List[(String, Long)],
290	input: String): (String, Double) = {
291	var recommendation: (Option[String], Double) = (None, 0)	1✔
292	val intersectWords =
293	wordsByFrequency.map(word => word._1).intersect(wordsByHamming.map(word => word._1))	1✔
294	if (wordsByFrequency.isEmpty && wordsByHamming.isEmpty) {	1✔
295	logger.debug("no intersection or frequent words found")	1✔
296	recommendation = (Some(input), 0)	1✔
297	} else if (wordsByFrequency.isEmpty \|\| wordsByHamming.isEmpty) {	1✔
298	logger.debug("no intersection but one recommendation found")	1✔
299	recommendation = getRecommendation(wordsByFrequency, wordsByHamming)	1✔
300	} else if (intersectWords.nonEmpty) {	1✔
301	logger.debug("hammer and frequency recommendations found")	1✔
302	val frequencyAndHammingRecommendation =
303	getFrequencyAndHammingRecommendation(wordsByFrequency, wordsByHamming, intersectWords)	1✔
304	recommendation =
305	(frequencyAndHammingRecommendation._1, frequencyAndHammingRecommendation._2)	1✔
306	} else {	1✔
307	logger.debug("no intersection of hammer and frequency")	1✔
308	recommendation =
309	getFrequencyOrHammingRecommendation(wordsByFrequency, wordsByHamming, input)	1✔
310	}
311	(recommendation._1.getOrElse(input), recommendation._2)	1✔
312	}
313
314	private def getRecommendation(
315	wordsByFrequency: List[(String, Long)],
316	wordsByHamming: List[(String, Long)]) = {
317	if (wordsByFrequency.nonEmpty) {	1✔
318	getResultByFrequency(wordsByFrequency)	×
319	} else {
320	getResultByHamming(wordsByHamming)	1✔
321	}
322	}
323
324	private def getFrequencyAndHammingRecommendation(
325	wordsByFrequency: List[(String, Long)],
326	wordsByHamming: List[(String, Long)],
327	intersectWords: List[String]): (Option[String], Double) = {
328	val wordsByFrequencyAndHamming = intersectWords.map { word =>	1✔
329	val frequency = wordsByFrequency.find(_._1 == word).get._2	1✔
330	val hamming = wordsByHamming.find(_._1 == word).get._2	1✔
331	(word, frequency, hamming)	1✔
332	}
333	val bestFrequencyValue = wordsByFrequencyAndHamming.maxBy(_._2)._2	1✔
334	val bestHammingValue = wordsByFrequencyAndHamming.minBy(_._3)._3	1✔
335	val bestRecommendations = wordsByFrequencyAndHamming.filter(word =>	1✔
336	word._2 == bestFrequencyValue && word._3 == bestHammingValue)	1✔
337	if (bestRecommendations.nonEmpty) {	1✔
338	val result = (	1✔
339	Utilities.getRandomValueFromList(bestRecommendations),	1✔
340	Utilities.computeConfidenceValue(bestRecommendations))	1✔
341	(Some(result._1.get._1), result._2)	1✔
342	} else {
343	if ($(frequencyPriority)) {	1✔
344	(Some(wordsByFrequencyAndHamming.sortBy(_._3).maxBy(_._2)._1), 1.toDouble)	1✔
345	} else {
346	(Some(wordsByFrequencyAndHamming.sortBy(_._2).reverse.minBy(_._3)._1), 1.toDouble)	1✔
347	}
348	}
349	}
350
351	def getResultByFrequency(wordsByFrequency: List[(String, Long)]): (Option[String], Double) = {
352	val bestFrequencyValue = wordsByFrequency.maxBy(_._2)._2	1✔
353	val bestRecommendations = wordsByFrequency.filter(_._2 == bestFrequencyValue).map(_._1)	1✔
354	(	1✔
355	Utilities.getRandomValueFromList(bestRecommendations),	1✔
356	Utilities.computeConfidenceValue(bestRecommendations))	1✔
357	}
358
359	def getResultByHamming(wordsByHamming: List[(String, Long)]): (Option[String], Double) = {
360	val bestHammingValue = wordsByHamming.minBy(_._2)._2	1✔
361	val bestRecommendations = wordsByHamming.filter(_._2 == bestHammingValue).map(_._1)	1✔
362	(	1✔
363	Utilities.getRandomValueFromList(bestRecommendations),	1✔
364	Utilities.computeConfidenceValue(bestRecommendations))	1✔
365	}
366
367	def getFrequencyOrHammingRecommendation(
368	wordsByFrequency: List[(String, Long)],
369	wordsByHamming: List[(String, Long)],
370	input: String): (Option[String], Double) = {
371	val frequencyResult: String = getResultByFrequency(wordsByFrequency)._1.getOrElse(input)	1✔
372	val hammingResult: String = getResultByHamming(wordsByHamming)._1.getOrElse(input)	1✔
373	var result = List(frequencyResult, hammingResult)	1✔
374	if (frequencyResult == input) {	1✔
375	result = List(hammingResult)	×
376	} else if (hammingResult == input) {	1✔
377	result = List(frequencyResult)	×
378	}
379
380	(Utilities.getRandomValueFromList(result), Utilities.computeConfidenceValue(result))	1✔
381	}
382
383	}
384
385	trait ReadablePretrainedNorvig
386	extends ParamsAndFeaturesReadable[NorvigSweetingModel]
387	with HasPretrained[NorvigSweetingModel] {
388	override val defaultModelName = Some("spellcheck_norvig")	1✔
389
390	/** Java compliant-overrides */
391	override def pretrained(): NorvigSweetingModel = super.pretrained()	×
392
393	override def pretrained(name: String): NorvigSweetingModel = super.pretrained(name)	×
394
395	override def pretrained(name: String, lang: String): NorvigSweetingModel =
396	super.pretrained(name, lang)	×
397
398	override def pretrained(name: String, lang: String, remoteLoc: String): NorvigSweetingModel =
399	super.pretrained(name, lang, remoteLoc)	×
400	}
401
402	/** This is the companion object of [[NorvigSweetingModel]]. Please refer to that class for the
403	* documentation.
404	*/
405	object NorvigSweetingModel extends ReadablePretrainedNorvig

JohnSnowLabs / spark-nlp / 2144239727

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous