3119159396

Build # 3119159396

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/eb046fa5090358bacfd0416eed6621f9469e6295">eb046fa50">&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/</a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/eb046fa5090358bacfd0416eed6621f9469e6295">eb046fa50</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/eb046fa5090358bacfd0416eed6621f9469e6295">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/eb046fa5090358bacfd0416eed6621f9469e6295&quot;&gt;eb046fa50&lt;/a&gt;&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/eb046fa5090358bacfd0416eed6621f9469e6295&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp... (continued)

Pull Request Pull Request #12763: Release/420 release candidate

Run Details

1045 of 1045 new or added lines in 30 files covered. (100.0%)

8692 of 12537 relevant lines covered (69.33%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.98

/src/main/scala/com/johnsnowlabs/nlp/annotators/er/EntityRulerModel.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators.er

import com.johnsnowlabs.nlp.AnnotatorType.{CHUNK, DOCUMENT, TOKEN}
import com.johnsnowlabs.nlp.annotators.common._
import com.johnsnowlabs.nlp.serialization.StructFeature
import com.johnsnowlabs.nlp.{Annotation, AnnotatorModel, HasPretrained, HasSimpleAnnotate}
import com.johnsnowlabs.storage.Database.{ENTITY_PATTERNS, ENTITY_REGEX_PATTERNS, Name}
import com.johnsnowlabs.storage._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.ml.PipelineModel
import org.apache.spark.ml.param.{BooleanParam, StringArrayParam}
import org.apache.spark.ml.util.Identifiable
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
import org.slf4j.{Logger, LoggerFactory}

/** Instantiated model of the [[EntityRulerApproach]]. For usage and examples see the
  * documentation of the main class.
  *
  * @param uid
  *   internally renquired UID to make it writable
  * @groupname anno Annotator types
  * @groupdesc anno
  *   Required input and expected output annotator types
  * @groupname Ungrouped Members
  * @groupname param Parameters
  * @groupname setParam Parameter setters
  * @groupname getParam Parameter getters
  * @groupname Ungrouped Members
  * @groupprio param  1
  * @groupprio anno  2
  * @groupprio Ungrouped 3
  * @groupprio setParam  4
  * @groupprio getParam  5
  * @groupdesc param
  *   A list of (hyper-)parameter keys this annotator can take. Users can set and get the
  *   parameter values through setters and getters, respectively.
  */
class EntityRulerModel(override val uid: String)
    extends AnnotatorModel[EntityRulerModel]
    with HasSimpleAnnotate[EntityRulerModel]
    with HasStorageModel {

  def this() = this(Identifiable.randomUID("ENTITY_RULER"))

  private val logger: Logger = LoggerFactory.getLogger("Credentials")

  @deprecated("Enabling pattern regex now is define on each pattern", "Since 4.2.0")
  private[er] val enablePatternRegex =
    new BooleanParam(this, "enablePatternRegex", "Enables regex pattern match")

  private[er] val useStorage =
    new BooleanParam(this, "useStorage", "Whether to use RocksDB storage to serialize patterns")

  private[er] val regexEntities =
    new StringArrayParam(this, "regexEntities", "entities defined in regex patterns")

  private[er] val entityRulerFeatures: StructFeature[EntityRulerFeatures] =
    new StructFeature[EntityRulerFeatures](
      this,
      "Structure to store data when RocksDB is not used")

  private[er] val sentenceMatch = new BooleanParam(
    this,
    "sentenceMatch",
    "Whether to find match at sentence level (regex only). True: sentence level. False: token level")

  private[er] val ahoCorasickAutomaton: StructFeature[Option[AhoCorasickAutomaton]] =
    new StructFeature[Option[AhoCorasickAutomaton]](this, "AhoCorasickAutomaton")

  @deprecated("Enabling pattern regex now is define on each pattern", "Since 4.2.0")
  private[er] def setEnablePatternRegex(value: Boolean): this.type =
    set(enablePatternRegex, value)

  private[er] def setRegexEntities(value: Array[String]): this.type = set(regexEntities, value)

  private[er] def setEntityRulerFeatures(value: EntityRulerFeatures): this.type =
    set(entityRulerFeatures, value)

  private[er] def setUseStorage(value: Boolean): this.type = set(useStorage, value)

  private[er] def setSentenceMatch(value: Boolean): this.type = set(sentenceMatch, value)

  private[er] def setAhoCorasickAutomaton(value: Option[AhoCorasickAutomaton]): this.type =
    set(ahoCorasickAutomaton, value)

  private var automatonModel: Option[Broadcast[AhoCorasickAutomaton]] = None

  def setAutomatonModelIfNotSet(
      spark: SparkSession,
      automaton: Option[AhoCorasickAutomaton]): this.type = {
    if (automatonModel.isEmpty && automaton.isDefined) {
      automatonModel = Some(spark.sparkContext.broadcast(automaton.get))
    }
    this
  }

  def getAutomatonModelIfNotSet: Option[AhoCorasickAutomaton] = {
    if (automatonModel.isDefined) {
      Some(automatonModel.get.value)
    } else {
      if ($$(ahoCorasickAutomaton).isDefined) $$(ahoCorasickAutomaton) else None
    }
  }

  setDefault(useStorage -> false, caseSensitive -> true)

  /** Annotator reference id. Used to identify elements in metadata or to refer to this annotator
    * type
    */
  val inputAnnotatorTypes: Array[String] = Array(DOCUMENT)
  override val optionalInputAnnotatorTypes: Array[String] = Array(TOKEN)
  val outputAnnotatorType: AnnotatorType = CHUNK

  override def _transform(
      dataset: Dataset[_],
      recursivePipeline: Option[PipelineModel]): DataFrame = {
    if ($(regexEntities).nonEmpty) {
      val structFields = dataset.schema.fields
        .filter(field => field.metadata.contains("annotatorType"))
        .filter(field => field.metadata.getString("annotatorType") == TOKEN)
      if (structFields.isEmpty) {
        throw new IllegalArgumentException(
          s"Missing $TOKEN annotator. Regex patterns requires it in your pipeline")
      } else {
        super._transform(dataset, recursivePipeline)
      }
    } else {
      super._transform(dataset, recursivePipeline)
    }
  }

  override def beforeAnnotate(dataset: Dataset[_]): Dataset[_] = {
    this.setAutomatonModelIfNotSet(dataset.sparkSession, $$(ahoCorasickAutomaton))
    dataset
  }

  /** takes a document and annotations and produces new annotations of this annotator's annotation
    * type
    *
    * @param annotations
    *   Annotations that correspond to inputAnnotationCols generated by previous annotators if any
    * @return
    *   any number of annotations processed for every input annotation. Not necessary one to one
    *   relationship
    */
  def annotate(annotations: Seq[Annotation]): Seq[Annotation] = {
    var annotatedEntitiesByKeywords: Seq[Annotation] = Seq()
    val sentences = SentenceSplit.unpack(annotations)
    val annotatedEntitiesByRegex = computeAnnotatedEntitiesByRegex(annotations, sentences)

    if (getAutomatonModelIfNotSet.isDefined) {
      annotatedEntitiesByKeywords = sentences.flatMap { sentence =>
        getAutomatonModelIfNotSet.get.searchPatternsInText(sentence)
      }
    }

    annotatedEntitiesByRegex ++ annotatedEntitiesByKeywords
  }

  private def computeAnnotatedEntitiesByRegex(
      annotations: Seq[Annotation],
      sentences: Seq[Sentence]): Seq[Annotation] = {
    if ($(regexEntities).nonEmpty) {
      val regexPatternsReader =
        if ($(useStorage))
          Some(getReader(Database.ENTITY_REGEX_PATTERNS).asInstanceOf[RegexPatternsReader])
        else None

      if ($(sentenceMatch)) {
        annotateEntitiesFromRegexPatternsBySentence(sentences, regexPatternsReader)
      } else {
        val tokenizedWithSentences = TokenizedWithSentence.unpack(annotations)
        annotateEntitiesFromRegexPatterns(tokenizedWithSentences, regexPatternsReader)
      }
    } else Seq()
  }

  private def annotateEntitiesFromRegexPatterns(
      tokenizedWithSentences: Seq[TokenizedSentence],
      regexPatternsReader: Option[RegexPatternsReader]): Seq[Annotation] = {

    val annotatedEntities = tokenizedWithSentences.flatMap { tokenizedWithSentence =>
      tokenizedWithSentence.indexedTokens.flatMap { indexedToken =>
        val entity = getMatchedEntity(indexedToken.token, regexPatternsReader)
        if (entity.isDefined) {
          val entityMetadata = getEntityMetadata(entity)
          Some(
            Annotation(
              CHUNK,
              indexedToken.begin,
              indexedToken.end,
              indexedToken.token,
              entityMetadata ++ Map("sentence" -> tokenizedWithSentence.sentenceIndex.toString)))
        } else None
      }
    }

    annotatedEntities
  }

  private def getMatchedEntity(
      token: String,
      regexPatternsReader: Option[RegexPatternsReader]): Option[String] = {

    val matchesByEntity = $(regexEntities).flatMap { regexEntity =>
      val regexPatterns: Option[Seq[String]] = regexPatternsReader match {
        case Some(rpr) => rpr.lookup(regexEntity)
        case None => $$(entityRulerFeatures).regexPatterns.get(regexEntity)
      }
      if (regexPatterns.isDefined) {
        val matches = regexPatterns.get.flatMap(regexPattern => regexPattern.r.findFirstIn(token))
        if (matches.nonEmpty) Some(regexEntity) else None
      } else None
    }.toSeq

    if (matchesByEntity.size > 1) {
      logger.warn("More than one entity found. Sending the first element of the array")
    }

    matchesByEntity.headOption
  }

  private def getMatchedEntityBySentence(
      sentence: Sentence,
      regexPatternsReader: Option[RegexPatternsReader]): Array[(IndexedToken, String)] = {

    val matchesByEntity = $(regexEntities)
      .flatMap { regexEntity =>
        val regexPatterns: Option[Seq[String]] = regexPatternsReader match {
          case Some(rpr) => rpr.lookup(regexEntity)
          case None => $$(entityRulerFeatures).regexPatterns.get(regexEntity)
        }
        if (regexPatterns.isDefined) {

          val resultMatches = regexPatterns.get.flatMap { regexPattern =>
            val matchedResult = regexPattern.r.findFirstMatchIn(sentence.content)
            if (matchedResult.isDefined) {
              val begin = matchedResult.get.start + sentence.start
              val end = matchedResult.get.end + sentence.start - 1
              Some(matchedResult.get.toString(), begin, end, regexEntity)
            } else None
          }

          val intervals =
            resultMatches.map(resultMatch => List(resultMatch._2, resultMatch._3)).toList
          val mergedIntervals = EntityRulerUtil.mergeIntervals(intervals)
          val filteredMatches =
            resultMatches.filter(x => mergedIntervals.contains(List(x._2, x._3)))

          if (filteredMatches.nonEmpty) Some(filteredMatches) else None
        } else None
      }
      .flatten
      .sortBy(_._2)

    matchesByEntity.map(matches => (IndexedToken(matches._1, matches._2, matches._3), matches._4))
  }

  private def annotateEntitiesFromRegexPatternsBySentence(
      sentences: Seq[Sentence],
      patternsReader: Option[RegexPatternsReader]): Seq[Annotation] = {

    val annotatedEntities = sentences.flatMap { sentence =>
      val matchedEntities = getMatchedEntityBySentence(sentence, patternsReader)
      matchedEntities.map { case (indexedToken, label) =>
        val entityMetadata = getEntityMetadata(Some(label))
        Annotation(
          CHUNK,
          indexedToken.begin,
          indexedToken.end,
          indexedToken.token,
          entityMetadata ++ Map("sentence" -> sentence.index.toString))
      }
    }
    annotatedEntities
  }

  private def getEntityMetadata(labelData: Option[String]): Map[String, String] = {

    val entityMetadata = labelData.get
      .split(",")
      .zipWithIndex
      .flatMap { case (metadata, index) =>
        if (index == 0) {
          Map("entity" -> metadata)
        } else Map("id" -> metadata)
      }
      .toMap

    entityMetadata
  }

  override def deserializeStorage(path: String, spark: SparkSession): Unit = {
    if ($(useStorage)) {
      super.deserializeStorage(path: String, spark: SparkSession)
    }
  }

  override def onWrite(path: String, spark: SparkSession): Unit = {
    if ($(useStorage)) {
      super.onWrite(path, spark)
    }
  }

  protected val databases: Array[Name] = EntityRulerModel.databases

  protected def createReader(database: Name, connection: RocksDBConnection): StorageReader[_] = {
    database match {
      case Database.ENTITY_PATTERNS => new PatternsReader(connection)
      case Database.ENTITY_REGEX_PATTERNS => new RegexPatternsReader(connection)
    }
  }
}

trait ReadablePretrainedEntityRuler
    extends StorageReadable[EntityRulerModel]
    with HasPretrained[EntityRulerModel] {

  override val databases: Array[Name] = Array(ENTITY_PATTERNS, ENTITY_REGEX_PATTERNS)

  override val defaultModelName: Option[String] = None

  override def pretrained(): EntityRulerModel = super.pretrained()

  override def pretrained(name: String): EntityRulerModel = super.pretrained(name)

  override def pretrained(name: String, lang: String): EntityRulerModel =
    super.pretrained(name, lang)

  override def pretrained(name: String, lang: String, remoteLoc: String): EntityRulerModel =
    super.pretrained(name, lang, remoteLoc)

}

object EntityRulerModel extends ReadablePretrainedEntityRuler

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.annotators.er
18
19	import com.johnsnowlabs.nlp.AnnotatorType.{CHUNK, DOCUMENT, TOKEN}
20	import com.johnsnowlabs.nlp.annotators.common._
21	import com.johnsnowlabs.nlp.serialization.StructFeature
22	import com.johnsnowlabs.nlp.{Annotation, AnnotatorModel, HasPretrained, HasSimpleAnnotate}
23	import com.johnsnowlabs.storage.Database.{ENTITY_PATTERNS, ENTITY_REGEX_PATTERNS, Name}
24	import com.johnsnowlabs.storage._
25	import org.apache.spark.broadcast.Broadcast
26	import org.apache.spark.ml.PipelineModel
27	import org.apache.spark.ml.param.{BooleanParam, StringArrayParam}
28	import org.apache.spark.ml.util.Identifiable
29	import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
30	import org.slf4j.{Logger, LoggerFactory}
31
32	/** Instantiated model of the [[EntityRulerApproach]]. For usage and examples see the
33	* documentation of the main class.
34	*
35	* @param uid
36	* internally renquired UID to make it writable
37	* @groupname anno Annotator types
38	* @groupdesc anno
39	* Required input and expected output annotator types
40	* @groupname Ungrouped Members
41	* @groupname param Parameters
42	* @groupname setParam Parameter setters
43	* @groupname getParam Parameter getters
44	* @groupname Ungrouped Members
45	* @groupprio param 1
46	* @groupprio anno 2
47	* @groupprio Ungrouped 3
48	* @groupprio setParam 4
49	* @groupprio getParam 5
50	* @groupdesc param
51	* A list of (hyper-)parameter keys this annotator can take. Users can set and get the
52	* parameter values through setters and getters, respectively.
53	*/
54	class EntityRulerModel(override val uid: String)
55	extends AnnotatorModel[EntityRulerModel]
56	with HasSimpleAnnotate[EntityRulerModel]
57	with HasStorageModel {
58
59	def this() = this(Identifiable.randomUID("ENTITY_RULER"))	1✔
60
61	private val logger: Logger = LoggerFactory.getLogger("Credentials")	1✔
62
63	@deprecated("Enabling pattern regex now is define on each pattern", "Since 4.2.0")
64	private[er] val enablePatternRegex =
65	new BooleanParam(this, "enablePatternRegex", "Enables regex pattern match")	1✔
66
67	private[er] val useStorage =
68	new BooleanParam(this, "useStorage", "Whether to use RocksDB storage to serialize patterns")	1✔
69
70	private[er] val regexEntities =
71	new StringArrayParam(this, "regexEntities", "entities defined in regex patterns")	1✔
72
73	private[er] val entityRulerFeatures: StructFeature[EntityRulerFeatures] =
74	new StructFeature[EntityRulerFeatures](	1✔
75	this,
76	"Structure to store data when RocksDB is not used")	1✔
77
78	private[er] val sentenceMatch = new BooleanParam(	1✔
79	this,
80	"sentenceMatch",	1✔
81	"Whether to find match at sentence level (regex only). True: sentence level. False: token level")	1✔
82
83	private[er] val ahoCorasickAutomaton: StructFeature[Option[AhoCorasickAutomaton]] =
84	new StructFeature[Option[AhoCorasickAutomaton]](this, "AhoCorasickAutomaton")	1✔
85
86	@deprecated("Enabling pattern regex now is define on each pattern", "Since 4.2.0")
87	private[er] def setEnablePatternRegex(value: Boolean): this.type =
88	set(enablePatternRegex, value)	×
89
90	private[er] def setRegexEntities(value: Array[String]): this.type = set(regexEntities, value)	1✔
91
92	private[er] def setEntityRulerFeatures(value: EntityRulerFeatures): this.type =
93	set(entityRulerFeatures, value)	1✔
94
95	private[er] def setUseStorage(value: Boolean): this.type = set(useStorage, value)	1✔
96
97	private[er] def setSentenceMatch(value: Boolean): this.type = set(sentenceMatch, value)	×
98
99	private[er] def setAhoCorasickAutomaton(value: Option[AhoCorasickAutomaton]): this.type =
100	set(ahoCorasickAutomaton, value)	1✔
101
102	private var automatonModel: Option[Broadcast[AhoCorasickAutomaton]] = None	1✔
103
104	def setAutomatonModelIfNotSet(
105	spark: SparkSession,
106	automaton: Option[AhoCorasickAutomaton]): this.type = {
107	if (automatonModel.isEmpty && automaton.isDefined) {	1✔
108	automatonModel = Some(spark.sparkContext.broadcast(automaton.get))	1✔
109	}
110	this
111	}
112
113	def getAutomatonModelIfNotSet: Option[AhoCorasickAutomaton] = {
114	if (automatonModel.isDefined) {	1✔
115	Some(automatonModel.get.value)	1✔
116	} else {
117	if ($$(ahoCorasickAutomaton).isDefined) $$(ahoCorasickAutomaton) else None	1✔
118	}
119	}
120
121	setDefault(useStorage -> false, caseSensitive -> true)	1✔
122
123	/** Annotator reference id. Used to identify elements in metadata or to refer to this annotator
124	* type
125	*/
126	val inputAnnotatorTypes: Array[String] = Array(DOCUMENT)	1✔
127	override val optionalInputAnnotatorTypes: Array[String] = Array(TOKEN)	1✔
128	val outputAnnotatorType: AnnotatorType = CHUNK	1✔
129
130	override def _transform(
131	dataset: Dataset[_],
132	recursivePipeline: Option[PipelineModel]): DataFrame = {
133	if ($(regexEntities).nonEmpty) {	1✔
134	val structFields = dataset.schema.fields	1✔
135	.filter(field => field.metadata.contains("annotatorType"))	1✔
136	.filter(field => field.metadata.getString("annotatorType") == TOKEN)	1✔
137	if (structFields.isEmpty) {	1✔
138	throw new IllegalArgumentException(	×
139	s"Missing $TOKEN annotator. Regex patterns requires it in your pipeline")
140	} else {
141	super._transform(dataset, recursivePipeline)	1✔
142	}
143	} else {
144	super._transform(dataset, recursivePipeline)	1✔
145	}
146	}
147
148	override def beforeAnnotate(dataset: Dataset[_]): Dataset[_] = {
149	this.setAutomatonModelIfNotSet(dataset.sparkSession, $$(ahoCorasickAutomaton))	1✔
150	dataset
151	}
152
153	/** takes a document and annotations and produces new annotations of this annotator's annotation
154	* type
155	*
156	* @param annotations
157	* Annotations that correspond to inputAnnotationCols generated by previous annotators if any
158	* @return
159	* any number of annotations processed for every input annotation. Not necessary one to one
160	* relationship
161	*/
162	def annotate(annotations: Seq[Annotation]): Seq[Annotation] = {
163	var annotatedEntitiesByKeywords: Seq[Annotation] = Seq()	1✔
164	val sentences = SentenceSplit.unpack(annotations)	1✔
165	val annotatedEntitiesByRegex = computeAnnotatedEntitiesByRegex(annotations, sentences)	1✔
166
167	if (getAutomatonModelIfNotSet.isDefined) {	1✔
168	annotatedEntitiesByKeywords = sentences.flatMap { sentence =>	1✔
169	getAutomatonModelIfNotSet.get.searchPatternsInText(sentence)	1✔
170	}
171	}
172
173	annotatedEntitiesByRegex ++ annotatedEntitiesByKeywords	1✔
174	}
175
176	private def computeAnnotatedEntitiesByRegex(
177	annotations: Seq[Annotation],
178	sentences: Seq[Sentence]): Seq[Annotation] = {
179	if ($(regexEntities).nonEmpty) {	1✔
180	val regexPatternsReader =
181	if ($(useStorage))	1✔
182	Some(getReader(Database.ENTITY_REGEX_PATTERNS).asInstanceOf[RegexPatternsReader])	1✔
183	else None	1✔
184
185	if ($(sentenceMatch)) {	1✔
186	annotateEntitiesFromRegexPatternsBySentence(sentences, regexPatternsReader)	1✔
187	} else {	1✔
188	val tokenizedWithSentences = TokenizedWithSentence.unpack(annotations)	1✔
189	annotateEntitiesFromRegexPatterns(tokenizedWithSentences, regexPatternsReader)	1✔
190	}
191	} else Seq()	1✔
192	}
193
194	private def annotateEntitiesFromRegexPatterns(
195	tokenizedWithSentences: Seq[TokenizedSentence],
196	regexPatternsReader: Option[RegexPatternsReader]): Seq[Annotation] = {
197
198	val annotatedEntities = tokenizedWithSentences.flatMap { tokenizedWithSentence =>	1✔
199	tokenizedWithSentence.indexedTokens.flatMap { indexedToken =>	1✔
200	val entity = getMatchedEntity(indexedToken.token, regexPatternsReader)	1✔
201	if (entity.isDefined) {	1✔
202	val entityMetadata = getEntityMetadata(entity)	1✔
203	Some(	1✔
204	Annotation(	1✔
205	CHUNK,	1✔
206	indexedToken.begin,	1✔
207	indexedToken.end,	1✔
208	indexedToken.token,	1✔
209	entityMetadata ++ Map("sentence" -> tokenizedWithSentence.sentenceIndex.toString)))	1✔
210	} else None	1✔
211	}
212	}
213
214	annotatedEntities
215	}
216
217	private def getMatchedEntity(
218	token: String,
219	regexPatternsReader: Option[RegexPatternsReader]): Option[String] = {
220
221	val matchesByEntity = $(regexEntities).flatMap { regexEntity =>	1✔
222	val regexPatterns: Option[Seq[String]] = regexPatternsReader match {
223	case Some(rpr) => rpr.lookup(regexEntity)	1✔
224	case None => $$(entityRulerFeatures).regexPatterns.get(regexEntity)	1✔
225	}
226	if (regexPatterns.isDefined) {	1✔
227	val matches = regexPatterns.get.flatMap(regexPattern => regexPattern.r.findFirstIn(token))	1✔
228	if (matches.nonEmpty) Some(regexEntity) else None	1✔
229	} else None	×
230	}.toSeq	1✔
231
232	if (matchesByEntity.size > 1) {	1✔
233	logger.warn("More than one entity found. Sending the first element of the array")	1✔
234	}
235
236	matchesByEntity.headOption	1✔
237	}
238
239	private def getMatchedEntityBySentence(
240	sentence: Sentence,
241	regexPatternsReader: Option[RegexPatternsReader]): Array[(IndexedToken, String)] = {
242
243	val matchesByEntity = $(regexEntities)	1✔
244	.flatMap { regexEntity =>	1✔
245	val regexPatterns: Option[Seq[String]] = regexPatternsReader match {
246	case Some(rpr) => rpr.lookup(regexEntity)	1✔
247	case None => $$(entityRulerFeatures).regexPatterns.get(regexEntity)	1✔
248	}
249	if (regexPatterns.isDefined) {	1✔
250
251	val resultMatches = regexPatterns.get.flatMap { regexPattern =>	1✔
252	val matchedResult = regexPattern.r.findFirstMatchIn(sentence.content)	1✔
253	if (matchedResult.isDefined) {	1✔
254	val begin = matchedResult.get.start + sentence.start	1✔
255	val end = matchedResult.get.end + sentence.start - 1	1✔
256	Some(matchedResult.get.toString(), begin, end, regexEntity)	1✔
257	} else None	×
258	}
259
260	val intervals =
261	resultMatches.map(resultMatch => List(resultMatch._2, resultMatch._3)).toList	1✔
262	val mergedIntervals = EntityRulerUtil.mergeIntervals(intervals)	1✔
263	val filteredMatches =
264	resultMatches.filter(x => mergedIntervals.contains(List(x._2, x._3)))	1✔
265
266	if (filteredMatches.nonEmpty) Some(filteredMatches) else None	1✔
267	} else None	×
268	}
269	.flatten	1✔
270	.sortBy(_._2)	×
271
272	matchesByEntity.map(matches => (IndexedToken(matches._1, matches._2, matches._3), matches._4))	1✔
273	}
274
275	private def annotateEntitiesFromRegexPatternsBySentence(
276	sentences: Seq[Sentence],
277	patternsReader: Option[RegexPatternsReader]): Seq[Annotation] = {
278
279	val annotatedEntities = sentences.flatMap { sentence =>	1✔
280	val matchedEntities = getMatchedEntityBySentence(sentence, patternsReader)	1✔
281	matchedEntities.map { case (indexedToken, label) =>	1✔
282	val entityMetadata = getEntityMetadata(Some(label))	1✔
283	Annotation(	1✔
284	CHUNK,	1✔
285	indexedToken.begin,	1✔
286	indexedToken.end,	1✔
287	indexedToken.token,	1✔
288	entityMetadata ++ Map("sentence" -> sentence.index.toString))	1✔
289	}
290	}
291	annotatedEntities
292	}
293
294	private def getEntityMetadata(labelData: Option[String]): Map[String, String] = {
295
296	val entityMetadata = labelData.get
297	.split(",")	1✔
298	.zipWithIndex	1✔
299	.flatMap { case (metadata, index) =>	1✔
300	if (index == 0) {	1✔
301	Map("entity" -> metadata)	1✔
302	} else Map("id" -> metadata)	1✔
303	}
304	.toMap	1✔
305
306	entityMetadata
307	}
308
309	override def deserializeStorage(path: String, spark: SparkSession): Unit = {
310	if ($(useStorage)) {	1✔
311	super.deserializeStorage(path: String, spark: SparkSession)	×
312	}
313	}
314
315	override def onWrite(path: String, spark: SparkSession): Unit = {
316	if ($(useStorage)) {	1✔
317	super.onWrite(path, spark)	×
318	}
319	}
320
321	protected val databases: Array[Name] = EntityRulerModel.databases	1✔
322
323	protected def createReader(database: Name, connection: RocksDBConnection): StorageReader[_] = {
324	database match {
325	case Database.ENTITY_PATTERNS => new PatternsReader(connection)	×
326	case Database.ENTITY_REGEX_PATTERNS => new RegexPatternsReader(connection)	1✔
327	}
328	}
329	}
330
331	trait ReadablePretrainedEntityRuler
332	extends StorageReadable[EntityRulerModel]
333	with HasPretrained[EntityRulerModel] {
334
335	override val databases: Array[Name] = Array(ENTITY_PATTERNS, ENTITY_REGEX_PATTERNS)	1✔
336
337	override val defaultModelName: Option[String] = None	1✔
338
339	override def pretrained(): EntityRulerModel = super.pretrained()	×
340
341	override def pretrained(name: String): EntityRulerModel = super.pretrained(name)	×
342
343	override def pretrained(name: String, lang: String): EntityRulerModel =
344	super.pretrained(name, lang)	×
345
346	override def pretrained(name: String, lang: String, remoteLoc: String): EntityRulerModel =
347	super.pretrained(name, lang, remoteLoc)	×
348
349	}
350
351	object EntityRulerModel extends ReadablePretrainedEntityRuler

JohnSnowLabs / spark-nlp / 3119159396

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous