3136088427

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/21b3902df0e1ed57a1551255dfd5458339972d9b">21b3902df">&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/</a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/21b3902df0e1ed57a1551255dfd5458339972d9b">21b3902df</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/21b3902df0e1ed57a1551255dfd5458339972d9b">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/21b3902df0e1ed57a1551255dfd5458339972d9b&quot;&gt;21b3902df&lt;/a&gt;&lt;a href=&q... (continued)

Pull Request Pull Request #12763: Release/420 release candidate

Run Details

1045 of 1045 new or added lines in 30 files covered. (100.0%)

8692 of 12537 relevant lines covered (69.33%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/nlp/annotators/ner/NerOverwriter.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators.ner

import com.johnsnowlabs.nlp.serialization.MapFeature
import com.johnsnowlabs.nlp.{Annotation, AnnotatorModel, HasSimpleAnnotate}
import org.apache.spark.ml.param.{Param, StringArrayParam}
import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}

import java.util
import scala.collection.JavaConverters._

/** Overwrites entities of specified strings.
  *
  * The input for this Annotator have to be entities that are already extracted, Annotator type
  * `NAMED_ENTITY`. The strings specified with `setStopWords` will have new entities assigned to,
  * specified with `setNewResult`.
  *
  * ==Example==
  * {{{
  * import spark.implicits._
  * import com.johnsnowlabs.nlp.base.DocumentAssembler
  * import com.johnsnowlabs.nlp.annotators.Tokenizer
  * import com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetector
  * import com.johnsnowlabs.nlp.embeddings.WordEmbeddingsModel
  * import com.johnsnowlabs.nlp.annotators.ner.dl.NerDLModel
  * import com.johnsnowlabs.nlp.annotators.ner.NerOverwriter
  * import org.apache.spark.ml.Pipeline
  *
  * // First extract the prerequisite Entities
  * val documentAssembler = new DocumentAssembler()
  *   .setInputCol("text")
  *   .setOutputCol("document")
  *
  * val sentence = new SentenceDetector()
  *   .setInputCols("document")
  *   .setOutputCol("sentence")
  *
  * val tokenizer = new Tokenizer()
  *   .setInputCols("sentence")
  *   .setOutputCol("token")
  *
  * val embeddings = WordEmbeddingsModel.pretrained()
  *   .setInputCols("sentence", "token")
  *   .setOutputCol("embeddings")
  *
  * val nerTagger = NerDLModel.pretrained()
  *   .setInputCols("sentence", "token", "embeddings")
  *   .setOutputCol("ner")
  *
  * val pipeline = new Pipeline().setStages(Array(
  *   documentAssembler,
  *   sentence,
  *   tokenizer,
  *   embeddings,
  *   nerTagger
  * ))
  *
  * val data = Seq("Spark NLP Crosses Five Million Downloads, John Snow Labs Announces.").toDF("text")
  * val result = pipeline.fit(data).transform(data)
  *
  * result.selectExpr("explode(ner)").show(false)
  * /*
  * +------------------------------------------------------+
  * |col                                                   |
  * +------------------------------------------------------+
  * |[named_entity, 0, 4, B-ORG, [word -> Spark], []]      |
  * |[named_entity, 6, 8, I-ORG, [word -> NLP], []]        |
  * |[named_entity, 10, 16, O, [word -> Crosses], []]      |
  * |[named_entity, 18, 21, O, [word -> Five], []]         |
  * |[named_entity, 23, 29, O, [word -> Million], []]      |
  * |[named_entity, 31, 39, O, [word -> Downloads], []]    |
  * |[named_entity, 40, 40, O, [word -> ,], []]            |
  * |[named_entity, 42, 45, B-ORG, [word -> John], []]     |
  * |[named_entity, 47, 50, I-ORG, [word -> Snow], []]     |
  * |[named_entity, 52, 55, I-ORG, [word -> Labs], []]     |
  * |[named_entity, 57, 65, I-ORG, [word -> Announces], []]|
  * |[named_entity, 66, 66, O, [word -> .], []]            |
  * +------------------------------------------------------+
  * */
  * // The recognized entities can then be overwritten
  * val nerOverwriter = new NerOverwriter()
  *   .setInputCols("ner")
  *   .setOutputCol("ner_overwritten")
  *   .setNerWords(Array("Million"))
  *   .setNerNewEntity,("B-CARDINAL")
  *
  * nerOverwriter.transform(result).selectExpr("explode(ner_overwritten)").show(false)
  * +---------------------------------------------------------+
  * |col                                                      |
  * +---------------------------------------------------------+
  * |[named_entity, 0, 4, B-ORG, [word -> Spark], []]         |
  * |[named_entity, 6, 8, I-ORG, [word -> NLP], []]           |
  * |[named_entity, 10, 16, O, [word -> Crosses], []]         |
  * |[named_entity, 18, 21, O, [word -> Five], []]            |
  * |[named_entity, 23, 29, B-CARDINAL, [word -> Million], []]|
  * |[named_entity, 31, 39, O, [word -> Downloads], []]       |
  * |[named_entity, 40, 40, O, [word -> ,], []]               |
  * |[named_entity, 42, 45, B-ORG, [word -> John], []]        |
  * |[named_entity, 47, 50, I-ORG, [word -> Snow], []]        |
  * |[named_entity, 52, 55, I-ORG, [word -> Labs], []]        |
  * |[named_entity, 57, 65, I-ORG, [word -> Announces], []]   |
  * |[named_entity, 66, 66, O, [word -> .], []]               |
  * +---------------------------------------------------------+
  * }}}
  *
  * @groupname anno Annotator types
  * @groupdesc anno
  *   Required input and expected output annotator types
  * @groupname Ungrouped Members
  * @groupname param Parameters
  * @groupname setParam Parameter setters
  * @groupname getParam Parameter getters
  * @groupname Ungrouped Members
  * @groupprio param  1
  * @groupprio anno  2
  * @groupprio Ungrouped 3
  * @groupprio setParam  4
  * @groupprio getParam  5
  * @groupdesc param
  *   A list of (hyper-)parameter keys this annotator can take. Users can set and get the
  *   parameter values through setters and getters, respectively.
  */
class NerOverwriter(override val uid: String)
    extends AnnotatorModel[NerOverwriter]
    with HasSimpleAnnotate[NerOverwriter] {

  import com.johnsnowlabs.nlp.AnnotatorType.NAMED_ENTITY

  /** Output Annotator Type : NAMED_ENTITY
    *
    * @group anno
    */
  override val outputAnnotatorType: AnnotatorType = NAMED_ENTITY

  /** Input Annotator Type : NAMED_ENTITY
    *
    * @group anno
    */
  override val inputAnnotatorTypes: Array[AnnotatorType] = Array(NAMED_ENTITY)

  def this() = this(Identifiable.randomUID("NER_OVERWRITER"))

  /** The words to be filtered out.
    *
    * @group param
    */
  val nerWords: StringArrayParam =
    new StringArrayParam(
      this,
      "nerWords",
      "The words to be filtered out to replace the current NER entity with a new NER Entity")

  /** The words to be filtered out.
    *
    * @group setParam
    */
  def setNerWords(value: Array[String]): this.type = set(nerWords, value)

  /** The words to be filtered out.
    *
    * @group getParam
    */
  def getNerWords: Array[String] = $(nerWords)

  /** New NER class to overwrite
    *
    * @group param
    */
  val newNerEntity: Param[String] = new Param(this, "newNerEntity", "New NER label to overwrite")

  /** New NER class to overwrite
    *
    * @group setParam
    */
  def setNewNerEntity(r: String): this.type = {
    set(newNerEntity, r)
  }

  /** New NER class to overwrite
    *
    * @group getParam
    */
  def getNewNerEntity: String = $(newNerEntity)

  val replaceEntities: MapFeature[String, String] =
    new MapFeature[String, String](this, "replaceEntities")

  def setReplaceEntities(w: Map[String, String]): this.type = set(replaceEntities, w)

  // for Python access
  /** @group setParam */
  def setReplaceEntities(w: util.HashMap[String, String]): this.type = {

    val ws = w.asScala.toMap
    set(replaceEntities, ws)
  }

  def getReplaceEntities: Map[String, String] = {
    if (!replaceEntities.isSet) {
      Map.empty[String, String]
    } else {
      $$(replaceEntities)
    }
  }

  setDefault(newNerEntity -> "I-OVERWRITE", nerWords -> Array())

  override def annotate(annotations: Seq[Annotation]): Seq[Annotation] = {

    val annotationsOverwritten = annotations
    val replace = getReplaceEntities
    annotationsOverwritten
      .map { tokenAnnotation =>
        val nerWordsSet = $(nerWords).toSet
        if (nerWordsSet.contains(tokenAnnotation.metadata("word"))) {
          Annotation(
            outputAnnotatorType,
            tokenAnnotation.begin,
            tokenAnnotation.end,
            $(newNerEntity),
            tokenAnnotation.metadata)
        } else {
          Annotation(
            outputAnnotatorType,
            tokenAnnotation.begin,
            tokenAnnotation.end,
            tokenAnnotation.result,
            tokenAnnotation.metadata)
        }
      }
      .map { ann =>
        ann.copy(result = replace.getOrElse(ann.result, ann.result))
      }

  }

}

/** This is the companion object of [[NerOverwriter]]. Please refer to that class for the
  * documentation.
  */
object NerOverwriter extends DefaultParamsReadable[NerOverwriter]

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.annotators.ner
18
19	import com.johnsnowlabs.nlp.serialization.MapFeature
20	import com.johnsnowlabs.nlp.{Annotation, AnnotatorModel, HasSimpleAnnotate}
21	import org.apache.spark.ml.param.{Param, StringArrayParam}
22	import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}
23
24	import java.util
25	import scala.collection.JavaConverters._
26
27	/** Overwrites entities of specified strings.
28	*
29	* The input for this Annotator have to be entities that are already extracted, Annotator type
30	* `NAMED_ENTITY`. The strings specified with `setStopWords` will have new entities assigned to,
31	* specified with `setNewResult`.
32	*
33	* ==Example==
34	* {{{
35	* import spark.implicits._
36	* import com.johnsnowlabs.nlp.base.DocumentAssembler
37	* import com.johnsnowlabs.nlp.annotators.Tokenizer
38	* import com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetector
39	* import com.johnsnowlabs.nlp.embeddings.WordEmbeddingsModel
40	* import com.johnsnowlabs.nlp.annotators.ner.dl.NerDLModel
41	* import com.johnsnowlabs.nlp.annotators.ner.NerOverwriter
42	* import org.apache.spark.ml.Pipeline
43	*
44	* // First extract the prerequisite Entities
45	* val documentAssembler = new DocumentAssembler()
46	* .setInputCol("text")
47	* .setOutputCol("document")
48	*
49	* val sentence = new SentenceDetector()
50	* .setInputCols("document")
51	* .setOutputCol("sentence")
52	*
53	* val tokenizer = new Tokenizer()
54	* .setInputCols("sentence")
55	* .setOutputCol("token")
56	*
57	* val embeddings = WordEmbeddingsModel.pretrained()
58	* .setInputCols("sentence", "token")
59	* .setOutputCol("embeddings")
60	*
61	* val nerTagger = NerDLModel.pretrained()
62	* .setInputCols("sentence", "token", "embeddings")
63	* .setOutputCol("ner")
64	*
65	* val pipeline = new Pipeline().setStages(Array(
66	* documentAssembler,
67	* sentence,
68	* tokenizer,
69	* embeddings,
70	* nerTagger
71	* ))
72	*
73	* val data = Seq("Spark NLP Crosses Five Million Downloads, John Snow Labs Announces.").toDF("text")
74	* val result = pipeline.fit(data).transform(data)
75	*
76	* result.selectExpr("explode(ner)").show(false)
77	* /*
78	* +------------------------------------------------------+
79	* \|col \|
80	* +------------------------------------------------------+
81	* \|[named_entity, 0, 4, B-ORG, [word -> Spark], []] \|
82	* \|[named_entity, 6, 8, I-ORG, [word -> NLP], []] \|
83	* \|[named_entity, 10, 16, O, [word -> Crosses], []] \|
84	* \|[named_entity, 18, 21, O, [word -> Five], []] \|
85	* \|[named_entity, 23, 29, O, [word -> Million], []] \|
86	* \|[named_entity, 31, 39, O, [word -> Downloads], []] \|
87	* \|[named_entity, 40, 40, O, [word -> ,], []] \|
88	* \|[named_entity, 42, 45, B-ORG, [word -> John], []] \|
89	* \|[named_entity, 47, 50, I-ORG, [word -> Snow], []] \|
90	* \|[named_entity, 52, 55, I-ORG, [word -> Labs], []] \|
91	* \|[named_entity, 57, 65, I-ORG, [word -> Announces], []]\|
92	* \|[named_entity, 66, 66, O, [word -> .], []] \|
93	* +------------------------------------------------------+
94	* */
95	* // The recognized entities can then be overwritten
96	* val nerOverwriter = new NerOverwriter()
97	* .setInputCols("ner")
98	* .setOutputCol("ner_overwritten")
99	* .setNerWords(Array("Million"))
100	* .setNerNewEntity,("B-CARDINAL")
101	*
102	* nerOverwriter.transform(result).selectExpr("explode(ner_overwritten)").show(false)
103	* +---------------------------------------------------------+
104	* \|col \|
105	* +---------------------------------------------------------+
106	* \|[named_entity, 0, 4, B-ORG, [word -> Spark], []] \|
107	* \|[named_entity, 6, 8, I-ORG, [word -> NLP], []] \|
108	* \|[named_entity, 10, 16, O, [word -> Crosses], []] \|
109	* \|[named_entity, 18, 21, O, [word -> Five], []] \|
110	* \|[named_entity, 23, 29, B-CARDINAL, [word -> Million], []]\|
111	* \|[named_entity, 31, 39, O, [word -> Downloads], []] \|
112	* \|[named_entity, 40, 40, O, [word -> ,], []] \|
113	* \|[named_entity, 42, 45, B-ORG, [word -> John], []] \|
114	* \|[named_entity, 47, 50, I-ORG, [word -> Snow], []] \|
115	* \|[named_entity, 52, 55, I-ORG, [word -> Labs], []] \|
116	* \|[named_entity, 57, 65, I-ORG, [word -> Announces], []] \|
117	* \|[named_entity, 66, 66, O, [word -> .], []] \|
118	* +---------------------------------------------------------+
119	* }}}
120	*
121	* @groupname anno Annotator types
122	* @groupdesc anno
123	* Required input and expected output annotator types
124	* @groupname Ungrouped Members
125	* @groupname param Parameters
126	* @groupname setParam Parameter setters
127	* @groupname getParam Parameter getters
128	* @groupname Ungrouped Members
129	* @groupprio param 1
130	* @groupprio anno 2
131	* @groupprio Ungrouped 3
132	* @groupprio setParam 4
133	* @groupprio getParam 5
134	* @groupdesc param
135	* A list of (hyper-)parameter keys this annotator can take. Users can set and get the
136	* parameter values through setters and getters, respectively.
137	*/
138	class NerOverwriter(override val uid: String)
139	extends AnnotatorModel[NerOverwriter]
140	with HasSimpleAnnotate[NerOverwriter] {
141
142	import com.johnsnowlabs.nlp.AnnotatorType.NAMED_ENTITY
143
144	/** Output Annotator Type : NAMED_ENTITY
145	*
146	* @group anno
147	*/
148	override val outputAnnotatorType: AnnotatorType = NAMED_ENTITY	×
149
150	/** Input Annotator Type : NAMED_ENTITY
151	*
152	* @group anno
153	*/
154	override val inputAnnotatorTypes: Array[AnnotatorType] = Array(NAMED_ENTITY)	×
155
156	def this() = this(Identifiable.randomUID("NER_OVERWRITER"))	×
157
158	/** The words to be filtered out.
159	*
160	* @group param
161	*/
162	val nerWords: StringArrayParam =
163	new StringArrayParam(	×
164	this,
165	"nerWords",	×
166	"The words to be filtered out to replace the current NER entity with a new NER Entity")	×
167
168	/** The words to be filtered out.
169	*
170	* @group setParam
171	*/
172	def setNerWords(value: Array[String]): this.type = set(nerWords, value)	×
173
174	/** The words to be filtered out.
175	*
176	* @group getParam
177	*/
178	def getNerWords: Array[String] = $(nerWords)	×
179
180	/** New NER class to overwrite
181	*
182	* @group param
183	*/
184	val newNerEntity: Param[String] = new Param(this, "newNerEntity", "New NER label to overwrite")	×
185
186	/** New NER class to overwrite
187	*
188	* @group setParam
189	*/
190	def setNewNerEntity(r: String): this.type = {
191	set(newNerEntity, r)	×
192	}
193
194	/** New NER class to overwrite
195	*
196	* @group getParam
197	*/
198	def getNewNerEntity: String = $(newNerEntity)	×
199
200	val replaceEntities: MapFeature[String, String] =
201	new MapFeature[String, String](this, "replaceEntities")	×
202
203	def setReplaceEntities(w: Map[String, String]): this.type = set(replaceEntities, w)	×
204
205	// for Python access
206	/** @group setParam */
207	def setReplaceEntities(w: util.HashMap[String, String]): this.type = {
208
209	val ws = w.asScala.toMap	×
210	set(replaceEntities, ws)	×
211	}
212
213	def getReplaceEntities: Map[String, String] = {
214	if (!replaceEntities.isSet) {	×
215	Map.empty[String, String]	×
216	} else {
217	$$(replaceEntities)	×
218	}
219	}
220
221	setDefault(newNerEntity -> "I-OVERWRITE", nerWords -> Array())	×
222
223	override def annotate(annotations: Seq[Annotation]): Seq[Annotation] = {
224
225	val annotationsOverwritten = annotations
226	val replace = getReplaceEntities	×
227	annotationsOverwritten
228	.map { tokenAnnotation =>	×
229	val nerWordsSet = $(nerWords).toSet	×
230	if (nerWordsSet.contains(tokenAnnotation.metadata("word"))) {	×
231	Annotation(	×
232	outputAnnotatorType,	×
233	tokenAnnotation.begin,	×
234	tokenAnnotation.end,	×
235	$(newNerEntity),	×
236	tokenAnnotation.metadata)	×
237	} else {
238	Annotation(	×
239	outputAnnotatorType,	×
240	tokenAnnotation.begin,	×
241	tokenAnnotation.end,	×
242	tokenAnnotation.result,	×
243	tokenAnnotation.metadata)	×
244	}
245	}
246	.map { ann =>	×
247	ann.copy(result = replace.getOrElse(ann.result, ann.result))	×
248	}
249
250	}
251
252	}
253
254	/** This is the companion object of [[NerOverwriter]]. Please refer to that class for the
255	* documentation.
256	*/
257	object NerOverwriter extends DefaultParamsReadable[NerOverwriter]

JohnSnowLabs / spark-nlp / 3136088427

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous