3122721559

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/eb0f9d8d66f5ad5461d7c1e2db0ede0f560ea409">eb0f9d8d6">&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/</a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/eb0f9d8d66f5ad5461d7c1e2db0ede0f560ea409">eb0f9d8d6</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/eb0f9d8d66f5ad5461d7c1e2db0ede0f560ea409">&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/eb0f9d8d66f5ad5461d7c1e2db0ede0f560ea409&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/JohnSnowLabs/spark-nlp/commit/eb0f9d8d66f5ad5461d7c1e2db0ede0f560ea409&amp;quot;&amp;gt;eb0f9d8d6&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/eb0f9d8d66f5ad5461d7c1e2db0ede0f560ea409&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/eb0f9d8d66f5ad5461d7c1e2db0ede0f&amp;lt;/a&amp;gt;560ea409&amp;q</a>uot;&gt;&amp;quot;&amp;gt;Merge &amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/Joh... (continued)

Pull Request Pull Request #12763: Release/420 release candidate

Run Details

1045 of 1045 new or added lines in 30 files covered. (100.0%)

8691 of 12537 relevant lines covered (69.32%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.64

/src/main/scala/com/johnsnowlabs/nlp/annotators/ner/crf/NerCrfApproach.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators.ner.crf

import com.johnsnowlabs.ml.crf.{CrfParams, LinearChainCrf}
import com.johnsnowlabs.nlp.AnnotatorType._
import com.johnsnowlabs.nlp.annotators.common.NerTagged
import com.johnsnowlabs.nlp.annotators.ner.{NerApproach, Verbose}
import com.johnsnowlabs.nlp.annotators.param.ExternalResourceParam
import com.johnsnowlabs.nlp.util.io.{ExternalResource, ReadAs}
import com.johnsnowlabs.nlp.{AnnotatorApproach, AnnotatorType}
import com.johnsnowlabs.storage.HasStorageRef
import org.apache.spark.ml.PipelineModel
import org.apache.spark.ml.param.{BooleanParam, DoubleParam, IntParam}
import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}
import org.apache.spark.sql.Dataset

/** Algorithm for training a Named Entity Recognition Model
  *
  * For instantiated/pretrained models, see [[NerCrfModel]].
  *
  * This Named Entity recognition annotator allows for a generic model to be trained by utilizing
  * a CRF machine learning algorithm. The training data should be a labeled Spark Dataset, e.g.
  * [[com.johnsnowlabs.nlp.training.CoNLL CoNLL]] 2003 IOB with `Annotation` type columns. The
  * data should have columns of type `DOCUMENT, TOKEN, POS, WORD_EMBEDDINGS` and an additional
  * label column of annotator type `NAMED_ENTITY`. Excluding the label, this can be done with for
  * example
  *   - a [[com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetector SentenceDetector]],
  *   - a [[com.johnsnowlabs.nlp.annotators.Tokenizer Tokenizer]],
  *   - a [[com.johnsnowlabs.nlp.annotators.pos.perceptron.PerceptronModel PerceptronModel]] and
  *   - a [[com.johnsnowlabs.nlp.embeddings.WordEmbeddingsModel WordEmbeddingsModel]].
  *
  * Optionally the user can provide an entity dictionary file with [[setExternalFeatures]] for
  * better accuracy.
  *
  * For extended examples of usage, see the
  * [[https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/training/english/crf-ner/ner_dl_crf.ipynb Spark NLP Workshop]]
  * and the
  * [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/ner/crf/NerCrfApproachTestSpec.scala NerCrfApproachTestSpec]].
  *
  * ==Example==
  * {{{
  * import com.johnsnowlabs.nlp.base.DocumentAssembler
  * import com.johnsnowlabs.nlp.annotators.Tokenizer
  * import com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetector
  * import com.johnsnowlabs.nlp.embeddings.WordEmbeddingsModel
  * import com.johnsnowlabs.nlp.annotators.pos.perceptron.PerceptronModel
  * import com.johnsnowlabs.nlp.training.CoNLL
  * import com.johnsnowlabs.nlp.annotator.NerCrfApproach
  * import org.apache.spark.ml.Pipeline
  *
  * // This CoNLL dataset already includes a sentence, token, POS tags and label
  * // column with their respective annotator types. If a custom dataset is used,
  * // these need to be defined with for example:
  *
  * val documentAssembler = new DocumentAssembler()
  *   .setInputCol("text")
  *   .setOutputCol("document")
  *
  * val sentence = new SentenceDetector()
  *   .setInputCols("document")
  *   .setOutputCol("sentence")
  *
  * val tokenizer = new Tokenizer()
  *   .setInputCols("sentence")
  *   .setOutputCol("token")
  *
  * val posTagger = PerceptronModel.pretrained()
  *   .setInputCols("sentence", "token")
  *   .setOutputCol("pos")
  *
  * // Then the training can start
  * val embeddings = WordEmbeddingsModel.pretrained()
  *   .setInputCols("sentence", "token")
  *   .setOutputCol("embeddings")
  *   .setCaseSensitive(false)
  *
  * val nerTagger = new NerCrfApproach()
  *   .setInputCols("sentence", "token", "pos", "embeddings")
  *   .setLabelColumn("label")
  *   .setMinEpochs(1)
  *   .setMaxEpochs(3)
  *   .setOutputCol("ner")
  *
  * val pipeline = new Pipeline().setStages(Array(
  *   embeddings,
  *   nerTagger
  * ))
  *
  * // We use the sentences, tokens, POS tags and labels from the CoNLL dataset.
  * val conll = CoNLL()
  * val trainingData = conll.readDataset(spark, "src/test/resources/conll2003/eng.train")
  *
  * val pipelineModel = pipeline.fit(trainingData)
  * }}}
  * @see
  *   [[com.johnsnowlabs.nlp.annotators.ner.dl.NerDLApproach NerDLApproach]] for a deep learning
  *   based approach
  * @see
  *   [[com.johnsnowlabs.nlp.annotators.ner.NerConverter NerConverter]] to further process the
  *   results
  * @param uid
  *   required uid for storing annotator to disk
  * @groupname anno Annotator types
  * @groupdesc anno
  *   Required input and expected output annotator types
  * @groupname Ungrouped Members
  * @groupname param Parameters
  * @groupname setParam Parameter setters
  * @groupname getParam Parameter getters
  * @groupname Ungrouped Members
  * @groupprio param  1
  * @groupprio anno  2
  * @groupprio Ungrouped 3
  * @groupprio setParam  4
  * @groupprio getParam  5
  * @groupdesc param
  *   A list of (hyper-)parameter keys this annotator can take. Users can set and get the
  *   parameter values through setters and getters, respectively.
  */
class NerCrfApproach(override val uid: String)
    extends AnnotatorApproach[NerCrfModel]
    with NerApproach[NerCrfApproach] {

  def this() = this(Identifiable.randomUID("NER"))

  /** CRF based Named Entity Recognition Tagger */
  override val description = "CRF based Named Entity Recognition Tagger"

  /** Input annotator types : DOCUMENT, TOKEN, POS, WORD_EMBEDDINGS
    *
    * @group anno
    */
  override val inputAnnotatorTypes: Array[String] = Array(DOCUMENT, TOKEN, POS, WORD_EMBEDDINGS)

  /** Output annotator types : NAMED_ENTITY
    *
    * @group anno
    */
  override val outputAnnotatorType: AnnotatorType = NAMED_ENTITY

  /** L2 regularization coefficient (Default: `1f`)
    *
    * @group param
    */
  val l2 = new DoubleParam(this, "l2", "L2 regularization coefficient")

  /** c0 params defining decay speed for gradient (Default: `2250000`)
    *
    * @group param
    */
  val c0 = new IntParam(this, "c0", "c0 params defining decay speed for gradient")

  /** If Epoch relative improvement is less than `lossEps` then training is stopped (Default:
    * `1e-3f`)
    *
    * @group param
    */
  val lossEps = new DoubleParam(
    this,
    "lossEps",
    "If Epoch relative improvement less than eps then training is stopped")

  /** Features with less weights then this param value will be filtered
    *
    * @group param
    */
  val minW = new DoubleParam(
    this,
    "minW",
    "Features with less weights then this param value will be filtered")

  /** Whether or not to calculate prediction confidence by token, included in metadata (Default:
    * `false`)
    *
    * @group param
    */
  val includeConfidence = new BooleanParam(
    this,
    "includeConfidence",
    "whether or not to calculate prediction confidence by token, includes in metadata")

  /** Additional dictionary to use for features
    *
    * @group param
    */
  val externalFeatures = new ExternalResourceParam(
    this,
    "externalFeatures",
    "Additional dictionary to use for features")

  /** Level of verbosity during training (Default: `Verbose.Silent.id`)
    *
    * @group param
    */
  val verbose = new IntParam(this, "verbose", "Level of verbosity during training")

  /** L2 regularization coefficient
    *
    * @group setParam
    */
  def setL2(l2: Double): this.type = set(this.l2, l2)

  /** c0 params defining decay speed for gradient
    *
    * @group setParam
    */
  def setC0(c0: Int): this.type = set(this.c0, c0)

  /** If Epoch relative improvement less than eps then training is stopped
    *
    * @group setParam
    */
  def setLossEps(eps: Double): this.type = set(this.lossEps, eps)

  /** Features with less weights then this param value will be filtered
    *
    * @group setParam
    */
  def setMinW(w: Double): this.type = set(this.minW, w)

  /** Whether or not to calculate prediction confidence by token, includes in metadata
    *
    * @group setParam
    */
  def setIncludeConfidence(c: Boolean): this.type = set(includeConfidence, c)

  /** Level of verbosity during training (Default: `Verbose.Silent.id`)
    *
    * @group setParam
    */
  def setVerbose(verbose: Int): this.type = set(this.verbose, verbose)

  /** Level of verbosity during training (Default: `Verbose.Silent.id`)
    *
    * @group setParam
    */
  def setVerbose(verbose: Verbose.Level): this.type =
    set(this.verbose, verbose.id)

  /** L2 regularization coefficient
    *
    * @group getParam
    */
  def getL2: Double = $(l2)

  /** c0 params defining decay speed for gradient
    *
    * @group getParam
    */
  def getC0: Int = $(c0)

  /** If Epoch relative improvement less than eps then training is stopped
    *
    * @group getParam
    */
  def getLossEps: Double = $(lossEps)

  /** Features with less weights then this param value will be filtered
    *
    * @group getParam
    */
  def getMinW: Double = $(minW)

  /** Whether or not to calculate prediction confidence by token, includes in metadata
    *
    * @group getParam
    */
  def getIncludeConfidence: Boolean = $(includeConfidence)

  /** Additional dictionary to use for features
    *
    * @group setParam
    */
  def setExternalFeatures(value: ExternalResource): this.type = {
    require(
      value.options.contains("delimiter"),
      "external features is a delimited text. needs 'delimiter' in options")
    set(externalFeatures, value)
  }

  /** Additional dictionary to use for features
    *
    * @group setParam
    */
  def setExternalFeatures(
      path: String,
      delimiter: String,
      readAs: ReadAs.Format = ReadAs.TEXT,
      options: Map[String, String] = Map("format" -> "text")): this.type =
    set(
      externalFeatures,
      ExternalResource(path, readAs, options ++ Map("delimiter" -> delimiter)))

  setDefault(
    minEpochs -> 0,
    maxEpochs -> 1000,
    l2 -> 1f,
    c0 -> 2250000,
    lossEps -> 1e-3f,
    verbose -> Verbose.Silent.id,
    includeConfidence -> false)

  override def train(
      dataset: Dataset[_],
      recursivePipeline: Option[PipelineModel]): NerCrfModel = {

    val rows = dataset.toDF()

    val trainDataset =
      NerTagged.collectTrainingInstancesWithPos(rows, getInputCols, $(labelColumn))

    val extraFeatures = get(externalFeatures)
    val dictFeatures = DictionaryFeatures.read(extraFeatures)
    val crfDataset = FeatureGenerator(dictFeatures)
      .generateDataset(trainDataset)

    val params = CrfParams(
      minEpochs = getOrDefault(minEpochs),
      maxEpochs = getOrDefault(maxEpochs),
      l2 = getOrDefault(l2).toFloat,
      c0 = getOrDefault(c0),
      lossEps = getOrDefault(lossEps).toFloat,
      verbose = Verbose.Epochs,
      randomSeed = get(randomSeed))

    val embeddingsRef =
      HasStorageRef.getStorageRefFromInput(dataset, $(inputCols), AnnotatorType.WORD_EMBEDDINGS)

    val crf = new LinearChainCrf(params)
    val crfModel = crf.trainSGD(crfDataset)

    var model = new NerCrfModel()
      .setModel(crfModel)
      .setDictionaryFeatures(dictFeatures)
      .setIncludeConfidence($(includeConfidence))
      .setStorageRef(embeddingsRef)

    if (isDefined(entities))
      model.setEntities($(entities))

    if (isDefined(minW))
      model = model.shrink($(minW).toFloat)

    model
  }
}

/** This is the companion object of [[NerCrfApproach]]. Please refer to that class for the
  * documentation.
  */
object NerCrfApproach extends DefaultParamsReadable[NerCrfApproach]

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.annotators.ner.crf
18
19	import com.johnsnowlabs.ml.crf.{CrfParams, LinearChainCrf}
20	import com.johnsnowlabs.nlp.AnnotatorType._
21	import com.johnsnowlabs.nlp.annotators.common.NerTagged
22	import com.johnsnowlabs.nlp.annotators.ner.{NerApproach, Verbose}
23	import com.johnsnowlabs.nlp.annotators.param.ExternalResourceParam
24	import com.johnsnowlabs.nlp.util.io.{ExternalResource, ReadAs}
25	import com.johnsnowlabs.nlp.{AnnotatorApproach, AnnotatorType}
26	import com.johnsnowlabs.storage.HasStorageRef
27	import org.apache.spark.ml.PipelineModel
28	import org.apache.spark.ml.param.{BooleanParam, DoubleParam, IntParam}
29	import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}
30	import org.apache.spark.sql.Dataset
31
32	/** Algorithm for training a Named Entity Recognition Model
33	*
34	* For instantiated/pretrained models, see [[NerCrfModel]].
35	*
36	* This Named Entity recognition annotator allows for a generic model to be trained by utilizing
37	* a CRF machine learning algorithm. The training data should be a labeled Spark Dataset, e.g.
38	* [[com.johnsnowlabs.nlp.training.CoNLL CoNLL]] 2003 IOB with `Annotation` type columns. The
39	* data should have columns of type `DOCUMENT, TOKEN, POS, WORD_EMBEDDINGS` and an additional
40	* label column of annotator type `NAMED_ENTITY`. Excluding the label, this can be done with for
41	* example
42	* - a [[com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetector SentenceDetector]],
43	* - a [[com.johnsnowlabs.nlp.annotators.Tokenizer Tokenizer]],
44	* - a [[com.johnsnowlabs.nlp.annotators.pos.perceptron.PerceptronModel PerceptronModel]] and
45	* - a [[com.johnsnowlabs.nlp.embeddings.WordEmbeddingsModel WordEmbeddingsModel]].
46	*
47	* Optionally the user can provide an entity dictionary file with [[setExternalFeatures]] for
48	* better accuracy.
49	*
50	* For extended examples of usage, see the
51	* [[https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/training/english/crf-ner/ner_dl_crf.ipynb Spark NLP Workshop]]
52	* and the
53	* [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/ner/crf/NerCrfApproachTestSpec.scala NerCrfApproachTestSpec]].
54	*
55	* ==Example==
56	* {{{
57	* import com.johnsnowlabs.nlp.base.DocumentAssembler
58	* import com.johnsnowlabs.nlp.annotators.Tokenizer
59	* import com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetector
60	* import com.johnsnowlabs.nlp.embeddings.WordEmbeddingsModel
61	* import com.johnsnowlabs.nlp.annotators.pos.perceptron.PerceptronModel
62	* import com.johnsnowlabs.nlp.training.CoNLL
63	* import com.johnsnowlabs.nlp.annotator.NerCrfApproach
64	* import org.apache.spark.ml.Pipeline
65	*
66	* // This CoNLL dataset already includes a sentence, token, POS tags and label
67	* // column with their respective annotator types. If a custom dataset is used,
68	* // these need to be defined with for example:
69	*
70	* val documentAssembler = new DocumentAssembler()
71	* .setInputCol("text")
72	* .setOutputCol("document")
73	*
74	* val sentence = new SentenceDetector()
75	* .setInputCols("document")
76	* .setOutputCol("sentence")
77	*
78	* val tokenizer = new Tokenizer()
79	* .setInputCols("sentence")
80	* .setOutputCol("token")
81	*
82	* val posTagger = PerceptronModel.pretrained()
83	* .setInputCols("sentence", "token")
84	* .setOutputCol("pos")
85	*
86	* // Then the training can start
87	* val embeddings = WordEmbeddingsModel.pretrained()
88	* .setInputCols("sentence", "token")
89	* .setOutputCol("embeddings")
90	* .setCaseSensitive(false)
91	*
92	* val nerTagger = new NerCrfApproach()
93	* .setInputCols("sentence", "token", "pos", "embeddings")
94	* .setLabelColumn("label")
95	* .setMinEpochs(1)
96	* .setMaxEpochs(3)
97	* .setOutputCol("ner")
98	*
99	* val pipeline = new Pipeline().setStages(Array(
100	* embeddings,
101	* nerTagger
102	* ))
103	*
104	* // We use the sentences, tokens, POS tags and labels from the CoNLL dataset.
105	* val conll = CoNLL()
106	* val trainingData = conll.readDataset(spark, "src/test/resources/conll2003/eng.train")
107	*
108	* val pipelineModel = pipeline.fit(trainingData)
109	* }}}
110	* @see
111	* [[com.johnsnowlabs.nlp.annotators.ner.dl.NerDLApproach NerDLApproach]] for a deep learning
112	* based approach
113	* @see
114	* [[com.johnsnowlabs.nlp.annotators.ner.NerConverter NerConverter]] to further process the
115	* results
116	* @param uid
117	* required uid for storing annotator to disk
118	* @groupname anno Annotator types
119	* @groupdesc anno
120	* Required input and expected output annotator types
121	* @groupname Ungrouped Members
122	* @groupname param Parameters
123	* @groupname setParam Parameter setters
124	* @groupname getParam Parameter getters
125	* @groupname Ungrouped Members
126	* @groupprio param 1
127	* @groupprio anno 2
128	* @groupprio Ungrouped 3
129	* @groupprio setParam 4
130	* @groupprio getParam 5
131	* @groupdesc param
132	* A list of (hyper-)parameter keys this annotator can take. Users can set and get the
133	* parameter values through setters and getters, respectively.
134	*/
135	class NerCrfApproach(override val uid: String)
136	extends AnnotatorApproach[NerCrfModel]
137	with NerApproach[NerCrfApproach] {
138
139	def this() = this(Identifiable.randomUID("NER"))	1✔
140
141	/** CRF based Named Entity Recognition Tagger */
142	override val description = "CRF based Named Entity Recognition Tagger"	1✔
143
144	/** Input annotator types : DOCUMENT, TOKEN, POS, WORD_EMBEDDINGS
145	*
146	* @group anno
147	*/
148	override val inputAnnotatorTypes: Array[String] = Array(DOCUMENT, TOKEN, POS, WORD_EMBEDDINGS)	1✔
149
150	/** Output annotator types : NAMED_ENTITY
151	*
152	* @group anno
153	*/
154	override val outputAnnotatorType: AnnotatorType = NAMED_ENTITY	1✔
155
156	/** L2 regularization coefficient (Default: `1f`)
157	*
158	* @group param
159	*/
160	val l2 = new DoubleParam(this, "l2", "L2 regularization coefficient")	1✔
161
162	/** c0 params defining decay speed for gradient (Default: `2250000`)
163	*
164	* @group param
165	*/
166	val c0 = new IntParam(this, "c0", "c0 params defining decay speed for gradient")	1✔
167
168	/** If Epoch relative improvement is less than `lossEps` then training is stopped (Default:
169	* `1e-3f`)
170	*
171	* @group param
172	*/
173	val lossEps = new DoubleParam(	1✔
174	this,
175	"lossEps",	1✔
176	"If Epoch relative improvement less than eps then training is stopped")	1✔
177
178	/** Features with less weights then this param value will be filtered
179	*
180	* @group param
181	*/
182	val minW = new DoubleParam(	1✔
183	this,
184	"minW",	1✔
185	"Features with less weights then this param value will be filtered")	1✔
186
187	/** Whether or not to calculate prediction confidence by token, included in metadata (Default:
188	* `false`)
189	*
190	* @group param
191	*/
192	val includeConfidence = new BooleanParam(	1✔
193	this,
194	"includeConfidence",	1✔
195	"whether or not to calculate prediction confidence by token, includes in metadata")	1✔
196
197	/** Additional dictionary to use for features
198	*
199	* @group param
200	*/
201	val externalFeatures = new ExternalResourceParam(	1✔
202	this,
203	"externalFeatures",	1✔
204	"Additional dictionary to use for features")	1✔
205
206	/** Level of verbosity during training (Default: `Verbose.Silent.id`)
207	*
208	* @group param
209	*/
210	val verbose = new IntParam(this, "verbose", "Level of verbosity during training")	1✔
211
212	/** L2 regularization coefficient
213	*
214	* @group setParam
215	*/
216	def setL2(l2: Double): this.type = set(this.l2, l2)	1✔
217
218	/** c0 params defining decay speed for gradient
219	*
220	* @group setParam
221	*/
222	def setC0(c0: Int): this.type = set(this.c0, c0)	1✔
223
224	/** If Epoch relative improvement less than eps then training is stopped
225	*
226	* @group setParam
227	*/
228	def setLossEps(eps: Double): this.type = set(this.lossEps, eps)	×
229
230	/** Features with less weights then this param value will be filtered
231	*
232	* @group setParam
233	*/
234	def setMinW(w: Double): this.type = set(this.minW, w)	×
235
236	/** Whether or not to calculate prediction confidence by token, includes in metadata
237	*
238	* @group setParam
239	*/
240	def setIncludeConfidence(c: Boolean): this.type = set(includeConfidence, c)	×
241
242	/** Level of verbosity during training (Default: `Verbose.Silent.id`)
243	*
244	* @group setParam
245	*/
246	def setVerbose(verbose: Int): this.type = set(this.verbose, verbose)	×
247
248	/** Level of verbosity during training (Default: `Verbose.Silent.id`)
249	*
250	* @group setParam
251	*/
252	def setVerbose(verbose: Verbose.Level): this.type =
253	set(this.verbose, verbose.id)	×
254
255	/** L2 regularization coefficient
256	*
257	* @group getParam
258	*/
259	def getL2: Double = $(l2)	×
260
261	/** c0 params defining decay speed for gradient
262	*
263	* @group getParam
264	*/
265	def getC0: Int = $(c0)	×
266
267	/** If Epoch relative improvement less than eps then training is stopped
268	*
269	* @group getParam
270	*/
271	def getLossEps: Double = $(lossEps)	×
272
273	/** Features with less weights then this param value will be filtered
274	*
275	* @group getParam
276	*/
277	def getMinW: Double = $(minW)	×
278
279	/** Whether or not to calculate prediction confidence by token, includes in metadata
280	*
281	* @group getParam
282	*/
283	def getIncludeConfidence: Boolean = $(includeConfidence)	×
284
285	/** Additional dictionary to use for features
286	*
287	* @group setParam
288	*/
289	def setExternalFeatures(value: ExternalResource): this.type = {
290	require(	×
291	value.options.contains("delimiter"),	×
292	"external features is a delimited text. needs 'delimiter' in options")	×
293	set(externalFeatures, value)	×
294	}
295
296	/** Additional dictionary to use for features
297	*
298	* @group setParam
299	*/
300	def setExternalFeatures(
301	path: String,
302	delimiter: String,
303	readAs: ReadAs.Format = ReadAs.TEXT,
304	options: Map[String, String] = Map("format" -> "text")): this.type =
305	set(	×
306	externalFeatures,	×
307	ExternalResource(path, readAs, options ++ Map("delimiter" -> delimiter)))	×
308
309	setDefault(	1✔
310	minEpochs -> 0,	1✔
311	maxEpochs -> 1000,	1✔
312	l2 -> 1f,	1✔
313	c0 -> 2250000,	1✔
314	lossEps -> 1e-3f,	1✔
315	verbose -> Verbose.Silent.id,	1✔
316	includeConfidence -> false)	1✔
317
318	override def train(
319	dataset: Dataset[_],
320	recursivePipeline: Option[PipelineModel]): NerCrfModel = {
321
322	val rows = dataset.toDF()	1✔
323
324	val trainDataset =
325	NerTagged.collectTrainingInstancesWithPos(rows, getInputCols, $(labelColumn))	1✔
326
327	val extraFeatures = get(externalFeatures)	1✔
328	val dictFeatures = DictionaryFeatures.read(extraFeatures)	1✔
329	val crfDataset = FeatureGenerator(dictFeatures)
330	.generateDataset(trainDataset)	1✔
331
332	val params = CrfParams(	1✔
333	minEpochs = getOrDefault(minEpochs),	1✔
334	maxEpochs = getOrDefault(maxEpochs),	1✔
335	l2 = getOrDefault(l2).toFloat,	1✔
336	c0 = getOrDefault(c0),	1✔
337	lossEps = getOrDefault(lossEps).toFloat,	1✔
338	verbose = Verbose.Epochs,	1✔
339	randomSeed = get(randomSeed))	1✔
340
341	val embeddingsRef =
342	HasStorageRef.getStorageRefFromInput(dataset, $(inputCols), AnnotatorType.WORD_EMBEDDINGS)	1✔
343
344	val crf = new LinearChainCrf(params)	1✔
345	val crfModel = crf.trainSGD(crfDataset)	1✔
346
347	var model = new NerCrfModel()
348	.setModel(crfModel)
349	.setDictionaryFeatures(dictFeatures)
350	.setIncludeConfidence($(includeConfidence))
351	.setStorageRef(embeddingsRef)	1✔
352
353	if (isDefined(entities))	1✔
354	model.setEntities($(entities))	×
355
356	if (isDefined(minW))	1✔
357	model = model.shrink($(minW).toFloat)	×
358
359	model
360	}
361	}
362
363	/** This is the companion object of [[NerCrfApproach]]. Please refer to that class for the
364	* documentation.
365	*/
366	object NerCrfApproach extends DefaultParamsReadable[NerCrfApproach]

JohnSnowLabs / spark-nlp / 3122721559

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous