2695891961

Build # 2695891961

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/31b1686f922865503d6ebadff7338269e0d5c01a">31b1686f9<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/31b1686f922865503d6ebadff7338269e0d5c01a">">Merge </a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/0600d172ceb61677d84b32dd17354ab43978b5fd">0600d172c</a>">0600d172c</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/31b1686f922865503d6ebadff7338269e0d5c01a"> into 262fd1bd1">262fd1bd1</a>

Pull Request Pull Request #10571: Release/402 release candidage

Run Details

42 of 42 new or added lines in 6 files covered. (100.0%)

8228 of 11259 relevant lines covered (73.08%)

0.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.72

/src/main/scala/com/johnsnowlabs/nlp/annotators/sbd/SentenceDetectorParams.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators.sbd

import org.apache.spark.ml.param.{BooleanParam, IntParam, Param, Params, StringArrayParam}

import scala.collection.mutable.ArrayBuffer

/** See
  * [[https://github.com/JohnSnowLabs/spark-nlp/tree/master/src/test/scala/com/johnsnowlabs/nlp/annotators/sbd/pragmatic]]
  * for further reference on how to use this API
  */
trait SentenceDetectorParams extends Params {

  /** Whether to apply abbreviations at sentence detection (Default: `true`)
    *
    * @group param
    */
  val useAbbrevations = new BooleanParam(
    this,
    "useAbbreviations",
    "whether to apply abbreviations at sentence detection")

  /** Whether take lists into consideration at sentence detection (Default: `true`)
    *
    * @group param
    */
  val detectLists = new BooleanParam(
    this,
    "detectLists",
    "whether take lists into consideration at sentence detection")

  /** Whether to only utilize custom bounds for sentence detection (Default: `false`)
    *
    * @group param
    */
  val useCustomBoundsOnly = new BooleanParam(
    this,
    "useCustomBoundsOnly",
    "whether to only utilize custom bounds for sentence detection")

  /** Whether to explode each sentence into a different row, for better parallelization (Default:
    * `false`)
    *
    * @group param
    */
  val explodeSentences = new BooleanParam(
    this,
    "explodeSentences",
    "whether to explode each sentence into a different row, for better parallelization. Defaults to false.")

  /** Characters used to explicitly mark sentence bounds (Default: None)
    *
    * @group param
    */
  val customBounds: StringArrayParam = new StringArrayParam(
    this,
    "customBounds",
    "characters used to explicitly mark sentence bounds")

  /** Length at which sentences will be forcibly split (Ignored if not set)
    *
    * @group param
    */
  val splitLength: IntParam =
    new IntParam(this, "splitLength", "length at which sentences will be forcibly split.")

  /** Set the minimum allowed length for each sentence (Default: `0`)
    *
    * @group param
    */
  val minLength =
    new IntParam(this, "minLength", "Set the minimum allowed length for each sentence")

  /** Set the maximum allowed length for each sentence (Ignored if not set)
    *
    * @group param
    */
  val maxLength =
    new IntParam(this, "maxLength", "Set the maximum allowed length for each sentence")

  /** How to return matched custom bounds (Default: `none`). Will have no effect if no custom
    * bounds are used. Possible values are:
    *
    *   - "none" - Will not return the matched bound
    *   - "prepend" - Prepends a sentence break to the match
    *   - "append" - Appends a sentence break to the match
    *
    * @group param
    */
  val customBoundsStrategy: Param[String] =
    new Param[String](this, "customBoundsStrategy", "How to return matched custom bounds")

  setDefault(
    useAbbrevations -> true,
    detectLists -> true,
    useCustomBoundsOnly -> false,
    explodeSentences -> false,
    customBounds -> Array.empty[String],
    minLength -> 0,
    customBoundsStrategy -> "none")

  /** Custom sentence separator text
    * @group setParam
    */
  def setCustomBounds(value: Array[String]): this.type = set(customBounds, value)

  /** Custom sentence separator text
    * @group getParam
    */
  def getCustomBounds: Array[String] = $(customBounds)

  /** Use only custom bounds without considering those of Pragmatic Segmenter. Defaults to false.
    * Needs customBounds.
    * @group setParam
    */
  def setUseCustomBoundsOnly(value: Boolean): this.type = set(useCustomBoundsOnly, value)

  /** Use only custom bounds without considering those of Pragmatic Segmenter. Defaults to false.
    * Needs customBounds.
    * @group getParam
    */
  def getUseCustomBoundsOnly: Boolean = $(useCustomBoundsOnly)

  /** Sets how to return matched custom bounds (Default: `none`). Will have no effect if no custom
    * bounds are used. Possible values are:
    *
    *   - "none" - Will not return the matched bound
    *   - "prepend" - Prepends a sentence break to the match
    *   - "append" - Appends a sentence break to the match
    *
    * @group setParam
    */
  def setCustomBoundsStrategy(value: String): this.type = {
    val possibleValues = Array("none", "prepend", "append")
    require(
      possibleValues.contains(value),
      s"$value is not a valid strategy for custom bounds. " +
        s"Possible Values: (${possibleValues.mkString(", ")}).")

    set(customBoundsStrategy, value)
  }

  /** Gets how to return matched custom bounds (Default: `none`).
    * @group getParam
    */
  def getCustomBoundsStrategy: String = $(customBoundsStrategy)

  /** Whether to consider abbreviation strategies for better accuracy but slower performance.
    * Defaults to true.
    * @group setParam
    */
  def setUseAbbreviations(value: Boolean): this.type = set(useAbbrevations, value)

  /** Whether to consider abbreviation strategies for better accuracy but slower performance.
    * Defaults to true.
    * @group getParam
    */
  def getUseAbbreviations: Boolean = $(useAbbrevations)

  /** Whether to take lists into consideration at sentence detection. Defaults to true.
    * @group setParam
    */
  def setDetectLists(value: Boolean): this.type = set(detectLists, value)

  /** Whether to take lists into consideration at sentence detection. Defaults to true.
    * @group getParam
    */
  def getDetectLists: Boolean = $(detectLists)

  /** Whether to split sentences into different Dataset rows. Useful for higher parallelism in fat
    * rows. Defaults to false.
    * @group setParam
    */
  def setExplodeSentences(value: Boolean): this.type = set(explodeSentences, value)

  /** Whether to split sentences into different Dataset rows. Useful for higher parallelism in fat
    * rows. Defaults to false.
    * @group getParam
    */
  def getExplodeSentences: Boolean = $(explodeSentences)

  /** Length at which sentences will be forcibly split
    * @group setParam
    */
  def setSplitLength(value: Int): this.type = set(splitLength, value)

  /** Length at which sentences will be forcibly split
    * @group getParam
    */
  def getSplitLength: Int = $(splitLength)

  /** Set the minimum allowed length for each sentence
    * @group setParam
    */
  def setMinLength(value: Int): this.type = {
    require(value >= 0, "minLength must be greater equal than 0")
    require(value.isValidInt, "minLength must be Int")
    set(minLength, value)
  }

  /** Get the minimum allowed length for each sentence
    * @group getParam
    */
  def getMinLength(value: Int): Int = $(minLength)

  /** Set the maximum allowed length for each sentence
    * @group setParam
    */
  def setMaxLength(value: Int): this.type = {
    require(
      value >= $ {
        minLength
      },
      "maxLength must be greater equal than minLength")
    require(value.isValidInt, "minLength must be Int")
    set(maxLength, value)
  }

  /** Get the maximum allowed length for each sentence
    * @group getParam
    */
  def getMaxLength(value: Int): Int = $(maxLength)

  def truncateSentence(sentence: String, maxLength: Int): Array[String] = {
    var currentLength = 0
    val allSentences = ArrayBuffer.empty[String]
    val currentSentence = ArrayBuffer.empty[String]

    def addWordToSentence(word: String): Unit = {

      /** Adds +1 because of the space joining words */
      currentLength += word.length + 1
      currentSentence.append(word)
    }

    sentence
      .split(" ")
      .foreach(word => {
        if (currentLength + word.length > maxLength) {
          allSentences.append(currentSentence.mkString(" "))
          currentSentence.clear()
          currentLength = 0
          addWordToSentence(word)
        } else {
          addWordToSentence(word)
        }
      })

    /** add leftovers */
    allSentences.append(currentSentence.mkString(" "))
    allSentences.toArray
  }

}

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.annotators.sbd
18
19	import org.apache.spark.ml.param.{BooleanParam, IntParam, Param, Params, StringArrayParam}
20
21	import scala.collection.mutable.ArrayBuffer
22
23	/** See
24	* [[https://github.com/JohnSnowLabs/spark-nlp/tree/master/src/test/scala/com/johnsnowlabs/nlp/annotators/sbd/pragmatic]]
25	* for further reference on how to use this API
26	*/
27	trait SentenceDetectorParams extends Params {
28
29	/** Whether to apply abbreviations at sentence detection (Default: `true`)
30	*
31	* @group param
32	*/
33	val useAbbrevations = new BooleanParam(	1✔
34	this,
35	"useAbbreviations",	1✔
36	"whether to apply abbreviations at sentence detection")	1✔
37
38	/** Whether take lists into consideration at sentence detection (Default: `true`)
39	*
40	* @group param
41	*/
42	val detectLists = new BooleanParam(	1✔
43	this,
44	"detectLists",	1✔
45	"whether take lists into consideration at sentence detection")	1✔
46
47	/** Whether to only utilize custom bounds for sentence detection (Default: `false`)
48	*
49	* @group param
50	*/
51	val useCustomBoundsOnly = new BooleanParam(	1✔
52	this,
53	"useCustomBoundsOnly",	1✔
54	"whether to only utilize custom bounds for sentence detection")	1✔
55
56	/** Whether to explode each sentence into a different row, for better parallelization (Default:
57	* `false`)
58	*
59	* @group param
60	*/
61	val explodeSentences = new BooleanParam(	1✔
62	this,
63	"explodeSentences",	1✔
64	"whether to explode each sentence into a different row, for better parallelization. Defaults to false.")	1✔
65
66	/** Characters used to explicitly mark sentence bounds (Default: None)
67	*
68	* @group param
69	*/
70	val customBounds: StringArrayParam = new StringArrayParam(	1✔
71	this,
72	"customBounds",	1✔
73	"characters used to explicitly mark sentence bounds")	1✔
74
75	/** Length at which sentences will be forcibly split (Ignored if not set)
76	*
77	* @group param
78	*/
79	val splitLength: IntParam =
80	new IntParam(this, "splitLength", "length at which sentences will be forcibly split.")	1✔
81
82	/** Set the minimum allowed length for each sentence (Default: `0`)
83	*
84	* @group param
85	*/
86	val minLength =
87	new IntParam(this, "minLength", "Set the minimum allowed length for each sentence")	1✔
88
89	/** Set the maximum allowed length for each sentence (Ignored if not set)
90	*
91	* @group param
92	*/
93	val maxLength =
94	new IntParam(this, "maxLength", "Set the maximum allowed length for each sentence")	1✔
95
96	/** How to return matched custom bounds (Default: `none`). Will have no effect if no custom
97	* bounds are used. Possible values are:
98	*
99	* - "none" - Will not return the matched bound
100	* - "prepend" - Prepends a sentence break to the match
101	* - "append" - Appends a sentence break to the match
102	*
103	* @group param
104	*/
105	val customBoundsStrategy: Param[String] =
106	new Param[String](this, "customBoundsStrategy", "How to return matched custom bounds")	1✔
107
108	setDefault(	1✔
109	useAbbrevations -> true,	1✔
110	detectLists -> true,	1✔
111	useCustomBoundsOnly -> false,	1✔
112	explodeSentences -> false,	1✔
113	customBounds -> Array.empty[String],	1✔
114	minLength -> 0,	1✔
115	customBoundsStrategy -> "none")	1✔
116
117	/** Custom sentence separator text
118	* @group setParam
119	*/
120	def setCustomBounds(value: Array[String]): this.type = set(customBounds, value)	1✔
121
122	/** Custom sentence separator text
123	* @group getParam
124	*/
125	def getCustomBounds: Array[String] = $(customBounds)	×
126
127	/** Use only custom bounds without considering those of Pragmatic Segmenter. Defaults to false.
128	* Needs customBounds.
129	* @group setParam
130	*/
131	def setUseCustomBoundsOnly(value: Boolean): this.type = set(useCustomBoundsOnly, value)	1✔
132
133	/** Use only custom bounds without considering those of Pragmatic Segmenter. Defaults to false.
134	* Needs customBounds.
135	* @group getParam
136	*/
137	def getUseCustomBoundsOnly: Boolean = $(useCustomBoundsOnly)	×
138
139	/** Sets how to return matched custom bounds (Default: `none`). Will have no effect if no custom
140	* bounds are used. Possible values are:
141	*
142	* - "none" - Will not return the matched bound
143	* - "prepend" - Prepends a sentence break to the match
144	* - "append" - Appends a sentence break to the match
145	*
146	* @group setParam
147	*/
148	def setCustomBoundsStrategy(value: String): this.type = {
149	val possibleValues = Array("none", "prepend", "append")	1✔
150	require(	1✔
151	possibleValues.contains(value),	1✔
152	s"$value is not a valid strategy for custom bounds. " +	1✔
153	s"Possible Values: (${possibleValues.mkString(", ")}).")	1✔
154
155	set(customBoundsStrategy, value)	1✔
156	}
157
158	/** Gets how to return matched custom bounds (Default: `none`).
159	* @group getParam
160	*/
161	def getCustomBoundsStrategy: String = $(customBoundsStrategy)	×
162
163	/** Whether to consider abbreviation strategies for better accuracy but slower performance.
164	* Defaults to true.
165	* @group setParam
166	*/
167	def setUseAbbreviations(value: Boolean): this.type = set(useAbbrevations, value)	1✔
168
169	/** Whether to consider abbreviation strategies for better accuracy but slower performance.
170	* Defaults to true.
171	* @group getParam
172	*/
173	def getUseAbbreviations: Boolean = $(useAbbrevations)	×
174
175	/** Whether to take lists into consideration at sentence detection. Defaults to true.
176	* @group setParam
177	*/
178	def setDetectLists(value: Boolean): this.type = set(detectLists, value)	×
179
180	/** Whether to take lists into consideration at sentence detection. Defaults to true.
181	* @group getParam
182	*/
183	def getDetectLists: Boolean = $(detectLists)	×
184
185	/** Whether to split sentences into different Dataset rows. Useful for higher parallelism in fat
186	* rows. Defaults to false.
187	* @group setParam
188	*/
189	def setExplodeSentences(value: Boolean): this.type = set(explodeSentences, value)	1✔
190
191	/** Whether to split sentences into different Dataset rows. Useful for higher parallelism in fat
192	* rows. Defaults to false.
193	* @group getParam
194	*/
195	def getExplodeSentences: Boolean = $(explodeSentences)	×
196
197	/** Length at which sentences will be forcibly split
198	* @group setParam
199	*/
200	def setSplitLength(value: Int): this.type = set(splitLength, value)	1✔
201
202	/** Length at which sentences will be forcibly split
203	* @group getParam
204	*/
205	def getSplitLength: Int = $(splitLength)	×
206
207	/** Set the minimum allowed length for each sentence
208	* @group setParam
209	*/
210	def setMinLength(value: Int): this.type = {
211	require(value >= 0, "minLength must be greater equal than 0")	1✔
212	require(value.isValidInt, "minLength must be Int")	1✔
213	set(minLength, value)	1✔
214	}
215
216	/** Get the minimum allowed length for each sentence
217	* @group getParam
218	*/
219	def getMinLength(value: Int): Int = $(minLength)	×
220
221	/** Set the maximum allowed length for each sentence
222	* @group setParam
223	*/
224	def setMaxLength(value: Int): this.type = {
225	require(	1✔
226	value >= $ {	1✔
227	minLength	1✔
228	},
229	"maxLength must be greater equal than minLength")	×
230	require(value.isValidInt, "minLength must be Int")	1✔
231	set(maxLength, value)	1✔
232	}
233
234	/** Get the maximum allowed length for each sentence
235	* @group getParam
236	*/
237	def getMaxLength(value: Int): Int = $(maxLength)	×
238
239	def truncateSentence(sentence: String, maxLength: Int): Array[String] = {
240	var currentLength = 0	1✔
241	val allSentences = ArrayBuffer.empty[String]	1✔
242	val currentSentence = ArrayBuffer.empty[String]	1✔
243
244	def addWordToSentence(word: String): Unit = {
245
246	/** Adds +1 because of the space joining words */
247	currentLength += word.length + 1	1✔
248	currentSentence.append(word)	1✔
249	}
250
251	sentence
252	.split(" ")	1✔
253	.foreach(word => {	1✔
254	if (currentLength + word.length > maxLength) {	1✔
255	allSentences.append(currentSentence.mkString(" "))	1✔
256	currentSentence.clear()	1✔
257	currentLength = 0	1✔
258	addWordToSentence(word)	1✔
259	} else {
260	addWordToSentence(word)	1✔
261	}
262	})
263
264	/** add leftovers */
265	allSentences.append(currentSentence.mkString(" "))	1✔
266	allSentences.toArray	1✔
267	}
268
269	}

JohnSnowLabs / spark-nlp / 2695891961

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous