11429325160

Committed 20 Oct 2024 08:18PM UTC coverage: 60.052% (-0.2%) from 60.216%

Build # 11429325160

Build Type

Pull #14439

github

Committed by

web-flow

Commit Message

Merge 1c191569d into 9db33328b

Pull Request Pull Request #14439: [SPARKNLP-1067] PromptAssembler

Run Details

0 of 50 new or added lines in 2 files covered. (0.0%)

48 existing lines in 26 files now uncovered.

8985 of 14962 relevant lines covered (60.05%)

0.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.51

/src/main/scala/com/johnsnowlabs/nlp/annotators/Tokenizer.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators

import com.johnsnowlabs.nlp.AnnotatorApproach
import com.johnsnowlabs.nlp.AnnotatorType.{DOCUMENT, TOKEN}
import com.johnsnowlabs.nlp.annotators.param.ExternalResourceParam
import com.johnsnowlabs.nlp.util.io.{ExternalResource, MatchStrategy, ReadAs, ResourceHelper}
import com.johnsnowlabs.nlp.util.regex.RuleFactory
import org.apache.spark.ml.PipelineModel
import org.apache.spark.ml.param.{BooleanParam, IntParam, Param, StringArrayParam}
import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}
import org.apache.spark.sql.Dataset

import java.util.regex.Pattern
import scala.collection.mutable.ArrayBuffer

/** Tokenizes raw text in document type columns into TokenizedSentence .
  *
  * This class represents a non fitted tokenizer. Fitting it will cause the internal RuleFactory
  * to construct the rules for tokenizing from the input configuration.
  *
  * Identifies tokens with tokenization open standards. A few rules will help customizing it if
  * defaults do not fit user needs.
  *
  * For extended examples of usage see the
  * [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/document-normalizer/document_normalizer_notebook.ipynb Examples]]
  * and
  * [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/TokenizerTestSpec.scala Tokenizer test class]]
  *
  * ==Example==
  * {{{
  * import spark.implicits._
  * import com.johnsnowlabs.nlp.DocumentAssembler
  * import com.johnsnowlabs.nlp.annotators.Tokenizer
  * import org.apache.spark.ml.Pipeline
  *
  * val data = Seq("I'd like to say we didn't expect that. Jane's boyfriend.").toDF("text")
  * val documentAssembler = new DocumentAssembler().setInputCol("text").setOutputCol("document")
  * val tokenizer = new Tokenizer().setInputCols("document").setOutputCol("token").fit(data)
  *
  * val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer)).fit(data)
  * val result = pipeline.transform(data)
  *
  * result.selectExpr("token.result").show(false)
  * +-----------------------------------------------------------------------+
  * |output                                                                 |
  * +-----------------------------------------------------------------------+
  * |[I'd, like, to, say, we, didn't, expect, that, ., Jane's, boyfriend, .]|
  * +-----------------------------------------------------------------------+
  * }}}
  *
  * @param uid
  *   required uid for storing annotator to disk
  * @groupname anno Annotator types
  * @groupdesc anno
  *   Required input and expected output annotator types
  * @groupname Ungrouped Members
  * @groupname param Parameters.
  * @groupname setParam Parameter setters
  * @groupname getParam Parameter getters
  * @groupname Ungrouped Members
  * @groupprio param  1
  * @groupprio anno  2
  * @groupprio Ungrouped 3
  * @groupprio setParam  4
  * @groupprio getParam  5
  * @groupdesc param
  *   A list of (hyper-)parameter keys this annotator can take. Users can set and get the
  *   parameter values through setters and getters, respectively.
  */
class Tokenizer(override val uid: String) extends AnnotatorApproach[TokenizerModel] {

  /** Output annotator type : TOKEN
    *
    * @group anno
    */
  override val outputAnnotatorType: AnnotatorType = TOKEN

  /** Input annotator type : DOCUMENT
    *
    * @group anno
    */
  override val inputAnnotatorTypes: Array[AnnotatorType] = Array[AnnotatorType](
    DOCUMENT
  ) // A Tokenizer could require only for now a SentenceDetector annotator

  def this() = this(Identifiable.randomUID("REGEX_TOKENIZER"))

  /** Annotator that identifies points of analysis in a useful manner */
  override val description: String =
    "Annotator that identifies points of analysis in a useful manner"

  /** Words that won't be affected by tokenization rules
    *
    * @group param
    */
  val exceptions: StringArrayParam =
    new StringArrayParam(this, "exceptions", "Words that won't be affected by tokenization rules")

  /** Path to file containing list of exceptions
    *
    * @group param
    */
  val exceptionsPath: ExternalResourceParam = new ExternalResourceParam(
    this,
    "exceptionsPath",
    "Path to file containing list of exceptions")

  /** Whether to care for case sensitiveness in exceptions (Default: `true`)
    *
    * @group param
    */
  val caseSensitiveExceptions: BooleanParam = new BooleanParam(
    this,
    "caseSensitiveExceptions",
    "Whether to care for case sensitiveness in exceptions")

  /** Character list used to separate from token boundaries (Default: `Array(".", ",", ";", ":",
    * "!", "?", "*", "-", "(", ")", "\"", "'")`)
    * @group param
    */
  val contextChars: StringArrayParam = new StringArrayParam(
    this,
    "contextChars",
    "Character list used to separate from token boundaries")

  /** Character list used to separate from the inside of tokens
    *
    * @group param
    */
  val splitChars: StringArrayParam = new StringArrayParam(
    this,
    "splitChars",
    "Character list used to separate from the inside of tokens")

  /** Pattern to separate from the inside of tokens. takes priority over splitChars.
    *
    * This pattern will be applied to the tokens which where extracted with the target pattern
    * previously
    *
    * ''' Example:'''
    *
    * {{{
    * import org.apache.spark.ml.Pipeline
    *
    * import com.johnsnowlabs.nlp.annotators.Tokenizer
    *
    * import com.johnsnowlabs.nlp.DocumentAssembler
    *
    * val textDf = sqlContext.sparkContext.parallelize(Array("Tokens in this-text will#be#split on hashtags-and#dashes")).toDF("text")
    *
    * val documentAssembler = new DocumentAssembler().setInputCol("text").setOutputCol("sentences")
    *
    * val tokenizer = new Tokenizer().setInputCols("sentences").setOutputCol("tokens").setSplitPattern("-|#")
    *
    * new Pipeline().setStages(Array(documentAssembler, tokenizer)).fit(textDf).transform(textDf).select("tokens.result").show(false)
    * }}}
    *
    * This will yield: `Tokens, in, this, text, will, be, split, on, hashtags, and, dashes`
    * @group param
    */
  val splitPattern: Param[String] = new Param(
    this,
    "splitPattern",
    "Pattern to separate from the inside of tokens. takes priority over splitChars.")

  /** Pattern to grab from text as token candidates. (Default: `"\\S+"`)
    *
    * Defaults to: "\\S+" which means anything not a space will be matched and considered as a
    * token candidate, This will cause text to be split on on white spaces to yield token
    * candidates.
    *
    * This rule will be added to the BREAK_PATTERN varaible, which is used to yield token
    * candidates.
    *
    * {{{
    * import org.apache.spark.ml.Pipeline
    * import com.johnsnowlabs.nlp.annotators.Tokenizer
    * import com.johnsnowlabs.nlp.DocumentAssembler
    *
    * val textDf = sqlContext.sparkContext.parallelize(Array("I only consider lowercase characters and NOT UPPERCASED and only the numbers 0,1, to 7 as tokens but not 8 or 9")).toDF("text")
    * val documentAssembler = new DocumentAssembler().setInputCol("text").setOutputCol("sentences")
    * val tokenizer = new Tokenizer().setInputCols("sentences").setOutputCol("tokens").setTargetPattern("a-z-0-7")
    * new Pipeline().setStages(Array(documentAssembler, tokenizer)).fit(textDf).transform(textDf).select("tokens.result").show(false)
    * }}}
    *
    * This will yield: `only, consider, lowercase, characters, and, and, only, the, numbers, 0, 1,
    * to, 7, as, tokens, but, not, or`
    * @group param
    */
  val targetPattern: Param[String] = new Param(
    this,
    "targetPattern",
    "Pattern to grab from text as token candidates. Defaults \\S+")

  /** Regex patterns that match tokens within a single target. groups identify different
    * sub-tokens. multiple defaults
    *
    * Infix patterns must use regex group. Notice each group will result in separate token
    *
    * '''Example:'''
    *
    * {{{
    * import org.apache.spark.ml.Pipeline
    * import com.johnsnowlabs.nlp.annotators.Tokenizer
    * import com.johnsnowlabs.nlp.DocumentAssembler
    *
    * val textDf = sqlContext.sparkContext.parallelize(Array("l'une d'un l'un, des l'extrême des l'extreme")).toDF("text")
    * val documentAssembler = new DocumentAssembler().setInputCol("text").setOutputCol("sentences")
    * val tokenizer = new Tokenizer().setInputCols("sentences").setOutputCol("tokens").setInfixPatterns(Array("([\\p{L}\\w]+'{1})([\\p{L}\\w]+)"))
    * new Pipeline().setStages(Array(documentAssembler, tokenizer)).fit(textDf).transform(textDf).select("tokens.result").show(false)
    *
    * }}}
    *
    * This will yield: `l', une, d', un, l', un, , , des, l', extrême, des, l', extreme`
    * @group param
    */
  val infixPatterns: StringArrayParam = new StringArrayParam(
    this,
    "infixPatterns",
    "Regex patterns that match tokens within a single target. groups identify different sub-tokens. multiple defaults")

  /** Regex with groups and begins with \\A to match target prefix. Overrides contextCharacters
    * Param
    *
    * @group param
    */
  val prefixPattern: Param[String] = new Param[String](
    this,
    "prefixPattern",
    "Regex with groups and begins with \\A to match target prefix. Overrides contextCharacters Param")

  /** Regex with groups and ends with \\z to match target suffix. Overrides contextCharacters
    * Param
    *
    * @group param
    */
  val suffixPattern: Param[String] = new Param[String](
    this,
    "suffixPattern",
    "Regex with groups and ends with \\z to match target suffix. Overrides contextCharacters Param")

  /** Set the minimum allowed length for each token
    *
    * @group param
    */
  val minLength = new IntParam(this, "minLength", "Set the minimum allowed length for each token")

  /** Set the minimum allowed length for each token
    * @group setParam
    */
  def setMinLength(value: Int): this.type = {
    require(value >= 0, "minLength must be greater equal than 0")
    require(value.isValidInt, "minLength must be Int")
    set(minLength, value)
  }

  /** Get the minimum allowed length for each token
    * @group getParam
    */
  def getMinLength(value: Int): Int = $(minLength)

  /** Set the maximum allowed length for each token
    *
    * @group param
    */
  val maxLength = new IntParam(this, "maxLength", "Set the maximum allowed length for each token")

  /** Get the maximum allowed length for each token
    * @group setParam
    */
  def setMaxLength(value: Int): this.type = {
    require(
      value >= $ {
        minLength
      },
      "maxLength must be greater equal than minLength")
    require(value.isValidInt, "minLength must be Int")
    set(maxLength, value)
  }

  /** Get the maximum allowed length for each token
    * @group getParam
    */
  def getMaxLength(value: Int): Int = $(maxLength)

  /** Set a basic regex rule to identify token candidates in text.
    * @group setParam
    */
  def setTargetPattern(value: String): this.type = set(targetPattern, value)

  /** Regex pattern to separate from the inside of tokens. Takes priority over splitChars.
    * @group setParam
    */
  def setSplitPattern(value: String): this.type = set(splitPattern, value)

  /** Set a list of Regex patterns that match tokens within a single target. Groups identify
    * different sub-tokens. multiple defaults
    * @group setParam
    */
  def setInfixPatterns(value: Array[String]): this.type = set(infixPatterns, value)

  /** Add an extension pattern regex with groups to the top of thsetExceptionse rules (will target
    * first, from more specific to the more general).
    *
    * @group setParam
    */
  def addInfixPattern(value: String): this.type = set(infixPatterns, value +: $(infixPatterns))

  /** Regex to identify subtokens that come in the beginning of the token. Regex has to start with
    * \\A and must contain groups (). Each group will become a separate token within the prefix.
    * Defaults to non-letter characters. e.g. quotes or parenthesis
    *
    * @group setParam
    */
  def setPrefixPattern(value: String): this.type = set(prefixPattern, value)

  /** Regex to identify subtokens that are in the end of the token. Regex has to end with \\z and
    * must contain groups (). Each group will become a separate token within the prefix. Defaults
    * to non-letter characters. e.g. quotes or parenthesis
    *
    * @group setParam
    */
  def setSuffixPattern(value: String): this.type = set(suffixPattern, value)

  /** List of tokens to not alter at all. Allows composite tokens like two worded tokens that the
    * user may not want to split.
    *
    * @group setParam
    */
  def setExceptions(value: Array[String]): this.type = set(exceptions, value)

  /** Add a single exception
    *
    * @group setParam
    */
  def addException(value: String): this.type =
    set(exceptions, get(exceptions).getOrElse(Array.empty[String]) :+ value)

  /** @group setParam */
  def getExceptions: Array[String] = $(exceptions)

  /** Path to txt file with list of token exceptions
    *
    * @group getParam
    */
  def setExceptionsPath(
      path: String,
      readAs: ReadAs.Format = ReadAs.TEXT,
      options: Map[String, String] = Map("format" -> "text")): this.type =
    set(exceptionsPath, ExternalResource(path, readAs, options))

  /** Whether to follow case sensitiveness for matching exceptions in text
    *
    * @group getParam
    */
  def setCaseSensitiveExceptions(value: Boolean): this.type = set(caseSensitiveExceptions, value)

  /** Whether to follow case sensitiveness for matching exceptions in text
    *
    * @group getParam
    */
  def getCaseSensitiveExceptions(value: Boolean): Boolean = $(caseSensitiveExceptions)

  /** Add an extension pattern regex with groups to the top of the rules (will target first, from
    * more specific to the more general).
    *
    * @group getParam
    */
  def getInfixPatterns: Array[String] = $(infixPatterns)

  /** Regex to identify subtokens that come in the beginning of the token. Regex has to start with
    * \\A and must contain groups (). Each group will become a separate token within the prefix.
    * Defaults to non-letter characters. e.g. quotes or parenthesis
    *
    * @group getParam
    */
  def getPrefixPattern: String = $(prefixPattern)

  /** Regex to identify subtokens that are in the end of the token. Regex has to end with \\z and
    * must contain groups (). Each group will become a separate token within the prefix. Defaults
    * to non-letter characters. e.g. quotes or parenthesis
    *
    * @group getParam
    */
  def getSuffixPattern: String = $(suffixPattern)

  /** Basic regex rule to identify a candidate for tokenization. Defaults to \\S+ which means
    * anything not a space
    *
    * @group getParam
    */
  def getTargetPattern: String = $(targetPattern)

  /** List of 1 character string to split tokens inside, such as hyphens. Ignored if using infix,
    * prefix or suffix patterns.
    *
    * @group getParam
    */
  def getSplitPattern: String = $(splitPattern)

  /** List of 1 character string to rip off from tokens, such as parenthesis or question marks.
    * Ignored if using prefix, infix or suffix patterns.
    *
    * @group setParam
    */
  def setContextChars(v: Array[String]): this.type = {
    require(v.forall(_.length == 1), "All elements in context chars must have length == 1")
    set(contextChars, v)
  }

  /** Add one character string to rip off from tokens, such as parenthesis or question marks.
    * Ignored if using prefix, infix or suffix patterns.
    *
    * @group setParam
    */
  def addContextChars(v: String): this.type = {
    require(v.length == 1, "Context char must have length == 1")
    set(contextChars, get(contextChars).getOrElse(Array.empty[String]) :+ v)
  }

  /** List of 1 character string to rip off from tokens, such as parenthesis or question marks.
    * Ignored if using prefix, infix or suffix patterns.
    *
    * @group getParam
    */
  def getContextChars: Array[String] = {
    $(contextChars)
  }

  /** List of 1 character string to split tokens inside, such as hyphens. Ignored if using infix,
    * prefix or suffix patterns.
    *
    * @group setParam
    */
  def setSplitChars(v: Array[String]): this.type = {
    require(
      v.forall(x => x.length == 1 || (x.length == 2 && x.substring(0, 1) == "\\")),
      "All elements in context chars must have length == 1")
    set(splitChars, v)
  }

  /** One character string to split tokens inside, such as hyphens. Ignored if using infix, prefix
    * or suffix patterns.
    *
    * @group setParam
    */
  def addSplitChars(v: String): this.type = {
    require(
      v.length == 1 || (v.length == 2 && v.substring(0, 1) == "\\"),
      "Context char must have length == 1")
    set(splitChars, get(splitChars).getOrElse(Array.empty[String]) :+ v)
  }

  /** List of 1 character string to split tokens inside, such as hyphens. Ignored if using infix,
    * prefix or suffix patterns.
    *
    * @group getParam
    */
  def getSplitChars: Array[String] = {
    $(splitChars)
  }

  setDefault(
    inputCols -> Array(DOCUMENT),
    outputCol -> "token",
    targetPattern -> "\\S+",
    contextChars -> Array(".", ",", ";", ":", "!", "?", "*", "-", "(", ")", "\"", "'"),
    caseSensitiveExceptions -> true,
    minLength -> 0)

  /** Build rule factory which combines all defined parameters to build regex that is applied to
    * tokens
    */
  def buildRuleFactory: RuleFactory = {
    val rules = ArrayBuffer.empty[String]

    lazy val quotedContext = Pattern.quote($(contextChars).mkString(""))

    val processedPrefix = get(prefixPattern).getOrElse(s"\\A([$quotedContext]*)")
    require(
      processedPrefix.startsWith("\\A"),
      "prefixPattern must begin with \\A to ensure it is the beginning of the string")

    val processedSuffix = get(suffixPattern).getOrElse(s"([$quotedContext]*)\\z")
    require(
      processedSuffix.endsWith("\\z"),
      "suffixPattern must end with \\z to ensure it is the end of the string")

    val processedInfixes =
      get(infixPatterns).getOrElse(Array(s"([^$quotedContext](?:.*[^$quotedContext])*)"))

    require(
      processedInfixes.forall(ip => ip.contains("(") && ip.contains(")")),
      "infix patterns must use regex group. Notice each group will result in separate token")
    processedInfixes.foreach(infix => {
      val ruleBuilder = new StringBuilder
      ruleBuilder.append(processedPrefix)
      ruleBuilder.append(infix)
      ruleBuilder.append(processedSuffix)
      rules.append(ruleBuilder.toString)
    })
    rules.foldLeft(new RuleFactory(MatchStrategy.MATCH_FIRST))((factory, rule) =>
      factory.addRule(rule.r, rule))
  }

  /** Clears out rules and constructs a new rule for every combination of rules provided . The
    * strategy is to catch one token per regex group. User may add its own groups if needs targets
    * to be tokenized separately from the rest
    */
  override def train(
      dataset: Dataset[_],
      recursivePipeline: Option[PipelineModel]): TokenizerModel = {

    /** Clears out rules and constructs a new rule for every combination of rules provided */
    /** The strategy is to catch one token per regex group */
    /** User may add its own groups if needs targets to be tokenized separately from the rest */
    val ruleFactory = buildRuleFactory

    val processedExceptions = get(exceptionsPath)
      .map(er => ResourceHelper.parseLines(er))
      .getOrElse(Array.empty[String]) ++ get(exceptions).getOrElse(Array.empty[String])

    val raw = new TokenizerModel()
      .setCaseSensitiveExceptions($(caseSensitiveExceptions))
      .setTargetPattern($(targetPattern))
      .setRules(ruleFactory)
      .setMinLength($(minLength))

    if (isDefined(maxLength))
      raw.setMaxLength($(maxLength))

    if (processedExceptions.nonEmpty)
      raw.setExceptions(processedExceptions)

    if (isSet(splitPattern)) raw.setSplitPattern($(splitPattern))

    if (isSet(splitChars)) raw.setSplitChars($(splitChars))

    raw

  }

}

/** This is the companion object of [[Tokenizer]]. Please refer to that class for the
  * documentation.
  */
object Tokenizer extends DefaultParamsReadable[Tokenizer]

JohnSnowLabs / spark-nlp / 11429325160

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous