2501539046

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/6935ab46c57dfdbf732422d4c677f2ff5cab89aa">6935ab46c">&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/</a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/6935ab46c57dfdbf732422d4c677f2ff5cab89aa">6935ab46c</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/6935ab46c57dfdbf732422d4c677f2ff5cab89aa">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/6935ab46c57dfdbf732422d4c677f2ff5cab89aa&quot;&gt;6935ab46c&lt;/a&gt;&lt;a href=&q... (continued)

Pull Request Pull Request #8320: Release/400 release candidate

Run Details

337 of 337 new or added lines in 18 files covered. (100.0%)

8203 of 11235 relevant lines covered (73.01%)

0.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.44

/src/main/scala/com/johnsnowlabs/nlp/annotators/DocumentNormalizer.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators

import com.johnsnowlabs.nlp.AnnotatorType.DOCUMENT
import com.johnsnowlabs.nlp.{Annotation, AnnotatorModel, AnnotatorType, HasSimpleAnnotate}
import org.apache.spark.ml.param.{BooleanParam, Param, StringArrayParam}
import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}

import java.nio.charset.{Charset, StandardCharsets}
import scala.util.{Failure, Success, Try}
import scala.xml.XML

/** Annotator which normalizes raw text from tagged text, e.g. scraped web pages or xml documents,
  * from document type columns into Sentence. Removes all dirty characters from text following one
  * or more input regex patterns. Can apply not wanted character removal with a specific policy.
  * Can apply lower case normalization.
  *
  * For extended examples of usage, see the
  * [[https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb Spark NLP Workshop]].
  *
  * ==Example==
  * {{{
  * import spark.implicits._
  * import com.johnsnowlabs.nlp.DocumentAssembler
  * import com.johnsnowlabs.nlp.annotator.DocumentNormalizer
  * import org.apache.spark.ml.Pipeline
  *
  * val documentAssembler = new DocumentAssembler()
  *   .setInputCol("text")
  *   .setOutputCol("document")
  *
  * val cleanUpPatterns = Array("<[^>]*>")
  *
  * val documentNormalizer = new DocumentNormalizer()
  *   .setInputCols("document")
  *   .setOutputCol("normalizedDocument")
  *   .setAction("clean")
  *   .setPatterns(cleanUpPatterns)
  *   .setReplacement(" ")
  *   .setPolicy("pretty_all")
  *   .setLowercase(true)
  *
  * val pipeline = new Pipeline().setStages(Array(
  *   documentAssembler,
  *   documentNormalizer
  * ))
  *
  * val text =
  *   """
  * <div id="theworldsgreatest" class='my-right my-hide-small my-wide toptext' style="font-family:'Segoe UI',Arial,sans-serif">
  *   THE WORLD'S LARGEST WEB DEVELOPER SITE
  *   <h1 style="font-size:300%;">THE WORLD'S LARGEST WEB DEVELOPER SITE</h1>
  *   <p style="font-size:160%;">Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum..</p>
  * </div>
  *
  * </div>"""
  * val data = Seq(text).toDF("text")
  * val pipelineModel = pipeline.fit(data)
  *
  * val result = pipelineModel.transform(data)
  * result.selectExpr("normalizedDocument.result").show(truncate=false)
  * +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
  * |result                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
  * +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
  * |[ the world's largest web developer site the world's largest web developer site lorem ipsum is simply dummy text of the printing and typesetting industry. lorem ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. it has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. it was popularised in the 1960s with the release of letraset sheets containing lorem ipsum passages, and more recently with desktop publishing software like aldus pagemaker including versions of lorem ipsum..]|
  * +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
  * }}}
  * @param uid
  *   required uid for storing annotator to disk
  * @groupname anno Annotator types
  * @groupdesc anno
  *   Required input and expected output annotator types
  * @groupname Ungrouped Members
  * @groupname param Parameters
  * @groupname setParam Parameter setters
  * @groupname getParam Parameter getters
  * @groupname Ungrouped Members
  * @groupprio param  1
  * @groupprio anno  2
  * @groupprio Ungrouped 3
  * @groupprio setParam  4
  * @groupprio getParam  5
  * @groupdesc param
  *   A list of (hyper-)parameter keys this annotator can take. Users can set and get the
  *   parameter values through setters and getters, respectively.
  */
class DocumentNormalizer(override val uid: String)
    extends AnnotatorModel[DocumentNormalizer]
    with HasSimpleAnnotate[DocumentNormalizer] {

  private val EMPTY_STR = ""
  private val BREAK_STR = "|##|"
  private val SPACE_STR = " "
  private val GENERIC_TAGS_REMOVAL_PATTERN = "<[^>]*>"

  /** Input annotator type : DOCUMENT
    *
    * @group anno
    */
  override val inputAnnotatorTypes: Array[AnnotatorType] = Array[AnnotatorType](DOCUMENT)

  /** Output annotator type : DOCUMENT
    *
    * @group anno
    */
  override val outputAnnotatorType: AnnotatorType = DOCUMENT

  def this() = this(Identifiable.randomUID("DOCUMENT_NORMALIZER"))

  /** Action to perform applying regex patterns on text
    *
    * @group param
    */
  val action: Param[String] =
    new Param(this, "action", "Action to perform applying regex patterns on text")

  /** Normalization regex patterns which match will be removed from document (Default:
    * `Array("<[^>]*>")`)
    *
    * @group param
    */
  val patterns: StringArrayParam = new StringArrayParam(
    this,
    "patterns",
    "Normalization regex patterns which match will be removed from document. Defaults is \"<[^>]*>\"")

  /** Replacement string to apply when regexes match (Default: `" "`)
    *
    * @group param
    */
  val replacement: Param[String] =
    new Param(this, "replacement", "Replacement string to apply when regexes match")

  /** Whether to convert strings to lowercase (Default: `false`)
    *
    * @group param
    */
  val lowercase = new BooleanParam(
    this,
    "lowercase",
    "Whether to convert strings to lowercase (Default: `false`)")

  /** RemovalPolicy to remove patterns from text with a given policy (Default: `"pretty_all"`).
    * Possible values are `"all", "pretty_all", "first", "pretty_first"`
    * @group param
    */
  val policy: Param[String] =
    new Param(this, "policy", "RemovalPolicy to remove pattern from text")

  /** File encoding to apply on normalized documents (Default: `"disable"`)
    *
    * @group param
    */
  val encoding: Param[String] = new Param(
    this,
    name = "encoding",
    "File encoding to apply on normalized documents (Default: `disable`)")

  //  Assuming non-html does not contain any < or > and that input string is correctly structured
  setDefault(
    inputCols -> Array(AnnotatorType.DOCUMENT),
    action -> "clean",
    patterns -> Array(GENERIC_TAGS_REMOVAL_PATTERN),
    replacement -> SPACE_STR,
    lowercase -> false,
    policy -> "pretty_all",
    encoding -> "disable")

  /** Action to perform on text. (Default `"clean"`).
    *
    * @group getParam
    */
  def getAction: String = $(action)

  /** Regular expressions list for normalization.
    *
    * @group getParam
    */
  def getPatterns: Array[String] = $(patterns)

  /** Replacement string to apply when regexes match (Default: `" "`)
    *
    * @group getParam
    */
  def getReplacement: String = $(replacement)

  /** Lowercase tokens (Default: `false`)
    *
    * @group getParam
    */
  def getLowercase: Boolean = $(lowercase)

  /** Policy to remove patterns from text (Default: `"pretty_all"`)
    *
    * @group getParam
    */
  def getPolicy: String = $(policy)

  /** Encoding to apply to normalized documents (Default: `"disable"`)
    *
    * @group getParam
    */
  def getEncoding: String = $(encoding)

  /** Action to perform on text. (Default `"clean"`).
    *
    * @group getParam
    */
  def setAction(value: String): this.type = set(action, value)

  /** Regular expressions list for normalization (Default: `Array("<[^>]*>")`)
    *
    * @group setParam
    */
  def setPatterns(value: Array[String]): this.type = set(patterns, value)

  /** Replacement string to apply when regexes match (Default: `" "`)
    *
    * @group getParam
    */
  def setReplacement(value: String): this.type = set(replacement, value)

  /** Lower case tokens default false
    *
    * @group setParam
    */
  def setLowercase(value: Boolean): this.type = set(lowercase, value)

  /** Removal policy to apply (Default: `"pretty_all"`). Valid policy values are: "all",
    * "pretty_all", "first", "pretty_first"
    *
    * @group setParam
    */
  def setPolicy(value: String): this.type = set(policy, value)

  /** Encoding to apply. Default is `"UTF-8"`. Valid encoding are values are: UTF_8, UTF_16,
    * US_ASCII, ISO-8859-1, UTF-16BE, UTF-16LE
    *
    * @group setParam
    */
  def setEncoding(value: String): this.type = set(encoding, value)

  /** Applying document normalization without pretty formatting (removing multiple spaces) */
  private def withAllFormatter(
      text: String,
      action: String,
      patterns: Array[String],
      replacement: String): String = {
    action match {
      case "clean" =>
        val patternsStr: String = patterns.mkString(BREAK_STR)
        text.replaceAll(patternsStr, replacement)
      case "extract" => {
        val htmlXml = XML.loadString(text)
        val textareaContents = (htmlXml \\ patterns.mkString).text
        textareaContents
      }
      case _ =>
        throw new Exception(
          "Unknown action parameter in DocumentNormalizer annotation." +
            "Please select either: clean or extract")
    }
  }

  /** pattern to grab from text as token candidates. Defaults \\S+ */
  private def withPrettyAllFormatter(
      text: String,
      action: String,
      patterns: Array[String],
      replacement: String): String = {
    withAllFormatter(text, action, patterns, replacement)
      .split("\\s+")
      .map(_.trim)
      .mkString(SPACE_STR)
  }

  /** Applying document normalization without pretty formatting (removing multiple spaces)
    * retrieving first element only
    */
  private def withFirstFormatter(
      text: String,
      action: String,
      patterns: Array[String],
      replacement: String): String = {
    action match {
      case "clean" =>
        val patternsStr: String = patterns.mkString(BREAK_STR)
        text.replaceFirst(patternsStr, replacement)
      case "extract" => {
        val htmlXml = XML.loadString(text)
        val textareaContents = htmlXml \\ patterns.mkString
        textareaContents.head.mkString
      }
      case _ =>
        throw new Exception(
          "Unknown action parameter in DocumentNormalizer annotation." +
            "Please select either: clean or extract")
    }
  }

  /** pattern to grab from text as token candidates. Defaults \\S+ */
  private def withPrettyFirstFormatter(
      text: String,
      action: String,
      patterns: Array[String],
      replacement: String): String = {
    withFirstFormatter(text, action, patterns, replacement)
      .split("\\s+")
      .map(_.trim)
      .mkString(SPACE_STR)
  }

  /** Apply a given encoding to the processed text
    *
    * US-ASCII Seven-bit ASCII, a.k.a. ISO646-US, a.k.a. the Basic Latin block of the Unicode
    * character set
    *
    * ISO-8859-1 ISO Latin Alphabet No. 1, a.k.a. ISO-LATIN-1
    *
    * UTF-8 Eight-bit UCS Transformation Format
    *
    * UTF-16BE Sixteen-bit UCS Transformation Format, big-endian byte order
    *
    * UTF-16LE Sixteen-bit UCS Transformation Format, little-endian byte order
    *
    * UTF-16 Sixteen-bit UCS Transformation Format, byte order identified by an optional
    * byte-order mark
    */
  private def withEncoding(text: String, encoding: Charset = StandardCharsets.UTF_8): String = {
    val defaultCharset: Charset = Charset.defaultCharset
    if (!Charset.defaultCharset.equals(encoding)) {
      log.warn("Requested encoding parameter is different from the default charset.")
    }
    new String(text.getBytes(defaultCharset), encoding)
  }

  /** Apply document normalization on text using action, patterns, policy, lowercase and encoding
    * parameters.
    */
  private def applyDocumentNormalization(
      text: String,
      action: String,
      patterns: Array[String],
      replacement: String,
      policy: String,
      lowercase: Boolean,
      encoding: String): String = {

    val processedWithActionPatterns: String = policy match {
      case "all" => withAllFormatter(text, action, patterns, replacement)
      case "pretty_all" => withPrettyAllFormatter(text, action, patterns, replacement)
      case "first" => withFirstFormatter(text, action, patterns, replacement)
      case "pretty_first" => withPrettyFirstFormatter(text, action, patterns, replacement)
      case _ =>
        throw new Exception(
          "Unknown policy parameter in DocumentNormalizer annotation." +
            "Please select either: all, pretty_all, first, or pretty_first")
    }

    val cased =
      if (lowercase)
        processedWithActionPatterns.toLowerCase
      else
        processedWithActionPatterns

    encoding match {
      case "disable" => cased
      case "UTF-8" => withEncoding(cased, StandardCharsets.UTF_8)
      case "UTF-16" => withEncoding(cased, StandardCharsets.UTF_16)
      case "US-ASCII" => withEncoding(cased, StandardCharsets.US_ASCII)
      case "ISO-8859-1" => withEncoding(cased, StandardCharsets.ISO_8859_1)
      case "UTF-16BE" => withEncoding(cased, StandardCharsets.UTF_16BE)
      case "UTF-16LE" => withEncoding(cased, StandardCharsets.UTF_16LE)
      case _ =>
        throw new Exception("Unknown encoding parameter in DocumentNormalizer annotation." +
          "Please select either: disable, UTF_8, UTF_16, US_ASCII, ISO-8859-1, UTF-16BE, UTF-16LE")
    }
  }

  override def annotate(annotations: Seq[Annotation]): Seq[Annotation] = annotations.map {
    annotation =>
      Try(
        applyDocumentNormalization(
          annotation.result,
          getAction,
          getPatterns,
          getReplacement,
          getPolicy,
          getLowercase,
          getEncoding)) match {
        case Success(cleanedDoc) =>
          Annotation(
            DOCUMENT,
            annotation.begin,
            cleanedDoc.length - 1,
            cleanedDoc,
            annotation.metadata)
        case Failure(_) =>
          Annotation.apply("")
      }
  }
}

/** This is the companion object of [[DocumentNormalizer]]. Please refer to that class for the
  * documentation.
  */
object DocumentNormalizer extends DefaultParamsReadable[DocumentNormalizer]

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.annotators
18
19	import com.johnsnowlabs.nlp.AnnotatorType.DOCUMENT
20	import com.johnsnowlabs.nlp.{Annotation, AnnotatorModel, AnnotatorType, HasSimpleAnnotate}
21	import org.apache.spark.ml.param.{BooleanParam, Param, StringArrayParam}
22	import org.apache.spark.ml.util.{DefaultParamsReadable, Identifiable}
23
24	import java.nio.charset.{Charset, StandardCharsets}
25	import scala.util.{Failure, Success, Try}
26	import scala.xml.XML
27
28	/** Annotator which normalizes raw text from tagged text, e.g. scraped web pages or xml documents,
29	* from document type columns into Sentence. Removes all dirty characters from text following one
30	* or more input regex patterns. Can apply not wanted character removal with a specific policy.
31	* Can apply lower case normalization.
32	*
33	* For extended examples of usage, see the
34	* [[https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb Spark NLP Workshop]].
35	*
36	* ==Example==
37	* {{{
38	* import spark.implicits._
39	* import com.johnsnowlabs.nlp.DocumentAssembler
40	* import com.johnsnowlabs.nlp.annotator.DocumentNormalizer
41	* import org.apache.spark.ml.Pipeline
42	*
43	* val documentAssembler = new DocumentAssembler()
44	* .setInputCol("text")
45	* .setOutputCol("document")
46	*
47	* val cleanUpPatterns = Array("<[^>]*>")
48	*
49	* val documentNormalizer = new DocumentNormalizer()
50	* .setInputCols("document")
51	* .setOutputCol("normalizedDocument")
52	* .setAction("clean")
53	* .setPatterns(cleanUpPatterns)
54	* .setReplacement(" ")
55	* .setPolicy("pretty_all")
56	* .setLowercase(true)
57	*
58	* val pipeline = new Pipeline().setStages(Array(
59	* documentAssembler,
60	* documentNormalizer
61	* ))
62	*
63	* val text =
64	* """
65	* <div id="theworldsgreatest" class='my-right my-hide-small my-wide toptext' style="font-family:'Segoe UI',Arial,sans-serif">
66	* THE WORLD'S LARGEST WEB DEVELOPER SITE
67	* <h1 style="font-size:300%;">THE WORLD'S LARGEST WEB DEVELOPER SITE</h1>
68	* <p style="font-size:160%;">Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum..</p>
69	* </div>
70	*
71	* </div>"""
72	* val data = Seq(text).toDF("text")
73	* val pipelineModel = pipeline.fit(data)
74	*
75	* val result = pipelineModel.transform(data)
76	* result.selectExpr("normalizedDocument.result").show(truncate=false)
77	* +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
78	* \|result \|
79	* +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
80	* \|[ the world's largest web developer site the world's largest web developer site lorem ipsum is simply dummy text of the printing and typesetting industry. lorem ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. it has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. it was popularised in the 1960s with the release of letraset sheets containing lorem ipsum passages, and more recently with desktop publishing software like aldus pagemaker including versions of lorem ipsum..]\|
81	* +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
82	* }}}
83	* @param uid
84	* required uid for storing annotator to disk
85	* @groupname anno Annotator types
86	* @groupdesc anno
87	* Required input and expected output annotator types
88	* @groupname Ungrouped Members
89	* @groupname param Parameters
90	* @groupname setParam Parameter setters
91	* @groupname getParam Parameter getters
92	* @groupname Ungrouped Members
93	* @groupprio param 1
94	* @groupprio anno 2
95	* @groupprio Ungrouped 3
96	* @groupprio setParam 4
97	* @groupprio getParam 5
98	* @groupdesc param
99	* A list of (hyper-)parameter keys this annotator can take. Users can set and get the
100	* parameter values through setters and getters, respectively.
101	*/
102	class DocumentNormalizer(override val uid: String)
103	extends AnnotatorModel[DocumentNormalizer]
104	with HasSimpleAnnotate[DocumentNormalizer] {
105
106	private val EMPTY_STR = ""	1✔
107	private val BREAK_STR = "\|##\|"	1✔
108	private val SPACE_STR = " "	1✔
109	private val GENERIC_TAGS_REMOVAL_PATTERN = "<[^>]*>"	1✔
110
111	/** Input annotator type : DOCUMENT
112	*
113	* @group anno
114	*/
115	override val inputAnnotatorTypes: Array[AnnotatorType] = Array[AnnotatorType](DOCUMENT)	1✔
116
117	/** Output annotator type : DOCUMENT
118	*
119	* @group anno
120	*/
121	override val outputAnnotatorType: AnnotatorType = DOCUMENT	1✔
122
123	def this() = this(Identifiable.randomUID("DOCUMENT_NORMALIZER"))	1✔
124
125	/** Action to perform applying regex patterns on text
126	*
127	* @group param
128	*/
129	val action: Param[String] =
130	new Param(this, "action", "Action to perform applying regex patterns on text")	1✔
131
132	/** Normalization regex patterns which match will be removed from document (Default:
133	* `Array("<[^>]*>")`)
134	*
135	* @group param
136	*/
137	val patterns: StringArrayParam = new StringArrayParam(	1✔
138	this,
139	"patterns",	1✔
140	"Normalization regex patterns which match will be removed from document. Defaults is \"<[^>]*>\"")	1✔
141
142	/** Replacement string to apply when regexes match (Default: `" "`)
143	*
144	* @group param
145	*/
146	val replacement: Param[String] =
147	new Param(this, "replacement", "Replacement string to apply when regexes match")	1✔
148
149	/** Whether to convert strings to lowercase (Default: `false`)
150	*
151	* @group param
152	*/
153	val lowercase = new BooleanParam(	1✔
154	this,
155	"lowercase",	1✔
156	"Whether to convert strings to lowercase (Default: `false`)")	1✔
157
158	/** RemovalPolicy to remove patterns from text with a given policy (Default: `"pretty_all"`).
159	* Possible values are `"all", "pretty_all", "first", "pretty_first"`
160	* @group param
161	*/
162	val policy: Param[String] =
163	new Param(this, "policy", "RemovalPolicy to remove pattern from text")	1✔
164
165	/** File encoding to apply on normalized documents (Default: `"disable"`)
166	*
167	* @group param
168	*/
169	val encoding: Param[String] = new Param(	1✔
170	this,
171	name = "encoding",	1✔
172	"File encoding to apply on normalized documents (Default: `disable`)")	1✔
173
174	// Assuming non-html does not contain any < or > and that input string is correctly structured
175	setDefault(	1✔
176	inputCols -> Array(AnnotatorType.DOCUMENT),	1✔
177	action -> "clean",	1✔
178	patterns -> Array(GENERIC_TAGS_REMOVAL_PATTERN),	1✔
179	replacement -> SPACE_STR,	1✔
180	lowercase -> false,	1✔
181	policy -> "pretty_all",	1✔
182	encoding -> "disable")	1✔
183
184	/** Action to perform on text. (Default `"clean"`).
185	*
186	* @group getParam
187	*/
188	def getAction: String = $(action)	1✔
189
190	/** Regular expressions list for normalization.
191	*
192	* @group getParam
193	*/
194	def getPatterns: Array[String] = $(patterns)	1✔
195
196	/** Replacement string to apply when regexes match (Default: `" "`)
197	*
198	* @group getParam
199	*/
200	def getReplacement: String = $(replacement)	1✔
201
202	/** Lowercase tokens (Default: `false`)
203	*
204	* @group getParam
205	*/
206	def getLowercase: Boolean = $(lowercase)	1✔
207
208	/** Policy to remove patterns from text (Default: `"pretty_all"`)
209	*
210	* @group getParam
211	*/
212	def getPolicy: String = $(policy)	1✔
213
214	/** Encoding to apply to normalized documents (Default: `"disable"`)
215	*
216	* @group getParam
217	*/
218	def getEncoding: String = $(encoding)	1✔
219
220	/** Action to perform on text. (Default `"clean"`).
221	*
222	* @group getParam
223	*/
224	def setAction(value: String): this.type = set(action, value)	1✔
225
226	/** Regular expressions list for normalization (Default: `Array("<[^>]*>")`)
227	*
228	* @group setParam
229	*/
230	def setPatterns(value: Array[String]): this.type = set(patterns, value)	1✔
231
232	/** Replacement string to apply when regexes match (Default: `" "`)
233	*
234	* @group getParam
235	*/
236	def setReplacement(value: String): this.type = set(replacement, value)	1✔
237
238	/** Lower case tokens default false
239	*
240	* @group setParam
241	*/
242	def setLowercase(value: Boolean): this.type = set(lowercase, value)	1✔
243
244	/** Removal policy to apply (Default: `"pretty_all"`). Valid policy values are: "all",
245	* "pretty_all", "first", "pretty_first"
246	*
247	* @group setParam
248	*/
249	def setPolicy(value: String): this.type = set(policy, value)	1✔
250
251	/** Encoding to apply. Default is `"UTF-8"`. Valid encoding are values are: UTF_8, UTF_16,
252	* US_ASCII, ISO-8859-1, UTF-16BE, UTF-16LE
253	*
254	* @group setParam
255	*/
256	def setEncoding(value: String): this.type = set(encoding, value)	1✔
257
258	/** Applying document normalization without pretty formatting (removing multiple spaces) */
259	private def withAllFormatter(
260	text: String,
261	action: String,
262	patterns: Array[String],
263	replacement: String): String = {
264	action match {
265	case "clean" =>
266	val patternsStr: String = patterns.mkString(BREAK_STR)	1✔
267	text.replaceAll(patternsStr, replacement)	1✔
268	case "extract" => {
269	val htmlXml = XML.loadString(text)	1✔
270	val textareaContents = (htmlXml \\ patterns.mkString).text	1✔
271	textareaContents
272	}
273	case _ =>
274	throw new Exception(	×
275	"Unknown action parameter in DocumentNormalizer annotation." +
276	"Please select either: clean or extract")
277	}
278	}
279
280	/** pattern to grab from text as token candidates. Defaults \\S+ */
281	private def withPrettyAllFormatter(
282	text: String,
283	action: String,
284	patterns: Array[String],
285	replacement: String): String = {
286	withAllFormatter(text, action, patterns, replacement)
287	.split("\\s+")	1✔
288	.map(_.trim)	1✔
289	.mkString(SPACE_STR)	1✔
290	}
291
292	/** Applying document normalization without pretty formatting (removing multiple spaces)
293	* retrieving first element only
294	*/
295	private def withFirstFormatter(
296	text: String,
297	action: String,
298	patterns: Array[String],
299	replacement: String): String = {
300	action match {
301	case "clean" =>
302	val patternsStr: String = patterns.mkString(BREAK_STR)	×
303	text.replaceFirst(patternsStr, replacement)	×
304	case "extract" => {
305	val htmlXml = XML.loadString(text)	×
306	val textareaContents = htmlXml \\ patterns.mkString	×
307	textareaContents.head.mkString	×
308	}
309	case _ =>
310	throw new Exception(	×
311	"Unknown action parameter in DocumentNormalizer annotation." +
312	"Please select either: clean or extract")
313	}
314	}
315
316	/** pattern to grab from text as token candidates. Defaults \\S+ */
317	private def withPrettyFirstFormatter(
318	text: String,
319	action: String,
320	patterns: Array[String],
321	replacement: String): String = {
322	withFirstFormatter(text, action, patterns, replacement)
323	.split("\\s+")	×
324	.map(_.trim)	×
325	.mkString(SPACE_STR)	×
326	}
327
328	/** Apply a given encoding to the processed text
329	*
330	* US-ASCII Seven-bit ASCII, a.k.a. ISO646-US, a.k.a. the Basic Latin block of the Unicode
331	* character set
332	*
333	* ISO-8859-1 ISO Latin Alphabet No. 1, a.k.a. ISO-LATIN-1
334	*
335	* UTF-8 Eight-bit UCS Transformation Format
336	*
337	* UTF-16BE Sixteen-bit UCS Transformation Format, big-endian byte order
338	*
339	* UTF-16LE Sixteen-bit UCS Transformation Format, little-endian byte order
340	*
341	* UTF-16 Sixteen-bit UCS Transformation Format, byte order identified by an optional
342	* byte-order mark
343	*/
344	private def withEncoding(text: String, encoding: Charset = StandardCharsets.UTF_8): String = {
345	val defaultCharset: Charset = Charset.defaultCharset	1✔
346	if (!Charset.defaultCharset.equals(encoding)) {	1✔
347	log.warn("Requested encoding parameter is different from the default charset.")	×
348	}
349	new String(text.getBytes(defaultCharset), encoding)	1✔
350	}
351
352	/** Apply document normalization on text using action, patterns, policy, lowercase and encoding
353	* parameters.
354	*/
355	private def applyDocumentNormalization(
356	text: String,
357	action: String,
358	patterns: Array[String],
359	replacement: String,
360	policy: String,
361	lowercase: Boolean,
362	encoding: String): String = {
363
364	val processedWithActionPatterns: String = policy match {
365	case "all" => withAllFormatter(text, action, patterns, replacement)	×
366	case "pretty_all" => withPrettyAllFormatter(text, action, patterns, replacement)	1✔
367	case "first" => withFirstFormatter(text, action, patterns, replacement)	×
368	case "pretty_first" => withPrettyFirstFormatter(text, action, patterns, replacement)	×
369	case _ =>
370	throw new Exception(	×
371	"Unknown policy parameter in DocumentNormalizer annotation." +
372	"Please select either: all, pretty_all, first, or pretty_first")
373	}
374
375	val cased =
376	if (lowercase)
377	processedWithActionPatterns.toLowerCase	×
378	else
379	processedWithActionPatterns	1✔
380
381	encoding match {
382	case "disable" => cased
383	case "UTF-8" => withEncoding(cased, StandardCharsets.UTF_8)	1✔
384	case "UTF-16" => withEncoding(cased, StandardCharsets.UTF_16)	×
385	case "US-ASCII" => withEncoding(cased, StandardCharsets.US_ASCII)	×
386	case "ISO-8859-1" => withEncoding(cased, StandardCharsets.ISO_8859_1)	×
387	case "UTF-16BE" => withEncoding(cased, StandardCharsets.UTF_16BE)	×
388	case "UTF-16LE" => withEncoding(cased, StandardCharsets.UTF_16LE)	×
389	case _ =>
390	throw new Exception("Unknown encoding parameter in DocumentNormalizer annotation." +	×
391	"Please select either: disable, UTF_8, UTF_16, US_ASCII, ISO-8859-1, UTF-16BE, UTF-16LE")
392	}
393	}
394
395	override def annotate(annotations: Seq[Annotation]): Seq[Annotation] = annotations.map {	1✔
396	annotation =>
397	Try(	1✔
398	applyDocumentNormalization(	1✔
399	annotation.result,	1✔
400	getAction,	1✔
401	getPatterns,	1✔
402	getReplacement,	1✔
403	getPolicy,	1✔
404	getLowercase,	1✔
405	getEncoding)) match {	1✔
406	case Success(cleanedDoc) =>
407	Annotation(	1✔
408	DOCUMENT,	1✔
409	annotation.begin,	1✔
410	cleanedDoc.length - 1,	1✔
411	cleanedDoc,
412	annotation.metadata)	1✔
413	case Failure(_) =>
414	Annotation.apply("")	×
415	}
416	}
417	}
418
419	/** This is the companion object of [[DocumentNormalizer]]. Please refer to that class for the
420	* documentation.
421	*/
422	object DocumentNormalizer extends DefaultParamsReadable[DocumentNormalizer]

JohnSnowLabs / spark-nlp / 2501539046

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous