5210642866

pending completion

Build # 5210642866

Build Type

Pull #13848

github

Committed by

web-flow

Commit Message

Merge d7636a247 into ddf39b36d

Pull Request Pull Request #13848: release/444-release-candidate

Run Details

3 of 3 new or added lines in 3 files covered. (100.0%)

8644 of 13124 relevant lines covered (65.86%)

0.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/nlp/SparkNLP.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp

import org.apache.spark.sql.SparkSession

object SparkNLP {

  val currentVersion = "4.4.4"
  val MavenSpark3 = s"com.johnsnowlabs.nlp:spark-nlp_2.12:$currentVersion"
  val MavenGpuSpark3 = s"com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:$currentVersion"
  val MavenSparkSilicon = s"com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:$currentVersion"
  val MavenSparkAarch64 = s"com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:$currentVersion"

  /** Start SparkSession with Spark NLP
    *
    * @param gpu
    *   start Spark NLP with GPU
    * @param apple_silicon
    *   start Spark NLP for Apple M1 & M2 systems
    * @param aarch64
    *   start Spark NLP for Linux Aarch64 systems
    * @param memory
    *   set driver memory for SparkSession
    * @param cache_folder
    *   The location to download and extract pretrained Models and Pipelines (by default, it will
    *   be in the users home directory under `cache_pretrained`.)
    * @param log_folder
    *   The location to use on a cluster for temporarily files such as unpacking indexes for
    *   WordEmbeddings. By default, this locations is the location of `hadoop.tmp.dir` set via
    *   Hadoop configuration for Apache Spark. NOTE: `S3` is not supported and it must be local,
    *   HDFS, or DBFS.
    * @param cluster_tmp_dir
    *   The location to save logs from annotators during training (By default, it will be in the
    *   users home directory under `annotator_logs`.)
    * @param params
    *   Custom parameters to set for the Spark configuration (Default: `Map.empty`)
    * @return
    *   SparkSession
    */
  def start(
      gpu: Boolean = false,
      apple_silicon: Boolean = false,
      aarch64: Boolean = false,
      memory: String = "16G",
      cache_folder: String = "",
      log_folder: String = "",
      cluster_tmp_dir: String = "",
      params: Map[String, String] = Map.empty): SparkSession = {

    if (SparkSession.getActiveSession.isDefined)
      println("Warning: Spark Session already created, some configs may not be applied.")

    val builder = SparkSession
      .builder()
      .appName("Spark NLP")
      .master("local[*]")
      .config("spark.driver.memory", memory)
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .config("spark.kryoserializer.buffer.max", "2000M")
      .config("spark.driver.maxResultSize", "0")

    val sparkNlpJar =
      if (apple_silicon) MavenSparkSilicon
      else if (aarch64) MavenSparkAarch64
      else if (gpu) MavenGpuSpark3
      else MavenSpark3

    if (!params.contains("spark.jars.packages")) {
      builder.config("spark.jars.packages", sparkNlpJar)
    }

    params.foreach {
      case (key, value) if key == "spark.jars.packages" =>
        builder.config(key, sparkNlpJar + "," + value)
      case (key, value) =>
        builder.config(key, value)
    }

    if (cache_folder.nonEmpty)
      builder.config("spark.jsl.settings.pretrained.cache_folder", cache_folder)

    if (log_folder.nonEmpty)
      builder.config("spark.jsl.settings.annotator.log_folder", log_folder)

    if (cluster_tmp_dir.nonEmpty)
      builder.config("spark.jsl.settings.storage.cluster_tmp_dir", cluster_tmp_dir)

    builder.getOrCreate()
  }

  def version(): String = {
    currentVersion
  }

}

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp
18
19	import org.apache.spark.sql.SparkSession
20
21	object SparkNLP {
22
23	val currentVersion = "4.4.4"	×
24	val MavenSpark3 = s"com.johnsnowlabs.nlp:spark-nlp_2.12:$currentVersion"	×
25	val MavenGpuSpark3 = s"com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:$currentVersion"	×
26	val MavenSparkSilicon = s"com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:$currentVersion"	×
27	val MavenSparkAarch64 = s"com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:$currentVersion"	×
28
29	/** Start SparkSession with Spark NLP
30	*
31	* @param gpu
32	* start Spark NLP with GPU
33	* @param apple_silicon
34	* start Spark NLP for Apple M1 & M2 systems
35	* @param aarch64
36	* start Spark NLP for Linux Aarch64 systems
37	* @param memory
38	* set driver memory for SparkSession
39	* @param cache_folder
40	* The location to download and extract pretrained Models and Pipelines (by default, it will
41	* be in the users home directory under `cache_pretrained`.)
42	* @param log_folder
43	* The location to use on a cluster for temporarily files such as unpacking indexes for
44	* WordEmbeddings. By default, this locations is the location of `hadoop.tmp.dir` set via
45	* Hadoop configuration for Apache Spark. NOTE: `S3` is not supported and it must be local,
46	* HDFS, or DBFS.
47	* @param cluster_tmp_dir
48	* The location to save logs from annotators during training (By default, it will be in the
49	* users home directory under `annotator_logs`.)
50	* @param params
51	* Custom parameters to set for the Spark configuration (Default: `Map.empty`)
52	* @return
53	* SparkSession
54	*/
55	def start(
56	gpu: Boolean = false,
57	apple_silicon: Boolean = false,
58	aarch64: Boolean = false,
59	memory: String = "16G",
60	cache_folder: String = "",
61	log_folder: String = "",
62	cluster_tmp_dir: String = "",
63	params: Map[String, String] = Map.empty): SparkSession = {
64
65	if (SparkSession.getActiveSession.isDefined)	×
66	println("Warning: Spark Session already created, some configs may not be applied.")	×
67
68	val builder = SparkSession
69	.builder()
70	.appName("Spark NLP")
71	.master("local[*]")
72	.config("spark.driver.memory", memory)
73	.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
74	.config("spark.kryoserializer.buffer.max", "2000M")
75	.config("spark.driver.maxResultSize", "0")	×
76
77	val sparkNlpJar =
78	if (apple_silicon) MavenSparkSilicon	×
79	else if (aarch64) MavenSparkAarch64	×
80	else if (gpu) MavenGpuSpark3	×
81	else MavenSpark3	×
82
83	if (!params.contains("spark.jars.packages")) {	×
84	builder.config("spark.jars.packages", sparkNlpJar)	×
85	}
86
87	params.foreach {	×
88	case (key, value) if key == "spark.jars.packages" =>	×
89	builder.config(key, sparkNlpJar + "," + value)	×
90	case (key, value) =>
91	builder.config(key, value)	×
92	}
93
94	if (cache_folder.nonEmpty)	×
95	builder.config("spark.jsl.settings.pretrained.cache_folder", cache_folder)	×
96
97	if (log_folder.nonEmpty)	×
98	builder.config("spark.jsl.settings.annotator.log_folder", log_folder)	×
99
100	if (cluster_tmp_dir.nonEmpty)	×
101	builder.config("spark.jsl.settings.storage.cluster_tmp_dir", cluster_tmp_dir)	×
102
103	builder.getOrCreate()	×
104	}
105
106	def version(): String = {
107	currentVersion	×
108	}
109
110	}

JohnSnowLabs / spark-nlp / 5210642866

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous