2597158328

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://git"><a class=hub.com/JohnSnowLabs/spark-nlp/commit/cc7bbd0fcb4351a192e47ff876dbef633f176c00">cc7bbd0fc">&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/</a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/JohnSnowLabs/spark-nlp/commit/cc7bbd0fcb4351a192e47ff876dbef633f176c00">cc7bbd0fc</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/cc7bbd0fcb4351a192e47ff876dbef633f176c00">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/JohnSnowLabs/spark-nlp/commit/cc7bbd0fcb4351a192e47ff876dbef633f176c00&quot;&gt;cc7bbd0fc&lt;/a&gt;&lt;a href=&quot;https://github.com/JohnSnowLabs/spark-nlp/commit/cc7bbd0fcb4351a192e47ff876dbef633f176c00&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/JohnSnowLabs/spark-nlp/commit/&amp;lt;/a&amp;gt;&amp;lt... (continued)

Pull Request Pull Request #9867: Release/401 release candidate

Run Details

11 of 11 new or added lines in 3 files covered. (100.0%)

8207 of 11238 relevant lines covered (73.03%)

0.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/nlp/SparkNLP.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp

import org.apache.spark.sql.SparkSession

object SparkNLP {

  val currentVersion = "4.0.1"
  val MavenSpark3 = s"com.johnsnowlabs.nlp:spark-nlp_2.12:$currentVersion"
  val MavenGpuSpark3 = s"com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:$currentVersion"
  val MavenSparkM1 = s"com.johnsnowlabs.nlp:spark-nlp-m1_2.12:$currentVersion"

  /** Start SparkSession with Spark NLP
    *
    * @param gpu
    *   start Spark NLP with GPU
    * @param m1
    *   start Spark NLP with M1
    * @param memory
    *   set driver memory for SparkSession
    * @param cache_folder
    *   The location to download and extract pretrained Models and Pipelines (by default, it will
    *   be in the users home directory under `cache_pretrained`.)
    * @param log_folder
    *   The location to use on a cluster for temporarily files such as unpacking indexes for
    *   WordEmbeddings. By default, this locations is the location of `hadoop.tmp.dir` set via
    *   Hadoop configuration for Apache Spark. NOTE: `S3` is not supported and it must be local,
    *   HDFS, or DBFS.
    * @param cluster_tmp_dir
    *   The location to save logs from annotators during training (By default, it will be in the
    *   users home directory under `annotator_logs`.)
    * @return
    *   SparkSession
    */
  def start(
      gpu: Boolean = false,
      m1: Boolean = false,
      memory: String = "16G",
      cache_folder: String = "",
      log_folder: String = "",
      cluster_tmp_dir: String = ""): SparkSession = {

    val build = SparkSession
      .builder()
      .appName("Spark NLP")
      .master("local[*]")
      .config("spark.driver.memory", memory)
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .config("spark.kryoserializer.buffer.max", "2000M")
      .config("spark.driver.maxResultSize", "0")

    if (m1) {
      build.config("spark.jars.packages", MavenSparkM1)
    } else if (gpu) {
      build.config("spark.jars.packages", MavenGpuSpark3)
    } else {
      build.config("spark.jars.packages", MavenSpark3)
    }

    if (cache_folder.nonEmpty)
      build.config("spark.jsl.settings.pretrained.cache_folder", cache_folder)

    if (log_folder.nonEmpty)
      build.config("spark.jsl.settings.annotator.log_folder", log_folder)

    if (cluster_tmp_dir.nonEmpty)
      build.config("spark.jsl.settings.storage.cluster_tmp_dir", cluster_tmp_dir)

    build.getOrCreate()
  }

  def version(): String = {
    currentVersion
  }

}

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp
18
19	import org.apache.spark.sql.SparkSession
20
21	object SparkNLP {
22
23	val currentVersion = "4.0.1"	×
24	val MavenSpark3 = s"com.johnsnowlabs.nlp:spark-nlp_2.12:$currentVersion"	×
25	val MavenGpuSpark3 = s"com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:$currentVersion"	×
26	val MavenSparkM1 = s"com.johnsnowlabs.nlp:spark-nlp-m1_2.12:$currentVersion"	×
27
28	/** Start SparkSession with Spark NLP
29	*
30	* @param gpu
31	* start Spark NLP with GPU
32	* @param m1
33	* start Spark NLP with M1
34	* @param memory
35	* set driver memory for SparkSession
36	* @param cache_folder
37	* The location to download and extract pretrained Models and Pipelines (by default, it will
38	* be in the users home directory under `cache_pretrained`.)
39	* @param log_folder
40	* The location to use on a cluster for temporarily files such as unpacking indexes for
41	* WordEmbeddings. By default, this locations is the location of `hadoop.tmp.dir` set via
42	* Hadoop configuration for Apache Spark. NOTE: `S3` is not supported and it must be local,
43	* HDFS, or DBFS.
44	* @param cluster_tmp_dir
45	* The location to save logs from annotators during training (By default, it will be in the
46	* users home directory under `annotator_logs`.)
47	* @return
48	* SparkSession
49	*/
50	def start(
51	gpu: Boolean = false,
52	m1: Boolean = false,
53	memory: String = "16G",
54	cache_folder: String = "",
55	log_folder: String = "",
56	cluster_tmp_dir: String = ""): SparkSession = {
57
58	val build = SparkSession
59	.builder()
60	.appName("Spark NLP")
61	.master("local[*]")
62	.config("spark.driver.memory", memory)
63	.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
64	.config("spark.kryoserializer.buffer.max", "2000M")
65	.config("spark.driver.maxResultSize", "0")	×
66
67	if (m1) {
68	build.config("spark.jars.packages", MavenSparkM1)	×
69	} else if (gpu) {	×
70	build.config("spark.jars.packages", MavenGpuSpark3)	×
71	} else {
72	build.config("spark.jars.packages", MavenSpark3)	×
73	}
74
75	if (cache_folder.nonEmpty)	×
76	build.config("spark.jsl.settings.pretrained.cache_folder", cache_folder)	×
77
78	if (log_folder.nonEmpty)	×
79	build.config("spark.jsl.settings.annotator.log_folder", log_folder)	×
80
81	if (cluster_tmp_dir.nonEmpty)	×
82	build.config("spark.jsl.settings.storage.cluster_tmp_dir", cluster_tmp_dir)	×
83
84	build.getOrCreate()	×
85	}
86
87	def version(): String = {
88	currentVersion	×
89	}
90
91	}

JohnSnowLabs / spark-nlp / 2597158328

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous