10675586955

Committed 03 Sep 2024 02:30AM UTC coverage: 61.821% (-0.06%) from 61.884%

Build # 10675586955

Build Type

Pull #14379

github

Committed by

web-flow

Commit Message

Merge 1f222af49 into 9285df8c6

Pull Request Pull Request #14379: SPARKNLP Introducing LLAMA 3

Run Details

0 of 27 new or added lines in 3 files covered. (0.0%)

15 existing lines in 11 files now uncovered.

8982 of 14529 relevant lines covered (61.82%)

0.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/src/main/scala/com/johnsnowlabs/nlp/util/SparkNlpConfig.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.util

import com.johnsnowlabs.util.Version
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{Dataset, Row}

/** Additional configure options that used by spark.nlp */
object SparkNlpConfig {

  def getEncoder(inputDataset: Dataset[_], newStructType: StructType): ExpressionEncoder[Row] = {
    val sparkVersion = Version.parse(inputDataset.sparkSession.version).toFloat
    if (sparkVersion >= 3.5f) {
      val expressionEncoderClass =
        Class.forName("org.apache.spark.sql.catalyst.encoders.ExpressionEncoder")
      val applyMethod = expressionEncoderClass.getMethod("apply", classOf[StructType])
      applyMethod.invoke(null, newStructType).asInstanceOf[ExpressionEncoder[Row]]
    } else {
      try {
        // Use reflection to access RowEncoder.apply in older Spark versions
        val rowEncoderClass = Class.forName("org.apache.spark.sql.catalyst.encoders.RowEncoder")
        val applyMethod = rowEncoderClass.getMethod("apply", classOf[StructType])
        applyMethod.invoke(null, newStructType).asInstanceOf[ExpressionEncoder[Row]]
      } catch {
        case _: Throwable =>
          throw new UnsupportedOperationException(
            "RowEncoder.apply is not supported in this Spark version.")
      }
    }
  }

}

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.util
18
19	import com.johnsnowlabs.util.Version
20	import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
21	import org.apache.spark.sql.types.StructType
22	import org.apache.spark.sql.{Dataset, Row}
23
24	/** Additional configure options that used by spark.nlp */
25	object SparkNlpConfig {
26
27	def getEncoder(inputDataset: Dataset[_], newStructType: StructType): ExpressionEncoder[Row] = {
28	val sparkVersion = Version.parse(inputDataset.sparkSession.version).toFloat	1✔
UNCOV 29	if (sparkVersion >= 3.5f) {	×
30	val expressionEncoderClass =
31	Class.forName("org.apache.spark.sql.catalyst.encoders.ExpressionEncoder")	×
32	val applyMethod = expressionEncoderClass.getMethod("apply", classOf[StructType])	×
33	applyMethod.invoke(null, newStructType).asInstanceOf[ExpressionEncoder[Row]]	×
34	} else {
35	try {	1✔
36	// Use reflection to access RowEncoder.apply in older Spark versions
37	val rowEncoderClass = Class.forName("org.apache.spark.sql.catalyst.encoders.RowEncoder")	1✔
38	val applyMethod = rowEncoderClass.getMethod("apply", classOf[StructType])	1✔
39	applyMethod.invoke(null, newStructType).asInstanceOf[ExpressionEncoder[Row]]	1✔
40	} catch {
41	case _: Throwable =>
42	throw new UnsupportedOperationException(	×
43	"RowEncoder.apply is not supported in this Spark version.")
44	}
45	}
46	}
47
48	}

JohnSnowLabs / spark-nlp / 10675586955

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous