9686

Build # 9686

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/651770445cdaf5dead8e2f0fc50fa6b8551bf0d4">651770445<a href="https://github.com/apache/carbondata/commit/651770445cdaf5dead8e2f0fc50fa6b8551bf0d4">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/9e2a62d6c2d72e131c86e401b52ffc7297ab6927">9e2a62d6c</a>">9e2a62d6c</a><a href="https://github.com/apache/carbondata/commit/651770445cdaf5dead8e2f0fc50fa6b8551bf0d4"> into 851dd2c88">851dd2c88</a>

Run Details

4 of 4 new or added lines in 1 file covered. (100.0%)

61300 of 76953 relevant lines covered (79.66%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.spark.thriftserver

import java.io.File

import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, ENDPOINT, SECRET_KEY}
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
import org.slf4j.{Logger, LoggerFactory}

import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties

/**
 * CarbonThriftServer support different modes:
 * 1. read/write data from/to HDFS or local,it only needs configurate storePath
 * 2. read/write data from/to S3, it needs provide access-key, secret-key, s3-endpoint
 */
object CarbonThriftServer {

  def main(args: Array[String]): Unit = {

    import org.apache.spark.sql.CarbonSession._

    val sparkConf = new SparkConf(loadDefaults = true)

    val logger: Logger = LoggerFactory.getLogger(this.getClass)
    if (args.length != 0 && args.length != 1 && args.length != 4) {
      logger.error("parameters: storePath [access-key] [secret-key] [s3-endpoint]")
      System.exit(0)
    }

    val master = Option(System.getProperty("spark.master"))
      .orElse(sys.env.get("MASTER"))
      .orElse(Option("local[8]"))

    val builder = SparkSession
      .builder()
      .master(master.get)
      .config(sparkConf)
      .appName("Carbon Thrift Server(uses CarbonSession)")
      .enableHiveSupport()

    if (!sparkConf.contains("carbon.properties.filepath")) {
      val sparkHome = System.getenv.get("SPARK_HOME")
      if (null != sparkHome) {
        val file = new File(sparkHome + '/' + "conf" + '/' + "carbon.properties")
        if (file.exists()) {
          builder.config("carbon.properties.filepath", file.getCanonicalPath)
          System.setProperty("carbon.properties.filepath", file.getCanonicalPath)
        }
      }
    } else {
      System.setProperty("carbon.properties.filepath", sparkConf.get("carbon.properties.filepath"))
    }

    val storePath = if (args.length > 0) args.head else null

    val spark = if (args.length <= 1) {
      builder.getOrCreateCarbonSession(storePath)
    } else {
      val (accessKey, secretKey, endpoint) = getKeyOnPrefix(args(0))
      builder.config(accessKey, args(1))
        .config(secretKey, args(2))
        .config(endpoint, getS3EndPoint(args))
        .getOrCreateCarbonSession(storePath)
    }

    val warmUpTime = CarbonProperties.getInstance().getProperty("carbon.spark.warmUpTime", "5000")
    try {
      Thread.sleep(Integer.parseInt(warmUpTime))
    } catch {
      case e: Exception =>
        val LOG = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
        LOG.error(s"Wrong value for carbon.spark.warmUpTime $warmUpTime " +
                  "Using default Value and proceeding")
        Thread.sleep(5000)
    }

    HiveThriftServer2.startWithContext(spark.sqlContext)
  }

  def getKeyOnPrefix(path: String): (String, String, String) = {
    val endPoint = "spark.hadoop." + ENDPOINT
    if (path.startsWith(CarbonCommonConstants.S3A_PREFIX)) {
      ("spark.hadoop." + ACCESS_KEY, "spark.hadoop." + SECRET_KEY, endPoint)
    } else if (path.startsWith(CarbonCommonConstants.S3N_PREFIX)) {
      ("spark.hadoop." + CarbonCommonConstants.S3N_ACCESS_KEY,
        "spark.hadoop." + CarbonCommonConstants.S3N_SECRET_KEY, endPoint)
    } else if (path.startsWith(CarbonCommonConstants.S3_PREFIX)) {
      ("spark.hadoop." + CarbonCommonConstants.S3_ACCESS_KEY,
        "spark.hadoop." + CarbonCommonConstants.S3_SECRET_KEY, endPoint)
    } else {
      throw new Exception("Incorrect Store Path")
    }
  }

  def getS3EndPoint(args: Array[String]): String = {
    if (args.length >= 4 && args(3).contains(".com")) args(3)
    else ""
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.spark.thriftserver
19
20	import java.io.File
21
22	import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, ENDPOINT, SECRET_KEY}
23	import org.apache.spark.SparkConf
24	import org.apache.spark.sql.SparkSession
25	import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
26	import org.slf4j.{Logger, LoggerFactory}
27
28	import org.apache.carbondata.common.logging.LogServiceFactory
29	import org.apache.carbondata.core.constants.CarbonCommonConstants
30	import org.apache.carbondata.core.util.CarbonProperties
31
32	/**
33	* CarbonThriftServer support different modes:
34	* 1. read/write data from/to HDFS or local,it only needs configurate storePath
35	* 2. read/write data from/to S3, it needs provide access-key, secret-key, s3-endpoint
36	*/
37	object CarbonThriftServer {
38
39	def main(args: Array[String]): Unit = {
40
41	import org.apache.spark.sql.CarbonSession._
42
43	val sparkConf = new SparkConf(loadDefaults = true)	×
44
45	val logger: Logger = LoggerFactory.getLogger(this.getClass)	×
46	if (args.length != 0 && args.length != 1 && args.length != 4) {	×
47	logger.error("parameters: storePath [access-key] [secret-key] [s3-endpoint]")	×
48	System.exit(0)	×
49	}
50
51	val master = Option(System.getProperty("spark.master"))	×
52	.orElse(sys.env.get("MASTER"))	×
53	.orElse(Option("local[8]"))	×
54
55	val builder = SparkSession	×
56	.builder()	×
57	.master(master.get)	×
58	.config(sparkConf)	×
59	.appName("Carbon Thrift Server(uses CarbonSession)")	×
60	.enableHiveSupport()	×
61
62	if (!sparkConf.contains("carbon.properties.filepath")) {	×
63	val sparkHome = System.getenv.get("SPARK_HOME")	×
64	if (null != sparkHome) {	×
65	val file = new File(sparkHome + '/' + "conf" + '/' + "carbon.properties")	×
66	if (file.exists()) {	×
67	builder.config("carbon.properties.filepath", file.getCanonicalPath)	×
68	System.setProperty("carbon.properties.filepath", file.getCanonicalPath)	×
69	}
70	}
71	} else {
72	System.setProperty("carbon.properties.filepath", sparkConf.get("carbon.properties.filepath"))	×
73	}
74
75	val storePath = if (args.length > 0) args.head else null	×
76
77	val spark = if (args.length <= 1) {	×
78	builder.getOrCreateCarbonSession(storePath)	×
79	} else {
80	val (accessKey, secretKey, endpoint) = getKeyOnPrefix(args(0))	×
81	builder.config(accessKey, args(1))	×
82	.config(secretKey, args(2))	×
83	.config(endpoint, getS3EndPoint(args))	×
84	.getOrCreateCarbonSession(storePath)	×
85	}
86
87	val warmUpTime = CarbonProperties.getInstance().getProperty("carbon.spark.warmUpTime", "5000")	×
88	try {	×
89	Thread.sleep(Integer.parseInt(warmUpTime))	×
90	} catch {
91	case e: Exception =>
92	val LOG = LogServiceFactory.getLogService(this.getClass.getCanonicalName)	×
93	LOG.error(s"Wrong value for carbon.spark.warmUpTime $warmUpTime " +	×
94	"Using default Value and proceeding")	×
95	Thread.sleep(5000)	×
96	}
97
98	HiveThriftServer2.startWithContext(spark.sqlContext)	×
99	}
100
101	def getKeyOnPrefix(path: String): (String, String, String) = {	×
102	val endPoint = "spark.hadoop." + ENDPOINT	×
103	if (path.startsWith(CarbonCommonConstants.S3A_PREFIX)) {	×
104	("spark.hadoop." + ACCESS_KEY, "spark.hadoop." + SECRET_KEY, endPoint)	×
105	} else if (path.startsWith(CarbonCommonConstants.S3N_PREFIX)) {	×
106	("spark.hadoop." + CarbonCommonConstants.S3N_ACCESS_KEY,	×
107	"spark.hadoop." + CarbonCommonConstants.S3N_SECRET_KEY, endPoint)	×
108	} else if (path.startsWith(CarbonCommonConstants.S3_PREFIX)) {	×
109	("spark.hadoop." + CarbonCommonConstants.S3_ACCESS_KEY,	×
110	"spark.hadoop." + CarbonCommonConstants.S3_SECRET_KEY, endPoint)	×
111	} else {
112	throw new Exception("Incorrect Store Path")	×
113	}
114	}
115
116	def getS3EndPoint(args: Array[String]): String = {
117	if (args.length >= 4 && args(3).contains(".com")) args(3)	×
118	else ""	×
119	}
120
121	}	×

apache / carbondata / 9686

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous