8464

Build # 8464

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/1d1d292d6a36f55e3cf8722b4a2a723b6aeea4d4">1d1d292d6">&lt;a href=&quot;https://github.com/apache/carbondata/commit/</a><a class="double-link" href="https://github.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/1d1d292d6a36f55e3cf8722b4a2a723b6aeea4d4">1d1d292d6</a><a href="https://github.com/apache/carbondata/commit/1d1d292d6a36f55e3cf8722b4a2a723b6aeea4d4">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/1d1d292d6a36f55e3cf8722b4a2a723b6aeea4d4&quot;&gt;1d1d292d6&lt;/a&gt;&lt;a href=&quot;https://github.com/apache/carbondata/commit/1d1d292d6a36f55e3cf8722b4a2a723b6aeea4d4&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/carbondata/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/carbondata/commit/&amp;amp;lt;a ... (continued)

Run Details

22 of 22 new or added lines in 11 files covered. (100.0%)

58209 of 72169 relevant lines covered (80.66%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

55.36

/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.util

import java.lang.reflect.Method

import scala.reflect.runtime._
import scala.reflect.runtime.universe._

import org.apache.spark.{SPARK_VERSION, SparkContext}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
import org.apache.spark.sql.catalyst.parser.AstBuilder
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, SubqueryAlias}
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
import org.apache.spark.sql.execution.{RowDataSourceScanExec, SparkPlan}
import org.apache.spark.sql.execution.datasources.{DataSource, LogicalRelation}
import org.apache.spark.sql.sources.{BaseRelation, Filter}

import org.apache.carbondata.core.constants.CarbonCommonConstants

/**
 * Reflection APIs
 */

object CarbonReflectionUtils {
  private val rm = universe.runtimeMirror(getClass.getClassLoader)

  /**
   * Returns the field val from a object through reflection.
   * @param name - name of the field being retrieved.
   * @param obj - Object from which the field has to be retrieved.
   * @tparam T
   * @return
   */
  def getField[T: TypeTag : reflect.ClassTag](name: String, obj: T): Any = {
    val im = rm.reflect(obj)
    im.symbol.typeSignature.members.find(_.name.toString.equals(name))
      .map(l => im.reflectField(l.asTerm).get).getOrElse(null)
  }

  def getUnresolvedRelation(
      tableIdentifier: TableIdentifier,
      tableAlias: Option[String] = None): UnresolvedRelation = {
    val className = "org.apache.spark.sql.catalyst.analysis.UnresolvedRelation"
    if (SparkUtil.isSparkVersionEqualTo("2.1")) {
      createObject(
        className,
        tableIdentifier,
        tableAlias)._1.asInstanceOf[UnresolvedRelation]
    } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
      createObject(
        className,
        tableIdentifier)._1.asInstanceOf[UnresolvedRelation]
    } else {
      throw new UnsupportedOperationException(s"Unsupported Spark version $SPARK_VERSION")
    }
  }

  def getSubqueryAlias(sparkSession: SparkSession, alias: Option[String],
      relation: LogicalPlan,
      view: Option[TableIdentifier]): SubqueryAlias = {
    val className = "org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias"
    if (SparkUtil.isSparkVersionEqualTo("2.1")) {
      createObject(
        className,
        alias.getOrElse(""),
        relation,
        Option(view))._1.asInstanceOf[SubqueryAlias]
    } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
      createObject(
        className,
        alias.getOrElse(""),
        relation)._1.asInstanceOf[SubqueryAlias]
    } else {
      throw new UnsupportedOperationException("Unsupported Spark version")
    }
  }

  def getInsertIntoCommand(table: LogicalPlan,
      partition: Map[String, Option[String]],
      query: LogicalPlan,
      overwrite: Boolean,
      ifPartitionNotExists: Boolean): InsertIntoTable = {
    val className = "org.apache.spark.sql.catalyst.plans.logical.InsertIntoTable"
    if (SparkUtil.isSparkVersionEqualTo("2.1")) {
      val overwriteOptions = createObject(
        "org.apache.spark.sql.catalyst.plans.logical.OverwriteOptions",
        overwrite.asInstanceOf[Object], Map.empty.asInstanceOf[Object])._1.asInstanceOf[Object]
      createObject(
        className,
        table,
        partition,
        query,
        overwriteOptions,
        ifPartitionNotExists.asInstanceOf[Object])._1.asInstanceOf[InsertIntoTable]
    } else if (SparkUtil.isSparkVersionXandAbove("2.2") ) {
      createObject(
        className,
        table,
        partition,
        query,
        overwrite.asInstanceOf[Object],
        ifPartitionNotExists.asInstanceOf[Object])._1.asInstanceOf[InsertIntoTable]
    } else {
      throw new UnsupportedOperationException("Unsupported Spark version")
    }
  }

  def getLogicalRelation(relation: BaseRelation,
      expectedOutputAttributes: Seq[Attribute],
      catalogTable: Option[CatalogTable],
      isStreaming: Boolean): LogicalRelation = {
    val className = "org.apache.spark.sql.execution.datasources.LogicalRelation"
    if (SparkUtil.isSparkVersionEqualTo("2.1")) {
      createObject(
        className,
        relation,
        Some(expectedOutputAttributes),
        catalogTable)._1.asInstanceOf[LogicalRelation]
    } else if (SparkUtil.isSparkVersionEqualTo("2.2")) {
      createObject(
        className,
        relation,
        expectedOutputAttributes,
        catalogTable)._1.asInstanceOf[LogicalRelation]
    } else if (SparkUtil.isSparkVersionEqualTo("2.3")) {
      createObject(
        className,
        relation,
        expectedOutputAttributes,
        catalogTable,
        isStreaming.asInstanceOf[Object])._1.asInstanceOf[LogicalRelation]
    } else {
      throw new UnsupportedOperationException("Unsupported Spark version")
    }
  }


  def getOverWriteOption[T: TypeTag : reflect.ClassTag](name: String, obj: T): Boolean = {
    var overwriteboolean: Boolean = false
    val im = rm.reflect(obj)
    for (m <- typeOf[T].members.filter(!_.isMethod)) {
      if (m.toString.contains("overwrite")) {
        val typ = m.typeSignature
        if (typ.toString.contains("Boolean")) {
          // Spark2.2
          overwriteboolean = im.reflectField(m.asTerm).get.asInstanceOf[Boolean]
        } else {
          overwriteboolean = getOverWrite("enabled", im.reflectField(m.asTerm).get)
        }
      }
    }
    overwriteboolean
  }

  private def getOverWrite[T: TypeTag : reflect.ClassTag](name: String, obj: T): Boolean = {
    var overwriteboolean: Boolean = false
    val im = rm.reflect(obj)
    for (l <- im.symbol.typeSignature.members.filter(_.name.toString.contains("enabled"))) {
      overwriteboolean = im.reflectField(l.asTerm).get.asInstanceOf[Boolean]
    }
    overwriteboolean
  }

  def getFieldOfCatalogTable[T: TypeTag : reflect.ClassTag](name: String, obj: T): Any = {
    val im = rm.reflect(obj)
    val sym = im.symbol.typeSignature.member(TermName(name))
    val tableMeta = im.reflectMethod(sym.asMethod).apply()
    tableMeta
  }

  def getAstBuilder(conf: Object,
      sqlParser: Object,
      sparkSession: SparkSession): AstBuilder = {
    val className = sparkSession.sparkContext.conf.get(
      CarbonCommonConstants.CARBON_SQLASTBUILDER_CLASSNAME,
      "org.apache.spark.sql.hive.CarbonSqlAstBuilder")
    createObject(className,
      conf,
      sqlParser, sparkSession)._1.asInstanceOf[AstBuilder]
  }

  def getSessionState(sparkContext: SparkContext,
      carbonSession: Object,
      useHiveMetaStore: Boolean): Any = {
    if (SparkUtil.isSparkVersionEqualTo("2.1")) {
      val className = sparkContext.conf.get(
        CarbonCommonConstants.CARBON_SESSIONSTATE_CLASSNAME,
        "org.apache.spark.sql.hive.CarbonSessionState")
      createObject(className, carbonSession)._1
    } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
      if (useHiveMetaStore) {
        val className = sparkContext.conf.get(
          CarbonCommonConstants.CARBON_SESSIONSTATE_CLASSNAME,
          "org.apache.spark.sql.hive.CarbonSessionStateBuilder")
        val tuple = createObject(className, carbonSession, None)
        val method = tuple._2.getMethod("build")
        method.invoke(tuple._1)
      } else {
        val className = sparkContext.conf.get(
          CarbonCommonConstants.CARBON_SESSIONSTATE_CLASSNAME,
          "org.apache.spark.sql.hive.CarbonInMemorySessionStateBuilder")
        val tuple = createObject(className, carbonSession, None)
        val method = tuple._2.getMethod("build")
        method.invoke(tuple._1)
      }
    } else {
      throw new UnsupportedOperationException("Spark version not supported")
    }
  }

  def hasPredicateSubquery(filterExp: Expression) : Boolean = {
    if (SparkUtil.isSparkVersionEqualTo("2.1")) {
      val tuple = Class.forName("org.apache.spark.sql.catalyst.expressions.PredicateSubquery")
      val method = tuple.getMethod("hasPredicateSubquery", classOf[Expression])
      val hasSubquery : Boolean = method.invoke(tuple, filterExp).asInstanceOf[Boolean]
      hasSubquery
    } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
      val tuple = Class.forName("org.apache.spark.sql.catalyst.expressions.SubqueryExpression")
      val method = tuple.getMethod("hasInOrExistsSubquery", classOf[Expression])
      val hasSubquery : Boolean = method.invoke(tuple, filterExp).asInstanceOf[Boolean]
      hasSubquery
    } else {
      throw new UnsupportedOperationException("Spark version not supported")
    }
  }

  def getDescribeTableFormattedField[T: TypeTag : reflect.ClassTag](obj: T): Boolean = {
    val im = rm.reflect(obj)
    val isFormatted = im.symbol.typeSignature.members
      .find(_.name.toString.equalsIgnoreCase("isFormatted"))
      .map(l => im.reflectField(l.asTerm).get).getOrElse("false").asInstanceOf[Boolean]
    isFormatted
  }

  def getRowDataSourceScanExecObj(relation: LogicalRelation,
      output: Seq[Attribute],
      pushedFilters: Seq[Filter],
      handledFilters: Seq[Filter],
      rdd: RDD[InternalRow],
      partition: Partitioning,
      metadata: Map[String, String]): RowDataSourceScanExec = {
    val className = "org.apache.spark.sql.execution.RowDataSourceScanExec"
    if (SparkUtil.isSparkVersionEqualTo("2.1") || SparkUtil.isSparkVersionEqualTo("2.2")) {
      createObject(className, output, rdd, relation.relation,
        partition, metadata,
        relation.catalogTable.map(_.identifier))._1.asInstanceOf[RowDataSourceScanExec]
    } else if (SparkUtil.isSparkVersionXandAbove("2.3")) {
      createObject(className, output, output.map(output.indexOf),
        pushedFilters.toSet, handledFilters.toSet, rdd,
        relation.relation,
        relation.catalogTable.map(_.identifier))._1.asInstanceOf[RowDataSourceScanExec]
    } else {
      throw new UnsupportedOperationException("Spark version not supported")
    }
  }

  def invokewriteAndReadMethod(dataSourceObj: DataSource,
      dataFrame: DataFrame,
      data: LogicalPlan,
      session: SparkSession,
      mode: SaveMode,
      query: LogicalPlan,
      physicalPlan: SparkPlan): BaseRelation = {
    if (SparkUtil.isSparkVersionEqualTo("2.2")) {
      val method: Method = dataSourceObj.getClass
        .getMethod("writeAndRead", classOf[SaveMode], classOf[DataFrame])
      method.invoke(dataSourceObj, mode, dataFrame)
        .asInstanceOf[BaseRelation]
    } else if (SparkUtil.isSparkVersionEqualTo("2.3")) {
      val method: Method = dataSourceObj.getClass
        .getMethod("writeAndRead",
          classOf[SaveMode],
          classOf[LogicalPlan],
          classOf[Seq[Attribute]],
          classOf[SparkPlan])
      method.invoke(dataSourceObj, mode, query, query.output, physicalPlan)
        .asInstanceOf[BaseRelation]
    } else {
      throw new UnsupportedOperationException("Spark version not supported")
    }
  }

  def createObject(className: String, conArgs: Object*): (Any, Class[_]) = {
    val clazz = Utils.classForName(className)
    val ctor = clazz.getConstructors.head
    ctor.setAccessible(true)
    (ctor.newInstance(conArgs: _*), clazz)
  }

  def createObjectOfPrivateConstructor(className: String, conArgs: Object*): (Any, Class[_]) = {
    val clazz = Utils.classForName(className)
    val ctor = clazz.getDeclaredConstructors.head
    ctor.setAccessible(true)
    (ctor.newInstance(conArgs: _*), clazz)
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.spark.util
19
20	import java.lang.reflect.Method
21
22	import scala.reflect.runtime._
23	import scala.reflect.runtime.universe._
24
25	import org.apache.spark.{SPARK_VERSION, SparkContext}
26	import org.apache.spark.rdd.RDD
27	import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
28	import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
29	import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
30	import org.apache.spark.sql.catalyst.catalog.CatalogTable
31	import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
32	import org.apache.spark.sql.catalyst.parser.AstBuilder
33	import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, SubqueryAlias}
34	import org.apache.spark.sql.catalyst.plans.physical.Partitioning
35	import org.apache.spark.sql.execution.{RowDataSourceScanExec, SparkPlan}
36	import org.apache.spark.sql.execution.datasources.{DataSource, LogicalRelation}
37	import org.apache.spark.sql.sources.{BaseRelation, Filter}
38
39	import org.apache.carbondata.core.constants.CarbonCommonConstants
40
41	/**
42	* Reflection APIs
43	*/
44
45	object CarbonReflectionUtils {
46	private val rm = universe.runtimeMirror(getClass.getClassLoader)	1✔
47
48	/**
49	* Returns the field val from a object through reflection.
50	* @param name - name of the field being retrieved.
51	* @param obj - Object from which the field has to be retrieved.
52	* @tparam T
53	* @return
54	*/
55	def getField[T: TypeTag : reflect.ClassTag](name: String, obj: T): Any = {
56	val im = rm.reflect(obj)	×
57	im.symbol.typeSignature.members.find(_.name.toString.equals(name))	×
58	.map(l => im.reflectField(l.asTerm).get).getOrElse(null)	×
59	}
60
61	def getUnresolvedRelation(
62	tableIdentifier: TableIdentifier,
63	tableAlias: Option[String] = None): UnresolvedRelation = {	1✔
64	val className = "org.apache.spark.sql.catalyst.analysis.UnresolvedRelation"	1✔
65	if (SparkUtil.isSparkVersionEqualTo("2.1")) {	1✔
66	createObject(	×
67	className,	×
68	tableIdentifier,	×
69	tableAlias)._1.asInstanceOf[UnresolvedRelation]	×
70	} else if (SparkUtil.isSparkVersionXandAbove("2.2")) {	1✔
71	createObject(	1✔
72	className,	1✔
73	tableIdentifier)._1.asInstanceOf[UnresolvedRelation]	1✔
74	} else {
75	throw new UnsupportedOperationException(s"Unsupported Spark version $SPARK_VERSION")	×
76	}
77	}
78
79	def getSubqueryAlias(sparkSession: SparkSession, alias: Option[String],
80	relation: LogicalPlan,
81	view: Option[TableIdentifier]): SubqueryAlias = {	1✔
82	val className = "org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias"	1✔
83	if (SparkUtil.isSparkVersionEqualTo("2.1")) {	1✔
84	createObject(	×
85	className,	×
86	alias.getOrElse(""),	×
87	relation,	×
88	Option(view))._1.asInstanceOf[SubqueryAlias]	×
89	} else if (SparkUtil.isSparkVersionXandAbove("2.2")) {	1✔
90	createObject(	1✔
91	className,	1✔
92	alias.getOrElse(""),	1✔
93	relation)._1.asInstanceOf[SubqueryAlias]	1✔
94	} else {
95	throw new UnsupportedOperationException("Unsupported Spark version")	×
96	}
97	}
98
99	def getInsertIntoCommand(table: LogicalPlan,
100	partition: Map[String, Option[String]],
101	query: LogicalPlan,
102	overwrite: Boolean,
103	ifPartitionNotExists: Boolean): InsertIntoTable = {	1✔
104	val className = "org.apache.spark.sql.catalyst.plans.logical.InsertIntoTable"	1✔
105	if (SparkUtil.isSparkVersionEqualTo("2.1")) {	1✔
106	val overwriteOptions = createObject(	×
107	"org.apache.spark.sql.catalyst.plans.logical.OverwriteOptions",	×
108	overwrite.asInstanceOf[Object], Map.empty.asInstanceOf[Object])._1.asInstanceOf[Object]	×
109	createObject(	×
110	className,	×
111	table,	×
112	partition,	×
113	query,	×
114	overwriteOptions,	×
115	ifPartitionNotExists.asInstanceOf[Object])._1.asInstanceOf[InsertIntoTable]	×
116	} else if (SparkUtil.isSparkVersionXandAbove("2.2") ) {	1✔
117	createObject(	1✔
118	className,	1✔
119	table,	1✔
120	partition,	1✔
121	query,	1✔
122	overwrite.asInstanceOf[Object],	1✔
123	ifPartitionNotExists.asInstanceOf[Object])._1.asInstanceOf[InsertIntoTable]	1✔
124	} else {
125	throw new UnsupportedOperationException("Unsupported Spark version")	×
126	}
127	}
128
129	def getLogicalRelation(relation: BaseRelation,
130	expectedOutputAttributes: Seq[Attribute],
131	catalogTable: Option[CatalogTable],
132	isStreaming: Boolean): LogicalRelation = {	1✔
133	val className = "org.apache.spark.sql.execution.datasources.LogicalRelation"	1✔
134	if (SparkUtil.isSparkVersionEqualTo("2.1")) {	1✔
135	createObject(	×
136	className,	×
137	relation,	×
138	Some(expectedOutputAttributes),	×
139	catalogTable)._1.asInstanceOf[LogicalRelation]	×
140	} else if (SparkUtil.isSparkVersionEqualTo("2.2")) {	1✔
141	createObject(	×
142	className,	×
143	relation,	×
144	expectedOutputAttributes,	×
145	catalogTable)._1.asInstanceOf[LogicalRelation]	×
146	} else if (SparkUtil.isSparkVersionEqualTo("2.3")) {	1✔
147	createObject(	1✔
148	className,	1✔
149	relation,	1✔
150	expectedOutputAttributes,	1✔
151	catalogTable,	1✔
152	isStreaming.asInstanceOf[Object])._1.asInstanceOf[LogicalRelation]	1✔
153	} else {
154	throw new UnsupportedOperationException("Unsupported Spark version")	×
155	}
156	}
157
158
159	def getOverWriteOption[T: TypeTag : reflect.ClassTag](name: String, obj: T): Boolean = {
160	var overwriteboolean: Boolean = false	1✔
161	val im = rm.reflect(obj)	1✔
162	for (m <- typeOf[T].members.filter(!_.isMethod)) {	1✔
163	if (m.toString.contains("overwrite")) {	1✔
164	val typ = m.typeSignature	1✔
165	if (typ.toString.contains("Boolean")) {	1✔
166	// Spark2.2
167	overwriteboolean = im.reflectField(m.asTerm).get.asInstanceOf[Boolean]	1✔
168	} else {
169	overwriteboolean = getOverWrite("enabled", im.reflectField(m.asTerm).get)	×
170	}
171	}
172	}
173	overwriteboolean	1✔
174	}
175
176	private def getOverWrite[T: TypeTag : reflect.ClassTag](name: String, obj: T): Boolean = {
177	var overwriteboolean: Boolean = false	×
178	val im = rm.reflect(obj)	×
179	for (l <- im.symbol.typeSignature.members.filter(_.name.toString.contains("enabled"))) {	×
180	overwriteboolean = im.reflectField(l.asTerm).get.asInstanceOf[Boolean]	×
181	}
182	overwriteboolean	×
183	}
184
185	def getFieldOfCatalogTable[T: TypeTag : reflect.ClassTag](name: String, obj: T): Any = {
186	val im = rm.reflect(obj)	1✔
187	val sym = im.symbol.typeSignature.member(TermName(name))	1✔
188	val tableMeta = im.reflectMethod(sym.asMethod).apply()	1✔
189	tableMeta	1✔
190	}
191
192	def getAstBuilder(conf: Object,
193	sqlParser: Object,
194	sparkSession: SparkSession): AstBuilder = {
195	val className = sparkSession.sparkContext.conf.get(	1✔
196	CarbonCommonConstants.CARBON_SQLASTBUILDER_CLASSNAME,	1✔
197	"org.apache.spark.sql.hive.CarbonSqlAstBuilder")	1✔
198	createObject(className,	1✔
199	conf,	1✔
200	sqlParser, sparkSession)._1.asInstanceOf[AstBuilder]	1✔
201	}
202
203	def getSessionState(sparkContext: SparkContext,
204	carbonSession: Object,
205	useHiveMetaStore: Boolean): Any = {
206	if (SparkUtil.isSparkVersionEqualTo("2.1")) {	1✔
207	val className = sparkContext.conf.get(	×
208	CarbonCommonConstants.CARBON_SESSIONSTATE_CLASSNAME,	×
209	"org.apache.spark.sql.hive.CarbonSessionState")	×
210	createObject(className, carbonSession)._1	×
211	} else if (SparkUtil.isSparkVersionXandAbove("2.2")) {	1✔
212	if (useHiveMetaStore) {	1✔
213	val className = sparkContext.conf.get(	1✔
214	CarbonCommonConstants.CARBON_SESSIONSTATE_CLASSNAME,	1✔
215	"org.apache.spark.sql.hive.CarbonSessionStateBuilder")	1✔
216	val tuple = createObject(className, carbonSession, None)	1✔
217	val method = tuple._2.getMethod("build")	1✔
218	method.invoke(tuple._1)	1✔
219	} else {
220	val className = sparkContext.conf.get(	×
221	CarbonCommonConstants.CARBON_SESSIONSTATE_CLASSNAME,	×
222	"org.apache.spark.sql.hive.CarbonInMemorySessionStateBuilder")	×
223	val tuple = createObject(className, carbonSession, None)	×
224	val method = tuple._2.getMethod("build")	×
225	method.invoke(tuple._1)	×
226	}
227	} else {
228	throw new UnsupportedOperationException("Spark version not supported")	×
229	}
230	}
231
232	def hasPredicateSubquery(filterExp: Expression) : Boolean = {
233	if (SparkUtil.isSparkVersionEqualTo("2.1")) {	1✔
234	val tuple = Class.forName("org.apache.spark.sql.catalyst.expressions.PredicateSubquery")	×
235	val method = tuple.getMethod("hasPredicateSubquery", classOf[Expression])	×
236	val hasSubquery : Boolean = method.invoke(tuple, filterExp).asInstanceOf[Boolean]	×
237	hasSubquery	×
238	} else if (SparkUtil.isSparkVersionXandAbove("2.2")) {	1✔
239	val tuple = Class.forName("org.apache.spark.sql.catalyst.expressions.SubqueryExpression")	1✔
240	val method = tuple.getMethod("hasInOrExistsSubquery", classOf[Expression])	1✔
241	val hasSubquery : Boolean = method.invoke(tuple, filterExp).asInstanceOf[Boolean]	1✔
242	hasSubquery	1✔
243	} else {
244	throw new UnsupportedOperationException("Spark version not supported")	×
245	}
246	}
247
248	def getDescribeTableFormattedField[T: TypeTag : reflect.ClassTag](obj: T): Boolean = {
249	val im = rm.reflect(obj)	×
250	val isFormatted = im.symbol.typeSignature.members	×
251	.find(_.name.toString.equalsIgnoreCase("isFormatted"))	×
252	.map(l => im.reflectField(l.asTerm).get).getOrElse("false").asInstanceOf[Boolean]	×
253	isFormatted	×
254	}
255
256	def getRowDataSourceScanExecObj(relation: LogicalRelation,
257	output: Seq[Attribute],
258	pushedFilters: Seq[Filter],
259	handledFilters: Seq[Filter],
260	rdd: RDD[InternalRow],
261	partition: Partitioning,
262	metadata: Map[String, String]): RowDataSourceScanExec = {	1✔
263	val className = "org.apache.spark.sql.execution.RowDataSourceScanExec"	1✔
264	if (SparkUtil.isSparkVersionEqualTo("2.1") \|\| SparkUtil.isSparkVersionEqualTo("2.2")) {	1✔
265	createObject(className, output, rdd, relation.relation,	×
266	partition, metadata,	×
267	relation.catalogTable.map(_.identifier))._1.asInstanceOf[RowDataSourceScanExec]	×
268	} else if (SparkUtil.isSparkVersionXandAbove("2.3")) {	1✔
269	createObject(className, output, output.map(output.indexOf),	1✔
270	pushedFilters.toSet, handledFilters.toSet, rdd,	1✔
271	relation.relation,	1✔
272	relation.catalogTable.map(_.identifier))._1.asInstanceOf[RowDataSourceScanExec]	1✔
273	} else {
274	throw new UnsupportedOperationException("Spark version not supported")	×
275	}
276	}
277
278	def invokewriteAndReadMethod(dataSourceObj: DataSource,
279	dataFrame: DataFrame,
280	data: LogicalPlan,
281	session: SparkSession,
282	mode: SaveMode,
283	query: LogicalPlan,
284	physicalPlan: SparkPlan): BaseRelation = {
285	if (SparkUtil.isSparkVersionEqualTo("2.2")) {	1✔
286	val method: Method = dataSourceObj.getClass	×
287	.getMethod("writeAndRead", classOf[SaveMode], classOf[DataFrame])	×
288	method.invoke(dataSourceObj, mode, dataFrame)	×
289	.asInstanceOf[BaseRelation]
290	} else if (SparkUtil.isSparkVersionEqualTo("2.3")) {	1✔
291	val method: Method = dataSourceObj.getClass	1✔
292	.getMethod("writeAndRead",	1✔
293	classOf[SaveMode],	1✔
294	classOf[LogicalPlan],	1✔
295	classOf[Seq[Attribute]],	1✔
296	classOf[SparkPlan])	1✔
297	method.invoke(dataSourceObj, mode, query, query.output, physicalPlan)	1✔
298	.asInstanceOf[BaseRelation]
299	} else {
300	throw new UnsupportedOperationException("Spark version not supported")	×
301	}
302	}
303
304	def createObject(className: String, conArgs: Object*): (Any, Class[_]) = {
305	val clazz = Utils.classForName(className)	1✔
306	val ctor = clazz.getConstructors.head	1✔
307	ctor.setAccessible(true)	1✔
308	(ctor.newInstance(conArgs: _*), clazz)	1✔
309	}
310
311	def createObjectOfPrivateConstructor(className: String, conArgs: Object*): (Any, Class[_]) = {
312	val clazz = Utils.classForName(className)	×
313	val ctor = clazz.getDeclaredConstructors.head	×
314	ctor.setAccessible(true)	×
315	(ctor.newInstance(conArgs: _*), clazz)	×
316	}
317	}	1✔

apache / carbondata / 8464

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous