6761

Build # 6761

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/26fca963fcd8fbaecc4bc358a4fe219d399b1d6b">26fca963f<a href="https://github.com/apache/carbondata/commit/26fca963fcd8fbaecc4bc358a4fe219d399b1d6b">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/ac75a7e01e7c4050104357552610f0b1ececc3bf">ac75a7e01</a>">ac75a7e01</a><a href="https://github.com/apache/carbondata/commit/26fca963fcd8fbaecc4bc358a4fe219d399b1d6b"> into 6029b2800">6029b2800</a>

Run Details

30 of 30 new or added lines in 2 files covered. (100.0%)

57218 of 70707 relevant lines covered (80.92%)

1.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.23

/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.spark.sql.hive

import java.util.LinkedHashSet

import scala.Array.canBuildFrom
import scala.collection.JavaConverters._
import scala.util.parsing.combinator.RegexParsers

import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
import org.apache.spark.sql.catalyst.expressions.AttributeReference
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan}
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.{CarbonMetastoreTypes, SparkTypeConverter}

import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.path.CarbonTablePath

/**
 * Represents logical plan for one carbon table
 */
case class CarbonRelation(
    databaseName: String,
    tableName: String,
    var metaData: CarbonMetaData,
    carbonTable: CarbonTable)
  extends LeafNode with MultiInstanceRelation {

  override def newInstance(): LogicalPlan = {
    CarbonRelation(databaseName, tableName, metaData, carbonTable)
      .asInstanceOf[this.type]
  }

  val dimensionsAttr: Seq[AttributeReference] = {
    val sett = new LinkedHashSet(
      carbonTable.getDimensionByTableName(carbonTable.getTableName)
        .asScala.asJava)
    sett.asScala.toSeq.map(dim => {
      val dimval = metaData.carbonTable
        .getDimensionByName(metaData.carbonTable.getTableName, dim.getColName)
      val output: DataType = dimval.getDataType.getName.toLowerCase match {
        case "array" =>
          CarbonMetastoreTypes.toDataType(
            s"array<${SparkTypeConverter.getArrayChildren(carbonTable, dim.getColName)}>")
        case "struct" =>
          CarbonMetastoreTypes.toDataType(
            s"struct<${SparkTypeConverter.getStructChildren(carbonTable, dim.getColName)}>")
        case dType =>
          val dataType = addDecimalScaleAndPrecision(dimval, dType)
          CarbonMetastoreTypes.toDataType(dataType)
      }

      AttributeReference(
        dim.getColName,
        output,
        nullable = true)()
    })
  }

  val measureAttr = {
    val factTable = carbonTable.getTableName
    new LinkedHashSet(
      carbonTable.getMeasureByTableName(carbonTable.getTableName).asScala.asJava).asScala.toSeq
      .map { x =>
      val metastoreType = metaData.carbonTable.getMeasureByName(factTable, x.getColName)
        .getDataType.getName.toLowerCase match {
        case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
        case others => others
      }
      AttributeReference(
        x.getColName,
        CarbonMetastoreTypes.toDataType(metastoreType),
        nullable = true)()
    }
  }

  override val output = {
    val columns = carbonTable.getCreateOrderColumn(carbonTable.getTableName)
      .asScala
    // convert each column to Attribute
    columns.filter(!_.isInvisible).map { column: CarbonColumn =>
      if (column.isDimension()) {
        val output: DataType = column.getDataType.getName.toLowerCase match {
          case "array" =>
            CarbonMetastoreTypes.toDataType(
              s"array<${SparkTypeConverter.getArrayChildren(carbonTable, column.getColName)}>")
          case "struct" =>
            CarbonMetastoreTypes.toDataType(
              s"struct<${SparkTypeConverter.getStructChildren(carbonTable, column.getColName)}>")
          case dType =>
            val dataType = SparkTypeConverter.addDecimalScaleAndPrecision(column, dType)
            CarbonMetastoreTypes.toDataType(dataType)
        }
        AttributeReference(column.getColName, output, nullable = true )(
          qualifier = Option(tableName + "." + column.getColName))
      } else {
        val output = CarbonMetastoreTypes.toDataType {
          column.getDataType.getName.toLowerCase match {
            case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column
              .getColumnSchema.getScale + ")"
            case others => others
          }
        }
        AttributeReference(column.getColName, output, nullable = true)(
          qualifier = Option(tableName + "." + column.getColName))
      }
    }
  }

  // TODO: Use data from the footers.
  // TODO For 2.1
  //  override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
  // Todo for 2.2
  //  override def computeStats(conf: SQLConf): Statistics = Statistics(sizeInBytes =
  //  this.sizeInBytes)

  // override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)

  override def equals(other: Any): Boolean = {
    other match {
      case p: CarbonRelation =>
        p.databaseName == databaseName && p.output == output && p.tableName == tableName
      case _ => false
    }
  }

  def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = {
    var dType = dataType
    if (DataTypes.isDecimal(dimval.getDataType)) {
      dType +=
      "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
    }
    dType
  }

  private var tableStatusLastUpdateTime = 0L

  private var sizeInBytesLocalValue = 0L

  def sizeInBytes: Long = {
    if (carbonTable.isExternalTable) {
      val tablePath = carbonTable.getTablePath
      val fileType = FileFactory.getFileType(tablePath)
      if (FileFactory.isFileExist(tablePath, fileType)) {
        sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
      }
    } else {
      val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(
        carbonTable.getAbsoluteTableIdentifier)
      if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
        if (new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
          .getValidAndInvalidSegments.getValidSegments.isEmpty) {
          sizeInBytesLocalValue = 0L
        } else {
          val tablePath = carbonTable.getTablePath
          val fileType = FileFactory.getFileType(tablePath)
          if (FileFactory.isFileExist(tablePath, fileType)) {
            // get the valid segments
            val segments = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
              .getValidAndInvalidSegments.getValidSegments.asScala
            var size = 0L
            // for each segment calculate the size
            segments.foreach { validSeg =>
              // for older store
              if (null != validSeg.getLoadMetadataDetails.getDataSize &&
                  null != validSeg.getLoadMetadataDetails.getIndexSize) {
                size = size + validSeg.getLoadMetadataDetails.getDataSize.toLong +
                       validSeg.getLoadMetadataDetails.getIndexSize.toLong
              } else {
                size = size + FileFactory.getDirectorySize(
                  CarbonTablePath.getSegmentPath(tablePath, validSeg.getSegmentNo))
              }
            }
            // update the new table status time
            tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
            // update the new size
            sizeInBytesLocalValue = size
          }
        }
      }
    }

    sizeInBytesLocalValue
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.spark.sql.hive
18
19	import java.util.LinkedHashSet
20
21	import scala.Array.canBuildFrom
22	import scala.collection.JavaConverters._
23	import scala.util.parsing.combinator.RegexParsers
24
25	import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
26	import org.apache.spark.sql.catalyst.expressions.AttributeReference
27	import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan}
28	import org.apache.spark.sql.types._
29	import org.apache.spark.sql.util.CarbonException
30	import org.apache.spark.util.{CarbonMetastoreTypes, SparkTypeConverter}
31
32	import org.apache.carbondata.core.datastore.impl.FileFactory
33	import org.apache.carbondata.core.metadata.datatype.DataTypes
34	import org.apache.carbondata.core.metadata.schema.table.CarbonTable
35	import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
36	import org.apache.carbondata.core.statusmanager.SegmentStatusManager
37	import org.apache.carbondata.core.util.path.CarbonTablePath
38
39	/**
40	* Represents logical plan for one carbon table
41	*/
42	case class CarbonRelation(	×
43	databaseName: String,	2✔
44	tableName: String,	2✔
45	var metaData: CarbonMetaData,	2✔
46	carbonTable: CarbonTable)	2✔
47	extends LeafNode with MultiInstanceRelation {	2✔
48
49	override def newInstance(): LogicalPlan = {
50	CarbonRelation(databaseName, tableName, metaData, carbonTable)	×
51	.asInstanceOf[this.type]
52	}
53
54	val dimensionsAttr: Seq[AttributeReference] = {	2✔
55	val sett = new LinkedHashSet(	2✔
56	carbonTable.getDimensionByTableName(carbonTable.getTableName)	2✔
57	.asScala.asJava)	2✔
58	sett.asScala.toSeq.map(dim => {	2✔
59	val dimval = metaData.carbonTable	2✔
60	.getDimensionByName(metaData.carbonTable.getTableName, dim.getColName)	2✔
61	val output: DataType = dimval.getDataType.getName.toLowerCase match {	2✔
62	case "array" =>	2✔
63	CarbonMetastoreTypes.toDataType(	2✔
64	s"array<${SparkTypeConverter.getArrayChildren(carbonTable, dim.getColName)}>")	2✔
65	case "struct" =>	2✔
66	CarbonMetastoreTypes.toDataType(	2✔
67	s"struct<${SparkTypeConverter.getStructChildren(carbonTable, dim.getColName)}>")	2✔
68	case dType =>	2✔
69	val dataType = addDecimalScaleAndPrecision(dimval, dType)	2✔
70	CarbonMetastoreTypes.toDataType(dataType)	2✔
71	}
72
73	AttributeReference(	2✔
74	dim.getColName,	2✔
75	output,	2✔
76	nullable = true)()	2✔
77	})
78	}
79
80	val measureAttr = {	2✔
81	val factTable = carbonTable.getTableName	2✔
82	new LinkedHashSet(	2✔
83	carbonTable.getMeasureByTableName(carbonTable.getTableName).asScala.asJava).asScala.toSeq	2✔
84	.map { x =>	2✔
85	val metastoreType = metaData.carbonTable.getMeasureByName(factTable, x.getColName)	2✔
86	.getDataType.getName.toLowerCase match {	2✔
87	case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"	2✔
88	case others => others	2✔
89	}
90	AttributeReference(	2✔
91	x.getColName,	2✔
92	CarbonMetastoreTypes.toDataType(metastoreType),	2✔
93	nullable = true)()	2✔
94	}
95	}
96
97	override val output = {	2✔
98	val columns = carbonTable.getCreateOrderColumn(carbonTable.getTableName)	2✔
99	.asScala	2✔
100	// convert each column to Attribute
101	columns.filter(!_.isInvisible).map { column: CarbonColumn =>	2✔
102	if (column.isDimension()) {	2✔
103	val output: DataType = column.getDataType.getName.toLowerCase match {	2✔
104	case "array" =>	2✔
105	CarbonMetastoreTypes.toDataType(	2✔
106	s"array<${SparkTypeConverter.getArrayChildren(carbonTable, column.getColName)}>")	2✔
107	case "struct" =>	2✔
108	CarbonMetastoreTypes.toDataType(	2✔
109	s"struct<${SparkTypeConverter.getStructChildren(carbonTable, column.getColName)}>")	2✔
110	case dType =>	2✔
111	val dataType = SparkTypeConverter.addDecimalScaleAndPrecision(column, dType)	2✔
112	CarbonMetastoreTypes.toDataType(dataType)	2✔
113	}
114	AttributeReference(column.getColName, output, nullable = true )(	2✔
115	qualifier = Option(tableName + "." + column.getColName))	2✔
116	} else {
117	val output = CarbonMetastoreTypes.toDataType {	2✔
118	column.getDataType.getName.toLowerCase match {	2✔
119	case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column	2✔
120	.getColumnSchema.getScale + ")"	2✔
121	case others => others	2✔
122	}
123	}
124	AttributeReference(column.getColName, output, nullable = true)(	2✔
125	qualifier = Option(tableName + "." + column.getColName))	2✔
126	}
127	}
128	}
129
130	// TODO: Use data from the footers.
131	// TODO For 2.1
132	// override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
133	// Todo for 2.2
134	// override def computeStats(conf: SQLConf): Statistics = Statistics(sizeInBytes =
135	// this.sizeInBytes)
136
137	// override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
138
139	override def equals(other: Any): Boolean = {
140	other match {	×
141	case p: CarbonRelation =>	×
142	p.databaseName == databaseName && p.output == output && p.tableName == tableName	×
143	case _ => false	×
144	}
145	}
146
147	def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = {
148	var dType = dataType	2✔
149	if (DataTypes.isDecimal(dimval.getDataType)) {	2✔
150	dType +=	2✔
151	"(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"	2✔
152	}
153	dType	2✔
154	}
155
156	private var tableStatusLastUpdateTime = 0L	2✔
157
158	private var sizeInBytesLocalValue = 0L	2✔
159
160	def sizeInBytes: Long = {
161	if (carbonTable.isExternalTable) {	2✔
162	val tablePath = carbonTable.getTablePath	1✔
163	val fileType = FileFactory.getFileType(tablePath)	1✔
164	if (FileFactory.isFileExist(tablePath, fileType)) {	1✔
165	sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)	1✔
166	}
167	} else {
168	val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(	2✔
169	carbonTable.getAbsoluteTableIdentifier)	2✔
170	if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {	2✔
171	if (new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)	2✔
172	.getValidAndInvalidSegments.getValidSegments.isEmpty) {	2✔
173	sizeInBytesLocalValue = 0L	1✔
174	} else {
175	val tablePath = carbonTable.getTablePath	2✔
176	val fileType = FileFactory.getFileType(tablePath)	2✔
177	if (FileFactory.isFileExist(tablePath, fileType)) {	2✔
178	// get the valid segments
179	val segments = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)	2✔
180	.getValidAndInvalidSegments.getValidSegments.asScala	2✔
181	var size = 0L	2✔
182	// for each segment calculate the size
183	segments.foreach { validSeg =>	2✔
184	// for older store
185	if (null != validSeg.getLoadMetadataDetails.getDataSize &&	2✔
186	null != validSeg.getLoadMetadataDetails.getIndexSize) {	2✔
187	size = size + validSeg.getLoadMetadataDetails.getDataSize.toLong +	2✔
188	validSeg.getLoadMetadataDetails.getIndexSize.toLong	2✔
189	} else {
190	size = size + FileFactory.getDirectorySize(	×
191	CarbonTablePath.getSegmentPath(tablePath, validSeg.getSegmentNo))	×
192	}
193	}
194	// update the new table status time
195	tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime	2✔
196	// update the new size
197	sizeInBytesLocalValue = size	2✔
198	}
199	}
200	}
201	}
202
203	sizeInBytesLocalValue	2✔
204	}
205
206	}

apache / carbondata / 6761

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous