8263

Build # 8263

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/14ba95e6596d9e8d573d7d559b141d85b65ee51b">14ba95e65<a href="https://github.com/apache/carbondata/commit/14ba95e6596d9e8d573d7d559b141d85b65ee51b">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/3f9b164fde482705859d0c4f405161d5d2245b14">3f9b164fd</a>">3f9b164fd</a><a href="https://github.com/apache/carbondata/commit/14ba95e6596d9e8d573d7d559b141d85b65ee51b"> into d980d4cd8">d980d4cd8</a>

Run Details

21 of 21 new or added lines in 6 files covered. (100.0%)

57509 of 71320 relevant lines covered (80.64%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.28

/integration/spark-datasource/src/main/scala/org/apache/spark/sql/util/SparkTypeConverter.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.sql.util

import java.util.Objects

import scala.collection.JavaConverters._
import scala.collection.mutable

import org.apache.spark.sql.types
import org.apache.spark.sql.types._

import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, ColumnSchema}

private[spark] object SparkTypeConverter {

  def createSparkSchema(table: CarbonTable, columns: Seq[String]): StructType = {
    Objects.requireNonNull(table)
    Objects.requireNonNull(columns)
    if (columns.isEmpty) {
      throw new IllegalArgumentException("column list is empty")
    }
    val fields = new java.util.ArrayList[StructField](columns.size)
    val allColumns = table.getTableInfo.getFactTable.getListOfColumns.asScala

    // find the column and add it to fields array
    columns.foreach { column =>
      val col = allColumns.find(_.getColumnName.equalsIgnoreCase(column)).getOrElse(
        throw new IllegalArgumentException(column + " does not exist")
      )
      fields.add(StructField(col.getColumnName, convertCarbonToSparkDataType(col, table)))
    }
    StructType(fields)
  }

  /**
   * Converts from carbon datatype to corresponding spark datatype.
   */
  def convertCarbonToSparkDataType(
      columnSchema: ColumnSchema,
      table: CarbonTable): types.DataType = {
    if (CarbonDataTypes.isDecimal(columnSchema.getDataType)) {
      val scale = columnSchema.getScale
      val precision = columnSchema.getPrecision
      if (scale == 0 && precision == 0) {
        DecimalType(18, 2)
      } else {
        DecimalType(precision, scale)
      }
    } else if (CarbonDataTypes.isArrayType(columnSchema.getDataType)) {
      CarbonMetastoreTypes
        .toDataType(s"array<${ getArrayChildren(table, columnSchema.getColumnName) }>")
    } else if (CarbonDataTypes.isStructType(columnSchema.getDataType)) {
      CarbonMetastoreTypes
        .toDataType(s"struct<${ getStructChildren(table, columnSchema.getColumnName) }>")
    } else if (CarbonDataTypes.isMapType(columnSchema.getDataType)) {
      CarbonMetastoreTypes
        .toDataType(s"map<${ getMapChildren(table, columnSchema.getColumnName) }>")
    } else {
      columnSchema.getDataType match {
        case CarbonDataTypes.STRING => StringType
        case CarbonDataTypes.SHORT => ShortType
        case CarbonDataTypes.INT => IntegerType
        case CarbonDataTypes.LONG => LongType
        case CarbonDataTypes.DOUBLE => DoubleType
        case CarbonDataTypes.BOOLEAN => BooleanType
        case CarbonDataTypes.TIMESTAMP => TimestampType
        case CarbonDataTypes.DATE => DateType
      }
    }
  }

  def getArrayChildren(table: CarbonTable, dimName: String): String = {
    table.getChildren(dimName).asScala.map(childDim => {
      childDim.getDataType.getName.toLowerCase match {
        case "array" => s"array<${ getArrayChildren(table, childDim.getColName) }>"
        case "struct" => s"struct<${ getStructChildren(table, childDim.getColName) }>"
        case "map" => s"map<${ getMapChildren(table, childDim.getColName) }>"
        case dType => addDecimalScaleAndPrecision(childDim, dType)
      }
    }).mkString(",")
  }

  def getStructChildren(table: CarbonTable, dimName: String): String = {
    table.getChildren(dimName).asScala.map(childDim => {
      childDim.getDataType.getName.toLowerCase match {
        case "array" => s"${
          childDim.getColName.substring(dimName.length + 1)
        }:array<${ getArrayChildren(table, childDim.getColName) }>"
        case "struct" => s"${
          childDim.getColName.substring(dimName.length + 1)
        }:struct<${ table.getChildren(childDim.getColName)
          .asScala.map(f => s"${ recursiveMethod(table, childDim.getColName, f) }").mkString(",")
        }>"
        case "map" => s"${
          childDim.getColName.substring(dimName.length + 1)
        }:map<${ getMapChildren(table, childDim.getColName) }>"
        case dType => s"${ childDim.getColName
          .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
      }
    }).mkString(",")
  }

  def getMapChildren(table: CarbonTable, dimName: String): String = {
    table.getChildren(dimName).asScala.flatMap { childDim =>
      // Map<String, String> is stored internally as Map<Struct<String, String>> in carbon schema
      // and stored as Array<Struct<String, String>> in the actual data storage. So while parsing
      // the map dataType we can ignore the struct child and directly get the children of struct
      // which are actual children of map
      val structChildren = table.getChildren(childDim.getColName).asScala
      structChildren.map { structChild =>
        structChild.getDataType.getName.toLowerCase match {
          case "array" => s"array<${ getArrayChildren(table, structChild.getColName) }>"
          case "struct" => s"struct<${ getStructChildren(table, structChild.getColName) }>"
          case "map" => s"map<${ getMapChildren(table, structChild.getColName) }>"
          case dType => addDecimalScaleAndPrecision(structChild, dType)
        }
      }
    }.mkString(",")
  }

  def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
    var dType = dataType
    if (CarbonDataTypes.isDecimal(dimval.getDataType)) {
      dType +=
      "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
    }
    dType
  }

  private def recursiveMethod(
      table: CarbonTable, dimName: String, childDim: CarbonDimension) = {
    childDim.getDataType.getName.toLowerCase match {
      case "array" => s"${
        childDim.getColName.substring(dimName.length + 1)
      }:array<${ getArrayChildren(table, childDim.getColName) }>"
      case "struct" => s"${
        childDim.getColName.substring(dimName.length + 1)
      }:struct<${ getStructChildren(table, childDim.getColName) }>"
      case "map" => s"${
        childDim.getColName.substring(dimName.length + 1)
      }:map<${ getMapChildren(table, childDim.getColName) }>"
      case dType => s"${
        childDim.getColName
          .substring(dimName.length + 1)
      }:${ addDecimalScaleAndPrecision(childDim, dType) }"
    }
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.spark.sql.util
19
20	import java.util.Objects
21
22	import scala.collection.JavaConverters._
23	import scala.collection.mutable
24
25	import org.apache.spark.sql.types
26	import org.apache.spark.sql.types._
27
28	import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
29	import org.apache.carbondata.core.metadata.schema.table.CarbonTable
30	import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, ColumnSchema}
31
32	private[spark] object SparkTypeConverter {
33
34	def createSparkSchema(table: CarbonTable, columns: Seq[String]): StructType = {
35	Objects.requireNonNull(table)	×
36	Objects.requireNonNull(columns)	×
37	if (columns.isEmpty) {	×
38	throw new IllegalArgumentException("column list is empty")	×
39	}
40	val fields = new java.util.ArrayList[StructField](columns.size)	×
41	val allColumns = table.getTableInfo.getFactTable.getListOfColumns.asScala	×
42
43	// find the column and add it to fields array
44	columns.foreach { column =>	×
45	val col = allColumns.find(_.getColumnName.equalsIgnoreCase(column)).getOrElse(	×
46	throw new IllegalArgumentException(column + " does not exist")	×
47	)
48	fields.add(StructField(col.getColumnName, convertCarbonToSparkDataType(col, table)))	×
49	}
50	StructType(fields)	×
51	}
52
53	/**
54	* Converts from carbon datatype to corresponding spark datatype.
55	*/
56	def convertCarbonToSparkDataType(
57	columnSchema: ColumnSchema,
58	table: CarbonTable): types.DataType = {
59	if (CarbonDataTypes.isDecimal(columnSchema.getDataType)) {	1✔
60	val scale = columnSchema.getScale	×
61	val precision = columnSchema.getPrecision	×
62	if (scale == 0 && precision == 0) {	×
63	DecimalType(18, 2)	×
64	} else {
65	DecimalType(precision, scale)	×
66	}
67	} else if (CarbonDataTypes.isArrayType(columnSchema.getDataType)) {	1✔
68	CarbonMetastoreTypes	×
69	.toDataType(s"array<${ getArrayChildren(table, columnSchema.getColumnName) }>")	×
70	} else if (CarbonDataTypes.isStructType(columnSchema.getDataType)) {	1✔
71	CarbonMetastoreTypes	1✔
72	.toDataType(s"struct<${ getStructChildren(table, columnSchema.getColumnName) }>")	1✔
73	} else if (CarbonDataTypes.isMapType(columnSchema.getDataType)) {	1✔
74	CarbonMetastoreTypes	×
75	.toDataType(s"map<${ getMapChildren(table, columnSchema.getColumnName) }>")	×
76	} else {
77	columnSchema.getDataType match {	1✔
78	case CarbonDataTypes.STRING => StringType	1✔
79	case CarbonDataTypes.SHORT => ShortType	1✔
80	case CarbonDataTypes.INT => IntegerType	1✔
81	case CarbonDataTypes.LONG => LongType	1✔
82	case CarbonDataTypes.DOUBLE => DoubleType	1✔
83	case CarbonDataTypes.BOOLEAN => BooleanType	×
84	case CarbonDataTypes.TIMESTAMP => TimestampType	×
85	case CarbonDataTypes.DATE => DateType	×
86	}
87	}
88	}
89
90	def getArrayChildren(table: CarbonTable, dimName: String): String = {
91	table.getChildren(dimName).asScala.map(childDim => {	1✔
92	childDim.getDataType.getName.toLowerCase match {	1✔
93	case "array" => s"array<${ getArrayChildren(table, childDim.getColName) }>"	1✔
94	case "struct" => s"struct<${ getStructChildren(table, childDim.getColName) }>"	1✔
95	case "map" => s"map<${ getMapChildren(table, childDim.getColName) }>"	1✔
96	case dType => addDecimalScaleAndPrecision(childDim, dType)	1✔
97	}
98	}).mkString(",")	1✔
99	}
100
101	def getStructChildren(table: CarbonTable, dimName: String): String = {
102	table.getChildren(dimName).asScala.map(childDim => {	1✔
103	childDim.getDataType.getName.toLowerCase match {	1✔
104	case "array" => s"${	1✔
105	childDim.getColName.substring(dimName.length + 1)	1✔
106	}:array<${ getArrayChildren(table, childDim.getColName) }>"	1✔
107	case "struct" => s"${	1✔
108	childDim.getColName.substring(dimName.length + 1)	1✔
109	}:struct<${ table.getChildren(childDim.getColName)	1✔
110	.asScala.map(f => s"${ recursiveMethod(table, childDim.getColName, f) }").mkString(",")	1✔
111	}>"	1✔
112	case "map" => s"${	×
113	childDim.getColName.substring(dimName.length + 1)	×
114	}:map<${ getMapChildren(table, childDim.getColName) }>"	×
115	case dType => s"${ childDim.getColName	×
116	.substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"	×
117	}
118	}).mkString(",")	1✔
119	}
120
121	def getMapChildren(table: CarbonTable, dimName: String): String = {
122	table.getChildren(dimName).asScala.flatMap { childDim =>	×
123	// Map<String, String> is stored internally as Map<Struct<String, String>> in carbon schema
124	// and stored as Array<Struct<String, String>> in the actual data storage. So while parsing
125	// the map dataType we can ignore the struct child and directly get the children of struct
126	// which are actual children of map
127	val structChildren = table.getChildren(childDim.getColName).asScala	×
128	structChildren.map { structChild =>	×
129	structChild.getDataType.getName.toLowerCase match {	×
130	case "array" => s"array<${ getArrayChildren(table, structChild.getColName) }>"	×
131	case "struct" => s"struct<${ getStructChildren(table, structChild.getColName) }>"	×
132	case "map" => s"map<${ getMapChildren(table, structChild.getColName) }>"	×
133	case dType => addDecimalScaleAndPrecision(structChild, dType)	×
134	}
135	}
136	}.mkString(",")	×
137	}
138
139	def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
140	var dType = dataType	1✔
141	if (CarbonDataTypes.isDecimal(dimval.getDataType)) {	1✔
142	dType +=	×
143	"(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"	×
144	}
145	dType	1✔
146	}
147
148	private def recursiveMethod(
149	table: CarbonTable, dimName: String, childDim: CarbonDimension) = {
150	childDim.getDataType.getName.toLowerCase match {	1✔
151	case "array" => s"${	1✔
152	childDim.getColName.substring(dimName.length + 1)	×
153	}:array<${ getArrayChildren(table, childDim.getColName) }>"	×
154	case "struct" => s"${	1✔
155	childDim.getColName.substring(dimName.length + 1)	×
156	}:struct<${ getStructChildren(table, childDim.getColName) }>"	×
157	case "map" => s"${	1✔
158	childDim.getColName.substring(dimName.length + 1)	×
159	}:map<${ getMapChildren(table, childDim.getColName) }>"	×
160	case dType => s"${	1✔
161	childDim.getColName	1✔
162	.substring(dimName.length + 1)	1✔
163	}:${ addDecimalScaleAndPrecision(childDim, dType) }"	1✔
164	}
165	}
166
167	}	1✔

apache / carbondata / 8263

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous