3601

Build # 3601

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/55b10a1d2ba25a09428314ee47ad54588ea6e9c8">55b10a1d2<a href="https://github.com/apache/carbondata/commit/55b10a1d2ba25a09428314ee47ad54588ea6e9c8">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/ca7cc1f65d31987affa90b6713924bbf17035fb4">ca7cc1f65</a>">ca7cc1f65</a><a href="https://github.com/apache/carbondata/commit/55b10a1d2ba25a09428314ee47ad54588ea6e9c8"> into c8c0211c3">c8c0211c3</a>

Run Details

12 of 12 new or added lines in 1 file covered. (100.0%)

63708 of 82464 relevant lines covered (77.26%)

1.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.46

/integration/spark2/src/main/scala/org/apache/spark/util/DataMapUtil.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.util

import java.io.IOException
import java.util

import scala.collection.JavaConverters._
import scala.collection.mutable

import org.apache.spark.sql.execution.command.{DataMapField, Field}

import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.MV
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}

/**
 * Utility class for keeping all the utility methods common for pre-aggregate and mv datamap
 */
object DataMapUtil {

  def inheritTablePropertiesFromMainTable(parentTable: CarbonTable,
      fields: Seq[Field],
      fieldRelationMap: scala.collection.mutable.LinkedHashMap[Field, DataMapField],
      tableProperties: mutable.Map[String, String]): Unit = {
    var neworder = Seq[String]()
    val parentOrder = parentTable.getSortColumns(parentTable.getTableName).asScala
    parentOrder.foreach(parentcol =>
      fields.filter(col => fieldRelationMap(col).aggregateFunction.isEmpty &&
                           fieldRelationMap(col).columnTableRelationList.size == 1 &&
                           parentcol.equals(fieldRelationMap(col).
                             columnTableRelationList.get(0).parentColumnName))
        .map(cols => neworder :+= cols.column))
    tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, neworder.mkString(","))
    tableProperties.put("sort_scope", parentTable.getTableInfo.getFactTable.
      getTableProperties.asScala.getOrElse("sort_scope", CarbonCommonConstants
      .LOAD_SORT_SCOPE_DEFAULT))
    tableProperties
      .put(CarbonCommonConstants.TABLE_BLOCKSIZE, parentTable.getBlockSizeInMB.toString)
    tableProperties.put(CarbonCommonConstants.FLAT_FOLDER,
      parentTable.getTableInfo.getFactTable.getTableProperties.asScala.getOrElse(
        CarbonCommonConstants.FLAT_FOLDER, CarbonCommonConstants.DEFAULT_FLAT_FOLDER))

    // Datamap table name and columns are automatically added prefix with parent table name
    // in carbon. For convenient, users can type column names same as the ones in select statement
    // when config dmproperties, and here we update column names with prefix.
    // If longStringColumn is not present in dm properties then we take long_string_columns from
    // the parent table.
    var longStringColumn = tableProperties.get(CarbonCommonConstants.LONG_STRING_COLUMNS)
    if (longStringColumn.isEmpty) {
      val longStringColumnInParents = parentTable.getTableInfo.getFactTable.getTableProperties
        .asScala
        .getOrElse(CarbonCommonConstants.LONG_STRING_COLUMNS, "").split(",").map(_.trim)
      val varcharDatamapFields = scala.collection.mutable.ArrayBuffer.empty[String]
      fieldRelationMap foreach (fields => {
        val aggFunc = fields._2.aggregateFunction
        val relationList = fields._2.columnTableRelationList
        // check if columns present in datamap are long_string_col in parent table. If they are
        // long_string_columns in parent, make them long_string_columns in datamap
        if (aggFunc.isEmpty && relationList.size == 1 && longStringColumnInParents
          .contains(relationList.head.head.parentColumnName)) {
          varcharDatamapFields += relationList.head.head.parentColumnName
        }
      })
      if (!varcharDatamapFields.isEmpty) {
        longStringColumn = Option(varcharDatamapFields.mkString(","))
      }
    }

    if (longStringColumn != None) {
      val fieldNames = fields.map(_.column)
      val newLongStringColumn = longStringColumn.get.split(",").map(_.trim).map { colName =>
        val newColName = parentTable.getTableName.toLowerCase() + "_" + colName
        if (!fieldNames.contains(newColName)) {
          throw new MalformedDataMapCommandException(
            CarbonCommonConstants.LONG_STRING_COLUMNS.toUpperCase() + ":" + colName
            + " does not in datamap")
        }
        newColName
      }
      tableProperties.put(CarbonCommonConstants.LONG_STRING_COLUMNS,
        newLongStringColumn.mkString(","))
    }

    // inherit the local dictionary properties of main parent table
    tableProperties
      .put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE,
        parentTable.getTableInfo.getFactTable.getTableProperties.asScala
          .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, "false"))
    tableProperties
      .put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,
        parentTable.getTableInfo.getFactTable.getTableProperties.asScala
          .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,
            CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT))
    val parentDictInclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala
      .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE, "").split(",")

    val parentDictExclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala
      .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE, "").split(",")

    val parentGlobalDictInclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala
      .getOrElse(CarbonCommonConstants.DICTIONARY_INCLUDE, "").split(",")

    val parentGlobalDictExclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala
      .getOrElse(CarbonCommonConstants.DICTIONARY_EXCLUDE, "").split(",")

    val newLocalDictInclude = getDataMapColumns(parentDictInclude, fields, fieldRelationMap)

    val newLocalDictExclude = getDataMapColumns(parentDictExclude, fields, fieldRelationMap)

    val newGlobalDictInclude = getDataMapColumns(parentGlobalDictInclude, fields, fieldRelationMap)

    val newGlobalDictExclude = getDataMapColumns(parentGlobalDictExclude, fields, fieldRelationMap)

    if (newLocalDictInclude.nonEmpty) {
      tableProperties
        .put(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE, newLocalDictInclude.mkString(","))
    }
    if (newLocalDictExclude.nonEmpty) {
      tableProperties
        .put(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE, newLocalDictExclude.mkString(","))
    }

    if (newGlobalDictInclude.nonEmpty) {
      tableProperties
        .put(CarbonCommonConstants.DICTIONARY_INCLUDE, newGlobalDictInclude.mkString(","))
    }
    if (newGlobalDictExclude.nonEmpty) {
      tableProperties
        .put(CarbonCommonConstants.DICTIONARY_EXCLUDE, newGlobalDictExclude.mkString(","))
    }

    val parentInvertedIndex = parentTable.getTableInfo.getFactTable.getTableProperties.asScala
      .getOrElse(CarbonCommonConstants.INVERTED_INDEX, "").split(",")

    val newInvertedIndex = getDataMapColumns(parentInvertedIndex, fields, fieldRelationMap)

    val parentNoInvertedIndex = parentTable.getTableInfo.getFactTable.getTableProperties.asScala
      .getOrElse(CarbonCommonConstants.NO_INVERTED_INDEX, "").split(",")

    val newNoInvertedIndex = getDataMapColumns(parentNoInvertedIndex, fields, fieldRelationMap)

    if (newInvertedIndex.nonEmpty) {
      tableProperties
        .put(CarbonCommonConstants.INVERTED_INDEX, newInvertedIndex.mkString(","))
    }
    if (newNoInvertedIndex.nonEmpty) {
      tableProperties
        .put(CarbonCommonConstants.NO_INVERTED_INDEX, newNoInvertedIndex.mkString(","))
    }

  }

  private def getDataMapColumns(parentColumns: Array[String], fields: Seq[Field],
      fieldRelationMap: scala.collection.mutable.LinkedHashMap[Field, DataMapField]) = {
    val dataMapColumns = parentColumns.flatMap(parentcol =>
      fields.collect {
        case col if fieldRelationMap(col).aggregateFunction.isEmpty &&
                    fieldRelationMap(col).columnTableRelationList.size == 1 &&
                    parentcol.equals(fieldRelationMap(col).
                      columnTableRelationList.get.head.parentColumnName) =>
          col.column
      })
    dataMapColumns
  }

  /**
   * Return true if MV datamap present in the specified table
   */
  @throws[IOException]
  def hasMVDataMap(carbonTable: CarbonTable): Boolean = {
    val dataMapSchemaList = DataMapStoreManager.getInstance.
      getDataMapSchemasOfTable(carbonTable).asScala
    dataMapSchemaList.foreach { dataMapSchema =>
      if (dataMapSchema.getProviderName.equalsIgnoreCase(MV.toString)) {
        return true
      }
    }
    false
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.spark.util
19
20	import java.io.IOException
21	import java.util
22
23	import scala.collection.JavaConverters._
24	import scala.collection.mutable
25
26	import org.apache.spark.sql.execution.command.{DataMapField, Field}
27
28	import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
29	import org.apache.carbondata.core.constants.CarbonCommonConstants
30	import org.apache.carbondata.core.datamap.DataMapStoreManager
31	import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.MV
32	import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
33
34	/**
35	* Utility class for keeping all the utility methods common for pre-aggregate and mv datamap
36	*/
37	object DataMapUtil {
38
39	def inheritTablePropertiesFromMainTable(parentTable: CarbonTable,
40	fields: Seq[Field],
41	fieldRelationMap: scala.collection.mutable.LinkedHashMap[Field, DataMapField],
42	tableProperties: mutable.Map[String, String]): Unit = {	2✔
43	var neworder = Seq[String]()	2✔
44	val parentOrder = parentTable.getSortColumns(parentTable.getTableName).asScala	2✔
45	parentOrder.foreach(parentcol =>	2✔
46	fields.filter(col => fieldRelationMap(col).aggregateFunction.isEmpty &&	1✔
47	fieldRelationMap(col).columnTableRelationList.size == 1 &&	1✔
48	parentcol.equals(fieldRelationMap(col).	1✔
49	columnTableRelationList.get(0).parentColumnName))	1✔
50	.map(cols => neworder :+= cols.column))	1✔
51	tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, neworder.mkString(","))	2✔
52	tableProperties.put("sort_scope", parentTable.getTableInfo.getFactTable.	2✔
53	getTableProperties.asScala.getOrElse("sort_scope", CarbonCommonConstants	2✔
54	.LOAD_SORT_SCOPE_DEFAULT))	2✔
55	tableProperties	2✔
56	.put(CarbonCommonConstants.TABLE_BLOCKSIZE, parentTable.getBlockSizeInMB.toString)	2✔
57	tableProperties.put(CarbonCommonConstants.FLAT_FOLDER,	2✔
58	parentTable.getTableInfo.getFactTable.getTableProperties.asScala.getOrElse(	2✔
59	CarbonCommonConstants.FLAT_FOLDER, CarbonCommonConstants.DEFAULT_FLAT_FOLDER))	2✔
60
61	// Datamap table name and columns are automatically added prefix with parent table name
62	// in carbon. For convenient, users can type column names same as the ones in select statement
63	// when config dmproperties, and here we update column names with prefix.
64	// If longStringColumn is not present in dm properties then we take long_string_columns from
65	// the parent table.
66	var longStringColumn = tableProperties.get(CarbonCommonConstants.LONG_STRING_COLUMNS)	2✔
67	if (longStringColumn.isEmpty) {	2✔
68	val longStringColumnInParents = parentTable.getTableInfo.getFactTable.getTableProperties	2✔
69	.asScala	2✔
70	.getOrElse(CarbonCommonConstants.LONG_STRING_COLUMNS, "").split(",").map(_.trim)	2✔
71	val varcharDatamapFields = scala.collection.mutable.ArrayBuffer.empty[String]	2✔
72	fieldRelationMap foreach (fields => {	2✔
73	val aggFunc = fields._2.aggregateFunction	2✔
74	val relationList = fields._2.columnTableRelationList	2✔
75	// check if columns present in datamap are long_string_col in parent table. If they are
76	// long_string_columns in parent, make them long_string_columns in datamap
77	if (aggFunc.isEmpty && relationList.size == 1 && longStringColumnInParents	2✔
78	.contains(relationList.head.head.parentColumnName)) {	2✔
79	varcharDatamapFields += relationList.head.head.parentColumnName	1✔
80	}
81	})
82	if (!varcharDatamapFields.isEmpty) {	2✔
83	longStringColumn = Option(varcharDatamapFields.mkString(","))	1✔
84	}
85	}
86
87	if (longStringColumn != None) {	2✔
88	val fieldNames = fields.map(_.column)	1✔
89	val newLongStringColumn = longStringColumn.get.split(",").map(_.trim).map { colName =>	1✔
90	val newColName = parentTable.getTableName.toLowerCase() + "_" + colName	1✔
91	if (!fieldNames.contains(newColName)) {	1✔
92	throw new MalformedDataMapCommandException(	×
93	CarbonCommonConstants.LONG_STRING_COLUMNS.toUpperCase() + ":" + colName	×
94	+ " does not in datamap")	×
95	}
96	newColName	1✔
97	}
98	tableProperties.put(CarbonCommonConstants.LONG_STRING_COLUMNS,	1✔
99	newLongStringColumn.mkString(","))	1✔
100	}
101
102	// inherit the local dictionary properties of main parent table
103	tableProperties	2✔
104	.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE,	2✔
105	parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
106	.getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, "false"))	2✔
107	tableProperties	2✔
108	.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,	2✔
109	parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
110	.getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,	2✔
111	CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT))	2✔
112	val parentDictInclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
113	.getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE, "").split(",")	2✔
114
115	val parentDictExclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
116	.getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE, "").split(",")	2✔
117
118	val parentGlobalDictInclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
119	.getOrElse(CarbonCommonConstants.DICTIONARY_INCLUDE, "").split(",")	2✔
120
121	val parentGlobalDictExclude = parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
122	.getOrElse(CarbonCommonConstants.DICTIONARY_EXCLUDE, "").split(",")	2✔
123
124	val newLocalDictInclude = getDataMapColumns(parentDictInclude, fields, fieldRelationMap)	2✔
125
126	val newLocalDictExclude = getDataMapColumns(parentDictExclude, fields, fieldRelationMap)	2✔
127
128	val newGlobalDictInclude = getDataMapColumns(parentGlobalDictInclude, fields, fieldRelationMap)	2✔
129
130	val newGlobalDictExclude = getDataMapColumns(parentGlobalDictExclude, fields, fieldRelationMap)	2✔
131
132	if (newLocalDictInclude.nonEmpty) {	2✔
133	tableProperties	×
134	.put(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE, newLocalDictInclude.mkString(","))	×
135	}
136	if (newLocalDictExclude.nonEmpty) {	2✔
137	tableProperties	×
138	.put(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE, newLocalDictExclude.mkString(","))	×
139	}
140
141	if (newGlobalDictInclude.nonEmpty) {	2✔
142	tableProperties	1✔
143	.put(CarbonCommonConstants.DICTIONARY_INCLUDE, newGlobalDictInclude.mkString(","))	1✔
144	}
145	if (newGlobalDictExclude.nonEmpty) {	2✔
146	tableProperties	1✔
147	.put(CarbonCommonConstants.DICTIONARY_EXCLUDE, newGlobalDictExclude.mkString(","))	1✔
148	}
149
150	val parentInvertedIndex = parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
151	.getOrElse(CarbonCommonConstants.INVERTED_INDEX, "").split(",")	2✔
152
153	val newInvertedIndex = getDataMapColumns(parentInvertedIndex, fields, fieldRelationMap)	2✔
154
155	val parentNoInvertedIndex = parentTable.getTableInfo.getFactTable.getTableProperties.asScala	2✔
156	.getOrElse(CarbonCommonConstants.NO_INVERTED_INDEX, "").split(",")	2✔
157
158	val newNoInvertedIndex = getDataMapColumns(parentNoInvertedIndex, fields, fieldRelationMap)	2✔
159
160	if (newInvertedIndex.nonEmpty) {	2✔
161	tableProperties	×
162	.put(CarbonCommonConstants.INVERTED_INDEX, newInvertedIndex.mkString(","))	×
163	}
164	if (newNoInvertedIndex.nonEmpty) {	2✔
165	tableProperties	×
166	.put(CarbonCommonConstants.NO_INVERTED_INDEX, newNoInvertedIndex.mkString(","))	×
167	}
168
169	}
170
171	private def getDataMapColumns(parentColumns: Array[String], fields: Seq[Field],
172	fieldRelationMap: scala.collection.mutable.LinkedHashMap[Field, DataMapField]) = {
173	val dataMapColumns = parentColumns.flatMap(parentcol =>	2✔
174	fields.collect {	2✔
175	case col if fieldRelationMap(col).aggregateFunction.isEmpty &&	2✔
176	fieldRelationMap(col).columnTableRelationList.size == 1 &&	2✔
177	parentcol.equals(fieldRelationMap(col).	2✔
178	columnTableRelationList.get.head.parentColumnName) =>	2✔
179	col.column	1✔
180	})
181	dataMapColumns	2✔
182	}
183
184	/**
185	* Return true if MV datamap present in the specified table
186	*/
187	@throws[IOException]
188	def hasMVDataMap(carbonTable: CarbonTable): Boolean = {	2✔
189	val dataMapSchemaList = DataMapStoreManager.getInstance.	2✔
190	getDataMapSchemasOfTable(carbonTable).asScala	2✔
191	dataMapSchemaList.foreach { dataMapSchema =>	2✔
192	if (dataMapSchema.getProviderName.equalsIgnoreCase(MV.toString)) {	2✔
193	return true	×
194	}
195	}
196	false	2✔
197	}
198	}	2✔

apache / carbondata / 3601

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous