apache / carbondata / 432

Build # 432

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/cf7939304963cfeaac80b3b7fbd305046fb26aaf">cf7939304<a href="https://github.com/apache/carbondata/commit/cf7939304963cfeaac80b3b7fbd305046fb26aaf">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/b146695317e593c034a2fc16c9f57f01336ede11">b14669531</a>">b14669531</a><a href="https://github.com/apache/carbondata/commit/cf7939304963cfeaac80b3b7fbd305046fb26aaf"> into 4c692d185">4c692d185</a>

Run Details

173 of 173 new or added lines in 12 files covered. (100.0%)

58760 of 72841 relevant lines covered (80.67%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.65

/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.scan.executor.util;

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;

import org.apache.commons.lang3.ArrayUtils;

/**
 * Utility class for restructuring
 */
public class RestructureUtil {

  /**
   * Below method will be used to get the updated query dimension updation
   * means, after restructuring some dimension will be not present in older
   * table blocks in that case we need to select only those dimension out of
   * query dimension which is present in the current table block
   *
   * @param blockExecutionInfo
   * @param queryDimensions
   * @param tableBlockDimensions
   * @param tableComplexDimension
   * @param isTransactionalTable
   * @return list of query dimension which is present in the table block
   */
  public static List<ProjectionDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(
      BlockExecutionInfo blockExecutionInfo, List<ProjectionDimension> queryDimensions,
      List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension,
      int measureCount, boolean isTransactionalTable) {
    List<ProjectionDimension> presentDimension =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    boolean[] isDimensionExists = new boolean[queryDimensions.size()];
    Object[] defaultValues = new Object[queryDimensions.size()];
    // create dimension information instance
    DimensionInfo dimensionInfo = new DimensionInfo(isDimensionExists, defaultValues);
    dimensionInfo.dataType = new DataType[queryDimensions.size() + measureCount];
    int newDictionaryColumnCount = 0;
    int newNoDictionaryColumnCount = 0;
    // selecting only those dimension which is present in the query
    int dimIndex = 0;
    for (ProjectionDimension queryDimension : queryDimensions) {
      if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
        presentDimension.add(queryDimension);
        isDimensionExists[dimIndex] = true;
        dimensionInfo.dataType[queryDimension.getOrdinal()] =
            queryDimension.getDimension().getDataType();
      } else {
        for (CarbonDimension tableDimension : tableBlockDimensions) {
          if (isColumnMatches(isTransactionalTable, queryDimension.getDimension(),
              tableDimension)) {
            ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);
            tableDimension.getColumnSchema()
                .setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision());
            tableDimension.getColumnSchema()
                .setScale(queryDimension.getDimension().getColumnSchema().getScale());
            tableDimension.getColumnSchema()
                .setDefaultValue(queryDimension.getDimension().getDefaultValue());
            currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
            presentDimension.add(currentBlockDimension);
            isDimensionExists[dimIndex] = true;
            dimensionInfo.dataType[currentBlockDimension.getOrdinal()] =
                currentBlockDimension.getDimension().getDataType();
            break;
          }
        }
        // if dimension is found then no need to search in the complex dimensions list
        if (isDimensionExists[dimIndex]) {
          dimIndex++;
          continue;
        }
        for (CarbonDimension tableDimension : tableComplexDimension) {
          if (isColumnMatches(isTransactionalTable, queryDimension.getDimension(),
              tableDimension)) {
            ProjectionDimension currentBlockDimension = null;
            // If projection dimension is child of struct field and contains Parent Ordinal
            if (null != queryDimension.getDimension().getComplexParentDimension()) {
              currentBlockDimension = new ProjectionDimension(queryDimension.getDimension());
            } else {
              currentBlockDimension = new ProjectionDimension(tableDimension);
            }
            // TODO: for complex dimension set scale and precision by traversing
            // the child dimensions
            currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
            presentDimension.add(currentBlockDimension);
            isDimensionExists[dimIndex] = true;
            dimensionInfo.dataType[currentBlockDimension.getOrdinal()] =
                currentBlockDimension.getDimension().getDataType();
            break;
          }
        }
        // add default value only in case query dimension is not found in the current block
        if (!isDimensionExists[dimIndex]) {
          defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension());
          blockExecutionInfo.setRestructuredBlock(true);
          // set the flag to say whether a new dictionary column or no dictionary column
          // has been added. This will be useful after restructure for compaction scenarios where
          // newly added columns data need to be filled
          if (queryDimension.getDimension().hasEncoding(Encoding.DICTIONARY)) {
            dimensionInfo.setDictionaryColumnAdded(true);
            newDictionaryColumnCount++;
          } else {
            dimensionInfo.setNoDictionaryColumnAdded(true);
            newNoDictionaryColumnCount++;
          }
        }
      }
      dimIndex++;
    }
    dimensionInfo.setNewDictionaryColumnCount(newDictionaryColumnCount);
    dimensionInfo.setNewNoDictionaryColumnCount(newNoDictionaryColumnCount);
    blockExecutionInfo.setDimensionInfo(dimensionInfo);
    return presentDimension;
  }

  /**
   * Match the columns for transactional and non transactional tables
   * @param isTransactionalTable
   * @param queryColumn
   * @param tableColumn
   * @return
   */
  private static boolean isColumnMatches(boolean isTransactionalTable,
      CarbonColumn queryColumn, CarbonColumn tableColumn) {
    // If it is non transactional table just check the column names, no need to validate
    // column id as multiple sdk's output placed in a single folder doesn't have same
    // column ID but can have same column name
    if (tableColumn.getDataType().isComplexType() && !(tableColumn.getDataType().getId()
        == DataTypes.ARRAY_TYPE_ID)) {
      if (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId())) {
        return true;
      } else {
        return isColumnMatchesStruct(tableColumn, queryColumn);
      }
    } else {
      return (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId()) || (
          !isTransactionalTable && tableColumn.getColName()
              .equalsIgnoreCase(queryColumn.getColName())));
    }
  }

  /**
   * In case of Multilevel Complex column - STRUCT/STRUCTofSTRUCT, traverse all the child dimension
   * to check column Id
   *
   * @param tableColumn
   * @param queryColumn
   * @return
   */
  private static boolean isColumnMatchesStruct(CarbonColumn tableColumn, CarbonColumn queryColumn) {
    if (tableColumn instanceof CarbonDimension) {
      List<CarbonDimension> parentDimension =
          ((CarbonDimension) tableColumn).getListOfChildDimensions();
      CarbonDimension carbonDimension = null;
      String[] colSplits = queryColumn.getColName().split("\\.");
      StringBuffer tempColName = new StringBuffer(colSplits[0]);
      for (String colSplit : colSplits) {
        if (!tempColName.toString().equalsIgnoreCase(colSplit)) {
          tempColName = tempColName.append(".").append(colSplit);
        }
        carbonDimension = CarbonTable.getCarbonDimension(tempColName.toString(), parentDimension);
        if (carbonDimension != null) {
          if (carbonDimension.getColumnSchema().getColumnUniqueId()
              .equalsIgnoreCase(queryColumn.getColumnId())) {
            return true;
          }
          if (carbonDimension.getListOfChildDimensions() != null) {
            parentDimension = carbonDimension.getListOfChildDimensions();
          }
        }
      }
    }
    return false;
  }

  /**
   * This method will validate and return the default value to be
   * filled at the time of result preparation
   *
   * @param queryDimension
   * @return
   */
  public static Object validateAndGetDefaultValue(CarbonDimension queryDimension) {
    byte[] defaultValue = queryDimension.getDefaultValue();
    Object defaultValueToBeConsidered = null;
    if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DICTIONARY)) {
      // direct dictionary case
      if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
        defaultValueToBeConsidered = getDirectDictionaryDefaultValue(queryDimension.getDataType(),
            queryDimension.getDefaultValue());
      } else {
        // dictionary case
        defaultValueToBeConsidered = getDictionaryDefaultValue(defaultValue);
      }
    } else {
      // no dictionary
      defaultValueToBeConsidered =
          getNoDictionaryDefaultValue(queryDimension.getDataType(), defaultValue);
    }
    return defaultValueToBeConsidered;
  }

  /**
   * Method for computing default value for dictionary column
   *
   * @param defaultValue
   * @return
   */
  private static Object getDictionaryDefaultValue(byte[] defaultValue) {
    Object dictionaryDefaultValue = null;
    // dictionary has 2 cases:
    // 1. If default value is specified then its surrogate key will be 2
    // 2.  If default value is not specified then its surrogate key will be
    // 1 which is for member default value null
    if (isDefaultValueNull(defaultValue)) {
      dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
    } else {
      dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1;
    }
    return dictionaryDefaultValue;
  }

  /**
   * Method for computing default value for direct dictionary
   *
   * @param dataType
   * @param defaultValue
   * @return
   */
  public static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) {
    Object directDictionaryDefaultValue = null;
    if (!isDefaultValueNull(defaultValue)) {
      DirectDictionaryGenerator directDictionaryGenerator =
          DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dataType);
      if (directDictionaryGenerator != null) {
        String value =
            new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        directDictionaryDefaultValue =
            directDictionaryGenerator.getValueFromSurrogate(Integer.parseInt(value));
      }
    }
    return directDictionaryDefaultValue;
  }

  /**
   * Method for computing default value for no dictionary
   *
   * @param defaultValue
   * @return
   */
  private static Object getNoDictionaryDefaultValue(DataType datatype, byte[] defaultValue) {
    Object noDictionaryDefaultValue = null;
    String value = null;
    if (!isDefaultValueNull(defaultValue)) {
      if (datatype == DataTypes.INT) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        noDictionaryDefaultValue = Integer.parseInt(value);
      } else if (datatype == DataTypes.LONG) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        noDictionaryDefaultValue = Long.parseLong(value);
      } else if (datatype == DataTypes.TIMESTAMP) {
        long timestampValue = ByteUtil.toXorLong(defaultValue, 0, defaultValue.length);
        noDictionaryDefaultValue = timestampValue * 1000L;
      } else {
        noDictionaryDefaultValue =
            DataTypeUtil.getDataTypeConverter().convertFromByteToUTF8Bytes(defaultValue);
      }
    }
    return noDictionaryDefaultValue;
  }

  /**
   * This method will validate whether a given value is empty or null
   *
   * @param defaultValue
   * @return
   */
  private static boolean isDefaultValueNull(byte[] defaultValue) {
    return null == defaultValue;
  }

  /**
   * Method for computing measure default value based on the data type
   *
   * @param columnSchema
   * @param defaultValue
   * @return
   */
  public static Object getMeasureDefaultValue(ColumnSchema columnSchema, byte[] defaultValue) {
    Object measureDefaultValue = null;
    if (!isDefaultValueNull(defaultValue)) {
      String value;
      DataType dataType = columnSchema.getDataType();
      if (dataType == DataTypes.SHORT) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Short.valueOf(value);
      } else if (dataType == DataTypes.LONG) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Long.parseLong(value);
      } else if (dataType == DataTypes.INT) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Integer.parseInt(value);
      } else if (dataType == DataTypes.BOOLEAN) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Boolean.valueOf(value);
      } else if (DataTypes.isDecimal(dataType)) {
        BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue);
        if (columnSchema.getScale() > decimal.scale()) {
          decimal = decimal.setScale(columnSchema.getScale(), RoundingMode.HALF_UP);
        }
        measureDefaultValue = decimal;
      } else {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        Double parsedValue = Double.valueOf(value);
        if (!Double.isInfinite(parsedValue) && !Double.isNaN(parsedValue)) {
          measureDefaultValue = parsedValue;
        }
      }
    }
    return measureDefaultValue;
  }

  /**
   * Gets the default value based on the column data type.
   *
   * @param columnSchema
   * @param defaultValue
   * @return
   */
  public static Object getMeasureDefaultValueByType(ColumnSchema columnSchema,
      byte[] defaultValue) {
    Object measureDefaultValue = null;
    if (!isDefaultValueNull(defaultValue)) {
      String value;
      DataType dataType = columnSchema.getDataType();
      if (dataType == DataTypes.SHORT) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Short.parseShort(value);
      } else if (dataType == DataTypes.INT) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Integer.parseInt(value);
      } else if (dataType == DataTypes.LONG) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Long.parseLong(value);
      } else if (dataType == DataTypes.BOOLEAN) {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        measureDefaultValue = Boolean.valueOf(value);
      } else if (DataTypes.isDecimal(dataType)) {
        BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue);
        if (columnSchema.getScale() > decimal.scale()) {
          decimal = decimal.setScale(columnSchema.getScale(), RoundingMode.HALF_UP);
        }
        measureDefaultValue = decimal;
      } else {
        value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        Double parsedValue = Double.valueOf(value);
        if (!Double.isInfinite(parsedValue) && !Double.isNaN(parsedValue)) {
          measureDefaultValue = parsedValue;
        }
      }
    }
    return measureDefaultValue;
  }

  /**
   * Below method will be used to prepare the measure info object
   * in this method some of the properties which will be extracted
   * from query measure and current block measures will be set
   *
   * @param blockExecutionInfo
   * @param queryMeasures        measures present in query
   * @param currentBlockMeasures current block measures
   * @param isTransactionalTable
   * @return measures present in the block
   */
  public static List<ProjectionMeasure> createMeasureInfoAndGetCurrentBlockQueryMeasures(
      BlockExecutionInfo blockExecutionInfo, List<ProjectionMeasure> queryMeasures,
      List<CarbonMeasure> currentBlockMeasures, boolean isTransactionalTable) {
    MeasureInfo measureInfo = new MeasureInfo();
    List<ProjectionMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
    int numberOfMeasureInQuery = queryMeasures.size();
    List<Integer> measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery);
    Object[] defaultValues = new Object[numberOfMeasureInQuery];
    boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
    int index = 0;
    for (ProjectionMeasure queryMeasure : queryMeasures) {
      // if query measure exists in current dimension measures
      // then setting measure exists is true
      // otherwise adding a default value of a measure
      for (CarbonMeasure carbonMeasure : currentBlockMeasures) {
        if (isColumnMatches(isTransactionalTable, carbonMeasure, queryMeasure.getMeasure())) {
          ProjectionMeasure currentBlockMeasure = new ProjectionMeasure(carbonMeasure);
          carbonMeasure.getColumnSchema().setDataType(queryMeasure.getMeasure().getDataType());
          carbonMeasure.getColumnSchema().setPrecision(queryMeasure.getMeasure().getPrecision());
          carbonMeasure.getColumnSchema().setScale(queryMeasure.getMeasure().getScale());
          carbonMeasure.getColumnSchema()
              .setDefaultValue(queryMeasure.getMeasure().getDefaultValue());
          currentBlockMeasure.setOrdinal(queryMeasure.getOrdinal());
          presentMeasure.add(currentBlockMeasure);
          measureOrdinalList.add(carbonMeasure.getOrdinal());
          measureExistsInCurrentBlock[index] = true;
          break;
        }
      }
      if (!measureExistsInCurrentBlock[index]) {
        defaultValues[index] = getMeasureDefaultValue(queryMeasure.getMeasure().getColumnSchema(),
            queryMeasure.getMeasure().getDefaultValue());
        blockExecutionInfo.setRestructuredBlock(true);
      }
      index++;
    }
    int[] measureOrdinals =
        ArrayUtils.toPrimitive(measureOrdinalList.toArray(new Integer[measureOrdinalList.size()]));
    measureInfo.setDefaultValues(defaultValues);
    measureInfo.setMeasureOrdinals(measureOrdinals);
    measureInfo.setMeasureExists(measureExistsInCurrentBlock);
    blockExecutionInfo.setMeasureInfo(measureInfo);
    return presentMeasure;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.scan.executor.util;
18
19	import java.math.BigDecimal;
20	import java.math.RoundingMode;
21	import java.nio.charset.Charset;
22	import java.util.ArrayList;
23	import java.util.List;
24
25	import org.apache.carbondata.core.constants.CarbonCommonConstants;
26	import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
27	import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
28	import org.apache.carbondata.core.metadata.datatype.DataType;
29	import org.apache.carbondata.core.metadata.datatype.DataTypes;
30	import org.apache.carbondata.core.metadata.encoder.Encoding;
31	import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
32	import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
33	import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
34	import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
35	import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
36	import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
37	import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
38	import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
39	import org.apache.carbondata.core.scan.model.ProjectionDimension;
40	import org.apache.carbondata.core.scan.model.ProjectionMeasure;
41	import org.apache.carbondata.core.util.ByteUtil;
42	import org.apache.carbondata.core.util.CarbonUtil;
43	import org.apache.carbondata.core.util.DataTypeUtil;
44
45	import org.apache.commons.lang3.ArrayUtils;
46
47	/**
48	* Utility class for restructuring
49	*/
50	public class RestructureUtil {	×
51
52	/**
53	* Below method will be used to get the updated query dimension updation
54	* means, after restructuring some dimension will be not present in older
55	* table blocks in that case we need to select only those dimension out of
56	* query dimension which is present in the current table block
57	*
58	* @param blockExecutionInfo
59	* @param queryDimensions
60	* @param tableBlockDimensions
61	* @param tableComplexDimension
62	* @param isTransactionalTable
63	* @return list of query dimension which is present in the table block
64	*/
65	public static List<ProjectionDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(
66	BlockExecutionInfo blockExecutionInfo, List<ProjectionDimension> queryDimensions,
67	List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension,
68	int measureCount, boolean isTransactionalTable) {
69	List<ProjectionDimension> presentDimension =	2✔
70	new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
71	boolean[] isDimensionExists = new boolean[queryDimensions.size()];	2✔
72	Object[] defaultValues = new Object[queryDimensions.size()];	2✔
73	// create dimension information instance
74	DimensionInfo dimensionInfo = new DimensionInfo(isDimensionExists, defaultValues);	2✔
75	dimensionInfo.dataType = new DataType[queryDimensions.size() + measureCount];	2✔
76	int newDictionaryColumnCount = 0;	2✔
77	int newNoDictionaryColumnCount = 0;	2✔
78	// selecting only those dimension which is present in the query
79	int dimIndex = 0;	2✔
80	for (ProjectionDimension queryDimension : queryDimensions) {	2✔
81	if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {	2✔
82	presentDimension.add(queryDimension);	1✔
83	isDimensionExists[dimIndex] = true;	1✔
84	dimensionInfo.dataType[queryDimension.getOrdinal()] =	1✔
85	queryDimension.getDimension().getDataType();	1✔
86	} else {
87	for (CarbonDimension tableDimension : tableBlockDimensions) {	2✔
88	if (isColumnMatches(isTransactionalTable, queryDimension.getDimension(),	2✔
89	tableDimension)) {
90	ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);	2✔
91	tableDimension.getColumnSchema()	2✔
92	.setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision());	2✔
93	tableDimension.getColumnSchema()	2✔
94	.setScale(queryDimension.getDimension().getColumnSchema().getScale());	2✔
95	tableDimension.getColumnSchema()	2✔
96	.setDefaultValue(queryDimension.getDimension().getDefaultValue());	2✔
97	currentBlockDimension.setOrdinal(queryDimension.getOrdinal());	2✔
98	presentDimension.add(currentBlockDimension);	2✔
99	isDimensionExists[dimIndex] = true;	2✔
100	dimensionInfo.dataType[currentBlockDimension.getOrdinal()] =	2✔
101	currentBlockDimension.getDimension().getDataType();	2✔
102	break;	2✔
103	}
104	}	2✔
105	// if dimension is found then no need to search in the complex dimensions list
106	if (isDimensionExists[dimIndex]) {	2✔
107	dimIndex++;	2✔
108	continue;	2✔
109	}
110	for (CarbonDimension tableDimension : tableComplexDimension) {	2✔
111	if (isColumnMatches(isTransactionalTable, queryDimension.getDimension(),	2✔
112	tableDimension)) {
113	ProjectionDimension currentBlockDimension = null;	2✔
114	// If projection dimension is child of struct field and contains Parent Ordinal
115	if (null != queryDimension.getDimension().getComplexParentDimension()) {	2✔
116	currentBlockDimension = new ProjectionDimension(queryDimension.getDimension());	1✔
117	} else {
118	currentBlockDimension = new ProjectionDimension(tableDimension);	2✔
119	}
120	// TODO: for complex dimension set scale and precision by traversing
121	// the child dimensions
122	currentBlockDimension.setOrdinal(queryDimension.getOrdinal());	2✔
123	presentDimension.add(currentBlockDimension);	2✔
124	isDimensionExists[dimIndex] = true;	2✔
125	dimensionInfo.dataType[currentBlockDimension.getOrdinal()] =	2✔
126	currentBlockDimension.getDimension().getDataType();	2✔
127	break;	2✔
128	}
129	}	2✔
130	// add default value only in case query dimension is not found in the current block
131	if (!isDimensionExists[dimIndex]) {	2✔
132	defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension());	2✔
133	blockExecutionInfo.setRestructuredBlock(true);	2✔
134	// set the flag to say whether a new dictionary column or no dictionary column
135	// has been added. This will be useful after restructure for compaction scenarios where
136	// newly added columns data need to be filled
137	if (queryDimension.getDimension().hasEncoding(Encoding.DICTIONARY)) {	2✔
138	dimensionInfo.setDictionaryColumnAdded(true);	2✔
139	newDictionaryColumnCount++;	2✔
140	} else {
141	dimensionInfo.setNoDictionaryColumnAdded(true);	1✔
142	newNoDictionaryColumnCount++;	1✔
143	}
144	}
145	}
146	dimIndex++;	2✔
147	}	2✔
148	dimensionInfo.setNewDictionaryColumnCount(newDictionaryColumnCount);	2✔
149	dimensionInfo.setNewNoDictionaryColumnCount(newNoDictionaryColumnCount);	2✔
150	blockExecutionInfo.setDimensionInfo(dimensionInfo);	2✔
151	return presentDimension;	2✔
152	}
153
154	/**
155	* Match the columns for transactional and non transactional tables
156	* @param isTransactionalTable
157	* @param queryColumn
158	* @param tableColumn
159	* @return
160	*/
161	private static boolean isColumnMatches(boolean isTransactionalTable,
162	CarbonColumn queryColumn, CarbonColumn tableColumn) {
163	// If it is non transactional table just check the column names, no need to validate
164	// column id as multiple sdk's output placed in a single folder doesn't have same
165	// column ID but can have same column name
166	if (tableColumn.getDataType().isComplexType() && !(tableColumn.getDataType().getId()	2✔
167	== DataTypes.ARRAY_TYPE_ID)) {
168	if (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId())) {	1✔
169	return true;	1✔
170	} else {
171	return isColumnMatchesStruct(tableColumn, queryColumn);	1✔
172	}
173	} else {
174	return (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId()) \|\| (	2✔
175	!isTransactionalTable && tableColumn.getColName()	1✔
176	.equalsIgnoreCase(queryColumn.getColName())));	2✔
177	}
178	}
179
180	/**
181	* In case of Multilevel Complex column - STRUCT/STRUCTofSTRUCT, traverse all the child dimension
182	* to check column Id
183	*
184	* @param tableColumn
185	* @param queryColumn
186	* @return
187	*/
188	private static boolean isColumnMatchesStruct(CarbonColumn tableColumn, CarbonColumn queryColumn) {
189	if (tableColumn instanceof CarbonDimension) {	1✔
190	List<CarbonDimension> parentDimension =	1✔
191	((CarbonDimension) tableColumn).getListOfChildDimensions();	1✔
192	CarbonDimension carbonDimension = null;	1✔
193	String[] colSplits = queryColumn.getColName().split("\\.");	1✔
194	StringBuffer tempColName = new StringBuffer(colSplits[0]);	1✔
195	for (String colSplit : colSplits) {	1✔
196	if (!tempColName.toString().equalsIgnoreCase(colSplit)) {	1✔
197	tempColName = tempColName.append(".").append(colSplit);	1✔
198	}
199	carbonDimension = CarbonTable.getCarbonDimension(tempColName.toString(), parentDimension);	1✔
200	if (carbonDimension != null) {	1✔
201	if (carbonDimension.getColumnSchema().getColumnUniqueId()	1✔
202	.equalsIgnoreCase(queryColumn.getColumnId())) {	1✔
203	return true;	1✔
204	}
205	if (carbonDimension.getListOfChildDimensions() != null) {	1✔
206	parentDimension = carbonDimension.getListOfChildDimensions();	1✔
207	}
208	}
209	}
210	}
211	return false;	1✔
212	}
213
214	/**
215	* This method will validate and return the default value to be
216	* filled at the time of result preparation
217	*
218	* @param queryDimension
219	* @return
220	*/
221	public static Object validateAndGetDefaultValue(CarbonDimension queryDimension) {
222	byte[] defaultValue = queryDimension.getDefaultValue();	2✔
223	Object defaultValueToBeConsidered = null;	2✔
224	if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DICTIONARY)) {	2✔
225	// direct dictionary case
226	if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DIRECT_DICTIONARY)) {	2✔
227	defaultValueToBeConsidered = getDirectDictionaryDefaultValue(queryDimension.getDataType(),	1✔
228	queryDimension.getDefaultValue());	1✔
229	} else {
230	// dictionary case
231	defaultValueToBeConsidered = getDictionaryDefaultValue(defaultValue);	2✔
232	}
233	} else {
234	// no dictionary
235	defaultValueToBeConsidered =	1✔
236	getNoDictionaryDefaultValue(queryDimension.getDataType(), defaultValue);	1✔
237	}
238	return defaultValueToBeConsidered;	2✔
239	}
240
241	/**
242	* Method for computing default value for dictionary column
243	*
244	* @param defaultValue
245	* @return
246	*/
247	private static Object getDictionaryDefaultValue(byte[] defaultValue) {
248	Object dictionaryDefaultValue = null;	2✔
249	// dictionary has 2 cases:
250	// 1. If default value is specified then its surrogate key will be 2
251	// 2. If default value is not specified then its surrogate key will be
252	// 1 which is for member default value null
253	if (isDefaultValueNull(defaultValue)) {	2✔
254	dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;	2✔
255	} else {
256	dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1;	1✔
257	}
258	return dictionaryDefaultValue;	2✔
259	}
260
261	/**
262	* Method for computing default value for direct dictionary
263	*
264	* @param dataType
265	* @param defaultValue
266	* @return
267	*/
268	public static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) {
269	Object directDictionaryDefaultValue = null;	1✔
270	if (!isDefaultValueNull(defaultValue)) {	1✔
271	DirectDictionaryGenerator directDictionaryGenerator =	1✔
272	DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dataType);	1✔
273	if (directDictionaryGenerator != null) {	1✔
274	String value =	1✔
275	new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
276	directDictionaryDefaultValue =	1✔
277	directDictionaryGenerator.getValueFromSurrogate(Integer.parseInt(value));	1✔
278	}
279	}
280	return directDictionaryDefaultValue;	1✔
281	}
282
283	/**
284	* Method for computing default value for no dictionary
285	*
286	* @param defaultValue
287	* @return
288	*/
289	private static Object getNoDictionaryDefaultValue(DataType datatype, byte[] defaultValue) {
290	Object noDictionaryDefaultValue = null;	1✔
291	String value = null;	1✔
292	if (!isDefaultValueNull(defaultValue)) {	1✔
293	if (datatype == DataTypes.INT) {	1✔
294	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
295	noDictionaryDefaultValue = Integer.parseInt(value);	1✔
296	} else if (datatype == DataTypes.LONG) {	1✔
297	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
298	noDictionaryDefaultValue = Long.parseLong(value);	1✔
299	} else if (datatype == DataTypes.TIMESTAMP) {	1✔
300	long timestampValue = ByteUtil.toXorLong(defaultValue, 0, defaultValue.length);	1✔
301	noDictionaryDefaultValue = timestampValue * 1000L;	1✔
302	} else {	1✔
303	noDictionaryDefaultValue =
304	DataTypeUtil.getDataTypeConverter().convertFromByteToUTF8Bytes(defaultValue);	1✔
305	}
306	}
307	return noDictionaryDefaultValue;	1✔
308	}
309
310	/**
311	* This method will validate whether a given value is empty or null
312	*
313	* @param defaultValue
314	* @return
315	*/
316	private static boolean isDefaultValueNull(byte[] defaultValue) {
317	return null == defaultValue;	2✔
318	}
319
320	/**
321	* Method for computing measure default value based on the data type
322	*
323	* @param columnSchema
324	* @param defaultValue
325	* @return
326	*/
327	public static Object getMeasureDefaultValue(ColumnSchema columnSchema, byte[] defaultValue) {
328	Object measureDefaultValue = null;	2✔
329	if (!isDefaultValueNull(defaultValue)) {	2✔
330	String value;
331	DataType dataType = columnSchema.getDataType();	2✔
332	if (dataType == DataTypes.SHORT) {	2✔
333	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
334	measureDefaultValue = Short.valueOf(value);	1✔
335	} else if (dataType == DataTypes.LONG) {	2✔
336	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
337	measureDefaultValue = Long.parseLong(value);	1✔
338	} else if (dataType == DataTypes.INT) {	2✔
339	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
340	measureDefaultValue = Integer.parseInt(value);	1✔
341	} else if (dataType == DataTypes.BOOLEAN) {	2✔
342	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
343	measureDefaultValue = Boolean.valueOf(value);	1✔
344	} else if (DataTypes.isDecimal(dataType)) {	2✔
345	BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue);	1✔
346	if (columnSchema.getScale() > decimal.scale()) {	1✔
347	decimal = decimal.setScale(columnSchema.getScale(), RoundingMode.HALF_UP);	×
348	}
349	measureDefaultValue = decimal;	1✔
350	} else {	1✔
351	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	2✔
352	Double parsedValue = Double.valueOf(value);	2✔
353	if (!Double.isInfinite(parsedValue) && !Double.isNaN(parsedValue)) {	2✔
354	measureDefaultValue = parsedValue;	2✔
355	}
356	}
357	}
358	return measureDefaultValue;	2✔
359	}
360
361	/**
362	* Gets the default value based on the column data type.
363	*
364	* @param columnSchema
365	* @param defaultValue
366	* @return
367	*/
368	public static Object getMeasureDefaultValueByType(ColumnSchema columnSchema,
369	byte[] defaultValue) {
370	Object measureDefaultValue = null;	1✔
371	if (!isDefaultValueNull(defaultValue)) {	1✔
372	String value;
373	DataType dataType = columnSchema.getDataType();	1✔
374	if (dataType == DataTypes.SHORT) {	1✔
375	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
376	measureDefaultValue = Short.parseShort(value);	1✔
377	} else if (dataType == DataTypes.INT) {	1✔
378	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
379	measureDefaultValue = Integer.parseInt(value);	1✔
380	} else if (dataType == DataTypes.LONG) {	1✔
381	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	×
382	measureDefaultValue = Long.parseLong(value);	×
383	} else if (dataType == DataTypes.BOOLEAN) {	1✔
384	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
385	measureDefaultValue = Boolean.valueOf(value);	1✔
386	} else if (DataTypes.isDecimal(dataType)) {	1✔
387	BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue);	1✔
388	if (columnSchema.getScale() > decimal.scale()) {	1✔
389	decimal = decimal.setScale(columnSchema.getScale(), RoundingMode.HALF_UP);	×
390	}
391	measureDefaultValue = decimal;	1✔
392	} else {	1✔
393	value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	1✔
394	Double parsedValue = Double.valueOf(value);	1✔
395	if (!Double.isInfinite(parsedValue) && !Double.isNaN(parsedValue)) {	1✔
396	measureDefaultValue = parsedValue;	1✔
397	}
398	}
399	}
400	return measureDefaultValue;	1✔
401	}
402
403	/**
404	* Below method will be used to prepare the measure info object
405	* in this method some of the properties which will be extracted
406	* from query measure and current block measures will be set
407	*
408	* @param blockExecutionInfo
409	* @param queryMeasures measures present in query
410	* @param currentBlockMeasures current block measures
411	* @param isTransactionalTable
412	* @return measures present in the block
413	*/
414	public static List<ProjectionMeasure> createMeasureInfoAndGetCurrentBlockQueryMeasures(
415	BlockExecutionInfo blockExecutionInfo, List<ProjectionMeasure> queryMeasures,
416	List<CarbonMeasure> currentBlockMeasures, boolean isTransactionalTable) {
417	MeasureInfo measureInfo = new MeasureInfo();	2✔
418	List<ProjectionMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());	2✔
419	int numberOfMeasureInQuery = queryMeasures.size();	2✔
420	List<Integer> measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery);	2✔
421	Object[] defaultValues = new Object[numberOfMeasureInQuery];	2✔
422	boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];	2✔
423	int index = 0;	2✔
424	for (ProjectionMeasure queryMeasure : queryMeasures) {	2✔
425	// if query measure exists in current dimension measures
426	// then setting measure exists is true
427	// otherwise adding a default value of a measure
428	for (CarbonMeasure carbonMeasure : currentBlockMeasures) {	2✔
429	if (isColumnMatches(isTransactionalTable, carbonMeasure, queryMeasure.getMeasure())) {	2✔
430	ProjectionMeasure currentBlockMeasure = new ProjectionMeasure(carbonMeasure);	2✔
431	carbonMeasure.getColumnSchema().setDataType(queryMeasure.getMeasure().getDataType());	2✔
432	carbonMeasure.getColumnSchema().setPrecision(queryMeasure.getMeasure().getPrecision());	2✔
433	carbonMeasure.getColumnSchema().setScale(queryMeasure.getMeasure().getScale());	2✔
434	carbonMeasure.getColumnSchema()	2✔
435	.setDefaultValue(queryMeasure.getMeasure().getDefaultValue());	2✔
436	currentBlockMeasure.setOrdinal(queryMeasure.getOrdinal());	2✔
437	presentMeasure.add(currentBlockMeasure);	2✔
438	measureOrdinalList.add(carbonMeasure.getOrdinal());	2✔
439	measureExistsInCurrentBlock[index] = true;	2✔
440	break;	2✔
441	}
442	}	2✔
443	if (!measureExistsInCurrentBlock[index]) {	2✔
444	defaultValues[index] = getMeasureDefaultValue(queryMeasure.getMeasure().getColumnSchema(),	2✔
445	queryMeasure.getMeasure().getDefaultValue());	2✔
446	blockExecutionInfo.setRestructuredBlock(true);	2✔
447	}
448	index++;	2✔
449	}	2✔
450	int[] measureOrdinals =	2✔
451	ArrayUtils.toPrimitive(measureOrdinalList.toArray(new Integer[measureOrdinalList.size()]));	2✔
452	measureInfo.setDefaultValues(defaultValues);	2✔
453	measureInfo.setMeasureOrdinals(measureOrdinals);	2✔
454	measureInfo.setMeasureExists(measureExistsInCurrentBlock);	2✔
455	blockExecutionInfo.setMeasureInfo(measureInfo);	2✔
456	return presentMeasure;	2✔
457	}
458	}

apache / carbondata / 432

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous