apache / carbondata / 710

Build # 710

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/560cc84777582e14ada76e44efbe0f6775866519">560cc8477<a href="https://github.com/apache/carbondata/commit/560cc84777582e14ada76e44efbe0f6775866519">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/a7fec5bbc8355aaa73f87af7152b3947b8fa9acd">a7fec5bbc</a>">a7fec5bbc</a><a href="https://github.com/apache/carbondata/commit/560cc84777582e14ada76e44efbe0f6775866519"> into 3cd8b947c">3cd8b947c</a>

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

60502 of 75051 relevant lines covered (80.61%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.17

/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.datastore.page.encoding;

import java.math.BigDecimal;

import org.apache.carbondata.core.datastore.ColumnType;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.DecimalColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaFloatingCodec;
import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaIntegralCodec;
import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveFloatingCodec;
import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveIntegralCodec;
import org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.ComplexDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DictDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DirectDictDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.HighCardDictDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.DecimalConverterFactory;
import org.apache.carbondata.core.util.DataTypeUtil;

/**
 * Default factory will select encoding base on column page data type and statistics
 */
public class DefaultEncodingFactory extends EncodingFactory {

  private static final int THREE_BYTES_MAX = (int) Math.pow(2, 23) - 1;
  private static final int THREE_BYTES_MIN = - THREE_BYTES_MAX - 1;

  private static final boolean newWay = false;

  private static EncodingFactory encodingFactory = new DefaultEncodingFactory();

  public static EncodingFactory getInstance() {
    // TODO: make it configurable after added new encodingFactory
    return encodingFactory;
  }

  @Override
  public ColumnPageEncoder createEncoder(TableSpec.ColumnSpec columnSpec, ColumnPage inputPage) {
    // TODO: add log
    // choose the encoding type for measure type and no dictionary primitive type columns
    if (columnSpec instanceof TableSpec.MeasureSpec || (
        DataTypeUtil.isPrimitiveColumn(columnSpec.getSchemaDataType())
            && columnSpec.getColumnType() == ColumnType.PLAIN_VALUE)) {
      return createEncoderForMeasureOrNoDictionaryPrimitive(inputPage, columnSpec);
    } else {
      if (newWay) {
        return createEncoderForDimension((TableSpec.DimensionSpec) columnSpec, inputPage);
      } else {
        assert columnSpec instanceof TableSpec.DimensionSpec;
        return createEncoderForDimensionLegacy((TableSpec.DimensionSpec) columnSpec);
      }
    }
  }

  private ColumnPageEncoder createEncoderForDimension(TableSpec.DimensionSpec columnSpec,
      ColumnPage inputPage) {
    switch (columnSpec.getColumnType()) {
      case GLOBAL_DICTIONARY:
      case DIRECT_DICTIONARY:
      case PLAIN_VALUE:
        return new DirectCompressCodec(inputPage.getDataType()).createEncoder(null);
      case COMPLEX:
        return new ComplexDimensionIndexCodec(false, false).createEncoder(null);
      default:
        throw new RuntimeException("unsupported dimension type: " + columnSpec.getColumnType());
    }
  }

  private ColumnPageEncoder createEncoderForDimensionLegacy(TableSpec.DimensionSpec dimensionSpec) {
    switch (dimensionSpec.getColumnType()) {
      case GLOBAL_DICTIONARY:
        return new DictDimensionIndexCodec(
            dimensionSpec.isInSortColumns(),
            dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex())
            .createEncoder(null);
      case DIRECT_DICTIONARY:
        return new DirectDictDimensionIndexCodec(
            dimensionSpec.isInSortColumns(),
            dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex())
            .createEncoder(null);
      case PLAIN_VALUE:
        return new HighCardDictDimensionIndexCodec(
            dimensionSpec.isInSortColumns(),
            dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex(),
            dimensionSpec.getSchemaDataType() == DataTypes.VARCHAR)
            .createEncoder(null);
      default:
        throw new RuntimeException("unsupported dimension type: " +
            dimensionSpec.getColumnType());
    }
  }

  private ColumnPageEncoder createEncoderForMeasureOrNoDictionaryPrimitive(ColumnPage columnPage,
      TableSpec.ColumnSpec columnSpec) {
    SimpleStatsResult stats = columnPage.getStatistics();
    DataType dataType = stats.getDataType();
    if (dataType == DataTypes.BOOLEAN) {
      return new DirectCompressCodec(columnPage.getDataType()).createEncoder(null);
    } else if (dataType == DataTypes.BYTE ||
        dataType == DataTypes.SHORT ||
        dataType == DataTypes.INT ||
        dataType == DataTypes.LONG ||
        dataType == DataTypes.TIMESTAMP) {
      return selectCodecByAlgorithmForIntegral(stats, false, columnSpec).createEncoder(null);
    } else if (DataTypes.isDecimal(dataType)) {
      return createEncoderForDecimalDataTypeMeasure(columnPage, columnSpec);
    } else if (dataType == DataTypes.FLOAT || dataType == DataTypes.DOUBLE) {
      return selectCodecByAlgorithmForFloating(stats, false, columnSpec).createEncoder(null);
    } else if (dataType == DataTypes.BYTE_ARRAY) {
      return new DirectCompressCodec(columnPage.getDataType()).createEncoder(null);
    } else {
      throw new RuntimeException("unsupported data type: " + stats.getDataType());
    }
  }

  private ColumnPageEncoder createEncoderForDecimalDataTypeMeasure(ColumnPage columnPage,
      TableSpec.ColumnSpec columnSpec) {
    DecimalConverterFactory.DecimalConverterType decimalConverterType =
        ((DecimalColumnPage) columnPage).getDecimalConverter().getDecimalConverterType();
    switch (decimalConverterType) {
      case DECIMAL_INT:
      case DECIMAL_LONG:
        return selectCodecByAlgorithmForDecimal(columnPage.getStatistics(), decimalConverterType,
            columnSpec)
            .createEncoder(null);
      default:
        return new DirectCompressCodec(columnPage.getDataType()).createEncoder(null);
    }
  }

  private static DataType fitLongMinMax(long max, long min) {
    if (max <= Byte.MAX_VALUE && min >= Byte.MIN_VALUE) {
      return DataTypes.BYTE;
    } else if (max <= Short.MAX_VALUE && min >= Short.MIN_VALUE) {
      return DataTypes.SHORT;
    } else if (max <= THREE_BYTES_MAX && min >= THREE_BYTES_MIN) {
      return DataTypes.SHORT_INT;
    } else if (max <= Integer.MAX_VALUE && min >= Integer.MIN_VALUE) {
      return DataTypes.INT;
    } else {
      return DataTypes.LONG;
    }
  }

  private static DataType fitMinMax(DataType dataType, Object max, Object min) {
    if (dataType == DataTypes.BYTE || dataType == DataTypes.BOOLEAN) {
      return fitLongMinMax((byte) max, (byte) min);
    } else if (dataType == DataTypes.SHORT) {
      return fitLongMinMax((short) max, (short) min);
    } else if (dataType == DataTypes.INT) {
      return fitLongMinMax((int) max, (int) min);
    } else if ((dataType == DataTypes.LONG) || (dataType == DataTypes.TIMESTAMP)) {
      return fitLongMinMax((long) max, (long) min);
    } else if (dataType == DataTypes.DOUBLE) {
      return fitLongMinMax((long) (double) max, (long) (double) min);
    } else {
      throw new RuntimeException("internal error: " + dataType);
    }
  }

  private static DataType fitMinMaxForDecimalType(DataType dataType, Object max, Object min,
      DecimalConverterFactory.DecimalConverterType decimalConverterType) {
    long maxValue = ((BigDecimal) max).unscaledValue().longValue();
    long minValue = ((BigDecimal) min).unscaledValue().longValue();
    switch (decimalConverterType) {
      case DECIMAL_INT:
        return fitLongMinMax((int) maxValue, (int) minValue);
      case DECIMAL_LONG:
        return fitLongMinMax(maxValue, minValue);
      default:
        throw new RuntimeException("internal error: " + dataType);
    }
  }

  private static DataType fitDeltaForDecimalType(DataType dataType, Object max, Object min,
      DecimalConverterFactory.DecimalConverterType decimalConverterType) {
    long maxValue = ((BigDecimal) max).unscaledValue().longValue();
    long minValue = ((BigDecimal) min).unscaledValue().longValue();
    switch (decimalConverterType) {
      case DECIMAL_INT:
        long value = maxValue - minValue;
        return compareMinMaxAndSelectDataType(value);
      case DECIMAL_LONG:
        return DataTypes.LONG;
      default:
        throw new RuntimeException("internal error: " + dataType);
    }
  }

  // fit the long input value into minimum data type
  private static DataType fitDelta(DataType dataType, Object max, Object min) {
    // use long data type to calculate delta to avoid overflow
    long value;
    if (dataType == DataTypes.BYTE || dataType == DataTypes.BOOLEAN) {
      value = (long) (byte) max - (long) (byte) min;
    } else if (dataType == DataTypes.SHORT) {
      value = (long) (short) max - (long) (short) min;
    } else if (dataType == DataTypes.INT) {
      value = (long) (int) max - (long) (int) min;
    } else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
      value = (long) max - (long) min;
      // The subtraction overflowed iff the operands have opposing signs
      // and the result's sign differs from the minuend.
      boolean overflow = (((long) max ^ (long) min) & ((long) max ^ value)) < 0;
      if (overflow) {
        return DataTypes.LONG;
      }
    } else if (dataType == DataTypes.DOUBLE) {
      return DataTypes.LONG;
    } else {
      throw new RuntimeException("internal error: " + dataType);
    }
    return compareMinMaxAndSelectDataType(value);
  }

  private static DataType compareMinMaxAndSelectDataType(long value) {
    if (value <= Byte.MAX_VALUE && value >= Byte.MIN_VALUE) {
      return DataTypes.BYTE;
    } else if (value <= Short.MAX_VALUE && value >= Short.MIN_VALUE) {
      return DataTypes.SHORT;
    } else if (value <= THREE_BYTES_MAX && value >= THREE_BYTES_MIN) {
      return DataTypes.SHORT_INT;
    } else if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) {
      return DataTypes.INT;
    } else {
      return DataTypes.LONG;
    }
  }

  /**
   * choose between adaptive encoder or delta adaptive encoder, based on whose target data type
   * size is smaller
   */
  static ColumnPageCodec selectCodecByAlgorithmForIntegral(SimpleStatsResult stats,
      boolean isComplexPrimitive, TableSpec.ColumnSpec columnSpec) {
    DataType srcDataType = stats.getDataType();
    DataType adaptiveDataType = fitMinMax(stats.getDataType(), stats.getMax(), stats.getMin());

    DataType deltaDataType = fitDelta(stats.getDataType(), stats.getMax(), stats.getMin());
    // for complex primitive, if source and destination data type is same, use adaptive encoding.
    if (!isComplexPrimitive) {
      // in case of decimal datatype, check if the decimal converter type is Int or Long and based
      // on that get size in bytes
      if (Math.min(adaptiveDataType.getSizeInBytes(), deltaDataType.getSizeInBytes()) == srcDataType
          .getSizeInBytes()) {
        // no effect to use adaptive or delta, use compression only
        return new DirectCompressCodec(stats.getDataType());
      }
    }
    boolean isInvertedIndex = isInvertedIndex(isComplexPrimitive, columnSpec);
    if (adaptiveDataType.getSizeInBytes() <= deltaDataType.getSizeInBytes()) {
      // choose adaptive encoding
      return new AdaptiveIntegralCodec(stats.getDataType(), adaptiveDataType, stats,
          isInvertedIndex);
    } else {
      // choose delta adaptive encoding
      return new AdaptiveDeltaIntegralCodec(stats.getDataType(), deltaDataType, stats,
          isInvertedIndex);
    }
  }

  /**
   * Check whether the column is sort column and inverted index column
   *
   * @param isComplexPrimitive
   * @param columnSpec
   * @return
   */
  private static boolean isInvertedIndex(boolean isComplexPrimitive,
      TableSpec.ColumnSpec columnSpec) {
    boolean isSort;
    boolean isInvertedIndex = false;
    if (columnSpec instanceof TableSpec.DimensionSpec && !isComplexPrimitive) {
      isSort = ((TableSpec.DimensionSpec) columnSpec).isInSortColumns();
      isInvertedIndex = isSort && ((TableSpec.DimensionSpec) columnSpec).isDoInvertedIndex();
    }
    return isInvertedIndex;
  }

  // choose between upscale adaptive encoder or upscale delta adaptive encoder,
  // based on whose target data type size is smaller
  static ColumnPageCodec selectCodecByAlgorithmForFloating(SimpleStatsResult stats,
      boolean isComplexPrimitive, TableSpec.ColumnSpec columnSpec) {
    DataType srcDataType = stats.getDataType();
    double maxValue;
    double minValue;
    if (srcDataType == DataTypes.FLOAT) {
      maxValue = (float) stats.getMax();
      minValue = (float) stats.getMin();
    } else {
      maxValue = (double) stats.getMax();
      minValue = (double) stats.getMin();
    }
    int decimalCount = stats.getDecimalCount();

    // For Complex Type primitive we should always choose adaptive path
    // as LV format will be reduced to only V format. Therefore inorder
    // to do that decimal count should be actual count instead of -1.
    if (isComplexPrimitive && decimalCount == -1 && stats instanceof PrimitivePageStatsCollector) {
      decimalCount = ((PrimitivePageStatsCollector)stats).getDecimalForComplexPrimitive();
    }

    //Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
    //but we can't use -1 to getDatatype, we should use -10000000.
    double absMaxValue = Math.max(Math.abs(maxValue), Math.abs(minValue));
    if (decimalCount == 0) {
      // short, int, long
      return selectCodecByAlgorithmForIntegral(stats, false, columnSpec);
    } else if (decimalCount < 0 && !isComplexPrimitive) {
      return new DirectCompressCodec(DataTypes.DOUBLE);
    } else {
      // double
      // If absMaxValue exceeds LONG.MAX_VALUE, then go for direct compression
      if ((Math.pow(10, decimalCount) * absMaxValue) > Long.MAX_VALUE) {
        return new DirectCompressCodec(DataTypes.DOUBLE);
      } else {
        long max = (long) (Math.pow(10, decimalCount) * absMaxValue);
        DataType adaptiveDataType = fitLongMinMax(max, 0);
        DataType deltaDataType = compareMinMaxAndSelectDataType(
            (long) (Math.pow(10, decimalCount) * (maxValue - minValue)));
        if (adaptiveDataType.getSizeInBytes() > deltaDataType.getSizeInBytes()) {
          return new AdaptiveDeltaFloatingCodec(srcDataType, deltaDataType, stats,
              isInvertedIndex(isComplexPrimitive, columnSpec));
        } else if (adaptiveDataType.getSizeInBytes() < DataTypes.DOUBLE.getSizeInBytes() || (
            (isComplexPrimitive) && (adaptiveDataType.getSizeInBytes() == DataTypes.DOUBLE
                .getSizeInBytes()))) {
          return new AdaptiveFloatingCodec(srcDataType, adaptiveDataType, stats,
              isInvertedIndex(isComplexPrimitive, columnSpec));
        } else {
          return new DirectCompressCodec(DataTypes.DOUBLE);
        }
      }
    }
  }

  /**
   * choose between adaptive encoder or delta adaptive encoder, based on whose target data type
   * size is smaller for decimal data type
   */
  static ColumnPageCodec selectCodecByAlgorithmForDecimal(SimpleStatsResult stats,
      DecimalConverterFactory.DecimalConverterType decimalConverterType,
      TableSpec.ColumnSpec columnSpec) {
    DataType srcDataType = stats.getDataType();
    DataType adaptiveDataType =
        fitMinMaxForDecimalType(stats.getDataType(), stats.getMax(), stats.getMin(),
            decimalConverterType);
    DataType deltaDataType;

    if (adaptiveDataType == DataTypes.LONG) {
      deltaDataType = DataTypes.LONG;
    } else {
      deltaDataType = fitDeltaForDecimalType(stats.getDataType(), stats.getMax(), stats.getMin(),
          decimalConverterType);
    }
    // in case of decimal data type check if the decimal converter type is Int or Long and based on
    // that get size in bytes
    if (Math.min(adaptiveDataType.getSizeInBytes(), deltaDataType.getSizeInBytes()) == srcDataType
        .getSizeInBytes()) {
      // no effect to use adaptive or delta, use compression only
      return new DirectCompressCodec(stats.getDataType());
    }
    if (adaptiveDataType.getSizeInBytes() <= deltaDataType.getSizeInBytes()) {
      // choose adaptive encoding
      return new AdaptiveIntegralCodec(stats.getDataType(), adaptiveDataType, stats,
          isInvertedIndex(columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE, columnSpec));
    } else {
      // choose delta adaptive encoding
      return new AdaptiveDeltaIntegralCodec(stats.getDataType(), deltaDataType, stats,
          isInvertedIndex(columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE, columnSpec));
    }
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.datastore.page.encoding;
19
20	import java.math.BigDecimal;
21
22	import org.apache.carbondata.core.datastore.ColumnType;
23	import org.apache.carbondata.core.datastore.TableSpec;
24	import org.apache.carbondata.core.datastore.page.ColumnPage;
25	import org.apache.carbondata.core.datastore.page.DecimalColumnPage;
26	import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaFloatingCodec;
27	import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaIntegralCodec;
28	import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveFloatingCodec;
29	import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveIntegralCodec;
30	import org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec;
31	import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.ComplexDimensionIndexCodec;
32	import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DictDimensionIndexCodec;
33	import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DirectDictDimensionIndexCodec;
34	import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.HighCardDictDimensionIndexCodec;
35	import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
36	import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
37	import org.apache.carbondata.core.metadata.datatype.DataType;
38	import org.apache.carbondata.core.metadata.datatype.DataTypes;
39	import org.apache.carbondata.core.metadata.datatype.DecimalConverterFactory;
40	import org.apache.carbondata.core.util.DataTypeUtil;
41
42	/**
43	* Default factory will select encoding base on column page data type and statistics
44	*/
45	public class DefaultEncodingFactory extends EncodingFactory {	2✔
46
47	private static final int THREE_BYTES_MAX = (int) Math.pow(2, 23) - 1;	2✔
48	private static final int THREE_BYTES_MIN = - THREE_BYTES_MAX - 1;	2✔
49
50	private static final boolean newWay = false;
51
52	private static EncodingFactory encodingFactory = new DefaultEncodingFactory();	2✔
53
54	public static EncodingFactory getInstance() {
55	// TODO: make it configurable after added new encodingFactory
56	return encodingFactory;	2✔
57	}
58
59	@Override
60	public ColumnPageEncoder createEncoder(TableSpec.ColumnSpec columnSpec, ColumnPage inputPage) {
61	// TODO: add log
62	// choose the encoding type for measure type and no dictionary primitive type columns
63	if (columnSpec instanceof TableSpec.MeasureSpec \|\| (	1✔
64	DataTypeUtil.isPrimitiveColumn(columnSpec.getSchemaDataType())	1✔
65	&& columnSpec.getColumnType() == ColumnType.PLAIN_VALUE)) {	1✔
66	return createEncoderForMeasureOrNoDictionaryPrimitive(inputPage, columnSpec);	1✔
67	} else {
68	if (newWay) {
69	return createEncoderForDimension((TableSpec.DimensionSpec) columnSpec, inputPage);
70	} else {
71	assert columnSpec instanceof TableSpec.DimensionSpec;	1✔
72	return createEncoderForDimensionLegacy((TableSpec.DimensionSpec) columnSpec);	1✔
73	}
74	}
75	}
76
77	private ColumnPageEncoder createEncoderForDimension(TableSpec.DimensionSpec columnSpec,
78	ColumnPage inputPage) {
79	switch (columnSpec.getColumnType()) {	1✔
80	case GLOBAL_DICTIONARY:
81	case DIRECT_DICTIONARY:
82	case PLAIN_VALUE:
83	return new DirectCompressCodec(inputPage.getDataType()).createEncoder(null);	×
84	case COMPLEX:
85	return new ComplexDimensionIndexCodec(false, false).createEncoder(null);	×
86	default:
87	throw new RuntimeException("unsupported dimension type: " + columnSpec.getColumnType());	×
88	}
89	}
90
91	private ColumnPageEncoder createEncoderForDimensionLegacy(TableSpec.DimensionSpec dimensionSpec) {
92	switch (dimensionSpec.getColumnType()) {	1✔
93	case GLOBAL_DICTIONARY:
94	return new DictDimensionIndexCodec(	1✔
95	dimensionSpec.isInSortColumns(),	1✔
96	dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex())	1✔
97	.createEncoder(null);	1✔
98	case DIRECT_DICTIONARY:
99	return new DirectDictDimensionIndexCodec(	1✔
100	dimensionSpec.isInSortColumns(),	1✔
101	dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex())	1✔
102	.createEncoder(null);	1✔
103	case PLAIN_VALUE:
104	return new HighCardDictDimensionIndexCodec(	1✔
105	dimensionSpec.isInSortColumns(),	1✔
106	dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex(),	1✔
107	dimensionSpec.getSchemaDataType() == DataTypes.VARCHAR)	1✔
108	.createEncoder(null);	1✔
109	default:
110	throw new RuntimeException("unsupported dimension type: " +	×
111	dimensionSpec.getColumnType());	×
112	}
113	}
114
115	private ColumnPageEncoder createEncoderForMeasureOrNoDictionaryPrimitive(ColumnPage columnPage,
116	TableSpec.ColumnSpec columnSpec) {
117	SimpleStatsResult stats = columnPage.getStatistics();	1✔
118	DataType dataType = stats.getDataType();	1✔
119	if (dataType == DataTypes.BOOLEAN) {	1✔
120	return new DirectCompressCodec(columnPage.getDataType()).createEncoder(null);	1✔
121	} else if (dataType == DataTypes.BYTE \|\|	1✔
122	dataType == DataTypes.SHORT \|\|
123	dataType == DataTypes.INT \|\|
124	dataType == DataTypes.LONG \|\|
125	dataType == DataTypes.TIMESTAMP) {
126	return selectCodecByAlgorithmForIntegral(stats, false, columnSpec).createEncoder(null);	1✔
127	} else if (DataTypes.isDecimal(dataType)) {	1✔
128	return createEncoderForDecimalDataTypeMeasure(columnPage, columnSpec);	1✔
129	} else if (dataType == DataTypes.FLOAT \|\| dataType == DataTypes.DOUBLE) {	1✔
130	return selectCodecByAlgorithmForFloating(stats, false, columnSpec).createEncoder(null);	1✔
131	} else if (dataType == DataTypes.BYTE_ARRAY) {	×
132	return new DirectCompressCodec(columnPage.getDataType()).createEncoder(null);	×
133	} else {
134	throw new RuntimeException("unsupported data type: " + stats.getDataType());	×
135	}
136	}
137
138	private ColumnPageEncoder createEncoderForDecimalDataTypeMeasure(ColumnPage columnPage,
139	TableSpec.ColumnSpec columnSpec) {
140	DecimalConverterFactory.DecimalConverterType decimalConverterType =	1✔
141	((DecimalColumnPage) columnPage).getDecimalConverter().getDecimalConverterType();	1✔
142	switch (decimalConverterType) {	1✔
143	case DECIMAL_INT:
144	case DECIMAL_LONG:
145	return selectCodecByAlgorithmForDecimal(columnPage.getStatistics(), decimalConverterType,	1✔
146	columnSpec)
147	.createEncoder(null);	1✔
148	default:
149	return new DirectCompressCodec(columnPage.getDataType()).createEncoder(null);	1✔
150	}
151	}
152
153	private static DataType fitLongMinMax(long max, long min) {
154	if (max <= Byte.MAX_VALUE && min >= Byte.MIN_VALUE) {	2✔
155	return DataTypes.BYTE;	2✔
156	} else if (max <= Short.MAX_VALUE && min >= Short.MIN_VALUE) {	2✔
157	return DataTypes.SHORT;	2✔
158	} else if (max <= THREE_BYTES_MAX && min >= THREE_BYTES_MIN) {	2✔
159	return DataTypes.SHORT_INT;	2✔
160	} else if (max <= Integer.MAX_VALUE && min >= Integer.MIN_VALUE) {	2✔
161	return DataTypes.INT;	2✔
162	} else {
163	return DataTypes.LONG;	2✔
164	}
165	}
166
167	private static DataType fitMinMax(DataType dataType, Object max, Object min) {
168	if (dataType == DataTypes.BYTE \|\| dataType == DataTypes.BOOLEAN) {	2✔
169	return fitLongMinMax((byte) max, (byte) min);	1✔
170	} else if (dataType == DataTypes.SHORT) {	2✔
171	return fitLongMinMax((short) max, (short) min);	1✔
172	} else if (dataType == DataTypes.INT) {	2✔
173	return fitLongMinMax((int) max, (int) min);	1✔
174	} else if ((dataType == DataTypes.LONG) \|\| (dataType == DataTypes.TIMESTAMP)) {	2✔
175	return fitLongMinMax((long) max, (long) min);	2✔
176	} else if (dataType == DataTypes.DOUBLE) {	1✔
177	return fitLongMinMax((long) (double) max, (long) (double) min);	1✔
178	} else {
179	throw new RuntimeException("internal error: " + dataType);	×
180	}
181	}
182
183	private static DataType fitMinMaxForDecimalType(DataType dataType, Object max, Object min,
184	DecimalConverterFactory.DecimalConverterType decimalConverterType) {
185	long maxValue = ((BigDecimal) max).unscaledValue().longValue();	1✔
186	long minValue = ((BigDecimal) min).unscaledValue().longValue();	1✔
187	switch (decimalConverterType) {	1✔
188	case DECIMAL_INT:
189	return fitLongMinMax((int) maxValue, (int) minValue);	1✔
190	case DECIMAL_LONG:
191	return fitLongMinMax(maxValue, minValue);	1✔
192	default:
193	throw new RuntimeException("internal error: " + dataType);	×
194	}
195	}
196
197	private static DataType fitDeltaForDecimalType(DataType dataType, Object max, Object min,
198	DecimalConverterFactory.DecimalConverterType decimalConverterType) {
199	long maxValue = ((BigDecimal) max).unscaledValue().longValue();	1✔
200	long minValue = ((BigDecimal) min).unscaledValue().longValue();	1✔
201	switch (decimalConverterType) {	1✔
202	case DECIMAL_INT:
203	long value = maxValue - minValue;	1✔
204	return compareMinMaxAndSelectDataType(value);	1✔
205	case DECIMAL_LONG:
206	return DataTypes.LONG;	1✔
207	default:
208	throw new RuntimeException("internal error: " + dataType);	×
209	}
210	}
211
212	// fit the long input value into minimum data type
213	private static DataType fitDelta(DataType dataType, Object max, Object min) {
214	// use long data type to calculate delta to avoid overflow
215	long value;
216	if (dataType == DataTypes.BYTE \|\| dataType == DataTypes.BOOLEAN) {	2✔
217	value = (long) (byte) max - (long) (byte) min;	1✔
218	} else if (dataType == DataTypes.SHORT) {	2✔
219	value = (long) (short) max - (long) (short) min;	1✔
220	} else if (dataType == DataTypes.INT) {	2✔
221	value = (long) (int) max - (long) (int) min;	1✔
222	} else if (dataType == DataTypes.LONG \|\| dataType == DataTypes.TIMESTAMP) {	2✔
223	value = (long) max - (long) min;	2✔
224	// The subtraction overflowed iff the operands have opposing signs
225	// and the result's sign differs from the minuend.
226	boolean overflow = (((long) max ^ (long) min) & ((long) max ^ value)) < 0;	2✔
227	if (overflow) {	2✔
228	return DataTypes.LONG;	2✔
229	}
230	} else if (dataType == DataTypes.DOUBLE) {	2✔
231	return DataTypes.LONG;	1✔
232	} else {
233	throw new RuntimeException("internal error: " + dataType);	×
234	}
235	return compareMinMaxAndSelectDataType(value);	2✔
236	}
237
238	private static DataType compareMinMaxAndSelectDataType(long value) {
239	if (value <= Byte.MAX_VALUE && value >= Byte.MIN_VALUE) {	2✔
240	return DataTypes.BYTE;	2✔
241	} else if (value <= Short.MAX_VALUE && value >= Short.MIN_VALUE) {	2✔
242	return DataTypes.SHORT;	2✔
243	} else if (value <= THREE_BYTES_MAX && value >= THREE_BYTES_MIN) {	2✔
244	return DataTypes.SHORT_INT;	2✔
245	} else if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) {	2✔
246	return DataTypes.INT;	2✔
247	} else {
248	return DataTypes.LONG;	2✔
249	}
250	}
251
252	/**
253	* choose between adaptive encoder or delta adaptive encoder, based on whose target data type
254	* size is smaller
255	*/
256	static ColumnPageCodec selectCodecByAlgorithmForIntegral(SimpleStatsResult stats,
257	boolean isComplexPrimitive, TableSpec.ColumnSpec columnSpec) {
258	DataType srcDataType = stats.getDataType();	2✔
259	DataType adaptiveDataType = fitMinMax(stats.getDataType(), stats.getMax(), stats.getMin());	2✔
260
261	DataType deltaDataType = fitDelta(stats.getDataType(), stats.getMax(), stats.getMin());	2✔
262	// for complex primitive, if source and destination data type is same, use adaptive encoding.
263	if (!isComplexPrimitive) {	2✔
264	// in case of decimal datatype, check if the decimal converter type is Int or Long and based
265	// on that get size in bytes
266	if (Math.min(adaptiveDataType.getSizeInBytes(), deltaDataType.getSizeInBytes()) == srcDataType	2✔
267	.getSizeInBytes()) {	2✔
268	// no effect to use adaptive or delta, use compression only
269	return new DirectCompressCodec(stats.getDataType());	2✔
270	}
271	}
272	boolean isInvertedIndex = isInvertedIndex(isComplexPrimitive, columnSpec);	2✔
273	if (adaptiveDataType.getSizeInBytes() <= deltaDataType.getSizeInBytes()) {	2✔
274	// choose adaptive encoding
275	return new AdaptiveIntegralCodec(stats.getDataType(), adaptiveDataType, stats,	2✔
276	isInvertedIndex);
277	} else {
278	// choose delta adaptive encoding
279	return new AdaptiveDeltaIntegralCodec(stats.getDataType(), deltaDataType, stats,	2✔
280	isInvertedIndex);
281	}
282	}
283
284	/**
285	* Check whether the column is sort column and inverted index column
286	*
287	* @param isComplexPrimitive
288	* @param columnSpec
289	* @return
290	*/
291	private static boolean isInvertedIndex(boolean isComplexPrimitive,
292	TableSpec.ColumnSpec columnSpec) {
293	boolean isSort;
294	boolean isInvertedIndex = false;	2✔
295	if (columnSpec instanceof TableSpec.DimensionSpec && !isComplexPrimitive) {	2✔
296	isSort = ((TableSpec.DimensionSpec) columnSpec).isInSortColumns();	1✔
297	isInvertedIndex = isSort && ((TableSpec.DimensionSpec) columnSpec).isDoInvertedIndex();	1✔
298	}
299	return isInvertedIndex;	2✔
300	}
301
302	// choose between upscale adaptive encoder or upscale delta adaptive encoder,
303	// based on whose target data type size is smaller
304	static ColumnPageCodec selectCodecByAlgorithmForFloating(SimpleStatsResult stats,
305	boolean isComplexPrimitive, TableSpec.ColumnSpec columnSpec) {
306	DataType srcDataType = stats.getDataType();	1✔
307	double maxValue;
308	double minValue;
309	if (srcDataType == DataTypes.FLOAT) {	1✔
310	maxValue = (float) stats.getMax();	1✔
311	minValue = (float) stats.getMin();	1✔
312	} else {
313	maxValue = (double) stats.getMax();	1✔
314	minValue = (double) stats.getMin();	1✔
315	}
316	int decimalCount = stats.getDecimalCount();	1✔
317
318	// For Complex Type primitive we should always choose adaptive path
319	// as LV format will be reduced to only V format. Therefore inorder
320	// to do that decimal count should be actual count instead of -1.
321	if (isComplexPrimitive && decimalCount == -1 && stats instanceof PrimitivePageStatsCollector) {	1✔
322	decimalCount = ((PrimitivePageStatsCollector)stats).getDecimalForComplexPrimitive();	1✔
323	}
324
325	//Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
326	//but we can't use -1 to getDatatype, we should use -10000000.
327	double absMaxValue = Math.max(Math.abs(maxValue), Math.abs(minValue));	1✔
328	if (decimalCount == 0) {	1✔
329	// short, int, long
330	return selectCodecByAlgorithmForIntegral(stats, false, columnSpec);	1✔
331	} else if (decimalCount < 0 && !isComplexPrimitive) {	1✔
332	return new DirectCompressCodec(DataTypes.DOUBLE);	1✔
333	} else {
334	// double
335	// If absMaxValue exceeds LONG.MAX_VALUE, then go for direct compression
336	if ((Math.pow(10, decimalCount) * absMaxValue) > Long.MAX_VALUE) {	1✔
337	return new DirectCompressCodec(DataTypes.DOUBLE);	1✔
338	} else {
339	long max = (long) (Math.pow(10, decimalCount) * absMaxValue);	1✔
340	DataType adaptiveDataType = fitLongMinMax(max, 0);	1✔
341	DataType deltaDataType = compareMinMaxAndSelectDataType(	1✔
342	(long) (Math.pow(10, decimalCount) * (maxValue - minValue)));	1✔
343	if (adaptiveDataType.getSizeInBytes() > deltaDataType.getSizeInBytes()) {	1✔
344	return new AdaptiveDeltaFloatingCodec(srcDataType, deltaDataType, stats,	1✔
345	isInvertedIndex(isComplexPrimitive, columnSpec));	1✔
346	} else if (adaptiveDataType.getSizeInBytes() < DataTypes.DOUBLE.getSizeInBytes() \|\| (	1✔
347	(isComplexPrimitive) && (adaptiveDataType.getSizeInBytes() == DataTypes.DOUBLE	1✔
348	.getSizeInBytes()))) {	1✔
349	return new AdaptiveFloatingCodec(srcDataType, adaptiveDataType, stats,	1✔
350	isInvertedIndex(isComplexPrimitive, columnSpec));	1✔
351	} else {
352	return new DirectCompressCodec(DataTypes.DOUBLE);	1✔
353	}
354	}
355	}
356	}
357
358	/**
359	* choose between adaptive encoder or delta adaptive encoder, based on whose target data type
360	* size is smaller for decimal data type
361	*/
362	static ColumnPageCodec selectCodecByAlgorithmForDecimal(SimpleStatsResult stats,
363	DecimalConverterFactory.DecimalConverterType decimalConverterType,
364	TableSpec.ColumnSpec columnSpec) {
365	DataType srcDataType = stats.getDataType();	1✔
366	DataType adaptiveDataType =	1✔
367	fitMinMaxForDecimalType(stats.getDataType(), stats.getMax(), stats.getMin(),	1✔
368	decimalConverterType);
369	DataType deltaDataType;
370
371	if (adaptiveDataType == DataTypes.LONG) {	1✔
372	deltaDataType = DataTypes.LONG;	1✔
373	} else {
374	deltaDataType = fitDeltaForDecimalType(stats.getDataType(), stats.getMax(), stats.getMin(),	1✔
375	decimalConverterType);
376	}
377	// in case of decimal data type check if the decimal converter type is Int or Long and based on
378	// that get size in bytes
379	if (Math.min(adaptiveDataType.getSizeInBytes(), deltaDataType.getSizeInBytes()) == srcDataType	1✔
380	.getSizeInBytes()) {	1✔
381	// no effect to use adaptive or delta, use compression only
382	return new DirectCompressCodec(stats.getDataType());	×
383	}
384	if (adaptiveDataType.getSizeInBytes() <= deltaDataType.getSizeInBytes()) {	1✔
385	// choose adaptive encoding
386	return new AdaptiveIntegralCodec(stats.getDataType(), adaptiveDataType, stats,	1✔
387	isInvertedIndex(columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE, columnSpec));	1✔
388	} else {
389	// choose delta adaptive encoding
390	return new AdaptiveDeltaIntegralCodec(stats.getDataType(), deltaDataType, stats,	1✔
391	isInvertedIndex(columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE, columnSpec));	1✔
392	}
393	}
394
395	}

apache / carbondata / 710

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous