8385

Build # 8385

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/f4721e8b0bb39f275a7a4b3f1c85dde2e5554f64">f4721e8b0<a href="https://github.com/apache/carbondata/commit/f4721e8b0bb39f275a7a4b3f1c85dde2e5554f64">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/18ccca494724c97d0a40ee3532a88ff3a69e1c59">18ccca494</a>">18ccca494</a><a href="https://github.com/apache/carbondata/commit/f4721e8b0bb39f275a7a4b3f1c85dde2e5554f64"> into b6bd90d80">b6bd90d80</a>

Run Details

53169 of 71816 relevant lines covered (74.04%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.71

/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.util;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.Map;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.BlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockIndex;

/**
 * Footer reader class
 */
public abstract class AbstractDataFileFooterConverter {

  /**
   * Below method will be used to convert the thrift presence meta to wrapper
   * presence meta
   *
   * @param presentMetadataThrift
   * @return wrapper presence meta
   */
  private static BitSet getPresenceMeta(
      org.apache.carbondata.format.PresenceMeta presentMetadataThrift) {
    final byte[] present_bit_stream = presentMetadataThrift.getPresent_bit_stream();
    if (null != present_bit_stream) {
      return BitSet.valueOf(present_bit_stream);
    } else {
      return new BitSet(1);
    }
  }

  /**
   * Below method will be used to get the index info from index file
   *
   * @param filePath           file path of the index file
   * @param tableBlockInfoList table block index
   * @return list of index info
   * @throws IOException problem while reading the index file
   */
  public List<DataFileFooter> getIndexInfo(String filePath, List<TableBlockInfo> tableBlockInfoList)
      throws IOException {
    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
    List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
    try {
      // open the reader
      indexReader.openThriftReader(filePath);
      // get the index header
      org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
      List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
      List<org.apache.carbondata.format.ColumnSchema> table_columns =
          readIndexHeader.getTable_columns();
      for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i)));
      }
      // get the segment info
      SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());
      BlockletIndex blockletIndex = null;
      int counter = 0;
      int index = 0;
      DataFileFooter dataFileFooter = null;
      // read the block info from file
      while (indexReader.hasNext()) {
        BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();
        blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());
        dataFileFooter = new DataFileFooter();
        TableBlockInfo tableBlockInfo = tableBlockInfoList.get(index);
        if (Integer.parseInt(CarbonTablePath.DataFileUtil.getPartNo(
            tableBlockInfo.getFilePath())) == counter++) {
          tableBlockInfo.setBlockOffset(readBlockIndexInfo.getOffset());
          tableBlockInfo.setVersion(
              ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion()));
          int blockletSize = getBlockletSize(readBlockIndexInfo);
          tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize);
          dataFileFooter.setBlockletIndex(blockletIndex);
          dataFileFooter.setColumnInTable(columnSchemaList);
          dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());
          dataFileFooter.setBlockInfo(new BlockInfo(tableBlockInfo));
          dataFileFooter.setSegmentInfo(segmentInfo);
          dataFileFooters.add(dataFileFooter);
          if (++index == tableBlockInfoList.size()) {
            break;
          }
        }
      }
    } finally {
      indexReader.closeThriftReader();
    }
    return dataFileFooters;
  }

  /**
   * Below method will be used to get the index info from index file
   *
   * @param filePath           file path of the index file
   * @return list of index info
   * @throws IOException problem while reading the index file
   */
  public List<DataFileFooter> getIndexInfo(String filePath, byte[] fileData) throws IOException {
    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
    List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
    String parentPath = filePath.substring(0, filePath.lastIndexOf("/"));
    try {
      // open the reader
      if (fileData != null) {
        indexReader.openThriftReader(fileData);
      } else {
        indexReader.openThriftReader(filePath);
      }
      // get the index header
      org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
      List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
      List<org.apache.carbondata.format.ColumnSchema> table_columns =
          readIndexHeader.getTable_columns();
      for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i)));
      }
      // get the segment info
      SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());
      BlockletIndex blockletIndex = null;
      DataFileFooter dataFileFooter = null;
      // read the block info from file
      while (indexReader.hasNext()) {
        BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();
        blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());
        dataFileFooter = new DataFileFooter();
        TableBlockInfo tableBlockInfo =
            getTableBlockInfo(readBlockIndexInfo, readIndexHeader, parentPath);
        dataFileFooter.setBlockletIndex(blockletIndex);
        dataFileFooter.setColumnInTable(columnSchemaList);
        dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());
        dataFileFooter.setBlockInfo(new BlockInfo(tableBlockInfo));
        dataFileFooter.setSegmentInfo(segmentInfo);
        dataFileFooter.setVersionId(tableBlockInfo.getVersion());
        // In case of old schema time stamp will not be found in the index header
        if (readIndexHeader.isSetSchema_time_stamp()) {
          dataFileFooter.setSchemaUpdatedTimeStamp(readIndexHeader.getSchema_time_stamp());
        }
        if (readBlockIndexInfo.isSetBlocklet_info()) {
          List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
          BlockletInfo blockletInfo = new DataFileFooterConverterV3()
              .getBlockletInfo(readBlockIndexInfo.getBlocklet_info(),
                  CarbonUtil.getNumberOfDimensionColumns(columnSchemaList));
          blockletInfo.setBlockletIndex(blockletIndex);
          blockletInfoList.add(blockletInfo);
          dataFileFooter.setBlockletList(blockletInfoList);
        }
        dataFileFooters.add(dataFileFooter);
      }
    } finally {
      indexReader.closeThriftReader();
    }
    return dataFileFooters;
  }

  /**
   * This method will create a table block info object from index file info
   *
   * @param readBlockIndexInfo
   * @param readIndexHeader
   * @param parentPath
   * @return
   */
  public TableBlockInfo getTableBlockInfo(BlockIndex readBlockIndexInfo,
      org.apache.carbondata.format.IndexHeader readIndexHeader, String parentPath) {
    TableBlockInfo tableBlockInfo = new TableBlockInfo();
    tableBlockInfo.setBlockOffset(readBlockIndexInfo.getOffset());
    ColumnarFormatVersion version =
        ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion());
    tableBlockInfo.setVersion(version);
    int blockletSize = getBlockletSize(readBlockIndexInfo);
    tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize);
    String fileName = readBlockIndexInfo.file_name;
    // Take only name of file.
    if (fileName.lastIndexOf("/") > 0) {
      fileName = fileName.substring(fileName.lastIndexOf("/"));
    }
    fileName = (CarbonCommonConstants.FILE_SEPARATOR + fileName).replaceAll("//", "/");
    tableBlockInfo.setFilePath(parentPath + fileName);
    return tableBlockInfo;
  }

  /**
   * the methods returns the number of blocklets in a block
   *
   * @param readBlockIndexInfo
   * @return
   */
  protected int getBlockletSize(BlockIndex readBlockIndexInfo) {
    long num_rows = readBlockIndexInfo.getNum_rows();
    int blockletSize = Integer.parseInt(CarbonProperties.getInstance()
        .getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
            CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
    int remainder = (int) (num_rows % blockletSize);
    int noOfBlockLet = (int) (num_rows / blockletSize);
    // there could be some blocklets which will not
    // contain the total records equal to the blockletSize
    if (remainder > 0) {
      noOfBlockLet = noOfBlockLet + 1;
    }
    return noOfBlockLet;
  }

  /**
   * Below method will be used to convert thrift file meta to wrapper file meta
   */
  public abstract DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo)
      throws IOException;

  public abstract List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException;

  /**
   * Below method will be used to get blocklet index for data file meta
   *
   * @param blockletIndexList
   * @return blocklet index
   */
  protected BlockletIndex getBlockletIndexForDataFileFooter(List<BlockletIndex> blockletIndexList) {
    BlockletIndex blockletIndex = new BlockletIndex();
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey());
    blockletBTreeIndex
        .setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey());
    blockletIndex.setBtreeIndex(blockletBTreeIndex);
    byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone();
    byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone();
    byte[][] minValue = null;
    byte[][] maxValue = null;
    for (int i = 1; i < blockletIndexList.size(); i++) {
      minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues();
      maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues();
      for (int j = 0; j < maxValue.length; j++) {
        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
          currentMinValue[j] = minValue[j].clone();
        }
        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
          currentMaxValue[j] = maxValue[j].clone();
        }
      }
    }

    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(currentMaxValue);
    minMax.setMinValues(currentMinValue);
    blockletIndex.setMinMaxIndex(minMax);
    return blockletIndex;
  }

  protected ColumnSchema thriftColumnSchemaToWrapperColumnSchema(
      org.apache.carbondata.format.ColumnSchema externalColumnSchema) {
    ColumnSchema wrapperColumnSchema = new ColumnSchema();
    wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
    wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
    DataType dataType = CarbonUtil.thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type);
    if (DataTypes.isDecimal(dataType)) {
      DecimalType decimalType = (DecimalType) dataType;
      decimalType.setPrecision(externalColumnSchema.getPrecision());
      decimalType.setScale(externalColumnSchema.getScale());
    }
    wrapperColumnSchema.setDataType(dataType);
    wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());
    List<Encoding> encoders = new ArrayList<Encoding>();
    for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {
      encoders.add(fromExternalToWrapperEncoding(encoder));
    }
    wrapperColumnSchema.setEncodingList(encoders);
    wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
    wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
    wrapperColumnSchema.setScale(externalColumnSchema.getScale());
    wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
    Map<String, String> properties = externalColumnSchema.getColumnProperties();
    if (properties != null) {
      if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {
        wrapperColumnSchema.setSortColumn(true);
      }
    }
    wrapperColumnSchema.setFunction(externalColumnSchema.getAggregate_function());
    List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation =
        externalColumnSchema.getParentColumnTableRelations();
    if (null != parentColumnTableRelation) {
      wrapperColumnSchema.setParentColumnTableRelations(
          fromThriftToWrapperParentTableColumnRelations(parentColumnTableRelation));
    }
    return wrapperColumnSchema;
  }

  private List<ParentColumnTableRelation> fromThriftToWrapperParentTableColumnRelations(
      List<org.apache.carbondata.format.ParentColumnTableRelation> thirftParentColumnRelation) {
    List<ParentColumnTableRelation> parentColumnTableRelationList = new ArrayList<>();
    for (org.apache.carbondata.format.ParentColumnTableRelation carbonTableRelation :
        thirftParentColumnRelation) {
      RelationIdentifier relationIdentifier =
          new RelationIdentifier(carbonTableRelation.getRelationIdentifier().getDatabaseName(),
              carbonTableRelation.getRelationIdentifier().getTableName(),
              carbonTableRelation.getRelationIdentifier().getTableId());
      ParentColumnTableRelation parentColumnTableRelation =
          new ParentColumnTableRelation(relationIdentifier, carbonTableRelation.getColumnId(),
              carbonTableRelation.getColumnName());
      parentColumnTableRelationList.add(parentColumnTableRelation);
    }
    return parentColumnTableRelationList;
  }



  /**
   * Below method is convert the thrift encoding to wrapper encoding
   *
   * @param encoderThrift thrift encoding
   * @return wrapper encoding
   */
  protected Encoding fromExternalToWrapperEncoding(
      org.apache.carbondata.format.Encoding encoderThrift) {
    switch (encoderThrift) {
      case DICTIONARY:
        return Encoding.DICTIONARY;
      case DELTA:
        return Encoding.DELTA;
      case RLE:
        return Encoding.RLE;
      case INVERTED_INDEX:
        return Encoding.INVERTED_INDEX;
      case BIT_PACKED:
        return Encoding.BIT_PACKED;
      case DIRECT_DICTIONARY:
        return Encoding.DIRECT_DICTIONARY;
      default:
        throw new IllegalArgumentException(encoderThrift.toString() + " is not supported");
    }
  }

  /**
   * Below method will be used to convert thrift segment object to wrapper
   * segment object
   *
   * @param segmentInfo thrift segment info object
   * @return wrapper segment info object
   */
  protected SegmentInfo getSegmentInfo(org.apache.carbondata.format.SegmentInfo segmentInfo) {
    SegmentInfo info = new SegmentInfo();
    int[] cardinality = new int[segmentInfo.getColumn_cardinalities().size()];
    for (int i = 0; i < cardinality.length; i++) {
      cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);
    }
    info.setColumnCardinality(cardinality);
    return info;
  }

  /**
   * Below method will be used to convert the blocklet index of thrift to
   * wrapper
   *
   * @param blockletIndexThrift
   * @return blocklet index wrapper
   */
  protected BlockletIndex getBlockletIndex(
      org.apache.carbondata.format.BlockletIndex blockletIndexThrift) {
    org.apache.carbondata.format.BlockletBTreeIndex btreeIndex =
        blockletIndexThrift.getB_tree_index();
    org.apache.carbondata.format.BlockletMinMaxIndex minMaxIndex =
        blockletIndexThrift.getMin_max_index();
    return new BlockletIndex(
        new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()),
        new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values()));
  }

  /**
   * Below method will be used to convert the thrift data chunk to wrapper
   * data chunk
   *
   * @param datachunkThrift
   * @return wrapper data chunk
   */
  protected DataChunk getDataChunk(org.apache.carbondata.format.DataChunk datachunkThrift,
      boolean isPresenceMetaPresent) {
    DataChunk dataChunk = new DataChunk();
    dataChunk.setDataPageLength(datachunkThrift.getData_page_length());
    dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset());
    if (isPresenceMetaPresent) {
      dataChunk.setNullValueIndexForColumn(getPresenceMeta(datachunkThrift.getPresence()));
    }
    dataChunk.setRlePageLength(datachunkThrift.getRle_page_length());
    dataChunk.setRlePageOffset(datachunkThrift.getRle_page_offset());
    dataChunk.setRowMajor(datachunkThrift.isRowMajor());
    dataChunk.setRowIdPageLength(datachunkThrift.getRowid_page_length());
    dataChunk.setRowIdPageOffset(datachunkThrift.getRowid_page_offset());
    List<Encoding> encodingList = new ArrayList<Encoding>(datachunkThrift.getEncoders().size());
    for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) {
      encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i)));
    }
    dataChunk.setEncodingList(encodingList);
    if (encodingList.contains(Encoding.DELTA)) {
      List<ByteBuffer> thriftEncoderMeta = datachunkThrift.getEncoder_meta();
      List<ValueEncoderMeta> encodeMetaList =
          new ArrayList<ValueEncoderMeta>(thriftEncoderMeta.size());
      for (int i = 0; i < thriftEncoderMeta.size(); i++) {
        encodeMetaList.add(CarbonUtil.deserializeEncoderMetaV2(thriftEncoderMeta.get(i).array()));
      }
      dataChunk.setValueEncoderMeta(encodeMetaList);
    }
    return dataChunk;
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.util;
18
19	import java.io.IOException;
20	import java.nio.ByteBuffer;
21	import java.util.ArrayList;
22	import java.util.BitSet;
23	import java.util.List;
24	import java.util.Map;
25
26	import org.apache.carbondata.core.constants.CarbonCommonConstants;
27	import org.apache.carbondata.core.datastore.block.BlockInfo;
28	import org.apache.carbondata.core.datastore.block.TableBlockInfo;
29	import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
30	import org.apache.carbondata.core.metadata.ValueEncoderMeta;
31	import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
32	import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
33	import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
34	import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
35	import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex;
36	import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
37	import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
38	import org.apache.carbondata.core.metadata.datatype.DataType;
39	import org.apache.carbondata.core.metadata.datatype.DataTypes;
40	import org.apache.carbondata.core.metadata.datatype.DecimalType;
41	import org.apache.carbondata.core.metadata.encoder.Encoding;
42	import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
43	import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
44	import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
45	import org.apache.carbondata.core.reader.CarbonIndexFileReader;
46	import org.apache.carbondata.core.util.path.CarbonTablePath;
47	import org.apache.carbondata.format.BlockIndex;
48
49	/**
50	* Footer reader class
51	*/
52	public abstract class AbstractDataFileFooterConverter {	2✔
53
54	/**
55	* Below method will be used to convert the thrift presence meta to wrapper
56	* presence meta
57	*
58	* @param presentMetadataThrift
59	* @return wrapper presence meta
60	*/
61	private static BitSet getPresenceMeta(
62	org.apache.carbondata.format.PresenceMeta presentMetadataThrift) {
63	final byte[] present_bit_stream = presentMetadataThrift.getPresent_bit_stream();	×
64	if (null != present_bit_stream) {	×
65	return BitSet.valueOf(present_bit_stream);	×
66	} else {
67	return new BitSet(1);	×
68	}
69	}
70
71	/**
72	* Below method will be used to get the index info from index file
73	*
74	* @param filePath file path of the index file
75	* @param tableBlockInfoList table block index
76	* @return list of index info
77	* @throws IOException problem while reading the index file
78	*/
79	public List<DataFileFooter> getIndexInfo(String filePath, List<TableBlockInfo> tableBlockInfoList)
80	throws IOException {
81	CarbonIndexFileReader indexReader = new CarbonIndexFileReader();	2✔
82	List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();	2✔
83	try {
84	// open the reader
85	indexReader.openThriftReader(filePath);	2✔
86	// get the index header
87	org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();	2✔
88	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	2✔
89	List<org.apache.carbondata.format.ColumnSchema> table_columns =	2✔
90	readIndexHeader.getTable_columns();	2✔
91	for (int i = 0; i < table_columns.size(); i++) {	2✔
92	columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i)));	2✔
93	}
94	// get the segment info
95	SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());	2✔
96	BlockletIndex blockletIndex = null;	2✔
97	int counter = 0;	2✔
98	int index = 0;	2✔
99	DataFileFooter dataFileFooter = null;	2✔
100	// read the block info from file
101	while (indexReader.hasNext()) {	2✔
102	BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();	2✔
103	blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());	2✔
104	dataFileFooter = new DataFileFooter();	2✔
105	TableBlockInfo tableBlockInfo = tableBlockInfoList.get(index);	2✔
106	if (Integer.parseInt(CarbonTablePath.DataFileUtil.getPartNo(	2✔
107	tableBlockInfo.getFilePath())) == counter++) {	2✔
108	tableBlockInfo.setBlockOffset(readBlockIndexInfo.getOffset());	2✔
109	tableBlockInfo.setVersion(	2✔
110	ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion()));	2✔
111	int blockletSize = getBlockletSize(readBlockIndexInfo);	2✔
112	tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize);	2✔
113	dataFileFooter.setBlockletIndex(blockletIndex);	2✔
114	dataFileFooter.setColumnInTable(columnSchemaList);	2✔
115	dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());	2✔
116	dataFileFooter.setBlockInfo(new BlockInfo(tableBlockInfo));	2✔
117	dataFileFooter.setSegmentInfo(segmentInfo);	2✔
118	dataFileFooters.add(dataFileFooter);	2✔
119	if (++index == tableBlockInfoList.size()) {	2✔
120	break;	2✔
121	}
122	}
123	}	×
124	} finally {
125	indexReader.closeThriftReader();	2✔
126	}	2✔
127	return dataFileFooters;	2✔
128	}
129
130	/**
131	* Below method will be used to get the index info from index file
132	*
133	* @param filePath file path of the index file
134	* @return list of index info
135	* @throws IOException problem while reading the index file
136	*/
137	public List<DataFileFooter> getIndexInfo(String filePath, byte[] fileData) throws IOException {
138	CarbonIndexFileReader indexReader = new CarbonIndexFileReader();	1✔
139	List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();	1✔
140	String parentPath = filePath.substring(0, filePath.lastIndexOf("/"));	1✔
141	try {
142	// open the reader
143	if (fileData != null) {	1✔
144	indexReader.openThriftReader(fileData);	1✔
145	} else {
146	indexReader.openThriftReader(filePath);	×
147	}
148	// get the index header
149	org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();	1✔
150	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	1✔
151	List<org.apache.carbondata.format.ColumnSchema> table_columns =	1✔
152	readIndexHeader.getTable_columns();	1✔
153	for (int i = 0; i < table_columns.size(); i++) {	1✔
154	columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(table_columns.get(i)));	1✔
155	}
156	// get the segment info
157	SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());	1✔
158	BlockletIndex blockletIndex = null;	1✔
159	DataFileFooter dataFileFooter = null;	1✔
160	// read the block info from file
161	while (indexReader.hasNext()) {	1✔
162	BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();	1✔
163	blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());	1✔
164	dataFileFooter = new DataFileFooter();	1✔
165	TableBlockInfo tableBlockInfo =	1✔
166	getTableBlockInfo(readBlockIndexInfo, readIndexHeader, parentPath);	1✔
167	dataFileFooter.setBlockletIndex(blockletIndex);	1✔
168	dataFileFooter.setColumnInTable(columnSchemaList);	1✔
169	dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());	1✔
170	dataFileFooter.setBlockInfo(new BlockInfo(tableBlockInfo));	1✔
171	dataFileFooter.setSegmentInfo(segmentInfo);	1✔
172	dataFileFooter.setVersionId(tableBlockInfo.getVersion());	1✔
173	// In case of old schema time stamp will not be found in the index header
174	if (readIndexHeader.isSetSchema_time_stamp()) {	1✔
175	dataFileFooter.setSchemaUpdatedTimeStamp(readIndexHeader.getSchema_time_stamp());	1✔
176	}
177	if (readBlockIndexInfo.isSetBlocklet_info()) {	1✔
178	List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();	1✔
179	BlockletInfo blockletInfo = new DataFileFooterConverterV3()	1✔
180	.getBlockletInfo(readBlockIndexInfo.getBlocklet_info(),	1✔
181	CarbonUtil.getNumberOfDimensionColumns(columnSchemaList));	1✔
182	blockletInfo.setBlockletIndex(blockletIndex);	1✔
183	blockletInfoList.add(blockletInfo);	1✔
184	dataFileFooter.setBlockletList(blockletInfoList);	1✔
185	}
186	dataFileFooters.add(dataFileFooter);	1✔
187	}	1✔
188	} finally {
189	indexReader.closeThriftReader();	1✔
190	}	1✔
191	return dataFileFooters;	1✔
192	}
193
194	/**
195	* This method will create a table block info object from index file info
196	*
197	* @param readBlockIndexInfo
198	* @param readIndexHeader
199	* @param parentPath
200	* @return
201	*/
202	public TableBlockInfo getTableBlockInfo(BlockIndex readBlockIndexInfo,
203	org.apache.carbondata.format.IndexHeader readIndexHeader, String parentPath) {
204	TableBlockInfo tableBlockInfo = new TableBlockInfo();	1✔
205	tableBlockInfo.setBlockOffset(readBlockIndexInfo.getOffset());	1✔
206	ColumnarFormatVersion version =	1✔
207	ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion());	1✔
208	tableBlockInfo.setVersion(version);	1✔
209	int blockletSize = getBlockletSize(readBlockIndexInfo);	1✔
210	tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize);	1✔
211	String fileName = readBlockIndexInfo.file_name;	1✔
212	// Take only name of file.
213	if (fileName.lastIndexOf("/") > 0) {	1✔
214	fileName = fileName.substring(fileName.lastIndexOf("/"));	×
215	}
216	fileName = (CarbonCommonConstants.FILE_SEPARATOR + fileName).replaceAll("//", "/");	1✔
217	tableBlockInfo.setFilePath(parentPath + fileName);	1✔
218	return tableBlockInfo;	1✔
219	}
220
221	/**
222	* the methods returns the number of blocklets in a block
223	*
224	* @param readBlockIndexInfo
225	* @return
226	*/
227	protected int getBlockletSize(BlockIndex readBlockIndexInfo) {
228	long num_rows = readBlockIndexInfo.getNum_rows();	2✔
229	int blockletSize = Integer.parseInt(CarbonProperties.getInstance()	2✔
230	.getProperty(CarbonCommonConstants.BLOCKLET_SIZE,	2✔
231	CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
232	int remainder = (int) (num_rows % blockletSize);	2✔
233	int noOfBlockLet = (int) (num_rows / blockletSize);	2✔
234	// there could be some blocklets which will not
235	// contain the total records equal to the blockletSize
236	if (remainder > 0) {	2✔
237	noOfBlockLet = noOfBlockLet + 1;	1✔
238	}
239	return noOfBlockLet;	2✔
240	}
241
242	/**
243	* Below method will be used to convert thrift file meta to wrapper file meta
244	*/
245	public abstract DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo)
246	throws IOException;
247
248	public abstract List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException;
249
250	/**
251	* Below method will be used to get blocklet index for data file meta
252	*
253	* @param blockletIndexList
254	* @return blocklet index
255	*/
256	protected BlockletIndex getBlockletIndexForDataFileFooter(List<BlockletIndex> blockletIndexList) {
257	BlockletIndex blockletIndex = new BlockletIndex();	2✔
258	BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();	2✔
259	blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey());	2✔
260	blockletBTreeIndex	2✔
261	.setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey());	2✔
262	blockletIndex.setBtreeIndex(blockletBTreeIndex);	2✔
263	byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone();	2✔
264	byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone();	2✔
265	byte[][] minValue = null;	2✔
266	byte[][] maxValue = null;	2✔
267	for (int i = 1; i < blockletIndexList.size(); i++) {	2✔
268	minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues();	×
269	maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues();	×
270	for (int j = 0; j < maxValue.length; j++) {	×
271	if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {	×
272	currentMinValue[j] = minValue[j].clone();	×
273	}
274	if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {	×
275	currentMaxValue[j] = maxValue[j].clone();	×
276	}
277	}
278	}
279
280	BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();	2✔
281	minMax.setMaxValues(currentMaxValue);	2✔
282	minMax.setMinValues(currentMinValue);	2✔
283	blockletIndex.setMinMaxIndex(minMax);	2✔
284	return blockletIndex;	2✔
285	}
286
287	protected ColumnSchema thriftColumnSchemaToWrapperColumnSchema(
288	org.apache.carbondata.format.ColumnSchema externalColumnSchema) {
289	ColumnSchema wrapperColumnSchema = new ColumnSchema();	2✔
290	wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());	2✔
291	wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());	2✔
292	DataType dataType = CarbonUtil.thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type);	2✔
293	if (DataTypes.isDecimal(dataType)) {	2✔
294	DecimalType decimalType = (DecimalType) dataType;	2✔
295	decimalType.setPrecision(externalColumnSchema.getPrecision());	2✔
296	decimalType.setScale(externalColumnSchema.getScale());	2✔
297	}
298	wrapperColumnSchema.setDataType(dataType);	2✔
299	wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());	2✔
300	List<Encoding> encoders = new ArrayList<Encoding>();	2✔
301	for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {	2✔
302	encoders.add(fromExternalToWrapperEncoding(encoder));	2✔
303	}	2✔
304	wrapperColumnSchema.setEncodingList(encoders);	2✔
305	wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());	2✔
306	wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());	2✔
307	wrapperColumnSchema.setScale(externalColumnSchema.getScale());	2✔
308	wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());	2✔
309	Map<String, String> properties = externalColumnSchema.getColumnProperties();	2✔
310	if (properties != null) {	2✔
311	if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {	1✔
312	wrapperColumnSchema.setSortColumn(true);	1✔
313	}
314	}
315	wrapperColumnSchema.setFunction(externalColumnSchema.getAggregate_function());	2✔
316	List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation =	2✔
317	externalColumnSchema.getParentColumnTableRelations();	2✔
318	if (null != parentColumnTableRelation) {	2✔
319	wrapperColumnSchema.setParentColumnTableRelations(	1✔
320	fromThriftToWrapperParentTableColumnRelations(parentColumnTableRelation));	1✔
321	}
322	return wrapperColumnSchema;	2✔
323	}
324
325	private List<ParentColumnTableRelation> fromThriftToWrapperParentTableColumnRelations(
326	List<org.apache.carbondata.format.ParentColumnTableRelation> thirftParentColumnRelation) {
327	List<ParentColumnTableRelation> parentColumnTableRelationList = new ArrayList<>();	1✔
328	for (org.apache.carbondata.format.ParentColumnTableRelation carbonTableRelation :
329	thirftParentColumnRelation) {	1✔
330	RelationIdentifier relationIdentifier =	1✔
331	new RelationIdentifier(carbonTableRelation.getRelationIdentifier().getDatabaseName(),	1✔
332	carbonTableRelation.getRelationIdentifier().getTableName(),	1✔
333	carbonTableRelation.getRelationIdentifier().getTableId());	1✔
334	ParentColumnTableRelation parentColumnTableRelation =	1✔
335	new ParentColumnTableRelation(relationIdentifier, carbonTableRelation.getColumnId(),	1✔
336	carbonTableRelation.getColumnName());	1✔
337	parentColumnTableRelationList.add(parentColumnTableRelation);	1✔
338	}	1✔
339	return parentColumnTableRelationList;	1✔
340	}
341
342
343
344	/**
345	* Below method is convert the thrift encoding to wrapper encoding
346	*
347	* @param encoderThrift thrift encoding
348	* @return wrapper encoding
349	*/
350	protected Encoding fromExternalToWrapperEncoding(
351	org.apache.carbondata.format.Encoding encoderThrift) {
352	switch (encoderThrift) {	2✔
353	case DICTIONARY:
354	return Encoding.DICTIONARY;	2✔
355	case DELTA:
356	return Encoding.DELTA;	2✔
357	case RLE:
358	return Encoding.RLE;	2✔
359	case INVERTED_INDEX:
360	return Encoding.INVERTED_INDEX;	2✔
361	case BIT_PACKED:
362	return Encoding.BIT_PACKED;	2✔
363	case DIRECT_DICTIONARY:
364	return Encoding.DIRECT_DICTIONARY;	2✔
365	default:
366	throw new IllegalArgumentException(encoderThrift.toString() + " is not supported");	×
367	}
368	}
369
370	/**
371	* Below method will be used to convert thrift segment object to wrapper
372	* segment object
373	*
374	* @param segmentInfo thrift segment info object
375	* @return wrapper segment info object
376	*/
377	protected SegmentInfo getSegmentInfo(org.apache.carbondata.format.SegmentInfo segmentInfo) {
378	SegmentInfo info = new SegmentInfo();	2✔
379	int[] cardinality = new int[segmentInfo.getColumn_cardinalities().size()];	2✔
380	for (int i = 0; i < cardinality.length; i++) {	2✔
381	cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);	2✔
382	}
383	info.setColumnCardinality(cardinality);	2✔
384	return info;	2✔
385	}
386
387	/**
388	* Below method will be used to convert the blocklet index of thrift to
389	* wrapper
390	*
391	* @param blockletIndexThrift
392	* @return blocklet index wrapper
393	*/
394	protected BlockletIndex getBlockletIndex(
395	org.apache.carbondata.format.BlockletIndex blockletIndexThrift) {
396	org.apache.carbondata.format.BlockletBTreeIndex btreeIndex =	2✔
397	blockletIndexThrift.getB_tree_index();	2✔
398	org.apache.carbondata.format.BlockletMinMaxIndex minMaxIndex =	2✔
399	blockletIndexThrift.getMin_max_index();	2✔
400	return new BlockletIndex(	2✔
401	new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()),	2✔
402	new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values()));	2✔
403	}
404
405	/**
406	* Below method will be used to convert the thrift data chunk to wrapper
407	* data chunk
408	*
409	* @param datachunkThrift
410	* @return wrapper data chunk
411	*/
412	protected DataChunk getDataChunk(org.apache.carbondata.format.DataChunk datachunkThrift,
413	boolean isPresenceMetaPresent) {
414	DataChunk dataChunk = new DataChunk();	×
415	dataChunk.setDataPageLength(datachunkThrift.getData_page_length());	×
416	dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset());	×
417	if (isPresenceMetaPresent) {	×
418	dataChunk.setNullValueIndexForColumn(getPresenceMeta(datachunkThrift.getPresence()));	×
419	}
420	dataChunk.setRlePageLength(datachunkThrift.getRle_page_length());	×
421	dataChunk.setRlePageOffset(datachunkThrift.getRle_page_offset());	×
422	dataChunk.setRowMajor(datachunkThrift.isRowMajor());	×
423	dataChunk.setRowIdPageLength(datachunkThrift.getRowid_page_length());	×
424	dataChunk.setRowIdPageOffset(datachunkThrift.getRowid_page_offset());	×
425	List<Encoding> encodingList = new ArrayList<Encoding>(datachunkThrift.getEncoders().size());	×
426	for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) {	×
427	encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i)));	×
428	}
429	dataChunk.setEncodingList(encodingList);	×
430	if (encodingList.contains(Encoding.DELTA)) {	×
431	List<ByteBuffer> thriftEncoderMeta = datachunkThrift.getEncoder_meta();	×
432	List<ValueEncoderMeta> encodeMetaList =	×
433	new ArrayList<ValueEncoderMeta>(thriftEncoderMeta.size());	×
434	for (int i = 0; i < thriftEncoderMeta.size(); i++) {	×
435	encodeMetaList.add(CarbonUtil.deserializeEncoderMetaV2(thriftEncoderMeta.get(i).array()));	×
436	}
437	dataChunk.setValueEncoderMeta(encodeMetaList);	×
438	}
439	return dataChunk;	×
440	}
441
442	}

apache / carbondata / 8385

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous