apache / carbondata / 184

Build # 184

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/671856600385e962017aa963125f937bef17e061">671856600<a href="https://github.com/apache/carbondata/commit/671856600385e962017aa963125f937bef17e061">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/fb04e69ced74685b216d27242a5d4bb74d83aa31">fb04e69ce</a>">fb04e69ce</a><a href="https://github.com/apache/carbondata/commit/671856600385e962017aa963125f937bef17e061"> into 884ac915b">884ac915b</a>

Run Details

294 of 294 new or added lines in 10 files covered. (100.0%)

57888 of 71662 relevant lines covered (80.78%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.66

/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.util;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage;
import org.apache.carbondata.core.datastore.blocklet.EncodedBlocklet;
import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
import org.apache.carbondata.core.datastore.page.statistics.TablePageStatistics;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.format.BlockIndex;
import org.apache.carbondata.format.BlockletBTreeIndex;
import org.apache.carbondata.format.BlockletIndex;
import org.apache.carbondata.format.BlockletInfo3;
import org.apache.carbondata.format.BlockletMinMaxIndex;
import org.apache.carbondata.format.ChunkCompressionMeta;
import org.apache.carbondata.format.ColumnSchema;
import org.apache.carbondata.format.CompressionCodec;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataChunk3;
import org.apache.carbondata.format.FileFooter3;
import org.apache.carbondata.format.FileHeader;
import org.apache.carbondata.format.IndexHeader;
import org.apache.carbondata.format.LocalDictionaryChunk;
import org.apache.carbondata.format.SegmentInfo;

/**
 * Util class to convert to thrift metdata classes
 */
public class CarbonMetadataUtil {

  private CarbonMetadataUtil() {
  }

  /**
   * Below method prepares the file footer object for carbon data file version 3
   *
   * @param infoList
   * @param blockletIndexs
   * @param cardinalities
   * @param numberOfColumns
   * @return FileFooter
   */
  public static FileFooter3 convertFileFooterVersion3(List<BlockletInfo3> infoList,
      List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns)
      throws IOException {
    FileFooter3 footer = getFileFooter3(infoList, blockletIndexs, cardinalities, numberOfColumns);
    for (BlockletInfo3 info : infoList) {
      footer.addToBlocklet_info_list3(info);
    }
    return footer;
  }

  /**
   * Below method will be used to get the file footer object
   *
   * @param infoList         blocklet info
   * @param blockletIndexs
   * @param cardinalities    cardinlaity of dimension columns
   * @param numberOfColumns
   * @return file footer
   */
  private static FileFooter3 getFileFooter3(List<BlockletInfo3> infoList,
      List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns) {
    SegmentInfo segmentInfo = new SegmentInfo();
    segmentInfo.setNum_cols(numberOfColumns);
    segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(cardinalities));
    FileFooter3 footer = new FileFooter3();
    footer.setNum_rows(getNumberOfRowForFooter(infoList));
    footer.setSegment_info(segmentInfo);
    for (BlockletIndex info : blockletIndexs) {
      footer.addToBlocklet_index_list(info);
    }
    return footer;
  }

  /**
   * convert external thrift BlockletMinMaxIndex to BlockletMinMaxIndex of carbon metadata
   */
  public static org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex
      convertExternalMinMaxIndex(BlockletMinMaxIndex minMaxIndex) {
    if (minMaxIndex == null) {
      return null;
    }

    return new org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex(
            minMaxIndex.getMin_values(), minMaxIndex.getMax_values());
  }

  /**
   * convert BlockletMinMaxIndex of carbon metadata to external thrift BlockletMinMaxIndex
   */
  public static BlockletMinMaxIndex convertMinMaxIndex(
      org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex minMaxIndex) {
    if (minMaxIndex == null) {
      return null;
    }

    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();

    for (int i = 0; i < minMaxIndex.getMaxValues().length; i++) {
      blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(minMaxIndex.getMaxValues()[i]));
      blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(minMaxIndex.getMinValues()[i]));
    }

    return blockletMinMaxIndex;
  }

  public static BlockletIndex getBlockletIndex(
      org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex info) {
    BlockletMinMaxIndex blockletMinMaxIndex = convertMinMaxIndex(info.getMinMaxIndex());
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key(info.getBtreeIndex().getStartKey());
    blockletBTreeIndex.setEnd_key(info.getBtreeIndex().getEndKey());
    BlockletIndex blockletIndex = new BlockletIndex();
    blockletIndex.setMin_max_index(blockletMinMaxIndex);
    blockletIndex.setB_tree_index(blockletBTreeIndex);
    return blockletIndex;
  }

  /**
   * Get total number of rows for the file.
   *
   * @param infoList
   * @return
   */
  private static long getNumberOfRowForFooter(List<BlockletInfo3> infoList) {
    long numberOfRows = 0;
    for (BlockletInfo3 info : infoList) {
      numberOfRows += info.num_rows;
    }
    return numberOfRows;
  }

  private static EncodedColumnPage[] getEncodedColumnPages(EncodedBlocklet encodedBlocklet,
      boolean isDimension, int pageIndex) {
    int size =
        isDimension ? encodedBlocklet.getNumberOfDimension() : encodedBlocklet.getNumberOfMeasure();
    EncodedColumnPage [] encodedPages = new EncodedColumnPage[size];

    for (int i = 0; i < size; i++) {
      if (isDimension) {
        encodedPages[i] =
            encodedBlocklet.getEncodedDimensionColumnPages().get(i).getEncodedColumnPageList()
                .get(pageIndex);
      } else {
        encodedPages[i] =
            encodedBlocklet.getEncodedMeasureColumnPages().get(i).getEncodedColumnPageList()
                .get(pageIndex);
      }
    }
    return encodedPages;
  }
  public static BlockletIndex getBlockletIndex(EncodedBlocklet encodedBlocklet,
      List<CarbonMeasure> carbonMeasureList) {
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();

    // Calculating min/max for every each column.
    TablePageStatistics stats =
        new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, 0),
            getEncodedColumnPages(encodedBlocklet, false, 0));
    byte[][] minCol = stats.getDimensionMinValue().clone();
    byte[][] maxCol = stats.getDimensionMaxValue().clone();

    for (int pageIndex = 0; pageIndex < encodedBlocklet.getNumberOfPages(); pageIndex++) {
      stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, pageIndex),
          getEncodedColumnPages(encodedBlocklet, false, pageIndex));
      byte[][] columnMaxData = stats.getDimensionMaxValue();
      byte[][] columnMinData = stats.getDimensionMinValue();
      for (int i = 0; i < maxCol.length; i++) {
        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMaxData[i], maxCol[i]) > 0) {
          maxCol[i] = columnMaxData[i];
        }
        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMinData[i], minCol[i]) < 0) {
          minCol[i] = columnMinData[i];
        }
      }
    }
    // Writing min/max to thrift file
    for (byte[] max : maxCol) {
      blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : minCol) {
      blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }

    stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, 0),
        getEncodedColumnPages(encodedBlocklet, false, 0));
    byte[][] measureMaxValue = stats.getMeasureMaxValue().clone();
    byte[][] measureMinValue = stats.getMeasureMinValue().clone();
    byte[] minVal = null;
    byte[] maxVal = null;
    for (int i = 1; i < encodedBlocklet.getNumberOfPages(); i++) {
      for (int j = 0; j < measureMinValue.length; j++) {
        stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, i),
            getEncodedColumnPages(encodedBlocklet, false, i));
        minVal = stats.getMeasureMinValue()[j];
        maxVal = stats.getMeasureMaxValue()[j];
        if (compareMeasureData(measureMaxValue[j], maxVal, carbonMeasureList.get(j).getDataType())
            < 0) {
          measureMaxValue[j] = maxVal.clone();
        }
        if (compareMeasureData(measureMinValue[j], minVal, carbonMeasureList.get(j).getDataType())
            > 0) {
          measureMinValue[j] = minVal.clone();
        }
      }
    }

    for (byte[] max : measureMaxValue) {
      blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : measureMinValue) {
      blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    byte[] startKey = encodedBlocklet.getPageMetadataList().get(0).serializeStartKey();
    blockletBTreeIndex.setStart_key(startKey);
    byte[] endKey =
        encodedBlocklet.getPageMetadataList().get(encodedBlocklet.getPageMetadataList().size() - 1)
            .serializeEndKey();
    blockletBTreeIndex.setEnd_key(endKey);
    BlockletIndex blockletIndex = new BlockletIndex();
    blockletIndex.setMin_max_index(blockletMinMaxIndex);
    blockletIndex.setB_tree_index(blockletBTreeIndex);
    return blockletIndex;
  }

  /**
   * Right now it is set to default values. We may use this in future
   */
  public static ChunkCompressionMeta getSnappyChunkCompressionMeta() {
    ChunkCompressionMeta chunkCompressionMeta = new ChunkCompressionMeta();
    chunkCompressionMeta.setCompression_codec(CompressionCodec.SNAPPY);
    chunkCompressionMeta.setTotal_compressed_size(0);
    chunkCompressionMeta.setTotal_uncompressed_size(0);
    return chunkCompressionMeta;
  }

  /**
   * Below method will be used to get the index header
   *
   * @param columnCardinality cardinality of each column
   * @param columnSchemaList  list of column present in the table
   * @param bucketNumber
   * @param schemaTimeStamp current timestamp of schema
   * @return Index header object
   */
  public static IndexHeader getIndexHeader(int[] columnCardinality,
      List<ColumnSchema> columnSchemaList, int bucketNumber, long schemaTimeStamp) {
    // create segment info object
    SegmentInfo segmentInfo = new SegmentInfo();
    // set the number of columns
    segmentInfo.setNum_cols(columnSchemaList.size());
    // setting the column cardinality
    segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(columnCardinality));
    // create index header object
    IndexHeader indexHeader = new IndexHeader();
    ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
    indexHeader.setVersion(version.number());
    // set the segment info
    indexHeader.setSegment_info(segmentInfo);
    // set the column names
    indexHeader.setTable_columns(columnSchemaList);
    // set the bucket number
    indexHeader.setBucket_id(bucketNumber);
    // set the current schema time stamp which will used for deciding the restructured block
    indexHeader.setSchema_time_stamp(schemaTimeStamp);
    return indexHeader;
  }

  /**
   * Below method will be used to get the block index info thrift object for
   * each block present in the segment
   *
   * @param blockIndexInfoList block index info list
   * @return list of block index
   */
  public static List<BlockIndex> getBlockIndexInfo(List<BlockIndexInfo> blockIndexInfoList) {
    List<BlockIndex> thriftBlockIndexList = new ArrayList<BlockIndex>();
    BlockIndex blockIndex = null;
    // below code to create block index info object for each block
    for (BlockIndexInfo blockIndexInfo : blockIndexInfoList) {
      blockIndex = new BlockIndex();
      blockIndex.setNum_rows(blockIndexInfo.getNumberOfRows());
      blockIndex.setOffset(blockIndexInfo.getOffset());
      blockIndex.setFile_name(blockIndexInfo.getFileName());
      blockIndex.setBlock_index(getBlockletIndex(blockIndexInfo.getBlockletIndex()));
      if (blockIndexInfo.getBlockletInfo() != null) {
        blockIndex.setBlocklet_info(getBlocletInfo3(blockIndexInfo.getBlockletInfo()));
      }
      thriftBlockIndexList.add(blockIndex);
    }
    return thriftBlockIndexList;
  }

  public static BlockletInfo3 getBlocletInfo3(
      org.apache.carbondata.core.metadata.blocklet.BlockletInfo blockletInfo) {
    List<Long> dimensionChunkOffsets = blockletInfo.getDimensionChunkOffsets();
    dimensionChunkOffsets.addAll(blockletInfo.getMeasureChunkOffsets());
    List<Integer> dimensionChunksLength = blockletInfo.getDimensionChunksLength();
    dimensionChunksLength.addAll(blockletInfo.getMeasureChunksLength());
    return new BlockletInfo3(blockletInfo.getNumberOfRows(), dimensionChunkOffsets,
        dimensionChunksLength, blockletInfo.getDimensionOffset(), blockletInfo.getMeasureOffsets(),
        blockletInfo.getNumberOfPages());
  }

  /**
   * return DataChunk3 that contains the input DataChunk2 list
   */
  public static DataChunk3 getDataChunk3(List<DataChunk2> dataChunksList,
      LocalDictionaryChunk encodedDictionary) {
    int offset = 0;
    DataChunk3 dataChunk = new DataChunk3();
    List<Integer> pageOffsets = new ArrayList<>();
    List<Integer> pageLengths = new ArrayList<>();
    int length = 0;
    for (DataChunk2 dataChunk2 : dataChunksList) {
      pageOffsets.add(offset);
      length = dataChunk2.getData_page_length() + dataChunk2.getRle_page_length() +
          dataChunk2.getRowid_page_length();
      pageLengths.add(length);
      offset += length;
    }
    dataChunk.setLocal_dictionary(encodedDictionary);
    dataChunk.setData_chunk_list(dataChunksList);
    dataChunk.setPage_length(pageLengths);
    dataChunk.setPage_offset(pageOffsets);
    return dataChunk;
  }

  /**
   * return DataChunk3 for the dimension column (specifed by `columnIndex`)
   * in `encodedTablePageList`
   */
  public static DataChunk3 getDimensionDataChunk3(EncodedBlocklet encodedBlocklet,
      int columnIndex) {
    List<DataChunk2> dataChunksList = new ArrayList<>();
    BlockletEncodedColumnPage blockletEncodedColumnPage =
        encodedBlocklet.getEncodedDimensionColumnPages().get(columnIndex);
    for (EncodedColumnPage encodedColumnPage : blockletEncodedColumnPage
        .getEncodedColumnPageList()) {
      dataChunksList.add(encodedColumnPage.getPageMetadata());
    }
    return CarbonMetadataUtil
        .getDataChunk3(dataChunksList, blockletEncodedColumnPage.getEncodedDictionary());
  }

  /**
   * return DataChunk3 for the measure column (specifed by `columnIndex`)
   * in `encodedTablePageList`
   */
  public static DataChunk3 getMeasureDataChunk3(EncodedBlocklet encodedBlocklet, int columnIndex) {
    List<DataChunk2> dataChunksList = new ArrayList<>();
    BlockletEncodedColumnPage blockletEncodedColumnPage =
        encodedBlocklet.getEncodedMeasureColumnPages().get(columnIndex);
    for (EncodedColumnPage encodedColumnPage : blockletEncodedColumnPage
        .getEncodedColumnPageList()) {
      dataChunksList.add(encodedColumnPage.getPageMetadata());
    }
    return CarbonMetadataUtil.getDataChunk3(dataChunksList, null);
  }

  private static int compareMeasureData(byte[] first, byte[] second, DataType dataType) {
    ByteBuffer firstBuffer = null;
    ByteBuffer secondBuffer = null;
    if (dataType == DataTypes.BOOLEAN) {
      return first[0] - second[0];
    } else if (dataType == DataTypes.DOUBLE) {
      firstBuffer = ByteBuffer.allocate(8);
      firstBuffer.put(first);
      secondBuffer = ByteBuffer.allocate(8);
      secondBuffer.put(second);
      firstBuffer.flip();
      secondBuffer.flip();
      double compare = firstBuffer.getDouble() - secondBuffer.getDouble();
      if (compare > 0) {
        compare = 1;
      } else if (compare < 0) {
        compare = -1;
      }
      return (int) compare;
    } else if (dataType == DataTypes.LONG || dataType == DataTypes.INT
        || dataType == DataTypes.SHORT) {
      firstBuffer = ByteBuffer.allocate(8);
      firstBuffer.put(first);
      secondBuffer = ByteBuffer.allocate(8);
      secondBuffer.put(second);
      firstBuffer.flip();
      secondBuffer.flip();
      long compare = firstBuffer.getLong() - secondBuffer.getLong();
      if (compare > 0) {
        compare = 1;
      } else if (compare < 0) {
        compare = -1;
      }
      return (int) compare;
    } else if (DataTypes.isDecimal(dataType)) {
      return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second));
    } else {
      throw new IllegalArgumentException("Invalid data type:" + dataType);
    }
  }

  /**
   * Below method will be used to prepare the file header object for carbondata file
   *
   * @param isFooterPresent  is footer present in carbon data file
   * @param columnSchemaList list of column schema
   * @param schemaUpdatedTimeStamp  schema updated time stamp to be used for restructure scenarios
   * @return file header thrift object
   */
  public static FileHeader getFileHeader(boolean isFooterPresent,
      List<ColumnSchema> columnSchemaList, long schemaUpdatedTimeStamp) {
    FileHeader fileHeader = new FileHeader();
    ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
    fileHeader.setIs_footer_present(isFooterPresent);
    fileHeader.setColumn_schema(columnSchemaList);
    fileHeader.setVersion(version.number());
    fileHeader.setTime_stamp(schemaUpdatedTimeStamp);
    return fileHeader;
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.util;
18
19	import java.io.IOException;
20	import java.nio.ByteBuffer;
21	import java.util.ArrayList;
22	import java.util.List;
23
24	import org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage;
25	import org.apache.carbondata.core.datastore.blocklet.EncodedBlocklet;
26	import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
27	import org.apache.carbondata.core.datastore.page.statistics.TablePageStatistics;
28	import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
29	import org.apache.carbondata.core.metadata.datatype.DataType;
30	import org.apache.carbondata.core.metadata.datatype.DataTypes;
31	import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
32	import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
33	import org.apache.carbondata.format.BlockIndex;
34	import org.apache.carbondata.format.BlockletBTreeIndex;
35	import org.apache.carbondata.format.BlockletIndex;
36	import org.apache.carbondata.format.BlockletInfo3;
37	import org.apache.carbondata.format.BlockletMinMaxIndex;
38	import org.apache.carbondata.format.ChunkCompressionMeta;
39	import org.apache.carbondata.format.ColumnSchema;
40	import org.apache.carbondata.format.CompressionCodec;
41	import org.apache.carbondata.format.DataChunk2;
42	import org.apache.carbondata.format.DataChunk3;
43	import org.apache.carbondata.format.FileFooter3;
44	import org.apache.carbondata.format.FileHeader;
45	import org.apache.carbondata.format.IndexHeader;
46	import org.apache.carbondata.format.LocalDictionaryChunk;
47	import org.apache.carbondata.format.SegmentInfo;
48
49	/**
50	* Util class to convert to thrift metdata classes
51	*/
52	public class CarbonMetadataUtil {
53
54	private CarbonMetadataUtil() {	×
55	}	×
56
57	/**
58	* Below method prepares the file footer object for carbon data file version 3
59	*
60	* @param infoList
61	* @param blockletIndexs
62	* @param cardinalities
63	* @param numberOfColumns
64	* @return FileFooter
65	*/
66	public static FileFooter3 convertFileFooterVersion3(List<BlockletInfo3> infoList,
67	List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns)
68	throws IOException {
69	FileFooter3 footer = getFileFooter3(infoList, blockletIndexs, cardinalities, numberOfColumns);	1✔
70	for (BlockletInfo3 info : infoList) {	1✔
71	footer.addToBlocklet_info_list3(info);	1✔
72	}	1✔
73	return footer;	1✔
74	}
75
76	/**
77	* Below method will be used to get the file footer object
78	*
79	* @param infoList blocklet info
80	* @param blockletIndexs
81	* @param cardinalities cardinlaity of dimension columns
82	* @param numberOfColumns
83	* @return file footer
84	*/
85	private static FileFooter3 getFileFooter3(List<BlockletInfo3> infoList,
86	List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns) {
87	SegmentInfo segmentInfo = new SegmentInfo();	1✔
88	segmentInfo.setNum_cols(numberOfColumns);	1✔
89	segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(cardinalities));	1✔
90	FileFooter3 footer = new FileFooter3();	1✔
91	footer.setNum_rows(getNumberOfRowForFooter(infoList));	1✔
92	footer.setSegment_info(segmentInfo);	1✔
93	for (BlockletIndex info : blockletIndexs) {	1✔
94	footer.addToBlocklet_index_list(info);	1✔
95	}	1✔
96	return footer;	1✔
97	}
98
99	/**
100	* convert external thrift BlockletMinMaxIndex to BlockletMinMaxIndex of carbon metadata
101	*/
102	public static org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex
103	convertExternalMinMaxIndex(BlockletMinMaxIndex minMaxIndex) {
104	if (minMaxIndex == null) {	1✔
105	return null;	×
106	}
107
108	return new org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex(	1✔
109	minMaxIndex.getMin_values(), minMaxIndex.getMax_values());	1✔
110	}
111
112	/**
113	* convert BlockletMinMaxIndex of carbon metadata to external thrift BlockletMinMaxIndex
114	*/
115	public static BlockletMinMaxIndex convertMinMaxIndex(
116	org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex minMaxIndex) {
117	if (minMaxIndex == null) {	2✔
118	return null;	×
119	}
120
121	BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();	2✔
122
123	for (int i = 0; i < minMaxIndex.getMaxValues().length; i++) {	2✔
124	blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(minMaxIndex.getMaxValues()[i]));	2✔
125	blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(minMaxIndex.getMinValues()[i]));	2✔
126	}
127
128	return blockletMinMaxIndex;	2✔
129	}
130
131	public static BlockletIndex getBlockletIndex(
132	org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex info) {
133	BlockletMinMaxIndex blockletMinMaxIndex = convertMinMaxIndex(info.getMinMaxIndex());	2✔
134	BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();	2✔
135	blockletBTreeIndex.setStart_key(info.getBtreeIndex().getStartKey());	2✔
136	blockletBTreeIndex.setEnd_key(info.getBtreeIndex().getEndKey());	2✔
137	BlockletIndex blockletIndex = new BlockletIndex();	2✔
138	blockletIndex.setMin_max_index(blockletMinMaxIndex);	2✔
139	blockletIndex.setB_tree_index(blockletBTreeIndex);	2✔
140	return blockletIndex;	2✔
141	}
142
143	/**
144	* Get total number of rows for the file.
145	*
146	* @param infoList
147	* @return
148	*/
149	private static long getNumberOfRowForFooter(List<BlockletInfo3> infoList) {
150	long numberOfRows = 0;	1✔
151	for (BlockletInfo3 info : infoList) {	1✔
152	numberOfRows += info.num_rows;	1✔
153	}	1✔
154	return numberOfRows;	1✔
155	}
156
157	private static EncodedColumnPage[] getEncodedColumnPages(EncodedBlocklet encodedBlocklet,
158	boolean isDimension, int pageIndex) {
159	int size =	1✔
160	isDimension ? encodedBlocklet.getNumberOfDimension() : encodedBlocklet.getNumberOfMeasure();	1✔
161	EncodedColumnPage [] encodedPages = new EncodedColumnPage[size];	1✔
162
163	for (int i = 0; i < size; i++) {	1✔
164	if (isDimension) {	1✔
165	encodedPages[i] =	1✔
166	encodedBlocklet.getEncodedDimensionColumnPages().get(i).getEncodedColumnPageList()	1✔
167	.get(pageIndex);	1✔
168	} else {
169	encodedPages[i] =	1✔
170	encodedBlocklet.getEncodedMeasureColumnPages().get(i).getEncodedColumnPageList()	1✔
171	.get(pageIndex);	1✔
172	}
173	}
174	return encodedPages;	1✔
175	}
176	public static BlockletIndex getBlockletIndex(EncodedBlocklet encodedBlocklet,
177	List<CarbonMeasure> carbonMeasureList) {
178	BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();	1✔
179
180	// Calculating min/max for every each column.
181	TablePageStatistics stats =	1✔
182	new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, 0),	1✔
183	getEncodedColumnPages(encodedBlocklet, false, 0));	1✔
184	byte[][] minCol = stats.getDimensionMinValue().clone();	1✔
185	byte[][] maxCol = stats.getDimensionMaxValue().clone();	1✔
186
187	for (int pageIndex = 0; pageIndex < encodedBlocklet.getNumberOfPages(); pageIndex++) {	1✔
188	stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, pageIndex),	1✔
189	getEncodedColumnPages(encodedBlocklet, false, pageIndex));	1✔
190	byte[][] columnMaxData = stats.getDimensionMaxValue();	1✔
191	byte[][] columnMinData = stats.getDimensionMinValue();	1✔
192	for (int i = 0; i < maxCol.length; i++) {	1✔
193	if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMaxData[i], maxCol[i]) > 0) {	1✔
194	maxCol[i] = columnMaxData[i];	1✔
195	}
196	if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMinData[i], minCol[i]) < 0) {	1✔
197	minCol[i] = columnMinData[i];	1✔
198	}
199	}
200	}
201	// Writing min/max to thrift file
202	for (byte[] max : maxCol) {	1✔
203	blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));	1✔
204	}
205	for (byte[] min : minCol) {	1✔
206	blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));	1✔
207	}
208
209	stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, 0),	1✔
210	getEncodedColumnPages(encodedBlocklet, false, 0));	1✔
211	byte[][] measureMaxValue = stats.getMeasureMaxValue().clone();	1✔
212	byte[][] measureMinValue = stats.getMeasureMinValue().clone();	1✔
213	byte[] minVal = null;	1✔
214	byte[] maxVal = null;	1✔
215	for (int i = 1; i < encodedBlocklet.getNumberOfPages(); i++) {	1✔
216	for (int j = 0; j < measureMinValue.length; j++) {	1✔
217	stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, i),	1✔
218	getEncodedColumnPages(encodedBlocklet, false, i));	1✔
219	minVal = stats.getMeasureMinValue()[j];	1✔
220	maxVal = stats.getMeasureMaxValue()[j];	1✔
221	if (compareMeasureData(measureMaxValue[j], maxVal, carbonMeasureList.get(j).getDataType())	1✔
222	< 0) {
223	measureMaxValue[j] = maxVal.clone();	1✔
224	}
225	if (compareMeasureData(measureMinValue[j], minVal, carbonMeasureList.get(j).getDataType())	1✔
226	> 0) {
227	measureMinValue[j] = minVal.clone();	1✔
228	}
229	}
230	}
231
232	for (byte[] max : measureMaxValue) {	1✔
233	blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));	1✔
234	}
235	for (byte[] min : measureMinValue) {	1✔
236	blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));	1✔
237	}
238	BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();	1✔
239	byte[] startKey = encodedBlocklet.getPageMetadataList().get(0).serializeStartKey();	1✔
240	blockletBTreeIndex.setStart_key(startKey);	1✔
241	byte[] endKey =	1✔
242	encodedBlocklet.getPageMetadataList().get(encodedBlocklet.getPageMetadataList().size() - 1)	1✔
243	.serializeEndKey();	1✔
244	blockletBTreeIndex.setEnd_key(endKey);	1✔
245	BlockletIndex blockletIndex = new BlockletIndex();	1✔
246	blockletIndex.setMin_max_index(blockletMinMaxIndex);	1✔
247	blockletIndex.setB_tree_index(blockletBTreeIndex);	1✔
248	return blockletIndex;	1✔
249	}
250
251	/**
252	* Right now it is set to default values. We may use this in future
253	*/
254	public static ChunkCompressionMeta getSnappyChunkCompressionMeta() {
255	ChunkCompressionMeta chunkCompressionMeta = new ChunkCompressionMeta();	2✔
256	chunkCompressionMeta.setCompression_codec(CompressionCodec.SNAPPY);	2✔
257	chunkCompressionMeta.setTotal_compressed_size(0);	2✔
258	chunkCompressionMeta.setTotal_uncompressed_size(0);	2✔
259	return chunkCompressionMeta;	2✔
260	}
261
262	/**
263	* Below method will be used to get the index header
264	*
265	* @param columnCardinality cardinality of each column
266	* @param columnSchemaList list of column present in the table
267	* @param bucketNumber
268	* @param schemaTimeStamp current timestamp of schema
269	* @return Index header object
270	*/
271	public static IndexHeader getIndexHeader(int[] columnCardinality,
272	List<ColumnSchema> columnSchemaList, int bucketNumber, long schemaTimeStamp) {
273	// create segment info object
274	SegmentInfo segmentInfo = new SegmentInfo();	2✔
275	// set the number of columns
276	segmentInfo.setNum_cols(columnSchemaList.size());	2✔
277	// setting the column cardinality
278	segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(columnCardinality));	2✔
279	// create index header object
280	IndexHeader indexHeader = new IndexHeader();	2✔
281	ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();	2✔
282	indexHeader.setVersion(version.number());	2✔
283	// set the segment info
284	indexHeader.setSegment_info(segmentInfo);	2✔
285	// set the column names
286	indexHeader.setTable_columns(columnSchemaList);	2✔
287	// set the bucket number
288	indexHeader.setBucket_id(bucketNumber);	2✔
289	// set the current schema time stamp which will used for deciding the restructured block
290	indexHeader.setSchema_time_stamp(schemaTimeStamp);	2✔
291	return indexHeader;	2✔
292	}
293
294	/**
295	* Below method will be used to get the block index info thrift object for
296	* each block present in the segment
297	*
298	* @param blockIndexInfoList block index info list
299	* @return list of block index
300	*/
301	public static List<BlockIndex> getBlockIndexInfo(List<BlockIndexInfo> blockIndexInfoList) {
302	List<BlockIndex> thriftBlockIndexList = new ArrayList<BlockIndex>();	2✔
303	BlockIndex blockIndex = null;	2✔
304	// below code to create block index info object for each block
305	for (BlockIndexInfo blockIndexInfo : blockIndexInfoList) {	2✔
306	blockIndex = new BlockIndex();	2✔
307	blockIndex.setNum_rows(blockIndexInfo.getNumberOfRows());	2✔
308	blockIndex.setOffset(blockIndexInfo.getOffset());	2✔
309	blockIndex.setFile_name(blockIndexInfo.getFileName());	2✔
310	blockIndex.setBlock_index(getBlockletIndex(blockIndexInfo.getBlockletIndex()));	2✔
311	if (blockIndexInfo.getBlockletInfo() != null) {	2✔
312	blockIndex.setBlocklet_info(getBlocletInfo3(blockIndexInfo.getBlockletInfo()));	1✔
313	}
314	thriftBlockIndexList.add(blockIndex);	2✔
315	}	2✔
316	return thriftBlockIndexList;	2✔
317	}
318
319	public static BlockletInfo3 getBlocletInfo3(
320	org.apache.carbondata.core.metadata.blocklet.BlockletInfo blockletInfo) {
321	List<Long> dimensionChunkOffsets = blockletInfo.getDimensionChunkOffsets();	1✔
322	dimensionChunkOffsets.addAll(blockletInfo.getMeasureChunkOffsets());	1✔
323	List<Integer> dimensionChunksLength = blockletInfo.getDimensionChunksLength();	1✔
324	dimensionChunksLength.addAll(blockletInfo.getMeasureChunksLength());	1✔
325	return new BlockletInfo3(blockletInfo.getNumberOfRows(), dimensionChunkOffsets,	1✔
326	dimensionChunksLength, blockletInfo.getDimensionOffset(), blockletInfo.getMeasureOffsets(),	1✔
327	blockletInfo.getNumberOfPages());	1✔
328	}
329
330	/**
331	* return DataChunk3 that contains the input DataChunk2 list
332	*/
333	public static DataChunk3 getDataChunk3(List<DataChunk2> dataChunksList,
334	LocalDictionaryChunk encodedDictionary) {
335	int offset = 0;	1✔
336	DataChunk3 dataChunk = new DataChunk3();	1✔
337	List<Integer> pageOffsets = new ArrayList<>();	1✔
338	List<Integer> pageLengths = new ArrayList<>();	1✔
339	int length = 0;	1✔
340	for (DataChunk2 dataChunk2 : dataChunksList) {	1✔
341	pageOffsets.add(offset);	1✔
342	length = dataChunk2.getData_page_length() + dataChunk2.getRle_page_length() +	1✔
343	dataChunk2.getRowid_page_length();	1✔
344	pageLengths.add(length);	1✔
345	offset += length;	1✔
346	}	1✔
347	dataChunk.setLocal_dictionary(encodedDictionary);	1✔
348	dataChunk.setData_chunk_list(dataChunksList);	1✔
349	dataChunk.setPage_length(pageLengths);	1✔
350	dataChunk.setPage_offset(pageOffsets);	1✔
351	return dataChunk;	1✔
352	}
353
354	/**
355	* return DataChunk3 for the dimension column (specifed by `columnIndex`)
356	* in `encodedTablePageList`
357	*/
358	public static DataChunk3 getDimensionDataChunk3(EncodedBlocklet encodedBlocklet,
359	int columnIndex) {
360	List<DataChunk2> dataChunksList = new ArrayList<>();	1✔
361	BlockletEncodedColumnPage blockletEncodedColumnPage =	1✔
362	encodedBlocklet.getEncodedDimensionColumnPages().get(columnIndex);	1✔
363	for (EncodedColumnPage encodedColumnPage : blockletEncodedColumnPage	1✔
364	.getEncodedColumnPageList()) {	1✔
365	dataChunksList.add(encodedColumnPage.getPageMetadata());	1✔
366	}	1✔
367	return CarbonMetadataUtil	1✔
368	.getDataChunk3(dataChunksList, blockletEncodedColumnPage.getEncodedDictionary());	1✔
369	}
370
371	/**
372	* return DataChunk3 for the measure column (specifed by `columnIndex`)
373	* in `encodedTablePageList`
374	*/
375	public static DataChunk3 getMeasureDataChunk3(EncodedBlocklet encodedBlocklet, int columnIndex) {
376	List<DataChunk2> dataChunksList = new ArrayList<>();	1✔
377	BlockletEncodedColumnPage blockletEncodedColumnPage =	1✔
378	encodedBlocklet.getEncodedMeasureColumnPages().get(columnIndex);	1✔
379	for (EncodedColumnPage encodedColumnPage : blockletEncodedColumnPage	1✔
380	.getEncodedColumnPageList()) {	1✔
381	dataChunksList.add(encodedColumnPage.getPageMetadata());	1✔
382	}	1✔
383	return CarbonMetadataUtil.getDataChunk3(dataChunksList, null);	1✔
384	}
385
386	private static int compareMeasureData(byte[] first, byte[] second, DataType dataType) {
387	ByteBuffer firstBuffer = null;	2✔
388	ByteBuffer secondBuffer = null;	2✔
389	if (dataType == DataTypes.BOOLEAN) {	2✔
390	return first[0] - second[0];	1✔
391	} else if (dataType == DataTypes.DOUBLE) {	2✔
392	firstBuffer = ByteBuffer.allocate(8);	1✔
393	firstBuffer.put(first);	1✔
394	secondBuffer = ByteBuffer.allocate(8);	1✔
395	secondBuffer.put(second);	1✔
396	firstBuffer.flip();	1✔
397	secondBuffer.flip();	1✔
398	double compare = firstBuffer.getDouble() - secondBuffer.getDouble();	1✔
399	if (compare > 0) {	1✔
400	compare = 1;	1✔
401	} else if (compare < 0) {	1✔
402	compare = -1;	1✔
403	}
404	return (int) compare;	1✔
405	} else if (dataType == DataTypes.LONG \|\| dataType == DataTypes.INT	2✔
406	\|\| dataType == DataTypes.SHORT) {
407	firstBuffer = ByteBuffer.allocate(8);	2✔
408	firstBuffer.put(first);	2✔
409	secondBuffer = ByteBuffer.allocate(8);	2✔
410	secondBuffer.put(second);	2✔
411	firstBuffer.flip();	2✔
412	secondBuffer.flip();	2✔
413	long compare = firstBuffer.getLong() - secondBuffer.getLong();	2✔
414	if (compare > 0) {	2✔
415	compare = 1;	2✔
416	} else if (compare < 0) {	1✔
417	compare = -1;	1✔
418	}
419	return (int) compare;	2✔
420	} else if (DataTypes.isDecimal(dataType)) {	1✔
421	return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second));	1✔
422	} else {
423	throw new IllegalArgumentException("Invalid data type:" + dataType);	×
424	}
425	}
426
427	/**
428	* Below method will be used to prepare the file header object for carbondata file
429	*
430	* @param isFooterPresent is footer present in carbon data file
431	* @param columnSchemaList list of column schema
432	* @param schemaUpdatedTimeStamp schema updated time stamp to be used for restructure scenarios
433	* @return file header thrift object
434	*/
435	public static FileHeader getFileHeader(boolean isFooterPresent,
436	List<ColumnSchema> columnSchemaList, long schemaUpdatedTimeStamp) {
437	FileHeader fileHeader = new FileHeader();	1✔
438	ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();	1✔
439	fileHeader.setIs_footer_present(isFooterPresent);	1✔
440	fileHeader.setColumn_schema(columnSchemaList);	1✔
441	fileHeader.setVersion(version.number());	1✔
442	fileHeader.setTime_stamp(schemaUpdatedTimeStamp);	1✔
443	return fileHeader;	1✔
444	}
445
446	}

apache / carbondata / 184

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous