3031

Build # 3031

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/e874fb22e564137b9b7c6e6fd818cbf1e70a3ada">e874fb22e">&lt;a href=&quot;https://github.com/apache/carbondata/commit/</a><a class="double-link" href="https://github.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/e874fb22e564137b9b7c6e6fd818cbf1e70a3ada">e874fb22e</a><a href="https://github.com/apache/carbondata/commit/e874fb22e564137b9b7c6e6fd818cbf1e70a3ada">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/e874fb22e564137b9b7c6e6fd818cbf1e70a3ada&quot;&gt;e874fb22e&lt;/a&gt;&lt;a href=&quot;https://github.com/apache/carbondata/commit/e874fb22e564137b9b7c6e6fd818cbf1e70a3ada&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/carbondata/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/carbondata/commit/&amp;amp;lt;a ... (continued)

Run Details

15 of 15 new or added lines in 2 files covered. (100.0%)

62897 of 79415 relevant lines covered (79.2%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.8

/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.indexstore.blockletindex;

import java.io.ByteArrayOutputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.SegmentPropertiesAndSchemaHolder;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.indexstore.BlockMetaInfo;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.row.DataMapRow;
import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.util.BlockletDataMapUtil;

/**
 * Datamap implementation for blocklet.
 */
public class BlockletDataMap extends BlockDataMap implements Serializable {

  private static final long serialVersionUID = -2170289352240810993L;
  // total block number in this datamap
  private int blockNum = 0;

  @Override
  public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
    super.init(dataMapModel);
  }

  /**
   * Method to check the cache level and load metadata based on that information
   *
   * @param blockletDataMapInfo
   * @param indexInfo
   * @throws IOException
   * @throws MemoryException
   */
  @Override
  protected DataMapRowImpl loadMetadata(CarbonRowSchema[] taskSummarySchema,
      SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo,
      List<DataFileFooter> indexInfo) throws IOException, MemoryException {
    if (isLegacyStore) {
      return loadBlockInfoForOldStore(taskSummarySchema, segmentProperties, blockletDataMapInfo,
          indexInfo);
    } else {
      return loadBlockletMetaInfo(taskSummarySchema, segmentProperties, blockletDataMapInfo,
          indexInfo);
    }
  }

  @Override
  protected CarbonRowSchema[] getTaskSummarySchema() {
    if (isLegacyStore) {
      return super.getTaskSummarySchema();
    }
    SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper segmentPropertiesWrapper =
        SegmentPropertiesAndSchemaHolder.getInstance()
            .getSegmentPropertiesWrapper(segmentPropertiesIndex);
    try {
      return segmentPropertiesWrapper.getTaskSummarySchemaForBlocklet(false, isFilePathStored);
    } catch (MemoryException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  protected CarbonRowSchema[] getFileFooterEntrySchema() {
    if (isLegacyStore) {
      return super.getFileFooterEntrySchema();
    }
    return SegmentPropertiesAndSchemaHolder.getInstance()
        .getSegmentPropertiesWrapper(segmentPropertiesIndex).getBlockletFileFooterEntrySchema();
  }

  /**
   * Method to load blocklet metadata information
   *
   * @param blockletDataMapInfo
   * @param indexInfo
   * @throws IOException
   * @throws MemoryException
   */
  private DataMapRowImpl loadBlockletMetaInfo(CarbonRowSchema[] taskSummarySchema,
      SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo,
      List<DataFileFooter> indexInfo) throws IOException, MemoryException {
    String tempFilePath = null;
    DataMapRowImpl summaryRow = null;
    CarbonRowSchema[] schema = getFileFooterEntrySchema();
    boolean[] summaryRowMinMaxFlag = new boolean[segmentProperties.getColumnsValueSize().length];
    Arrays.fill(summaryRowMinMaxFlag, true);
    // Relative blocklet ID is the id assigned to a blocklet within a part file
    int relativeBlockletId = 0;
    for (DataFileFooter fileFooter : indexInfo) {
      // update the min max flag for summary row
      updateMinMaxFlag(fileFooter, summaryRowMinMaxFlag);
      TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
      BlockMetaInfo blockMetaInfo =
          blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
      // Here it loads info about all blocklets of index
      // Only add if the file exists physically. There are scenarios which index file exists inside
      // merge index but related carbondata files are deleted. In that case we first check whether
      // the file exists physically or not
      if (blockMetaInfo != null) {
        // this case is for CACHE_LEVEL = BLOCKLET
        // blocklet ID will start from 0 again only when part file path is changed
        if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
          tempFilePath = blockInfo.getFilePath();
          relativeBlockletId = 0;
          blockNum++;
        }
        summaryRow = loadToUnsafe(schema, taskSummarySchema, fileFooter, segmentProperties,
            getMinMaxCacheColumns(), blockInfo.getFilePath(), summaryRow,
            blockMetaInfo, relativeBlockletId);
        // this is done because relative blocklet id need to be incremented based on the
        // total number of blocklets
        relativeBlockletId += fileFooter.getBlockletList().size();
      }
    }
    summaryRow.setLong(0L, TASK_ROW_COUNT);
    setMinMaxFlagForTaskSummary(summaryRow, taskSummarySchema, segmentProperties,
        summaryRowMinMaxFlag);
    return summaryRow;
  }

  private DataMapRowImpl loadToUnsafe(CarbonRowSchema[] schema, CarbonRowSchema[] taskSummarySchema,
      DataFileFooter fileFooter, SegmentProperties segmentProperties,
      List<CarbonColumn> minMaxCacheColumns, String filePath, DataMapRowImpl summaryRow,
      BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
    List<BlockletInfo> blockletList = fileFooter.getBlockletList();
    // Add one row to maintain task level min max for segment pruning
    if (!blockletList.isEmpty() && summaryRow == null) {
      summaryRow = new DataMapRowImpl(taskSummarySchema);
    }
    for (int index = 0; index < blockletList.size(); index++) {
      DataMapRow row = new DataMapRowImpl(schema);
      int ordinal = 0;
      int taskMinMaxOrdinal = 1;
      BlockletInfo blockletInfo = blockletList.get(index);
      blockletInfo.setSorted(fileFooter.isSorted());
      BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
      // get min max values for columns to be cached
      byte[][] minValuesForColumnsToBeCached = BlockletDataMapUtil
          .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns,
              minMaxIndex.getMinValues());
      byte[][] maxValuesForColumnsToBeCached = BlockletDataMapUtil
          .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns,
              minMaxIndex.getMaxValues());
      boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletDataMapUtil
          .getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, minMaxCacheColumns,
              fileFooter.getBlockletIndex().getMinMaxIndex().getIsMinMaxSet());
      row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal);
      // compute and set task level min values
      addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal,
          minValuesForColumnsToBeCached, TASK_MIN_VALUES_INDEX, true);
      ordinal++;
      taskMinMaxOrdinal++;
      row.setRow(addMinMax(schema[ordinal], maxValuesForColumnsToBeCached), ordinal);
      // compute and set task level max values
      addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal,
          maxValuesForColumnsToBeCached, TASK_MAX_VALUES_INDEX, false);
      ordinal++;
      row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
      // add file name
      byte[] filePathBytes =
          getFileNameFromPath(filePath).getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
      row.setByteArray(filePathBytes, ordinal++);
      // add version number
      row.setShort(fileFooter.getVersionId().number(), ordinal++);
      // add schema updated time
      row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
      byte[] serializedData;
      try {
        // Add block footer offset, it is used if we need to read footer of block
        row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
        setLocations(blockMetaInfo.getLocationInfo(), row, ordinal++);
        // Store block size
        row.setLong(blockMetaInfo.getSize(), ordinal++);
        // add min max flag for all the dimension columns
        addMinMaxFlagValues(row, schema[ordinal], minMaxFlagValuesForColumnsToBeCached, ordinal);
        ordinal++;
        // add blocklet info
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        DataOutput dataOutput = new DataOutputStream(stream);
        blockletInfo.write(dataOutput);
        serializedData = stream.toByteArray();
        row.setByteArray(serializedData, ordinal++);
        // add pages
        row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
        // for relative blocklet id i.e blocklet id that belongs to a particular carbondata file
        row.setShort((short) relativeBlockletId++, ordinal);
        memoryDMStore.addIndexRow(schema, row);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }
    return summaryRow;
  }

  @Override
  public ExtendedBlocklet getDetailedBlocklet(String blockletId) {
    if (isLegacyStore) {
      return super.getDetailedBlocklet(blockletId);
    }
    int absoluteBlockletId = Integer.parseInt(blockletId);
    DataMapRow row = memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), absoluteBlockletId);
    short relativeBlockletId = row.getShort(BLOCKLET_ID_INDEX);
    String filePath = getFilePath();
    return createBlocklet(row, getFileNameWithFilePath(row, filePath), relativeBlockletId,
        false);
  }

  @Override
  protected short getBlockletId(DataMapRow dataMapRow) {
    if (isLegacyStore) {
      return super.getBlockletId(dataMapRow);
    }
    return dataMapRow.getShort(BLOCKLET_ID_INDEX);
  }

  protected boolean useMinMaxForExecutorPruning(FilterResolverIntf filterResolverIntf) {
    if (isLegacyStore) {
      return super.useMinMaxForExecutorPruning(filterResolverIntf);
    }
    return BlockletDataMapUtil
        .useMinMaxForBlockletPruning(filterResolverIntf, getMinMaxCacheColumns());
  }

  @Override
  protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId,
      boolean useMinMaxForPruning) {
    if (isLegacyStore) {
      return super.createBlocklet(row, fileName, blockletId, useMinMaxForPruning);
    }
    short versionNumber = row.getShort(VERSION_INDEX);
    ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + "",
        ColumnarFormatVersion.valueOf(versionNumber));
    blocklet.setColumnSchema(getColumnSchema());
    blocklet.setUseMinMaxForPruning(useMinMaxForPruning);
    blocklet.setIsBlockCache(false);
    blocklet.setColumnCardinality(getColumnCardinality());
    blocklet.setLegacyStore(isLegacyStore);
    blocklet.setDataMapRow(row);
    return blocklet;
  }

  @Override
  protected short getBlockletNumOfEntry(int index) {
    if (isLegacyStore) {
      return super.getBlockletNumOfEntry(index);
    } else {
      //in blocklet datamap, each entry contains info of one blocklet
      return 1;
    }
  }

  @Override public int getTotalBlocks() {
    if (isLegacyStore) {
      return super.getTotalBlocklets();
    } else {
      return blockNum;
    }
  }

  @Override
  protected int getTotalBlocklets() {
    if (isLegacyStore) {
      return super.getTotalBlocklets();
    } else {
      return memoryDMStore.getRowCount();
    }
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.indexstore.blockletindex;
18
19	import java.io.ByteArrayOutputStream;
20	import java.io.DataOutput;
21	import java.io.DataOutputStream;
22	import java.io.IOException;
23	import java.io.Serializable;
24	import java.util.Arrays;
25	import java.util.List;
26
27	import org.apache.carbondata.core.constants.CarbonCommonConstants;
28	import org.apache.carbondata.core.datamap.dev.DataMapModel;
29	import org.apache.carbondata.core.datastore.block.SegmentProperties;
30	import org.apache.carbondata.core.datastore.block.SegmentPropertiesAndSchemaHolder;
31	import org.apache.carbondata.core.datastore.block.TableBlockInfo;
32	import org.apache.carbondata.core.indexstore.BlockMetaInfo;
33	import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
34	import org.apache.carbondata.core.indexstore.row.DataMapRow;
35	import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
36	import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
37	import org.apache.carbondata.core.memory.MemoryException;
38	import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
39	import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
40	import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
41	import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
42	import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
43	import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
44	import org.apache.carbondata.core.util.BlockletDataMapUtil;
45
46	/**
47	* Datamap implementation for blocklet.
48	*/
49	public class BlockletDataMap extends BlockDataMap implements Serializable {	2✔
50
51	private static final long serialVersionUID = -2170289352240810993L;
52	// total block number in this datamap
53	private int blockNum = 0;	2✔
54
55	@Override
56	public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
57	super.init(dataMapModel);	1✔
58	}	1✔
59
60	/**
61	* Method to check the cache level and load metadata based on that information
62	*
63	* @param blockletDataMapInfo
64	* @param indexInfo
65	* @throws IOException
66	* @throws MemoryException
67	*/
68	@Override
69	protected DataMapRowImpl loadMetadata(CarbonRowSchema[] taskSummarySchema,
70	SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo,
71	List<DataFileFooter> indexInfo) throws IOException, MemoryException {
72	if (isLegacyStore) {	1✔
73	return loadBlockInfoForOldStore(taskSummarySchema, segmentProperties, blockletDataMapInfo,	×
74	indexInfo);
75	} else {
76	return loadBlockletMetaInfo(taskSummarySchema, segmentProperties, blockletDataMapInfo,	1✔
77	indexInfo);
78	}
79	}
80
81	@Override
82	protected CarbonRowSchema[] getTaskSummarySchema() {
83	if (isLegacyStore) {	1✔
84	return super.getTaskSummarySchema();	×
85	}
86	SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper segmentPropertiesWrapper =
87	SegmentPropertiesAndSchemaHolder.getInstance()	1✔
88	.getSegmentPropertiesWrapper(segmentPropertiesIndex);	1✔
89	try {
90	return segmentPropertiesWrapper.getTaskSummarySchemaForBlocklet(false, isFilePathStored);	1✔
91	} catch (MemoryException e) {	×
92	throw new RuntimeException(e);	×
93	}
94	}
95
96	@Override
97	protected CarbonRowSchema[] getFileFooterEntrySchema() {
98	if (isLegacyStore) {	1✔
99	return super.getFileFooterEntrySchema();	×
100	}
101	return SegmentPropertiesAndSchemaHolder.getInstance()	1✔
102	.getSegmentPropertiesWrapper(segmentPropertiesIndex).getBlockletFileFooterEntrySchema();	1✔
103	}
104
105	/**
106	* Method to load blocklet metadata information
107	*
108	* @param blockletDataMapInfo
109	* @param indexInfo
110	* @throws IOException
111	* @throws MemoryException
112	*/
113	private DataMapRowImpl loadBlockletMetaInfo(CarbonRowSchema[] taskSummarySchema,
114	SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo,
115	List<DataFileFooter> indexInfo) throws IOException, MemoryException {
116	String tempFilePath = null;	1✔
117	DataMapRowImpl summaryRow = null;	1✔
118	CarbonRowSchema[] schema = getFileFooterEntrySchema();	1✔
119	boolean[] summaryRowMinMaxFlag = new boolean[segmentProperties.getColumnsValueSize().length];	1✔
120	Arrays.fill(summaryRowMinMaxFlag, true);	1✔
121	// Relative blocklet ID is the id assigned to a blocklet within a part file
122	int relativeBlockletId = 0;	1✔
123	for (DataFileFooter fileFooter : indexInfo) {	1✔
124	// update the min max flag for summary row
125	updateMinMaxFlag(fileFooter, summaryRowMinMaxFlag);	1✔
126	TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();	1✔
127	BlockMetaInfo blockMetaInfo =	1✔
128	blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());	1✔
129	// Here it loads info about all blocklets of index
130	// Only add if the file exists physically. There are scenarios which index file exists inside
131	// merge index but related carbondata files are deleted. In that case we first check whether
132	// the file exists physically or not
133	if (blockMetaInfo != null) {	1✔
134	// this case is for CACHE_LEVEL = BLOCKLET
135	// blocklet ID will start from 0 again only when part file path is changed
136	if (null == tempFilePath \|\| !tempFilePath.equals(blockInfo.getFilePath())) {	1✔
137	tempFilePath = blockInfo.getFilePath();	1✔
138	relativeBlockletId = 0;	1✔
139	blockNum++;	1✔
140	}
141	summaryRow = loadToUnsafe(schema, taskSummarySchema, fileFooter, segmentProperties,	1✔
142	getMinMaxCacheColumns(), blockInfo.getFilePath(), summaryRow,	1✔
143	blockMetaInfo, relativeBlockletId);
144	// this is done because relative blocklet id need to be incremented based on the
145	// total number of blocklets
146	relativeBlockletId += fileFooter.getBlockletList().size();	1✔
147	}
148	}	1✔
149	summaryRow.setLong(0L, TASK_ROW_COUNT);	1✔
150	setMinMaxFlagForTaskSummary(summaryRow, taskSummarySchema, segmentProperties,	1✔
151	summaryRowMinMaxFlag);
152	return summaryRow;	1✔
153	}
154
155	private DataMapRowImpl loadToUnsafe(CarbonRowSchema[] schema, CarbonRowSchema[] taskSummarySchema,
156	DataFileFooter fileFooter, SegmentProperties segmentProperties,
157	List<CarbonColumn> minMaxCacheColumns, String filePath, DataMapRowImpl summaryRow,
158	BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
159	List<BlockletInfo> blockletList = fileFooter.getBlockletList();	1✔
160	// Add one row to maintain task level min max for segment pruning
161	if (!blockletList.isEmpty() && summaryRow == null) {	1✔
162	summaryRow = new DataMapRowImpl(taskSummarySchema);	1✔
163	}
164	for (int index = 0; index < blockletList.size(); index++) {	1✔
165	DataMapRow row = new DataMapRowImpl(schema);	1✔
166	int ordinal = 0;	1✔
167	int taskMinMaxOrdinal = 1;	1✔
168	BlockletInfo blockletInfo = blockletList.get(index);	1✔
169	blockletInfo.setSorted(fileFooter.isSorted());	1✔
170	BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();	1✔
171	// get min max values for columns to be cached
172	byte[][] minValuesForColumnsToBeCached = BlockletDataMapUtil	1✔
173	.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns,	1✔
174	minMaxIndex.getMinValues());	1✔
175	byte[][] maxValuesForColumnsToBeCached = BlockletDataMapUtil	1✔
176	.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns,	1✔
177	minMaxIndex.getMaxValues());	1✔
178	boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletDataMapUtil	1✔
179	.getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, minMaxCacheColumns,	1✔
180	fileFooter.getBlockletIndex().getMinMaxIndex().getIsMinMaxSet());	1✔
181	row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal);	1✔
182	// compute and set task level min values
183	addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal,	1✔
184	minValuesForColumnsToBeCached, TASK_MIN_VALUES_INDEX, true);
185	ordinal++;	1✔
186	taskMinMaxOrdinal++;	1✔
187	row.setRow(addMinMax(schema[ordinal], maxValuesForColumnsToBeCached), ordinal);	1✔
188	// compute and set task level max values
189	addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal,	1✔
190	maxValuesForColumnsToBeCached, TASK_MAX_VALUES_INDEX, false);
191	ordinal++;	1✔
192	row.setInt(blockletInfo.getNumberOfRows(), ordinal++);	1✔
193	// add file name
194	byte[] filePathBytes =	1✔
195	getFileNameFromPath(filePath).getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);	1✔
196	row.setByteArray(filePathBytes, ordinal++);	1✔
197	// add version number
198	row.setShort(fileFooter.getVersionId().number(), ordinal++);	1✔
199	// add schema updated time
200	row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);	1✔
201	byte[] serializedData;
202	try {
203	// Add block footer offset, it is used if we need to read footer of block
204	row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);	1✔
205	setLocations(blockMetaInfo.getLocationInfo(), row, ordinal++);	1✔
206	// Store block size
207	row.setLong(blockMetaInfo.getSize(), ordinal++);	1✔
208	// add min max flag for all the dimension columns
209	addMinMaxFlagValues(row, schema[ordinal], minMaxFlagValuesForColumnsToBeCached, ordinal);	1✔
210	ordinal++;	1✔
211	// add blocklet info
212	ByteArrayOutputStream stream = new ByteArrayOutputStream();	1✔
213	DataOutput dataOutput = new DataOutputStream(stream);	1✔
214	blockletInfo.write(dataOutput);	1✔
215	serializedData = stream.toByteArray();	1✔
216	row.setByteArray(serializedData, ordinal++);	1✔
217	// add pages
218	row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);	1✔
219	// for relative blocklet id i.e blocklet id that belongs to a particular carbondata file
220	row.setShort((short) relativeBlockletId++, ordinal);	1✔
221	memoryDMStore.addIndexRow(schema, row);	1✔
222	} catch (Exception e) {	×
223	throw new RuntimeException(e);	×
224	}	1✔
225	}
226	return summaryRow;	1✔
227	}
228
229	@Override
230	public ExtendedBlocklet getDetailedBlocklet(String blockletId) {
231	if (isLegacyStore) {	1✔
232	return super.getDetailedBlocklet(blockletId);	×
233	}
234	int absoluteBlockletId = Integer.parseInt(blockletId);	1✔
235	DataMapRow row = memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), absoluteBlockletId);	1✔
236	short relativeBlockletId = row.getShort(BLOCKLET_ID_INDEX);	1✔
237	String filePath = getFilePath();	1✔
238	return createBlocklet(row, getFileNameWithFilePath(row, filePath), relativeBlockletId,	1✔
239	false);
240	}
241
242	@Override
243	protected short getBlockletId(DataMapRow dataMapRow) {
244	if (isLegacyStore) {	1✔
245	return super.getBlockletId(dataMapRow);	×
246	}
247	return dataMapRow.getShort(BLOCKLET_ID_INDEX);	1✔
248	}
249
250	protected boolean useMinMaxForExecutorPruning(FilterResolverIntf filterResolverIntf) {
251	if (isLegacyStore) {	1✔
252	return super.useMinMaxForExecutorPruning(filterResolverIntf);	×
253	}
254	return BlockletDataMapUtil	1✔
255	.useMinMaxForBlockletPruning(filterResolverIntf, getMinMaxCacheColumns());	1✔
256	}
257
258	@Override
259	protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId,
260	boolean useMinMaxForPruning) {
261	if (isLegacyStore) {	1✔
262	return super.createBlocklet(row, fileName, blockletId, useMinMaxForPruning);	×
263	}
264	short versionNumber = row.getShort(VERSION_INDEX);	1✔
265	ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + "",	1✔
266	ColumnarFormatVersion.valueOf(versionNumber));	1✔
267	blocklet.setColumnSchema(getColumnSchema());	1✔
268	blocklet.setUseMinMaxForPruning(useMinMaxForPruning);	1✔
269	blocklet.setIsBlockCache(false);	1✔
270	blocklet.setColumnCardinality(getColumnCardinality());	1✔
271	blocklet.setLegacyStore(isLegacyStore);	1✔
272	blocklet.setDataMapRow(row);	1✔
273	return blocklet;	1✔
274	}
275
276	@Override
277	protected short getBlockletNumOfEntry(int index) {
278	if (isLegacyStore) {	1✔
279	return super.getBlockletNumOfEntry(index);	×
280	} else {
281	//in blocklet datamap, each entry contains info of one blocklet
282	return 1;	1✔
283	}
284	}
285
286	@Override public int getTotalBlocks() {
287	if (isLegacyStore) {	1✔
288	return super.getTotalBlocklets();	×
289	} else {
290	return blockNum;	1✔
291	}
292	}
293
294	@Override
295	protected int getTotalBlocklets() {
296	if (isLegacyStore) {	1✔
297	return super.getTotalBlocklets();	×
298	} else {
299	return memoryDMStore.getRowCount();	1✔
300	}
301	}
302
303	}

apache / carbondata / 3031

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous