10200

Build # 10200

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/3df24f7d2e648c45fc4eca9248c154fc11df4f2a">3df24f7d2<a href="https://github.com/apache/carbondata/commit/3df24f7d2e648c45fc4eca9248c154fc11df4f2a">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/349e901c8f5c8d658859089ab3acaf6377107150">349e901c8</a>">349e901c8</a><a href="https://github.com/apache/carbondata/commit/3df24f7d2e648c45fc4eca9248c154fc11df4f2a"> into 128a6c867">128a6c867</a>

Run Details

56620 of 77859 relevant lines covered (72.72%)

0.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRestructureBasedRawResultCollector.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.scan.collector.impl;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;

/**
 * It is not a collector it is just a scanned result holder.
 * This class returns all the dimensions in a ByteArrayWrapper and append
 * blockletNo/PageId/RowId at end of the row.
 */
@InterfaceAudience.Internal
public class RowIdRestructureBasedRawResultCollector extends RestructureBasedRawResultCollector {

  public RowIdRestructureBasedRawResultCollector(BlockExecutionInfo blockExecutionInfos) {
    super(blockExecutionInfos);
  }

  @Override
  protected void scanAndFillData(BlockletScannedResult scannedResult, int batchSize,
                               List<Object[]> listBasedResult, ProjectionMeasure[] queryMeasures) {
    int numberOfPages = scannedResult.numberOfpages();
    // loop will exit once the batchSize data has been read or the pages have been exhausted
    while (scannedResult.getCurrentPageCounter() < numberOfPages) {
      int currentPageRowCount = scannedResult.getCurrentPageRowCount();
      if (currentPageRowCount == 0) {
        scannedResult.incrementPageCounter();
        continue;
      }
      int rowCounter = scannedResult.getRowCounter();
      // getRowCounter holds total number rows processed. Calculate the
      // Left over space through getRowCounter only.
      int availableRows = currentPageRowCount - rowCounter;
      // rows available in current page that can be processed from current page
      int availableBatchRowCount = Math.min(batchSize, availableRows);
      // this condition will be true if no data left in the current block/blocklet to be scanned
      if (availableBatchRowCount < 1) {
        break;
      }
      if (batchSize > availableRows) {
        batchSize = batchSize - availableRows;
      } else {
        // this is done because in IUD cases actuals rows fetch can be less than batch size as
        // some of the rows could have deleted. So in those cases batchSize need to be
        // re initialized with left over value
        batchSize = 0;
      }
      // for every iteration of available rows filling newly created list of Object[] and add it to
      // the final list so there is no mismatch in the counter while filling dimension and
      // measure data
      List<Object[]> collectedData = new ArrayList<>(availableBatchRowCount);
      // fill dimension data
      fillDimensionData(scannedResult, collectedData, queryMeasures, availableBatchRowCount);
      fillMeasureData(scannedResult, collectedData);
      // increment the number of rows scanned in scanned result statistics
      // incrementScannedResultRowCounter(scannedResult, availableBatchRowCount);
      // assign the left over rows to batch size if the number of rows fetched are lesser
      // than batchSize
      if (collectedData.size() < availableBatchRowCount) {
        batchSize += availableBatchRowCount - listBasedResult.size();
      }
      // add the collected data to the final list
      listBasedResult.addAll(collectedData);
    }
  }

  private void fillDimensionData(BlockletScannedResult scannedResult,
               List<Object[]> listBasedResult, ProjectionMeasure[] queryMeasures, int batchSize) {
    long startTime = System.currentTimeMillis();
    List<byte[]> dictionaryKeyArrayBatch = scannedResult.getDictionaryKeyArrayBatch(batchSize);
    List<byte[][]> noDictionaryKeyArrayBatch =
        scannedResult.getNoDictionaryKeyArrayBatch(batchSize);
    List<byte[][]> complexTypeKeyArrayBatch = scannedResult.getComplexTypeKeyArrayBatch(batchSize);
    // it will same for one blocklet so can be computed only once
    byte[] implicitColumnByteArray = scannedResult.getBlockletId()
        .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
    // Note: size check in for loop is for dictionaryKeyArrayBatch as this size can be lesser than
    // batch size in case of IUD scenarios
    for (int i = 0; i < dictionaryKeyArrayBatch.size(); i++) {
      // 1 for ByteArrayWrapper object which will contain dictionary and no dictionary data
      // 3 for blockletId, pageId, rowId
      Object[] row = new Object[1 + queryMeasures.length + 3];
      scannedResult.incrementCounter();
      row[1 + queryMeasures.length] = scannedResult.getBlockletNumber();
      row[1 + queryMeasures.length + 1] = scannedResult.getCurrentPageCounter();
      ByteArrayWrapper wrapper = new ByteArrayWrapper();
      wrapper.setDictionaryKey(dictionaryKeyArrayBatch.get(i));
      wrapper.setNoDictionaryKeys(noDictionaryKeyArrayBatch.get(i));
      wrapper.setComplexTypesKeys(complexTypeKeyArrayBatch.get(i));
      wrapper.setImplicitColumnByteArray(implicitColumnByteArray);
      row[0] = wrapper;
      row[1 + queryMeasures.length + 2] = scannedResult.getCurrentRowId();
      listBasedResult.add(row);
    }
    QueryStatistic keyColumnFillingTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
        .get(QueryStatisticsConstants.KEY_COLUMN_FILLING_TIME);
    keyColumnFillingTime.addCountStatistic(QueryStatisticsConstants.KEY_COLUMN_FILLING_TIME,
        keyColumnFillingTime.getCount() + (System.currentTimeMillis() - startTime));
  }

  private void fillMeasureData(BlockletScannedResult scannedResult,
                               List<Object[]> listBasedResult) {
    long startTime = System.currentTimeMillis();
    // if list is not empty after filling the dimension data then only fill the measure data
    if (!listBasedResult.isEmpty()) {
      fillMeasureDataBatch(listBasedResult, 1, scannedResult);
    }
    QueryStatistic measureFillingTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
        .get(QueryStatisticsConstants.MEASURE_FILLING_TIME);
    measureFillingTime.addCountStatistic(QueryStatisticsConstants.MEASURE_FILLING_TIME,
        measureFillingTime.getCount() + (System.currentTimeMillis() - startTime));
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.scan.collector.impl;
18
19	import java.nio.charset.Charset;
20	import java.util.ArrayList;
21	import java.util.List;
22
23	import org.apache.carbondata.common.annotations.InterfaceAudience;
24	import org.apache.carbondata.core.constants.CarbonCommonConstants;
25	import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
26	import org.apache.carbondata.core.scan.model.ProjectionMeasure;
27	import org.apache.carbondata.core.scan.result.BlockletScannedResult;
28	import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
29	import org.apache.carbondata.core.stats.QueryStatistic;
30	import org.apache.carbondata.core.stats.QueryStatisticsConstants;
31
32	/**
33	* It is not a collector it is just a scanned result holder.
34	* This class returns all the dimensions in a ByteArrayWrapper and append
35	* blockletNo/PageId/RowId at end of the row.
36	*/
37	@InterfaceAudience.Internal
38	public class RowIdRestructureBasedRawResultCollector extends RestructureBasedRawResultCollector {
39
40	public RowIdRestructureBasedRawResultCollector(BlockExecutionInfo blockExecutionInfos) {
41	super(blockExecutionInfos);	×
42	}	×
43
44	@Override
45	protected void scanAndFillData(BlockletScannedResult scannedResult, int batchSize,
46	List<Object[]> listBasedResult, ProjectionMeasure[] queryMeasures) {
47	int numberOfPages = scannedResult.numberOfpages();	×
48	// loop will exit once the batchSize data has been read or the pages have been exhausted
49	while (scannedResult.getCurrentPageCounter() < numberOfPages) {	×
50	int currentPageRowCount = scannedResult.getCurrentPageRowCount();	×
51	if (currentPageRowCount == 0) {	×
52	scannedResult.incrementPageCounter();	×
53	continue;	×
54	}
55	int rowCounter = scannedResult.getRowCounter();	×
56	// getRowCounter holds total number rows processed. Calculate the
57	// Left over space through getRowCounter only.
58	int availableRows = currentPageRowCount - rowCounter;	×
59	// rows available in current page that can be processed from current page
60	int availableBatchRowCount = Math.min(batchSize, availableRows);	×
61	// this condition will be true if no data left in the current block/blocklet to be scanned
62	if (availableBatchRowCount < 1) {	×
63	break;	×
64	}
65	if (batchSize > availableRows) {	×
66	batchSize = batchSize - availableRows;	×
67	} else {
68	// this is done because in IUD cases actuals rows fetch can be less than batch size as
69	// some of the rows could have deleted. So in those cases batchSize need to be
70	// re initialized with left over value
71	batchSize = 0;	×
72	}
73	// for every iteration of available rows filling newly created list of Object[] and add it to
74	// the final list so there is no mismatch in the counter while filling dimension and
75	// measure data
76	List<Object[]> collectedData = new ArrayList<>(availableBatchRowCount);	×
77	// fill dimension data
78	fillDimensionData(scannedResult, collectedData, queryMeasures, availableBatchRowCount);	×
79	fillMeasureData(scannedResult, collectedData);	×
80	// increment the number of rows scanned in scanned result statistics
81	// incrementScannedResultRowCounter(scannedResult, availableBatchRowCount);
82	// assign the left over rows to batch size if the number of rows fetched are lesser
83	// than batchSize
84	if (collectedData.size() < availableBatchRowCount) {	×
85	batchSize += availableBatchRowCount - listBasedResult.size();	×
86	}
87	// add the collected data to the final list
88	listBasedResult.addAll(collectedData);	×
89	}	×
90	}	×
91
92	private void fillDimensionData(BlockletScannedResult scannedResult,
93	List<Object[]> listBasedResult, ProjectionMeasure[] queryMeasures, int batchSize) {
94	long startTime = System.currentTimeMillis();	×
95	List<byte[]> dictionaryKeyArrayBatch = scannedResult.getDictionaryKeyArrayBatch(batchSize);	×
96	List<byte[][]> noDictionaryKeyArrayBatch =	×
97	scannedResult.getNoDictionaryKeyArrayBatch(batchSize);	×
98	List<byte[][]> complexTypeKeyArrayBatch = scannedResult.getComplexTypeKeyArrayBatch(batchSize);	×
99	// it will same for one blocklet so can be computed only once
100	byte[] implicitColumnByteArray = scannedResult.getBlockletId()	×
101	.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));	×
102	// Note: size check in for loop is for dictionaryKeyArrayBatch as this size can be lesser than
103	// batch size in case of IUD scenarios
104	for (int i = 0; i < dictionaryKeyArrayBatch.size(); i++) {	×
105	// 1 for ByteArrayWrapper object which will contain dictionary and no dictionary data
106	// 3 for blockletId, pageId, rowId
107	Object[] row = new Object[1 + queryMeasures.length + 3];	×
108	scannedResult.incrementCounter();	×
109	row[1 + queryMeasures.length] = scannedResult.getBlockletNumber();	×
110	row[1 + queryMeasures.length + 1] = scannedResult.getCurrentPageCounter();	×
111	ByteArrayWrapper wrapper = new ByteArrayWrapper();	×
112	wrapper.setDictionaryKey(dictionaryKeyArrayBatch.get(i));	×
113	wrapper.setNoDictionaryKeys(noDictionaryKeyArrayBatch.get(i));	×
114	wrapper.setComplexTypesKeys(complexTypeKeyArrayBatch.get(i));	×
115	wrapper.setImplicitColumnByteArray(implicitColumnByteArray);	×
116	row[0] = wrapper;	×
117	row[1 + queryMeasures.length + 2] = scannedResult.getCurrentRowId();	×
118	listBasedResult.add(row);	×
119	}
120	QueryStatistic keyColumnFillingTime = queryStatisticsModel.getStatisticsTypeAndObjMap()	×
121	.get(QueryStatisticsConstants.KEY_COLUMN_FILLING_TIME);	×
122	keyColumnFillingTime.addCountStatistic(QueryStatisticsConstants.KEY_COLUMN_FILLING_TIME,	×
123	keyColumnFillingTime.getCount() + (System.currentTimeMillis() - startTime));	×
124	}	×
125
126	private void fillMeasureData(BlockletScannedResult scannedResult,
127	List<Object[]> listBasedResult) {
128	long startTime = System.currentTimeMillis();	×
129	// if list is not empty after filling the dimension data then only fill the measure data
130	if (!listBasedResult.isEmpty()) {	×
131	fillMeasureDataBatch(listBasedResult, 1, scannedResult);	×
132	}
133	QueryStatistic measureFillingTime = queryStatisticsModel.getStatisticsTypeAndObjMap()	×
134	.get(QueryStatisticsConstants.MEASURE_FILLING_TIME);	×
135	measureFillingTime.addCountStatistic(QueryStatisticsConstants.MEASURE_FILLING_TIME,	×
136	measureFillingTime.getCount() + (System.currentTimeMillis() - startTime));	×
137	}	×
138	}

apache / carbondata / 10200

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous