9324

Build # 9324

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/88d5a146d58a2ac3a904ad6d8625a868234b94a1">88d5a146d<a href="https://github.com/apache/carbondata/commit/88d5a146d58a2ac3a904ad6d8625a868234b94a1">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/69b633f6c291e1dcda5c33872e6f3db63e1064a6">69b633f6c</a>">69b633f6c</a><a href="https://github.com/apache/carbondata/commit/88d5a146d58a2ac3a904ad6d8625a868234b94a1"> into 019f5cd06">019f5cd06</a>

Run Details

288 of 288 new or added lines in 15 files covered. (100.0%)

61049 of 77085 relevant lines covered (79.2%)

1.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.2

/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyBlockletLoader.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.scan.scanner;

import java.io.IOException;

import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
import org.apache.carbondata.core.stats.QueryStatisticsModel;

/**
 * Reads the blocklet column chunks lazily, it means it reads the column chunks from disk when
 * execution engine wants to access it.
 * It is useful in case of filter queries with high cardinality columns.
 */
public class LazyBlockletLoader {

  private RawBlockletColumnChunks rawBlockletColumnChunks;

  private BlockExecutionInfo blockExecutionInfo;

  private LazyChunkWrapper[] dimLazyWrapperChunks;

  private LazyChunkWrapper[] msrLazyWrapperChunks;

  private boolean isLoaded;

  private QueryStatisticsModel queryStatisticsModel;

  public LazyBlockletLoader(RawBlockletColumnChunks rawBlockletColumnChunks,
      BlockExecutionInfo blockExecutionInfo, DimensionRawColumnChunk[] dimensionRawColumnChunks,
      MeasureRawColumnChunk[] measureRawColumnChunks, QueryStatisticsModel queryStatisticsModel) {
    this.rawBlockletColumnChunks = rawBlockletColumnChunks;
    this.blockExecutionInfo = blockExecutionInfo;
    this.dimLazyWrapperChunks = new LazyChunkWrapper[dimensionRawColumnChunks.length];
    this.msrLazyWrapperChunks = new LazyChunkWrapper[measureRawColumnChunks.length];
    for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
      dimLazyWrapperChunks[i] = new LazyChunkWrapper(dimensionRawColumnChunks[i]);
    }
    for (int i = 0; i < measureRawColumnChunks.length; i++) {
      msrLazyWrapperChunks[i] = new LazyChunkWrapper(measureRawColumnChunks[i]);
    }
    this.queryStatisticsModel = queryStatisticsModel;
  }

  public void load() throws IOException {
    if (!isLoaded) {
      readBlocklet();
    }
  }

  public LazyChunkWrapper getLazyChunkWrapper(int index, boolean isMeasure) {
    if (isMeasure) {
      return msrLazyWrapperChunks[index];
    } else {
      return dimLazyWrapperChunks[index];
    }
  }

  private synchronized void readBlocklet() throws IOException {
    FileReader fileReader = rawBlockletColumnChunks.getFileReader();

    long readTime = System.currentTimeMillis();
    int[][] allSelectedDimensionColumnIndexRange =
        blockExecutionInfo.getAllSelectedDimensionColumnIndexRange();
    DimensionRawColumnChunk[] projectionListDimensionChunk = rawBlockletColumnChunks.getDataBlock()
        .readDimensionChunks(fileReader, allSelectedDimensionColumnIndexRange);
    for (int[] columnIndexRange : allSelectedDimensionColumnIndexRange) {
      for (int i = columnIndexRange[0]; i < columnIndexRange[1] + 1; i++) {
        dimLazyWrapperChunks[i].rawColumnChunk = projectionListDimensionChunk[i];
      }
    }
    /*
     * in case projection if the projected dimension are not loaded in the dimensionColumnDataChunk
     * then loading them
     */
    int[] projectionListDimensionIndexes = blockExecutionInfo.getProjectionListDimensionIndexes();
    for (int projectionListDimensionIndex : projectionListDimensionIndexes) {
      if (null == dimLazyWrapperChunks[projectionListDimensionIndex].rawColumnChunk) {
        dimLazyWrapperChunks[projectionListDimensionIndex].rawColumnChunk =
            rawBlockletColumnChunks.getDataBlock()
                .readDimensionChunk(fileReader, projectionListDimensionIndex);
      }
    }


    int[][] allSelectedMeasureColumnIndexRange =
        blockExecutionInfo.getAllSelectedMeasureIndexRange();
    MeasureRawColumnChunk[] projectionListMeasureChunk = rawBlockletColumnChunks.getDataBlock()
        .readMeasureChunks(fileReader, allSelectedMeasureColumnIndexRange);
    for (int[] columnIndexRange : allSelectedMeasureColumnIndexRange) {
      for (int i = columnIndexRange[0]; i < columnIndexRange[1] + 1; i++) {
        msrLazyWrapperChunks[i].rawColumnChunk = projectionListMeasureChunk[i];
      }
    }
    /*
     * in case projection if the projected measure are not loaded in the ColumnPage
     * then loading them
     */
    int[] projectionListMeasureIndexes = blockExecutionInfo.getProjectionListMeasureIndexes();
    for (int projectionListMeasureIndex : projectionListMeasureIndexes) {
      if (null == msrLazyWrapperChunks[projectionListMeasureIndex].rawColumnChunk) {
        msrLazyWrapperChunks[projectionListMeasureIndex].rawColumnChunk =
            rawBlockletColumnChunks.getDataBlock()
                .readMeasureChunk(fileReader, projectionListMeasureIndex);
      }
    }
    readTime = System.currentTimeMillis() - readTime;
    QueryStatistic time = queryStatisticsModel.getStatisticsTypeAndObjMap()
        .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
    time.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
        time.getCount() + readTime);
    isLoaded = true;
  }

  public QueryStatisticsModel getQueryStatisticsModel() {
    return queryStatisticsModel;
  }

  public static class LazyChunkWrapper {

    private AbstractRawColumnChunk rawColumnChunk;

    public LazyChunkWrapper(AbstractRawColumnChunk rawColumnChunk) {
      this.rawColumnChunk = rawColumnChunk;
    }

    public AbstractRawColumnChunk getRawColumnChunk() {
      return rawColumnChunk;
    }

    public void setRawColumnChunk(AbstractRawColumnChunk rawColumnChunk) {
      this.rawColumnChunk = rawColumnChunk;
    }
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.scan.scanner;
19
20	import java.io.IOException;
21
22	import org.apache.carbondata.core.datastore.FileReader;
23	import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
24	import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
25	import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
26	import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
27	import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
28	import org.apache.carbondata.core.stats.QueryStatistic;
29	import org.apache.carbondata.core.stats.QueryStatisticsConstants;
30	import org.apache.carbondata.core.stats.QueryStatisticsModel;
31
32	/**
33	* Reads the blocklet column chunks lazily, it means it reads the column chunks from disk when
34	* execution engine wants to access it.
35	* It is useful in case of filter queries with high cardinality columns.
36	*/
37	public class LazyBlockletLoader {
38
39	private RawBlockletColumnChunks rawBlockletColumnChunks;
40
41	private BlockExecutionInfo blockExecutionInfo;
42
43	private LazyChunkWrapper[] dimLazyWrapperChunks;
44
45	private LazyChunkWrapper[] msrLazyWrapperChunks;
46
47	private boolean isLoaded;
48
49	private QueryStatisticsModel queryStatisticsModel;
50
51	public LazyBlockletLoader(RawBlockletColumnChunks rawBlockletColumnChunks,
52	BlockExecutionInfo blockExecutionInfo, DimensionRawColumnChunk[] dimensionRawColumnChunks,
53	MeasureRawColumnChunk[] measureRawColumnChunks, QueryStatisticsModel queryStatisticsModel) {	1✔
54	this.rawBlockletColumnChunks = rawBlockletColumnChunks;	1✔
55	this.blockExecutionInfo = blockExecutionInfo;	1✔
56	this.dimLazyWrapperChunks = new LazyChunkWrapper[dimensionRawColumnChunks.length];	1✔
57	this.msrLazyWrapperChunks = new LazyChunkWrapper[measureRawColumnChunks.length];	1✔
58	for (int i = 0; i < dimensionRawColumnChunks.length; i++) {	1✔
59	dimLazyWrapperChunks[i] = new LazyChunkWrapper(dimensionRawColumnChunks[i]);	1✔
60	}
61	for (int i = 0; i < measureRawColumnChunks.length; i++) {	1✔
62	msrLazyWrapperChunks[i] = new LazyChunkWrapper(measureRawColumnChunks[i]);	1✔
63	}
64	this.queryStatisticsModel = queryStatisticsModel;	1✔
65	}	1✔
66
67	public void load() throws IOException {
68	if (!isLoaded) {	×
69	readBlocklet();	×
70	}
71	}	×
72
73	public LazyChunkWrapper getLazyChunkWrapper(int index, boolean isMeasure) {
74	if (isMeasure) {	1✔
75	return msrLazyWrapperChunks[index];	1✔
76	} else {
77	return dimLazyWrapperChunks[index];	1✔
78	}
79	}
80
81	private synchronized void readBlocklet() throws IOException {
82	FileReader fileReader = rawBlockletColumnChunks.getFileReader();	×
83
84	long readTime = System.currentTimeMillis();	×
85	int[][] allSelectedDimensionColumnIndexRange =	×
86	blockExecutionInfo.getAllSelectedDimensionColumnIndexRange();	×
87	DimensionRawColumnChunk[] projectionListDimensionChunk = rawBlockletColumnChunks.getDataBlock()	×
88	.readDimensionChunks(fileReader, allSelectedDimensionColumnIndexRange);	×
89	for (int[] columnIndexRange : allSelectedDimensionColumnIndexRange) {	×
90	for (int i = columnIndexRange[0]; i < columnIndexRange[1] + 1; i++) {	×
91	dimLazyWrapperChunks[i].rawColumnChunk = projectionListDimensionChunk[i];	×
92	}
93	}
94	/*
95	* in case projection if the projected dimension are not loaded in the dimensionColumnDataChunk
96	* then loading them
97	*/
98	int[] projectionListDimensionIndexes = blockExecutionInfo.getProjectionListDimensionIndexes();	×
99	for (int projectionListDimensionIndex : projectionListDimensionIndexes) {	×
100	if (null == dimLazyWrapperChunks[projectionListDimensionIndex].rawColumnChunk) {	×
101	dimLazyWrapperChunks[projectionListDimensionIndex].rawColumnChunk =	×
102	rawBlockletColumnChunks.getDataBlock()	×
103	.readDimensionChunk(fileReader, projectionListDimensionIndex);	×
104	}
105	}
106
107
108	int[][] allSelectedMeasureColumnIndexRange =	×
109	blockExecutionInfo.getAllSelectedMeasureIndexRange();	×
110	MeasureRawColumnChunk[] projectionListMeasureChunk = rawBlockletColumnChunks.getDataBlock()	×
111	.readMeasureChunks(fileReader, allSelectedMeasureColumnIndexRange);	×
112	for (int[] columnIndexRange : allSelectedMeasureColumnIndexRange) {	×
113	for (int i = columnIndexRange[0]; i < columnIndexRange[1] + 1; i++) {	×
114	msrLazyWrapperChunks[i].rawColumnChunk = projectionListMeasureChunk[i];	×
115	}
116	}
117	/*
118	* in case projection if the projected measure are not loaded in the ColumnPage
119	* then loading them
120	*/
121	int[] projectionListMeasureIndexes = blockExecutionInfo.getProjectionListMeasureIndexes();	×
122	for (int projectionListMeasureIndex : projectionListMeasureIndexes) {	×
123	if (null == msrLazyWrapperChunks[projectionListMeasureIndex].rawColumnChunk) {	×
124	msrLazyWrapperChunks[projectionListMeasureIndex].rawColumnChunk =	×
125	rawBlockletColumnChunks.getDataBlock()	×
126	.readMeasureChunk(fileReader, projectionListMeasureIndex);	×
127	}
128	}
129	readTime = System.currentTimeMillis() - readTime;	×
130	QueryStatistic time = queryStatisticsModel.getStatisticsTypeAndObjMap()	×
131	.get(QueryStatisticsConstants.READ_BLOCKlET_TIME);	×
132	time.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,	×
133	time.getCount() + readTime);	×
134	isLoaded = true;	×
135	}	×
136
137	public QueryStatisticsModel getQueryStatisticsModel() {
138	return queryStatisticsModel;	1✔
139	}
140
141	public static class LazyChunkWrapper {
142
143	private AbstractRawColumnChunk rawColumnChunk;
144
145	public LazyChunkWrapper(AbstractRawColumnChunk rawColumnChunk) {	1✔
146	this.rawColumnChunk = rawColumnChunk;	1✔
147	}	1✔
148
149	public AbstractRawColumnChunk getRawColumnChunk() {
150	return rawColumnChunk;	1✔
151	}
152
153	public void setRawColumnChunk(AbstractRawColumnChunk rawColumnChunk) {
154	this.rawColumnChunk = rawColumnChunk;	×
155	}	×
156	}
157
158	}

apache / carbondata / 9324

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous