8301

Build # 8301

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/889e93a75f342c47b976022f44f828e171300e71">889e93a75<a href="https://github.com/apache/carbondata/commit/889e93a75f342c47b976022f44f828e171300e71">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/10ccff8d4a6c3d14aa5feef7251e1b155dc4d0c5">10ccff8d4</a>">10ccff8d4</a><a href="https://github.com/apache/carbondata/commit/889e93a75f342c47b976022f44f828e171300e71"> into 526e3bfa1">526e3bfa1</a>

Run Details

588 of 588 new or added lines in 63 files covered. (100.0%)

55401 of 71643 relevant lines covered (77.33%)

1.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.29

/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.datastore.chunk.impl;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
import org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
import org.apache.carbondata.core.scan.result.vector.impl.CarbonDictionaryImpl;
import org.apache.carbondata.format.Encoding;
import org.apache.carbondata.format.LocalDictionaryChunk;

/**
 * Contains raw dimension data,
 * 1. The read uncompressed raw data of column chunk with all pages is stored in this instance.
 * 2. The raw data can be converted to processed chunk using decodeColumnPage method
 *  by specifying page number.
 */
public class DimensionRawColumnChunk extends AbstractRawColumnChunk {

  private DimensionColumnPage[] dataChunks;

  private DimensionColumnChunkReader chunkReader;

  private FileReader fileReader;

  private CarbonDictionary localDictionary;

  public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
      DimensionColumnChunkReader columnChunkReader) {
    super(columnIndex, rawData, offSet, length);
    this.chunkReader = columnChunkReader;
  }

  /**
   * Convert all raw data with all pages to processed DimensionColumnPage's
   * @return
   */
  public DimensionColumnPage[] decodeAllColumnPages() {
    if (dataChunks == null) {
      dataChunks = new DimensionColumnPage[pagesCount];
    }
    for (int i = 0; i < pagesCount; i++) {
      try {
        if (dataChunks[i] == null) {
          dataChunks[i] = chunkReader.decodeColumnPage(this, i);
        }
      } catch (IOException | MemoryException e) {
        throw new RuntimeException(e);
      }
    }
    return dataChunks;
  }

  /**
   * Convert raw data with specified page number processed to DimensionColumnPage
   * @param pageNumber
   * @return
   */
  public DimensionColumnPage decodeColumnPage(int pageNumber) {
    assert pageNumber < pagesCount;
    if (dataChunks == null) {
      dataChunks = new DimensionColumnPage[pagesCount];
    }
    if (dataChunks[pageNumber] == null) {
      try {
        dataChunks[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber);
      } catch (IOException | MemoryException e) {
        throw new RuntimeException(e);
      }
    }

    return dataChunks[pageNumber];
  }

  /**
   * Convert raw data with specified page number processed to DimensionColumnDataChunk
   *
   * @param index
   * @return
   */
  public DimensionColumnPage convertToDimColDataChunkWithOutCache(int index) {
    assert index < pagesCount;
    // in case of filter query filter column if filter column is decoded and stored.
    // then return the same
    if (dataChunks != null && null != dataChunks[index]) {
      return dataChunks[index];
    }
    try {
      return chunkReader.decodeColumnPage(this, index);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  @Override public void freeMemory() {
    super.freeMemory();
    if (null != dataChunks) {
      for (int i = 0; i < dataChunks.length; i++) {
        if (dataChunks[i] != null) {
          dataChunks[i].freeMemory();
          dataChunks[i] = null;
        }
      }
    }
    rawData = null;
  }

  public void setFileReader(FileReader fileReader) {
    this.fileReader = fileReader;
  }

  public FileReader getFileReader() {
    return fileReader;
  }

  public CarbonDictionary getLocalDictionary() {
    if (null != getDataChunkV3() && null != getDataChunkV3().local_dictionary
        && null == localDictionary) {
      try {
        String compressorName =
            getDataChunkV3().data_chunk_list.get(0).chunk_meta.getCompression_codec().name();
        Compressor compressor = CompressorFactory.getInstance().getCompressor(compressorName);
        localDictionary = getDictionary(getDataChunkV3().local_dictionary, compressor);
      } catch (IOException | MemoryException e) {
        throw new RuntimeException(e);
      }
    }
    return localDictionary;
  }

  /**
   * Below method will be used to get the local dictionary for a blocklet
   * @param localDictionaryChunk
   * local dictionary chunk thrift object
   * @return local dictionary
   * @throws IOException
   * @throws MemoryException
   */
  private CarbonDictionary getDictionary(LocalDictionaryChunk localDictionaryChunk,
      Compressor compressor) throws IOException, MemoryException {
    if (null != localDictionaryChunk) {
      List<Encoding> encodings = localDictionaryChunk.getDictionary_meta().getEncoders();
      List<ByteBuffer> encoderMetas = localDictionaryChunk.getDictionary_meta().getEncoder_meta();
      ColumnPageDecoder decoder = DefaultEncodingFactory.getInstance().createDecoder(
          encodings, encoderMetas, compressor.getName());
      ColumnPage decode = decoder.decode(localDictionaryChunk.getDictionary_data(), 0,
          localDictionaryChunk.getDictionary_data().length);
      BitSet usedDictionary = BitSet.valueOf(compressor.unCompressByte(
          localDictionaryChunk.getDictionary_values()));
      int length = usedDictionary.length();
      int index = 0;
      byte[][] dictionary = new byte[length][];
      for (int i = 0; i < length; i++) {
        if (usedDictionary.get(i)) {
          dictionary[i] = decode.getBytes(index++);
        } else {
          dictionary[i] = null;
        }
      }
      decode.freeMemory();
      // as dictionary values starts from 1 setting null default value
      dictionary[1] = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
      return new CarbonDictionaryImpl(dictionary, usedDictionary.cardinality());
    }
    return null;
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.datastore.chunk.impl;
18
19	import java.io.IOException;
20	import java.nio.ByteBuffer;
21	import java.util.BitSet;
22	import java.util.List;
23
24	import org.apache.carbondata.core.constants.CarbonCommonConstants;
25	import org.apache.carbondata.core.datastore.FileReader;
26	import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
27	import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
28	import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
29	import org.apache.carbondata.core.datastore.compression.Compressor;
30	import org.apache.carbondata.core.datastore.compression.CompressorFactory;
31	import org.apache.carbondata.core.datastore.page.ColumnPage;
32	import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
33	import org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingFactory;
34	import org.apache.carbondata.core.memory.MemoryException;
35	import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
36	import org.apache.carbondata.core.scan.result.vector.impl.CarbonDictionaryImpl;
37	import org.apache.carbondata.format.Encoding;
38	import org.apache.carbondata.format.LocalDictionaryChunk;
39
40	/**
41	* Contains raw dimension data,
42	* 1. The read uncompressed raw data of column chunk with all pages is stored in this instance.
43	* 2. The raw data can be converted to processed chunk using decodeColumnPage method
44	* by specifying page number.
45	*/
46	public class DimensionRawColumnChunk extends AbstractRawColumnChunk {	1✔
47
48	private DimensionColumnPage[] dataChunks;
49
50	private DimensionColumnChunkReader chunkReader;
51
52	private FileReader fileReader;
53
54	private CarbonDictionary localDictionary;
55
56	public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
57	DimensionColumnChunkReader columnChunkReader) {
58	super(columnIndex, rawData, offSet, length);	1✔
59	this.chunkReader = columnChunkReader;	1✔
60	}	1✔
61
62	/**
63	* Convert all raw data with all pages to processed DimensionColumnPage's
64	* @return
65	*/
66	public DimensionColumnPage[] decodeAllColumnPages() {
67	if (dataChunks == null) {	1✔
68	dataChunks = new DimensionColumnPage[pagesCount];	1✔
69	}
70	for (int i = 0; i < pagesCount; i++) {	1✔
71	try {
72	if (dataChunks[i] == null) {	1✔
73	dataChunks[i] = chunkReader.decodeColumnPage(this, i);	1✔
74	}
75	} catch (IOException \| MemoryException e) {	×
76	throw new RuntimeException(e);	×
77	}	1✔
78	}
79	return dataChunks;	1✔
80	}
81
82	/**
83	* Convert raw data with specified page number processed to DimensionColumnPage
84	* @param pageNumber
85	* @return
86	*/
87	public DimensionColumnPage decodeColumnPage(int pageNumber) {
88	assert pageNumber < pagesCount;	1✔
89	if (dataChunks == null) {	1✔
90	dataChunks = new DimensionColumnPage[pagesCount];	1✔
91	}
92	if (dataChunks[pageNumber] == null) {	1✔
93	try {
94	dataChunks[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber);	1✔
95	} catch (IOException \| MemoryException e) {	×
96	throw new RuntimeException(e);	×
97	}	1✔
98	}
99
100	return dataChunks[pageNumber];	1✔
101	}
102
103	/**
104	* Convert raw data with specified page number processed to DimensionColumnDataChunk
105	*
106	* @param index
107	* @return
108	*/
109	public DimensionColumnPage convertToDimColDataChunkWithOutCache(int index) {
110	assert index < pagesCount;	1✔
111	// in case of filter query filter column if filter column is decoded and stored.
112	// then return the same
113	if (dataChunks != null && null != dataChunks[index]) {	1✔
114	return dataChunks[index];	1✔
115	}
116	try {
117	return chunkReader.decodeColumnPage(this, index);	1✔
118	} catch (Exception e) {	×
119	throw new RuntimeException(e);	×
120	}
121	}
122
123	@Override public void freeMemory() {
124	super.freeMemory();	1✔
125	if (null != dataChunks) {	1✔
126	for (int i = 0; i < dataChunks.length; i++) {	1✔
127	if (dataChunks[i] != null) {	1✔
128	dataChunks[i].freeMemory();	1✔
129	dataChunks[i] = null;	1✔
130	}
131	}
132	}
133	rawData = null;	1✔
134	}	1✔
135
136	public void setFileReader(FileReader fileReader) {
137	this.fileReader = fileReader;	1✔
138	}	1✔
139
140	public FileReader getFileReader() {
141	return fileReader;	×
142	}
143
144	public CarbonDictionary getLocalDictionary() {
145	if (null != getDataChunkV3() && null != getDataChunkV3().local_dictionary	1✔
146	&& null == localDictionary) {
147	try {
148	String compressorName =	1✔
149	getDataChunkV3().data_chunk_list.get(0).chunk_meta.getCompression_codec().name();	1✔
150	Compressor compressor = CompressorFactory.getInstance().getCompressor(compressorName);	1✔
151	localDictionary = getDictionary(getDataChunkV3().local_dictionary, compressor);	1✔
152	} catch (IOException \| MemoryException e) {	×
153	throw new RuntimeException(e);	×
154	}	1✔
155	}
156	return localDictionary;	1✔
157	}
158
159	/**
160	* Below method will be used to get the local dictionary for a blocklet
161	* @param localDictionaryChunk
162	* local dictionary chunk thrift object
163	* @return local dictionary
164	* @throws IOException
165	* @throws MemoryException
166	*/
167	private CarbonDictionary getDictionary(LocalDictionaryChunk localDictionaryChunk,
168	Compressor compressor) throws IOException, MemoryException {
169	if (null != localDictionaryChunk) {	1✔
170	List<Encoding> encodings = localDictionaryChunk.getDictionary_meta().getEncoders();	1✔
171	List<ByteBuffer> encoderMetas = localDictionaryChunk.getDictionary_meta().getEncoder_meta();	1✔
172	ColumnPageDecoder decoder = DefaultEncodingFactory.getInstance().createDecoder(	1✔
173	encodings, encoderMetas, compressor.getName());	1✔
174	ColumnPage decode = decoder.decode(localDictionaryChunk.getDictionary_data(), 0,	1✔
175	localDictionaryChunk.getDictionary_data().length);	1✔
176	BitSet usedDictionary = BitSet.valueOf(compressor.unCompressByte(	1✔
177	localDictionaryChunk.getDictionary_values()));	1✔
178	int length = usedDictionary.length();	1✔
179	int index = 0;	1✔
180	byte[][] dictionary = new byte[length][];	1✔
181	for (int i = 0; i < length; i++) {	1✔
182	if (usedDictionary.get(i)) {	1✔
183	dictionary[i] = decode.getBytes(index++);	1✔
184	} else {
185	dictionary[i] = null;	1✔
186	}
187	}
188	decode.freeMemory();	1✔
189	// as dictionary values starts from 1 setting null default value
190	dictionary[1] = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;	1✔
191	return new CarbonDictionaryImpl(dictionary, usedDictionary.cardinality());	1✔
192	}
193	return null;	×
194	}
195
196	}

apache / carbondata / 8301

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous