8549

Build # 8549

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/29e4533053186ddda4c1e5d519560a48e3f36cf9">29e453305<a href="https://github.com/apache/carbondata/commit/29e4533053186ddda4c1e5d519560a48e3f36cf9">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/2a94b69ce2d1dab0acee1b4339b01a9f8233e2ea">2a94b69ce</a>">2a94b69ce</a><a href="https://github.com/apache/carbondata/commit/29e4533053186ddda4c1e5d519560a48e3f36cf9"> into 476e6b2a5">476e6b2a5</a>

Run Details

1169 of 1169 new or added lines in 70 files covered. (100.0%)

59412 of 73712 relevant lines covered (80.6%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.3

/core/src/main/java/org/apache/carbondata/core/localdictionary/PageLevelDictionary.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.localdictionary;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.BitSet;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.ColumnType;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
import org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec;
import org.apache.carbondata.core.datastore.page.statistics.DummyStatsCollector;
import org.apache.carbondata.core.localdictionary.exception.DictionaryThresholdReachedException;
import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.format.LocalDictionaryChunk;

/**
 * Class to maintain page level dictionary. It will store all unique dictionary values
 * used in a page. This is required while writing blocklet level dictionary in carbondata
 * file
 */
public class PageLevelDictionary {

  /**
   * dictionary generator to generate dictionary values for page data
   */
  private LocalDictionaryGenerator localDictionaryGenerator;

  /**
   * set of dictionary surrogate key in this page
   */
  private BitSet usedDictionaryValues;

  private String columnName;

  private DataType dataType;

  private boolean isComplexTypePrimitive;
  // compressor to be used for the dictionary. The compressor is the same as column compressor.
  private String columnCompressor;

  public PageLevelDictionary(LocalDictionaryGenerator localDictionaryGenerator, String columnName,
      DataType dataType, boolean isComplexTypePrimitive, String columnCompressor) {
    this.localDictionaryGenerator = localDictionaryGenerator;
    this.usedDictionaryValues = new BitSet();
    this.columnName = columnName;
    this.dataType = dataType;
    this.isComplexTypePrimitive = isComplexTypePrimitive;
    this.columnCompressor = columnCompressor;
  }

  /**
   * Below method will be used to get the dictionary value
   *
   * @param data column data
   * @return dictionary value
   * @throws DictionaryThresholdReachedException when threshold crossed for column
   */
  public int getDictionaryValue(byte[] data) throws DictionaryThresholdReachedException {
    int dictionaryValue = localDictionaryGenerator.generateDictionary(data);
    this.usedDictionaryValues.set(dictionaryValue);
    return dictionaryValue;
  }

  /**
   * Method to merge the dictionary value across pages
   *
   * @param pageLevelDictionary other page level dictionary
   */
  public void mergerDictionaryValues(PageLevelDictionary pageLevelDictionary) {
    usedDictionaryValues.or(pageLevelDictionary.usedDictionaryValues);
  }

  /**
   * Below method will be used to get the local dictionary chunk for writing
   * @TODO Support for numeric data type dictionary exclude columns
   * @return encoded local dictionary chunk
   * @throws MemoryException
   * in case of problem in encoding
   * @throws IOException
   * in case of problem in encoding
   */
  public LocalDictionaryChunk getLocalDictionaryChunkForBlocklet()
      throws MemoryException, IOException {
    // TODO support for actual data type dictionary ColumnSPEC
    ColumnType columnType = ColumnType.PLAIN_VALUE;
    boolean isVarcharType = false;
    int lvSize = CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
    if (DataTypes.VARCHAR == dataType) {
      columnType = ColumnType.PLAIN_LONG_VALUE;
      lvSize = CarbonCommonConstants.INT_SIZE_IN_BYTE;
      isVarcharType = true;
    }
    TableSpec.ColumnSpec spec =
        TableSpec.ColumnSpec.newInstance(columnName, DataTypes.BYTE_ARRAY, columnType);
    ColumnPage dictionaryColumnPage = ColumnPage.newPage(
        new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor),
        usedDictionaryValues.cardinality());
    // TODO support data type specific stats collector for numeric data types
    dictionaryColumnPage.setStatsCollector(new DummyStatsCollector());
    int rowId = 0;
    ByteBuffer byteBuffer = null;
    for (int i = usedDictionaryValues.nextSetBit(0);
         i >= 0; i = usedDictionaryValues.nextSetBit(i + 1)) {
      if (!isComplexTypePrimitive) {
        dictionaryColumnPage
            .putData(rowId++, localDictionaryGenerator.getDictionaryKeyBasedOnValue(i));
      } else {
        byte[] dictionaryKeyBasedOnValue = localDictionaryGenerator.getDictionaryKeyBasedOnValue(i);
        byteBuffer = ByteBuffer.allocate(lvSize + dictionaryKeyBasedOnValue.length);
        if (!isVarcharType) {
          byteBuffer.putShort((short) dictionaryKeyBasedOnValue.length);
        } else {
          byteBuffer.putInt(dictionaryKeyBasedOnValue.length);
        }
        byteBuffer.put(dictionaryKeyBasedOnValue);
        dictionaryColumnPage.putData(rowId++, byteBuffer.array());
      }
    }
    // creating a encoder
    ColumnPageEncoder encoder = new DirectCompressCodec(DataTypes.BYTE_ARRAY).createEncoder(null);
    // get encoded dictionary values
    LocalDictionaryChunk localDictionaryChunk = encoder.encodeDictionary(dictionaryColumnPage);
    // set compressed dictionary values
    localDictionaryChunk.setDictionary_values(
        CompressorFactory.getInstance().getCompressor(columnCompressor).compressByte(
            usedDictionaryValues.toByteArray()));
    // free the dictionary page memory
    dictionaryColumnPage.freeMemory();
    return localDictionaryChunk;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.localdictionary;
18
19	import java.io.IOException;
20	import java.nio.ByteBuffer;
21	import java.util.BitSet;
22
23	import org.apache.carbondata.core.constants.CarbonCommonConstants;
24	import org.apache.carbondata.core.datastore.ColumnType;
25	import org.apache.carbondata.core.datastore.TableSpec;
26	import org.apache.carbondata.core.datastore.compression.CompressorFactory;
27	import org.apache.carbondata.core.datastore.page.ColumnPage;
28	import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder;
29	import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
30	import org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec;
31	import org.apache.carbondata.core.datastore.page.statistics.DummyStatsCollector;
32	import org.apache.carbondata.core.localdictionary.exception.DictionaryThresholdReachedException;
33	import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator;
34	import org.apache.carbondata.core.memory.MemoryException;
35	import org.apache.carbondata.core.metadata.datatype.DataType;
36	import org.apache.carbondata.core.metadata.datatype.DataTypes;
37	import org.apache.carbondata.format.LocalDictionaryChunk;
38
39	/**
40	* Class to maintain page level dictionary. It will store all unique dictionary values
41	* used in a page. This is required while writing blocklet level dictionary in carbondata
42	* file
43	*/
44	public class PageLevelDictionary {
45
46	/**
47	* dictionary generator to generate dictionary values for page data
48	*/
49	private LocalDictionaryGenerator localDictionaryGenerator;
50
51	/**
52	* set of dictionary surrogate key in this page
53	*/
54	private BitSet usedDictionaryValues;
55
56	private String columnName;
57
58	private DataType dataType;
59
60	private boolean isComplexTypePrimitive;
61	// compressor to be used for the dictionary. The compressor is the same as column compressor.
62	private String columnCompressor;
63
64	public PageLevelDictionary(LocalDictionaryGenerator localDictionaryGenerator, String columnName,
65	DataType dataType, boolean isComplexTypePrimitive, String columnCompressor) {	2✔
66	this.localDictionaryGenerator = localDictionaryGenerator;	2✔
67	this.usedDictionaryValues = new BitSet();	2✔
68	this.columnName = columnName;	2✔
69	this.dataType = dataType;	2✔
70	this.isComplexTypePrimitive = isComplexTypePrimitive;	2✔
71	this.columnCompressor = columnCompressor;	2✔
72	}	2✔
73
74	/**
75	* Below method will be used to get the dictionary value
76	*
77	* @param data column data
78	* @return dictionary value
79	* @throws DictionaryThresholdReachedException when threshold crossed for column
80	*/
81	public int getDictionaryValue(byte[] data) throws DictionaryThresholdReachedException {
82	int dictionaryValue = localDictionaryGenerator.generateDictionary(data);	2✔
83	this.usedDictionaryValues.set(dictionaryValue);	2✔
84	return dictionaryValue;	2✔
85	}
86
87	/**
88	* Method to merge the dictionary value across pages
89	*
90	* @param pageLevelDictionary other page level dictionary
91	*/
92	public void mergerDictionaryValues(PageLevelDictionary pageLevelDictionary) {
93	usedDictionaryValues.or(pageLevelDictionary.usedDictionaryValues);	1✔
94	}	1✔
95
96	/**
97	* Below method will be used to get the local dictionary chunk for writing
98	* @TODO Support for numeric data type dictionary exclude columns
99	* @return encoded local dictionary chunk
100	* @throws MemoryException
101	* in case of problem in encoding
102	* @throws IOException
103	* in case of problem in encoding
104	*/
105	public LocalDictionaryChunk getLocalDictionaryChunkForBlocklet()
106	throws MemoryException, IOException {
107	// TODO support for actual data type dictionary ColumnSPEC
108	ColumnType columnType = ColumnType.PLAIN_VALUE;	2✔
109	boolean isVarcharType = false;	2✔
110	int lvSize = CarbonCommonConstants.SHORT_SIZE_IN_BYTE;	2✔
111	if (DataTypes.VARCHAR == dataType) {	2✔
112	columnType = ColumnType.PLAIN_LONG_VALUE;	×
113	lvSize = CarbonCommonConstants.INT_SIZE_IN_BYTE;	×
114	isVarcharType = true;	×
115	}
116	TableSpec.ColumnSpec spec =	2✔
117	TableSpec.ColumnSpec.newInstance(columnName, DataTypes.BYTE_ARRAY, columnType);	2✔
118	ColumnPage dictionaryColumnPage = ColumnPage.newPage(	2✔
119	new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor),
120	usedDictionaryValues.cardinality());	2✔
121	// TODO support data type specific stats collector for numeric data types
122	dictionaryColumnPage.setStatsCollector(new DummyStatsCollector());	2✔
123	int rowId = 0;	2✔
124	ByteBuffer byteBuffer = null;	2✔
125	for (int i = usedDictionaryValues.nextSetBit(0);	2✔
126	i >= 0; i = usedDictionaryValues.nextSetBit(i + 1)) {	2✔
127	if (!isComplexTypePrimitive) {	2✔
128	dictionaryColumnPage	2✔
129	.putData(rowId++, localDictionaryGenerator.getDictionaryKeyBasedOnValue(i));	2✔
130	} else {
131	byte[] dictionaryKeyBasedOnValue = localDictionaryGenerator.getDictionaryKeyBasedOnValue(i);	1✔
132	byteBuffer = ByteBuffer.allocate(lvSize + dictionaryKeyBasedOnValue.length);	1✔
133	if (!isVarcharType) {	1✔
134	byteBuffer.putShort((short) dictionaryKeyBasedOnValue.length);	1✔
135	} else {
136	byteBuffer.putInt(dictionaryKeyBasedOnValue.length);	×
137	}
138	byteBuffer.put(dictionaryKeyBasedOnValue);	1✔
139	dictionaryColumnPage.putData(rowId++, byteBuffer.array());	1✔
140	}
141	}
142	// creating a encoder
143	ColumnPageEncoder encoder = new DirectCompressCodec(DataTypes.BYTE_ARRAY).createEncoder(null);	2✔
144	// get encoded dictionary values
145	LocalDictionaryChunk localDictionaryChunk = encoder.encodeDictionary(dictionaryColumnPage);	2✔
146	// set compressed dictionary values
147	localDictionaryChunk.setDictionary_values(	2✔
148	CompressorFactory.getInstance().getCompressor(columnCompressor).compressByte(	2✔
149	usedDictionaryValues.toByteArray()));	2✔
150	// free the dictionary page memory
151	dictionaryColumnPage.freeMemory();	2✔
152	return localDictionaryChunk;	2✔
153	}
154	}

apache / carbondata / 8549

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous