1094

Build # 1094

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/b637471a486349c34405e561703a7c3effe96c5e">b637471a4<a href="https://github.com/apache/carbondata/commit/b637471a486349c34405e561703a7c3effe96c5e">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/1be27964bcfbff2658cccc7e1d9686e611061aad">1be27964b</a>">1be27964b</a><a href="https://github.com/apache/carbondata/commit/b637471a486349c34405e561703a7c3effe96c5e"> into a9e405f47">a9e405f47</a>

Run Details

41 of 41 new or added lines in 6 files covered. (100.0%)

24578 of 75182 relevant lines covered (32.69%)

0.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.53

/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.datastore.page;

import java.util.List;
import java.util.Map;

import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
import org.apache.carbondata.core.datastore.page.statistics.DummyStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
import org.apache.carbondata.core.datastore.row.ComplexColumnInfo;
import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.DataTypeUtil;

/**
 * holds the complex columndata and its children data
 */
public class ComplexColumnPage {

  /**
   * number of columns
   */
  private int complexColumnIndex;

  /**
   * type of each column
   */
  private List<ComplexColumnInfo> complexColumnInfoList;

  /**
   * column page for each type
   */
  private ColumnPage[] columnPages;

  /**
   * to maintain the number of record added for each type
   */
  private int[] currentRowIdList;

  public ComplexColumnPage(List<ComplexColumnInfo> complexColumnInfoList) {
    this.complexColumnIndex = complexColumnInfoList.size();
    this.complexColumnInfoList = complexColumnInfoList;
    this.columnPages = new ColumnPage[this.complexColumnIndex];
    this.currentRowIdList = new int[complexColumnIndex];
  }

  /**
   * below method will be used to initlize the column page of complex type
   * @param columnToDictMap
   * dictionary map
   * @param pageSize
   * number of records
   * @throws MemoryException
   * if memory is not sufficient
   */
  public void initialize(Map<String, LocalDictionaryGenerator> columnToDictMap, int pageSize,
      String columnCompressor) throws MemoryException {
    DataType dataType;
    for (int i = 0; i < this.columnPages.length; i++) {
      LocalDictionaryGenerator localDictionaryGenerator =
          columnToDictMap.get(complexColumnInfoList.get(i).getColumnNames());
      TableSpec.ColumnSpec spec = getColumnSpec(i, localDictionaryGenerator);
      if (null == localDictionaryGenerator) {
        dataType = complexColumnInfoList.get(i).getColumnDataTypes();
        if (isColumnPageBasedOnDataType(i)) {
          // no dictionary primitive types need adaptive encoding,
          // hence store as actual value instead of byte array
          this.columnPages[i] = ColumnPage.newPage(
              new ColumnPageEncoderMeta(spec, dataType, columnCompressor), pageSize);
          this.columnPages[i].setStatsCollector(PrimitivePageStatsCollector.newInstance(dataType));
        } else {
          this.columnPages[i] = ColumnPage.newPage(
              new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize);
          this.columnPages[i].setStatsCollector(new DummyStatsCollector());
        }
      } else {
        this.columnPages[i] = ColumnPage.newLocalDictPage(
            new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize,
            localDictionaryGenerator, true);
        this.columnPages[i].setStatsCollector(new DummyStatsCollector());
      }
    }
  }

  private TableSpec.ColumnSpec getColumnSpec(int columnPageIndex,
      LocalDictionaryGenerator localDictionaryGenerator) {
    if ((localDictionaryGenerator == null) && isColumnPageBasedOnDataType(columnPageIndex)) {
      return TableSpec.ColumnSpec
          .newInstance(complexColumnInfoList.get(columnPageIndex).getColumnNames(),
              complexColumnInfoList.get(columnPageIndex).getColumnDataTypes(),
              complexColumnInfoList.get(columnPageIndex).getComplexColumnType());
    } else {
      return TableSpec.ColumnSpec
          .newInstance(complexColumnInfoList.get(columnPageIndex).getColumnNames(),
              DataTypes.BYTE_ARRAY,
              complexColumnInfoList.get(columnPageIndex).getComplexColumnType());
    }
  }

  private boolean isColumnPageBasedOnDataType(int columnPageIndex) {
    DataType dataType = complexColumnInfoList.get(columnPageIndex).getColumnDataTypes();
    if ((complexColumnInfoList.get(columnPageIndex).isNoDictionary() &&
        !((DataTypes.isStructType(dataType) ||
            DataTypes.isArrayType(dataType) ||
            DataTypes.isMapType(dataType) ||
            (dataType == DataTypes.STRING) ||
            (dataType == DataTypes.VARCHAR) ||
            (dataType == DataTypes.DATE) ||
            DataTypes.isDecimal(dataType))))) {
      // For all these above condition the ColumnPage should be Taken as BYTE_ARRAY
      // for all other cases make Column Page Based on each DataType.
      return true;
    } else {
      return false;
    }
  }

  /**
   *
   * @return complexColumnIndex
   */
  public int getComplexColumnIndex() {
    return complexColumnIndex;
  }

  /**
   * method to add complex column data
   * @param depth
   * complexColumnIndex of column
   * @param dataList
   * dataList
   */
  public void putComplexData(int depth, List<byte[]> dataList) {
    assert (depth <= this.complexColumnIndex);
    int positionNumber = currentRowIdList[depth];
    for (byte[] value : dataList) {
      if (columnPages[depth].getDataType() != DataTypes.BYTE_ARRAY) {
        if ((value == null) || (value.length == 0)) {
          columnPages[depth].putNull(positionNumber);
          columnPages[depth].statsCollector.updateNull(positionNumber);
          columnPages[depth].nullBitSet.set(positionNumber);
        } else {
          columnPages[depth].putData(positionNumber, DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn(value,
                  columnPages[depth].getColumnSpec().getSchemaDataType(), false));
        }
      } else {
        columnPages[depth].putData(positionNumber, value);
      }
      positionNumber++;
    }
    currentRowIdList[depth] = positionNumber;
  }

  /**
   * to free the used memory
   */
  public void freeMemory() {
    for (int i = 0; i < complexColumnIndex; i++) {
      columnPages[i].freeMemory();
    }
  }

  /**
   * return the column page
   * @param complexColumnIndex
   * complexColumnIndex of column
   * @return colum page
   */
  public ColumnPage getColumnPage(int complexColumnIndex) {
    assert (complexColumnIndex <= this.complexColumnIndex);
    return columnPages[complexColumnIndex];
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.datastore.page;
19
20	import java.util.List;
21	import java.util.Map;
22
23	import org.apache.carbondata.core.datastore.TableSpec;
24	import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
25	import org.apache.carbondata.core.datastore.page.statistics.DummyStatsCollector;
26	import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
27	import org.apache.carbondata.core.datastore.row.ComplexColumnInfo;
28	import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator;
29	import org.apache.carbondata.core.memory.MemoryException;
30	import org.apache.carbondata.core.metadata.datatype.DataType;
31	import org.apache.carbondata.core.metadata.datatype.DataTypes;
32	import org.apache.carbondata.core.util.DataTypeUtil;
33
34	/**
35	* holds the complex columndata and its children data
36	*/
37	public class ComplexColumnPage {	1✔
38
39	/**
40	* number of columns
41	*/
42	private int complexColumnIndex;
43
44	/**
45	* type of each column
46	*/
47	private List<ComplexColumnInfo> complexColumnInfoList;
48
49	/**
50	* column page for each type
51	*/
52	private ColumnPage[] columnPages;
53
54	/**
55	* to maintain the number of record added for each type
56	*/
57	private int[] currentRowIdList;
58
59	public ComplexColumnPage(List<ComplexColumnInfo> complexColumnInfoList) {	1✔
60	this.complexColumnIndex = complexColumnInfoList.size();	1✔
61	this.complexColumnInfoList = complexColumnInfoList;	1✔
62	this.columnPages = new ColumnPage[this.complexColumnIndex];	1✔
63	this.currentRowIdList = new int[complexColumnIndex];	1✔
64	}	1✔
65
66	/**
67	* below method will be used to initlize the column page of complex type
68	* @param columnToDictMap
69	* dictionary map
70	* @param pageSize
71	* number of records
72	* @throws MemoryException
73	* if memory is not sufficient
74	*/
75	public void initialize(Map<String, LocalDictionaryGenerator> columnToDictMap, int pageSize,
76	String columnCompressor) throws MemoryException {
77	DataType dataType;
78	for (int i = 0; i < this.columnPages.length; i++) {	1✔
79	LocalDictionaryGenerator localDictionaryGenerator =	1✔
80	columnToDictMap.get(complexColumnInfoList.get(i).getColumnNames());	1✔
81	TableSpec.ColumnSpec spec = getColumnSpec(i, localDictionaryGenerator);	1✔
82	if (null == localDictionaryGenerator) {	1✔
83	dataType = complexColumnInfoList.get(i).getColumnDataTypes();	1✔
84	if (isColumnPageBasedOnDataType(i)) {	1✔
85	// no dictionary primitive types need adaptive encoding,
86	// hence store as actual value instead of byte array
87	this.columnPages[i] = ColumnPage.newPage(	1✔
88	new ColumnPageEncoderMeta(spec, dataType, columnCompressor), pageSize);
89	this.columnPages[i].setStatsCollector(PrimitivePageStatsCollector.newInstance(dataType));	1✔
90	} else {
91	this.columnPages[i] = ColumnPage.newPage(	1✔
92	new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize);
93	this.columnPages[i].setStatsCollector(new DummyStatsCollector());	1✔
94	}
95	} else {
96	this.columnPages[i] = ColumnPage.newLocalDictPage(	×
97	new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize,
98	localDictionaryGenerator, true);
99	this.columnPages[i].setStatsCollector(new DummyStatsCollector());	×
100	}
101	}
102	}	1✔
103
104	private TableSpec.ColumnSpec getColumnSpec(int columnPageIndex,
105	LocalDictionaryGenerator localDictionaryGenerator) {
106	if ((localDictionaryGenerator == null) && isColumnPageBasedOnDataType(columnPageIndex)) {	1✔
107	return TableSpec.ColumnSpec	1✔
108	.newInstance(complexColumnInfoList.get(columnPageIndex).getColumnNames(),	1✔
109	complexColumnInfoList.get(columnPageIndex).getColumnDataTypes(),	1✔
110	complexColumnInfoList.get(columnPageIndex).getComplexColumnType());	1✔
111	} else {
112	return TableSpec.ColumnSpec	1✔
113	.newInstance(complexColumnInfoList.get(columnPageIndex).getColumnNames(),	1✔
114	DataTypes.BYTE_ARRAY,
115	complexColumnInfoList.get(columnPageIndex).getComplexColumnType());	1✔
116	}
117	}
118
119	private boolean isColumnPageBasedOnDataType(int columnPageIndex) {
120	DataType dataType = complexColumnInfoList.get(columnPageIndex).getColumnDataTypes();	1✔
121	if ((complexColumnInfoList.get(columnPageIndex).isNoDictionary() &&	1✔
122	!((DataTypes.isStructType(dataType) \|\|	1✔
123	DataTypes.isArrayType(dataType) \|\|	1✔
124	DataTypes.isMapType(dataType) \|\|	1✔
125	(dataType == DataTypes.STRING) \|\|
126	(dataType == DataTypes.VARCHAR) \|\|
127	(dataType == DataTypes.DATE) \|\|
128	DataTypes.isDecimal(dataType))))) {	1✔
129	// For all these above condition the ColumnPage should be Taken as BYTE_ARRAY
130	// for all other cases make Column Page Based on each DataType.
131	return true;	1✔
132	} else {
133	return false;	1✔
134	}
135	}
136
137	/**
138	*
139	* @return complexColumnIndex
140	*/
141	public int getComplexColumnIndex() {
142	return complexColumnIndex;	1✔
143	}
144
145	/**
146	* method to add complex column data
147	* @param depth
148	* complexColumnIndex of column
149	* @param dataList
150	* dataList
151	*/
152	public void putComplexData(int depth, List<byte[]> dataList) {
153	assert (depth <= this.complexColumnIndex);	1✔
154	int positionNumber = currentRowIdList[depth];	1✔
155	for (byte[] value : dataList) {	1✔
156	if (columnPages[depth].getDataType() != DataTypes.BYTE_ARRAY) {	1✔
157	if ((value == null) \|\| (value.length == 0)) {	1✔
158	columnPages[depth].putNull(positionNumber);	×
159	columnPages[depth].statsCollector.updateNull(positionNumber);	×
160	columnPages[depth].nullBitSet.set(positionNumber);	×
161	} else {
162	columnPages[depth].putData(positionNumber, DataTypeUtil	1✔
163	.getDataBasedOnDataTypeForNoDictionaryColumn(value,	1✔
164	columnPages[depth].getColumnSpec().getSchemaDataType(), false));	1✔
165	}
166	} else {
167	columnPages[depth].putData(positionNumber, value);	1✔
168	}
169	positionNumber++;	1✔
170	}	1✔
171	currentRowIdList[depth] = positionNumber;	1✔
172	}	1✔
173
174	/**
175	* to free the used memory
176	*/
177	public void freeMemory() {
178	for (int i = 0; i < complexColumnIndex; i++) {	1✔
179	columnPages[i].freeMemory();	1✔
180	}
181	}	1✔
182
183	/**
184	* return the column page
185	* @param complexColumnIndex
186	* complexColumnIndex of column
187	* @return colum page
188	*/
189	public ColumnPage getColumnPage(int complexColumnIndex) {
190	assert (complexColumnIndex <= this.complexColumnIndex);	1✔
191	return columnPages[complexColumnIndex];	1✔
192	}
193	}

apache / carbondata / 1094

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous