3292

Build # 3292

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/24b3da4816d0e54f3e9b5cd68a49262ce00ce7c9">24b3da481<a href="https://github.com/apache/carbondata/commit/24b3da4816d0e54f3e9b5cd68a49262ce00ce7c9">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/43082203e80f021351c4550c497870be139614df">43082203e</a>">43082203e</a><a href="https://github.com/apache/carbondata/commit/24b3da4816d0e54f3e9b5cd68a49262ce00ce7c9"> into eb7a83350">eb7a83350</a>

Run Details

6 of 6 new or added lines in 1 file covered. (100.0%)

63416 of 80115 relevant lines covered (79.16%)

1.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.46

/core/src/main/java/org/apache/carbondata/core/indexstore/schema/SchemaGenerator.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.indexstore.schema;

import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.util.BlockletDataMapUtil;

/**
 * class for creating schema for a given DataMap
 */
public class SchemaGenerator {

  /**
   * Method for creating blocklet Schema. Each blocklet row will share the same schema
   *
   * @param segmentProperties
   * @return
   */
  public static CarbonRowSchema[] createBlockSchema(SegmentProperties segmentProperties,
      List<CarbonColumn> minMaxCacheColumns) {
    List<CarbonRowSchema> indexSchemas = new ArrayList<>();
    // get MinMax Schema
    getMinMaxSchema(segmentProperties, indexSchemas, minMaxCacheColumns);
    // for number of rows.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));
    // for table block path
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for version number.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
    // for schema updated time.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for block footer offset.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for locations
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for storing block length.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for storing min max flag for each column which reflects whether min max for a column is
    // written in the metadata or not.
    addMinMaxFlagSchema(segmentProperties, indexSchemas, minMaxCacheColumns);
    CarbonRowSchema[] schema = indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]);
    updateBytePosition(schema);
    return schema;
  }

  /**
   * Method to update the byte position which will be used in case of unsafe dm store
   * @see org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java:87
   *
   * @param schema
   */
  private static void updateBytePosition(CarbonRowSchema[] schema) {
    int currentSize;
    int bytePosition = 0;
    // First assign byte postion to all the fixed length schema
    for (int i = 0; i < schema.length; i++) {
      switch (schema[i].getSchemaType()) {
        case STRUCT:
          CarbonRowSchema[] childSchemas =
              ((CarbonRowSchema.StructCarbonRowSchema) schema[i]).getChildSchemas();
          for (int j = 0; j < childSchemas.length; j++) {
            currentSize = getSchemaSize(childSchemas[j]);
            if (currentSize != -1) {
              childSchemas[j].setBytePosition(bytePosition);
              bytePosition += currentSize;
            }
          }
          break;
        default:
          currentSize = getSchemaSize(schema[i]);
          if (currentSize != -1) {
            schema[i].setBytePosition(bytePosition);
            bytePosition += currentSize;
          }
          break;
      }
    }
    // adding byte position for storing offset in case of variable length columns
    for (int i = 0; i < schema.length; i++) {
      switch (schema[i].getSchemaType()) {
        case STRUCT:
          CarbonRowSchema[] childSchemas =
              ((CarbonRowSchema.StructCarbonRowSchema) schema[i]).getChildSchemas();
          for (int j = 0; j < childSchemas.length; j++) {
            if (childSchemas[j].getBytePosition() == -1) {
              childSchemas[j].setBytePosition(bytePosition);
              bytePosition += CarbonCommonConstants.INT_SIZE_IN_BYTE;
            }
          }
          break;
        default:
          if (schema[i].getBytePosition() == -1) {
            schema[i].setBytePosition(bytePosition);
            bytePosition += CarbonCommonConstants.INT_SIZE_IN_BYTE;
          }
          break;
      }
    }
  }
  private static int getSchemaSize(CarbonRowSchema schema) {
    switch (schema.getSchemaType()) {
      case FIXED:
        return schema.getLength();
      case VARIABLE_SHORT:
      case VARIABLE_INT:
        return -1;
      default:
        throw new UnsupportedOperationException("Invalid Type");
    }
  }

  /**
   * Method for creating blocklet Schema. Each blocklet row will share the same schema
   *
   * @param segmentProperties
   * @return
   */
  public static CarbonRowSchema[] createBlockletSchema(SegmentProperties segmentProperties,
      List<CarbonColumn> minMaxCacheColumns) {
    List<CarbonRowSchema> indexSchemas = new ArrayList<>();
    // get MinMax Schema
    getMinMaxSchema(segmentProperties, indexSchemas, minMaxCacheColumns);
    // for number of rows.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));
    // for table block path
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for version number.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
    // for schema updated time.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for block footer offset.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for locations
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for storing block length.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for storing min max flag for each column which reflects whether min max for a column is
    // written in the metadata or not.
    addMinMaxFlagSchema(segmentProperties, indexSchemas, minMaxCacheColumns);
    //for blocklet info
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for number of pages.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
    // for relative blocklet id i.e. blocklet id that belongs to a particular part file
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
    CarbonRowSchema[] schema = indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]);
    updateBytePosition(schema);
    return schema;
  }

  /**
   * Creates the schema to store summary information or the information which can be stored only
   * once per datamap. It stores datamap level max/min of each column and partition information of
   * datamap
   *
   * @param segmentProperties
   * @throws MemoryException
   */
  public static CarbonRowSchema[] createTaskSummarySchema(SegmentProperties segmentProperties,
      List<CarbonColumn> minMaxCacheColumns,
      boolean storeBlockletCount, boolean filePathToBeStored) throws MemoryException {
    List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();
    // for number of rows.
    taskMinMaxSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // get MinMax Schema
    getMinMaxSchema(segmentProperties, taskMinMaxSchemas, minMaxCacheColumns);
    // for storing file name
    taskMinMaxSchemas
        .add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for storing segmentid
    taskMinMaxSchemas
        .add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for storing min max flag for each column which reflects whether min max for a column is
    // written in the metadata or not.
    addMinMaxFlagSchema(segmentProperties, taskMinMaxSchemas, minMaxCacheColumns);
    // store path only in case of partition table or non transactional table
    if (filePathToBeStored) {
      // for storing file path
      taskMinMaxSchemas
          .add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    }
    // flag to check whether it is required to store blocklet count of each carbondata file as
    // binary in summary schema. This will be true when it is not a legacy store (>1.1 version)
    // and CACHE_LEVEL=BLOCK
    if (storeBlockletCount) {
      taskMinMaxSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    }
    CarbonRowSchema[] schema =
        taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]);
    updateBytePosition(schema);
    return schema;
  }

  /**
   * Method to create schema for storing min/max data
   *
   * @param segmentProperties
   * @param minMaxSchemas
   */
  private static void getMinMaxSchema(SegmentProperties segmentProperties,
      List<CarbonRowSchema> minMaxSchemas, List<CarbonColumn> minMaxCacheColumns) {
    // Index key
    int[] minMaxLen = getMinMaxLength(segmentProperties, minMaxCacheColumns);
    int[] columnOrdinals = getColumnOrdinalsToAccess(segmentProperties, minMaxCacheColumns);
    // do it 2 times, one for min and one for max.
    for (int k = 0; k < 2; k++) {
      CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length];
      for (int i = 0; i < minMaxLen.length; i++) {
        if (minMaxLen[i] <= 0) {
          boolean isVarchar = false;
          if (columnOrdinals[i] < segmentProperties.getDimensions().size()
              && segmentProperties.getDimensions().get(columnOrdinals[i]).getDataType()
              == DataTypes.VARCHAR) {
            isVarchar = true;
          }
          mapSchemas[i] =
              new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY, isVarchar);
        } else {
          mapSchemas[i] =
              new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
        }
      }
      CarbonRowSchema mapSchema =
          new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
              mapSchemas);
      minMaxSchemas.add(mapSchema);
    }
  }

  /**
   * Method to add min max flag schema for all the dimensions
   * @param segmentProperties
   * @param indexSchemas
   * @param minMaxCacheColumns
   */
  private static void addMinMaxFlagSchema(SegmentProperties segmentProperties,
      List<CarbonRowSchema> indexSchemas, List<CarbonColumn> minMaxCacheColumns) {
    int minMaxFlagLength = segmentProperties.getColumnsValueSize().length;
    if (null != minMaxCacheColumns) {
      minMaxFlagLength = minMaxCacheColumns.size();
    }
    CarbonRowSchema[] minMaxFlagSchemas = new CarbonRowSchema[minMaxFlagLength];
    for (int i = 0; i < minMaxFlagLength; i++) {
      minMaxFlagSchemas[i] = new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BOOLEAN);
    }
    CarbonRowSchema structMinMaxFlagSchema =
        new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
            minMaxFlagSchemas);
    indexSchemas.add(structMinMaxFlagSchema);
  }

  /**
   * Method to get the min max length of each column. It will return the length of only column
   * which will be cached
   *
   * @param segmentProperties
   * @param minMaxCacheColumns
   * @return
   */
  private static int[] getMinMaxLength(SegmentProperties segmentProperties,
      List<CarbonColumn> minMaxCacheColumns) {
    int[] minMaxLen = null;
    if (null != minMaxCacheColumns) {
      minMaxLen = new int[minMaxCacheColumns.size()];
      int counter = 0;
      for (CarbonColumn column : minMaxCacheColumns) {
        minMaxLen[counter++] = segmentProperties.getColumnsValueSize()[BlockletDataMapUtil
            .getColumnOrdinal(segmentProperties, column)];
      }
    } else {
      minMaxLen = segmentProperties.getColumnsValueSize();
    }
    return minMaxLen;
  }

  /**
   * Method to fill the column ordinals to access based on the columns to be cached
   *
   * @param segmentProperties
   * @param minMaxCacheColumns
   * @return
   */
  private static int[] getColumnOrdinalsToAccess(SegmentProperties segmentProperties,
      List<CarbonColumn> minMaxCacheColumns) {
    int[] columnOrdinalsTOAccess = null;
    if (null != minMaxCacheColumns) {
      columnOrdinalsTOAccess = new int[minMaxCacheColumns.size()];
      int counter = 0;
      for (CarbonColumn column : minMaxCacheColumns) {
        columnOrdinalsTOAccess[counter++] =
            BlockletDataMapUtil.getColumnOrdinal(segmentProperties, column);
      }
    } else {
      // when columns to cache is not specified then column access order will be same as the array
      // index of min max length
      columnOrdinalsTOAccess = new int[segmentProperties.getColumnsValueSize().length];
      for (int i = 0; i < columnOrdinalsTOAccess.length; i++) {
        columnOrdinalsTOAccess[i] = i;
      }
    }
    return columnOrdinalsTOAccess;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.indexstore.schema;
19
20	import java.util.ArrayList;
21	import java.util.List;
22
23	import org.apache.carbondata.core.constants.CarbonCommonConstants;
24	import org.apache.carbondata.core.datastore.block.SegmentProperties;
25	import org.apache.carbondata.core.memory.MemoryException;
26	import org.apache.carbondata.core.metadata.datatype.DataTypes;
27	import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
28	import org.apache.carbondata.core.util.BlockletDataMapUtil;
29
30	/**
31	* class for creating schema for a given DataMap
32	*/
33	public class SchemaGenerator {	×
34
35	/**
36	* Method for creating blocklet Schema. Each blocklet row will share the same schema
37	*
38	* @param segmentProperties
39	* @return
40	*/
41	public static CarbonRowSchema[] createBlockSchema(SegmentProperties segmentProperties,
42	List<CarbonColumn> minMaxCacheColumns) {
43	List<CarbonRowSchema> indexSchemas = new ArrayList<>();	1✔
44	// get MinMax Schema
45	getMinMaxSchema(segmentProperties, indexSchemas, minMaxCacheColumns);	1✔
46	// for number of rows.
47	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));	1✔
48	// for table block path
49	indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
50	// for version number.
51	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));	1✔
52	// for schema updated time.
53	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));	1✔
54	// for block footer offset.
55	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));	1✔
56	// for locations
57	indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
58	// for storing block length.
59	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));	1✔
60	// for storing min max flag for each column which reflects whether min max for a column is
61	// written in the metadata or not.
62	addMinMaxFlagSchema(segmentProperties, indexSchemas, minMaxCacheColumns);	1✔
63	CarbonRowSchema[] schema = indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]);	1✔
64	updateBytePosition(schema);	1✔
65	return schema;	1✔
66	}
67
68	/**
69	* Method to update the byte position which will be used in case of unsafe dm store
70	* @see org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java:87
71	*
72	* @param schema
73	*/
74	private static void updateBytePosition(CarbonRowSchema[] schema) {
75	int currentSize;
76	int bytePosition = 0;	1✔
77	// First assign byte postion to all the fixed length schema
78	for (int i = 0; i < schema.length; i++) {	1✔
79	switch (schema[i].getSchemaType()) {	1✔
80	case STRUCT:
81	CarbonRowSchema[] childSchemas =	1✔
82	((CarbonRowSchema.StructCarbonRowSchema) schema[i]).getChildSchemas();	1✔
83	for (int j = 0; j < childSchemas.length; j++) {	1✔
84	currentSize = getSchemaSize(childSchemas[j]);	1✔
85	if (currentSize != -1) {	1✔
86	childSchemas[j].setBytePosition(bytePosition);	1✔
87	bytePosition += currentSize;	1✔
88	}
89	}
90	break;	1✔
91	default:
92	currentSize = getSchemaSize(schema[i]);	1✔
93	if (currentSize != -1) {	1✔
94	schema[i].setBytePosition(bytePosition);	1✔
95	bytePosition += currentSize;	1✔
96	}
97	break;
98	}
99	}
100	// adding byte position for storing offset in case of variable length columns
101	for (int i = 0; i < schema.length; i++) {	1✔
102	switch (schema[i].getSchemaType()) {	1✔
103	case STRUCT:
104	CarbonRowSchema[] childSchemas =	1✔
105	((CarbonRowSchema.StructCarbonRowSchema) schema[i]).getChildSchemas();	1✔
106	for (int j = 0; j < childSchemas.length; j++) {	1✔
107	if (childSchemas[j].getBytePosition() == -1) {	1✔
108	childSchemas[j].setBytePosition(bytePosition);	1✔
109	bytePosition += CarbonCommonConstants.INT_SIZE_IN_BYTE;	1✔
110	}
111	}
112	break;	1✔
113	default:
114	if (schema[i].getBytePosition() == -1) {	1✔
115	schema[i].setBytePosition(bytePosition);	1✔
116	bytePosition += CarbonCommonConstants.INT_SIZE_IN_BYTE;	1✔
117	}
118	break;
119	}
120	}
121	}	1✔
122	private static int getSchemaSize(CarbonRowSchema schema) {
123	switch (schema.getSchemaType()) {	1✔
124	case FIXED:
125	return schema.getLength();	1✔
126	case VARIABLE_SHORT:
127	case VARIABLE_INT:
128	return -1;	1✔
129	default:
130	throw new UnsupportedOperationException("Invalid Type");	×
131	}
132	}
133
134	/**
135	* Method for creating blocklet Schema. Each blocklet row will share the same schema
136	*
137	* @param segmentProperties
138	* @return
139	*/
140	public static CarbonRowSchema[] createBlockletSchema(SegmentProperties segmentProperties,
141	List<CarbonColumn> minMaxCacheColumns) {
142	List<CarbonRowSchema> indexSchemas = new ArrayList<>();	1✔
143	// get MinMax Schema
144	getMinMaxSchema(segmentProperties, indexSchemas, minMaxCacheColumns);	1✔
145	// for number of rows.
146	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));	1✔
147	// for table block path
148	indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
149	// for version number.
150	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));	1✔
151	// for schema updated time.
152	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));	1✔
153	// for block footer offset.
154	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));	1✔
155	// for locations
156	indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
157	// for storing block length.
158	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));	1✔
159	// for storing min max flag for each column which reflects whether min max for a column is
160	// written in the metadata or not.
161	addMinMaxFlagSchema(segmentProperties, indexSchemas, minMaxCacheColumns);	1✔
162	//for blocklet info
163	indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
164	// for number of pages.
165	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));	1✔
166	// for relative blocklet id i.e. blocklet id that belongs to a particular part file
167	indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));	1✔
168	CarbonRowSchema[] schema = indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]);	1✔
169	updateBytePosition(schema);	1✔
170	return schema;	1✔
171	}
172
173	/**
174	* Creates the schema to store summary information or the information which can be stored only
175	* once per datamap. It stores datamap level max/min of each column and partition information of
176	* datamap
177	*
178	* @param segmentProperties
179	* @throws MemoryException
180	*/
181	public static CarbonRowSchema[] createTaskSummarySchema(SegmentProperties segmentProperties,
182	List<CarbonColumn> minMaxCacheColumns,
183	boolean storeBlockletCount, boolean filePathToBeStored) throws MemoryException {
184	List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();	1✔
185	// for number of rows.
186	taskMinMaxSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));	1✔
187	// get MinMax Schema
188	getMinMaxSchema(segmentProperties, taskMinMaxSchemas, minMaxCacheColumns);	1✔
189	// for storing file name
190	taskMinMaxSchemas	1✔
191	.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
192	// for storing segmentid
193	taskMinMaxSchemas	1✔
194	.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
195	// for storing min max flag for each column which reflects whether min max for a column is
196	// written in the metadata or not.
197	addMinMaxFlagSchema(segmentProperties, taskMinMaxSchemas, minMaxCacheColumns);	1✔
198	// store path only in case of partition table or non transactional table
199	if (filePathToBeStored) {	1✔
200	// for storing file path
201	taskMinMaxSchemas	1✔
202	.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
203	}
204	// flag to check whether it is required to store blocklet count of each carbondata file as
205	// binary in summary schema. This will be true when it is not a legacy store (>1.1 version)
206	// and CACHE_LEVEL=BLOCK
207	if (storeBlockletCount) {	1✔
208	taskMinMaxSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));	1✔
209	}
210	CarbonRowSchema[] schema =	1✔
211	taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]);	1✔
212	updateBytePosition(schema);	1✔
213	return schema;	1✔
214	}
215
216	/**
217	* Method to create schema for storing min/max data
218	*
219	* @param segmentProperties
220	* @param minMaxSchemas
221	*/
222	private static void getMinMaxSchema(SegmentProperties segmentProperties,
223	List<CarbonRowSchema> minMaxSchemas, List<CarbonColumn> minMaxCacheColumns) {
224	// Index key
225	int[] minMaxLen = getMinMaxLength(segmentProperties, minMaxCacheColumns);	1✔
226	int[] columnOrdinals = getColumnOrdinalsToAccess(segmentProperties, minMaxCacheColumns);	1✔
227	// do it 2 times, one for min and one for max.
228	for (int k = 0; k < 2; k++) {	1✔
229	CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length];	1✔
230	for (int i = 0; i < minMaxLen.length; i++) {	1✔
231	if (minMaxLen[i] <= 0) {	1✔
232	boolean isVarchar = false;	1✔
233	if (columnOrdinals[i] < segmentProperties.getDimensions().size()	1✔
234	&& segmentProperties.getDimensions().get(columnOrdinals[i]).getDataType()	1✔
235	== DataTypes.VARCHAR) {
236	isVarchar = true;	1✔
237	}
238	mapSchemas[i] =	1✔
239	new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY, isVarchar);
240	} else {	1✔
241	mapSchemas[i] =	1✔
242	new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
243	}
244	}
245	CarbonRowSchema mapSchema =	1✔
246	new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),	1✔
247	mapSchemas);
248	minMaxSchemas.add(mapSchema);	1✔
249	}
250	}	1✔
251
252	/**
253	* Method to add min max flag schema for all the dimensions
254	* @param segmentProperties
255	* @param indexSchemas
256	* @param minMaxCacheColumns
257	*/
258	private static void addMinMaxFlagSchema(SegmentProperties segmentProperties,
259	List<CarbonRowSchema> indexSchemas, List<CarbonColumn> minMaxCacheColumns) {
260	int minMaxFlagLength = segmentProperties.getColumnsValueSize().length;	1✔
261	if (null != minMaxCacheColumns) {	1✔
262	minMaxFlagLength = minMaxCacheColumns.size();	1✔
263	}
264	CarbonRowSchema[] minMaxFlagSchemas = new CarbonRowSchema[minMaxFlagLength];	1✔
265	for (int i = 0; i < minMaxFlagLength; i++) {	1✔
266	minMaxFlagSchemas[i] = new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BOOLEAN);	1✔
267	}
268	CarbonRowSchema structMinMaxFlagSchema =	1✔
269	new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),	1✔
270	minMaxFlagSchemas);
271	indexSchemas.add(structMinMaxFlagSchema);	1✔
272	}	1✔
273
274	/**
275	* Method to get the min max length of each column. It will return the length of only column
276	* which will be cached
277	*
278	* @param segmentProperties
279	* @param minMaxCacheColumns
280	* @return
281	*/
282	private static int[] getMinMaxLength(SegmentProperties segmentProperties,
283	List<CarbonColumn> minMaxCacheColumns) {
284	int[] minMaxLen = null;	1✔
285	if (null != minMaxCacheColumns) {	1✔
286	minMaxLen = new int[minMaxCacheColumns.size()];	1✔
287	int counter = 0;	1✔
288	for (CarbonColumn column : minMaxCacheColumns) {	1✔
289	minMaxLen[counter++] = segmentProperties.getColumnsValueSize()[BlockletDataMapUtil	1✔
290	.getColumnOrdinal(segmentProperties, column)];	1✔
291	}	1✔
292	} else {	1✔
293	minMaxLen = segmentProperties.getColumnsValueSize();	1✔
294	}
295	return minMaxLen;	1✔
296	}
297
298	/**
299	* Method to fill the column ordinals to access based on the columns to be cached
300	*
301	* @param segmentProperties
302	* @param minMaxCacheColumns
303	* @return
304	*/
305	private static int[] getColumnOrdinalsToAccess(SegmentProperties segmentProperties,
306	List<CarbonColumn> minMaxCacheColumns) {
307	int[] columnOrdinalsTOAccess = null;	1✔
308	if (null != minMaxCacheColumns) {	1✔
309	columnOrdinalsTOAccess = new int[minMaxCacheColumns.size()];	1✔
310	int counter = 0;	1✔
311	for (CarbonColumn column : minMaxCacheColumns) {	1✔
312	columnOrdinalsTOAccess[counter++] =	1✔
313	BlockletDataMapUtil.getColumnOrdinal(segmentProperties, column);	1✔
314	}	1✔
315	} else {	1✔
316	// when columns to cache is not specified then column access order will be same as the array
317	// index of min max length
318	columnOrdinalsTOAccess = new int[segmentProperties.getColumnsValueSize().length];	1✔
319	for (int i = 0; i < columnOrdinalsTOAccess.length; i++) {	1✔
320	columnOrdinalsTOAccess[i] = i;	1✔
321	}
322	}
323	return columnOrdinalsTOAccess;	1✔
324	}
325	}

apache / carbondata / 3292

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous