1394

Build # 1394

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/321ccc6b2f432173c096aa43785536b99f3d0e90">321ccc6b2<a href="https://github.com/apache/carbondata/commit/321ccc6b2f432173c096aa43785536b99f3d0e90">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/1a2acdf24aabf344ff2c73ed8ac3a08ba2d6e5c1">1a2acdf24</a>">1a2acdf24</a><a href="https://github.com/apache/carbondata/commit/321ccc6b2f432173c096aa43785536b99f3d0e90"> into 0e39abf81">0e39abf81</a>

Run Details

46 of 46 new or added lines in 2 files covered. (100.0%)

61822 of 77185 relevant lines covered (80.1%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.4

/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.processing.merger;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.carbondata.processing.store.CarbonFactHandler;
import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;

import org.apache.log4j.Logger;

/**
 * This class will process the query result and convert the data
 * into a format compatible for data load
 */
public class CompactionResultSortProcessor extends AbstractResultProcessor {

  /**
   * LOGGER
   */
  private static final Logger LOGGER =
      LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());
  /**
   * carbon load model that contains all the required information for load
   */
  private CarbonLoadModel carbonLoadModel;
  /**
   * carbon table
   */
  private CarbonTable carbonTable;
  /**
   * sortDataRows instance for sorting each row read ad writing to sort temp file
   */
  private SortDataRows sortDataRows;
  /**
   * final merger for merge sort
   */
  private SingleThreadFinalSortFilesMerger finalMerger;
  /**
   * data handler VO object
   */
  private CarbonFactHandler dataHandler;
  /**
   * segment properties for getting dimension cardinality and other required information of a block
   */
  private SegmentProperties segmentProperties;
  /**
   * compaction type to decide whether taskID need to be extracted from carbondata files
   */
  private CompactionType compactionType;
  /**
   * boolean mapping for no dictionary columns in schema
   */
  private boolean[] noDictionaryColMapping;

  private boolean[] sortColumnMapping;
  /**
   * boolean mapping for long string dimension
   */
  private boolean[] isVarcharDimMapping;
  /**
   * agg type defined for measures
   */
  private DataType[] dataTypes;
  /**
   * segment id
   */
  private String segmentId;
  /**
   * temp store location to be sued during data load
   */
  private String[] tempStoreLocation;
  /**
   * table name
   */
  private String tableName;
  /**
   * no dictionary column count in schema
   */
  private int noDictionaryCount;
  /**
   * total count of measures in schema
   */
  private int measureCount;
  /**
   * dimension count excluding complex dimension and no dictionary column count
   */
  private int dimensionColumnCount;
  /**
   * whether the allocated tasks has any record
   */
  private boolean isRecordFound;
  /**
   * intermediate sort merger
   */
  private SortIntermediateFileMerger intermediateFileMerger;

  private PartitionSpec partitionSpec;

  private SortParameters sortParameters;

  public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
      SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
      PartitionSpec partitionSpec) {
    this.carbonLoadModel = carbonLoadModel;
    this.carbonTable = carbonTable;
    this.segmentProperties = segmentProperties;
    this.segmentId = carbonLoadModel.getSegmentId();
    this.compactionType = compactionType;
    this.tableName = tableName;
    this.partitionSpec = partitionSpec;
  }

  /**
   * This method will iterate over the query result and convert it into a format compatible
   * for data loading
   *
   * @param resultIteratorList
   */
  public boolean execute(List<RawResultIterator> resultIteratorList) throws Exception {
    boolean isCompactionSuccess = false;
    try {
      initTempStoreLocation();
      initSortDataRows();
      dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
      processResult(resultIteratorList);
      // After delete command, if no records are fetched from one split,
      // below steps are not required to be initialized.
      if (isRecordFound) {
        initializeFinalThreadMergerForMergeSort();
        initDataHandler();
        readAndLoadDataFromSortTempFiles();
      }
      isCompactionSuccess = true;
    } catch (Exception e) {
      LOGGER.error(e.getLocalizedMessage(), e);
      throw e;
    } finally {
      if (partitionSpec != null) {
        try {
          SegmentFileStore
              .writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),
                  partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",
                  partitionSpec.getPartitions());
        } catch (IOException e) {
          throw e;
        }
      }
      // clear temp files and folders created during compaction
      deleteTempStoreLocation();
    }
    return isCompactionSuccess;
  }

  @Override
  public void close() {
    // close the sorter executor service
    if (null != sortDataRows) {
      sortDataRows.close();
    }
    // close the final merger
    if (null != finalMerger) {
      finalMerger.close();
    }
    // close data handler
    if (null != dataHandler) {
      dataHandler.closeHandler();
    }
  }

  /**
   * This method will clean up the local folders and files created during compaction process
   */
  private void deleteTempStoreLocation() {
    if (null != tempStoreLocation) {
      for (String tempLoc : tempStoreLocation) {
        try {
          CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));
        } catch (IOException | InterruptedException e) {
          LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());
        }
      }
    }
  }

  /**
   * This method will iterate over the query result and perform row sorting operation
   *
   * @param resultIteratorList
   */
  private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
    for (RawResultIterator resultIterator : resultIteratorList) {
      if (CompactionType.STREAMING == compactionType) {
        while (resultIterator.hasNext()) {
          // the input iterator of streaming segment is already using raw row
          addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      } else {
        while (resultIterator.hasNext()) {
          addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      }
      resultIterator.close();
    }
    try {
      sortDataRows.startSorting();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
    List<CarbonDimension> dimensions = segmentProperties.getDimensions();
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY)) {
        // dictionary
        preparedRow[i] = row[i];
      } else {
        // no dictionary dims
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = row[i];
        }
      }
    }
    // fill all the measures
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];
    }
    return preparedRow;
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareRowObjectForSorting(Object[] row) {
    ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];
    // ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
    List<CarbonDimension> dimensions = segmentProperties.getDimensions();
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    // convert the dictionary from MDKey to surrogate key
    byte[] dictionaryKey = wrapper.getDictionaryKey();
    long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);
    Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];
    for (int i = 0; i < keyArray.length; i++) {
      dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();
    }
    int noDictionaryIndex = 0;
    int dictionaryIndex = 0;
    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY)) {
        // dictionary
        preparedRow[i] = dictionaryValues[dictionaryIndex++];
      } else {
        // no dictionary dims
        byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,
                  dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = noDictionaryKeyByIndex;
        }
      }
    }
    // fill all the measures
    // measures will always start from 1st index in the row object array
    int measureIndexInRow = 1;
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] =
          getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);
    }
    return preparedRow;
  }

  /**
   * This method will convert the spark decimal to java big decimal type
   *
   * @param value
   * @param type
   * @return
   */
  private Object getConvertedMeasureValue(Object value, DataType type) {
    if (DataTypes.isDecimal(type)) {
      if (value != null) {
        value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);
      }
      return value;
    } else {
      return value;
    }
  }

  /**
   * This method will read sort temp files, perform merge sort and add it to store for data loading
   */
  private void readAndLoadDataFromSortTempFiles() throws Exception {
    try {
      intermediateFileMerger.finish();
      finalMerger.startFinalMerge();
      while (finalMerger.hasNext()) {
        Object[] row = finalMerger.next();
        dataHandler.addDataToStore(new CarbonRow(row));
      }
      dataHandler.finish();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } catch (Exception e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } finally {
      if (null != dataHandler) {
        try {
          dataHandler.closeHandler();
        } catch (CarbonDataWriterException e) {
          LOGGER.error("Error in close data handler", e);
          throw new Exception("Error in close data handler", e);
        }
      }
    }
  }

  /**
   * add row to a temp array which will we written to a sort temp file after sorting
   *
   * @param row
   */
  private void addRowForSorting(Object[] row) throws Exception {
    try {
      sortDataRows.addRow(row);
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());
    }
  }

  /**
   * create an instance of sort data rows
   */
  private void initSortDataRows() throws Exception {
    measureCount = carbonTable.getMeasureByTableName(tableName).size();
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
    noDictionaryColMapping = new boolean[dimensions.size()];
    sortColumnMapping = new boolean[dimensions.size()];
    isVarcharDimMapping = new boolean[dimensions.size()];
    int i = 0;
    for (CarbonDimension dimension : dimensions) {
      if (dimension.isSortColumn()) {
        sortColumnMapping[i] = true;
      }
      if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
        i++;
        continue;
      }
      noDictionaryColMapping[i] = true;
      if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {
        isVarcharDimMapping[i] = true;
      }
      i++;
      noDictionaryCount++;
    }
    dimensionColumnCount = dimensions.size();
    sortParameters = createSortParameters();
    intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
    // TODO: Now it is only supported onheap merge, but we can have unsafe merge
    // as well by using UnsafeSortDataRows.
    this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
    try {
      this.sortDataRows.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception(
          "Error initializing sort data rows object during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will create the sort parameters VO object
   *
   * @return
   */
  private SortParameters createSortParameters() {
    return SortParameters
        .createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,
            dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,
            noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,
            sortColumnMapping, isVarcharDimMapping, true);
  }

  /**
   * create an instance of finalThread merger which will perform merge sort on all the
   * sort temp files
   */
  private void initializeFinalThreadMergerForMergeSort() {
    boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil
        .getNoDictSortColMapping(carbonTable);
    sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
    String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,
        CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    finalMerger =
        new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
  }

  /**
   * initialise carbon data writer instance
   */
  private void initDataHandler() throws Exception {
    String carbonStoreLocation;
    if (partitionSpec != null) {
      carbonStoreLocation =
          partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR
              + carbonLoadModel.getFactTimeStamp() + ".tmp";
    } else {
      carbonStoreLocation = CarbonDataProcessorUtil
          .createCarbonStoreLocation(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(),
              carbonLoadModel.getSegmentId());
    }
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
        .getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
            tempStoreLocation, carbonStoreLocation);
    carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());
    setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);
    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);
    try {
      dataHandler.initialise();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());
    }
  }

  /**
   * initialise temporary store location
   */
  private void initTempStoreLocation() {
    tempStoreLocation = CarbonDataProcessorUtil
        .getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),
           segmentId, true, false);
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.processing.merger;
18
19	import java.io.File;
20	import java.io.IOException;
21	import java.util.List;
22
23	import org.apache.carbondata.common.logging.LogServiceFactory;
24	import org.apache.carbondata.core.constants.CarbonCommonConstants;
25	import org.apache.carbondata.core.datastore.block.SegmentProperties;
26	import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
27	import org.apache.carbondata.core.datastore.row.CarbonRow;
28	import org.apache.carbondata.core.indexstore.PartitionSpec;
29	import org.apache.carbondata.core.metadata.SegmentFileStore;
30	import org.apache.carbondata.core.metadata.datatype.DataType;
31	import org.apache.carbondata.core.metadata.datatype.DataTypes;
32	import org.apache.carbondata.core.metadata.encoder.Encoding;
33	import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
34	import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
35	import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
36	import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
37	import org.apache.carbondata.core.util.CarbonUtil;
38	import org.apache.carbondata.core.util.DataTypeUtil;
39	import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
40	import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
41	import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
42	import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
43	import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
44	import org.apache.carbondata.processing.sort.sortdata.SortParameters;
45	import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
46	import org.apache.carbondata.processing.store.CarbonFactHandler;
47	import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
48	import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
49
50	import org.apache.log4j.Logger;
51
52	/**
53	* This class will process the query result and convert the data
54	* into a format compatible for data load
55	*/
56	public class CompactionResultSortProcessor extends AbstractResultProcessor {
57
58	/**
59	* LOGGER
60	*/
61	private static final Logger LOGGER =	1✔
62	LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());	1✔
63	/**
64	* carbon load model that contains all the required information for load
65	*/
66	private CarbonLoadModel carbonLoadModel;
67	/**
68	* carbon table
69	*/
70	private CarbonTable carbonTable;
71	/**
72	* sortDataRows instance for sorting each row read ad writing to sort temp file
73	*/
74	private SortDataRows sortDataRows;
75	/**
76	* final merger for merge sort
77	*/
78	private SingleThreadFinalSortFilesMerger finalMerger;
79	/**
80	* data handler VO object
81	*/
82	private CarbonFactHandler dataHandler;
83	/**
84	* segment properties for getting dimension cardinality and other required information of a block
85	*/
86	private SegmentProperties segmentProperties;
87	/**
88	* compaction type to decide whether taskID need to be extracted from carbondata files
89	*/
90	private CompactionType compactionType;
91	/**
92	* boolean mapping for no dictionary columns in schema
93	*/
94	private boolean[] noDictionaryColMapping;
95
96	private boolean[] sortColumnMapping;
97	/**
98	* boolean mapping for long string dimension
99	*/
100	private boolean[] isVarcharDimMapping;
101	/**
102	* agg type defined for measures
103	*/
104	private DataType[] dataTypes;
105	/**
106	* segment id
107	*/
108	private String segmentId;
109	/**
110	* temp store location to be sued during data load
111	*/
112	private String[] tempStoreLocation;
113	/**
114	* table name
115	*/
116	private String tableName;
117	/**
118	* no dictionary column count in schema
119	*/
120	private int noDictionaryCount;
121	/**
122	* total count of measures in schema
123	*/
124	private int measureCount;
125	/**
126	* dimension count excluding complex dimension and no dictionary column count
127	*/
128	private int dimensionColumnCount;
129	/**
130	* whether the allocated tasks has any record
131	*/
132	private boolean isRecordFound;
133	/**
134	* intermediate sort merger
135	*/
136	private SortIntermediateFileMerger intermediateFileMerger;
137
138	private PartitionSpec partitionSpec;
139
140	private SortParameters sortParameters;
141
142	public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
143	SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
144	PartitionSpec partitionSpec) {	1✔
145	this.carbonLoadModel = carbonLoadModel;	1✔
146	this.carbonTable = carbonTable;	1✔
147	this.segmentProperties = segmentProperties;	1✔
148	this.segmentId = carbonLoadModel.getSegmentId();	1✔
149	this.compactionType = compactionType;	1✔
150	this.tableName = tableName;	1✔
151	this.partitionSpec = partitionSpec;	1✔
152	}	1✔
153
154	/**
155	* This method will iterate over the query result and convert it into a format compatible
156	* for data loading
157	*
158	* @param resultIteratorList
159	*/
160	public boolean execute(List<RawResultIterator> resultIteratorList) throws Exception {
161	boolean isCompactionSuccess = false;	1✔
162	try {
163	initTempStoreLocation();	1✔
164	initSortDataRows();	1✔
165	dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);	1✔
166	processResult(resultIteratorList);	1✔
167	// After delete command, if no records are fetched from one split,
168	// below steps are not required to be initialized.
169	if (isRecordFound) {	1✔
170	initializeFinalThreadMergerForMergeSort();	1✔
171	initDataHandler();	1✔
172	readAndLoadDataFromSortTempFiles();	1✔
173	}
174	isCompactionSuccess = true;	1✔
175	} catch (Exception e) {	×
176	LOGGER.error(e.getLocalizedMessage(), e);	×
177	throw e;	×
178	} finally {
179	if (partitionSpec != null) {	1✔
180	try {
181	SegmentFileStore	×
182	.writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),	×
183	partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",	×
184	partitionSpec.getPartitions());	×
185	} catch (IOException e) {	×
186	throw e;	×
187	}	×
188	}
189	// clear temp files and folders created during compaction
190	deleteTempStoreLocation();	1✔
191	}	1✔
192	return isCompactionSuccess;	1✔
193	}
194
195	@Override
196	public void close() {
197	// close the sorter executor service
198	if (null != sortDataRows) {	1✔
199	sortDataRows.close();	1✔
200	}
201	// close the final merger
202	if (null != finalMerger) {	1✔
203	finalMerger.close();	1✔
204	}
205	// close data handler
206	if (null != dataHandler) {	1✔
207	dataHandler.closeHandler();	1✔
208	}
209	}	1✔
210
211	/**
212	* This method will clean up the local folders and files created during compaction process
213	*/
214	private void deleteTempStoreLocation() {
215	if (null != tempStoreLocation) {	1✔
216	for (String tempLoc : tempStoreLocation) {	1✔
217	try {
218	CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));	1✔
219	} catch (IOException \| InterruptedException e) {	×
220	LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());	×
221	}	1✔
222	}
223	}
224	}	1✔
225
226	/**
227	* This method will iterate over the query result and perform row sorting operation
228	*
229	* @param resultIteratorList
230	*/
231	private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
232	for (RawResultIterator resultIterator : resultIteratorList) {	1✔
233	if (CompactionType.STREAMING == compactionType) {	1✔
234	while (resultIterator.hasNext()) {	1✔
235	// the input iterator of streaming segment is already using raw row
236	addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));	1✔
237	isRecordFound = true;	1✔
238	}
239	} else {
240	while (resultIterator.hasNext()) {	1✔
241	addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));	1✔
242	isRecordFound = true;	1✔
243	}
244	}
245	resultIterator.close();	1✔
246	}	1✔
247	try {
248	sortDataRows.startSorting();	1✔
249	} catch (CarbonSortKeyAndGroupByException e) {	×
250	LOGGER.error(e);	×
251	throw new Exception("Problem loading data during compaction: " + e.getMessage());	×
252	}	1✔
253	}	1✔
254
255	/**
256	* This method will prepare the data from raw object that will take part in sorting
257	*
258	* @param row
259	* @return
260	*/
261	private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
262	List<CarbonDimension> dimensions = segmentProperties.getDimensions();	1✔
263	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
264	for (int i = 0; i < dimensions.size(); i++) {	1✔
265	CarbonDimension dims = dimensions.get(i);	1✔
266	if (dims.hasEncoding(Encoding.DICTIONARY)) {	1✔
267	// dictionary
268	preparedRow[i] = row[i];	1✔
269	} else {
270	// no dictionary dims
271	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {	1✔
272	// no dictionary measure columns are expected as original data
273	preparedRow[i] = DataTypeUtil	1✔
274	.getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());	1✔
275	// for timestamp the above method will give the original data, so it should be
276	// converted again to the format to be loaded (without micros)
277	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
278	preparedRow[i] = (long) preparedRow[i] / 1000L;	1✔
279	}
280	} else {
281	preparedRow[i] = row[i];	1✔
282	}
283	}
284	}
285	// fill all the measures
286	for (int i = 0; i < measureCount; i++) {	1✔
287	preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];	1✔
288	}
289	return preparedRow;	1✔
290	}
291
292	/**
293	* This method will prepare the data from raw object that will take part in sorting
294	*
295	* @param row
296	* @return
297	*/
298	private Object[] prepareRowObjectForSorting(Object[] row) {
299	ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];	1✔
300	// ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
301	List<CarbonDimension> dimensions = segmentProperties.getDimensions();	1✔
302	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
303	// convert the dictionary from MDKey to surrogate key
304	byte[] dictionaryKey = wrapper.getDictionaryKey();	1✔
305	long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);	1✔
306	Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];	1✔
307	for (int i = 0; i < keyArray.length; i++) {	1✔
308	dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();	×
309	}
310	int noDictionaryIndex = 0;	1✔
311	int dictionaryIndex = 0;	1✔
312	for (int i = 0; i < dimensions.size(); i++) {	1✔
313	CarbonDimension dims = dimensions.get(i);	1✔
314	if (dims.hasEncoding(Encoding.DICTIONARY)) {	1✔
315	// dictionary
316	preparedRow[i] = dictionaryValues[dictionaryIndex++];	×
317	} else {
318	// no dictionary dims
319	byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);	1✔
320	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {	1✔
321	// no dictionary measure columns are expected as original data
322	preparedRow[i] = DataTypeUtil	1✔
323	.getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,	1✔
324	dims.getDataType());	1✔
325	// for timestamp the above method will give the original data, so it should be
326	// converted again to the format to be loaded (without micros)
327	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
328	preparedRow[i] = (long) preparedRow[i] / 1000L;	1✔
329	}
330	} else {
331	preparedRow[i] = noDictionaryKeyByIndex;	1✔
332	}
333	}
334	}
335	// fill all the measures
336	// measures will always start from 1st index in the row object array
337	int measureIndexInRow = 1;	1✔
338	for (int i = 0; i < measureCount; i++) {	1✔
339	preparedRow[dimensionColumnCount + i] =	1✔
340	getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);	1✔
341	}
342	return preparedRow;	1✔
343	}
344
345	/**
346	* This method will convert the spark decimal to java big decimal type
347	*
348	* @param value
349	* @param type
350	* @return
351	*/
352	private Object getConvertedMeasureValue(Object value, DataType type) {
353	if (DataTypes.isDecimal(type)) {	1✔
354	if (value != null) {	1✔
355	value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);	1✔
356	}
357	return value;	1✔
358	} else {
359	return value;	1✔
360	}
361	}
362
363	/**
364	* This method will read sort temp files, perform merge sort and add it to store for data loading
365	*/
366	private void readAndLoadDataFromSortTempFiles() throws Exception {
367	try {
368	intermediateFileMerger.finish();	1✔
369	finalMerger.startFinalMerge();	1✔
370	while (finalMerger.hasNext()) {	1✔
371	Object[] row = finalMerger.next();	1✔
372	dataHandler.addDataToStore(new CarbonRow(row));	1✔
373	}	1✔
374	dataHandler.finish();	1✔
375	} catch (CarbonDataWriterException e) {	×
376	LOGGER.error(e);	×
377	throw new Exception("Problem loading data during compaction.", e);	×
378	} catch (Exception e) {	×
379	LOGGER.error(e);	×
380	throw new Exception("Problem loading data during compaction.", e);	×
381	} finally {
382	if (null != dataHandler) {	1✔
383	try {
384	dataHandler.closeHandler();	1✔
385	} catch (CarbonDataWriterException e) {	×
386	LOGGER.error("Error in close data handler", e);	×
387	throw new Exception("Error in close data handler", e);	×
388	}	1✔
389	}
390	}	×
391	}	1✔
392
393	/**
394	* add row to a temp array which will we written to a sort temp file after sorting
395	*
396	* @param row
397	*/
398	private void addRowForSorting(Object[] row) throws Exception {
399	try {
400	sortDataRows.addRow(row);	1✔
401	} catch (CarbonSortKeyAndGroupByException e) {	×
402	LOGGER.error(e);	×
403	throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());	×
404	}	1✔
405	}	1✔
406
407	/**
408	* create an instance of sort data rows
409	*/
410	private void initSortDataRows() throws Exception {
411	measureCount = carbonTable.getMeasureByTableName(tableName).size();	1✔
412	List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);	1✔
413	noDictionaryColMapping = new boolean[dimensions.size()];	1✔
414	sortColumnMapping = new boolean[dimensions.size()];	1✔
415	isVarcharDimMapping = new boolean[dimensions.size()];	1✔
416	int i = 0;	1✔
417	for (CarbonDimension dimension : dimensions) {	1✔
418	if (dimension.isSortColumn()) {	1✔
419	sortColumnMapping[i] = true;	1✔
420	}
421	if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {	1✔
422	i++;	1✔
423	continue;	1✔
424	}
425	noDictionaryColMapping[i] = true;	1✔
426	if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {	1✔
427	isVarcharDimMapping[i] = true;	×
428	}
429	i++;	1✔
430	noDictionaryCount++;	1✔
431	}	1✔
432	dimensionColumnCount = dimensions.size();	1✔
433	sortParameters = createSortParameters();	1✔
434	intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);	1✔
435	// TODO: Now it is only supported onheap merge, but we can have unsafe merge
436	// as well by using UnsafeSortDataRows.
437	this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);	1✔
438	try {
439	this.sortDataRows.initialize();	1✔
440	} catch (CarbonSortKeyAndGroupByException e) {	×
441	LOGGER.error(e);	×
442	throw new Exception(	×
443	"Error initializing sort data rows object during compaction: " + e.getMessage());	×
444	}	1✔
445	}	1✔
446
447	/**
448	* This method will create the sort parameters VO object
449	*
450	* @return
451	*/
452	private SortParameters createSortParameters() {
453	return SortParameters	1✔
454	.createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,	1✔
455	dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,	1✔
456	noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,	1✔
457	sortColumnMapping, isVarcharDimMapping, true);
458	}
459
460	/**
461	* create an instance of finalThread merger which will perform merge sort on all the
462	* sort temp files
463	*/
464	private void initializeFinalThreadMergerForMergeSort() {
465	boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil	1✔
466	.getNoDictSortColMapping(carbonTable);	1✔
467	sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);	1✔
468	String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,	1✔
469	CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
470	finalMerger =	1✔
471	new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
472	}	1✔
473
474	/**
475	* initialise carbon data writer instance
476	*/
477	private void initDataHandler() throws Exception {
478	String carbonStoreLocation;
479	if (partitionSpec != null) {	1✔
480	carbonStoreLocation =	×
481	partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR	×
482	+ carbonLoadModel.getFactTimeStamp() + ".tmp";	×
483	} else {
484	carbonStoreLocation = CarbonDataProcessorUtil	1✔
485	.createCarbonStoreLocation(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(),	1✔
486	carbonLoadModel.getSegmentId());	1✔
487	}
488	CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel	1✔
489	.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,	1✔
490	tempStoreLocation, carbonStoreLocation);
491	carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());	1✔
492	setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);	1✔
493	dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);	1✔
494	try {
495	dataHandler.initialise();	1✔
496	} catch (CarbonDataWriterException e) {	×
497	LOGGER.error(e);	×
498	throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());	×
499	}	1✔
500	}	1✔
501
502	/**
503	* initialise temporary store location
504	*/
505	private void initTempStoreLocation() {
506	tempStoreLocation = CarbonDataProcessorUtil	1✔
507	.getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),	1✔
508	segmentId, true, false);
509	}	1✔
510	}

apache / carbondata / 1394

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous