1825

Build # 1825

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/68300ecd0729c03382240e52127a47d3854a05db">68300ecd0<a href="https://github.com/apache/carbondata/commit/68300ecd0729c03382240e52127a47d3854a05db">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/c2fc6d5964b56d4a111a13b6f9dad7ed6748934f">c2fc6d596</a>">c2fc6d596</a><a href="https://github.com/apache/carbondata/commit/68300ecd0729c03382240e52127a47d3854a05db"> into 382ce430a">382ce430a</a>

Run Details

21 of 21 new or added lines in 2 files covered. (100.0%)

61607 of 77489 relevant lines covered (79.5%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.5

/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.processing.merger;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.carbondata.processing.store.CarbonFactHandler;
import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;

import org.apache.log4j.Logger;

/**
 * This class will process the query result and convert the data
 * into a format compatible for data load
 */
public class CompactionResultSortProcessor extends AbstractResultProcessor {

  /**
   * LOGGER
   */
  private static final Logger LOGGER =
      LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());
  /**
   * carbon load model that contains all the required information for load
   */
  private CarbonLoadModel carbonLoadModel;
  /**
   * carbon table
   */
  private CarbonTable carbonTable;
  /**
   * sortDataRows instance for sorting each row read ad writing to sort temp file
   */
  private SortDataRows sortDataRows;
  /**
   * final merger for merge sort
   */
  private SingleThreadFinalSortFilesMerger finalMerger;
  /**
   * data handler VO object
   */
  private CarbonFactHandler dataHandler;
  /**
   * segment properties for getting dimension cardinality and other required information of a block
   */
  private SegmentProperties segmentProperties;
  /**
   * compaction type to decide whether taskID need to be extracted from carbondata files
   */
  private CompactionType compactionType;
  /**
   * boolean mapping for no dictionary columns in schema
   */
  private boolean[] noDictionaryColMapping;

  private boolean[] sortColumnMapping;
  /**
   * boolean mapping for long string dimension
   */
  private boolean[] isVarcharDimMapping;
  /**
   * agg type defined for measures
   */
  private DataType[] dataTypes;
  /**
   * segment id
   */
  private String segmentId;
  /**
   * temp store location to be sued during data load
   */
  private String[] tempStoreLocation;
  /**
   * table name
   */
  private String tableName;
  /**
   * no dictionary column count in schema
   */
  private int noDictionaryCount;
  /**
   * total count of measures in schema
   */
  private int measureCount;
  /**
   * dimension count excluding complex dimension and no dictionary column count
   */
  private int dimensionColumnCount;
  /**
   * whether the allocated tasks has any record
   */
  private boolean isRecordFound;
  /**
   * intermediate sort merger
   */
  private SortIntermediateFileMerger intermediateFileMerger;

  private PartitionSpec partitionSpec;

  private SortParameters sortParameters;

  public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
      SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
      PartitionSpec partitionSpec) {
    this.carbonLoadModel = carbonLoadModel;
    this.carbonTable = carbonTable;
    this.segmentProperties = segmentProperties;
    this.segmentId = carbonLoadModel.getSegmentId();
    this.compactionType = compactionType;
    this.tableName = tableName;
    this.partitionSpec = partitionSpec;
  }

  /**
   * This method will iterate over the query result and convert it into a format compatible
   * for data loading
   *
   * @param resultIteratorList
   */
  public boolean execute(List<RawResultIterator> resultIteratorList) throws Exception {
    boolean isCompactionSuccess = false;
    try {
      initTempStoreLocation();
      initSortDataRows();
      dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
      processResult(resultIteratorList);
      // After delete command, if no records are fetched from one split,
      // below steps are not required to be initialized.
      if (isRecordFound) {
        initializeFinalThreadMergerForMergeSort();
        initDataHandler();
        readAndLoadDataFromSortTempFiles();
      }
      isCompactionSuccess = true;
    } catch (Exception e) {
      LOGGER.error(e.getLocalizedMessage(), e);
      throw e;
    } finally {
      if (partitionSpec != null) {
        try {
          SegmentFileStore
              .writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),
                  partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",
                  partitionSpec.getPartitions());
        } catch (IOException e) {
          throw e;
        }
      }
      // clear temp files and folders created during compaction
      deleteTempStoreLocation();
    }
    return isCompactionSuccess;
  }

  @Override
  public void close() {
    // close the sorter executor service
    if (null != sortDataRows) {
      sortDataRows.close();
    }
    // close the final merger
    if (null != finalMerger) {
      finalMerger.close();
    }
    // close data handler
    if (null != dataHandler) {
      dataHandler.closeHandler();
    }
  }

  /**
   * This method will clean up the local folders and files created during compaction process
   */
  private void deleteTempStoreLocation() {
    if (null != tempStoreLocation) {
      for (String tempLoc : tempStoreLocation) {
        try {
          CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));
        } catch (IOException | InterruptedException e) {
          LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());
        }
      }
    }
  }

  /**
   * This method will iterate over the query result and perform row sorting operation
   *
   * @param resultIteratorList
   */
  private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
    for (RawResultIterator resultIterator : resultIteratorList) {
      if (CompactionType.STREAMING == compactionType) {
        while (resultIterator.hasNext()) {
          // the input iterator of streaming segment is already using raw row
          addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      } else {
        while (resultIterator.hasNext()) {
          addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      }
      resultIterator.close();
    }
    try {
      sortDataRows.startSorting();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
    List<CarbonDimension> dimensions = segmentProperties.getDimensions();
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY)) {
        // dictionary
        preparedRow[i] = row[i];
      } else {
        // no dictionary dims
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = row[i];
        }
      }
    }
    // fill all the measures
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];
    }
    return preparedRow;
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareRowObjectForSorting(Object[] row) {
    ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];
    // ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
    List<CarbonDimension> dimensions = segmentProperties.getDimensions();
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    // convert the dictionary from MDKey to surrogate key
    byte[] dictionaryKey = wrapper.getDictionaryKey();
    long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);
    Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];
    for (int i = 0; i < keyArray.length; i++) {
      dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();
    }
    int noDictionaryIndex = 0;
    int dictionaryIndex = 0;
    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY)) {
        // dictionary
        preparedRow[i] = dictionaryValues[dictionaryIndex++];
      } else {
        // no dictionary dims
        byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,
                  dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = noDictionaryKeyByIndex;
        }
      }
    }
    // fill all the measures
    // measures will always start from 1st index in the row object array
    int measureIndexInRow = 1;
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] =
          getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);
    }
    return preparedRow;
  }

  /**
   * This method will convert the spark decimal to java big decimal type
   *
   * @param value
   * @param type
   * @return
   */
  private Object getConvertedMeasureValue(Object value, DataType type) {
    if (DataTypes.isDecimal(type)) {
      if (value != null) {
        value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);
      }
      return value;
    } else {
      return value;
    }
  }

  /**
   * This method will read sort temp files, perform merge sort and add it to store for data loading
   */
  private void readAndLoadDataFromSortTempFiles() throws Exception {
    try {
      intermediateFileMerger.finish();
      finalMerger.startFinalMerge();
      while (finalMerger.hasNext()) {
        Object[] row = finalMerger.next();
        dataHandler.addDataToStore(new CarbonRow(row));
      }
      dataHandler.finish();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } catch (Exception e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } finally {
      if (null != dataHandler) {
        try {
          dataHandler.closeHandler();
        } catch (CarbonDataWriterException e) {
          LOGGER.error("Error in close data handler", e);
          throw new Exception("Error in close data handler", e);
        }
      }
    }
  }

  /**
   * add row to a temp array which will we written to a sort temp file after sorting
   *
   * @param row
   */
  private void addRowForSorting(Object[] row) throws Exception {
    try {
      sortDataRows.addRow(row);
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());
    }
  }

  /**
   * create an instance of sort data rows
   */
  private void initSortDataRows() throws Exception {
    measureCount = carbonTable.getMeasureByTableName(tableName).size();
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
    noDictionaryColMapping = new boolean[dimensions.size()];
    sortColumnMapping = new boolean[dimensions.size()];
    isVarcharDimMapping = new boolean[dimensions.size()];
    int i = 0;
    for (CarbonDimension dimension : dimensions) {
      if (dimension.isSortColumn()) {
        sortColumnMapping[i] = true;
      }
      if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
        i++;
        continue;
      }
      noDictionaryColMapping[i] = true;
      if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {
        isVarcharDimMapping[i] = true;
      }
      i++;
      noDictionaryCount++;
    }
    dimensionColumnCount = dimensions.size();
    sortParameters = createSortParameters();
    intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
    // TODO: Now it is only supported onheap merge, but we can have unsafe merge
    // as well by using UnsafeSortDataRows.
    this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
    try {
      this.sortDataRows.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception(
          "Error initializing sort data rows object during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will create the sort parameters VO object
   *
   * @return
   */
  private SortParameters createSortParameters() {
    int numberOfCompactingCores = CarbonProperties.getInstance().getNumberOfCompactingCores();
    return SortParameters
        .createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,
            dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,
            noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,
            sortColumnMapping, isVarcharDimMapping, true, numberOfCompactingCores / 2);
  }

  /**
   * create an instance of finalThread merger which will perform merge sort on all the
   * sort temp files
   */
  private void initializeFinalThreadMergerForMergeSort() {
    boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil
        .getNoDictSortColMapping(carbonTable);
    sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
    String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,
        CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    finalMerger =
        new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
  }

  /**
   * initialise carbon data writer instance
   */
  private void initDataHandler() throws Exception {
    String carbonStoreLocation;
    if (partitionSpec != null) {
      carbonStoreLocation =
          partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR
              + carbonLoadModel.getFactTimeStamp() + ".tmp";
    } else {
      carbonStoreLocation = CarbonDataProcessorUtil
          .createCarbonStoreLocation(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(),
              carbonLoadModel.getSegmentId());
    }
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
        .getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
            tempStoreLocation, carbonStoreLocation);
    carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());
    setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);
    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);
    try {
      dataHandler.initialise();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());
    }
  }

  /**
   * initialise temporary store location
   */
  private void initTempStoreLocation() {
    tempStoreLocation = CarbonDataProcessorUtil
        .getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),
           segmentId, true, false);
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.processing.merger;
18
19	import java.io.File;
20	import java.io.IOException;
21	import java.util.List;
22
23	import org.apache.carbondata.common.logging.LogServiceFactory;
24	import org.apache.carbondata.core.constants.CarbonCommonConstants;
25	import org.apache.carbondata.core.datastore.block.SegmentProperties;
26	import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
27	import org.apache.carbondata.core.datastore.row.CarbonRow;
28	import org.apache.carbondata.core.indexstore.PartitionSpec;
29	import org.apache.carbondata.core.metadata.SegmentFileStore;
30	import org.apache.carbondata.core.metadata.datatype.DataType;
31	import org.apache.carbondata.core.metadata.datatype.DataTypes;
32	import org.apache.carbondata.core.metadata.encoder.Encoding;
33	import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
34	import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
35	import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
36	import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
37	import org.apache.carbondata.core.util.CarbonProperties;
38	import org.apache.carbondata.core.util.CarbonUtil;
39	import org.apache.carbondata.core.util.DataTypeUtil;
40	import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
41	import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
42	import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
43	import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
44	import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
45	import org.apache.carbondata.processing.sort.sortdata.SortParameters;
46	import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
47	import org.apache.carbondata.processing.store.CarbonFactHandler;
48	import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
49	import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
50
51	import org.apache.log4j.Logger;
52
53	/**
54	* This class will process the query result and convert the data
55	* into a format compatible for data load
56	*/
57	public class CompactionResultSortProcessor extends AbstractResultProcessor {
58
59	/**
60	* LOGGER
61	*/
62	private static final Logger LOGGER =	1✔
63	LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());	1✔
64	/**
65	* carbon load model that contains all the required information for load
66	*/
67	private CarbonLoadModel carbonLoadModel;
68	/**
69	* carbon table
70	*/
71	private CarbonTable carbonTable;
72	/**
73	* sortDataRows instance for sorting each row read ad writing to sort temp file
74	*/
75	private SortDataRows sortDataRows;
76	/**
77	* final merger for merge sort
78	*/
79	private SingleThreadFinalSortFilesMerger finalMerger;
80	/**
81	* data handler VO object
82	*/
83	private CarbonFactHandler dataHandler;
84	/**
85	* segment properties for getting dimension cardinality and other required information of a block
86	*/
87	private SegmentProperties segmentProperties;
88	/**
89	* compaction type to decide whether taskID need to be extracted from carbondata files
90	*/
91	private CompactionType compactionType;
92	/**
93	* boolean mapping for no dictionary columns in schema
94	*/
95	private boolean[] noDictionaryColMapping;
96
97	private boolean[] sortColumnMapping;
98	/**
99	* boolean mapping for long string dimension
100	*/
101	private boolean[] isVarcharDimMapping;
102	/**
103	* agg type defined for measures
104	*/
105	private DataType[] dataTypes;
106	/**
107	* segment id
108	*/
109	private String segmentId;
110	/**
111	* temp store location to be sued during data load
112	*/
113	private String[] tempStoreLocation;
114	/**
115	* table name
116	*/
117	private String tableName;
118	/**
119	* no dictionary column count in schema
120	*/
121	private int noDictionaryCount;
122	/**
123	* total count of measures in schema
124	*/
125	private int measureCount;
126	/**
127	* dimension count excluding complex dimension and no dictionary column count
128	*/
129	private int dimensionColumnCount;
130	/**
131	* whether the allocated tasks has any record
132	*/
133	private boolean isRecordFound;
134	/**
135	* intermediate sort merger
136	*/
137	private SortIntermediateFileMerger intermediateFileMerger;
138
139	private PartitionSpec partitionSpec;
140
141	private SortParameters sortParameters;
142
143	public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
144	SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
145	PartitionSpec partitionSpec) {	1✔
146	this.carbonLoadModel = carbonLoadModel;	1✔
147	this.carbonTable = carbonTable;	1✔
148	this.segmentProperties = segmentProperties;	1✔
149	this.segmentId = carbonLoadModel.getSegmentId();	1✔
150	this.compactionType = compactionType;	1✔
151	this.tableName = tableName;	1✔
152	this.partitionSpec = partitionSpec;	1✔
153	}	1✔
154
155	/**
156	* This method will iterate over the query result and convert it into a format compatible
157	* for data loading
158	*
159	* @param resultIteratorList
160	*/
161	public boolean execute(List<RawResultIterator> resultIteratorList) throws Exception {
162	boolean isCompactionSuccess = false;	1✔
163	try {
164	initTempStoreLocation();	1✔
165	initSortDataRows();	1✔
166	dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);	1✔
167	processResult(resultIteratorList);	1✔
168	// After delete command, if no records are fetched from one split,
169	// below steps are not required to be initialized.
170	if (isRecordFound) {	1✔
171	initializeFinalThreadMergerForMergeSort();	1✔
172	initDataHandler();	1✔
173	readAndLoadDataFromSortTempFiles();	1✔
174	}
175	isCompactionSuccess = true;	1✔
176	} catch (Exception e) {	×
177	LOGGER.error(e.getLocalizedMessage(), e);	×
178	throw e;	×
179	} finally {
180	if (partitionSpec != null) {	1✔
181	try {
182	SegmentFileStore	×
183	.writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),	×
184	partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",	×
185	partitionSpec.getPartitions());	×
186	} catch (IOException e) {	×
187	throw e;	×
188	}	×
189	}
190	// clear temp files and folders created during compaction
191	deleteTempStoreLocation();	1✔
192	}	1✔
193	return isCompactionSuccess;	1✔
194	}
195
196	@Override
197	public void close() {
198	// close the sorter executor service
199	if (null != sortDataRows) {	1✔
200	sortDataRows.close();	1✔
201	}
202	// close the final merger
203	if (null != finalMerger) {	1✔
204	finalMerger.close();	1✔
205	}
206	// close data handler
207	if (null != dataHandler) {	1✔
208	dataHandler.closeHandler();	1✔
209	}
210	}	1✔
211
212	/**
213	* This method will clean up the local folders and files created during compaction process
214	*/
215	private void deleteTempStoreLocation() {
216	if (null != tempStoreLocation) {	1✔
217	for (String tempLoc : tempStoreLocation) {	1✔
218	try {
219	CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));	1✔
220	} catch (IOException \| InterruptedException e) {	×
221	LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());	×
222	}	1✔
223	}
224	}
225	}	1✔
226
227	/**
228	* This method will iterate over the query result and perform row sorting operation
229	*
230	* @param resultIteratorList
231	*/
232	private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
233	for (RawResultIterator resultIterator : resultIteratorList) {	1✔
234	if (CompactionType.STREAMING == compactionType) {	1✔
235	while (resultIterator.hasNext()) {	1✔
236	// the input iterator of streaming segment is already using raw row
237	addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));	1✔
238	isRecordFound = true;	1✔
239	}
240	} else {
241	while (resultIterator.hasNext()) {	1✔
242	addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));	1✔
243	isRecordFound = true;	1✔
244	}
245	}
246	resultIterator.close();	1✔
247	}	1✔
248	try {
249	sortDataRows.startSorting();	1✔
250	} catch (CarbonSortKeyAndGroupByException e) {	×
251	LOGGER.error(e);	×
252	throw new Exception("Problem loading data during compaction: " + e.getMessage());	×
253	}	1✔
254	}	1✔
255
256	/**
257	* This method will prepare the data from raw object that will take part in sorting
258	*
259	* @param row
260	* @return
261	*/
262	private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
263	List<CarbonDimension> dimensions = segmentProperties.getDimensions();	1✔
264	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
265	for (int i = 0; i < dimensions.size(); i++) {	1✔
266	CarbonDimension dims = dimensions.get(i);	1✔
267	if (dims.hasEncoding(Encoding.DICTIONARY)) {	1✔
268	// dictionary
269	preparedRow[i] = row[i];	1✔
270	} else {
271	// no dictionary dims
272	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {	1✔
273	// no dictionary measure columns are expected as original data
274	preparedRow[i] = DataTypeUtil	1✔
275	.getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());	1✔
276	// for timestamp the above method will give the original data, so it should be
277	// converted again to the format to be loaded (without micros)
278	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
279	preparedRow[i] = (long) preparedRow[i] / 1000L;	1✔
280	}
281	} else {
282	preparedRow[i] = row[i];	1✔
283	}
284	}
285	}
286	// fill all the measures
287	for (int i = 0; i < measureCount; i++) {	1✔
288	preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];	1✔
289	}
290	return preparedRow;	1✔
291	}
292
293	/**
294	* This method will prepare the data from raw object that will take part in sorting
295	*
296	* @param row
297	* @return
298	*/
299	private Object[] prepareRowObjectForSorting(Object[] row) {
300	ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];	1✔
301	// ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
302	List<CarbonDimension> dimensions = segmentProperties.getDimensions();	1✔
303	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
304	// convert the dictionary from MDKey to surrogate key
305	byte[] dictionaryKey = wrapper.getDictionaryKey();	1✔
306	long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);	1✔
307	Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];	1✔
308	for (int i = 0; i < keyArray.length; i++) {	1✔
309	dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();	×
310	}
311	int noDictionaryIndex = 0;	1✔
312	int dictionaryIndex = 0;	1✔
313	for (int i = 0; i < dimensions.size(); i++) {	1✔
314	CarbonDimension dims = dimensions.get(i);	1✔
315	if (dims.hasEncoding(Encoding.DICTIONARY)) {	1✔
316	// dictionary
317	preparedRow[i] = dictionaryValues[dictionaryIndex++];	×
318	} else {
319	// no dictionary dims
320	byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);	1✔
321	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {	1✔
322	// no dictionary measure columns are expected as original data
323	preparedRow[i] = DataTypeUtil	1✔
324	.getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,	1✔
325	dims.getDataType());	1✔
326	// for timestamp the above method will give the original data, so it should be
327	// converted again to the format to be loaded (without micros)
328	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
329	preparedRow[i] = (long) preparedRow[i] / 1000L;	1✔
330	}
331	} else {
332	preparedRow[i] = noDictionaryKeyByIndex;	1✔
333	}
334	}
335	}
336	// fill all the measures
337	// measures will always start from 1st index in the row object array
338	int measureIndexInRow = 1;	1✔
339	for (int i = 0; i < measureCount; i++) {	1✔
340	preparedRow[dimensionColumnCount + i] =	1✔
341	getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);	1✔
342	}
343	return preparedRow;	1✔
344	}
345
346	/**
347	* This method will convert the spark decimal to java big decimal type
348	*
349	* @param value
350	* @param type
351	* @return
352	*/
353	private Object getConvertedMeasureValue(Object value, DataType type) {
354	if (DataTypes.isDecimal(type)) {	1✔
355	if (value != null) {	1✔
356	value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);	1✔
357	}
358	return value;	1✔
359	} else {
360	return value;	1✔
361	}
362	}
363
364	/**
365	* This method will read sort temp files, perform merge sort and add it to store for data loading
366	*/
367	private void readAndLoadDataFromSortTempFiles() throws Exception {
368	try {
369	intermediateFileMerger.finish();	1✔
370	finalMerger.startFinalMerge();	1✔
371	while (finalMerger.hasNext()) {	1✔
372	Object[] row = finalMerger.next();	1✔
373	dataHandler.addDataToStore(new CarbonRow(row));	1✔
374	}	1✔
375	dataHandler.finish();	1✔
376	} catch (CarbonDataWriterException e) {	×
377	LOGGER.error(e);	×
378	throw new Exception("Problem loading data during compaction.", e);	×
379	} catch (Exception e) {	×
380	LOGGER.error(e);	×
381	throw new Exception("Problem loading data during compaction.", e);	×
382	} finally {
383	if (null != dataHandler) {	1✔
384	try {
385	dataHandler.closeHandler();	1✔
386	} catch (CarbonDataWriterException e) {	×
387	LOGGER.error("Error in close data handler", e);	×
388	throw new Exception("Error in close data handler", e);	×
389	}	1✔
390	}
391	}	×
392	}	1✔
393
394	/**
395	* add row to a temp array which will we written to a sort temp file after sorting
396	*
397	* @param row
398	*/
399	private void addRowForSorting(Object[] row) throws Exception {
400	try {
401	sortDataRows.addRow(row);	1✔
402	} catch (CarbonSortKeyAndGroupByException e) {	×
403	LOGGER.error(e);	×
404	throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());	×
405	}	1✔
406	}	1✔
407
408	/**
409	* create an instance of sort data rows
410	*/
411	private void initSortDataRows() throws Exception {
412	measureCount = carbonTable.getMeasureByTableName(tableName).size();	1✔
413	List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);	1✔
414	noDictionaryColMapping = new boolean[dimensions.size()];	1✔
415	sortColumnMapping = new boolean[dimensions.size()];	1✔
416	isVarcharDimMapping = new boolean[dimensions.size()];	1✔
417	int i = 0;	1✔
418	for (CarbonDimension dimension : dimensions) {	1✔
419	if (dimension.isSortColumn()) {	1✔
420	sortColumnMapping[i] = true;	1✔
421	}
422	if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {	1✔
423	i++;	1✔
424	continue;	1✔
425	}
426	noDictionaryColMapping[i] = true;	1✔
427	if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {	1✔
428	isVarcharDimMapping[i] = true;	×
429	}
430	i++;	1✔
431	noDictionaryCount++;	1✔
432	}	1✔
433	dimensionColumnCount = dimensions.size();	1✔
434	sortParameters = createSortParameters();	1✔
435	intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);	1✔
436	// TODO: Now it is only supported onheap merge, but we can have unsafe merge
437	// as well by using UnsafeSortDataRows.
438	this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);	1✔
439	try {
440	this.sortDataRows.initialize();	1✔
441	} catch (CarbonSortKeyAndGroupByException e) {	×
442	LOGGER.error(e);	×
443	throw new Exception(	×
444	"Error initializing sort data rows object during compaction: " + e.getMessage());	×
445	}	1✔
446	}	1✔
447
448	/**
449	* This method will create the sort parameters VO object
450	*
451	* @return
452	*/
453	private SortParameters createSortParameters() {
454	int numberOfCompactingCores = CarbonProperties.getInstance().getNumberOfCompactingCores();	1✔
455	return SortParameters	1✔
456	.createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,	1✔
457	dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,	1✔
458	noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,	1✔
459	sortColumnMapping, isVarcharDimMapping, true, numberOfCompactingCores / 2);
460	}
461
462	/**
463	* create an instance of finalThread merger which will perform merge sort on all the
464	* sort temp files
465	*/
466	private void initializeFinalThreadMergerForMergeSort() {
467	boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil	1✔
468	.getNoDictSortColMapping(carbonTable);	1✔
469	sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);	1✔
470	String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,	1✔
471	CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
472	finalMerger =	1✔
473	new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
474	}	1✔
475
476	/**
477	* initialise carbon data writer instance
478	*/
479	private void initDataHandler() throws Exception {
480	String carbonStoreLocation;
481	if (partitionSpec != null) {	1✔
482	carbonStoreLocation =	×
483	partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR	×
484	+ carbonLoadModel.getFactTimeStamp() + ".tmp";	×
485	} else {
486	carbonStoreLocation = CarbonDataProcessorUtil	1✔
487	.createCarbonStoreLocation(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(),	1✔
488	carbonLoadModel.getSegmentId());	1✔
489	}
490	CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel	1✔
491	.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,	1✔
492	tempStoreLocation, carbonStoreLocation);
493	carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());	1✔
494	setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);	1✔
495	dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);	1✔
496	try {
497	dataHandler.initialise();	1✔
498	} catch (CarbonDataWriterException e) {	×
499	LOGGER.error(e);	×
500	throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());	×
501	}	1✔
502	}	1✔
503
504	/**
505	* initialise temporary store location
506	*/
507	private void initTempStoreLocation() {
508	tempStoreLocation = CarbonDataProcessorUtil	1✔
509	.getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),	1✔
510	segmentId, true, false);
511	}	1✔
512	}

apache / carbondata / 1825

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous