apache / carbondata / 876

Build # 876

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/d3576b9eda9003ccff9b0d00af081120c256aaf3">d3576b9ed<a href="https://github.com/apache/carbondata/commit/d3576b9eda9003ccff9b0d00af081120c256aaf3">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/07cea072ef9baa79da5db6238514199a35345f20">07cea072e</a>">07cea072e</a><a href="https://github.com/apache/carbondata/commit/d3576b9eda9003ccff9b0d00af081120c256aaf3"> into 2081bc87a">2081bc87a</a>

Run Details

7 of 7 new or added lines in 1 file covered. (100.0%)

60524 of 75087 relevant lines covered (80.61%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.8

/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.processing.merger;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.carbondata.processing.store.CarbonFactHandler;
import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;

/**
 * This class will process the query result and convert the data
 * into a format compatible for data load
 */
public class CompactionResultSortProcessor extends AbstractResultProcessor {

  /**
   * LOGGER
   */
  private static final LogService LOGGER =
      LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());
  /**
   * carbon load model that contains all the required information for load
   */
  private CarbonLoadModel carbonLoadModel;
  /**
   * carbon table
   */
  private CarbonTable carbonTable;
  /**
   * sortDataRows instance for sorting each row read ad writing to sort temp file
   */
  private SortDataRows sortDataRows;
  /**
   * final merger for merge sort
   */
  private SingleThreadFinalSortFilesMerger finalMerger;
  /**
   * data handler VO object
   */
  private CarbonFactHandler dataHandler;
  /**
   * segment properties for getting dimension cardinality and other required information of a block
   */
  private SegmentProperties segmentProperties;
  /**
   * compaction type to decide whether taskID need to be extracted from carbondata files
   */
  private CompactionType compactionType;
  /**
   * boolean mapping for no dictionary columns in schema
   */
  private boolean[] noDictionaryColMapping;

  private boolean[] sortColumnMapping;
  /**
   * boolean mapping for long string dimension
   */
  private boolean[] isVarcharDimMapping;
  /**
   * agg type defined for measures
   */
  private DataType[] dataTypes;
  /**
   * segment id
   */
  private String segmentId;
  /**
   * temp store location to be sued during data load
   */
  private String[] tempStoreLocation;
  /**
   * table name
   */
  private String tableName;
  /**
   * no dictionary column count in schema
   */
  private int noDictionaryCount;
  /**
   * total count of measures in schema
   */
  private int measureCount;
  /**
   * dimension count excluding complex dimension and no dictionary column count
   */
  private int dimensionColumnCount;
  /**
   * whether the allocated tasks has any record
   */
  private boolean isRecordFound;
  /**
   * intermediate sort merger
   */
  private SortIntermediateFileMerger intermediateFileMerger;

  private PartitionSpec partitionSpec;

  private SortParameters sortParameters;

  public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
      SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
      PartitionSpec partitionSpec) {
    this.carbonLoadModel = carbonLoadModel;
    this.carbonTable = carbonTable;
    this.segmentProperties = segmentProperties;
    this.segmentId = carbonLoadModel.getSegmentId();
    this.compactionType = compactionType;
    this.tableName = tableName;
    this.partitionSpec = partitionSpec;
  }

  /**
   * This method will iterate over the query result and convert it into a format compatible
   * for data loading
   *
   * @param resultIteratorList
   */
  public boolean execute(List<RawResultIterator> resultIteratorList) throws Exception {
    boolean isCompactionSuccess = false;
    try {
      initTempStoreLocation();
      initSortDataRows();
      dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
      processResult(resultIteratorList);
      // After delete command, if no records are fetched from one split,
      // below steps are not required to be initialized.
      if (isRecordFound) {
        initializeFinalThreadMergerForMergeSort();
        initDataHandler();
        readAndLoadDataFromSortTempFiles();
      }
      isCompactionSuccess = true;
    } catch (Exception e) {
      throw e;
    } finally {
      if (partitionSpec != null) {
        try {
          SegmentFileStore
              .writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),
                  partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",
                  partitionSpec.getPartitions());
        } catch (IOException e) {
          throw e;
        }
      }
      // clear temp files and folders created during compaction
      deleteTempStoreLocation();
    }
    return isCompactionSuccess;
  }

  @Override
  public void close() {
    // close the sorter executor service
    if (null != sortDataRows) {
      sortDataRows.close();
    }
    // close the final merger
    if (null != finalMerger) {
      finalMerger.close();
    }
    // close data handler
    if (null != dataHandler) {
      dataHandler.closeHandler();
    }
  }

  /**
   * This method will clean up the local folders and files created during compaction process
   */
  private void deleteTempStoreLocation() {
    if (null != tempStoreLocation) {
      for (String tempLoc : tempStoreLocation) {
        try {
          CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));
        } catch (IOException | InterruptedException e) {
          LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());
        }
      }
    }
  }

  /**
   * This method will iterate over the query result and perform row sorting operation
   *
   * @param resultIteratorList
   */
  private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
    for (RawResultIterator resultIterator : resultIteratorList) {
      if (CompactionType.STREAMING == compactionType) {
        while (resultIterator.hasNext()) {
          // the input iterator of streaming segment is already using raw row
          addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      } else {
        while (resultIterator.hasNext()) {
          addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      }
      resultIterator.close();
    }
    try {
      sortDataRows.startSorting();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
    List<CarbonDimension> dimensions = segmentProperties.getDimensions();
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY)) {
        // dictionary
        preparedRow[i] = row[i];
      } else {
        // no dictionary dims
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = row[i];
        }
      }
    }
    // fill all the measures
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];
    }
    return preparedRow;
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareRowObjectForSorting(Object[] row) {
    ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];
    // ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
    List<CarbonDimension> dimensions = segmentProperties.getDimensions();
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    // convert the dictionary from MDKey to surrogate key
    byte[] dictionaryKey = wrapper.getDictionaryKey();
    long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);
    Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];
    for (int i = 0; i < keyArray.length; i++) {
      dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();
    }
    int noDictionaryIndex = 0;
    int dictionaryIndex = 0;
    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY)) {
        // dictionary
        preparedRow[i] = dictionaryValues[dictionaryIndex++];
      } else {
        // no dictionary dims
        byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,
                  dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = noDictionaryKeyByIndex;
        }
      }
    }
    // fill all the measures
    // measures will always start from 1st index in the row object array
    int measureIndexInRow = 1;
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] =
          getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);
    }
    return preparedRow;
  }

  /**
   * This method will convert the spark decimal to java big decimal type
   *
   * @param value
   * @param type
   * @return
   */
  private Object getConvertedMeasureValue(Object value, DataType type) {
    if (DataTypes.isDecimal(type)) {
      if (value != null) {
        value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);
      }
      return value;
    } else {
      return value;
    }
  }

  /**
   * This method will read sort temp files, perform merge sort and add it to store for data loading
   */
  private void readAndLoadDataFromSortTempFiles() throws Exception {
    try {
      intermediateFileMerger.finish();
      finalMerger.startFinalMerge();
      while (finalMerger.hasNext()) {
        Object[] row = finalMerger.next();
        dataHandler.addDataToStore(new CarbonRow(row));
      }
      dataHandler.finish();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } catch (Exception e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } finally {
      if (null != dataHandler) {
        try {
          dataHandler.closeHandler();
        } catch (CarbonDataWriterException e) {
          LOGGER.error(e, "Error in close data handler");
          throw new Exception("Error in close data handler", e);
        }
      }
    }
  }

  /**
   * add row to a temp array which will we written to a sort temp file after sorting
   *
   * @param row
   */
  private void addRowForSorting(Object[] row) throws Exception {
    try {
      sortDataRows.addRow(row);
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());
    }
  }

  /**
   * create an instance of sort data rows
   */
  private void initSortDataRows() throws Exception {
    measureCount = carbonTable.getMeasureByTableName(tableName).size();
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
    noDictionaryColMapping = new boolean[dimensions.size()];
    sortColumnMapping = new boolean[dimensions.size()];
    isVarcharDimMapping = new boolean[dimensions.size()];
    int i = 0;
    for (CarbonDimension dimension : dimensions) {
      if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
        i++;
        continue;
      }
      if (dimension.isSortColumn()) {
        sortColumnMapping[i] = true;
      }
      noDictionaryColMapping[i] = true;
      if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {
        isVarcharDimMapping[i] = true;
      }
      i++;
      noDictionaryCount++;
    }
    dimensionColumnCount = dimensions.size();
    sortParameters = createSortParameters();
    intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
    // TODO: Now it is only supported onheap merge, but we can have unsafe merge
    // as well by using UnsafeSortDataRows.
    this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
    try {
      this.sortDataRows.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception(
          "Error initializing sort data rows object during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will create the sort parameters VO object
   *
   * @return
   */
  private SortParameters createSortParameters() {
    return SortParameters
        .createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,
            dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,
            noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,
            sortColumnMapping, isVarcharDimMapping, true);
  }

  /**
   * create an instance of finalThread merger which will perform merge sort on all the
   * sort temp files
   */
  private void initializeFinalThreadMergerForMergeSort() {
    boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil
        .getNoDictSortColMapping(carbonTable.getDatabaseName(), carbonTable.getTableName());
    sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
    String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,
        CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    finalMerger =
        new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
  }

  /**
   * initialise carbon data writer instance
   */
  private void initDataHandler() throws Exception {
    String carbonStoreLocation;
    if (partitionSpec != null) {
      carbonStoreLocation =
          partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR
              + carbonLoadModel.getFactTimeStamp() + ".tmp";
    } else {
      carbonStoreLocation = CarbonDataProcessorUtil
          .createCarbonStoreLocation(carbonLoadModel.getDatabaseName(), tableName,
              carbonLoadModel.getSegmentId());
    }
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
        .getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
            tempStoreLocation, carbonStoreLocation);
    carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());
    setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);
    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);
    try {
      dataHandler.initialise();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());
    }
  }

  /**
   * initialise temporary store location
   */
  private void initTempStoreLocation() {
    tempStoreLocation = CarbonDataProcessorUtil
        .getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),
           segmentId, true, false);
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.processing.merger;
18
19	import java.io.File;
20	import java.io.IOException;
21	import java.util.List;
22
23	import org.apache.carbondata.common.logging.LogService;
24	import org.apache.carbondata.common.logging.LogServiceFactory;
25	import org.apache.carbondata.core.constants.CarbonCommonConstants;
26	import org.apache.carbondata.core.datastore.block.SegmentProperties;
27	import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
28	import org.apache.carbondata.core.datastore.row.CarbonRow;
29	import org.apache.carbondata.core.indexstore.PartitionSpec;
30	import org.apache.carbondata.core.metadata.SegmentFileStore;
31	import org.apache.carbondata.core.metadata.datatype.DataType;
32	import org.apache.carbondata.core.metadata.datatype.DataTypes;
33	import org.apache.carbondata.core.metadata.encoder.Encoding;
34	import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
35	import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
36	import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
37	import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
38	import org.apache.carbondata.core.util.CarbonUtil;
39	import org.apache.carbondata.core.util.DataTypeUtil;
40	import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
41	import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
42	import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
43	import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
44	import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
45	import org.apache.carbondata.processing.sort.sortdata.SortParameters;
46	import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
47	import org.apache.carbondata.processing.store.CarbonFactHandler;
48	import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
49	import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
50
51	/**
52	* This class will process the query result and convert the data
53	* into a format compatible for data load
54	*/
55	public class CompactionResultSortProcessor extends AbstractResultProcessor {
56
57	/**
58	* LOGGER
59	*/
60	private static final LogService LOGGER =	1✔
61	LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());	1✔
62	/**
63	* carbon load model that contains all the required information for load
64	*/
65	private CarbonLoadModel carbonLoadModel;
66	/**
67	* carbon table
68	*/
69	private CarbonTable carbonTable;
70	/**
71	* sortDataRows instance for sorting each row read ad writing to sort temp file
72	*/
73	private SortDataRows sortDataRows;
74	/**
75	* final merger for merge sort
76	*/
77	private SingleThreadFinalSortFilesMerger finalMerger;
78	/**
79	* data handler VO object
80	*/
81	private CarbonFactHandler dataHandler;
82	/**
83	* segment properties for getting dimension cardinality and other required information of a block
84	*/
85	private SegmentProperties segmentProperties;
86	/**
87	* compaction type to decide whether taskID need to be extracted from carbondata files
88	*/
89	private CompactionType compactionType;
90	/**
91	* boolean mapping for no dictionary columns in schema
92	*/
93	private boolean[] noDictionaryColMapping;
94
95	private boolean[] sortColumnMapping;
96	/**
97	* boolean mapping for long string dimension
98	*/
99	private boolean[] isVarcharDimMapping;
100	/**
101	* agg type defined for measures
102	*/
103	private DataType[] dataTypes;
104	/**
105	* segment id
106	*/
107	private String segmentId;
108	/**
109	* temp store location to be sued during data load
110	*/
111	private String[] tempStoreLocation;
112	/**
113	* table name
114	*/
115	private String tableName;
116	/**
117	* no dictionary column count in schema
118	*/
119	private int noDictionaryCount;
120	/**
121	* total count of measures in schema
122	*/
123	private int measureCount;
124	/**
125	* dimension count excluding complex dimension and no dictionary column count
126	*/
127	private int dimensionColumnCount;
128	/**
129	* whether the allocated tasks has any record
130	*/
131	private boolean isRecordFound;
132	/**
133	* intermediate sort merger
134	*/
135	private SortIntermediateFileMerger intermediateFileMerger;
136
137	private PartitionSpec partitionSpec;
138
139	private SortParameters sortParameters;
140
141	public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
142	SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
143	PartitionSpec partitionSpec) {	1✔
144	this.carbonLoadModel = carbonLoadModel;	1✔
145	this.carbonTable = carbonTable;	1✔
146	this.segmentProperties = segmentProperties;	1✔
147	this.segmentId = carbonLoadModel.getSegmentId();	1✔
148	this.compactionType = compactionType;	1✔
149	this.tableName = tableName;	1✔
150	this.partitionSpec = partitionSpec;	1✔
151	}	1✔
152
153	/**
154	* This method will iterate over the query result and convert it into a format compatible
155	* for data loading
156	*
157	* @param resultIteratorList
158	*/
159	public boolean execute(List<RawResultIterator> resultIteratorList) throws Exception {
160	boolean isCompactionSuccess = false;	1✔
161	try {
162	initTempStoreLocation();	1✔
163	initSortDataRows();	1✔
164	dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);	1✔
165	processResult(resultIteratorList);	1✔
166	// After delete command, if no records are fetched from one split,
167	// below steps are not required to be initialized.
168	if (isRecordFound) {	1✔
169	initializeFinalThreadMergerForMergeSort();	1✔
170	initDataHandler();	1✔
171	readAndLoadDataFromSortTempFiles();	1✔
172	}
173	isCompactionSuccess = true;	1✔
174	} catch (Exception e) {	×
175	throw e;	×
176	} finally {
177	if (partitionSpec != null) {	1✔
178	try {
179	SegmentFileStore	×
180	.writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),	×
181	partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",	×
182	partitionSpec.getPartitions());	×
183	} catch (IOException e) {	×
184	throw e;	×
185	}	×
186	}
187	// clear temp files and folders created during compaction
188	deleteTempStoreLocation();	1✔
189	}	1✔
190	return isCompactionSuccess;	1✔
191	}
192
193	@Override
194	public void close() {
195	// close the sorter executor service
196	if (null != sortDataRows) {	1✔
197	sortDataRows.close();	1✔
198	}
199	// close the final merger
200	if (null != finalMerger) {	1✔
201	finalMerger.close();	1✔
202	}
203	// close data handler
204	if (null != dataHandler) {	1✔
205	dataHandler.closeHandler();	1✔
206	}
207	}	1✔
208
209	/**
210	* This method will clean up the local folders and files created during compaction process
211	*/
212	private void deleteTempStoreLocation() {
213	if (null != tempStoreLocation) {	1✔
214	for (String tempLoc : tempStoreLocation) {	1✔
215	try {
216	CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));	1✔
217	} catch (IOException \| InterruptedException e) {	×
218	LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());	×
219	}	1✔
220	}
221	}
222	}	1✔
223
224	/**
225	* This method will iterate over the query result and perform row sorting operation
226	*
227	* @param resultIteratorList
228	*/
229	private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
230	for (RawResultIterator resultIterator : resultIteratorList) {	1✔
231	if (CompactionType.STREAMING == compactionType) {	1✔
232	while (resultIterator.hasNext()) {	1✔
233	// the input iterator of streaming segment is already using raw row
234	addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));	1✔
235	isRecordFound = true;	1✔
236	}
237	} else {
238	while (resultIterator.hasNext()) {	1✔
239	addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));	1✔
240	isRecordFound = true;	1✔
241	}
242	}
243	resultIterator.close();	1✔
244	}	1✔
245	try {
246	sortDataRows.startSorting();	1✔
247	} catch (CarbonSortKeyAndGroupByException e) {	×
248	LOGGER.error(e);	×
249	throw new Exception("Problem loading data during compaction: " + e.getMessage());	×
250	}	1✔
251	}	1✔
252
253	/**
254	* This method will prepare the data from raw object that will take part in sorting
255	*
256	* @param row
257	* @return
258	*/
259	private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
260	List<CarbonDimension> dimensions = segmentProperties.getDimensions();	1✔
261	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
262	for (int i = 0; i < dimensions.size(); i++) {	1✔
263	CarbonDimension dims = dimensions.get(i);	1✔
264	if (dims.hasEncoding(Encoding.DICTIONARY)) {	1✔
265	// dictionary
266	preparedRow[i] = row[i];	1✔
267	} else {
268	// no dictionary dims
269	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {	1✔
270	// no dictionary measure columns are expected as original data
271	preparedRow[i] = DataTypeUtil	1✔
272	.getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());	1✔
273	// for timestamp the above method will give the original data, so it should be
274	// converted again to the format to be loaded (without micros)
275	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
276	preparedRow[i] = (long) preparedRow[i] / 1000L;	1✔
277	}
278	} else {
279	preparedRow[i] = row[i];	1✔
280	}
281	}
282	}
283	// fill all the measures
284	for (int i = 0; i < measureCount; i++) {	1✔
285	preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];	1✔
286	}
287	return preparedRow;	1✔
288	}
289
290	/**
291	* This method will prepare the data from raw object that will take part in sorting
292	*
293	* @param row
294	* @return
295	*/
296	private Object[] prepareRowObjectForSorting(Object[] row) {
297	ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];	1✔
298	// ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
299	List<CarbonDimension> dimensions = segmentProperties.getDimensions();	1✔
300	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
301	// convert the dictionary from MDKey to surrogate key
302	byte[] dictionaryKey = wrapper.getDictionaryKey();	1✔
303	long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);	1✔
304	Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];	1✔
305	for (int i = 0; i < keyArray.length; i++) {	1✔
306	dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();	×
307	}
308	int noDictionaryIndex = 0;	1✔
309	int dictionaryIndex = 0;	1✔
310	for (int i = 0; i < dimensions.size(); i++) {	1✔
311	CarbonDimension dims = dimensions.get(i);	1✔
312	if (dims.hasEncoding(Encoding.DICTIONARY)) {	1✔
313	// dictionary
314	preparedRow[i] = dictionaryValues[dictionaryIndex++];	×
315	} else {
316	// no dictionary dims
317	byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);	1✔
318	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {	1✔
319	// no dictionary measure columns are expected as original data
320	preparedRow[i] = DataTypeUtil	1✔
321	.getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,	1✔
322	dims.getDataType());	1✔
323	// for timestamp the above method will give the original data, so it should be
324	// converted again to the format to be loaded (without micros)
325	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
326	preparedRow[i] = (long) preparedRow[i] / 1000L;	1✔
327	}
328	} else {
329	preparedRow[i] = noDictionaryKeyByIndex;	1✔
330	}
331	}
332	}
333	// fill all the measures
334	// measures will always start from 1st index in the row object array
335	int measureIndexInRow = 1;	1✔
336	for (int i = 0; i < measureCount; i++) {	1✔
337	preparedRow[dimensionColumnCount + i] =	1✔
338	getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);	1✔
339	}
340	return preparedRow;	1✔
341	}
342
343	/**
344	* This method will convert the spark decimal to java big decimal type
345	*
346	* @param value
347	* @param type
348	* @return
349	*/
350	private Object getConvertedMeasureValue(Object value, DataType type) {
351	if (DataTypes.isDecimal(type)) {	1✔
352	if (value != null) {	1✔
353	value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);	1✔
354	}
355	return value;	1✔
356	} else {
357	return value;	1✔
358	}
359	}
360
361	/**
362	* This method will read sort temp files, perform merge sort and add it to store for data loading
363	*/
364	private void readAndLoadDataFromSortTempFiles() throws Exception {
365	try {
366	intermediateFileMerger.finish();	1✔
367	finalMerger.startFinalMerge();	1✔
368	while (finalMerger.hasNext()) {	1✔
369	Object[] row = finalMerger.next();	1✔
370	dataHandler.addDataToStore(new CarbonRow(row));	1✔
371	}	1✔
372	dataHandler.finish();	1✔
373	} catch (CarbonDataWriterException e) {	×
374	LOGGER.error(e);	×
375	throw new Exception("Problem loading data during compaction.", e);	×
376	} catch (Exception e) {	×
377	LOGGER.error(e);	×
378	throw new Exception("Problem loading data during compaction.", e);	×
379	} finally {
380	if (null != dataHandler) {	1✔
381	try {
382	dataHandler.closeHandler();	1✔
383	} catch (CarbonDataWriterException e) {	×
384	LOGGER.error(e, "Error in close data handler");	×
385	throw new Exception("Error in close data handler", e);	×
386	}	1✔
387	}
388	}	×
389	}	1✔
390
391	/**
392	* add row to a temp array which will we written to a sort temp file after sorting
393	*
394	* @param row
395	*/
396	private void addRowForSorting(Object[] row) throws Exception {
397	try {
398	sortDataRows.addRow(row);	1✔
399	} catch (CarbonSortKeyAndGroupByException e) {	×
400	LOGGER.error(e);	×
401	throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());	×
402	}	1✔
403	}	1✔
404
405	/**
406	* create an instance of sort data rows
407	*/
408	private void initSortDataRows() throws Exception {
409	measureCount = carbonTable.getMeasureByTableName(tableName).size();	1✔
410	List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);	1✔
411	noDictionaryColMapping = new boolean[dimensions.size()];	1✔
412	sortColumnMapping = new boolean[dimensions.size()];	1✔
413	isVarcharDimMapping = new boolean[dimensions.size()];	1✔
414	int i = 0;	1✔
415	for (CarbonDimension dimension : dimensions) {	1✔
416	if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {	1✔
417	i++;	1✔
418	continue;	1✔
419	}
420	if (dimension.isSortColumn()) {	1✔
421	sortColumnMapping[i] = true;	1✔
422	}
423	noDictionaryColMapping[i] = true;	1✔
424	if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {	1✔
425	isVarcharDimMapping[i] = true;	×
426	}
427	i++;	1✔
428	noDictionaryCount++;	1✔
429	}	1✔
430	dimensionColumnCount = dimensions.size();	1✔
431	sortParameters = createSortParameters();	1✔
432	intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);	1✔
433	// TODO: Now it is only supported onheap merge, but we can have unsafe merge
434	// as well by using UnsafeSortDataRows.
435	this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);	1✔
436	try {
437	this.sortDataRows.initialize();	1✔
438	} catch (CarbonSortKeyAndGroupByException e) {	×
439	LOGGER.error(e);	×
440	throw new Exception(	×
441	"Error initializing sort data rows object during compaction: " + e.getMessage());	×
442	}	1✔
443	}	1✔
444
445	/**
446	* This method will create the sort parameters VO object
447	*
448	* @return
449	*/
450	private SortParameters createSortParameters() {
451	return SortParameters	1✔
452	.createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,	1✔
453	dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,	1✔
454	noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,	1✔
455	sortColumnMapping, isVarcharDimMapping, true);
456	}
457
458	/**
459	* create an instance of finalThread merger which will perform merge sort on all the
460	* sort temp files
461	*/
462	private void initializeFinalThreadMergerForMergeSort() {
463	boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil	1✔
464	.getNoDictSortColMapping(carbonTable.getDatabaseName(), carbonTable.getTableName());	1✔
465	sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);	1✔
466	String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,	1✔
467	CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
468	finalMerger =	1✔
469	new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
470	}	1✔
471
472	/**
473	* initialise carbon data writer instance
474	*/
475	private void initDataHandler() throws Exception {
476	String carbonStoreLocation;
477	if (partitionSpec != null) {	1✔
478	carbonStoreLocation =	×
479	partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR	×
480	+ carbonLoadModel.getFactTimeStamp() + ".tmp";	×
481	} else {
482	carbonStoreLocation = CarbonDataProcessorUtil	1✔
483	.createCarbonStoreLocation(carbonLoadModel.getDatabaseName(), tableName,	1✔
484	carbonLoadModel.getSegmentId());	1✔
485	}
486	CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel	1✔
487	.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,	1✔
488	tempStoreLocation, carbonStoreLocation);
489	carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());	1✔
490	setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);	1✔
491	dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);	1✔
492	try {
493	dataHandler.initialise();	1✔
494	} catch (CarbonDataWriterException e) {	×
495	LOGGER.error(e);	×
496	throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());	×
497	}	1✔
498	}	1✔
499
500	/**
501	* initialise temporary store location
502	*/
503	private void initTempStoreLocation() {
504	tempStoreLocation = CarbonDataProcessorUtil	1✔
505	.getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),	1✔
506	segmentId, true, false);
507	}	1✔
508	}

apache / carbondata / 876

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous