2720

Build # 2720

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/3cc7cf14869a553ab9c1ec1f38a9b3722c05ded1">3cc7cf148<a href="https://github.com/apache/carbondata/commit/3cc7cf14869a553ab9c1ec1f38a9b3722c05ded1">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/a8b837dcfd1d3561fcc025b814d01df681ef08e1">a8b837dcf</a>">a8b837dcf</a><a href="https://github.com/apache/carbondata/commit/3cc7cf14869a553ab9c1ec1f38a9b3722c05ded1"> into c55f1d7f6">c55f1d7f6</a>

Run Details

3 of 3 new or added lines in 1 file covered. (100.0%)

62393 of 78858 relevant lines covered (79.12%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.95

/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.processing.merger;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.carbondata.processing.store.CarbonFactHandler;
import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;

import org.apache.log4j.Logger;

/**
 * This class will process the query result and convert the data
 * into a format compatible for data load
 */
public class CompactionResultSortProcessor extends AbstractResultProcessor {

  /**
   * LOGGER
   */
  private static final Logger LOGGER =
      LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());
  /**
   * carbon load model that contains all the required information for load
   */
  private CarbonLoadModel carbonLoadModel;
  /**
   * carbon table
   */
  private CarbonTable carbonTable;
  /**
   * sortDataRows instance for sorting each row read ad writing to sort temp file
   */
  private SortDataRows sortDataRows;
  /**
   * final merger for merge sort
   */
  private SingleThreadFinalSortFilesMerger finalMerger;
  /**
   * data handler VO object
   */
  private CarbonFactHandler dataHandler;
  /**
   * segment properties for getting dimension cardinality and other required information of a block
   */
  private SegmentProperties segmentProperties;
  /**
   * compaction type to decide whether taskID need to be extracted from carbondata files
   */
  private CompactionType compactionType;
  /**
   * boolean mapping for no dictionary columns in schema
   */
  private boolean[] noDictionaryColMapping;

  private boolean[] sortColumnMapping;
  /**
   * boolean mapping for long string dimension
   */
  private boolean[] isVarcharDimMapping;
  /**
   * agg type defined for measures
   */
  private DataType[] dataTypes;
  /**
   * segment id
   */
  private String segmentId;
  /**
   * temp store location to be sued during data load
   */
  private String[] tempStoreLocation;
  /**
   * table name
   */
  private String tableName;
  /**
   * no dictionary column count in schema
   */
  private int noDictionaryCount;
  /**
   * total count of measures in schema
   */
  private int measureCount;
  /**
   * dimension count excluding complex dimension and no dictionary column count
   */
  private int dimensionColumnCount;
  /**
   * whether the allocated tasks has any record
   */
  private boolean isRecordFound;
  /**
   * intermediate sort merger
   */
  private SortIntermediateFileMerger intermediateFileMerger;

  private PartitionSpec partitionSpec;

  private SortParameters sortParameters;

  private CarbonColumn[] noDicAndComplexColumns;

  public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
      SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
      PartitionSpec partitionSpec) {
    this.carbonLoadModel = carbonLoadModel;
    this.carbonTable = carbonTable;
    this.segmentProperties = segmentProperties;
    this.segmentId = carbonLoadModel.getSegmentId();
    this.compactionType = compactionType;
    this.tableName = tableName;
    this.partitionSpec = partitionSpec;
  }

  /**
   * This method will iterate over the query result and convert it into a format compatible
   * for data loading
   *
   * @param unsortedResultIteratorList
   * @param sortedResultIteratorList
   * @return if the compaction is success or not
   * @throws Exception
   */
  public boolean execute(List<RawResultIterator> unsortedResultIteratorList,
      List<RawResultIterator> sortedResultIteratorList) throws Exception {
    boolean isCompactionSuccess = false;
    try {
      initTempStoreLocation();
      initSortDataRows();
      dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
      processResult(unsortedResultIteratorList);
      // After delete command, if no records are fetched from one split,
      // below steps are not required to be initialized.
      if (isRecordFound) {
        initializeFinalThreadMergerForMergeSort();
        initDataHandler();
        readAndLoadDataFromSortTempFiles(sortedResultIteratorList);
      }
      isCompactionSuccess = true;
    } catch (Exception e) {
      LOGGER.error(e.getLocalizedMessage(), e);
      throw e;
    } finally {
      if (partitionSpec != null) {
        try {
          SegmentFileStore
              .writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),
                  partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",
                  partitionSpec.getPartitions());
        } catch (IOException e) {
          throw e;
        }
      }
      // clear temp files and folders created during compaction
      deleteTempStoreLocation();
    }
    return isCompactionSuccess;
  }

  @Override
  public void close() {
    // close the sorter executor service
    if (null != sortDataRows) {
      sortDataRows.close();
    }
    // close the final merger
    if (null != finalMerger) {
      finalMerger.close();
    }
    // close data handler
    if (null != dataHandler) {
      dataHandler.closeHandler();
    }
  }

  /**
   * This method will clean up the local folders and files created during compaction process
   */
  private void deleteTempStoreLocation() {
    if (null != tempStoreLocation) {
      for (String tempLoc : tempStoreLocation) {
        try {
          CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));
        } catch (IOException | InterruptedException e) {
          LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());
        }
      }
    }
  }

  /**
   * This method will iterate over the query result and perform row sorting operation
   *
   * @param resultIteratorList
   */
  private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
    for (RawResultIterator resultIterator : resultIteratorList) {
      if (CompactionType.STREAMING == compactionType) {
        while (resultIterator.hasNext()) {
          // the input iterator of streaming segment is already using raw row
          addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      } else {
        while (resultIterator.hasNext()) {
          addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
          isRecordFound = true;
        }
      }
      resultIterator.close();
    }
    try {
      sortDataRows.startSorting();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
    List<CarbonDimension> dimensions = segmentProperties.getDimensions();
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY)) {
        // dictionary
        preparedRow[i] = row[i];
      } else {
        // no dictionary dims
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = row[i];
        }
      }
    }
    // fill all the measures
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];
    }
    return preparedRow;
  }

  /**
   * This method will prepare the data from raw object that will take part in sorting
   *
   * @param row
   * @return
   */
  private Object[] prepareRowObjectForSorting(Object[] row) {
    ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];
    // ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
    List<CarbonDimension> dimensions = new ArrayList<>();
    dimensions.addAll(segmentProperties.getDimensions());
    dimensions.addAll(segmentProperties.getComplexDimensions());
    Object[] preparedRow = new Object[dimensions.size() + measureCount];
    // convert the dictionary from MDKey to surrogate key
    byte[] dictionaryKey = wrapper.getDictionaryKey();
    long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);
    Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];
    for (int i = 0; i < keyArray.length; i++) {
      dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();
    }
    int noDictionaryIndex = 0;
    int dictionaryIndex = 0;
    int complexIndex = 0;

    for (int i = 0; i < dimensions.size(); i++) {
      CarbonDimension dims = dimensions.get(i);
      if (dims.hasEncoding(Encoding.DICTIONARY) && !dims.isComplex()) {
        // dictionary
        preparedRow[i] = dictionaryValues[dictionaryIndex++];
      } else if (!dims.isComplex()) {
        // no dictionary dims
        byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);
        if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {
          // no dictionary measure columns are expected as original data
          preparedRow[i] = DataTypeUtil
              .getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,
                  dims.getDataType());
          // for timestamp the above method will give the original data, so it should be
          // converted again to the format to be loaded (without micros)
          if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {
            preparedRow[i] = (long) preparedRow[i] / 1000L;
          }
        } else {
          preparedRow[i] = noDictionaryKeyByIndex;
        }
      } else {
        preparedRow[i] = wrapper.getComplexKeyByIndex(complexIndex++);
      }
    }

    // fill all the measures
    // measures will always start from 1st index in the row object array
    int measureIndexInRow = 1;
    for (int i = 0; i < measureCount; i++) {
      preparedRow[dimensionColumnCount + i] =
          getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);
    }
    return preparedRow;
  }

  /**
   * This method will convert the spark decimal to java big decimal type
   *
   * @param value
   * @param type
   * @return
   */
  private Object getConvertedMeasureValue(Object value, DataType type) {
    if (DataTypes.isDecimal(type)) {
      if (value != null) {
        value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);
      }
      return value;
    } else {
      return value;
    }
  }

  /**
   * This method will read sort temp files, perform merge sort and add it to store for data loading
   */
  private void readAndLoadDataFromSortTempFiles(List<RawResultIterator> sortedRawResultIterator)
      throws Exception {
    try {
      intermediateFileMerger.finish();
      finalMerger.startFinalMerge();
      if (sortedRawResultIterator != null && sortedRawResultIterator.size() > 0) {
        finalMerger.addInMemoryRawResultIterator(sortedRawResultIterator, segmentProperties,
            noDicAndComplexColumns, dataTypes);
      }
      while (finalMerger.hasNext()) {
        Object[] row = finalMerger.next();
        dataHandler.addDataToStore(new CarbonRow(row));
      }
      dataHandler.finish();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } catch (Exception e) {
      LOGGER.error(e);
      throw new Exception("Problem loading data during compaction.", e);
    } finally {
      if (null != dataHandler) {
        try {
          dataHandler.closeHandler();
        } catch (CarbonDataWriterException e) {
          LOGGER.error("Error in close data handler", e);
          throw new Exception("Error in close data handler", e);
        }
      }
    }
  }

  /**
   * add row to a temp array which will we written to a sort temp file after sorting
   *
   * @param row
   */
  private void addRowForSorting(Object[] row) throws Exception {
    try {
      sortDataRows.addRow(row);
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());
    }
  }

  /**
   * create an instance of sort data rows
   */
  private void initSortDataRows() throws Exception {
    measureCount = carbonTable.getMeasureByTableName(tableName).size();
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
    noDictionaryColMapping = new boolean[dimensions.size()];
    sortColumnMapping = new boolean[dimensions.size()];
    isVarcharDimMapping = new boolean[dimensions.size()];
    int i = 0;
    for (CarbonDimension dimension : dimensions) {
      if (dimension.isSortColumn()) {
        sortColumnMapping[i] = true;
      }
      if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
        i++;
        continue;
      }
      noDictionaryColMapping[i] = true;
      if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {
        isVarcharDimMapping[i] = true;
      }
      i++;
      noDictionaryCount++;
    }
    dimensionColumnCount = dimensions.size();
    sortParameters = createSortParameters();
    intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
    // TODO: Now it is only supported onheap merge, but we can have unsafe merge
    // as well by using UnsafeSortDataRows.
    this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
    try {
      this.sortDataRows.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
      LOGGER.error(e);
      throw new Exception(
          "Error initializing sort data rows object during compaction: " + e.getMessage());
    }
  }

  /**
   * This method will create the sort parameters VO object
   *
   * @return
   */
  private SortParameters createSortParameters() {
    int numberOfCompactingCores = CarbonProperties.getInstance().getNumberOfCompactingCores();
    return SortParameters
        .createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,
            dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,
            noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,
            sortColumnMapping, isVarcharDimMapping, true, numberOfCompactingCores / 2);
  }

  /**
   * create an instance of finalThread merger which will perform merge sort on all the
   * sort temp files
   */
  private void initializeFinalThreadMergerForMergeSort() {
    boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil
        .getNoDictSortColMapping(carbonTable);
    sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
    String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,
        CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    finalMerger =
        new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
  }

  /**
   * initialise carbon data writer instance
   */
  private void initDataHandler() throws Exception {
    String carbonStoreLocation;
    if (partitionSpec != null) {
      carbonStoreLocation =
          partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR
              + carbonLoadModel.getFactTimeStamp() + ".tmp";
    } else {
      carbonStoreLocation = CarbonDataProcessorUtil
          .createCarbonStoreLocation(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(),
              carbonLoadModel.getSegmentId());
    }
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
        .getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
            tempStoreLocation, carbonStoreLocation);
    carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());
    setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);
    this.noDicAndComplexColumns = carbonFactDataHandlerModel.getNoDictAndComplexColumns();
    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);
    try {
      dataHandler.initialise();
    } catch (CarbonDataWriterException e) {
      LOGGER.error(e);
      throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());
    }
  }

  /**
   * initialise temporary store location
   */
  private void initTempStoreLocation() {
    tempStoreLocation = CarbonDataProcessorUtil
        .getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),
           segmentId, true, false);
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.processing.merger;
18
19	import java.io.File;
20	import java.io.IOException;
21	import java.util.ArrayList;
22	import java.util.List;
23
24	import org.apache.carbondata.common.logging.LogServiceFactory;
25	import org.apache.carbondata.core.constants.CarbonCommonConstants;
26	import org.apache.carbondata.core.datastore.block.SegmentProperties;
27	import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
28	import org.apache.carbondata.core.datastore.row.CarbonRow;
29	import org.apache.carbondata.core.indexstore.PartitionSpec;
30	import org.apache.carbondata.core.metadata.SegmentFileStore;
31	import org.apache.carbondata.core.metadata.datatype.DataType;
32	import org.apache.carbondata.core.metadata.datatype.DataTypes;
33	import org.apache.carbondata.core.metadata.encoder.Encoding;
34	import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
35	import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
36	import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
37	import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
38	import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
39	import org.apache.carbondata.core.util.CarbonProperties;
40	import org.apache.carbondata.core.util.CarbonUtil;
41	import org.apache.carbondata.core.util.DataTypeUtil;
42	import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
43	import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
44	import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
45	import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
46	import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
47	import org.apache.carbondata.processing.sort.sortdata.SortParameters;
48	import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
49	import org.apache.carbondata.processing.store.CarbonFactHandler;
50	import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
51	import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
52
53	import org.apache.log4j.Logger;
54
55	/**
56	* This class will process the query result and convert the data
57	* into a format compatible for data load
58	*/
59	public class CompactionResultSortProcessor extends AbstractResultProcessor {
60
61	/**
62	* LOGGER
63	*/
64	private static final Logger LOGGER =	1✔
65	LogServiceFactory.getLogService(CompactionResultSortProcessor.class.getName());	1✔
66	/**
67	* carbon load model that contains all the required information for load
68	*/
69	private CarbonLoadModel carbonLoadModel;
70	/**
71	* carbon table
72	*/
73	private CarbonTable carbonTable;
74	/**
75	* sortDataRows instance for sorting each row read ad writing to sort temp file
76	*/
77	private SortDataRows sortDataRows;
78	/**
79	* final merger for merge sort
80	*/
81	private SingleThreadFinalSortFilesMerger finalMerger;
82	/**
83	* data handler VO object
84	*/
85	private CarbonFactHandler dataHandler;
86	/**
87	* segment properties for getting dimension cardinality and other required information of a block
88	*/
89	private SegmentProperties segmentProperties;
90	/**
91	* compaction type to decide whether taskID need to be extracted from carbondata files
92	*/
93	private CompactionType compactionType;
94	/**
95	* boolean mapping for no dictionary columns in schema
96	*/
97	private boolean[] noDictionaryColMapping;
98
99	private boolean[] sortColumnMapping;
100	/**
101	* boolean mapping for long string dimension
102	*/
103	private boolean[] isVarcharDimMapping;
104	/**
105	* agg type defined for measures
106	*/
107	private DataType[] dataTypes;
108	/**
109	* segment id
110	*/
111	private String segmentId;
112	/**
113	* temp store location to be sued during data load
114	*/
115	private String[] tempStoreLocation;
116	/**
117	* table name
118	*/
119	private String tableName;
120	/**
121	* no dictionary column count in schema
122	*/
123	private int noDictionaryCount;
124	/**
125	* total count of measures in schema
126	*/
127	private int measureCount;
128	/**
129	* dimension count excluding complex dimension and no dictionary column count
130	*/
131	private int dimensionColumnCount;
132	/**
133	* whether the allocated tasks has any record
134	*/
135	private boolean isRecordFound;
136	/**
137	* intermediate sort merger
138	*/
139	private SortIntermediateFileMerger intermediateFileMerger;
140
141	private PartitionSpec partitionSpec;
142
143	private SortParameters sortParameters;
144
145	private CarbonColumn[] noDicAndComplexColumns;
146
147	public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
148	SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
149	PartitionSpec partitionSpec) {	1✔
150	this.carbonLoadModel = carbonLoadModel;	1✔
151	this.carbonTable = carbonTable;	1✔
152	this.segmentProperties = segmentProperties;	1✔
153	this.segmentId = carbonLoadModel.getSegmentId();	1✔
154	this.compactionType = compactionType;	1✔
155	this.tableName = tableName;	1✔
156	this.partitionSpec = partitionSpec;	1✔
157	}	1✔
158
159	/**
160	* This method will iterate over the query result and convert it into a format compatible
161	* for data loading
162	*
163	* @param unsortedResultIteratorList
164	* @param sortedResultIteratorList
165	* @return if the compaction is success or not
166	* @throws Exception
167	*/
168	public boolean execute(List<RawResultIterator> unsortedResultIteratorList,
169	List<RawResultIterator> sortedResultIteratorList) throws Exception {
170	boolean isCompactionSuccess = false;	1✔
171	try {
172	initTempStoreLocation();	1✔
173	initSortDataRows();	1✔
174	dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);	1✔
175	processResult(unsortedResultIteratorList);	1✔
176	// After delete command, if no records are fetched from one split,
177	// below steps are not required to be initialized.
178	if (isRecordFound) {	1✔
179	initializeFinalThreadMergerForMergeSort();	1✔
180	initDataHandler();	1✔
181	readAndLoadDataFromSortTempFiles(sortedResultIteratorList);	1✔
182	}
183	isCompactionSuccess = true;	1✔
184	} catch (Exception e) {	×
185	LOGGER.error(e.getLocalizedMessage(), e);	×
186	throw e;	×
187	} finally {
188	if (partitionSpec != null) {	1✔
189	try {
190	SegmentFileStore	×
191	.writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),	×
192	partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",	×
193	partitionSpec.getPartitions());	×
194	} catch (IOException e) {	×
195	throw e;	×
196	}	×
197	}
198	// clear temp files and folders created during compaction
199	deleteTempStoreLocation();	1✔
200	}	1✔
201	return isCompactionSuccess;	1✔
202	}
203
204	@Override
205	public void close() {
206	// close the sorter executor service
207	if (null != sortDataRows) {	1✔
208	sortDataRows.close();	1✔
209	}
210	// close the final merger
211	if (null != finalMerger) {	1✔
212	finalMerger.close();	1✔
213	}
214	// close data handler
215	if (null != dataHandler) {	1✔
216	dataHandler.closeHandler();	1✔
217	}
218	}	1✔
219
220	/**
221	* This method will clean up the local folders and files created during compaction process
222	*/
223	private void deleteTempStoreLocation() {
224	if (null != tempStoreLocation) {	1✔
225	for (String tempLoc : tempStoreLocation) {	1✔
226	try {
227	CarbonUtil.deleteFoldersAndFiles(new File(tempLoc));	1✔
228	} catch (IOException \| InterruptedException e) {	×
229	LOGGER.error("Problem deleting local folders during compaction: " + e.getMessage());	×
230	}	1✔
231	}
232	}
233	}	1✔
234
235	/**
236	* This method will iterate over the query result and perform row sorting operation
237	*
238	* @param resultIteratorList
239	*/
240	private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
241	for (RawResultIterator resultIterator : resultIteratorList) {	1✔
242	if (CompactionType.STREAMING == compactionType) {	1✔
243	while (resultIterator.hasNext()) {	1✔
244	// the input iterator of streaming segment is already using raw row
245	addRowForSorting(prepareStreamingRowObjectForSorting(resultIterator.next()));	1✔
246	isRecordFound = true;	1✔
247	}
248	} else {
249	while (resultIterator.hasNext()) {	1✔
250	addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));	1✔
251	isRecordFound = true;	1✔
252	}
253	}
254	resultIterator.close();	1✔
255	}	1✔
256	try {
257	sortDataRows.startSorting();	1✔
258	} catch (CarbonSortKeyAndGroupByException e) {	×
259	LOGGER.error(e);	×
260	throw new Exception("Problem loading data during compaction: " + e.getMessage());	×
261	}	1✔
262	}	1✔
263
264	/**
265	* This method will prepare the data from raw object that will take part in sorting
266	*
267	* @param row
268	* @return
269	*/
270	private Object[] prepareStreamingRowObjectForSorting(Object[] row) {
271	List<CarbonDimension> dimensions = segmentProperties.getDimensions();	1✔
272	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
273	for (int i = 0; i < dimensions.size(); i++) {	1✔
274	CarbonDimension dims = dimensions.get(i);	1✔
275	if (dims.hasEncoding(Encoding.DICTIONARY)) {	1✔
276	// dictionary
277	preparedRow[i] = row[i];	1✔
278	} else {
279	// no dictionary dims
280	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {	1✔
281	// no dictionary measure columns are expected as original data
282	preparedRow[i] = DataTypeUtil	1✔
283	.getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType());	1✔
284	// for timestamp the above method will give the original data, so it should be
285	// converted again to the format to be loaded (without micros)
286	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
287	preparedRow[i] = (long) preparedRow[i] / 1000L;	1✔
288	}
289	} else {
290	preparedRow[i] = row[i];	1✔
291	}
292	}
293	}
294	// fill all the measures
295	for (int i = 0; i < measureCount; i++) {	1✔
296	preparedRow[dimensionColumnCount + i] = row[dimensionColumnCount + i];	1✔
297	}
298	return preparedRow;	1✔
299	}
300
301	/**
302	* This method will prepare the data from raw object that will take part in sorting
303	*
304	* @param row
305	* @return
306	*/
307	private Object[] prepareRowObjectForSorting(Object[] row) {
308	ByteArrayWrapper wrapper = (ByteArrayWrapper) row[0];	1✔
309	// ByteBuffer[] noDictionaryBuffer = new ByteBuffer[noDictionaryCount];
310	List<CarbonDimension> dimensions = new ArrayList<>();	1✔
311	dimensions.addAll(segmentProperties.getDimensions());	1✔
312	dimensions.addAll(segmentProperties.getComplexDimensions());	1✔
313	Object[] preparedRow = new Object[dimensions.size() + measureCount];	1✔
314	// convert the dictionary from MDKey to surrogate key
315	byte[] dictionaryKey = wrapper.getDictionaryKey();	1✔
316	long[] keyArray = segmentProperties.getDimensionKeyGenerator().getKeyArray(dictionaryKey);	1✔
317	Object[] dictionaryValues = new Object[dimensionColumnCount + measureCount];	1✔
318	for (int i = 0; i < keyArray.length; i++) {	1✔
319	dictionaryValues[i] = Long.valueOf(keyArray[i]).intValue();	1✔
320	}
321	int noDictionaryIndex = 0;	1✔
322	int dictionaryIndex = 0;	1✔
323	int complexIndex = 0;	1✔
324
325	for (int i = 0; i < dimensions.size(); i++) {	1✔
326	CarbonDimension dims = dimensions.get(i);	1✔
327	if (dims.hasEncoding(Encoding.DICTIONARY) && !dims.isComplex()) {	1✔
328	// dictionary
329	preparedRow[i] = dictionaryValues[dictionaryIndex++];	1✔
330	} else if (!dims.isComplex()) {	1✔
331	// no dictionary dims
332	byte[] noDictionaryKeyByIndex = wrapper.getNoDictionaryKeyByIndex(noDictionaryIndex++);	1✔
333	if (DataTypeUtil.isPrimitiveColumn(dims.getDataType())) {	1✔
334	// no dictionary measure columns are expected as original data
335	preparedRow[i] = DataTypeUtil	1✔
336	.getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeyByIndex,	1✔
337	dims.getDataType());	1✔
338	// for timestamp the above method will give the original data, so it should be
339	// converted again to the format to be loaded (without micros)
340	if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) {	1✔
341	preparedRow[i] = (long) preparedRow[i] / 1000L;	×
342	}
343	} else {
344	preparedRow[i] = noDictionaryKeyByIndex;	1✔
345	}
346	} else {	1✔
347	preparedRow[i] = wrapper.getComplexKeyByIndex(complexIndex++);	×
348	}
349	}
350
351	// fill all the measures
352	// measures will always start from 1st index in the row object array
353	int measureIndexInRow = 1;	1✔
354	for (int i = 0; i < measureCount; i++) {	1✔
355	preparedRow[dimensionColumnCount + i] =	1✔
356	getConvertedMeasureValue(row[measureIndexInRow++], dataTypes[i]);	1✔
357	}
358	return preparedRow;	1✔
359	}
360
361	/**
362	* This method will convert the spark decimal to java big decimal type
363	*
364	* @param value
365	* @param type
366	* @return
367	*/
368	private Object getConvertedMeasureValue(Object value, DataType type) {
369	if (DataTypes.isDecimal(type)) {	1✔
370	if (value != null) {	×
371	value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);	×
372	}
373	return value;	×
374	} else {
375	return value;	1✔
376	}
377	}
378
379	/**
380	* This method will read sort temp files, perform merge sort and add it to store for data loading
381	*/
382	private void readAndLoadDataFromSortTempFiles(List<RawResultIterator> sortedRawResultIterator)
383	throws Exception {
384	try {
385	intermediateFileMerger.finish();	1✔
386	finalMerger.startFinalMerge();	1✔
387	if (sortedRawResultIterator != null && sortedRawResultIterator.size() > 0) {	1✔
388	finalMerger.addInMemoryRawResultIterator(sortedRawResultIterator, segmentProperties,	1✔
389	noDicAndComplexColumns, dataTypes);
390	}
391	while (finalMerger.hasNext()) {	1✔
392	Object[] row = finalMerger.next();	1✔
393	dataHandler.addDataToStore(new CarbonRow(row));	1✔
394	}	1✔
395	dataHandler.finish();	1✔
396	} catch (CarbonDataWriterException e) {	×
397	LOGGER.error(e);	×
398	throw new Exception("Problem loading data during compaction.", e);	×
399	} catch (Exception e) {	×
400	LOGGER.error(e);	×
401	throw new Exception("Problem loading data during compaction.", e);	×
402	} finally {
403	if (null != dataHandler) {	1✔
404	try {
405	dataHandler.closeHandler();	1✔
406	} catch (CarbonDataWriterException e) {	×
407	LOGGER.error("Error in close data handler", e);	×
408	throw new Exception("Error in close data handler", e);	×
409	}	1✔
410	}
411	}	×
412	}	1✔
413
414	/**
415	* add row to a temp array which will we written to a sort temp file after sorting
416	*
417	* @param row
418	*/
419	private void addRowForSorting(Object[] row) throws Exception {
420	try {
421	sortDataRows.addRow(row);	1✔
422	} catch (CarbonSortKeyAndGroupByException e) {	×
423	LOGGER.error(e);	×
424	throw new Exception("Row addition for sorting failed during compaction: " + e.getMessage());	×
425	}	1✔
426	}	1✔
427
428	/**
429	* create an instance of sort data rows
430	*/
431	private void initSortDataRows() throws Exception {
432	measureCount = carbonTable.getMeasureByTableName(tableName).size();	1✔
433	List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);	1✔
434	noDictionaryColMapping = new boolean[dimensions.size()];	1✔
435	sortColumnMapping = new boolean[dimensions.size()];	1✔
436	isVarcharDimMapping = new boolean[dimensions.size()];	1✔
437	int i = 0;	1✔
438	for (CarbonDimension dimension : dimensions) {	1✔
439	if (dimension.isSortColumn()) {	1✔
440	sortColumnMapping[i] = true;	1✔
441	}
442	if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {	1✔
443	i++;	1✔
444	continue;	1✔
445	}
446	noDictionaryColMapping[i] = true;	1✔
447	if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {	1✔
448	isVarcharDimMapping[i] = true;	×
449	}
450	i++;	1✔
451	noDictionaryCount++;	1✔
452	}	1✔
453	dimensionColumnCount = dimensions.size();	1✔
454	sortParameters = createSortParameters();	1✔
455	intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);	1✔
456	// TODO: Now it is only supported onheap merge, but we can have unsafe merge
457	// as well by using UnsafeSortDataRows.
458	this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);	1✔
459	try {
460	this.sortDataRows.initialize();	1✔
461	} catch (CarbonSortKeyAndGroupByException e) {	×
462	LOGGER.error(e);	×
463	throw new Exception(	×
464	"Error initializing sort data rows object during compaction: " + e.getMessage());	×
465	}	1✔
466	}	1✔
467
468	/**
469	* This method will create the sort parameters VO object
470	*
471	* @return
472	*/
473	private SortParameters createSortParameters() {
474	int numberOfCompactingCores = CarbonProperties.getInstance().getNumberOfCompactingCores();	1✔
475	return SortParameters	1✔
476	.createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,	1✔
477	dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,	1✔
478	noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping,	1✔
479	sortColumnMapping, isVarcharDimMapping, true, numberOfCompactingCores / 2);
480	}
481
482	/**
483	* create an instance of finalThread merger which will perform merge sort on all the
484	* sort temp files
485	*/
486	private void initializeFinalThreadMergerForMergeSort() {
487	boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil	1✔
488	.getNoDictSortColMapping(carbonTable);	1✔
489	sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);	1✔
490	String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,	1✔
491	CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
492	finalMerger =	1✔
493	new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
494	}	1✔
495
496	/**
497	* initialise carbon data writer instance
498	*/
499	private void initDataHandler() throws Exception {
500	String carbonStoreLocation;
501	if (partitionSpec != null) {	1✔
502	carbonStoreLocation =	×
503	partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR	×
504	+ carbonLoadModel.getFactTimeStamp() + ".tmp";	×
505	} else {
506	carbonStoreLocation = CarbonDataProcessorUtil	1✔
507	.createCarbonStoreLocation(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(),	1✔
508	carbonLoadModel.getSegmentId());	1✔
509	}
510	CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel	1✔
511	.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,	1✔
512	tempStoreLocation, carbonStoreLocation);
513	carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());	1✔
514	setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);	1✔
515	this.noDicAndComplexColumns = carbonFactDataHandlerModel.getNoDictAndComplexColumns();	1✔
516	dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);	1✔
517	try {
518	dataHandler.initialise();	1✔
519	} catch (CarbonDataWriterException e) {	×
520	LOGGER.error(e);	×
521	throw new Exception("Problem initialising data handler during compaction: " + e.getMessage());	×
522	}	1✔
523	}	1✔
524
525	/**
526	* initialise temporary store location
527	*/
528	private void initTempStoreLocation() {
529	tempStoreLocation = CarbonDataProcessorUtil	1✔
530	.getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(),	1✔
531	segmentId, true, false);
532	}	1✔
533	}

apache / carbondata / 2720

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous