11886

Build # 11886

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/da7b719a6a24ed1c0ea15f5c655d392c9456e53a">da7b719a6<a href="https://github.com/apache/carbondata/commit/da7b719a6a24ed1c0ea15f5c655d392c9456e53a">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/9a729433153b2e716c43e023bcb2717c65fb9851">9a7294331</a>">9a7294331</a><a href="https://github.com/apache/carbondata/commit/da7b719a6a24ed1c0ea15f5c655d392c9456e53a"> into 581591a38">581591a38</a>

Run Details

64103 of 82866 relevant lines covered (77.36%)

1.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.62

/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.writer;

import java.io.IOException;
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;

import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.fileoperations.FileWriteOperation;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.MergedBlockIndex;
import org.apache.carbondata.format.MergedBlockIndexHeader;

import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;

public class CarbonIndexFileMergeWriter {

  /**
   * table handle
   */
  private CarbonTable table;

  /**
   * thrift writer object
   */
  private ThriftWriter thriftWriter;

  private Logger LOGGER = LogServiceFactory.getLogService(this.getClass().getCanonicalName());

  public CarbonIndexFileMergeWriter(CarbonTable table) {
    this.table = table;
  }

  /**
   * Merge all the carbonindex files of segment to a  merged file
   * @param tablePath
   * @param indexFileNamesTobeAdded while merging it comsiders only these files.
   *                                If null then consider all
   * @param readFileFooterFromCarbonDataFile flag to read file footer information from carbondata
   *                                         file. This will used in case of upgrade from version
   *                                         which do not store the blocklet info to current version
   * @throws IOException
   */
  private String mergeCarbonIndexFilesOfSegment(String segmentId,
      String tablePath, List<String> indexFileNamesTobeAdded,
      boolean readFileFooterFromCarbonDataFile, String uuid, String partitionPath) {
    try {
      Segment segment = Segment.getSegment(segmentId, tablePath);
      String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
      CarbonFile[] indexFiles;
      SegmentFileStore sfs = null;
      if (segment != null && segment.getSegmentFileName() != null) {
        sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());
        List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();
        if (table.isHivePartitionTable()) {
          // in case of partition table, merge index files of a partition
          List<CarbonFile> indexFilesInPartition = new ArrayList<>();
          for (CarbonFile indexCarbonFile : indexCarbonFiles) {
            if (indexCarbonFile.getParentFile().getPath().equals(partitionPath)) {
              indexFilesInPartition.add(indexCarbonFile);
            }
          }
          indexFiles = indexFilesInPartition.toArray(new CarbonFile[indexFilesInPartition.size()]);
        } else {
          indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);
        }
      } else {
        indexFiles =
            SegmentIndexFileStore.getCarbonIndexFiles(segmentPath, FileFactory.getConfiguration());
      }
      if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) {
        if (sfs == null) {
          return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded,
              readFileFooterFromCarbonDataFile, segmentPath, indexFiles, segmentId);
        } else {
          return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, sfs,
              indexFiles, uuid, partitionPath);
        }
      }
    } catch (Exception e) {
      LOGGER.error(
          "Failed to merge index files in path: " + tablePath, e);
    }
    return null;
  }

  private String writeMergeIndexFileBasedOnSegmentFolder(List<String> indexFileNamesTobeAdded,
      boolean readFileFooterFromCarbonDataFile, String segmentPath, CarbonFile[] indexFiles,
      String segmentId) throws IOException {
    SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
    if (readFileFooterFromCarbonDataFile) {
      // this case will be used in case of upgrade where old store will not have the blocklet
      // info in the index file and therefore blocklet info need to be read from the file footer
      // in the carbondata file
      fileStore.readAllIndexAndFillBolckletInfo(segmentPath);
    } else {
      fileStore.readAllIIndexOfSegment(segmentPath);
    }
    Map<String, byte[]> indexMap = fileStore.getCarbonIndexMap();
    writeMergeIndexFile(indexFileNamesTobeAdded, segmentPath, indexMap, segmentId);
    for (CarbonFile indexFile : indexFiles) {
      indexFile.delete();
    }
    return null;
  }

  private String writeMergeIndexFileBasedOnSegmentFile(String segmentId,
      List<String> indexFileNamesTobeAdded, SegmentFileStore segmentFileStore,
      CarbonFile[] indexFiles, String uuid, String partitionPath) throws IOException {
    SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
    // in case of partition table, merge index file to be created for each partition
    if (null != partitionPath) {
      for (CarbonFile indexFile : indexFiles) {
        fileStore.readIndexFile(indexFile);
      }
    } else {
      fileStore.readAllIIndexOfSegment(segmentFileStore.getSegmentFile(),
          segmentFileStore.getTablePath(), SegmentStatus.SUCCESS, true);
    }
    Map<String, byte[]> indexMap = fileStore.getCarbonIndexMapWithFullPath();
    Map<String, Map<String, byte[]>> indexLocationMap = new HashMap<>();
    for (Map.Entry<String, byte[]> entry: indexMap.entrySet()) {
      Path path = new Path(entry.getKey());
      Map<String, byte[]> map = indexLocationMap.get(path.getParent().toString());
      if (map == null) {
        map = new HashMap<>();
        indexLocationMap.put(path.getParent().toString(), map);
      }
      map.put(path.getName(), entry.getValue());
    }
    List<PartitionSpec> partitionSpecs = SegmentFileStore
        .getPartitionSpecs(segmentId, table.getTablePath(), SegmentStatusManager
            .readLoadMetadata(CarbonTablePath.getMetadataPath(table.getTablePath())));
    for (Map.Entry<String, Map<String, byte[]>> entry : indexLocationMap.entrySet()) {
      String mergeIndexFile =
          writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), entry.getValue(), segmentId);
      for (Map.Entry<String, SegmentFileStore.FolderDetails> segentry : segmentFileStore
          .getLocationMap().entrySet()) {
        String location = segentry.getKey();
        if (segentry.getValue().isRelative()) {
          location =
              segmentFileStore.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + location;
        }
        if (new Path(entry.getKey()).equals(new Path(location))) {
          segentry.getValue().setMergeFileName(mergeIndexFile);
          segentry.getValue().setFiles(new HashSet<String>());
          break;
        }
      }
      if (table.isHivePartitionTable()) {
        for (PartitionSpec partitionSpec : partitionSpecs) {
          if (partitionSpec.getLocation().toString().equals(partitionPath)) {
            SegmentFileStore.writeSegmentFile(table.getTablePath(), mergeIndexFile, partitionPath,
                segmentId + "_" + uuid + "", partitionSpec.getPartitions(), true);
          }
        }
      }
    }
    String newSegmentFileName = SegmentFileStore.genSegmentFileName(segmentId, uuid)
        + CarbonTablePath.SEGMENT_EXT;
    String path = CarbonTablePath.getSegmentFilesLocation(table.getTablePath())
        + CarbonCommonConstants.FILE_SEPARATOR + newSegmentFileName;
    if (!table.isHivePartitionTable()) {
      SegmentFileStore.writeSegmentFile(segmentFileStore.getSegmentFile(), path);
      SegmentFileStore.updateSegmentFile(table, segmentId, newSegmentFileName,
          table.getCarbonTableIdentifier().getTableId(), segmentFileStore);
    }

    for (CarbonFile file : indexFiles) {
      file.delete();
    }

    return uuid;
  }

  private String writeMergeIndexFile(List<String> indexFileNamesTobeAdded, String segmentPath,
      Map<String, byte[]> indexMap, String segment_id) throws IOException {
    MergedBlockIndexHeader indexHeader = new MergedBlockIndexHeader();
    MergedBlockIndex mergedBlockIndex = new MergedBlockIndex();
    List<String> fileNames = new ArrayList<>(indexMap.size());
    List<ByteBuffer> data = new ArrayList<>(indexMap.size());
    for (Map.Entry<String, byte[]> entry : indexMap.entrySet()) {
      if (indexFileNamesTobeAdded == null ||
          indexFileNamesTobeAdded.contains(entry.getKey())) {
        fileNames.add(entry.getKey());
        data.add(ByteBuffer.wrap(entry.getValue()));
      }
    }
    if (fileNames.size() > 0) {
      String mergeIndexName =
          segment_id + '_' + System.currentTimeMillis() + CarbonTablePath.MERGE_INDEX_FILE_EXT;
      openThriftWriter(segmentPath + "/" + mergeIndexName);
      indexHeader.setFile_names(fileNames);
      mergedBlockIndex.setFileData(data);
      writeMergedBlockIndexHeader(indexHeader);
      writeMergedBlockIndex(mergedBlockIndex);
      close();
      return mergeIndexName;
    }
    return null;
  }

  /**
   * Merge all the carbonindex files of segment to a  merged file
   *
   * @param segmentId
   * @throws IOException
   */
  public String mergeCarbonIndexFilesOfSegment(String segmentId, String uuid, String tablePath,
      String partitionPath) throws IOException {
    return mergeCarbonIndexFilesOfSegment(segmentId, tablePath, null, false, uuid, partitionPath);
  }

  /**
   * Merge all the carbonindex files of segment to a  merged file
   *
   * @param segmentId
   * @param readFileFooterFromCarbonDataFile
   * @throws IOException
   */
  public String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath,
      boolean readFileFooterFromCarbonDataFile, String uuid) throws IOException {
    return mergeCarbonIndexFilesOfSegment(segmentId, tablePath, null,
        readFileFooterFromCarbonDataFile, uuid, null);
  }

  private boolean isCarbonIndexFilePresent(CarbonFile[] indexFiles) {
    for (CarbonFile file : indexFiles) {
      if (file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
        return true;
      }
    }
    return false;
  }

  /**
   * It writes thrift object to file
   *
   * @throws IOException
   */
  private void writeMergedBlockIndexHeader(MergedBlockIndexHeader indexObject) throws IOException {
    thriftWriter.write(indexObject);
  }

  /**
   * It writes thrift object to file
   *
   * @throws IOException
   */
  private void writeMergedBlockIndex(MergedBlockIndex indexObject) throws IOException {
    thriftWriter.write(indexObject);
  }

  /**
   * Below method will be used to open the thrift writer
   *
   * @param filePath file path where data need to be written
   * @throws IOException throws io exception in case of any failure
   */
  private void openThriftWriter(String filePath) throws IOException {
    // create thrift writer instance
    thriftWriter = new ThriftWriter(filePath, false);
    // open the file stream
    thriftWriter.open(FileWriteOperation.OVERWRITE);
  }

  /**
   * Below method will be used to close the thrift object
   */
  private void close() throws IOException {
    thriftWriter.close();
  }

  public static class SegmentIndexFIleMergeStatus implements Serializable {

    private SegmentFileStore.SegmentFile segmentFile;

    private List<String> filesTobeDeleted;

    public SegmentIndexFIleMergeStatus(SegmentFileStore.SegmentFile segmentFile,
        List<String> filesTobeDeleted) {
      this.segmentFile = segmentFile;
      this.filesTobeDeleted = filesTobeDeleted;
    }

    public SegmentFileStore.SegmentFile getSegmentFile() {
      return segmentFile;
    }

    public List<String> getFilesTobeDeleted() {
      return filesTobeDeleted;
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.writer;
18
19	import java.io.IOException;
20	import java.io.Serializable;
21	import java.nio.ByteBuffer;
22	import java.util.ArrayList;
23	import java.util.HashMap;
24	import java.util.HashSet;
25	import java.util.List;
26	import java.util.Map;
27
28	import org.apache.carbondata.common.logging.LogServiceFactory;
29	import org.apache.carbondata.core.constants.CarbonCommonConstants;
30	import org.apache.carbondata.core.datamap.Segment;
31	import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
32	import org.apache.carbondata.core.datastore.impl.FileFactory;
33	import org.apache.carbondata.core.fileoperations.FileWriteOperation;
34	import org.apache.carbondata.core.indexstore.PartitionSpec;
35	import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
36	import org.apache.carbondata.core.metadata.SegmentFileStore;
37	import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
38	import org.apache.carbondata.core.statusmanager.SegmentStatus;
39	import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
40	import org.apache.carbondata.core.util.path.CarbonTablePath;
41	import org.apache.carbondata.format.MergedBlockIndex;
42	import org.apache.carbondata.format.MergedBlockIndexHeader;
43
44	import org.apache.hadoop.fs.Path;
45	import org.apache.log4j.Logger;
46
47	public class CarbonIndexFileMergeWriter {
48
49	/**
50	* table handle
51	*/
52	private CarbonTable table;
53
54	/**
55	* thrift writer object
56	*/
57	private ThriftWriter thriftWriter;
58
59	private Logger LOGGER = LogServiceFactory.getLogService(this.getClass().getCanonicalName());	1✔
60
61	public CarbonIndexFileMergeWriter(CarbonTable table) {	1✔
62	this.table = table;	1✔
63	}	1✔
64
65	/**
66	* Merge all the carbonindex files of segment to a merged file
67	* @param tablePath
68	* @param indexFileNamesTobeAdded while merging it comsiders only these files.
69	* If null then consider all
70	* @param readFileFooterFromCarbonDataFile flag to read file footer information from carbondata
71	* file. This will used in case of upgrade from version
72	* which do not store the blocklet info to current version
73	* @throws IOException
74	*/
75	private String mergeCarbonIndexFilesOfSegment(String segmentId,
76	String tablePath, List<String> indexFileNamesTobeAdded,
77	boolean readFileFooterFromCarbonDataFile, String uuid, String partitionPath) {
78	try {
79	Segment segment = Segment.getSegment(segmentId, tablePath);	1✔
80	String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);	1✔
81	CarbonFile[] indexFiles;
82	SegmentFileStore sfs = null;	1✔
83	if (segment != null && segment.getSegmentFileName() != null) {	1✔
84	sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());	1✔
85	List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();	1✔
86	if (table.isHivePartitionTable()) {	1✔
87	// in case of partition table, merge index files of a partition
88	List<CarbonFile> indexFilesInPartition = new ArrayList<>();	1✔
89	for (CarbonFile indexCarbonFile : indexCarbonFiles) {	1✔
90	if (indexCarbonFile.getParentFile().getPath().equals(partitionPath)) {	1✔
91	indexFilesInPartition.add(indexCarbonFile);	1✔
92	}
93	}	1✔
94	indexFiles = indexFilesInPartition.toArray(new CarbonFile[indexFilesInPartition.size()]);	1✔
95	} else {	1✔
96	indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);	1✔
97	}
98	} else {	1✔
99	indexFiles =	1✔
100	SegmentIndexFileStore.getCarbonIndexFiles(segmentPath, FileFactory.getConfiguration());	1✔
101	}
102	if (isCarbonIndexFilePresent(indexFiles) \|\| indexFileNamesTobeAdded != null) {	1✔
103	if (sfs == null) {	1✔
104	return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded,	×
105	readFileFooterFromCarbonDataFile, segmentPath, indexFiles, segmentId);
106	} else {
107	return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, sfs,	1✔
108	indexFiles, uuid, partitionPath);
109	}
110	}
111	} catch (Exception e) {	×
112	LOGGER.error(	×
113	"Failed to merge index files in path: " + tablePath, e);
114	}	1✔
115	return null;	1✔
116	}
117
118	private String writeMergeIndexFileBasedOnSegmentFolder(List<String> indexFileNamesTobeAdded,
119	boolean readFileFooterFromCarbonDataFile, String segmentPath, CarbonFile[] indexFiles,
120	String segmentId) throws IOException {
121	SegmentIndexFileStore fileStore = new SegmentIndexFileStore();	×
122	if (readFileFooterFromCarbonDataFile) {	×
123	// this case will be used in case of upgrade where old store will not have the blocklet
124	// info in the index file and therefore blocklet info need to be read from the file footer
125	// in the carbondata file
126	fileStore.readAllIndexAndFillBolckletInfo(segmentPath);	×
127	} else {
128	fileStore.readAllIIndexOfSegment(segmentPath);	×
129	}
130	Map<String, byte[]> indexMap = fileStore.getCarbonIndexMap();	×
131	writeMergeIndexFile(indexFileNamesTobeAdded, segmentPath, indexMap, segmentId);	×
132	for (CarbonFile indexFile : indexFiles) {	×
133	indexFile.delete();	×
134	}
135	return null;	×
136	}
137
138	private String writeMergeIndexFileBasedOnSegmentFile(String segmentId,
139	List<String> indexFileNamesTobeAdded, SegmentFileStore segmentFileStore,
140	CarbonFile[] indexFiles, String uuid, String partitionPath) throws IOException {
141	SegmentIndexFileStore fileStore = new SegmentIndexFileStore();	1✔
142	// in case of partition table, merge index file to be created for each partition
143	if (null != partitionPath) {	1✔
144	for (CarbonFile indexFile : indexFiles) {	1✔
145	fileStore.readIndexFile(indexFile);	1✔
146	}
147	} else {
148	fileStore.readAllIIndexOfSegment(segmentFileStore.getSegmentFile(),	1✔
149	segmentFileStore.getTablePath(), SegmentStatus.SUCCESS, true);	1✔
150	}
151	Map<String, byte[]> indexMap = fileStore.getCarbonIndexMapWithFullPath();	1✔
152	Map<String, Map<String, byte[]>> indexLocationMap = new HashMap<>();	1✔
153	for (Map.Entry<String, byte[]> entry: indexMap.entrySet()) {	1✔
154	Path path = new Path(entry.getKey());	1✔
155	Map<String, byte[]> map = indexLocationMap.get(path.getParent().toString());	1✔
156	if (map == null) {	1✔
157	map = new HashMap<>();	1✔
158	indexLocationMap.put(path.getParent().toString(), map);	1✔
159	}
160	map.put(path.getName(), entry.getValue());	1✔
161	}	1✔
162	List<PartitionSpec> partitionSpecs = SegmentFileStore	1✔
163	.getPartitionSpecs(segmentId, table.getTablePath(), SegmentStatusManager	1✔
164	.readLoadMetadata(CarbonTablePath.getMetadataPath(table.getTablePath())));	1✔
165	for (Map.Entry<String, Map<String, byte[]>> entry : indexLocationMap.entrySet()) {	1✔
166	String mergeIndexFile =	1✔
167	writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), entry.getValue(), segmentId);	1✔
168	for (Map.Entry<String, SegmentFileStore.FolderDetails> segentry : segmentFileStore	1✔
169	.getLocationMap().entrySet()) {	1✔
170	String location = segentry.getKey();	1✔
171	if (segentry.getValue().isRelative()) {	1✔
172	location =	1✔
173	segmentFileStore.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + location;	1✔
174	}
175	if (new Path(entry.getKey()).equals(new Path(location))) {	1✔
176	segentry.getValue().setMergeFileName(mergeIndexFile);	1✔
177	segentry.getValue().setFiles(new HashSet<String>());	1✔
178	break;	1✔
179	}
180	}	1✔
181	if (table.isHivePartitionTable()) {	1✔
182	for (PartitionSpec partitionSpec : partitionSpecs) {	1✔
183	if (partitionSpec.getLocation().toString().equals(partitionPath)) {	1✔
184	SegmentFileStore.writeSegmentFile(table.getTablePath(), mergeIndexFile, partitionPath,	1✔
185	segmentId + "_" + uuid + "", partitionSpec.getPartitions(), true);	1✔
186	}
187	}	1✔
188	}
189	}	1✔
190	String newSegmentFileName = SegmentFileStore.genSegmentFileName(segmentId, uuid)	1✔
191	+ CarbonTablePath.SEGMENT_EXT;
192	String path = CarbonTablePath.getSegmentFilesLocation(table.getTablePath())	1✔
193	+ CarbonCommonConstants.FILE_SEPARATOR + newSegmentFileName;
194	if (!table.isHivePartitionTable()) {	1✔
195	SegmentFileStore.writeSegmentFile(segmentFileStore.getSegmentFile(), path);	1✔
196	SegmentFileStore.updateSegmentFile(table, segmentId, newSegmentFileName,	1✔
197	table.getCarbonTableIdentifier().getTableId(), segmentFileStore);	1✔
198	}
199
200	for (CarbonFile file : indexFiles) {	1✔
201	file.delete();	1✔
202	}
203
204	return uuid;	1✔
205	}
206
207	private String writeMergeIndexFile(List<String> indexFileNamesTobeAdded, String segmentPath,
208	Map<String, byte[]> indexMap, String segment_id) throws IOException {
209	MergedBlockIndexHeader indexHeader = new MergedBlockIndexHeader();	1✔
210	MergedBlockIndex mergedBlockIndex = new MergedBlockIndex();	1✔
211	List<String> fileNames = new ArrayList<>(indexMap.size());	1✔
212	List<ByteBuffer> data = new ArrayList<>(indexMap.size());	1✔
213	for (Map.Entry<String, byte[]> entry : indexMap.entrySet()) {	1✔
214	if (indexFileNamesTobeAdded == null \|\|	1✔
215	indexFileNamesTobeAdded.contains(entry.getKey())) {	×
216	fileNames.add(entry.getKey());	1✔
217	data.add(ByteBuffer.wrap(entry.getValue()));	1✔
218	}
219	}	1✔
220	if (fileNames.size() > 0) {	1✔
221	String mergeIndexName =	1✔
222	segment_id + '_' + System.currentTimeMillis() + CarbonTablePath.MERGE_INDEX_FILE_EXT;	1✔
223	openThriftWriter(segmentPath + "/" + mergeIndexName);	1✔
224	indexHeader.setFile_names(fileNames);	1✔
225	mergedBlockIndex.setFileData(data);	1✔
226	writeMergedBlockIndexHeader(indexHeader);	1✔
227	writeMergedBlockIndex(mergedBlockIndex);	1✔
228	close();	1✔
229	return mergeIndexName;	1✔
230	}
231	return null;	×
232	}
233
234	/**
235	* Merge all the carbonindex files of segment to a merged file
236	*
237	* @param segmentId
238	* @throws IOException
239	*/
240	public String mergeCarbonIndexFilesOfSegment(String segmentId, String uuid, String tablePath,
241	String partitionPath) throws IOException {
242	return mergeCarbonIndexFilesOfSegment(segmentId, tablePath, null, false, uuid, partitionPath);	1✔
243	}
244
245	/**
246	* Merge all the carbonindex files of segment to a merged file
247	*
248	* @param segmentId
249	* @param readFileFooterFromCarbonDataFile
250	* @throws IOException
251	*/
252	public String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath,
253	boolean readFileFooterFromCarbonDataFile, String uuid) throws IOException {
254	return mergeCarbonIndexFilesOfSegment(segmentId, tablePath, null,	1✔
255	readFileFooterFromCarbonDataFile, uuid, null);
256	}
257
258	private boolean isCarbonIndexFilePresent(CarbonFile[] indexFiles) {
259	for (CarbonFile file : indexFiles) {	1✔
260	if (file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {	1✔
261	return true;	1✔
262	}
263	}
264	return false;	1✔
265	}
266
267	/**
268	* It writes thrift object to file
269	*
270	* @throws IOException
271	*/
272	private void writeMergedBlockIndexHeader(MergedBlockIndexHeader indexObject) throws IOException {
273	thriftWriter.write(indexObject);	1✔
274	}	1✔
275
276	/**
277	* It writes thrift object to file
278	*
279	* @throws IOException
280	*/
281	private void writeMergedBlockIndex(MergedBlockIndex indexObject) throws IOException {
282	thriftWriter.write(indexObject);	1✔
283	}	1✔
284
285	/**
286	* Below method will be used to open the thrift writer
287	*
288	* @param filePath file path where data need to be written
289	* @throws IOException throws io exception in case of any failure
290	*/
291	private void openThriftWriter(String filePath) throws IOException {
292	// create thrift writer instance
293	thriftWriter = new ThriftWriter(filePath, false);	1✔
294	// open the file stream
295	thriftWriter.open(FileWriteOperation.OVERWRITE);	1✔
296	}	1✔
297
298	/**
299	* Below method will be used to close the thrift object
300	*/
301	private void close() throws IOException {
302	thriftWriter.close();	1✔
303	}	1✔
304
305	public static class SegmentIndexFIleMergeStatus implements Serializable {
306
307	private SegmentFileStore.SegmentFile segmentFile;
308
309	private List<String> filesTobeDeleted;
310
311	public SegmentIndexFIleMergeStatus(SegmentFileStore.SegmentFile segmentFile,
312	List<String> filesTobeDeleted) {	×
313	this.segmentFile = segmentFile;	×
314	this.filesTobeDeleted = filesTobeDeleted;	×
315	}	×
316
317	public SegmentFileStore.SegmentFile getSegmentFile() {
318	return segmentFile;	×
319	}
320
321	public List<String> getFilesTobeDeleted() {
322	return filesTobeDeleted;	×
323	}
324	}
325	}

apache / carbondata / 11886

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous