11557

Build # 11557

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/d41b97d25615359a1041a44dbbc75e0c2d0f1f19">d41b97d25<a href="https://github.com/apache/carbondata/commit/d41b97d25615359a1041a44dbbc75e0c2d0f1f19">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/3c8c6dd439e2df4af7e82d4eba7c30fc9b98fe39">3c8c6dd43</a>">3c8c6dd43</a><a href="https://github.com/apache/carbondata/commit/d41b97d25615359a1041a44dbbc75e0c2d0f1f19"> into c2d4b3e0a">c2d4b3e0a</a>

Run Details

69 of 69 new or added lines in 3 files covered. (100.0%)

64069 of 81551 relevant lines covered (78.56%)

1.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.63

/tools/cli/src/main/java/org/apache/carbondata/tool/FileCollector.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.tool;

import java.io.IOException;
import java.util.*;

import org.apache.carbondata.common.Strings;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockletInfo3;
import org.apache.carbondata.format.ColumnSchema;
import org.apache.carbondata.format.FileFooter3;
import org.apache.carbondata.format.IndexHeader;

/**
 * A helper to collect all data files, schema file, table status file in a given folder
 */
class FileCollector {

  private long numBlock;
  private long numShard;
  private long numBlocklet;
  private long numPage;
  private long numRow;
  private long totalDataSize;
  private List<String> outPuts;

  // file path mapping to file object
  private LinkedHashMap<String, DataFile> dataFiles = new LinkedHashMap<>();
  private CarbonFile tableStatusFile;
  private CarbonFile schemaFile;


  FileCollector(List<String> outPuts) {
    this.outPuts = outPuts;
  }

  void collectFiles(String dataFolder) throws IOException {
    Set<String> shards = new HashSet<>();
    CarbonFile folder = FileFactory.getCarbonFile(dataFolder);
    List<CarbonFile> files = folder.listFiles(true);
    List<DataFile> unsortedFiles = new ArrayList<>();
    for (CarbonFile file : files) {
      if (isColumnarFile(file.getName())) {
        DataFile dataFile = new DataFile(file);
        dataFile.collectAllMeta();
        unsortedFiles.add(dataFile);
        collectNum(dataFile.getFooter());
        shards.add(dataFile.getShardName());
        totalDataSize += file.getSize();
      } else if (file.getName().endsWith(CarbonTablePath.TABLE_STATUS_FILE)) {
        tableStatusFile = file;
      } else if (file.getName().startsWith(CarbonTablePath.SCHEMA_FILE)) {
        schemaFile = file;
      } else if (isStreamFile(file.getName())) {
        outPuts.add(("WARN: input path contains streaming file, this tool does not support it yet, "
            + "skipping it..."));

      }
    }

    Collections.sort(unsortedFiles, new Comparator<DataFile>() {
      @Override public int compare(DataFile o1, DataFile o2) {
        if (o1.getShardName().equalsIgnoreCase(o2.getShardName())) {
          return Integer.parseInt(o1.getPartNo()) - Integer.parseInt(o2.getPartNo());
        } else {
          return o1.getShardName().compareTo(o2.getShardName());
        }
      }
    });

    for (DataFile collectedFile : unsortedFiles) {
      this.dataFiles.put(collectedFile.getFilePath(), collectedFile);
    }
    numShard = shards.size();
  }

  private void collectNum(FileFooter3 footer) {
    numBlock++;
    numBlocklet += footer.blocklet_index_list.size();
    numRow += footer.num_rows;
    for (BlockletInfo3 blockletInfo3 : footer.blocklet_info_list3) {
      numPage += blockletInfo3.number_number_of_pages;
    }
  }

  private boolean isColumnarFile(String fileName) {
    // if the timestamp in file name is "0", it is a streaming file
    return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&
        !CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");
  }

  private boolean isStreamFile(String fileName) {
    // if the timestamp in file name is "0", it is a streaming file
    return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&
        CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");
  }

  // return file path mapping to file object
  LinkedHashMap<String, DataFile> getDataFiles() {
    return dataFiles;
  }

  CarbonFile getTableStatusFile() {
    return tableStatusFile;
  }

  CarbonFile getSchemaFile() {
    return schemaFile;
  }

  int getNumDataFiles() {
    return dataFiles.size();
  }

  void printBasicStats() {
    if (dataFiles.size() == 0) {
      System.out.println("no data file found");
      return;
    }
    outPuts.add("## Summary");
    String format = String
        .format("total: %,d blocks, %,d shards, %,d blocklets, %,d pages, %,d rows, %s", numBlock,
            numShard, numBlocklet, numPage, numRow, Strings.formatSize(totalDataSize));
    outPuts.add(format);

    String format1 = String.format("avg: %s/block, %s/blocklet, %,d rows/block, %,d rows/blocklet",
        Strings.formatSize((float) totalDataSize / numBlock),
        Strings.formatSize((float) totalDataSize / numBlocklet), numRow / numBlock,
        numRow / numBlocklet);
    outPuts.add(format1);
  }

  private String makeSortColumnsString(List<ColumnSchema> columnList) {
    StringBuilder builder = new StringBuilder();
    for (ColumnSchema column : columnList) {
      if (column.isDimension()) {
        Map<String, String> properties = column.getColumnProperties();
        if (properties != null) {
          if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {
            builder.append(column.column_name).append(",");
          }
        }
      }
    }
    if (builder.length() > 1) {
      return builder.substring(0, builder.length() - 1);
    } else {
      return "";
    }
  }

  public void collectSortColumns(String segmentFolder) throws IOException {
    CarbonFile[] files = SegmentIndexFileStore.getCarbonIndexFiles(
        segmentFolder, FileFactory.getConfiguration());
    Set<Boolean> isSortSet = new HashSet<>();
    Set<String> sortColumnsSet = new HashSet<>();
    if (files != null) {
      for (CarbonFile file : files) {
        IndexHeader indexHeader = SegmentIndexFileStore.readIndexHeader(
            file.getCanonicalPath(), FileFactory.getConfiguration());
        if (indexHeader != null) {
          if (indexHeader.isSetIs_sort()) {
            isSortSet.add(indexHeader.is_sort);
            if (indexHeader.is_sort) {
              sortColumnsSet.add(makeSortColumnsString(indexHeader.getTable_columns()));
            }
          } else {
            // if is_sort is not set, it will be old store and consider as local_sort by default.
            sortColumnsSet.add(makeSortColumnsString(indexHeader.getTable_columns()));
          }
        }
        if (isSortSet.size() >= 2 || sortColumnsSet.size() >= 2) {
          break;
        }
      }
    }
    // for all index files, sort_columns should be same
    if (isSortSet.size() <= 1 && sortColumnsSet.size() == 1) {
      outPuts.add("sorted by " + sortColumnsSet.iterator().next());
    } else {
      outPuts.add("unsorted");
    }
  }

  public void close() throws IOException {
    for (DataFile file : dataFiles.values()) {
      file.close();
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.tool;
19
20	import java.io.IOException;
21	import java.util.*;
22
23	import org.apache.carbondata.common.Strings;
24	import org.apache.carbondata.core.constants.CarbonCommonConstants;
25	import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
26	import org.apache.carbondata.core.datastore.impl.FileFactory;
27	import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
28	import org.apache.carbondata.core.util.path.CarbonTablePath;
29	import org.apache.carbondata.format.BlockletInfo3;
30	import org.apache.carbondata.format.ColumnSchema;
31	import org.apache.carbondata.format.FileFooter3;
32	import org.apache.carbondata.format.IndexHeader;
33
34	/**
35	* A helper to collect all data files, schema file, table status file in a given folder
36	*/
37	class FileCollector {
38
39	private long numBlock;
40	private long numShard;
41	private long numBlocklet;
42	private long numPage;
43	private long numRow;
44	private long totalDataSize;
45	private List<String> outPuts;
46
47	// file path mapping to file object
48	private LinkedHashMap<String, DataFile> dataFiles = new LinkedHashMap<>();	1✔
49	private CarbonFile tableStatusFile;
50	private CarbonFile schemaFile;
51
52
53	FileCollector(List<String> outPuts) {	1✔
54	this.outPuts = outPuts;	1✔
55	}	1✔
56
57	void collectFiles(String dataFolder) throws IOException {
58	Set<String> shards = new HashSet<>();	1✔
59	CarbonFile folder = FileFactory.getCarbonFile(dataFolder);	1✔
60	List<CarbonFile> files = folder.listFiles(true);	1✔
61	List<DataFile> unsortedFiles = new ArrayList<>();	1✔
62	for (CarbonFile file : files) {	1✔
63	if (isColumnarFile(file.getName())) {	1✔
64	DataFile dataFile = new DataFile(file);	1✔
65	dataFile.collectAllMeta();	1✔
66	unsortedFiles.add(dataFile);	1✔
67	collectNum(dataFile.getFooter());	1✔
68	shards.add(dataFile.getShardName());	1✔
69	totalDataSize += file.getSize();	1✔
70	} else if (file.getName().endsWith(CarbonTablePath.TABLE_STATUS_FILE)) {	1✔
71	tableStatusFile = file;	×
72	} else if (file.getName().startsWith(CarbonTablePath.SCHEMA_FILE)) {	1✔
73	schemaFile = file;	×
74	} else if (isStreamFile(file.getName())) {	1✔
75	outPuts.add(("WARN: input path contains streaming file, this tool does not support it yet, "	×
76	+ "skipping it..."));
77
78	}
79	}	1✔
80
81	Collections.sort(unsortedFiles, new Comparator<DataFile>() {	1✔
82	@Override public int compare(DataFile o1, DataFile o2) {
83	if (o1.getShardName().equalsIgnoreCase(o2.getShardName())) {	1✔
84	return Integer.parseInt(o1.getPartNo()) - Integer.parseInt(o2.getPartNo());	1✔
85	} else {
86	return o1.getShardName().compareTo(o2.getShardName());	1✔
87	}
88	}
89	});
90
91	for (DataFile collectedFile : unsortedFiles) {	1✔
92	this.dataFiles.put(collectedFile.getFilePath(), collectedFile);	1✔
93	}	1✔
94	numShard = shards.size();	1✔
95	}	1✔
96
97	private void collectNum(FileFooter3 footer) {
98	numBlock++;	1✔
99	numBlocklet += footer.blocklet_index_list.size();	1✔
100	numRow += footer.num_rows;	1✔
101	for (BlockletInfo3 blockletInfo3 : footer.blocklet_info_list3) {	1✔
102	numPage += blockletInfo3.number_number_of_pages;	1✔
103	}	1✔
104	}	1✔
105
106	private boolean isColumnarFile(String fileName) {
107	// if the timestamp in file name is "0", it is a streaming file
108	return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&	1✔
109	!CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");	1✔
110	}
111
112	private boolean isStreamFile(String fileName) {
113	// if the timestamp in file name is "0", it is a streaming file
114	return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&	1✔
115	CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");	1✔
116	}
117
118	// return file path mapping to file object
119	LinkedHashMap<String, DataFile> getDataFiles() {
120	return dataFiles;	1✔
121	}
122
123	CarbonFile getTableStatusFile() {
124	return tableStatusFile;	1✔
125	}
126
127	CarbonFile getSchemaFile() {
128	return schemaFile;	1✔
129	}
130
131	int getNumDataFiles() {
132	return dataFiles.size();	1✔
133	}
134
135	void printBasicStats() {
136	if (dataFiles.size() == 0) {	1✔
137	System.out.println("no data file found");	×
138	return;	×
139	}
140	outPuts.add("## Summary");	1✔
141	String format = String	1✔
142	.format("total: %,d blocks, %,d shards, %,d blocklets, %,d pages, %,d rows, %s", numBlock,	1✔
143	numShard, numBlocklet, numPage, numRow, Strings.formatSize(totalDataSize));	1✔
144	outPuts.add(format);	1✔
145
146	String format1 = String.format("avg: %s/block, %s/blocklet, %,d rows/block, %,d rows/blocklet",	1✔
147	Strings.formatSize((float) totalDataSize / numBlock),	1✔
148	Strings.formatSize((float) totalDataSize / numBlocklet), numRow / numBlock,	1✔
149	numRow / numBlocklet);	1✔
150	outPuts.add(format1);	1✔
151	}	1✔
152
153	private String makeSortColumnsString(List<ColumnSchema> columnList) {
154	StringBuilder builder = new StringBuilder();	1✔
155	for (ColumnSchema column : columnList) {	1✔
156	if (column.isDimension()) {	1✔
157	Map<String, String> properties = column.getColumnProperties();	1✔
158	if (properties != null) {	1✔
159	if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {	1✔
160	builder.append(column.column_name).append(",");	1✔
161	}
162	}
163	}
164	}	1✔
165	if (builder.length() > 1) {	1✔
166	return builder.substring(0, builder.length() - 1);	1✔
167	} else {
168	return "";	×
169	}
170	}
171
172	public void collectSortColumns(String segmentFolder) throws IOException {
173	CarbonFile[] files = SegmentIndexFileStore.getCarbonIndexFiles(	1✔
174	segmentFolder, FileFactory.getConfiguration());	1✔
175	Set<Boolean> isSortSet = new HashSet<>();	1✔
176	Set<String> sortColumnsSet = new HashSet<>();	1✔
177	if (files != null) {	1✔
178	for (CarbonFile file : files) {	1✔
179	IndexHeader indexHeader = SegmentIndexFileStore.readIndexHeader(	1✔
180	file.getCanonicalPath(), FileFactory.getConfiguration());	1✔
181	if (indexHeader != null) {	1✔
182	if (indexHeader.isSetIs_sort()) {	1✔
183	isSortSet.add(indexHeader.is_sort);	1✔
184	if (indexHeader.is_sort) {	1✔
185	sortColumnsSet.add(makeSortColumnsString(indexHeader.getTable_columns()));	1✔
186	}
187	} else {
188	// if is_sort is not set, it will be old store and consider as local_sort by default.
189	sortColumnsSet.add(makeSortColumnsString(indexHeader.getTable_columns()));	×
190	}
191	}
192	if (isSortSet.size() >= 2 \|\| sortColumnsSet.size() >= 2) {	1✔
193	break;	×
194	}
195	}
196	}
197	// for all index files, sort_columns should be same
198	if (isSortSet.size() <= 1 && sortColumnsSet.size() == 1) {	1✔
199	outPuts.add("sorted by " + sortColumnsSet.iterator().next());	1✔
200	} else {
201	outPuts.add("unsorted");	×
202	}
203	}	1✔
204
205	public void close() throws IOException {
206	for (DataFile file : dataFiles.values()) {	1✔
207	file.close();	1✔
208	}	1✔
209	}	1✔
210	}

apache / carbondata / 11557

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous