8701

Build # 8701

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/10347882d7e86c8c839342893771233686b93846">10347882d<a href="https://github.com/apache/carbondata/commit/10347882d7e86c8c839342893771233686b93846">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/9fed41fcfb9b1f2900029db5ab0a3a8610b4085e">9fed41fcf</a>">9fed41fcf</a><a href="https://github.com/apache/carbondata/commit/10347882d7e86c8c839342893771233686b93846"> into e3eb03054">e3eb03054</a>

Run Details

269 of 269 new or added lines in 6 files covered. (100.0%)

59965 of 74926 relevant lines covered (80.03%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

27.87

/tools/cli/src/main/java/org/apache/carbondata/tool/FileCollector.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.tool;

import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Set;

import org.apache.carbondata.common.Strings;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockletInfo3;
import org.apache.carbondata.format.FileFooter3;

/**
 * A helper to collect all data files, schema file, table status file in a given folder
 */
class FileCollector {

  private long numBlock;
  private long numShard;
  private long numBlocklet;
  private long numPage;
  private long numRow;
  private long totalDataSize;

  // file path mapping to file object
  private LinkedHashMap<String, DataFile> dataFiles = new LinkedHashMap<>();
  private CarbonFile tableStatusFile;
  private CarbonFile schemaFile;

  private PrintStream out;

  FileCollector(PrintStream out) {
    this.out = out;
  }

  void collectFiles(String dataFolder) throws IOException {
    Set<String> shards = new HashSet<>();
    CarbonFile folder = FileFactory.getCarbonFile(dataFolder);
    List<CarbonFile> files = folder.listFiles(true);
    List<DataFile> unsortedFiles = new ArrayList<>();
    for (CarbonFile file : files) {
      if (isColumnarFile(file.getName())) {
        DataFile dataFile = new DataFile(file);
        dataFile.collectAllMeta();
        unsortedFiles.add(dataFile);
        collectNum(dataFile.getFooter());
        shards.add(dataFile.getShardName());
        totalDataSize += file.getSize();
      } else if (file.getName().endsWith(CarbonTablePath.TABLE_STATUS_FILE)) {
        tableStatusFile = file;
      } else if (file.getName().startsWith(CarbonTablePath.SCHEMA_FILE)) {
        schemaFile = file;
      } else if (isStreamFile(file.getName())) {
        out.println("WARN: input path contains streaming file, this tool does not support it yet, "
            + "skipping it...");
      }
    }
    unsortedFiles.sort((o1, o2) -> {
      if (o1.getShardName().equalsIgnoreCase(o2.getShardName())) {
        return Integer.parseInt(o1.getPartNo()) - Integer.parseInt(o2.getPartNo());
      } else {
        return o1.getShardName().compareTo(o2.getShardName());
      }
    });
    for (DataFile collectedFile : unsortedFiles) {
      this.dataFiles.put(collectedFile.getFilePath(), collectedFile);
    }
    numShard = shards.size();
  }

  private void collectNum(FileFooter3 footer) {
    numBlock++;
    numBlocklet += footer.blocklet_index_list.size();
    numRow += footer.num_rows;
    for (BlockletInfo3 blockletInfo3 : footer.blocklet_info_list3) {
      numPage += blockletInfo3.number_number_of_pages;
    }
  }

  private boolean isColumnarFile(String fileName) {
    // if the timestamp in file name is "0", it is a streaming file
    return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&
        !CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");
  }

  private boolean isStreamFile(String fileName) {
    // if the timestamp in file name is "0", it is a streaming file
    return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&
        CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");
  }

  // return file path mapping to file object
  LinkedHashMap<String, DataFile> getDataFiles() {
    return dataFiles;
  }

  CarbonFile getTableStatusFile() {
    return tableStatusFile;
  }

  CarbonFile getSchemaFile() {
    return schemaFile;
  }

  int getNumDataFiles() {
    return dataFiles.size();
  }

  void printBasicStats() {
    if (dataFiles.size() == 0) {
      System.out.println("no data file found");
      return;
    }
    out.println("## Summary");
    out.println(
        String.format("total: %,d blocks, %,d shards, %,d blocklets, %,d pages, %,d rows, %s",
            numBlock, numShard, numBlocklet, numPage, numRow, Strings.formatSize(totalDataSize)));
    out.println(
        String.format("avg: %s/block, %s/blocklet, %,d rows/block, %,d rows/blocklet",
            Strings.formatSize((float) totalDataSize / numBlock),
            Strings.formatSize((float) totalDataSize / numBlocklet),
            numRow / numBlock,
            numRow / numBlocklet));
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.tool;
19
20	import java.io.IOException;
21	import java.io.PrintStream;
22	import java.util.ArrayList;
23	import java.util.HashSet;
24	import java.util.LinkedHashMap;
25	import java.util.List;
26	import java.util.Set;
27
28	import org.apache.carbondata.common.Strings;
29	import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
30	import org.apache.carbondata.core.datastore.impl.FileFactory;
31	import org.apache.carbondata.core.util.path.CarbonTablePath;
32	import org.apache.carbondata.format.BlockletInfo3;
33	import org.apache.carbondata.format.FileFooter3;
34
35	/**
36	* A helper to collect all data files, schema file, table status file in a given folder
37	*/
38	class FileCollector {
39
40	private long numBlock;
41	private long numShard;
42	private long numBlocklet;
43	private long numPage;
44	private long numRow;
45	private long totalDataSize;
46
47	// file path mapping to file object
48	private LinkedHashMap<String, DataFile> dataFiles = new LinkedHashMap<>();	1✔
49	private CarbonFile tableStatusFile;
50	private CarbonFile schemaFile;
51
52	private PrintStream out;
53
54	FileCollector(PrintStream out) {	1✔
55	this.out = out;	1✔
56	}	1✔
57
58	void collectFiles(String dataFolder) throws IOException {
59	Set<String> shards = new HashSet<>();	1✔
60	CarbonFile folder = FileFactory.getCarbonFile(dataFolder);	1✔
61	List<CarbonFile> files = folder.listFiles(true);	1✔
62	List<DataFile> unsortedFiles = new ArrayList<>();	1✔
63	for (CarbonFile file : files) {	1✔
64	if (isColumnarFile(file.getName())) {	×
65	DataFile dataFile = new DataFile(file);	×
66	dataFile.collectAllMeta();	×
67	unsortedFiles.add(dataFile);	×
68	collectNum(dataFile.getFooter());	×
69	shards.add(dataFile.getShardName());	×
70	totalDataSize += file.getSize();	×
71	} else if (file.getName().endsWith(CarbonTablePath.TABLE_STATUS_FILE)) {	×
72	tableStatusFile = file;	×
73	} else if (file.getName().startsWith(CarbonTablePath.SCHEMA_FILE)) {	×
74	schemaFile = file;	×
75	} else if (isStreamFile(file.getName())) {	×
76	out.println("WARN: input path contains streaming file, this tool does not support it yet, "	×
77	+ "skipping it...");
78	}
79	}	×
80	unsortedFiles.sort((o1, o2) -> {	1✔
81	if (o1.getShardName().equalsIgnoreCase(o2.getShardName())) {	×
82	return Integer.parseInt(o1.getPartNo()) - Integer.parseInt(o2.getPartNo());	×
83	} else {
84	return o1.getShardName().compareTo(o2.getShardName());	×
85	}
86	});
87	for (DataFile collectedFile : unsortedFiles) {	1✔
88	this.dataFiles.put(collectedFile.getFilePath(), collectedFile);	×
89	}	×
90	numShard = shards.size();	1✔
91	}	1✔
92
93	private void collectNum(FileFooter3 footer) {
94	numBlock++;	×
95	numBlocklet += footer.blocklet_index_list.size();	×
96	numRow += footer.num_rows;	×
97	for (BlockletInfo3 blockletInfo3 : footer.blocklet_info_list3) {	×
98	numPage += blockletInfo3.number_number_of_pages;	×
99	}	×
100	}	×
101
102	private boolean isColumnarFile(String fileName) {
103	// if the timestamp in file name is "0", it is a streaming file
104	return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&	×
105	!CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");	×
106	}
107
108	private boolean isStreamFile(String fileName) {
109	// if the timestamp in file name is "0", it is a streaming file
110	return fileName.endsWith(CarbonTablePath.CARBON_DATA_EXT) &&	×
111	CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileName).equals("0");	×
112	}
113
114	// return file path mapping to file object
115	LinkedHashMap<String, DataFile> getDataFiles() {
116	return dataFiles;	×
117	}
118
119	CarbonFile getTableStatusFile() {
120	return tableStatusFile;	×
121	}
122
123	CarbonFile getSchemaFile() {
124	return schemaFile;	×
125	}
126
127	int getNumDataFiles() {
128	return dataFiles.size();	1✔
129	}
130
131	void printBasicStats() {
132	if (dataFiles.size() == 0) {	1✔
133	System.out.println("no data file found");	1✔
134	return;	1✔
135	}
136	out.println("## Summary");	×
137	out.println(	×
138	String.format("total: %,d blocks, %,d shards, %,d blocklets, %,d pages, %,d rows, %s",	×
139	numBlock, numShard, numBlocklet, numPage, numRow, Strings.formatSize(totalDataSize)));	×
140	out.println(	×
141	String.format("avg: %s/block, %s/blocklet, %,d rows/block, %,d rows/blocklet",	×
142	Strings.formatSize((float) totalDataSize / numBlock),	×
143	Strings.formatSize((float) totalDataSize / numBlocklet),	×
144	numRow / numBlock,	×
145	numRow / numBlocklet));	×
146	}	×
147	}

apache / carbondata / 8701

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous