9391

Build # 9391

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/d941b6790fed2e53fee75c690e07d5e5229ea918">d941b6790">&lt;a href=&quot;https://github.com/apache/carbondata/commit/</a><a class="double-link" href="https://github.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/d941b6790fed2e53fee75c690e07d5e5229ea918">d941b6790</a><a href="https://github.com/apache/carbondata/commit/d941b6790fed2e53fee75c690e07d5e5229ea918">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/d941b6790fed2e53fee75c690e07d5e5229ea918&quot;&gt;d941b6790&lt;/a&gt;&lt;a href=&quot;https://github.com/apache/carbondata/commit/d941b6790fed2e53fee75c690e07d5e5229ea918&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/carbondata/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/carbondata/commit/&amp;amp;lt;a ... (continued)

Run Details

20 of 20 new or added lines in 5 files covered. (100.0%)

61821 of 77146 relevant lines covered (80.14%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.63

/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.tool;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.carbondata.common.Strings;
import org.apache.carbondata.core.datastore.block.BlockletInfos;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.util.DataFileFooterConverterV3;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.FileFooter3;
import org.apache.carbondata.format.FileHeader;

import org.apache.commons.cli.CommandLine;

class ScanBenchmark implements Command {

  private String dataFolder;
  private DataFile file;
  private List<String> outPuts;

  ScanBenchmark(String dataFolder, List<String> outPuts) {
    this.dataFolder = dataFolder;
    this.outPuts = outPuts;
  }

  @Override
  public void run(CommandLine line) throws IOException, MemoryException {
    if (line.hasOption("f")) {
      String filePath = line.getOptionValue("f");
      file = new DataFile(FileFactory.getCarbonFile(filePath));
    } else {
      FileCollector collector = new FileCollector(outPuts);
      collector.collectFiles(dataFolder);
      if (collector.getNumDataFiles() == 0) {
        return;
      }
      Map<String, DataFile> dataFiles = collector.getDataFiles();
      Iterator<DataFile> iterator = dataFiles.values().iterator();
      // use the first file and close the rest
      file = iterator.next();
      while (iterator.hasNext()) {
        iterator.next().close();
      }
    }

    outPuts.add("\n## Benchmark");
    final AtomicReference<FileHeader> fileHeaderRef = new AtomicReference<>();
    final AtomicReference<FileFooter3> fileFoorterRef = new AtomicReference<>();
    final AtomicReference<DataFileFooter> convertedFooterRef = new AtomicReference<>();

    // benchmark read header and footer time
    benchmarkOperation("ReadHeaderAndFooter", new Operation() {
      @Override public void run() throws IOException, MemoryException {
        fileHeaderRef.set(file.readHeader());
        fileFoorterRef.set(file.readFooter());
      }
    });
    final FileHeader fileHeader = fileHeaderRef.get();
    final FileFooter3 fileFooter = fileFoorterRef.get();

    // benchmark convert footer
    benchmarkOperation("ConvertFooter", new Operation() {
      @Override public void run() throws IOException, MemoryException {
        convertFooter(fileHeader, fileFooter);
      }
    });

    // benchmark read all meta and convert footer
    benchmarkOperation("ReadAllMetaAndConvertFooter", new Operation() {
      @Override public void run() throws IOException, MemoryException {
        DataFileFooter footer = readAndConvertFooter(file);
        convertedFooterRef.set(footer);
      }
    });

    if (line.hasOption("c")) {
      String columnName = line.getOptionValue("c");
      outPuts.add("\nScan column '" + columnName + "'");

      final DataFileFooter footer = convertedFooterRef.get();
      final AtomicReference<AbstractRawColumnChunk> columnChunk = new AtomicReference<>();
      final int columnIndex = file.getColumnIndex(columnName);
      final boolean dimension = file.getColumn(columnName).isDimensionColumn();
      for (int i = 0; i < footer.getBlockletList().size(); i++) {
        final int blockletId = i;
        outPuts.add(String.format("Blocklet#%d: total size %s, %,d pages, %,d rows",
            blockletId,
            Strings.formatSize(file.getColumnDataSizeInBytes(blockletId, columnIndex)),
            footer.getBlockletList().get(blockletId).getNumberOfPages(),
            footer.getBlockletList().get(blockletId).getNumberOfRows()));
        benchmarkOperation("\tColumnChunk IO", new Operation() {
          @Override public void run() throws IOException, MemoryException {
            columnChunk.set(readBlockletColumnChunkIO(footer, blockletId, columnIndex, dimension));
          }
        });

        if (dimensionColumnChunkReader != null) {
          benchmarkOperation("\tDecompress Pages", new Operation() {
            @Override public void run() throws IOException, MemoryException {
              decompressDimensionPages(columnChunk.get(),
                  footer.getBlockletList().get(blockletId).getNumberOfPages());
            }
          });
        } else {
          benchmarkOperation("\tDecompress Pages", new Operation() {
            @Override public void run() throws IOException, MemoryException {
              decompressMeasurePages(columnChunk.get(),
                  footer.getBlockletList().get(blockletId).getNumberOfPages());
            }
          });
        }
      }
    }
    file.close();
  }

  interface Operation {
    void run() throws IOException, MemoryException;
  }

  private void benchmarkOperation(String opName, Operation op) throws IOException, MemoryException {
    long start, end;
    start = System.nanoTime();
    op.run();
    end = System.nanoTime();
    outPuts.add(String.format("%s takes %,d us", opName, (end - start) / 1000));
  }

  private DataFileFooter readAndConvertFooter(DataFile file) throws IOException {
    int numBlocklets = file.getNumBlocklet();
    BlockletInfos blockletInfos = new BlockletInfos(numBlocklets, 0, numBlocklets);
    String segmentId = CarbonTablePath.DataFileUtil.getSegmentNo(file.getFilePath());
    TableBlockInfo blockInfo =
        new TableBlockInfo(file.getFilePath(), file.getFooterOffset(),
            segmentId, new String[]{"localhost"}, file.getFileSizeInBytes(),
            blockletInfos, ColumnarFormatVersion.V3, new String[0]);

    DataFileFooterConverterV3 converter = new DataFileFooterConverterV3();
    return converter.readDataFileFooter(blockInfo);
  }

  private DataFileFooter convertFooter(FileHeader fileHeader, FileFooter3 fileFooter) {
    DataFileFooterConverterV3 converter = new DataFileFooterConverterV3();
    return converter.convertDataFileFooter(fileHeader, fileFooter);
  }

  private DimensionColumnChunkReader dimensionColumnChunkReader;
  private MeasureColumnChunkReader measureColumnChunkReader;

  private AbstractRawColumnChunk readBlockletColumnChunkIO(
      DataFileFooter footer, int blockletId, int columnIndex, boolean dimension)
      throws IOException {
    BlockletInfo blockletInfo = footer.getBlockletList().get(blockletId);
    if (dimension) {
      dimensionColumnChunkReader = CarbonDataReaderFactory.getInstance()
          .getDimensionColumnChunkReader(ColumnarFormatVersion.V3, blockletInfo,
              footer.getSegmentInfo().getColumnCardinality(), file.getFilePath(), false);
      return dimensionColumnChunkReader.readRawDimensionChunk(file.getFileReader(), columnIndex);
    } else {
      columnIndex = columnIndex - file.numDimensions();
      assert (columnIndex >= 0);
      measureColumnChunkReader = CarbonDataReaderFactory.getInstance()
          .getMeasureColumnChunkReader(ColumnarFormatVersion.V3, blockletInfo,
              file.getFilePath(), false);
      return measureColumnChunkReader.readRawMeasureChunk(file.getFileReader(), columnIndex);
    }
  }

  private DimensionColumnPage[] decompressDimensionPages(
      AbstractRawColumnChunk rawColumnChunk, int numPages) throws IOException, MemoryException {
    DimensionColumnPage[] pages = new DimensionColumnPage[numPages];
    for (int i = 0; i < pages.length; i++) {
      pages[i] = dimensionColumnChunkReader.decodeColumnPage(
          (DimensionRawColumnChunk) rawColumnChunk, i);
    }
    return pages;
  }

  private ColumnPage[] decompressMeasurePages(
      AbstractRawColumnChunk rawColumnChunk, int numPages) throws IOException, MemoryException {
    ColumnPage[] pages = new ColumnPage[numPages];
    for (int i = 0; i < pages.length; i++) {
      pages[i] = measureColumnChunkReader.decodeColumnPage(
          (MeasureRawColumnChunk) rawColumnChunk, i);
    }
    return pages;
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.tool;
19
20	import java.io.IOException;
21	import java.util.Iterator;
22	import java.util.List;
23	import java.util.Map;
24	import java.util.concurrent.atomic.AtomicReference;
25
26	import org.apache.carbondata.common.Strings;
27	import org.apache.carbondata.core.datastore.block.BlockletInfos;
28	import org.apache.carbondata.core.datastore.block.TableBlockInfo;
29	import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
30	import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
31	import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
32	import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
33	import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
34	import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
35	import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
36	import org.apache.carbondata.core.datastore.impl.FileFactory;
37	import org.apache.carbondata.core.datastore.page.ColumnPage;
38	import org.apache.carbondata.core.memory.MemoryException;
39	import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
40	import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
41	import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
42	import org.apache.carbondata.core.util.DataFileFooterConverterV3;
43	import org.apache.carbondata.core.util.path.CarbonTablePath;
44	import org.apache.carbondata.format.FileFooter3;
45	import org.apache.carbondata.format.FileHeader;
46
47	import org.apache.commons.cli.CommandLine;
48
49	class ScanBenchmark implements Command {	1✔
50
51	private String dataFolder;
52	private DataFile file;
53	private List<String> outPuts;
54
55	ScanBenchmark(String dataFolder, List<String> outPuts) {	1✔
56	this.dataFolder = dataFolder;	1✔
57	this.outPuts = outPuts;	1✔
58	}	1✔
59
60	@Override
61	public void run(CommandLine line) throws IOException, MemoryException {
62	if (line.hasOption("f")) {	1✔
63	String filePath = line.getOptionValue("f");	×
64	file = new DataFile(FileFactory.getCarbonFile(filePath));	×
65	} else {	×
66	FileCollector collector = new FileCollector(outPuts);	1✔
67	collector.collectFiles(dataFolder);	1✔
68	if (collector.getNumDataFiles() == 0) {	1✔
69	return;	×
70	}
71	Map<String, DataFile> dataFiles = collector.getDataFiles();	1✔
72	Iterator<DataFile> iterator = dataFiles.values().iterator();	1✔
73	// use the first file and close the rest
74	file = iterator.next();	1✔
75	while (iterator.hasNext()) {	1✔
76	iterator.next().close();	1✔
77	}
78	}
79
80	outPuts.add("\n## Benchmark");	1✔
81	final AtomicReference<FileHeader> fileHeaderRef = new AtomicReference<>();	1✔
82	final AtomicReference<FileFooter3> fileFoorterRef = new AtomicReference<>();	1✔
83	final AtomicReference<DataFileFooter> convertedFooterRef = new AtomicReference<>();	1✔
84
85	// benchmark read header and footer time
86	benchmarkOperation("ReadHeaderAndFooter", new Operation() {	1✔
87	@Override public void run() throws IOException, MemoryException {
88	fileHeaderRef.set(file.readHeader());	1✔
89	fileFoorterRef.set(file.readFooter());	1✔
90	}	1✔
91	});
92	final FileHeader fileHeader = fileHeaderRef.get();	1✔
93	final FileFooter3 fileFooter = fileFoorterRef.get();	1✔
94
95	// benchmark convert footer
96	benchmarkOperation("ConvertFooter", new Operation() {	1✔
97	@Override public void run() throws IOException, MemoryException {
98	convertFooter(fileHeader, fileFooter);	1✔
99	}	1✔
100	});
101
102	// benchmark read all meta and convert footer
103	benchmarkOperation("ReadAllMetaAndConvertFooter", new Operation() {	1✔
104	@Override public void run() throws IOException, MemoryException {
105	DataFileFooter footer = readAndConvertFooter(file);	1✔
106	convertedFooterRef.set(footer);	1✔
107	}	1✔
108	});
109
110	if (line.hasOption("c")) {	1✔
111	String columnName = line.getOptionValue("c");	1✔
112	outPuts.add("\nScan column '" + columnName + "'");	1✔
113
114	final DataFileFooter footer = convertedFooterRef.get();	1✔
115	final AtomicReference<AbstractRawColumnChunk> columnChunk = new AtomicReference<>();	1✔
116	final int columnIndex = file.getColumnIndex(columnName);	1✔
117	final boolean dimension = file.getColumn(columnName).isDimensionColumn();	1✔
118	for (int i = 0; i < footer.getBlockletList().size(); i++) {	1✔
119	final int blockletId = i;	1✔
120	outPuts.add(String.format("Blocklet#%d: total size %s, %,d pages, %,d rows",	1✔
121	blockletId,	1✔
122	Strings.formatSize(file.getColumnDataSizeInBytes(blockletId, columnIndex)),	1✔
123	footer.getBlockletList().get(blockletId).getNumberOfPages(),	1✔
124	footer.getBlockletList().get(blockletId).getNumberOfRows()));	1✔
125	benchmarkOperation("\tColumnChunk IO", new Operation() {	1✔
126	@Override public void run() throws IOException, MemoryException {
127	columnChunk.set(readBlockletColumnChunkIO(footer, blockletId, columnIndex, dimension));	1✔
128	}	1✔
129	});
130
131	if (dimensionColumnChunkReader != null) {	1✔
132	benchmarkOperation("\tDecompress Pages", new Operation() {	1✔
133	@Override public void run() throws IOException, MemoryException {
134	decompressDimensionPages(columnChunk.get(),	1✔
135	footer.getBlockletList().get(blockletId).getNumberOfPages());	1✔
136	}	1✔
137	});
138	} else {
139	benchmarkOperation("\tDecompress Pages", new Operation() {	×
140	@Override public void run() throws IOException, MemoryException {
141	decompressMeasurePages(columnChunk.get(),	×
142	footer.getBlockletList().get(blockletId).getNumberOfPages());	×
143	}	×
144	});
145	}
146	}
147	}
148	file.close();	1✔
149	}	1✔
150
151	interface Operation {
152	void run() throws IOException, MemoryException;
153	}
154
155	private void benchmarkOperation(String opName, Operation op) throws IOException, MemoryException {
156	long start, end;
157	start = System.nanoTime();	1✔
158	op.run();	1✔
159	end = System.nanoTime();	1✔
160	outPuts.add(String.format("%s takes %,d us", opName, (end - start) / 1000));	1✔
161	}	1✔
162
163	private DataFileFooter readAndConvertFooter(DataFile file) throws IOException {
164	int numBlocklets = file.getNumBlocklet();	1✔
165	BlockletInfos blockletInfos = new BlockletInfos(numBlocklets, 0, numBlocklets);	1✔
166	String segmentId = CarbonTablePath.DataFileUtil.getSegmentNo(file.getFilePath());	1✔
167	TableBlockInfo blockInfo =	1✔
168	new TableBlockInfo(file.getFilePath(), file.getFooterOffset(),	1✔
169	segmentId, new String[]{"localhost"}, file.getFileSizeInBytes(),	1✔
170	blockletInfos, ColumnarFormatVersion.V3, new String[0]);
171
172	DataFileFooterConverterV3 converter = new DataFileFooterConverterV3();	1✔
173	return converter.readDataFileFooter(blockInfo);	1✔
174	}
175
176	private DataFileFooter convertFooter(FileHeader fileHeader, FileFooter3 fileFooter) {
177	DataFileFooterConverterV3 converter = new DataFileFooterConverterV3();	1✔
178	return converter.convertDataFileFooter(fileHeader, fileFooter);	1✔
179	}
180
181	private DimensionColumnChunkReader dimensionColumnChunkReader;
182	private MeasureColumnChunkReader measureColumnChunkReader;
183
184	private AbstractRawColumnChunk readBlockletColumnChunkIO(
185	DataFileFooter footer, int blockletId, int columnIndex, boolean dimension)
186	throws IOException {
187	BlockletInfo blockletInfo = footer.getBlockletList().get(blockletId);	1✔
188	if (dimension) {	1✔
189	dimensionColumnChunkReader = CarbonDataReaderFactory.getInstance()	1✔
190	.getDimensionColumnChunkReader(ColumnarFormatVersion.V3, blockletInfo,	1✔
191	footer.getSegmentInfo().getColumnCardinality(), file.getFilePath(), false);	1✔
192	return dimensionColumnChunkReader.readRawDimensionChunk(file.getFileReader(), columnIndex);	1✔
193	} else {
194	columnIndex = columnIndex - file.numDimensions();	×
195	assert (columnIndex >= 0);	×
196	measureColumnChunkReader = CarbonDataReaderFactory.getInstance()	×
197	.getMeasureColumnChunkReader(ColumnarFormatVersion.V3, blockletInfo,	×
198	file.getFilePath(), false);	×
199	return measureColumnChunkReader.readRawMeasureChunk(file.getFileReader(), columnIndex);	×
200	}
201	}
202
203	private DimensionColumnPage[] decompressDimensionPages(
204	AbstractRawColumnChunk rawColumnChunk, int numPages) throws IOException, MemoryException {
205	DimensionColumnPage[] pages = new DimensionColumnPage[numPages];	1✔
206	for (int i = 0; i < pages.length; i++) {	1✔
207	pages[i] = dimensionColumnChunkReader.decodeColumnPage(	1✔
208	(DimensionRawColumnChunk) rawColumnChunk, i);
209	}
210	return pages;	1✔
211	}
212
213	private ColumnPage[] decompressMeasurePages(
214	AbstractRawColumnChunk rawColumnChunk, int numPages) throws IOException, MemoryException {
215	ColumnPage[] pages = new ColumnPage[numPages];	×
216	for (int i = 0; i < pages.length; i++) {	×
217	pages[i] = measureColumnChunkReader.decodeColumnPage(	×
218	(MeasureRawColumnChunk) rawColumnChunk, i);
219	}
220	return pages;	×
221	}
222
223	}

apache / carbondata / 9391

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous