9476

Build # 9476

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/4c82f14ffbdb2cee25d6cca066a028f1425f53d9">4c82f14ff<a href="https://github.com/apache/carbondata/commit/4c82f14ffbdb2cee25d6cca066a028f1425f53d9">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/b5cb8f657198ad75d62c12986c2f6aaec14551b4">b5cb8f657</a>">b5cb8f657</a><a href="https://github.com/apache/carbondata/commit/4c82f14ffbdb2cee25d6cca066a028f1425f53d9"> into 26d1cdd9c">26d1cdd9c</a>

Run Details

3 of 3 new or added lines in 3 files covered. (100.0%)

61663 of 77104 relevant lines covered (79.97%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.59

/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.sdk.file;

import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
import org.apache.carbondata.core.reader.CarbonHeaderReader;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.FileFooter3;

import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;

/**
 * Schema reader for carbon files, including carbondata file, carbonindex file, and schema file
 */
public class CarbonSchemaReader {

  /**
   * Read schema file and return the schema
   *
   * @param schemaFilePath complete path including schema file name
   * @return schema object
   * @throws IOException
   */
  public static Schema readSchemaInSchemaFile(String schemaFilePath) throws IOException {
    org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    List<ColumnSchema> schemaList = schemaConverter
        .fromExternalToWrapperTableInfo(tableInfo, "", "", "")
        .getFactTable()
        .getListOfColumns();
    return new Schema(schemaList);
  }

  /**
   * Read carbondata file and return the schema
   *
   * @param dataFilePath complete path including carbondata file name
   * @return Schema object
   * @throws IOException
   */
  public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
    CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath);
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<ColumnSchema> schemaList = reader.readSchema();
    for (int i = 0; i < schemaList.size(); i++) {
      ColumnSchema columnSchema = schemaList.get(i);
      if (!(columnSchema.getColumnName().contains("."))) {
        columnSchemaList.add(columnSchema);
      }
    }
    return new Schema(columnSchemaList);
  }

  /**
   * This method return the version details in formatted string by reading from carbondata file
   * @param dataFilePath
   * @return
   * @throws IOException
   */
  public static String getVersionDetails(String dataFilePath) throws IOException {
    long fileSize =
        FileFactory.getCarbonFile(dataFilePath, FileFactory.getFileType(dataFilePath)).getSize();
    FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath));
    ByteBuffer buffer =
        fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8);
    fileReader.finish();
    CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong());
    FileFooter3 footer = footerReader.readFooterVersion3();
    if (null != footer.getExtra_info()) {
      return footer.getExtra_info().get(CarbonCommonConstants.CARBON_WRITTEN_BY_FOOTER_INFO)
          + " in version: " + footer.getExtra_info()
          .get(CarbonCommonConstants.CARBON_WRITTEN_VERSION);
    } else {
      return "Version Details are not found in carbondata file";
    }
  }

  /**
   * Read carbonindex file and return the schema
   *
   * @param indexFilePath complete path including index file name
   * @return schema object
   * @throws IOException
   */
  public static Schema readSchemaInIndexFile(String indexFilePath) throws IOException {
    CarbonFile indexFile =
        FileFactory.getCarbonFile(indexFilePath, FileFactory.getFileType(indexFilePath));
    if (!indexFile.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
      throw new IOException("Not an index file name");
    }
    // read schema from the first index file
    DataInputStream dataInputStream =
        FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath));
    byte[] bytes = new byte[(int) indexFile.getSize()];
    try {
      //get the file in byte buffer
      dataInputStream.readFully(bytes);
      CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
      // read from byte buffer.
      indexReader.openThriftReader(bytes);
      // get the index header
      org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
      List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
      List<org.apache.carbondata.format.ColumnSchema> table_columns =
          readIndexHeader.getTable_columns();
      for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {
        if (!(columnSchema.column_name.contains("."))) {
          columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));
        }
      }
      return new Schema(columnSchemaList);
    } finally {
      dataInputStream.close();
    }
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.sdk.file;
19
20	import java.io.DataInputStream;
21	import java.io.IOException;
22	import java.nio.ByteBuffer;
23	import java.util.ArrayList;
24	import java.util.List;
25
26	import org.apache.carbondata.core.constants.CarbonCommonConstants;
27	import org.apache.carbondata.core.datastore.FileReader;
28	import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
29	import org.apache.carbondata.core.datastore.impl.FileFactory;
30	import org.apache.carbondata.core.metadata.converter.SchemaConverter;
31	import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
32	import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
33	import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
34	import org.apache.carbondata.core.reader.CarbonHeaderReader;
35	import org.apache.carbondata.core.reader.CarbonIndexFileReader;
36	import org.apache.carbondata.core.util.CarbonUtil;
37	import org.apache.carbondata.core.util.path.CarbonTablePath;
38	import org.apache.carbondata.format.FileFooter3;
39
40	import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
41
42	/**
43	* Schema reader for carbon files, including carbondata file, carbonindex file, and schema file
44	*/
45	public class CarbonSchemaReader {	×
46
47	/**
48	* Read schema file and return the schema
49	*
50	* @param schemaFilePath complete path including schema file name
51	* @return schema object
52	* @throws IOException
53	*/
54	public static Schema readSchemaInSchemaFile(String schemaFilePath) throws IOException {
55	org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);	×
56	SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();	×
57	List<ColumnSchema> schemaList = schemaConverter	×
58	.fromExternalToWrapperTableInfo(tableInfo, "", "", "")	×
59	.getFactTable()	×
60	.getListOfColumns();	×
61	return new Schema(schemaList);	×
62	}
63
64	/**
65	* Read carbondata file and return the schema
66	*
67	* @param dataFilePath complete path including carbondata file name
68	* @return Schema object
69	* @throws IOException
70	*/
71	public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
72	CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath);	2✔
73	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	2✔
74	List<ColumnSchema> schemaList = reader.readSchema();	2✔
75	for (int i = 0; i < schemaList.size(); i++) {	2✔
76	ColumnSchema columnSchema = schemaList.get(i);	2✔
77	if (!(columnSchema.getColumnName().contains("."))) {	2✔
78	columnSchemaList.add(columnSchema);	2✔
79	}
80	}
81	return new Schema(columnSchemaList);	2✔
82	}
83
84	/**
85	* This method return the version details in formatted string by reading from carbondata file
86	* @param dataFilePath
87	* @return
88	* @throws IOException
89	*/
90	public static String getVersionDetails(String dataFilePath) throws IOException {
91	long fileSize =	2✔
92	FileFactory.getCarbonFile(dataFilePath, FileFactory.getFileType(dataFilePath)).getSize();	2✔
93	FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath));	2✔
94	ByteBuffer buffer =	2✔
95	fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8);	2✔
96	fileReader.finish();	2✔
97	CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong());	2✔
98	FileFooter3 footer = footerReader.readFooterVersion3();	2✔
99	if (null != footer.getExtra_info()) {	2✔
100	return footer.getExtra_info().get(CarbonCommonConstants.CARBON_WRITTEN_BY_FOOTER_INFO)	2✔
101	+ " in version: " + footer.getExtra_info()	2✔
102	.get(CarbonCommonConstants.CARBON_WRITTEN_VERSION);	2✔
103	} else {
104	return "Version Details are not found in carbondata file";	×
105	}
106	}
107
108	/**
109	* Read carbonindex file and return the schema
110	*
111	* @param indexFilePath complete path including index file name
112	* @return schema object
113	* @throws IOException
114	*/
115	public static Schema readSchemaInIndexFile(String indexFilePath) throws IOException {
116	CarbonFile indexFile =	2✔
117	FileFactory.getCarbonFile(indexFilePath, FileFactory.getFileType(indexFilePath));	2✔
118	if (!indexFile.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {	2✔
119	throw new IOException("Not an index file name");	×
120	}
121	// read schema from the first index file
122	DataInputStream dataInputStream =	2✔
123	FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath));	2✔
124	byte[] bytes = new byte[(int) indexFile.getSize()];	2✔
125	try {
126	//get the file in byte buffer
127	dataInputStream.readFully(bytes);	2✔
128	CarbonIndexFileReader indexReader = new CarbonIndexFileReader();	2✔
129	// read from byte buffer.
130	indexReader.openThriftReader(bytes);	2✔
131	// get the index header
132	org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();	2✔
133	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	2✔
134	List<org.apache.carbondata.format.ColumnSchema> table_columns =	2✔
135	readIndexHeader.getTable_columns();	2✔
136	for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {	2✔
137	if (!(columnSchema.column_name.contains("."))) {	2✔
138	columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));	2✔
139	}
140	}	2✔
141	return new Schema(columnSchemaList);	2✔
142	} finally {
143	dataInputStream.close();	2✔
144	}
145	}
146
147	}

apache / carbondata / 9476

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous