9507

Build # 9507

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/f31d929d4eca454031806e6664168c8aa59237b3">f31d929d4<a href="https://github.com/apache/carbondata/commit/f31d929d4eca454031806e6664168c8aa59237b3">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/c9dd7b69997747d690cc85918660c7c508e7f847">c9dd7b699</a>">c9dd7b699</a><a href="https://github.com/apache/carbondata/commit/f31d929d4eca454031806e6664168c8aa59237b3"> into f51f5cde1">f51f5cde1</a>

Run Details

74 of 74 new or added lines in 3 files covered. (100.0%)

61853 of 77181 relevant lines covered (80.14%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.11

/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.sdk.file;

import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
import org.apache.carbondata.core.reader.CarbonHeaderReader;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.FileFooter3;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;

import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
import static org.apache.carbondata.core.util.path.CarbonTablePath.CARBON_DATA_EXT;
import static org.apache.carbondata.core.util.path.CarbonTablePath.INDEX_FILE_EXT;

/**
 * Schema reader for carbon files, including carbondata file, carbonindex file, and schema file
 */
public class CarbonSchemaReader {

  /**
   * Read schema file and return the schema
   *
   * @param schemaFilePath complete path including schema file name
   * @return schema object
   * @throws IOException
   */
  @Deprecated
  public static Schema readSchemaInSchemaFile(String schemaFilePath) throws IOException {
    org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    List<ColumnSchema> schemaList = schemaConverter
        .fromExternalToWrapperTableInfo(tableInfo, "", "", "")
        .getFactTable()
        .getListOfColumns();
    return new Schema(schemaList);
  }

  /**
   * get carbondata/carbonindex file in path
   *
   * @param path carbon file path
   * @return CarbonFile array
   */
  private static CarbonFile[] getCarbonFile(String path, final String extension) {
    String dataFilePath = path;
    if (!(dataFilePath.contains(extension))) {
      CarbonFile[] carbonFiles = FileFactory
          .getCarbonFile(path)
          .listFiles(new CarbonFileFilter() {
            @Override
            public boolean accept(CarbonFile file) {
              if (file == null) {
                return false;
              }
              return file.getName().endsWith(extension);
            }
          });
      if (carbonFiles == null || carbonFiles.length < 1) {
        throw new RuntimeException("Carbon file not exists.");
      }
      return carbonFiles;
    }
    return null;
  }

  /**
   * read schema from path,
   * path can be folder path, carbonindex file path, and carbondata file path
   * and will not check all files schema
   *
   * @param path file/folder path
   * @return schema
   * @throws IOException
   */
  public static Schema readSchema(String path) throws IOException {
    return readSchema(path, false);
  }

  /**
   * read schema from path,
   * path can be folder path, carbonindex file path, and carbondata file path
   * and user can decide whether check all files schema
   *
   * @param path             file/folder path
   * @param checkFilesSchema whether check all files schema
   * @return schema
   * @throws IOException
   */
  public static Schema readSchema(String path, boolean checkFilesSchema) throws IOException {
    if (path.endsWith(INDEX_FILE_EXT)) {
      return readSchemaFromIndexFile(path);
    } else if (path.endsWith(CARBON_DATA_EXT)) {
      return readSchemaFromDataFile(path);
    } else if (checkFilesSchema) {
      CarbonFile[] carbonIndexFiles = getCarbonFile(path, INDEX_FILE_EXT);
      Schema schema;
      if (carbonIndexFiles != null && carbonIndexFiles.length != 0) {
        schema = readSchemaFromIndexFile(carbonIndexFiles[0].getAbsolutePath());
        for (int i = 1; i < carbonIndexFiles.length; i++) {
          Schema schema2 = readSchemaFromIndexFile(carbonIndexFiles[i].getAbsolutePath());
          if (schema != schema2) {
            throw new CarbonDataLoadingException("Schema is different between different files.");
          }
        }
        CarbonFile[] carbonDataFiles = getCarbonFile(path, CARBON_DATA_EXT);
        for (int i = 0; i < carbonDataFiles.length; i++) {
          Schema schema2 = readSchemaFromDataFile(carbonDataFiles[i].getAbsolutePath());
          if (!schema.equals(schema2)) {
            throw new CarbonDataLoadingException("Schema is different between different files.");
          }
        }
        return schema;
      } else {
        throw new CarbonDataLoadingException("No carbonindex file in this path.");
      }
    } else {
      String indexFilePath = getCarbonFile(path, INDEX_FILE_EXT)[0].getAbsolutePath();
      if (indexFilePath != null) {
        return readSchemaFromIndexFile(indexFilePath);
      } else {
        String dataFilePath = getCarbonFile(path, CARBON_DATA_EXT)[0].getAbsolutePath();
        if (dataFilePath == null) {
          throw new CarbonDataLoadingException("No carbonindex and carbondata file in the path");
        } else {
          return readSchemaFromDataFile(dataFilePath);
        }
      }
    }
  }

  /**
   * Read carbondata file and return the schema
   * This interface will be removed,
   * please use readSchema instead of this interface
   *
   * @param dataFilePath carbondata file store path
   * @return Schema object
   * @throws IOException
   */
  @Deprecated
  public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
    return readSchema(dataFilePath, false);
  }

  /**
   * Read schema from carbondata file
   *
   * @param dataFilePath carbondata file path
   * @return carbon data schema
   * @throws IOException
   */
  public static Schema readSchemaFromDataFile(String dataFilePath) throws IOException {
    CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath);
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<ColumnSchema> schemaList = reader.readSchema();
    for (int i = 0; i < schemaList.size(); i++) {
      ColumnSchema columnSchema = schemaList.get(i);
      if (!(columnSchema.getColumnName().contains("."))) {
        columnSchemaList.add(columnSchema);
      }
    }
    return new Schema(columnSchemaList);
  }

  /**
   * Read carbonindex file and return the schema
   * This interface will be removed,
   * please use readSchema instead of this interface
   *
   * @param indexFilePath complete path including index file name
   * @return schema object
   * @throws IOException
   */
  @Deprecated
  public static Schema readSchemaInIndexFile(String indexFilePath) throws IOException {
    return readSchema(indexFilePath, false);
  }

  /**
   * Read schema from carbonindex file
   *
   * @param indexFilePath carbonindex file path
   * @return carbon data Schema
   * @throws IOException
   */
  private static Schema readSchemaFromIndexFile(String indexFilePath) throws IOException {
    CarbonFile indexFile =
        FileFactory.getCarbonFile(indexFilePath, FileFactory.getFileType(indexFilePath));
    if (!indexFile.getName().endsWith(INDEX_FILE_EXT)) {
      throw new IOException("Not an index file name");
    }
    // read schema from the first index file
    DataInputStream dataInputStream =
        FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath));
    byte[] bytes = new byte[(int) indexFile.getSize()];
    try {
      //get the file in byte buffer
      dataInputStream.readFully(bytes);
      CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
      // read from byte buffer.
      indexReader.openThriftReader(bytes);
      // get the index header
      org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
      List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
      List<org.apache.carbondata.format.ColumnSchema> table_columns =
          readIndexHeader.getTable_columns();
      for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {
        if (!(columnSchema.column_name.contains("."))) {
          columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));
        }
      }
      return new Schema(columnSchemaList);
    } finally {
      dataInputStream.close();
    }
  }

  /**
   * This method return the version details in formatted string by reading from carbondata file
   *
   * @param dataFilePath
   * @return
   * @throws IOException
   */
  public static String getVersionDetails(String dataFilePath) throws IOException {
    long fileSize =
        FileFactory.getCarbonFile(dataFilePath, FileFactory.getFileType(dataFilePath)).getSize();
    FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath));
    ByteBuffer buffer =
        fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8);
    fileReader.finish();
    CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong());
    FileFooter3 footer = footerReader.readFooterVersion3();
    if (null != footer.getExtra_info()) {
      return footer.getExtra_info().get(CarbonCommonConstants.CARBON_WRITTEN_BY_FOOTER_INFO)
          + " in version: " + footer.getExtra_info()
          .get(CarbonCommonConstants.CARBON_WRITTEN_VERSION);
    } else {
      return "Version Details are not found in carbondata file";
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.sdk.file;
19
20	import java.io.DataInputStream;
21	import java.io.IOException;
22	import java.nio.ByteBuffer;
23	import java.util.ArrayList;
24	import java.util.List;
25
26	import org.apache.carbondata.core.constants.CarbonCommonConstants;
27	import org.apache.carbondata.core.datastore.FileReader;
28	import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
29	import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
30	import org.apache.carbondata.core.datastore.impl.FileFactory;
31	import org.apache.carbondata.core.metadata.converter.SchemaConverter;
32	import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
33	import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
34	import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
35	import org.apache.carbondata.core.reader.CarbonHeaderReader;
36	import org.apache.carbondata.core.reader.CarbonIndexFileReader;
37	import org.apache.carbondata.core.util.CarbonUtil;
38	import org.apache.carbondata.format.FileFooter3;
39	import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
40
41	import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
42	import static org.apache.carbondata.core.util.path.CarbonTablePath.CARBON_DATA_EXT;
43	import static org.apache.carbondata.core.util.path.CarbonTablePath.INDEX_FILE_EXT;
44
45	/**
46	* Schema reader for carbon files, including carbondata file, carbonindex file, and schema file
47	*/
48	public class CarbonSchemaReader {	×
49
50	/**
51	* Read schema file and return the schema
52	*
53	* @param schemaFilePath complete path including schema file name
54	* @return schema object
55	* @throws IOException
56	*/
57	@Deprecated
58	public static Schema readSchemaInSchemaFile(String schemaFilePath) throws IOException {
59	org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);	×
60	SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();	×
61	List<ColumnSchema> schemaList = schemaConverter	×
62	.fromExternalToWrapperTableInfo(tableInfo, "", "", "")	×
63	.getFactTable()	×
64	.getListOfColumns();	×
65	return new Schema(schemaList);	×
66	}
67
68	/**
69	* get carbondata/carbonindex file in path
70	*
71	* @param path carbon file path
72	* @return CarbonFile array
73	*/
74	private static CarbonFile[] getCarbonFile(String path, final String extension) {
75	String dataFilePath = path;	2✔
76	if (!(dataFilePath.contains(extension))) {	2✔
77	CarbonFile[] carbonFiles = FileFactory	2✔
78	.getCarbonFile(path)	2✔
79	.listFiles(new CarbonFileFilter() {	2✔
80	@Override
81	public boolean accept(CarbonFile file) {
82	if (file == null) {	2✔
83	return false;	×
84	}
85	return file.getName().endsWith(extension);	2✔
86	}
87	});
88	if (carbonFiles == null \|\| carbonFiles.length < 1) {	2✔
89	throw new RuntimeException("Carbon file not exists.");	×
90	}
91	return carbonFiles;	2✔
92	}
93	return null;	×
94	}
95
96	/**
97	* read schema from path,
98	* path can be folder path, carbonindex file path, and carbondata file path
99	* and will not check all files schema
100	*
101	* @param path file/folder path
102	* @return schema
103	* @throws IOException
104	*/
105	public static Schema readSchema(String path) throws IOException {
106	return readSchema(path, false);	2✔
107	}
108
109	/**
110	* read schema from path,
111	* path can be folder path, carbonindex file path, and carbondata file path
112	* and user can decide whether check all files schema
113	*
114	* @param path file/folder path
115	* @param checkFilesSchema whether check all files schema
116	* @return schema
117	* @throws IOException
118	*/
119	public static Schema readSchema(String path, boolean checkFilesSchema) throws IOException {
120	if (path.endsWith(INDEX_FILE_EXT)) {	2✔
121	return readSchemaFromIndexFile(path);	2✔
122	} else if (path.endsWith(CARBON_DATA_EXT)) {	2✔
123	return readSchemaFromDataFile(path);	2✔
124	} else if (checkFilesSchema) {	2✔
125	CarbonFile[] carbonIndexFiles = getCarbonFile(path, INDEX_FILE_EXT);	2✔
126	Schema schema;
127	if (carbonIndexFiles != null && carbonIndexFiles.length != 0) {	2✔
128	schema = readSchemaFromIndexFile(carbonIndexFiles[0].getAbsolutePath());	2✔
129	for (int i = 1; i < carbonIndexFiles.length; i++) {	2✔
130	Schema schema2 = readSchemaFromIndexFile(carbonIndexFiles[i].getAbsolutePath());	2✔
131	if (schema != schema2) {	2✔
132	throw new CarbonDataLoadingException("Schema is different between different files.");	2✔
133	}
134	}
135	CarbonFile[] carbonDataFiles = getCarbonFile(path, CARBON_DATA_EXT);	×
136	for (int i = 0; i < carbonDataFiles.length; i++) {	×
137	Schema schema2 = readSchemaFromDataFile(carbonDataFiles[i].getAbsolutePath());	×
138	if (!schema.equals(schema2)) {	×
139	throw new CarbonDataLoadingException("Schema is different between different files.");	×
140	}
141	}
142	return schema;	×
143	} else {
144	throw new CarbonDataLoadingException("No carbonindex file in this path.");	×
145	}
146	} else {
147	String indexFilePath = getCarbonFile(path, INDEX_FILE_EXT)[0].getAbsolutePath();	2✔
148	if (indexFilePath != null) {	2✔
149	return readSchemaFromIndexFile(indexFilePath);	2✔
150	} else {
151	String dataFilePath = getCarbonFile(path, CARBON_DATA_EXT)[0].getAbsolutePath();	×
152	if (dataFilePath == null) {	×
153	throw new CarbonDataLoadingException("No carbonindex and carbondata file in the path");	×
154	} else {
155	return readSchemaFromDataFile(dataFilePath);	×
156	}
157	}
158	}
159	}
160
161	/**
162	* Read carbondata file and return the schema
163	* This interface will be removed,
164	* please use readSchema instead of this interface
165	*
166	* @param dataFilePath carbondata file store path
167	* @return Schema object
168	* @throws IOException
169	*/
170	@Deprecated
171	public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
172	return readSchema(dataFilePath, false);	×
173	}
174
175	/**
176	* Read schema from carbondata file
177	*
178	* @param dataFilePath carbondata file path
179	* @return carbon data schema
180	* @throws IOException
181	*/
182	public static Schema readSchemaFromDataFile(String dataFilePath) throws IOException {
183	CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath);	2✔
184	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	2✔
185	List<ColumnSchema> schemaList = reader.readSchema();	2✔
186	for (int i = 0; i < schemaList.size(); i++) {	2✔
187	ColumnSchema columnSchema = schemaList.get(i);	2✔
188	if (!(columnSchema.getColumnName().contains("."))) {	2✔
189	columnSchemaList.add(columnSchema);	2✔
190	}
191	}
192	return new Schema(columnSchemaList);	2✔
193	}
194
195	/**
196	* Read carbonindex file and return the schema
197	* This interface will be removed,
198	* please use readSchema instead of this interface
199	*
200	* @param indexFilePath complete path including index file name
201	* @return schema object
202	* @throws IOException
203	*/
204	@Deprecated
205	public static Schema readSchemaInIndexFile(String indexFilePath) throws IOException {
206	return readSchema(indexFilePath, false);	×
207	}
208
209	/**
210	* Read schema from carbonindex file
211	*
212	* @param indexFilePath carbonindex file path
213	* @return carbon data Schema
214	* @throws IOException
215	*/
216	private static Schema readSchemaFromIndexFile(String indexFilePath) throws IOException {
217	CarbonFile indexFile =	2✔
218	FileFactory.getCarbonFile(indexFilePath, FileFactory.getFileType(indexFilePath));	2✔
219	if (!indexFile.getName().endsWith(INDEX_FILE_EXT)) {	2✔
220	throw new IOException("Not an index file name");	×
221	}
222	// read schema from the first index file
223	DataInputStream dataInputStream =	2✔
224	FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath));	2✔
225	byte[] bytes = new byte[(int) indexFile.getSize()];	2✔
226	try {
227	//get the file in byte buffer
228	dataInputStream.readFully(bytes);	2✔
229	CarbonIndexFileReader indexReader = new CarbonIndexFileReader();	2✔
230	// read from byte buffer.
231	indexReader.openThriftReader(bytes);	2✔
232	// get the index header
233	org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();	2✔
234	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	2✔
235	List<org.apache.carbondata.format.ColumnSchema> table_columns =	2✔
236	readIndexHeader.getTable_columns();	2✔
237	for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {	2✔
238	if (!(columnSchema.column_name.contains("."))) {	2✔
239	columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));	2✔
240	}
241	}	2✔
242	return new Schema(columnSchemaList);	2✔
243	} finally {
244	dataInputStream.close();	2✔
245	}
246	}
247
248	/**
249	* This method return the version details in formatted string by reading from carbondata file
250	*
251	* @param dataFilePath
252	* @return
253	* @throws IOException
254	*/
255	public static String getVersionDetails(String dataFilePath) throws IOException {
256	long fileSize =	2✔
257	FileFactory.getCarbonFile(dataFilePath, FileFactory.getFileType(dataFilePath)).getSize();	2✔
258	FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath));	2✔
259	ByteBuffer buffer =	2✔
260	fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8);	2✔
261	fileReader.finish();	2✔
262	CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong());	2✔
263	FileFooter3 footer = footerReader.readFooterVersion3();	2✔
264	if (null != footer.getExtra_info()) {	2✔
265	return footer.getExtra_info().get(CarbonCommonConstants.CARBON_WRITTEN_BY_FOOTER_INFO)	2✔
266	+ " in version: " + footer.getExtra_info()	2✔
267	.get(CarbonCommonConstants.CARBON_WRITTEN_VERSION);	2✔
268	} else {
269	return "Version Details are not found in carbondata file";	×
270	}
271	}
272	}

apache / carbondata / 9507

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous