2118

Build # 2118

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/1818d6dbe5ca1d168a2b5cfa3e46bfe658e85c0e">1818d6dbe">&lt;a href=&quot;https://github.com/apache/carbondata/commit/</a><a class="double-link" href="https://github.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/1818d6dbe5ca1d168a2b5cfa3e46bfe658e85c0e">1818d6dbe</a><a href="https://github.com/apache/carbondata/commit/1818d6dbe5ca1d168a2b5cfa3e46bfe658e85c0e">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/1818d6dbe5ca1d168a2b5cfa3e46bfe658e85c0e&quot;&gt;1818d6dbe&lt;/a&gt;&lt;a href=&quot;https://github.com/apache/carbondata/commit/1818d6dbe5ca1d168a2b5cfa3e46bfe658e85c0e&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/carbondata/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/carbondata/commit/&amp;amp;lt;a ... (continued)

Run Details

6 of 6 new or added lines in 1 file covered. (100.0%)

61513 of 77889 relevant lines covered (78.98%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.89

/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.sdk.file;

import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
import org.apache.carbondata.core.reader.CarbonHeaderReader;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.FileFooter3;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;

import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
import static org.apache.carbondata.core.util.path.CarbonTablePath.CARBON_DATA_EXT;
import static org.apache.carbondata.core.util.path.CarbonTablePath.INDEX_FILE_EXT;

import org.apache.hadoop.conf.Configuration;

/**
 * Schema reader for carbon files, including carbondata file, carbonindex file, and schema file
 */
public class CarbonSchemaReader {

  /**
   * Read schema file and return the schema
   *
   * @param schemaFilePath complete path including schema file name
   * @return schema object
   * @throws IOException
   */
  @Deprecated
  public static Schema readSchemaInSchemaFile(String schemaFilePath) throws IOException {
    org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    List<ColumnSchema> schemaList = schemaConverter
        .fromExternalToWrapperTableInfo(tableInfo, "", "", "")
        .getFactTable()
        .getListOfColumns();
    return new Schema(schemaList);
  }

  /**
   * get carbondata/carbonindex file in path
   *
   * @param path      carbon file path
   * @param extension carbon file extension
   * @param conf      hadoop configuration support, can set s3a AK,SK,
   *                  end point and other conf with this
   * @return CarbonFile array
   */
  private static CarbonFile[] getCarbonFile(String path, final String extension, Configuration conf)
      throws IOException {
    String dataFilePath = path;
    if (!(dataFilePath.endsWith(extension))) {
      CarbonFile[] carbonFiles = FileFactory
          .getCarbonFile(path, conf)
          .listFiles(new CarbonFileFilter() {
            @Override
            public boolean accept(CarbonFile file) {
              if (file == null) {
                return false;
              }
              return file.getName().endsWith(extension);
            }
          });
      if (carbonFiles == null || carbonFiles.length < 1) {
        throw new IOException("Carbon file not exists.");
      }
      return carbonFiles;
    } else {
      throw new CarbonDataLoadingException("Please ensure path "
          + path + " end with " + extension);
    }
  }

  /**
   * read schema from path,
   * path can be folder path, carbonindex file path, and carbondata file path
   * and will not check all files schema
   *
   * @param path file/folder path
   * @return schema
   * @throws IOException
   */
  public static Schema readSchema(String path) throws IOException {
    Configuration conf = new Configuration();
    return readSchema(path, false, conf);
  }

  /**
   * read schema from path,
   * path can be folder path, carbonindex file path, and carbondata file path
   * and will not check all files schema
   *
   * @param path file/folder path
   * @param conf hadoop configuration support, can set s3a AK,SK,end point and other conf with this
   * @return schema
   * @throws IOException
   */
  public static Schema readSchema(String path, Configuration conf) throws IOException {
    return readSchema(path, false, conf);
  }

  /**
   * read schema from path,
   * path can be folder path, carbonindex file path, and carbondata file path
   * and user can decide whether check all files schema
   *
   * @param path           file/folder path
   * @param validateSchema whether check all files schema
   * @param conf           hadoop configuration support, can set s3a AK,SK,
   *                       end point and other conf with this
   * @return schema
   * @throws IOException
   */
  public static Schema readSchema(String path, boolean validateSchema, Configuration conf)
      throws IOException {
    if (path.endsWith(INDEX_FILE_EXT)) {
      return readSchemaFromIndexFile(path, conf);
    } else if (path.endsWith(CARBON_DATA_EXT)) {
      return readSchemaFromDataFile(path, conf);
    } else if (validateSchema) {
      CarbonFile[] carbonIndexFiles = getCarbonFile(path, INDEX_FILE_EXT, conf);
      Schema schema;
      if (carbonIndexFiles != null && carbonIndexFiles.length != 0) {
        schema = readSchemaFromIndexFile(carbonIndexFiles[0].getAbsolutePath(), conf);
        for (int i = 1; i < carbonIndexFiles.length; i++) {
          Schema schema2 = readSchemaFromIndexFile(carbonIndexFiles[i].getAbsolutePath(), conf);
          if (!schema.equals(schema2)) {
            throw new CarbonDataLoadingException("Schema is different between different files.");
          }
        }
        CarbonFile[] carbonDataFiles = getCarbonFile(path, CARBON_DATA_EXT, conf);
        for (int i = 0; i < carbonDataFiles.length; i++) {
          Schema schema2 = readSchemaFromDataFile(carbonDataFiles[i].getAbsolutePath(), conf);
          if (!schema.equals(schema2)) {
            throw new CarbonDataLoadingException("Schema is different between different files.");
          }
        }
        return schema;
      } else {
        throw new CarbonDataLoadingException("No carbonindex file in this path.");
      }
    } else {
      String indexFilePath = getCarbonFile(path, INDEX_FILE_EXT, conf)[0].getAbsolutePath();
      return readSchemaFromIndexFile(indexFilePath, conf);
    }
  }

  /**
   * read schema from path,
   * path can be folder path, carbonindex file path, and carbondata file path
   * and user can decide whether check all files schema
   *
   * @param path           file/folder path
   * @param validateSchema whether check all files schema
   * @return schema
   * @throws IOException
   */
  public static Schema readSchema(String path, boolean validateSchema) throws IOException {
    Configuration conf = new Configuration();
    return readSchema(path, validateSchema, conf);
  }

  /**
   * Read carbondata file and return the schema
   * This interface will be removed,
   * please use readSchema instead of this interface
   *
   * @param dataFilePath carbondata file store path
   * @return Schema object
   * @throws IOException
   */
  @Deprecated
  public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
    return readSchema(dataFilePath, false);
  }

  /**
   * Read schema from carbondata file
   *
   * @param dataFilePath carbondata file path
   * @param conf         hadoop configuration support, can set s3a AK,SK,
   *                     end point and other conf with this
   * @return carbon data schema
   * @throws IOException
   */
  private static Schema readSchemaFromDataFile(String dataFilePath, Configuration conf)
      throws IOException {
    CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath, conf);
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<ColumnSchema> schemaList = reader.readSchema();
    for (int i = 0; i < schemaList.size(); i++) {
      ColumnSchema columnSchema = schemaList.get(i);
      if (!(columnSchema.getColumnName().contains("."))) {
        columnSchemaList.add(columnSchema);
      }
    }
    return new Schema(columnSchemaList);
  }

  /**
   * Read carbonindex file and return the schema
   * This interface will be removed,
   * please use readSchema instead of this interface
   *
   * @param indexFilePath complete path including index file name
   * @return schema object
   * @throws IOException
   */
  @Deprecated
  public static Schema readSchemaInIndexFile(String indexFilePath) throws IOException {
    return readSchema(indexFilePath, false);
  }

  /**
   * Read schema from carbonindex file
   *
   * @param indexFilePath carbonindex file path
   * @param conf          hadoop configuration support, can set s3a AK,SK,
   *                      end point and other conf with this
   * @return carbon data Schema
   * @throws IOException
   */
  private static Schema readSchemaFromIndexFile(String indexFilePath, Configuration conf)
      throws IOException {
    CarbonFile indexFile =
        FileFactory.getCarbonFile(indexFilePath, conf);
    if (!indexFile.getName().endsWith(INDEX_FILE_EXT)) {
      throw new IOException("Not an index file name");
    }
    // read schema from the first index file
    DataInputStream dataInputStream =
        FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath), conf);
    byte[] bytes = new byte[(int) indexFile.getSize()];
    try {
      //get the file in byte buffer
      dataInputStream.readFully(bytes);
      CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
      // read from byte buffer.
      indexReader.openThriftReader(bytes);
      // get the index header
      org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
      List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
      List<org.apache.carbondata.format.ColumnSchema> table_columns =
          readIndexHeader.getTable_columns();
      for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {
        if (!(columnSchema.column_name.contains("."))) {
          columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));
        }
      }
      return new Schema(columnSchemaList);
    } finally {
      dataInputStream.close();
    }
  }

  /**
   * This method return the version details in formatted string by reading from carbondata file
   *
   * @param dataFilePath
   * @return
   * @throws IOException
   */
  public static String getVersionDetails(String dataFilePath) throws IOException {
    long fileSize =
        FileFactory.getCarbonFile(dataFilePath, FileFactory.getFileType(dataFilePath)).getSize();
    FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath));
    ByteBuffer buffer =
        fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8);
    fileReader.finish();
    CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong());
    FileFooter3 footer = footerReader.readFooterVersion3();
    if (null != footer.getExtra_info()) {
      return footer.getExtra_info().get(CarbonCommonConstants.CARBON_WRITTEN_BY_FOOTER_INFO)
          + " in version: " + footer.getExtra_info()
          .get(CarbonCommonConstants.CARBON_WRITTEN_VERSION);
    } else {
      return "Version Details are not found in carbondata file";
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.sdk.file;
19
20	import java.io.DataInputStream;
21	import java.io.IOException;
22	import java.nio.ByteBuffer;
23	import java.util.ArrayList;
24	import java.util.List;
25
26	import org.apache.carbondata.core.constants.CarbonCommonConstants;
27	import org.apache.carbondata.core.datastore.FileReader;
28	import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
29	import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
30	import org.apache.carbondata.core.datastore.impl.FileFactory;
31	import org.apache.carbondata.core.metadata.converter.SchemaConverter;
32	import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
33	import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
34	import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
35	import org.apache.carbondata.core.reader.CarbonHeaderReader;
36	import org.apache.carbondata.core.reader.CarbonIndexFileReader;
37	import org.apache.carbondata.core.util.CarbonUtil;
38	import org.apache.carbondata.format.FileFooter3;
39	import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
40
41	import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
42	import static org.apache.carbondata.core.util.path.CarbonTablePath.CARBON_DATA_EXT;
43	import static org.apache.carbondata.core.util.path.CarbonTablePath.INDEX_FILE_EXT;
44
45	import org.apache.hadoop.conf.Configuration;
46
47	/**
48	* Schema reader for carbon files, including carbondata file, carbonindex file, and schema file
49	*/
50	public class CarbonSchemaReader {	×
51
52	/**
53	* Read schema file and return the schema
54	*
55	* @param schemaFilePath complete path including schema file name
56	* @return schema object
57	* @throws IOException
58	*/
59	@Deprecated
60	public static Schema readSchemaInSchemaFile(String schemaFilePath) throws IOException {
61	org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);	×
62	SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();	×
63	List<ColumnSchema> schemaList = schemaConverter	×
64	.fromExternalToWrapperTableInfo(tableInfo, "", "", "")	×
65	.getFactTable()	×
66	.getListOfColumns();	×
67	return new Schema(schemaList);	×
68	}
69
70	/**
71	* get carbondata/carbonindex file in path
72	*
73	* @param path carbon file path
74	* @param extension carbon file extension
75	* @param conf hadoop configuration support, can set s3a AK,SK,
76	* end point and other conf with this
77	* @return CarbonFile array
78	*/
79	private static CarbonFile[] getCarbonFile(String path, final String extension, Configuration conf)
80	throws IOException {
81	String dataFilePath = path;	2✔
82	if (!(dataFilePath.endsWith(extension))) {	2✔
83	CarbonFile[] carbonFiles = FileFactory	2✔
84	.getCarbonFile(path, conf)	2✔
85	.listFiles(new CarbonFileFilter() {	2✔
86	@Override
87	public boolean accept(CarbonFile file) {
88	if (file == null) {	2✔
89	return false;	×
90	}
91	return file.getName().endsWith(extension);	2✔
92	}
93	});
94	if (carbonFiles == null \|\| carbonFiles.length < 1) {	2✔
95	throw new IOException("Carbon file not exists.");	×
96	}
97	return carbonFiles;	2✔
98	} else {
99	throw new CarbonDataLoadingException("Please ensure path "	×
100	+ path + " end with " + extension);
101	}
102	}
103
104	/**
105	* read schema from path,
106	* path can be folder path, carbonindex file path, and carbondata file path
107	* and will not check all files schema
108	*
109	* @param path file/folder path
110	* @return schema
111	* @throws IOException
112	*/
113	public static Schema readSchema(String path) throws IOException {
114	Configuration conf = new Configuration();	2✔
115	return readSchema(path, false, conf);	2✔
116	}
117
118	/**
119	* read schema from path,
120	* path can be folder path, carbonindex file path, and carbondata file path
121	* and will not check all files schema
122	*
123	* @param path file/folder path
124	* @param conf hadoop configuration support, can set s3a AK,SK,end point and other conf with this
125	* @return schema
126	* @throws IOException
127	*/
128	public static Schema readSchema(String path, Configuration conf) throws IOException {
129	return readSchema(path, false, conf);	×
130	}
131
132	/**
133	* read schema from path,
134	* path can be folder path, carbonindex file path, and carbondata file path
135	* and user can decide whether check all files schema
136	*
137	* @param path file/folder path
138	* @param validateSchema whether check all files schema
139	* @param conf hadoop configuration support, can set s3a AK,SK,
140	* end point and other conf with this
141	* @return schema
142	* @throws IOException
143	*/
144	public static Schema readSchema(String path, boolean validateSchema, Configuration conf)
145	throws IOException {
146	if (path.endsWith(INDEX_FILE_EXT)) {	2✔
147	return readSchemaFromIndexFile(path, conf);	2✔
148	} else if (path.endsWith(CARBON_DATA_EXT)) {	2✔
149	return readSchemaFromDataFile(path, conf);	2✔
150	} else if (validateSchema) {	2✔
151	CarbonFile[] carbonIndexFiles = getCarbonFile(path, INDEX_FILE_EXT, conf);	2✔
152	Schema schema;
153	if (carbonIndexFiles != null && carbonIndexFiles.length != 0) {	2✔
154	schema = readSchemaFromIndexFile(carbonIndexFiles[0].getAbsolutePath(), conf);	2✔
155	for (int i = 1; i < carbonIndexFiles.length; i++) {	2✔
156	Schema schema2 = readSchemaFromIndexFile(carbonIndexFiles[i].getAbsolutePath(), conf);	2✔
157	if (!schema.equals(schema2)) {	2✔
158	throw new CarbonDataLoadingException("Schema is different between different files.");	2✔
159	}
160	}
161	CarbonFile[] carbonDataFiles = getCarbonFile(path, CARBON_DATA_EXT, conf);	2✔
162	for (int i = 0; i < carbonDataFiles.length; i++) {	2✔
163	Schema schema2 = readSchemaFromDataFile(carbonDataFiles[i].getAbsolutePath(), conf);	2✔
164	if (!schema.equals(schema2)) {	2✔
165	throw new CarbonDataLoadingException("Schema is different between different files.");	×
166	}
167	}
168	return schema;	2✔
169	} else {
170	throw new CarbonDataLoadingException("No carbonindex file in this path.");	×
171	}
172	} else {
173	String indexFilePath = getCarbonFile(path, INDEX_FILE_EXT, conf)[0].getAbsolutePath();	2✔
174	return readSchemaFromIndexFile(indexFilePath, conf);	2✔
175	}
176	}
177
178	/**
179	* read schema from path,
180	* path can be folder path, carbonindex file path, and carbondata file path
181	* and user can decide whether check all files schema
182	*
183	* @param path file/folder path
184	* @param validateSchema whether check all files schema
185	* @return schema
186	* @throws IOException
187	*/
188	public static Schema readSchema(String path, boolean validateSchema) throws IOException {
189	Configuration conf = new Configuration();	2✔
190	return readSchema(path, validateSchema, conf);	2✔
191	}
192
193	/**
194	* Read carbondata file and return the schema
195	* This interface will be removed,
196	* please use readSchema instead of this interface
197	*
198	* @param dataFilePath carbondata file store path
199	* @return Schema object
200	* @throws IOException
201	*/
202	@Deprecated
203	public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
204	return readSchema(dataFilePath, false);	×
205	}
206
207	/**
208	* Read schema from carbondata file
209	*
210	* @param dataFilePath carbondata file path
211	* @param conf hadoop configuration support, can set s3a AK,SK,
212	* end point and other conf with this
213	* @return carbon data schema
214	* @throws IOException
215	*/
216	private static Schema readSchemaFromDataFile(String dataFilePath, Configuration conf)
217	throws IOException {
218	CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath, conf);	2✔
219	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	2✔
220	List<ColumnSchema> schemaList = reader.readSchema();	2✔
221	for (int i = 0; i < schemaList.size(); i++) {	2✔
222	ColumnSchema columnSchema = schemaList.get(i);	2✔
223	if (!(columnSchema.getColumnName().contains("."))) {	2✔
224	columnSchemaList.add(columnSchema);	2✔
225	}
226	}
227	return new Schema(columnSchemaList);	2✔
228	}
229
230	/**
231	* Read carbonindex file and return the schema
232	* This interface will be removed,
233	* please use readSchema instead of this interface
234	*
235	* @param indexFilePath complete path including index file name
236	* @return schema object
237	* @throws IOException
238	*/
239	@Deprecated
240	public static Schema readSchemaInIndexFile(String indexFilePath) throws IOException {
241	return readSchema(indexFilePath, false);	×
242	}
243
244	/**
245	* Read schema from carbonindex file
246	*
247	* @param indexFilePath carbonindex file path
248	* @param conf hadoop configuration support, can set s3a AK,SK,
249	* end point and other conf with this
250	* @return carbon data Schema
251	* @throws IOException
252	*/
253	private static Schema readSchemaFromIndexFile(String indexFilePath, Configuration conf)
254	throws IOException {
255	CarbonFile indexFile =	2✔
256	FileFactory.getCarbonFile(indexFilePath, conf);	2✔
257	if (!indexFile.getName().endsWith(INDEX_FILE_EXT)) {	2✔
258	throw new IOException("Not an index file name");	×
259	}
260	// read schema from the first index file
261	DataInputStream dataInputStream =	2✔
262	FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath), conf);	2✔
263	byte[] bytes = new byte[(int) indexFile.getSize()];	2✔
264	try {
265	//get the file in byte buffer
266	dataInputStream.readFully(bytes);	2✔
267	CarbonIndexFileReader indexReader = new CarbonIndexFileReader();	2✔
268	// read from byte buffer.
269	indexReader.openThriftReader(bytes);	2✔
270	// get the index header
271	org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();	2✔
272	List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();	2✔
273	List<org.apache.carbondata.format.ColumnSchema> table_columns =	2✔
274	readIndexHeader.getTable_columns();	2✔
275	for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {	2✔
276	if (!(columnSchema.column_name.contains("."))) {	2✔
277	columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));	2✔
278	}
279	}	2✔
280	return new Schema(columnSchemaList);	2✔
281	} finally {
282	dataInputStream.close();	2✔
283	}	×
284	}
285
286	/**
287	* This method return the version details in formatted string by reading from carbondata file
288	*
289	* @param dataFilePath
290	* @return
291	* @throws IOException
292	*/
293	public static String getVersionDetails(String dataFilePath) throws IOException {
294	long fileSize =	2✔
295	FileFactory.getCarbonFile(dataFilePath, FileFactory.getFileType(dataFilePath)).getSize();	2✔
296	FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath));	2✔
297	ByteBuffer buffer =	2✔
298	fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8);	2✔
299	fileReader.finish();	2✔
300	CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong());	2✔
301	FileFooter3 footer = footerReader.readFooterVersion3();	2✔
302	if (null != footer.getExtra_info()) {	2✔
303	return footer.getExtra_info().get(CarbonCommonConstants.CARBON_WRITTEN_BY_FOOTER_INFO)	2✔
304	+ " in version: " + footer.getExtra_info()	2✔
305	.get(CarbonCommonConstants.CARBON_WRITTEN_VERSION);	2✔
306	} else {
307	return "Version Details are not found in carbondata file";	×
308	}
309	}
310	}

apache / carbondata / 2118

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous