apache / carbondata / 572

Build # 572

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/d9984cfc751627da48fb426597054f9e1872050a">d9984cfc7">&lt;a href=&quot;https://github.com/apache/carbondata/commit/</a><a class="double-link" href="https://github.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/d9984cfc751627da48fb426597054f9e1872050a">d9984cfc7</a><a href="https://github.com/apache/carbondata/commit/d9984cfc751627da48fb426597054f9e1872050a">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/d9984cfc751627da48fb426597054f9e1872050a&quot;&gt;d9984cfc7&lt;/a&gt;&lt;a href=&quot;https://github.com/apache/carbondata/commit/d9984cfc751627da48fb426597054f9e1872050a&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/carbondata/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/carbondata/commit/&amp;amp;lt;a ... (continued)

Run Details

274 of 274 new or added lines in 37 files covered. (100.0%)

59750 of 74231 relevant lines covered (80.49%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.21

/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.sdk.file;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;

import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.path.CarbonTablePath;

import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.JsonDecoder;
import org.apache.hadoop.conf.Configuration;

@InterfaceAudience.Developer("Test")
public class TestUtil {

  public static final Configuration configuration = new Configuration();

  public static GenericData.Record jsonToAvro(String json, String avroSchema) throws IOException {
    InputStream input = null;
    DataFileWriter writer = null;
    ByteArrayOutputStream output = null;
    try {
      org.apache.avro.Schema schema = new org.apache.avro.Schema.Parser().parse(avroSchema);
      GenericDatumReader reader = new GenericDatumReader(schema);
      input = new ByteArrayInputStream(json.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
      output = new ByteArrayOutputStream();
      DataInputStream din = new DataInputStream(input);
      writer = new DataFileWriter(new GenericDatumWriter());
      writer.create(schema, output);
      JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, din);
      return (GenericData.Record) reader.read(null, decoder);
    } finally {
      if (input != null) {
        input.close();
      }
      if (writer != null) {
        writer.close();
      }
    }
  }

  static void writeFilesAndVerify(Schema schema, String path) {
    writeFilesAndVerify(schema, path, null);
  }

  static void writeFilesAndVerify(Schema schema, String path, String[] sortColumns) {
    writeFilesAndVerify(
        100, schema, path, sortColumns, false, -1, -1, true);
  }

  public static void writeFilesAndVerify(
      int rows, Schema schema, String path, boolean persistSchema) {
    writeFilesAndVerify(
        rows, schema, path, null, persistSchema, -1, -1, true);
  }

  public static void writeFilesAndVerify(Schema schema, String path, boolean persistSchema,
      boolean isTransactionalTable) {
    writeFilesAndVerify(
        100, schema, path, null, persistSchema, -1, -1, isTransactionalTable);
  }

  /**
   * write file and verify
   *
   * @param rows                 number of rows
   * @param schema               schema
   * @param path                 table store path
   * @param persistSchema        whether persist schema
   * @param isTransactionalTable whether is transactional table
   */
  public static void writeFilesAndVerify(
      int rows, Schema schema, String path, boolean persistSchema, boolean isTransactionalTable) {
    writeFilesAndVerify(rows, schema, path, null, persistSchema, -1, -1, isTransactionalTable);
  }

  /**
   * Invoke CarbonWriter API to write carbon files and assert the file is rewritten
   * @param rows number of rows to write
   * @param schema schema of the file
   * @param path local write path
   * @param sortColumns sort columns
   * @param persistSchema true if want to persist schema file
   * @param blockletSize blockletSize in the file, -1 for default size
   * @param blockSize blockSize in the file, -1 for default size
   * @param isTransactionalTable set to true if this is written for Transactional Table.
   */
  public static void writeFilesAndVerify(int rows, Schema schema, String path, String[] sortColumns,
      boolean persistSchema, int blockletSize, int blockSize, boolean isTransactionalTable) {
    try {
      CarbonWriterBuilder builder = CarbonWriter.builder()
          .isTransactionalTable(isTransactionalTable)
          .outputPath(path);
      if (sortColumns != null) {
        builder = builder.sortBy(sortColumns);
      }
      if (persistSchema) {
        builder = builder.persistSchemaFile(true);
      }
      if (blockletSize != -1) {
        builder = builder.withBlockletSize(blockletSize);
      }
      if (blockSize != -1) {
        builder = builder.withBlockSize(blockSize);
      }

      CarbonWriter writer = builder.buildWriterForCSVInput(schema, configuration);

      for (int i = 0; i < rows; i++) {
        writer.write(new String[]{
            "robot" + (i % 10), String.valueOf(i % 3000000), String.valueOf((double) i / 2)});
      }
      writer.close();
    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e);
    }

    File segmentFolder = null;
    if (isTransactionalTable) {
      segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null"));
      if (!segmentFolder.exists()) {
        throw new RuntimeException("Test failed: file not exists");
      }
    } else {
      segmentFolder = new File(path);
      if (!segmentFolder.exists()) {
        throw new RuntimeException("Test failed: file not exists");
      }
    }

    File[] dataFiles = segmentFolder.listFiles(new FileFilter() {
      @Override public boolean accept(File pathname) {
        return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
      }
    });
    if (dataFiles == null) {
      throw new RuntimeException("Test failed: dataFiles is null");
    }

    if (dataFiles.length == 0) {
      throw new RuntimeException("Test failed: dataFiles is empty");
    }
  }

  /**
   * verify whether the file exists
   * if delete the file success or file not exists, then return true; otherwise return false
   *
   * @return boolean
   */
  public static boolean cleanMdtFile() {
    String fileName = CarbonProperties.getInstance().getSystemFolderLocation()
            + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
    try {
      if (FileFactory.isFileExist(fileName)) {
        File file = new File(fileName);
        return file.delete();
      } else {
        return true;
      }
    } catch (IOException e) {
      e.printStackTrace();
      return false;
    }
  }

  /**
   * verify whether the mdt file exists
   * if the file exists, then return true; otherwise return false
   *
   * @return boolean
   */
  public static boolean verifyMdtFile() {
    String fileName = CarbonProperties.getInstance().getSystemFolderLocation()
            + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
    try {
      if (FileFactory.isFileExist(fileName)) {
        return true;
      }
      return false;
    } catch (IOException e) {
      throw new RuntimeException("IO exception:", e);
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.sdk.file;
19
20	import java.io.ByteArrayInputStream;
21	import java.io.ByteArrayOutputStream;
22	import java.io.DataInputStream;
23	import java.io.File;
24	import java.io.FileFilter;
25	import java.io.IOException;
26	import java.io.InputStream;
27
28	import org.apache.carbondata.common.annotations.InterfaceAudience;
29	import org.apache.carbondata.core.constants.CarbonCommonConstants;
30	import org.apache.carbondata.core.datastore.impl.FileFactory;
31	import org.apache.carbondata.core.util.CarbonProperties;
32	import org.apache.carbondata.core.util.path.CarbonTablePath;
33
34	import org.apache.avro.file.DataFileWriter;
35	import org.apache.avro.generic.GenericData;
36	import org.apache.avro.generic.GenericDatumReader;
37	import org.apache.avro.generic.GenericDatumWriter;
38	import org.apache.avro.io.DecoderFactory;
39	import org.apache.avro.io.JsonDecoder;
40	import org.apache.hadoop.conf.Configuration;
41
42	@InterfaceAudience.Developer("Test")
43	public class TestUtil {	×
44
45	public static final Configuration configuration = new Configuration();	2✔
46
47	public static GenericData.Record jsonToAvro(String json, String avroSchema) throws IOException {
48	InputStream input = null;	2✔
49	DataFileWriter writer = null;	2✔
50	ByteArrayOutputStream output = null;	2✔
51	try {
52	org.apache.avro.Schema schema = new org.apache.avro.Schema.Parser().parse(avroSchema);	2✔
53	GenericDatumReader reader = new GenericDatumReader(schema);	2✔
54	input = new ByteArrayInputStream(json.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));	2✔
55	output = new ByteArrayOutputStream();	2✔
56	DataInputStream din = new DataInputStream(input);	2✔
57	writer = new DataFileWriter(new GenericDatumWriter());	2✔
58	writer.create(schema, output);	2✔
59	JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, din);	2✔
60	return (GenericData.Record) reader.read(null, decoder);	2✔
61	} finally {
62	if (input != null) {	2✔
63	input.close();	2✔
64	}
65	if (writer != null) {	2✔
66	writer.close();	2✔
67	}
68	}	×
69	}
70
71	static void writeFilesAndVerify(Schema schema, String path) {
72	writeFilesAndVerify(schema, path, null);	2✔
73	}	2✔
74
75	static void writeFilesAndVerify(Schema schema, String path, String[] sortColumns) {
76	writeFilesAndVerify(	2✔
77	100, schema, path, sortColumns, false, -1, -1, true);
78	}	2✔
79
80	public static void writeFilesAndVerify(
81	int rows, Schema schema, String path, boolean persistSchema) {
82	writeFilesAndVerify(	2✔
83	rows, schema, path, null, persistSchema, -1, -1, true);
84	}	2✔
85
86	public static void writeFilesAndVerify(Schema schema, String path, boolean persistSchema,
87	boolean isTransactionalTable) {
88	writeFilesAndVerify(	2✔
89	100, schema, path, null, persistSchema, -1, -1, isTransactionalTable);
90	}	2✔
91
92	/**
93	* write file and verify
94	*
95	* @param rows number of rows
96	* @param schema schema
97	* @param path table store path
98	* @param persistSchema whether persist schema
99	* @param isTransactionalTable whether is transactional table
100	*/
101	public static void writeFilesAndVerify(
102	int rows, Schema schema, String path, boolean persistSchema, boolean isTransactionalTable) {
103	writeFilesAndVerify(rows, schema, path, null, persistSchema, -1, -1, isTransactionalTable);	2✔
104	}	2✔
105
106	/**
107	* Invoke CarbonWriter API to write carbon files and assert the file is rewritten
108	* @param rows number of rows to write
109	* @param schema schema of the file
110	* @param path local write path
111	* @param sortColumns sort columns
112	* @param persistSchema true if want to persist schema file
113	* @param blockletSize blockletSize in the file, -1 for default size
114	* @param blockSize blockSize in the file, -1 for default size
115	* @param isTransactionalTable set to true if this is written for Transactional Table.
116	*/
117	public static void writeFilesAndVerify(int rows, Schema schema, String path, String[] sortColumns,
118	boolean persistSchema, int blockletSize, int blockSize, boolean isTransactionalTable) {
119	try {
120	CarbonWriterBuilder builder = CarbonWriter.builder()	2✔
121	.isTransactionalTable(isTransactionalTable)	2✔
122	.outputPath(path);	2✔
123	if (sortColumns != null) {	2✔
124	builder = builder.sortBy(sortColumns);	2✔
125	}
126	if (persistSchema) {	2✔
127	builder = builder.persistSchemaFile(true);	2✔
128	}
129	if (blockletSize != -1) {	2✔
130	builder = builder.withBlockletSize(blockletSize);	2✔
131	}
132	if (blockSize != -1) {	2✔
133	builder = builder.withBlockSize(blockSize);	2✔
134	}
135
136	CarbonWriter writer = builder.buildWriterForCSVInput(schema, configuration);	2✔
137
138	for (int i = 0; i < rows; i++) {	2✔
139	writer.write(new String[]{	2✔
140	"robot" + (i % 10), String.valueOf(i % 3000000), String.valueOf((double) i / 2)});	2✔
141	}
142	writer.close();	2✔
143	} catch (Exception e) {	×
144	e.printStackTrace();	×
145	throw new RuntimeException(e);	×
146	}	2✔
147
148	File segmentFolder = null;	2✔
149	if (isTransactionalTable) {	2✔
150	segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null"));	2✔
151	if (!segmentFolder.exists()) {	2✔
152	throw new RuntimeException("Test failed: file not exists");	×
153	}
154	} else {
155	segmentFolder = new File(path);	2✔
156	if (!segmentFolder.exists()) {	2✔
157	throw new RuntimeException("Test failed: file not exists");	×
158	}
159	}
160
161	File[] dataFiles = segmentFolder.listFiles(new FileFilter() {	2✔
162	@Override public boolean accept(File pathname) {
163	return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);	2✔
164	}
165	});
166	if (dataFiles == null) {	2✔
167	throw new RuntimeException("Test failed: dataFiles is null");	×
168	}
169
170	if (dataFiles.length == 0) {	2✔
171	throw new RuntimeException("Test failed: dataFiles is empty");	×
172	}
173	}	2✔
174
175	/**
176	* verify whether the file exists
177	* if delete the file success or file not exists, then return true; otherwise return false
178	*
179	* @return boolean
180	*/
181	public static boolean cleanMdtFile() {
182	String fileName = CarbonProperties.getInstance().getSystemFolderLocation()	2✔
183	+ CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
184	try {
185	if (FileFactory.isFileExist(fileName)) {	2✔
186	File file = new File(fileName);	×
187	return file.delete();	×
188	} else {
189	return true;	2✔
190	}
191	} catch (IOException e) {	×
192	e.printStackTrace();	×
193	return false;	×
194	}
195	}
196
197	/**
198	* verify whether the mdt file exists
199	* if the file exists, then return true; otherwise return false
200	*
201	* @return boolean
202	*/
203	public static boolean verifyMdtFile() {
204	String fileName = CarbonProperties.getInstance().getSystemFolderLocation()	2✔
205	+ CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
206	try {
207	if (FileFactory.isFileExist(fileName)) {	2✔
208	return true;	×
209	}
210	return false;	2✔
211	} catch (IOException e) {	×
212	throw new RuntimeException("IO exception:", e);	×
213	}
214	}
215	}

apache / carbondata / 572

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous