3837

Build # 3837

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/847d19c5c2dd4a595ab157e07806e18ecce02d0f">847d19c5c<a href="https://github.com/apache/carbondata/commit/847d19c5c2dd4a595ab157e07806e18ecce02d0f">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/b6e7bf1788e3acf894db233ed31b400acf9314fc">b6e7bf178</a>">b6e7bf178</a><a href="https://github.com/apache/carbondata/commit/847d19c5c2dd4a595ab157e07806e18ecce02d0f"> into fc8c9d067">fc8c9d067</a>

Run Details

164 of 164 new or added lines in 4 files covered. (100.0%)

64257 of 83027 relevant lines covered (77.39%)

1.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.19

/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.sdk.file;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;

import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonProperties;

import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.JsonDecoder;
import org.apache.hadoop.conf.Configuration;

@InterfaceAudience.Developer("Test")
public class TestUtil {

  public static final Configuration configuration = new Configuration();

  public static GenericData.Record jsonToAvro(String json, String avroSchema) throws IOException {
    InputStream input = null;
    DataFileWriter writer = null;
    ByteArrayOutputStream output = null;
    try {
      org.apache.avro.Schema schema = new org.apache.avro.Schema.Parser().parse(avroSchema);
      GenericDatumReader reader = new GenericDatumReader(schema);
      input = new ByteArrayInputStream(json.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
      output = new ByteArrayOutputStream();
      DataInputStream din = new DataInputStream(input);
      writer = new DataFileWriter(new GenericDatumWriter());
      writer.create(schema, output);
      JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, din);
      return (GenericData.Record) reader.read(null, decoder);
    } finally {
      if (input != null) {
        input.close();
      }
      if (writer != null) {
        writer.close();
      }
    }
  }

  static void writeFilesAndVerify(Schema schema, String path, String[] sortColumns) {
    writeFilesAndVerify(
        100, schema, path, sortColumns, -1, -1);
  }

  public static void writeFilesAndVerify(Schema schema, String path) {
    writeFilesAndVerify(
        100, schema, path, null, -1, -1);
  }

  /**
   * write file and verify
   *
   * @param rows   number of rows
   * @param schema schema
   * @param path   table store path
   */
  public static void writeFilesAndVerify(
      int rows, Schema schema, String path) {
    writeFilesAndVerify(rows, schema, path, null, -1, -1);
  }

  /**
   * Invoke CarbonWriter API to write carbon files and assert the file is rewritten
   * @param rows number of rows to write
   * @param schema schema of the file
   * @param path local write path
   * @param sortColumns sort columns
   * @param blockletSize blockletSize in the file, -1 for default size
   * @param blockSize blockSize in the file, -1 for default size
   */
  public static void writeFilesAndVerify(int rows, Schema schema, String path, String[] sortColumns,
      int blockletSize, int blockSize) {
    try {
      CarbonWriterBuilder builder = CarbonWriter.builder()
          .outputPath(path);
      if (sortColumns != null) {
        builder = builder.sortBy(sortColumns);
      }
      if (blockletSize != -1) {
        builder = builder.withBlockletSize(blockletSize);
      }
      if (blockSize != -1) {
        builder = builder.withBlockSize(blockSize);
      }

      CarbonWriter writer = builder.withCsvInput(schema).writtenBy("TestUtil").build();

      for (int i = 0; i < rows; i++) {
        writer.write(new String[]{
            "robot" + (i % 10), String.valueOf(i % 3000000), String.valueOf((double) i / 2)});
      }
      writer.close();
    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e);
    }

    File[] dataFiles = new File(path).listFiles(new FileFilter() {
      @Override public boolean accept(File pathname) {
        return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
      }
    });
    if (dataFiles == null) {
      throw new RuntimeException("Test failed: dataFiles is null");
    }

    if (dataFiles.length == 0) {
      throw new RuntimeException("Test failed: dataFiles is empty");
    }
  }

  /**
   * verify whether the file exists
   * if delete the file success or file not exists, then return true; otherwise return false
   *
   * @return boolean
   */
  public static boolean cleanMdtFile() {
    String fileName = CarbonProperties.getInstance().getSystemFolderLocation()
            + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
    try {
      if (FileFactory.isFileExist(fileName)) {
        File file = new File(fileName);
        return file.delete();
      } else {
        return true;
      }
    } catch (IOException e) {
      e.printStackTrace();
      return false;
    }
  }

  /**
   * verify whether the mdt file exists
   * if the file exists, then return true; otherwise return false
   *
   * @return boolean
   */
  public static boolean verifyMdtFile() {
    String fileName = CarbonProperties.getInstance().getSystemFolderLocation()
            + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
    try {
      if (FileFactory.isFileExist(fileName)) {
        return true;
      }
      return false;
    } catch (IOException e) {
      throw new RuntimeException("IO exception:", e);
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.sdk.file;
19
20	import java.io.ByteArrayInputStream;
21	import java.io.ByteArrayOutputStream;
22	import java.io.DataInputStream;
23	import java.io.File;
24	import java.io.FileFilter;
25	import java.io.IOException;
26	import java.io.InputStream;
27
28	import org.apache.carbondata.common.annotations.InterfaceAudience;
29	import org.apache.carbondata.core.constants.CarbonCommonConstants;
30	import org.apache.carbondata.core.datastore.impl.FileFactory;
31	import org.apache.carbondata.core.util.CarbonProperties;
32
33	import org.apache.avro.file.DataFileWriter;
34	import org.apache.avro.generic.GenericData;
35	import org.apache.avro.generic.GenericDatumReader;
36	import org.apache.avro.generic.GenericDatumWriter;
37	import org.apache.avro.io.DecoderFactory;
38	import org.apache.avro.io.JsonDecoder;
39	import org.apache.hadoop.conf.Configuration;
40
41	@InterfaceAudience.Developer("Test")
42	public class TestUtil {	×
43
44	public static final Configuration configuration = new Configuration();	2✔
45
46	public static GenericData.Record jsonToAvro(String json, String avroSchema) throws IOException {
47	InputStream input = null;	2✔
48	DataFileWriter writer = null;	2✔
49	ByteArrayOutputStream output = null;	2✔
50	try {
51	org.apache.avro.Schema schema = new org.apache.avro.Schema.Parser().parse(avroSchema);	2✔
52	GenericDatumReader reader = new GenericDatumReader(schema);	2✔
53	input = new ByteArrayInputStream(json.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));	2✔
54	output = new ByteArrayOutputStream();	2✔
55	DataInputStream din = new DataInputStream(input);	2✔
56	writer = new DataFileWriter(new GenericDatumWriter());	2✔
57	writer.create(schema, output);	2✔
58	JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, din);	2✔
59	return (GenericData.Record) reader.read(null, decoder);	2✔
60	} finally {
61	if (input != null) {	2✔
62	input.close();	2✔
63	}
64	if (writer != null) {	2✔
65	writer.close();	2✔
66	}
67	}	×
68	}
69
70	static void writeFilesAndVerify(Schema schema, String path, String[] sortColumns) {
71	writeFilesAndVerify(	2✔
72	100, schema, path, sortColumns, -1, -1);
73	}	2✔
74
75	public static void writeFilesAndVerify(Schema schema, String path) {
76	writeFilesAndVerify(	2✔
77	100, schema, path, null, -1, -1);
78	}	2✔
79
80	/**
81	* write file and verify
82	*
83	* @param rows number of rows
84	* @param schema schema
85	* @param path table store path
86	*/
87	public static void writeFilesAndVerify(
88	int rows, Schema schema, String path) {
89	writeFilesAndVerify(rows, schema, path, null, -1, -1);	2✔
90	}	2✔
91
92	/**
93	* Invoke CarbonWriter API to write carbon files and assert the file is rewritten
94	* @param rows number of rows to write
95	* @param schema schema of the file
96	* @param path local write path
97	* @param sortColumns sort columns
98	* @param blockletSize blockletSize in the file, -1 for default size
99	* @param blockSize blockSize in the file, -1 for default size
100	*/
101	public static void writeFilesAndVerify(int rows, Schema schema, String path, String[] sortColumns,
102	int blockletSize, int blockSize) {
103	try {
104	CarbonWriterBuilder builder = CarbonWriter.builder()	2✔
105	.outputPath(path);	2✔
106	if (sortColumns != null) {	2✔
107	builder = builder.sortBy(sortColumns);	2✔
108	}
109	if (blockletSize != -1) {	2✔
110	builder = builder.withBlockletSize(blockletSize);	2✔
111	}
112	if (blockSize != -1) {	2✔
113	builder = builder.withBlockSize(blockSize);	2✔
114	}
115
116	CarbonWriter writer = builder.withCsvInput(schema).writtenBy("TestUtil").build();	2✔
117
118	for (int i = 0; i < rows; i++) {	2✔
119	writer.write(new String[]{	2✔
120	"robot" + (i % 10), String.valueOf(i % 3000000), String.valueOf((double) i / 2)});	2✔
121	}
122	writer.close();	2✔
123	} catch (Exception e) {	×
124	e.printStackTrace();	×
125	throw new RuntimeException(e);	×
126	}	2✔
127
128	File[] dataFiles = new File(path).listFiles(new FileFilter() {	2✔
129	@Override public boolean accept(File pathname) {
130	return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);	2✔
131	}
132	});
133	if (dataFiles == null) {	2✔
134	throw new RuntimeException("Test failed: dataFiles is null");	×
135	}
136
137	if (dataFiles.length == 0) {	2✔
138	throw new RuntimeException("Test failed: dataFiles is empty");	×
139	}
140	}	2✔
141
142	/**
143	* verify whether the file exists
144	* if delete the file success or file not exists, then return true; otherwise return false
145	*
146	* @return boolean
147	*/
148	public static boolean cleanMdtFile() {
149	String fileName = CarbonProperties.getInstance().getSystemFolderLocation()	2✔
150	+ CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
151	try {
152	if (FileFactory.isFileExist(fileName)) {	2✔
153	File file = new File(fileName);	×
154	return file.delete();	×
155	} else {
156	return true;	2✔
157	}
158	} catch (IOException e) {	×
159	e.printStackTrace();	×
160	return false;	×
161	}
162	}
163
164	/**
165	* verify whether the mdt file exists
166	* if the file exists, then return true; otherwise return false
167	*
168	* @return boolean
169	*/
170	public static boolean verifyMdtFile() {
171	String fileName = CarbonProperties.getInstance().getSystemFolderLocation()	2✔
172	+ CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
173	try {
174	if (FileFactory.isFileExist(fileName)) {	2✔
175	return true;	×
176	}
177	return false;	2✔
178	} catch (IOException e) {	×
179	throw new RuntimeException("IO exception:", e);	×
180	}
181	}
182	}

apache / carbondata / 3837

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous