9708

Build # 9708

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/180b3dec512f37e0cd4c4a39b2f90479d9234f0a">180b3dec5<a href="https://github.com/apache/carbondata/commit/180b3dec512f37e0cd4c4a39b2f90479d9234f0a">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/180b3dec512f37e0cd4c4a39b2f90479d9234f0a&quot;&gt;180b3dec5&lt;/a&gt;&lt;a href=&quot;https://github.com/apache/carbondata/commit/180b3dec512f37e0cd4c4a39b2f90479d9234f0a&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/180b3dec512f37e0cd4c4a39b2f90479&lt;/a&gt;d9234f0a">&quot;&gt;Merge &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/b1733604c5d46ece03a9a0cfe24d1ee416cd6dae">b1733604c</a><a href="https://github.com/apache/carbondata/commit/180b3dec512f37e0cd4c4a39b2f90479d9234f0a">&amp;quot;&amp;gt;b1733604c&amp;lt;/a&amp;gt;&amp;quot;&amp;gt;b1733604c&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/180b3dec512f37e0cd4c4a39b2f90479d9234&lt;/a&gt;f0a&quot;&gt; into &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/d4e8ba441">d4e8ba441">d4e8ba441</a>

Run Details

161 of 161 new or added lines in 68 files covered. (100.0%)

61301 of 76956 relevant lines covered (79.66%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/S3CarbonFile.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.datastore.filesystem;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;

import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;

public class S3CarbonFile extends HDFSCarbonFile {

  private static final Logger LOGGER =
      LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());

  public S3CarbonFile(String filePath) {
    super(filePath);
  }

  public S3CarbonFile(String filePath, Configuration hadoopConf) {
    super(filePath, hadoopConf);
  }

  public S3CarbonFile(Path path, Configuration hadoopConf) {
    super(path, hadoopConf);
  }

  /**
    TODO: The current implementation of renameForce is not correct as it deletes the destination
          object and then performs rename(copy).
          If the copy fails then there is not way to recover the old file.
           This can happen when tablestatus.write is renamed to tablestatus.
          One solution can be to read the content and rewrite the file as write with the same name
          will overwrite the file by default. Need to discuss.
          Refer CARBONDATA-2670 for tracking this.
   */
  @Override
  public boolean renameForce(String changeToName) {
    FileSystem fs;
    try {
      deleteFile(changeToName, FileFactory.getFileType(changeToName));
      fs = fileStatus.getPath().getFileSystem(hadoopConf);
      return fs.rename(fileStatus.getPath(), new Path(changeToName));
    } catch (IOException e) {
      LOGGER.error("Exception occured: " + e.getMessage(), e);
      return false;
    }
  }

  /**
   * @param listStatus
   * @return
   */
  @Override
  protected CarbonFile[] getFiles(FileStatus[] listStatus) {
    if (listStatus == null) {
      return new CarbonFile[0];
    }
    CarbonFile[] files = new CarbonFile[listStatus.length];
    for (int i = 0; i < files.length; i++) {
      files[i] = new HDFSCarbonFile(listStatus[i]);
    }
    return files;
  }

  @Override
  public DataOutputStream getDataOutputStreamUsingAppend(String path, FileFactory.FileType fileType)
      throws IOException {
    return getDataOutputStream(path, fileType, CarbonCommonConstants.BYTEBUFFER_SIZE, true);
  }

  @Override public DataOutputStream getDataOutputStream(String path, FileFactory.FileType fileType,
      int bufferSize, boolean append) throws IOException {
    Path pt = new Path(path);
    FileSystem fileSystem = pt.getFileSystem(FileFactory.getConfiguration());
    FSDataOutputStream stream;
    if (append) {
      // append to a file only if file already exists else file not found
      // exception will be thrown by hdfs
      if (CarbonUtil.isFileExists(path)) {
        DataInputStream dataInputStream = fileSystem.open(pt);
        int count = dataInputStream.available();
        // create buffer
        byte[] byteStreamBuffer = new byte[count];
        int bytesRead = dataInputStream.read(byteStreamBuffer);
        dataInputStream.close();
        stream = fileSystem.create(pt, true, bufferSize);
        if (bytesRead > 0) {
          stream.write(byteStreamBuffer, 0, bytesRead);
        }
      } else {
        stream = fileSystem.create(pt, true, bufferSize);
      }
    } else {
      stream = fileSystem.create(pt, true, bufferSize);
    }
    return stream;
  }

  @Override
  public CarbonFile getParentFile() {
    Path parent = fileStatus.getPath().getParent();
    return null == parent ? null : new S3CarbonFile(parent, hadoopConf);
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.datastore.filesystem;
18
19	import java.io.DataInputStream;
20	import java.io.DataOutputStream;
21	import java.io.IOException;
22
23	import org.apache.carbondata.common.logging.LogServiceFactory;
24	import org.apache.carbondata.core.constants.CarbonCommonConstants;
25	import org.apache.carbondata.core.datastore.impl.FileFactory;
26	import org.apache.carbondata.core.util.CarbonUtil;
27
28	import org.apache.hadoop.conf.Configuration;
29	import org.apache.hadoop.fs.FSDataOutputStream;
30	import org.apache.hadoop.fs.FileStatus;
31	import org.apache.hadoop.fs.FileSystem;
32	import org.apache.hadoop.fs.Path;
33	import org.apache.log4j.Logger;
34
35	public class S3CarbonFile extends HDFSCarbonFile {
36
37	private static final Logger LOGGER =	×
38	LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());	×
39
40	public S3CarbonFile(String filePath) {
41	super(filePath);	×
42	}	×
43
44	public S3CarbonFile(String filePath, Configuration hadoopConf) {
45	super(filePath, hadoopConf);	×
46	}	×
47
48	public S3CarbonFile(Path path, Configuration hadoopConf) {
49	super(path, hadoopConf);	×
50	}	×
51
52	/**
53	TODO: The current implementation of renameForce is not correct as it deletes the destination
54	object and then performs rename(copy).
55	If the copy fails then there is not way to recover the old file.
56	This can happen when tablestatus.write is renamed to tablestatus.
57	One solution can be to read the content and rewrite the file as write with the same name
58	will overwrite the file by default. Need to discuss.
59	Refer CARBONDATA-2670 for tracking this.
60	*/
61	@Override
62	public boolean renameForce(String changeToName) {
63	FileSystem fs;
64	try {
65	deleteFile(changeToName, FileFactory.getFileType(changeToName));	×
66	fs = fileStatus.getPath().getFileSystem(hadoopConf);	×
67	return fs.rename(fileStatus.getPath(), new Path(changeToName));	×
68	} catch (IOException e) {	×
69	LOGGER.error("Exception occured: " + e.getMessage(), e);	×
70	return false;	×
71	}
72	}
73
74	/**
75	* @param listStatus
76	* @return
77	*/
78	@Override
79	protected CarbonFile[] getFiles(FileStatus[] listStatus) {
80	if (listStatus == null) {	×
81	return new CarbonFile[0];	×
82	}
83	CarbonFile[] files = new CarbonFile[listStatus.length];	×
84	for (int i = 0; i < files.length; i++) {	×
85	files[i] = new HDFSCarbonFile(listStatus[i]);	×
86	}
87	return files;	×
88	}
89
90	@Override
91	public DataOutputStream getDataOutputStreamUsingAppend(String path, FileFactory.FileType fileType)
92	throws IOException {
93	return getDataOutputStream(path, fileType, CarbonCommonConstants.BYTEBUFFER_SIZE, true);	×
94	}
95
96	@Override public DataOutputStream getDataOutputStream(String path, FileFactory.FileType fileType,
97	int bufferSize, boolean append) throws IOException {
98	Path pt = new Path(path);	×
99	FileSystem fileSystem = pt.getFileSystem(FileFactory.getConfiguration());	×
100	FSDataOutputStream stream;
101	if (append) {	×
102	// append to a file only if file already exists else file not found
103	// exception will be thrown by hdfs
104	if (CarbonUtil.isFileExists(path)) {	×
105	DataInputStream dataInputStream = fileSystem.open(pt);	×
106	int count = dataInputStream.available();	×
107	// create buffer
108	byte[] byteStreamBuffer = new byte[count];	×
109	int bytesRead = dataInputStream.read(byteStreamBuffer);	×
110	dataInputStream.close();	×
111	stream = fileSystem.create(pt, true, bufferSize);	×
112	if (bytesRead > 0) {	×
113	stream.write(byteStreamBuffer, 0, bytesRead);	×
114	}
115	} else {	×
116	stream = fileSystem.create(pt, true, bufferSize);	×
117	}
118	} else {
119	stream = fileSystem.create(pt, true, bufferSize);	×
120	}
121	return stream;	×
122	}
123
124	@Override
125	public CarbonFile getParentFile() {
126	Path parent = fileStatus.getPath().getParent();	×
127	return null == parent ? null : new S3CarbonFile(parent, hadoopConf);	×
128	}
129
130	}

apache / carbondata / 9708

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous