9359

Build # 9359

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/4fcc5b149c36500283566b6d30d652c28a3ffcfd">4fcc5b149<a href="https://github.com/apache/carbondata/commit/4fcc5b149c36500283566b6d30d652c28a3ffcfd">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/9ff037c6ae9fad15020930106bae4fa29d5529e4">9ff037c6a</a>">9ff037c6a</a><a href="https://github.com/apache/carbondata/commit/4fcc5b149c36500283566b6d30d652c28a3ffcfd"> into 9de946673">9de946673</a>

Run Details

67 of 67 new or added lines in 2 files covered. (100.0%)

61635 of 77112 relevant lines covered (79.93%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.69

/core/src/main/java/org/apache/carbondata/core/datastore/compression/Lz4Compressor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.datastore.compression;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.DoubleBuffer;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
import java.util.Arrays;

import org.apache.carbondata.core.util.ByteUtil;

import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.lz4.LZ4FastDecompressor;

/**
 * This class is implementation of lz4 compressor for column compression.
 *
 * Since lz4 decompression needs original size of data,
 * we specially add it at the beginning of compressed data
 */
public class Lz4Compressor implements Compressor {

  private LZ4Compressor compressor;
  private LZ4FastDecompressor decompressor;

  public Lz4Compressor() {
    LZ4Factory factory = LZ4Factory.fastestInstance();
    compressor = factory.fastCompressor();
    decompressor = factory.fastDecompressor();
  }

  @Override
  public String getName() {
    return "lz4";
  }

  /**
   * here add original size in the very beginning for decompress
   * implementation refers to https://github.com/lz4/lz4-java/issues/119
   */
  @Override
  public byte[] compressByte(byte[] unCompInput) {
    // get max compressed length
    int maxCompressedLength = compressor.maxCompressedLength(unCompInput.length);
    // prepare space, add 4 bytes for length
    byte[] compressed = new byte[maxCompressedLength + ByteUtil.SIZEOF_INT];
    // compress to temp space
    int compressedLength = compressor.compress(unCompInput, 0, unCompInput.length,
        compressed, ByteUtil.SIZEOF_INT , maxCompressedLength);
    // add src length in the very beginning
    compressed[0] = (byte)unCompInput.length;
    compressed[1] = (byte)(unCompInput.length >> 8);
    compressed[2] = (byte)(unCompInput.length >> 16);
    compressed[3] = (byte)(unCompInput.length >> 24);
    return Arrays.copyOf(compressed, compressedLength + ByteUtil.SIZEOF_INT);
  }

  @Override
  public byte[] compressByte(byte[] unCompInput, int byteSize) {
    return compressor.compress(unCompInput);
  }

  /**
   * Returns the decompressed length of compressed data
   */
  public int getDecompressedLength(byte[] src, int srcOff) {
    return (src[srcOff] & 0xFF) | (src[srcOff + 1] & 0xFF) << 8 |
        (src[srcOff + 2] & 0xFF) << 16 | src[srcOff + 3] << 24;
  }

  @Override
  public byte[] unCompressByte(byte[] compInput) {
    int destLen = getDecompressedLength(compInput, 0);
    return decompressor.decompress(compInput, ByteUtil.SIZEOF_INT, destLen);
  }

  @Override
  public byte[] unCompressByte(byte[] compInput, int offset, int length) {
    int destLen = getDecompressedLength(compInput, offset);
    return decompressor.decompress(compInput, offset + ByteUtil.SIZEOF_INT, destLen);
  }

  @Override
  public byte[] compressShort(short[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_SHORT);
    unCompBuffer.asShortBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public short[] unCompressShort(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    ShortBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asShortBuffer();
    short[] shorts = new short[unCompArray.length / ByteUtil.SIZEOF_SHORT];
    unCompBuffer.get(shorts);
    return shorts;
  }

  @Override
  public byte[] compressInt(int[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_INT);
    unCompBuffer.asIntBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public int[] unCompressInt(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    IntBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asIntBuffer();
    int[] ints = new int[unCompArray.length / ByteUtil.SIZEOF_INT];
    unCompBuffer.get(ints);
    return ints;
  }

  @Override
  public byte[] compressLong(long[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_LONG);
    unCompBuffer.asLongBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public long[] unCompressLong(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    LongBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asLongBuffer();
    long[] longs = new long[unCompArray.length / ByteUtil.SIZEOF_LONG];
    unCompBuffer.get(longs);
    return longs;
  }

  @Override
  public byte[] compressFloat(float[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_FLOAT);
    unCompBuffer.asFloatBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public float[] unCompressFloat(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    FloatBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asFloatBuffer();
    float[] floats = new float[unCompArray.length / ByteUtil.SIZEOF_FLOAT];
    unCompBuffer.get(floats);
    return floats;
  }

  @Override
  public byte[] compressDouble(double[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_DOUBLE);
    unCompBuffer.asDoubleBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public double[] unCompressDouble(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    DoubleBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asDoubleBuffer();
    double[] doubles = new double[unCompArray.length / ByteUtil.SIZEOF_DOUBLE];
    unCompBuffer.get(doubles);
    return doubles;
  }

  @Override
  public long rawCompress(long inputAddress, int inputSize, long outputAddress) throws IOException {
    throw new RuntimeException("Not implemented rawCompress for lz4 yet");
  }

  @Override
  public long rawUncompress(byte[] input, byte[] output) throws IOException {
    decompressor.decompress(input, output);
    return getDecompressedLength(input, 0);
  }

  @Override
  public long maxCompressedLength(long inputSize) {
    // input value will be checked inside the method
    return compressor.maxCompressedLength((int)inputSize);
  }

  @Override
  public boolean supportUnsafe() {
    return false;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.datastore.compression;
19
20	import java.io.IOException;
21	import java.nio.ByteBuffer;
22	import java.nio.DoubleBuffer;
23	import java.nio.FloatBuffer;
24	import java.nio.IntBuffer;
25	import java.nio.LongBuffer;
26	import java.nio.ShortBuffer;
27	import java.util.Arrays;
28
29	import org.apache.carbondata.core.util.ByteUtil;
30
31	import net.jpountz.lz4.LZ4Compressor;
32	import net.jpountz.lz4.LZ4Factory;
33	import net.jpountz.lz4.LZ4FastDecompressor;
34
35	/**
36	* This class is implementation of lz4 compressor for column compression.
37	*
38	* Since lz4 decompression needs original size of data,
39	* we specially add it at the beginning of compressed data
40	*/
41	public class Lz4Compressor implements Compressor {
42
43	private LZ4Compressor compressor;
44	private LZ4FastDecompressor decompressor;
45
46	public Lz4Compressor() {	1✔
47	LZ4Factory factory = LZ4Factory.fastestInstance();	1✔
48	compressor = factory.fastCompressor();	1✔
49	decompressor = factory.fastDecompressor();	1✔
50	}	1✔
51
52	@Override
53	public String getName() {
54	return "lz4";	1✔
55	}
56
57	/**
58	* here add original size in the very beginning for decompress
59	* implementation refers to https://github.com/lz4/lz4-java/issues/119
60	*/
61	@Override
62	public byte[] compressByte(byte[] unCompInput) {
63	// get max compressed length
64	int maxCompressedLength = compressor.maxCompressedLength(unCompInput.length);	1✔
65	// prepare space, add 4 bytes for length
66	byte[] compressed = new byte[maxCompressedLength + ByteUtil.SIZEOF_INT];	1✔
67	// compress to temp space
68	int compressedLength = compressor.compress(unCompInput, 0, unCompInput.length,	1✔
69	compressed, ByteUtil.SIZEOF_INT , maxCompressedLength);
70	// add src length in the very beginning
71	compressed[0] = (byte)unCompInput.length;	1✔
72	compressed[1] = (byte)(unCompInput.length >> 8);	1✔
73	compressed[2] = (byte)(unCompInput.length >> 16);	1✔
74	compressed[3] = (byte)(unCompInput.length >> 24);	1✔
75	return Arrays.copyOf(compressed, compressedLength + ByteUtil.SIZEOF_INT);	1✔
76	}
77
78	@Override
79	public byte[] compressByte(byte[] unCompInput, int byteSize) {
80	return compressor.compress(unCompInput);	×
81	}
82
83	/**
84	* Returns the decompressed length of compressed data
85	*/
86	public int getDecompressedLength(byte[] src, int srcOff) {
87	return (src[srcOff] & 0xFF) \| (src[srcOff + 1] & 0xFF) << 8 \|	1✔
88	(src[srcOff + 2] & 0xFF) << 16 \| src[srcOff + 3] << 24;
89	}
90
91	@Override
92	public byte[] unCompressByte(byte[] compInput) {
93	int destLen = getDecompressedLength(compInput, 0);	1✔
94	return decompressor.decompress(compInput, ByteUtil.SIZEOF_INT, destLen);	1✔
95	}
96
97	@Override
98	public byte[] unCompressByte(byte[] compInput, int offset, int length) {
99	int destLen = getDecompressedLength(compInput, offset);	1✔
100	return decompressor.decompress(compInput, offset + ByteUtil.SIZEOF_INT, destLen);	1✔
101	}
102
103	@Override
104	public byte[] compressShort(short[] unCompInput) {
105	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_SHORT);	1✔
106	unCompBuffer.asShortBuffer().put(unCompInput);	1✔
107	return compressByte(unCompBuffer.array());	1✔
108	}
109
110	@Override
111	public short[] unCompressShort(byte[] compInput, int offset, int length) {
112	byte[] unCompArray = unCompressByte(compInput, offset, length);	1✔
113	ShortBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asShortBuffer();	1✔
114	short[] shorts = new short[unCompArray.length / ByteUtil.SIZEOF_SHORT];	1✔
115	unCompBuffer.get(shorts);	1✔
116	return shorts;	1✔
117	}
118
119	@Override
120	public byte[] compressInt(int[] unCompInput) {
121	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_INT);	1✔
122	unCompBuffer.asIntBuffer().put(unCompInput);	1✔
123	return compressByte(unCompBuffer.array());	1✔
124	}
125
126	@Override
127	public int[] unCompressInt(byte[] compInput, int offset, int length) {
128	byte[] unCompArray = unCompressByte(compInput, offset, length);	1✔
129	IntBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asIntBuffer();	1✔
130	int[] ints = new int[unCompArray.length / ByteUtil.SIZEOF_INT];	1✔
131	unCompBuffer.get(ints);	1✔
132	return ints;	1✔
133	}
134
135	@Override
136	public byte[] compressLong(long[] unCompInput) {
137	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_LONG);	1✔
138	unCompBuffer.asLongBuffer().put(unCompInput);	1✔
139	return compressByte(unCompBuffer.array());	1✔
140	}
141
142	@Override
143	public long[] unCompressLong(byte[] compInput, int offset, int length) {
144	byte[] unCompArray = unCompressByte(compInput, offset, length);	1✔
145	LongBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asLongBuffer();	1✔
146	long[] longs = new long[unCompArray.length / ByteUtil.SIZEOF_LONG];	1✔
147	unCompBuffer.get(longs);	1✔
148	return longs;	1✔
149	}
150
151	@Override
152	public byte[] compressFloat(float[] unCompInput) {
153	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_FLOAT);	×
154	unCompBuffer.asFloatBuffer().put(unCompInput);	×
155	return compressByte(unCompBuffer.array());	×
156	}
157
158	@Override
159	public float[] unCompressFloat(byte[] compInput, int offset, int length) {
160	byte[] unCompArray = unCompressByte(compInput, offset, length);	×
161	FloatBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asFloatBuffer();	×
162	float[] floats = new float[unCompArray.length / ByteUtil.SIZEOF_FLOAT];	×
163	unCompBuffer.get(floats);	×
164	return floats;	×
165	}
166
167	@Override
168	public byte[] compressDouble(double[] unCompInput) {
169	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_DOUBLE);	×
170	unCompBuffer.asDoubleBuffer().put(unCompInput);	×
171	return compressByte(unCompBuffer.array());	×
172	}
173
174	@Override
175	public double[] unCompressDouble(byte[] compInput, int offset, int length) {
176	byte[] unCompArray = unCompressByte(compInput, offset, length);	×
177	DoubleBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asDoubleBuffer();	×
178	double[] doubles = new double[unCompArray.length / ByteUtil.SIZEOF_DOUBLE];	×
179	unCompBuffer.get(doubles);	×
180	return doubles;	×
181	}
182
183	@Override
184	public long rawCompress(long inputAddress, int inputSize, long outputAddress) throws IOException {
185	throw new RuntimeException("Not implemented rawCompress for lz4 yet");	×
186	}
187
188	@Override
189	public long rawUncompress(byte[] input, byte[] output) throws IOException {
190	decompressor.decompress(input, output);	×
191	return getDecompressedLength(input, 0);	×
192	}
193
194	@Override
195	public long maxCompressedLength(long inputSize) {
196	// input value will be checked inside the method
197	return compressor.maxCompressedLength((int)inputSize);	×
198	}
199
200	@Override
201	public boolean supportUnsafe() {
202	return false;	1✔
203	}
204	}

apache / carbondata / 9359

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous