9681

Build # 9681

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/4bb3473bbdb09f1212bae1c445cb827b68c72d9b">4bb3473bb<a href="https://github.com/apache/carbondata/commit/4bb3473bbdb09f1212bae1c445cb827b68c72d9b">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/be9172ddf4088b9b2d8869738e60978c5cc24d9f">be9172ddf</a>">be9172ddf</a><a href="https://github.com/apache/carbondata/commit/4bb3473bbdb09f1212bae1c445cb827b68c72d9b"> into 851dd2c88">851dd2c88</a>

Run Details

3 of 3 new or added lines in 2 files covered. (100.0%)

56242 of 76994 relevant lines covered (73.05%)

0.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.96

/core/src/main/java/org/apache/carbondata/core/datastore/compression/ZstdCompressor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.datastore.compression;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.DoubleBuffer;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;

import org.apache.carbondata.core.util.ByteUtil;

import com.github.luben.zstd.Zstd;

public class ZstdCompressor implements Compressor {
  private static final int COMPRESS_LEVEL = 3;

  public ZstdCompressor() {
  }

  @Override
  public String getName() {
    return "zstd";
  }

  @Override
  public byte[] compressByte(byte[] unCompInput) {
    return Zstd.compress(unCompInput, COMPRESS_LEVEL);
  }

  @Override
  public byte[] compressByte(byte[] unCompInput, int byteSize) {
    return Zstd.compress(unCompInput, COMPRESS_LEVEL);
  }

  @Override
  public byte[] unCompressByte(byte[] compInput) {
    long decompressedSize = Zstd.decompressedSize(compInput);
    return Zstd.decompress(compInput, (int) decompressedSize);
  }

  @Override
  public byte[] unCompressByte(byte[] compInput, int offset, int length) {
    // todo: how to avoid memory copy
    byte[] dstBytes = new byte[length];
    System.arraycopy(compInput, offset, dstBytes, 0, length);
    return unCompressByte(dstBytes);
  }

  @Override
  public byte[] compressShort(short[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_SHORT);
    unCompBuffer.asShortBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public short[] unCompressShort(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    ShortBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asShortBuffer();
    short[] shorts = new short[unCompArray.length / ByteUtil.SIZEOF_SHORT];
    unCompBuffer.get(shorts);
    return shorts;
  }

  @Override
  public byte[] compressInt(int[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_INT);
    unCompBuffer.asIntBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public int[] unCompressInt(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    IntBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asIntBuffer();
    int[] ints = new int[unCompArray.length / ByteUtil.SIZEOF_INT];
    unCompBuffer.get(ints);
    return ints;
  }

  @Override
  public byte[] compressLong(long[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_LONG);
    unCompBuffer.asLongBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public long[] unCompressLong(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    LongBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asLongBuffer();
    long[] longs = new long[unCompArray.length / ByteUtil.SIZEOF_LONG];
    unCompBuffer.get(longs);
    return longs;
  }

  @Override
  public byte[] compressFloat(float[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_FLOAT);
    unCompBuffer.asFloatBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public float[] unCompressFloat(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    FloatBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asFloatBuffer();
    float[] floats = new float[unCompArray.length / ByteUtil.SIZEOF_FLOAT];
    unCompBuffer.get(floats);
    return floats;
  }

  @Override
  public byte[] compressDouble(double[] unCompInput) {
    ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_DOUBLE);
    unCompBuffer.asDoubleBuffer().put(unCompInput);
    return compressByte(unCompBuffer.array());
  }

  @Override
  public double[] unCompressDouble(byte[] compInput, int offset, int length) {
    byte[] unCompArray = unCompressByte(compInput, offset, length);
    DoubleBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asDoubleBuffer();
    double[] doubles = new double[unCompArray.length / ByteUtil.SIZEOF_DOUBLE];
    unCompBuffer.get(doubles);
    return doubles;
  }

  @Override
  public long rawCompress(long inputAddress, int inputSize, long outputAddress) throws IOException {
    throw new RuntimeException("Not implemented rawCompress for zstd yet");
  }

  @Override
  public long rawUncompress(byte[] input, byte[] output) throws IOException {
    return Zstd.decompress(output, input);
  }

  @Override
  public long maxCompressedLength(long inputSize) {
    return Zstd.compressBound(inputSize);
  }

  /**
   * currently java version of zstd does not support this feature.
   * It may support it in upcoming release 1.3.5-3, then we can optimize this accordingly.
   */
  @Override
  public boolean supportUnsafe() {
    return false;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.datastore.compression;
19
20	import java.io.IOException;
21	import java.nio.ByteBuffer;
22	import java.nio.DoubleBuffer;
23	import java.nio.FloatBuffer;
24	import java.nio.IntBuffer;
25	import java.nio.LongBuffer;
26	import java.nio.ShortBuffer;
27
28	import org.apache.carbondata.core.util.ByteUtil;
29
30	import com.github.luben.zstd.Zstd;
31
32	public class ZstdCompressor implements Compressor {
33	private static final int COMPRESS_LEVEL = 3;
34
35	public ZstdCompressor() {	2✔
36	}	2✔
37
38	@Override
39	public String getName() {
40	return "zstd";	1✔
41	}
42
43	@Override
44	public byte[] compressByte(byte[] unCompInput) {
45	return Zstd.compress(unCompInput, COMPRESS_LEVEL);	1✔
46	}
47
48	@Override
49	public byte[] compressByte(byte[] unCompInput, int byteSize) {
50	return Zstd.compress(unCompInput, COMPRESS_LEVEL);	×
51	}
52
53	@Override
54	public byte[] unCompressByte(byte[] compInput) {
55	long decompressedSize = Zstd.decompressedSize(compInput);	1✔
56	return Zstd.decompress(compInput, (int) decompressedSize);	1✔
57	}
58
59	@Override
60	public byte[] unCompressByte(byte[] compInput, int offset, int length) {
61	// todo: how to avoid memory copy
62	byte[] dstBytes = new byte[length];	1✔
63	System.arraycopy(compInput, offset, dstBytes, 0, length);	1✔
64	return unCompressByte(dstBytes);	1✔
65	}
66
67	@Override
68	public byte[] compressShort(short[] unCompInput) {
69	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_SHORT);	1✔
70	unCompBuffer.asShortBuffer().put(unCompInput);	1✔
71	return compressByte(unCompBuffer.array());	1✔
72	}
73
74	@Override
75	public short[] unCompressShort(byte[] compInput, int offset, int length) {
76	byte[] unCompArray = unCompressByte(compInput, offset, length);	1✔
77	ShortBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asShortBuffer();	1✔
78	short[] shorts = new short[unCompArray.length / ByteUtil.SIZEOF_SHORT];	1✔
79	unCompBuffer.get(shorts);	1✔
80	return shorts;	1✔
81	}
82
83	@Override
84	public byte[] compressInt(int[] unCompInput) {
85	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_INT);	1✔
86	unCompBuffer.asIntBuffer().put(unCompInput);	1✔
87	return compressByte(unCompBuffer.array());	1✔
88	}
89
90	@Override
91	public int[] unCompressInt(byte[] compInput, int offset, int length) {
92	byte[] unCompArray = unCompressByte(compInput, offset, length);	1✔
93	IntBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asIntBuffer();	1✔
94	int[] ints = new int[unCompArray.length / ByteUtil.SIZEOF_INT];	1✔
95	unCompBuffer.get(ints);	1✔
96	return ints;	1✔
97	}
98
99	@Override
100	public byte[] compressLong(long[] unCompInput) {
101	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_LONG);	1✔
102	unCompBuffer.asLongBuffer().put(unCompInput);	1✔
103	return compressByte(unCompBuffer.array());	1✔
104	}
105
106	@Override
107	public long[] unCompressLong(byte[] compInput, int offset, int length) {
108	byte[] unCompArray = unCompressByte(compInput, offset, length);	1✔
109	LongBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asLongBuffer();	1✔
110	long[] longs = new long[unCompArray.length / ByteUtil.SIZEOF_LONG];	1✔
111	unCompBuffer.get(longs);	1✔
112	return longs;	1✔
113	}
114
115	@Override
116	public byte[] compressFloat(float[] unCompInput) {
117	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_FLOAT);	×
118	unCompBuffer.asFloatBuffer().put(unCompInput);	×
119	return compressByte(unCompBuffer.array());	×
120	}
121
122	@Override
123	public float[] unCompressFloat(byte[] compInput, int offset, int length) {
124	byte[] unCompArray = unCompressByte(compInput, offset, length);	×
125	FloatBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asFloatBuffer();	×
126	float[] floats = new float[unCompArray.length / ByteUtil.SIZEOF_FLOAT];	×
127	unCompBuffer.get(floats);	×
128	return floats;	×
129	}
130
131	@Override
132	public byte[] compressDouble(double[] unCompInput) {
133	ByteBuffer unCompBuffer = ByteBuffer.allocate(unCompInput.length * ByteUtil.SIZEOF_DOUBLE);	×
134	unCompBuffer.asDoubleBuffer().put(unCompInput);	×
135	return compressByte(unCompBuffer.array());	×
136	}
137
138	@Override
139	public double[] unCompressDouble(byte[] compInput, int offset, int length) {
140	byte[] unCompArray = unCompressByte(compInput, offset, length);	×
141	DoubleBuffer unCompBuffer = ByteBuffer.wrap(unCompArray).asDoubleBuffer();	×
142	double[] doubles = new double[unCompArray.length / ByteUtil.SIZEOF_DOUBLE];	×
143	unCompBuffer.get(doubles);	×
144	return doubles;	×
145	}
146
147	@Override
148	public long rawCompress(long inputAddress, int inputSize, long outputAddress) throws IOException {
149	throw new RuntimeException("Not implemented rawCompress for zstd yet");	×
150	}
151
152	@Override
153	public long rawUncompress(byte[] input, byte[] output) throws IOException {
154	return Zstd.decompress(output, input);	×
155	}
156
157	@Override
158	public long maxCompressedLength(long inputSize) {
159	return Zstd.compressBound(inputSize);	×
160	}
161
162	/**
163	* currently java version of zstd does not support this feature.
164	* It may support it in upcoming release 1.3.5-3, then we can optimize this accordingly.
165	*/
166	@Override
167	public boolean supportUnsafe() {
168	return false;	1✔
169	}
170	}

apache / carbondata / 9681

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous