2593

Build # 2593

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/8cfba8a6a700780dac44540bcb0db75cd085b741">8cfba8a6a<a href="https://github.com/apache/carbondata/commit/8cfba8a6a700780dac44540bcb0db75cd085b741">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/0e4b26a35b5132d50b56ff1eab173ed519e2aa7a">0e4b26a35</a>">0e4b26a35</a><a href="https://github.com/apache/carbondata/commit/8cfba8a6a700780dac44540bcb0db75cd085b741"> into 6605b54d5">6605b54d5</a>

Run Details

61 of 61 new or added lines in 6 files covered. (100.0%)

62514 of 78758 relevant lines covered (79.37%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

41.76

/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.datastore.compression;

import java.io.IOException;
import java.lang.reflect.Field;

import org.apache.carbondata.common.logging.LogServiceFactory;

import org.apache.log4j.Logger;
import org.xerial.snappy.Snappy;
import org.xerial.snappy.SnappyNative;

public class SnappyCompressor extends AbstractCompressor {

  private static final Logger LOGGER =
      LogServiceFactory.getLogService(SnappyCompressor.class.getName());

  // snappy estimate max compressed length as 32 + source_len + source_len/6
  public static final int MAX_BYTE_TO_COMPRESS = (int)((Integer.MAX_VALUE - 32) / 7.0 * 6);

  private final transient SnappyNative snappyNative;

  public SnappyCompressor() {
    Snappy snappy = new Snappy();
    Field privateField = null;
    try {
      privateField = snappy.getClass().getDeclaredField("impl");
    } catch (NoSuchFieldException | SecurityException e) {
      throw new RuntimeException(e);
    }
    privateField.setAccessible(true);
    try {
      snappyNative = (SnappyNative) privateField.get(snappy);
    } catch (IllegalArgumentException | IllegalAccessException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public String getName() {
    return "snappy";
  }

  @Override public byte[] compressByte(byte[] unCompInput) {
    try {
      return Snappy.rawCompress(unCompInput, unCompInput.length);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public byte[] compressByte(byte[] unCompInput, int byteSize) {
    try {
      return Snappy.rawCompress(unCompInput, byteSize);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public byte[] unCompressByte(byte[] compInput) {
    try {
      return Snappy.uncompress(compInput);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
    int uncompressedLength = 0;
    byte[] data;
    try {
      uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
      data = new byte[uncompressedLength];
      snappyNative.rawUncompress(compInput, offset, length, data, 0);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
    return data;
  }

  @Override public byte[] compressShort(short[] unCompInput) {
    try {
      return Snappy.compress(unCompInput);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public short[] unCompressShort(byte[] compInput, int offset, int length) {
    try {
      return Snappy.uncompressShortArray(compInput, offset, length);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public byte[] compressInt(int[] unCompInput) {
    try {
      return Snappy.compress(unCompInput);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
    try {
      return Snappy.uncompressIntArray(compInput, offset, length);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public byte[] compressLong(long[] unCompInput) {
    try {
      return Snappy.compress(unCompInput);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
    try {
      return Snappy.uncompressLongArray(compInput, offset, length);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public byte[] compressFloat(float[] unCompInput) {
    try {
      return Snappy.compress(unCompInput);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
    try {
      return Snappy.uncompressFloatArray(compInput, offset, length);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public byte[] compressDouble(double[] unCompInput) {
    try {
      return Snappy.compress(unCompInput);
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
    try {
      int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
      double[] result = new double[uncompressedLength / 8];
      snappyNative.rawUncompress(compInput, offset, length, result, 0);
      return result;
    } catch (IOException e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }
  }

  @Override
  public long rawCompress(long inputAddress, int inputSize, long outputAddress) throws IOException {
    return snappyNative.rawCompress(inputAddress, inputSize, outputAddress);
  }

  @Override
  public long rawUncompress(byte[] input, byte[] output) throws IOException {
    return snappyNative.rawUncompress(input, 0, input.length, output, 0);
  }

  @Override
  public long maxCompressedLength(long inputSize) {
    return snappyNative.maxCompressedLength((int) inputSize);
  }

  @Override
  public boolean supportUnsafe() {
    return true;
  }

  @Override public int unCompressedLength(byte[] data, int offset, int length) {
    try {
      return Snappy.uncompressedLength(data, offset, length);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  @Override public int rawUncompress(byte[] data, int offset, int length, byte[] output) {
    try {
      return Snappy.rawUncompress(data, offset, length, output, 0);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  @Override public boolean supportReusableBuffer() {
    return true;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.datastore.compression;
19
20	import java.io.IOException;
21	import java.lang.reflect.Field;
22
23	import org.apache.carbondata.common.logging.LogServiceFactory;
24
25	import org.apache.log4j.Logger;
26	import org.xerial.snappy.Snappy;
27	import org.xerial.snappy.SnappyNative;
28
29	public class SnappyCompressor extends AbstractCompressor {
30
31	private static final Logger LOGGER =	2✔
32	LogServiceFactory.getLogService(SnappyCompressor.class.getName());	2✔
33
34	// snappy estimate max compressed length as 32 + source_len + source_len/6
35	public static final int MAX_BYTE_TO_COMPRESS = (int)((Integer.MAX_VALUE - 32) / 7.0 * 6);
36
37	private final transient SnappyNative snappyNative;
38
39	public SnappyCompressor() {	2✔
40	Snappy snappy = new Snappy();	2✔
41	Field privateField = null;	2✔
42	try {
43	privateField = snappy.getClass().getDeclaredField("impl");	2✔
44	} catch (NoSuchFieldException \| SecurityException e) {	×
45	throw new RuntimeException(e);	×
46	}	2✔
47	privateField.setAccessible(true);	2✔
48	try {
49	snappyNative = (SnappyNative) privateField.get(snappy);	2✔
50	} catch (IllegalArgumentException \| IllegalAccessException e) {	×
51	throw new RuntimeException(e);	×
52	}	2✔
53	}	2✔
54
55	@Override
56	public String getName() {
57	return "snappy";	2✔
58	}
59
60	@Override public byte[] compressByte(byte[] unCompInput) {
61	try {
62	return Snappy.rawCompress(unCompInput, unCompInput.length);	2✔
63	} catch (IOException e) {	×
64	LOGGER.error(e.getMessage(), e);	×
65	throw new RuntimeException(e);	×
66	}
67	}
68
69	@Override public byte[] compressByte(byte[] unCompInput, int byteSize) {
70	try {
71	return Snappy.rawCompress(unCompInput, byteSize);	1✔
72	} catch (IOException e) {	×
73	LOGGER.error(e.getMessage(), e);	×
74	throw new RuntimeException(e);	×
75	}
76	}
77
78	@Override public byte[] unCompressByte(byte[] compInput) {
79	try {
80	return Snappy.uncompress(compInput);	2✔
81	} catch (IOException e) {	×
82	LOGGER.error(e.getMessage(), e);	×
83	throw new RuntimeException(e);	×
84	}
85	}
86
87	@Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
88	int uncompressedLength = 0;	2✔
89	byte[] data;
90	try {
91	uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);	2✔
92	data = new byte[uncompressedLength];	2✔
93	snappyNative.rawUncompress(compInput, offset, length, data, 0);	2✔
94	} catch (IOException e) {	×
95	LOGGER.error(e.getMessage(), e);	×
96	throw new RuntimeException(e);	×
97	}	2✔
98	return data;	2✔
99	}
100
101	@Override public byte[] compressShort(short[] unCompInput) {
102	try {
103	return Snappy.compress(unCompInput);	1✔
104	} catch (IOException e) {	×
105	LOGGER.error(e.getMessage(), e);	×
106	throw new RuntimeException(e);	×
107	}
108	}
109
110	@Override public short[] unCompressShort(byte[] compInput, int offset, int length) {
111	try {
112	return Snappy.uncompressShortArray(compInput, offset, length);	1✔
113	} catch (IOException e) {	×
114	LOGGER.error(e.getMessage(), e);	×
115	throw new RuntimeException(e);	×
116	}
117	}
118
119	@Override public byte[] compressInt(int[] unCompInput) {
120	try {
121	return Snappy.compress(unCompInput);	1✔
122	} catch (IOException e) {	×
123	LOGGER.error(e.getMessage(), e);	×
124	throw new RuntimeException(e);	×
125	}
126	}
127
128	@Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
129	try {
130	return Snappy.uncompressIntArray(compInput, offset, length);	1✔
131	} catch (IOException e) {	×
132	LOGGER.error(e.getMessage(), e);	×
133	throw new RuntimeException(e);	×
134	}
135	}
136
137	@Override public byte[] compressLong(long[] unCompInput) {
138	try {
139	return Snappy.compress(unCompInput);	1✔
140	} catch (IOException e) {	×
141	LOGGER.error(e.getMessage(), e);	×
142	throw new RuntimeException(e);	×
143	}
144	}
145
146	@Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
147	try {
148	return Snappy.uncompressLongArray(compInput, offset, length);	1✔
149	} catch (IOException e) {	×
150	LOGGER.error(e.getMessage(), e);	×
151	throw new RuntimeException(e);	×
152	}
153	}
154
155	@Override public byte[] compressFloat(float[] unCompInput) {
156	try {
157	return Snappy.compress(unCompInput);	1✔
158	} catch (IOException e) {	×
159	LOGGER.error(e.getMessage(), e);	×
160	throw new RuntimeException(e);	×
161	}
162	}
163
164	@Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
165	try {
166	return Snappy.uncompressFloatArray(compInput, offset, length);	1✔
167	} catch (IOException e) {	×
168	LOGGER.error(e.getMessage(), e);	×
169	throw new RuntimeException(e);	×
170	}
171	}
172
173	@Override public byte[] compressDouble(double[] unCompInput) {
174	try {
175	return Snappy.compress(unCompInput);	1✔
176	} catch (IOException e) {	×
177	LOGGER.error(e.getMessage(), e);	×
178	throw new RuntimeException(e);	×
179	}
180	}
181
182	@Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
183	try {
184	int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);	1✔
185	double[] result = new double[uncompressedLength / 8];	1✔
186	snappyNative.rawUncompress(compInput, offset, length, result, 0);	1✔
187	return result;	1✔
188	} catch (IOException e) {	×
189	LOGGER.error(e.getMessage(), e);	×
190	throw new RuntimeException(e);	×
191	}
192	}
193
194	@Override
195	public long rawCompress(long inputAddress, int inputSize, long outputAddress) throws IOException {
196	return snappyNative.rawCompress(inputAddress, inputSize, outputAddress);	×
197	}
198
199	@Override
200	public long rawUncompress(byte[] input, byte[] output) throws IOException {
201	return snappyNative.rawUncompress(input, 0, input.length, output, 0);	1✔
202	}
203
204	@Override
205	public long maxCompressedLength(long inputSize) {
206	return snappyNative.maxCompressedLength((int) inputSize);	×
207	}
208
209	@Override
210	public boolean supportUnsafe() {
211	return true;	×
212	}
213
214	@Override public int unCompressedLength(byte[] data, int offset, int length) {
215	try {
216	return Snappy.uncompressedLength(data, offset, length);	1✔
217	} catch (IOException e) {	×
218	throw new RuntimeException(e);	×
219	}
220	}
221
222	@Override public int rawUncompress(byte[] data, int offset, int length, byte[] output) {
223	try {
224	return Snappy.rawUncompress(data, offset, length, output, 0);	1✔
225	} catch (IOException e) {	×
226	throw new RuntimeException(e);	×
227	}
228	}
229
230	@Override public boolean supportReusableBuffer() {
231	return true;	1✔
232	}
233	}

apache / carbondata / 2593

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous