8132

Build # 8132

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/e2d6518a1e299644bc6f108edb926dd193c9b52b">e2d6518a1<a href="https://github.com/apache/carbondata/commit/e2d6518a1e299644bc6f108edb926dd193c9b52b">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/30ec01e354b13fd72dca4a90d67db254eb49903b">30ec01e35</a>">30ec01e35</a><a href="https://github.com/apache/carbondata/commit/e2d6518a1e299644bc6f108edb926dd193c9b52b"> into 2a9604cd8">2a9604cd8</a>

Run Details

236 of 236 new or added lines in 15 files covered. (100.0%)

57417 of 71122 relevant lines covered (80.73%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.23

/datamap/bloom/src/main/java/org/apache/hadoop/util/bloom/CarbonBloomFilter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.util.bloom;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.BitSet;

import org.roaringbitmap.RoaringBitmap;

/**
 * It is the extendable class to hadoop bloomfilter, it is extendable to implement compressed bloom
 * and fast serialize and deserialize of bloom.
 */
public class CarbonBloomFilter extends BloomFilter {

  private RoaringBitmap bitmap;

  private boolean compress;

  private int blockletNo;

  // used for building blocklet when query
  private String shardName;

  public CarbonBloomFilter() {
  }

  public CarbonBloomFilter(int vectorSize, int nbHash, int hashType, boolean compress) {
    super(vectorSize, nbHash, hashType);
    this.compress = compress;
  }

  @Override
  public boolean membershipTest(Key key) {
    if (key == null) {
      throw new NullPointerException("key cannot be null");
    }

    int[] h = hash.hash(key);
    hash.clear();
    if (compress) {
      // If it is compressed check in roaring bitmap
      for (int i = 0; i < nbHash; i++) {
        if (!bitmap.contains(h[i])) {
          return false;
        }
      }
    } else {
      for (int i = 0; i < nbHash; i++) {
        if (!bits.get(h[i])) {
          return false;
        }
      }
    }
    return true;
  }

  @Override
  public void write(DataOutput out) throws IOException {
    out.writeInt(blockletNo);
    out.writeInt(this.nbHash);
    out.writeByte(this.hashType);
    out.writeInt(this.vectorSize);
    out.writeBoolean(compress);
    BitSet bits = getBitSet();
    if (!compress) {
      byte[] bytes = bits.toByteArray();
      out.writeInt(bytes.length);
      out.write(bytes);
    } else {
      RoaringBitmap bitmap = new RoaringBitmap();
      int length = bits.cardinality();
      int nextSetBit = bits.nextSetBit(0);
      for (int i = 0; i < length; ++i) {
        bitmap.add(nextSetBit);
        nextSetBit = bits.nextSetBit(nextSetBit + 1);
      }
      bitmap.serialize(out);
    }
  }

  @Override
  public void readFields(DataInput in) throws IOException {
    this.blockletNo = in.readInt();
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
    this.vectorSize = in.readInt();
    this.compress = in.readBoolean();
    if (!compress) {
      int len = in.readInt();
      byte[] bytes = new byte[len];
      in.readFully(bytes);
      setBitSet(BitSet.valueOf(bytes));
    } else {
      this.bitmap = new RoaringBitmap();
      bitmap.deserialize(in);
    }
    this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
  }

  public int getSize() {
    int size = 14; // size of nbHash,hashType, vectorSize, compress
    if (compress) {
      size += bitmap.getSizeInBytes();
    } else {
      try {
        size += getBitSet().toLongArray().length * 8;
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
    return size;
  }

  /**
   * Get bitset from super class using reflection, in some cases java cannot access the fields if
   * jars are loaded in separte class loaders.
   *
   * @return
   * @throws IOException
   */
  private BitSet getBitSet() throws IOException {
    try {
      Field field = BloomFilter.class.getDeclaredField("bits");
      field.setAccessible(true);
      return (BitSet)field.get(this);
    } catch (Exception e) {
      throw new IOException(e);
    }
  }

  /**
   * Set bitset from super class using reflection, in some cases java cannot access the fields if
   * jars are loaded in separte class loaders.
   * @param bitSet
   * @throws IOException
   */
  private void setBitSet(BitSet bitSet) throws IOException {
    try {
      Field field = BloomFilter.class.getDeclaredField("bits");
      field.setAccessible(true);
      field.set(this, bitSet);
    } catch (Exception e) {
      throw new IOException(e);
    }
  }

  public void setBlockletNo(int blockletNo) {
    this.blockletNo = blockletNo;
  }

  public int getBlockletNo() {
    return blockletNo;
  }

  public String getShardName() {
    return shardName;
  }

  public void setShardName(String shardName) {
    this.shardName = shardName;
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.hadoop.util.bloom;
18
19	import java.io.DataInput;
20	import java.io.DataOutput;
21	import java.io.IOException;
22	import java.lang.reflect.Field;
23	import java.util.BitSet;
24
25	import org.roaringbitmap.RoaringBitmap;
26
27	/**
28	* It is the extendable class to hadoop bloomfilter, it is extendable to implement compressed bloom
29	* and fast serialize and deserialize of bloom.
30	*/
31	public class CarbonBloomFilter extends BloomFilter {
32
33	private RoaringBitmap bitmap;
34
35	private boolean compress;
36
37	private int blockletNo;
38
39	// used for building blocklet when query
40	private String shardName;
41
42	public CarbonBloomFilter() {	1✔
43	}	1✔
44
45	public CarbonBloomFilter(int vectorSize, int nbHash, int hashType, boolean compress) {
46	super(vectorSize, nbHash, hashType);	1✔
47	this.compress = compress;	1✔
48	}	1✔
49
50	@Override
51	public boolean membershipTest(Key key) {
52	if (key == null) {	1✔
53	throw new NullPointerException("key cannot be null");	×
54	}
55
56	int[] h = hash.hash(key);	1✔
57	hash.clear();	1✔
58	if (compress) {	1✔
59	// If it is compressed check in roaring bitmap
60	for (int i = 0; i < nbHash; i++) {	1✔
61	if (!bitmap.contains(h[i])) {	1✔
62	return false;	1✔
63	}
64	}
65	} else {
66	for (int i = 0; i < nbHash; i++) {	×
67	if (!bits.get(h[i])) {	×
68	return false;	×
69	}
70	}
71	}
72	return true;	1✔
73	}
74
75	@Override
76	public void write(DataOutput out) throws IOException {
77	out.writeInt(blockletNo);	1✔
78	out.writeInt(this.nbHash);	1✔
79	out.writeByte(this.hashType);	1✔
80	out.writeInt(this.vectorSize);	1✔
81	out.writeBoolean(compress);	1✔
82	BitSet bits = getBitSet();	1✔
83	if (!compress) {	1✔
84	byte[] bytes = bits.toByteArray();	×
85	out.writeInt(bytes.length);	×
86	out.write(bytes);	×
87	} else {	×
88	RoaringBitmap bitmap = new RoaringBitmap();	1✔
89	int length = bits.cardinality();	1✔
90	int nextSetBit = bits.nextSetBit(0);	1✔
91	for (int i = 0; i < length; ++i) {	1✔
92	bitmap.add(nextSetBit);	1✔
93	nextSetBit = bits.nextSetBit(nextSetBit + 1);	1✔
94	}
95	bitmap.serialize(out);	1✔
96	}
97	}	1✔
98
99	@Override
100	public void readFields(DataInput in) throws IOException {
101	this.blockletNo = in.readInt();	1✔
102	this.nbHash = in.readInt();	1✔
103	this.hashType = in.readByte();	1✔
104	this.vectorSize = in.readInt();	1✔
105	this.compress = in.readBoolean();	1✔
106	if (!compress) {	1✔
107	int len = in.readInt();	×
108	byte[] bytes = new byte[len];	×
109	in.readFully(bytes);	×
110	setBitSet(BitSet.valueOf(bytes));	×
111	} else {	×
112	this.bitmap = new RoaringBitmap();	1✔
113	bitmap.deserialize(in);	1✔
114	}
115	this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);	1✔
116	}	1✔
117
118	public int getSize() {
119	int size = 14; // size of nbHash,hashType, vectorSize, compress	1✔
120	if (compress) {	1✔
121	size += bitmap.getSizeInBytes();	1✔
122	} else {
123	try {
124	size += getBitSet().toLongArray().length * 8;	×
125	} catch (IOException e) {	×
126	throw new RuntimeException(e);	×
127	}	×
128	}
129	return size;	1✔
130	}
131
132	/**
133	* Get bitset from super class using reflection, in some cases java cannot access the fields if
134	* jars are loaded in separte class loaders.
135	*
136	* @return
137	* @throws IOException
138	*/
139	private BitSet getBitSet() throws IOException {
140	try {
141	Field field = BloomFilter.class.getDeclaredField("bits");	1✔
142	field.setAccessible(true);	1✔
143	return (BitSet)field.get(this);	1✔
144	} catch (Exception e) {	×
145	throw new IOException(e);	×
146	}
147	}
148
149	/**
150	* Set bitset from super class using reflection, in some cases java cannot access the fields if
151	* jars are loaded in separte class loaders.
152	* @param bitSet
153	* @throws IOException
154	*/
155	private void setBitSet(BitSet bitSet) throws IOException {
156	try {
157	Field field = BloomFilter.class.getDeclaredField("bits");	×
158	field.setAccessible(true);	×
159	field.set(this, bitSet);	×
160	} catch (Exception e) {	×
161	throw new IOException(e);	×
162	}	×
163	}	×
164
165	public void setBlockletNo(int blockletNo) {
166	this.blockletNo = blockletNo;	1✔
167	}	1✔
168
169	public int getBlockletNo() {
170	return blockletNo;	1✔
171	}
172
173	public String getShardName() {
174	return shardName;	1✔
175	}
176
177	public void setShardName(String shardName) {
178	this.shardName = shardName;	1✔
179	}	1✔
180
181	}

apache / carbondata / 8132

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous