10098

Build # 10098

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/17f7cdae89fdffc764f339c3ff2f927da348ec79">17f7cdae8<a href="https://github.com/apache/carbondata/commit/17f7cdae89fdffc764f339c3ff2f927da348ec79">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/b7df6abe73a633c0df79c5b426df9416d7a4c1bc">b7df6abe7</a>">b7df6abe7</a><a href="https://github.com/apache/carbondata/commit/17f7cdae89fdffc764f339c3ff2f927da348ec79"> into 96cc1a6ab">96cc1a6ab</a>

Run Details

6 of 6 new or added lines in 1 file covered. (100.0%)

61413 of 77700 relevant lines covered (79.04%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.0

/datamap/bloom/src/main/java/org/apache/hadoop/util/bloom/CarbonBloomFilter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.util.bloom;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.BitSet;

import org.roaringbitmap.RoaringBitmap;

/**
 * It is the extendable class to hadoop bloomfilter, it is extendable to implement compressed bloom
 * and fast serialize and deserialize of bloom.
 */
public class CarbonBloomFilter extends BloomFilter {

  private RoaringBitmap bitmap;

  private boolean compress;

  private int blockletNo;

  // used for building blocklet when query
  private String shardName;

  public CarbonBloomFilter() {
  }

  public CarbonBloomFilter(int vectorSize, int nbHash, int hashType, boolean compress) {
    super(vectorSize, nbHash, hashType);
    this.compress = compress;
  }

  @Override
  public boolean membershipTest(Key key) {
    if (compress) {
      // If it is compressed check in roaring bitmap
      if (key == null) {
        throw new NullPointerException("key cannot be null");
      }
      int[] h = hash.hash(key);
      hash.clear();
      for (int i = 0; i < nbHash; i++) {
        if (!bitmap.contains(h[i])) {
          return false;
        }
      }
      return true;
    } else {
      // call super method to avoid IllegalAccessError for `bits` field
      return super.membershipTest(key);
    }
  }

  @Override
  public void write(DataOutput out) throws IOException {
    out.writeInt(blockletNo);
    out.writeInt(this.nbHash);
    out.writeByte(this.hashType);
    out.writeInt(this.vectorSize);
    out.writeBoolean(compress);
    BitSet bits = getBitSet();
    if (!compress) {
      byte[] bytes = bits.toByteArray();
      out.writeInt(bytes.length);
      out.write(bytes);
    } else {
      RoaringBitmap bitmap = new RoaringBitmap();
      int length = bits.cardinality();
      int nextSetBit = bits.nextSetBit(0);
      for (int i = 0; i < length; ++i) {
        bitmap.add(nextSetBit);
        nextSetBit = bits.nextSetBit(nextSetBit + 1);
      }
      bitmap.serialize(out);
    }
  }

  @Override
  public void readFields(DataInput in) throws IOException {
    this.blockletNo = in.readInt();
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
    this.vectorSize = in.readInt();
    this.compress = in.readBoolean();
    if (!compress) {
      int len = in.readInt();
      byte[] bytes = new byte[len];
      in.readFully(bytes);
      setBitSet(BitSet.valueOf(bytes));
    } else {
      this.bitmap = new RoaringBitmap();
      bitmap.deserialize(in);
    }
    this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
  }

  public int getSize() {
    int size = 14; // size of nbHash,hashType, vectorSize, compress
    if (compress) {
      size += bitmap.getSizeInBytes();
    } else {
      try {
        size += getBitSet().toLongArray().length * 8;
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
    return size;
  }

  /**
   * Get bitset from super class using reflection, in some cases java cannot access the fields if
   * jars are loaded in separte class loaders.
   *
   * @return
   * @throws IOException
   */
  private BitSet getBitSet() throws IOException {
    try {
      Field field = BloomFilter.class.getDeclaredField("bits");
      field.setAccessible(true);
      return (BitSet)field.get(this);
    } catch (Exception e) {
      throw new IOException(e);
    }
  }

  /**
   * Set bitset from super class using reflection, in some cases java cannot access the fields if
   * jars are loaded in separte class loaders.
   * @param bitSet
   * @throws IOException
   */
  private void setBitSet(BitSet bitSet) throws IOException {
    try {
      Field field = BloomFilter.class.getDeclaredField("bits");
      field.setAccessible(true);
      field.set(this, bitSet);
    } catch (Exception e) {
      throw new IOException(e);
    }
  }

  public void setBlockletNo(int blockletNo) {
    this.blockletNo = blockletNo;
  }

  public int getBlockletNo() {
    return blockletNo;
  }

  public String getShardName() {
    return shardName;
  }

  public void setShardName(String shardName) {
    this.shardName = shardName;
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.hadoop.util.bloom;
18
19	import java.io.DataInput;
20	import java.io.DataOutput;
21	import java.io.IOException;
22	import java.lang.reflect.Field;
23	import java.util.BitSet;
24
25	import org.roaringbitmap.RoaringBitmap;
26
27	/**
28	* It is the extendable class to hadoop bloomfilter, it is extendable to implement compressed bloom
29	* and fast serialize and deserialize of bloom.
30	*/
31	public class CarbonBloomFilter extends BloomFilter {
32
33	private RoaringBitmap bitmap;
34
35	private boolean compress;
36
37	private int blockletNo;
38
39	// used for building blocklet when query
40	private String shardName;
41
42	public CarbonBloomFilter() {	1✔
43	}	1✔
44
45	public CarbonBloomFilter(int vectorSize, int nbHash, int hashType, boolean compress) {
46	super(vectorSize, nbHash, hashType);	1✔
47	this.compress = compress;	1✔
48	}	1✔
49
50	@Override
51	public boolean membershipTest(Key key) {
52	if (compress) {	1✔
53	// If it is compressed check in roaring bitmap
54	if (key == null) {	1✔
55	throw new NullPointerException("key cannot be null");	×
56	}
57	int[] h = hash.hash(key);	1✔
58	hash.clear();	1✔
59	for (int i = 0; i < nbHash; i++) {	1✔
60	if (!bitmap.contains(h[i])) {	1✔
61	return false;	1✔
62	}
63	}
64	return true;	1✔
65	} else {
66	// call super method to avoid IllegalAccessError for `bits` field
67	return super.membershipTest(key);	×
68	}
69	}
70
71	@Override
72	public void write(DataOutput out) throws IOException {
73	out.writeInt(blockletNo);	1✔
74	out.writeInt(this.nbHash);	1✔
75	out.writeByte(this.hashType);	1✔
76	out.writeInt(this.vectorSize);	1✔
77	out.writeBoolean(compress);	1✔
78	BitSet bits = getBitSet();	1✔
79	if (!compress) {	1✔
80	byte[] bytes = bits.toByteArray();	×
81	out.writeInt(bytes.length);	×
82	out.write(bytes);	×
83	} else {	×
84	RoaringBitmap bitmap = new RoaringBitmap();	1✔
85	int length = bits.cardinality();	1✔
86	int nextSetBit = bits.nextSetBit(0);	1✔
87	for (int i = 0; i < length; ++i) {	1✔
88	bitmap.add(nextSetBit);	1✔
89	nextSetBit = bits.nextSetBit(nextSetBit + 1);	1✔
90	}
91	bitmap.serialize(out);	1✔
92	}
93	}	1✔
94
95	@Override
96	public void readFields(DataInput in) throws IOException {
97	this.blockletNo = in.readInt();	1✔
98	this.nbHash = in.readInt();	1✔
99	this.hashType = in.readByte();	1✔
100	this.vectorSize = in.readInt();	1✔
101	this.compress = in.readBoolean();	1✔
102	if (!compress) {	1✔
103	int len = in.readInt();	×
104	byte[] bytes = new byte[len];	×
105	in.readFully(bytes);	×
106	setBitSet(BitSet.valueOf(bytes));	×
107	} else {	×
108	this.bitmap = new RoaringBitmap();	1✔
109	bitmap.deserialize(in);	1✔
110	}
111	this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);	1✔
112	}	1✔
113
114	public int getSize() {
115	int size = 14; // size of nbHash,hashType, vectorSize, compress	1✔
116	if (compress) {	1✔
117	size += bitmap.getSizeInBytes();	1✔
118	} else {
119	try {
120	size += getBitSet().toLongArray().length * 8;	×
121	} catch (IOException e) {	×
122	throw new RuntimeException(e);	×
123	}	×
124	}
125	return size;	1✔
126	}
127
128	/**
129	* Get bitset from super class using reflection, in some cases java cannot access the fields if
130	* jars are loaded in separte class loaders.
131	*
132	* @return
133	* @throws IOException
134	*/
135	private BitSet getBitSet() throws IOException {
136	try {
137	Field field = BloomFilter.class.getDeclaredField("bits");	1✔
138	field.setAccessible(true);	1✔
139	return (BitSet)field.get(this);	1✔
140	} catch (Exception e) {	×
141	throw new IOException(e);	×
142	}
143	}
144
145	/**
146	* Set bitset from super class using reflection, in some cases java cannot access the fields if
147	* jars are loaded in separte class loaders.
148	* @param bitSet
149	* @throws IOException
150	*/
151	private void setBitSet(BitSet bitSet) throws IOException {
152	try {
153	Field field = BloomFilter.class.getDeclaredField("bits");	×
154	field.setAccessible(true);	×
155	field.set(this, bitSet);	×
156	} catch (Exception e) {	×
157	throw new IOException(e);	×
158	}	×
159	}	×
160
161	public void setBlockletNo(int blockletNo) {
162	this.blockletNo = blockletNo;	1✔
163	}	1✔
164
165	public int getBlockletNo() {
166	return blockletNo;	1✔
167	}
168
169	public String getShardName() {
170	return shardName;	1✔
171	}
172
173	public void setShardName(String shardName) {
174	this.shardName = shardName;	1✔
175	}	1✔
176
177	}

apache / carbondata / 10098

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous