#306

Committed 30 Apr 2024 10:01PM UTC coverage: 97.645% (-0.5%) from 98.139%

Build # #306

Build Type

push

Committed by

Commit Message

Merge pull request #555 from apache/fix_pom_xml_header

Fix pom xml header

Run Details

26865 of 27513 relevant lines covered (97.64%)

0.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.81

/src/main/java/org/apache/datasketches/quantiles/DirectCompactDoublesSketch.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.datasketches.quantiles;

import static org.apache.datasketches.quantiles.ClassicUtil.DOUBLES_SER_VER;
import static org.apache.datasketches.quantiles.ClassicUtil.checkFamilyID;
import static org.apache.datasketches.quantiles.ClassicUtil.checkK;
import static org.apache.datasketches.quantiles.ClassicUtil.computeBaseBufferItems;
import static org.apache.datasketches.quantiles.ClassicUtil.computeBitPattern;
import static org.apache.datasketches.quantiles.ClassicUtil.computeRetainedItems;
import static org.apache.datasketches.quantiles.PreambleUtil.COMBINED_BUFFER;
import static org.apache.datasketches.quantiles.PreambleUtil.COMPACT_FLAG_MASK;
import static org.apache.datasketches.quantiles.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.quantiles.PreambleUtil.MAX_DOUBLE;
import static org.apache.datasketches.quantiles.PreambleUtil.MIN_DOUBLE;
import static org.apache.datasketches.quantiles.PreambleUtil.N_LONG;
import static org.apache.datasketches.quantiles.PreambleUtil.ORDERED_FLAG_MASK;
import static org.apache.datasketches.quantiles.PreambleUtil.READ_ONLY_FLAG_MASK;
import static org.apache.datasketches.quantiles.PreambleUtil.extractFamilyID;
import static org.apache.datasketches.quantiles.PreambleUtil.extractFlags;
import static org.apache.datasketches.quantiles.PreambleUtil.extractK;
import static org.apache.datasketches.quantiles.PreambleUtil.extractN;
import static org.apache.datasketches.quantiles.PreambleUtil.extractPreLongs;
import static org.apache.datasketches.quantiles.PreambleUtil.extractSerVer;
import static org.apache.datasketches.quantiles.PreambleUtil.insertFamilyID;
import static org.apache.datasketches.quantiles.PreambleUtil.insertFlags;
import static org.apache.datasketches.quantiles.PreambleUtil.insertK;
import static org.apache.datasketches.quantiles.PreambleUtil.insertMaxDouble;
import static org.apache.datasketches.quantiles.PreambleUtil.insertMinDouble;
import static org.apache.datasketches.quantiles.PreambleUtil.insertN;
import static org.apache.datasketches.quantiles.PreambleUtil.insertPreLongs;
import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer;

import java.util.Arrays;

import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.quantilescommon.QuantilesAPI;

/**
 * Implements the DoublesSketch off-heap.
 *
 * @author Kevin Lang
 * @author Lee Rhodes
 * @author Jon Malkin
 */
final class DirectCompactDoublesSketch extends CompactDoublesSketch {
  private static final int MIN_DIRECT_DOUBLES_SER_VER = 3;
  private WritableMemory mem_;

  //**CONSTRUCTORS**********************************************************
  private DirectCompactDoublesSketch(final int k) {
    super(k); //Checks k
  }

  /**
   * Converts the given UpdateDoublesSketch to this compact form.
   *
   * @param sketch the sketch to convert
   * @param dstMem the WritableMemory to use for the destination
   * @return a DirectCompactDoublesSketch created from an UpdateDoublesSketch
   */
  static DirectCompactDoublesSketch createFromUpdateSketch(final UpdateDoublesSketch sketch,
                                                           final WritableMemory dstMem) {
    final long memCap = dstMem.getCapacity();
    final int k = sketch.getK();
    final long n = sketch.getN();
    checkDirectMemCapacity(k, n, memCap);

    //initialize dstMem
    dstMem.putLong(0, 0L); //clear pre0
    insertPreLongs(dstMem, 2);
    insertSerVer(dstMem, DOUBLES_SER_VER);
    insertFamilyID(dstMem, Family.QUANTILES.getID());
    insertK(dstMem, k);

    final int flags = COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK; // true for all compact sketches

    if (sketch.isEmpty()) {
      insertFlags(dstMem, flags | EMPTY_FLAG_MASK);
    } else {
      insertFlags(dstMem, flags);
      insertN(dstMem, n);
      insertMinDouble(dstMem, sketch.getMinItem());
      insertMaxDouble(dstMem, sketch.getMaxItem());

      final int bbCount = computeBaseBufferItems(k, n);

      final DoublesSketchAccessor inputAccessor = DoublesSketchAccessor.wrap(sketch);
      assert bbCount == inputAccessor.numItems();

      long dstMemOffset = COMBINED_BUFFER;

      // copy and sort base buffer
      final double[] bbArray = inputAccessor.getArray(0, bbCount);
      Arrays.sort(bbArray);
      dstMem.putDoubleArray(dstMemOffset, bbArray, 0, bbCount);
      dstMemOffset += bbCount << 3;

      long bitPattern = computeBitPattern(k, n);
      for (int lvl = 0; bitPattern > 0; ++lvl, bitPattern >>>= 1) {
        if ((bitPattern & 1L) > 0L) {
          inputAccessor.setLevel(lvl);
          dstMem.putDoubleArray(dstMemOffset, inputAccessor.getArray(0, k), 0, k);
          dstMemOffset += k << 3;
        }
      }
    }

    final DirectCompactDoublesSketch dcds = new DirectCompactDoublesSketch(k);
    dcds.mem_ = dstMem;

    return dcds;
  }

  /**
   * Wrap this sketch around the given compact Memory image of a DoublesSketch.
   *
   * @param srcMem the given compact Memory image of a DoublesSketch that may have data,
   * @return a sketch that wraps the given srcMem
   */
  static DirectCompactDoublesSketch wrapInstance(final Memory srcMem) {
    final long memCap = srcMem.getCapacity();

    final int preLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final int familyID = extractFamilyID(srcMem);
    final int flags = extractFlags(srcMem);
    final int k = extractK(srcMem);

    final boolean empty = (flags & EMPTY_FLAG_MASK) > 0;
    final long n = empty ? 0 : extractN(srcMem);

    //VALIDITY CHECKS
    DirectUpdateDoublesSketchR.checkPreLongs(preLongs);
    checkFamilyID(familyID);
    DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER);
    checkCompact(serVer, flags);
    checkK(k);
    checkDirectMemCapacity(k, n, memCap);
    DirectUpdateDoublesSketchR.checkEmptyAndN(empty, n);

    final DirectCompactDoublesSketch dds = new DirectCompactDoublesSketch(k);
    dds.mem_ = (WritableMemory) srcMem;
    return dds;
  }

  @Override
  public double getMaxItem() {
    if (isEmpty()) { throw new IllegalArgumentException(QuantilesAPI.EMPTY_MSG); }
    return mem_.getDouble(MAX_DOUBLE);
  }

  @Override
  public double getMinItem() {
    if (isEmpty()) { throw new IllegalArgumentException(QuantilesAPI.EMPTY_MSG); }
    return mem_.getDouble(MIN_DOUBLE);
  }

  @Override
  public long getN() {
    return (mem_.getCapacity() < COMBINED_BUFFER) ? 0 : mem_.getLong(N_LONG);
  }

  @Override
  public boolean hasMemory() {
    return (mem_ != null);
  }

  @Override
  public boolean isDirect() {
    return (mem_ != null) ? mem_.isDirect() : false;
  }

  @Override
  public boolean isSameResource(final Memory that) {
    return mem_.isSameResource(that);
  }

  //Restricted overrides
  //Gets

  @Override
  int getBaseBufferCount() {
    return computeBaseBufferItems(getK(), getN());
  }

  @Override
  int getCombinedBufferItemCapacity() {
    return ((int)mem_.getCapacity() - COMBINED_BUFFER) / 8;
  }

  @Override
  double[] getCombinedBuffer() {
    final int k = getK();
    if (isEmpty()) { return new double[k << 1]; } //2K
    final long n = getN();
    final int itemCap = computeRetainedItems(k, n);
    final double[] combinedBuffer = new double[itemCap];
    mem_.getDoubleArray(COMBINED_BUFFER, combinedBuffer, 0, itemCap);
    return combinedBuffer;
  }

  @Override
  long getBitPattern() {
    final int k = getK();
    final long n = getN();
    return computeBitPattern(k, n);
  }

  @Override
  WritableMemory getMemory() {
    return mem_;
  }

  //Checks

  /**
   * Checks the validity of the direct memory capacity assuming n, k.
   * @param k the given k
   * @param n the given n
   * @param memCapBytes the current memory capacity in bytes
   */
  static void checkDirectMemCapacity(final int k, final long n, final long memCapBytes) {
    final int reqBufBytes = getCompactSerialiedSizeBytes(k, n);

    if (memCapBytes < reqBufBytes) {
      throw new SketchesArgumentException("Possible corruption: Memory capacity too small: "
          + memCapBytes + " < " + reqBufBytes);
    }
  }

  /**
   * Checks a sketch's serial version and flags to see if the sketch can be wrapped as a
   * DirectCompactDoubleSketch. Throws an exception if the sketch is neither empty nor compact
   * and ordered, unles the sketch uses serialization version 2.
   * @param serVer the serialization version
   * @param flags Flags from the sketch to evaluate
   */
  static void checkCompact(final int serVer, final int flags) {
    final int compactFlagMask = COMPACT_FLAG_MASK | ORDERED_FLAG_MASK;
    if ((serVer != 2)
            && ((flags & EMPTY_FLAG_MASK) == 0)
            && ((flags & compactFlagMask) != compactFlagMask)) {
      throw new SketchesArgumentException(
              "Possible corruption: Must be v2, empty, or compact and ordered. Flags field: "
                      + Integer.toBinaryString(flags) + ", SerVer: " + serVer);
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one
3	* or more contributor license agreements. See the NOTICE file
4	* distributed with this work for additional information
5	* regarding copyright ownership. The ASF licenses this file
6	* to you under the Apache License, Version 2.0 (the
7	* "License"); you may not use this file except in compliance
8	* with the License. You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing,
13	* software distributed under the License is distributed on an
14	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15	* KIND, either express or implied. See the License for the
16	* specific language governing permissions and limitations
17	* under the License.
18	*/
19
20	package org.apache.datasketches.quantiles;
21
22	import static org.apache.datasketches.quantiles.ClassicUtil.DOUBLES_SER_VER;
23	import static org.apache.datasketches.quantiles.ClassicUtil.checkFamilyID;
24	import static org.apache.datasketches.quantiles.ClassicUtil.checkK;
25	import static org.apache.datasketches.quantiles.ClassicUtil.computeBaseBufferItems;
26	import static org.apache.datasketches.quantiles.ClassicUtil.computeBitPattern;
27	import static org.apache.datasketches.quantiles.ClassicUtil.computeRetainedItems;
28	import static org.apache.datasketches.quantiles.PreambleUtil.COMBINED_BUFFER;
29	import static org.apache.datasketches.quantiles.PreambleUtil.COMPACT_FLAG_MASK;
30	import static org.apache.datasketches.quantiles.PreambleUtil.EMPTY_FLAG_MASK;
31	import static org.apache.datasketches.quantiles.PreambleUtil.MAX_DOUBLE;
32	import static org.apache.datasketches.quantiles.PreambleUtil.MIN_DOUBLE;
33	import static org.apache.datasketches.quantiles.PreambleUtil.N_LONG;
34	import static org.apache.datasketches.quantiles.PreambleUtil.ORDERED_FLAG_MASK;
35	import static org.apache.datasketches.quantiles.PreambleUtil.READ_ONLY_FLAG_MASK;
36	import static org.apache.datasketches.quantiles.PreambleUtil.extractFamilyID;
37	import static org.apache.datasketches.quantiles.PreambleUtil.extractFlags;
38	import static org.apache.datasketches.quantiles.PreambleUtil.extractK;
39	import static org.apache.datasketches.quantiles.PreambleUtil.extractN;
40	import static org.apache.datasketches.quantiles.PreambleUtil.extractPreLongs;
41	import static org.apache.datasketches.quantiles.PreambleUtil.extractSerVer;
42	import static org.apache.datasketches.quantiles.PreambleUtil.insertFamilyID;
43	import static org.apache.datasketches.quantiles.PreambleUtil.insertFlags;
44	import static org.apache.datasketches.quantiles.PreambleUtil.insertK;
45	import static org.apache.datasketches.quantiles.PreambleUtil.insertMaxDouble;
46	import static org.apache.datasketches.quantiles.PreambleUtil.insertMinDouble;
47	import static org.apache.datasketches.quantiles.PreambleUtil.insertN;
48	import static org.apache.datasketches.quantiles.PreambleUtil.insertPreLongs;
49	import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer;
50
51	import java.util.Arrays;
52
53	import org.apache.datasketches.common.Family;
54	import org.apache.datasketches.common.SketchesArgumentException;
55	import org.apache.datasketches.memory.Memory;
56	import org.apache.datasketches.memory.WritableMemory;
57	import org.apache.datasketches.quantilescommon.QuantilesAPI;
58
59	/**
60	* Implements the DoublesSketch off-heap.
61	*
62	* @author Kevin Lang
63	* @author Lee Rhodes
64	* @author Jon Malkin
65	*/
66	final class DirectCompactDoublesSketch extends CompactDoublesSketch {	1✔
67	private static final int MIN_DIRECT_DOUBLES_SER_VER = 3;
68	private WritableMemory mem_;
69
70	//CONSTRUCTORS********************************************************
71	private DirectCompactDoublesSketch(final int k) {
72	super(k); //Checks k	1✔
73	}	1✔
74
75	/**
76	* Converts the given UpdateDoublesSketch to this compact form.
77	*
78	* @param sketch the sketch to convert
79	* @param dstMem the WritableMemory to use for the destination
80	* @return a DirectCompactDoublesSketch created from an UpdateDoublesSketch
81	*/
82	static DirectCompactDoublesSketch createFromUpdateSketch(final UpdateDoublesSketch sketch,
83	final WritableMemory dstMem) {
84	final long memCap = dstMem.getCapacity();	1✔
85	final int k = sketch.getK();	1✔
86	final long n = sketch.getN();	1✔
87	checkDirectMemCapacity(k, n, memCap);	1✔
88
89	//initialize dstMem
90	dstMem.putLong(0, 0L); //clear pre0	1✔
91	insertPreLongs(dstMem, 2);	1✔
92	insertSerVer(dstMem, DOUBLES_SER_VER);	1✔
93	insertFamilyID(dstMem, Family.QUANTILES.getID());	1✔
94	insertK(dstMem, k);	1✔
95
96	final int flags = COMPACT_FLAG_MASK \| READ_ONLY_FLAG_MASK; // true for all compact sketches	1✔
97
98	if (sketch.isEmpty()) {	1✔
99	insertFlags(dstMem, flags \| EMPTY_FLAG_MASK);	1✔
100	} else {
101	insertFlags(dstMem, flags);	1✔
102	insertN(dstMem, n);	1✔
103	insertMinDouble(dstMem, sketch.getMinItem());	1✔
104	insertMaxDouble(dstMem, sketch.getMaxItem());	1✔
105
106	final int bbCount = computeBaseBufferItems(k, n);	1✔
107
108	final DoublesSketchAccessor inputAccessor = DoublesSketchAccessor.wrap(sketch);	1✔
109	assert bbCount == inputAccessor.numItems();	1✔
110
111	long dstMemOffset = COMBINED_BUFFER;	1✔
112
113	// copy and sort base buffer
114	final double[] bbArray = inputAccessor.getArray(0, bbCount);	1✔
115	Arrays.sort(bbArray);	1✔
116	dstMem.putDoubleArray(dstMemOffset, bbArray, 0, bbCount);	1✔
117	dstMemOffset += bbCount << 3;	1✔
118
119	long bitPattern = computeBitPattern(k, n);	1✔
120	for (int lvl = 0; bitPattern > 0; ++lvl, bitPattern >>>= 1) {	1✔
121	if ((bitPattern & 1L) > 0L) {	1✔
122	inputAccessor.setLevel(lvl);	1✔
123	dstMem.putDoubleArray(dstMemOffset, inputAccessor.getArray(0, k), 0, k);	1✔
124	dstMemOffset += k << 3;	1✔
125	}
126	}
127	}
128
129	final DirectCompactDoublesSketch dcds = new DirectCompactDoublesSketch(k);	1✔
130	dcds.mem_ = dstMem;	1✔
131
132	return dcds;	1✔
133	}
134
135	/**
136	* Wrap this sketch around the given compact Memory image of a DoublesSketch.
137	*
138	* @param srcMem the given compact Memory image of a DoublesSketch that may have data,
139	* @return a sketch that wraps the given srcMem
140	*/
141	static DirectCompactDoublesSketch wrapInstance(final Memory srcMem) {
142	final long memCap = srcMem.getCapacity();	1✔
143
144	final int preLongs = extractPreLongs(srcMem);	1✔
145	final int serVer = extractSerVer(srcMem);	1✔
146	final int familyID = extractFamilyID(srcMem);	1✔
147	final int flags = extractFlags(srcMem);	1✔
148	final int k = extractK(srcMem);	1✔
149
150	final boolean empty = (flags & EMPTY_FLAG_MASK) > 0;	1✔
151	final long n = empty ? 0 : extractN(srcMem);	1✔
152
153	//VALIDITY CHECKS
154	DirectUpdateDoublesSketchR.checkPreLongs(preLongs);	1✔
155	checkFamilyID(familyID);	1✔
156	DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER);	1✔
157	checkCompact(serVer, flags);	1✔
158	checkK(k);	1✔
159	checkDirectMemCapacity(k, n, memCap);	1✔
160	DirectUpdateDoublesSketchR.checkEmptyAndN(empty, n);	1✔
161
162	final DirectCompactDoublesSketch dds = new DirectCompactDoublesSketch(k);	1✔
163	dds.mem_ = (WritableMemory) srcMem;	1✔
164	return dds;	1✔
165	}
166
167	@Override
168	public double getMaxItem() {
169	if (isEmpty()) { throw new IllegalArgumentException(QuantilesAPI.EMPTY_MSG); }	1✔
170	return mem_.getDouble(MAX_DOUBLE);	1✔
171	}
172
173	@Override
174	public double getMinItem() {
175	if (isEmpty()) { throw new IllegalArgumentException(QuantilesAPI.EMPTY_MSG); }	1✔
176	return mem_.getDouble(MIN_DOUBLE);	1✔
177	}
178
179	@Override
180	public long getN() {
181	return (mem_.getCapacity() < COMBINED_BUFFER) ? 0 : mem_.getLong(N_LONG);	1✔
182	}
183
184	@Override
185	public boolean hasMemory() {
186	return (mem_ != null);	1✔
187	}
188
189	@Override
190	public boolean isDirect() {
191	return (mem_ != null) ? mem_.isDirect() : false;	×
192	}
193
194	@Override
195	public boolean isSameResource(final Memory that) {
196	return mem_.isSameResource(that);	1✔
197	}
198
199	//Restricted overrides
200	//Gets
201
202	@Override
203	int getBaseBufferCount() {
204	return computeBaseBufferItems(getK(), getN());	1✔
205	}
206
207	@Override
208	int getCombinedBufferItemCapacity() {
209	return ((int)mem_.getCapacity() - COMBINED_BUFFER) / 8;	1✔
210	}
211
212	@Override
213	double[] getCombinedBuffer() {
214	final int k = getK();	1✔
215	if (isEmpty()) { return new double[k << 1]; } //2K	1✔
216	final long n = getN();	1✔
217	final int itemCap = computeRetainedItems(k, n);	1✔
218	final double[] combinedBuffer = new double[itemCap];	1✔
219	mem_.getDoubleArray(COMBINED_BUFFER, combinedBuffer, 0, itemCap);	1✔
220	return combinedBuffer;	1✔
221	}
222
223	@Override
224	long getBitPattern() {
225	final int k = getK();	1✔
226	final long n = getN();	1✔
227	return computeBitPattern(k, n);	1✔
228	}
229
230	@Override
231	WritableMemory getMemory() {
232	return mem_;	1✔
233	}
234
235	//Checks
236
237	/**
238	* Checks the validity of the direct memory capacity assuming n, k.
239	* @param k the given k
240	* @param n the given n
241	* @param memCapBytes the current memory capacity in bytes
242	*/
243	static void checkDirectMemCapacity(final int k, final long n, final long memCapBytes) {
244	final int reqBufBytes = getCompactSerialiedSizeBytes(k, n);	1✔
245
246	if (memCapBytes < reqBufBytes) {	1✔
247	throw new SketchesArgumentException("Possible corruption: Memory capacity too small: "	1✔
248	+ memCapBytes + " < " + reqBufBytes);
249	}
250	}	1✔
251
252	/**
253	* Checks a sketch's serial version and flags to see if the sketch can be wrapped as a
254	* DirectCompactDoubleSketch. Throws an exception if the sketch is neither empty nor compact
255	* and ordered, unles the sketch uses serialization version 2.
256	* @param serVer the serialization version
257	* @param flags Flags from the sketch to evaluate
258	*/
259	static void checkCompact(final int serVer, final int flags) {
260	final int compactFlagMask = COMPACT_FLAG_MASK \| ORDERED_FLAG_MASK;	1✔
261	if ((serVer != 2)	1✔
262	&& ((flags & EMPTY_FLAG_MASK) == 0)
263	&& ((flags & compactFlagMask) != compactFlagMask)) {
264	throw new SketchesArgumentException(	1✔
265	"Possible corruption: Must be v2, empty, or compact and ordered. Flags field: "
266	+ Integer.toBinaryString(flags) + ", SerVer: " + serVer);	1✔
267	}
268	}	1✔
269	}

apache / datasketches-java / #306

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous