#306

Committed 30 Apr 2024 10:01PM UTC coverage: 97.645% (-0.5%) from 98.139%

Build # #306

Build Type

push

Committed by

Commit Message

Merge pull request #555 from apache/fix_pom_xml_header

Fix pom xml header

Run Details

26865 of 27513 relevant lines covered (97.64%)

0.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.02

/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.datasketches.quantiles;

import static org.apache.datasketches.quantiles.ClassicUtil.DOUBLES_SER_VER;
import static org.apache.datasketches.quantiles.ClassicUtil.checkFamilyID;
import static org.apache.datasketches.quantiles.ClassicUtil.checkK;
import static org.apache.datasketches.quantiles.ClassicUtil.computeBitPattern;
import static org.apache.datasketches.quantiles.PreambleUtil.COMBINED_BUFFER;
import static org.apache.datasketches.quantiles.PreambleUtil.EMPTY_FLAG_MASK;
import static org.apache.datasketches.quantiles.PreambleUtil.FLAGS_BYTE;
import static org.apache.datasketches.quantiles.PreambleUtil.MAX_DOUBLE;
import static org.apache.datasketches.quantiles.PreambleUtil.MIN_DOUBLE;
import static org.apache.datasketches.quantiles.PreambleUtil.N_LONG;
import static org.apache.datasketches.quantiles.PreambleUtil.extractFamilyID;
import static org.apache.datasketches.quantiles.PreambleUtil.extractFlags;
import static org.apache.datasketches.quantiles.PreambleUtil.extractK;
import static org.apache.datasketches.quantiles.PreambleUtil.extractN;
import static org.apache.datasketches.quantiles.PreambleUtil.extractPreLongs;
import static org.apache.datasketches.quantiles.PreambleUtil.extractSerVer;
import static org.apache.datasketches.quantiles.PreambleUtil.insertFamilyID;
import static org.apache.datasketches.quantiles.PreambleUtil.insertFlags;
import static org.apache.datasketches.quantiles.PreambleUtil.insertK;
import static org.apache.datasketches.quantiles.PreambleUtil.insertMaxDouble;
import static org.apache.datasketches.quantiles.PreambleUtil.insertMinDouble;
import static org.apache.datasketches.quantiles.PreambleUtil.insertN;
import static org.apache.datasketches.quantiles.PreambleUtil.insertPreLongs;
import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer;

import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;

/**
 * Implements the DoublesSketch off-heap.
 *
 * @author Kevin Lang
 * @author Lee Rhodes
 *
 */
final class DirectUpdateDoublesSketch extends DirectUpdateDoublesSketchR {
  MemoryRequestServer memReqSvr = null;

  private DirectUpdateDoublesSketch(final int k) {
    super(k); //Checks k
  }

  /**
   * Obtains a new Direct instance of a DoublesSketch, which may be off-heap.
   *
   * @param k Parameter that controls space usage of sketch and accuracy of estimates.
   * Must be greater than 1 and less than 65536 and a power of 2.
   * @param dstMem the destination Memory that will be initialized to hold the data for this sketch.
   * It must initially be at least (16 * MIN_K + 32) bytes, where MIN_K defaults to 2. As it grows
   * it will request more memory using the MemoryRequest callback.
   * @return a DirectUpdateDoublesSketch
   */
  static DirectUpdateDoublesSketch newInstance(final int k, final WritableMemory dstMem) {
    // must be able to hold at least an empty sketch
    final long memCap = dstMem.getCapacity();
    checkDirectMemCapacity(k, 0, memCap);

    //initialize dstMem
    dstMem.putLong(0, 0L); //clear pre0
    insertPreLongs(dstMem, 2);
    insertSerVer(dstMem, DOUBLES_SER_VER);
    insertFamilyID(dstMem, Family.QUANTILES.getID());
    insertFlags(dstMem, EMPTY_FLAG_MASK);
    insertK(dstMem, k);

    if (memCap >= COMBINED_BUFFER) {
      insertN(dstMem, 0L);
      insertMinDouble(dstMem, Double.NaN);
      insertMaxDouble(dstMem, Double.NaN);
    }

    final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k);
    dds.mem_ = dstMem;
    return dds;
  }

  /**
   * Wrap this sketch around the given non-compact Memory image of a DoublesSketch.
   *
   * @param srcMem the given non-compact Memory image of a DoublesSketch that may have data
   * @return a sketch that wraps the given srcMem
   */
  static DirectUpdateDoublesSketch wrapInstance(final WritableMemory srcMem) {
    final long memCap = srcMem.getCapacity();

    final int preLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final int familyID = extractFamilyID(srcMem);
    final int flags = extractFlags(srcMem);
    final int k = extractK(srcMem);

    final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; //Preamble flags empty state
    final long n = empty ? 0 : extractN(srcMem);

    //VALIDITY CHECKS
    checkPreLongs(preLongs);
    checkFamilyID(familyID);
    DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER);
    checkDirectFlags(flags); //Cannot be compact
    checkK(k);
    checkCompact(serVer, flags);
    checkDirectMemCapacity(k, n, memCap);
    checkEmptyAndN(empty, n);

    final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k);
    dds.mem_ = srcMem;
    return dds;
  }

  @Override
  public boolean isReadOnly() {
    return false;
  }

  @Override
  public void update(final double dataItem) {
    if (Double.isNaN(dataItem)) { return; }

    final int curBBCount = getBaseBufferCount();
    final int newBBCount = curBBCount + 1; //derived, not stored

    //must check memory capacity before we put anything in it
    final int combBufItemCap = getCombinedBufferItemCapacity();
    if (newBBCount > combBufItemCap) {
      //only changes combinedBuffer when it is only a base buffer
      mem_ = growCombinedMemBuffer(2 * getK());
    }

    final long curN = getN();
    final long newN = curN + 1;

    if (curN == 0) { //set min and max quantiles
      putMaxItem(dataItem);
      putMinItem(dataItem);
    } else {
      if (dataItem > getMaxItem()) { putMaxItem(dataItem); }
      if (dataItem < getMinItem()) { putMinItem(dataItem); }
    }

    mem_.putDouble(COMBINED_BUFFER + ((long) curBBCount * Double.BYTES), dataItem); //put the item
    mem_.putByte(FLAGS_BYTE, (byte) 0); //not compact, not ordered, not empty

    if (newBBCount == (2 * k_)) { //Propagate
      // make sure there will be enough levels for the propagation
      final int curMemItemCap = getCombinedBufferItemCapacity();
      final int itemSpaceNeeded = DoublesUpdateImpl.getRequiredItemCapacity(k_, newN);

      //check mem has capacity to accommodate new level
      if (itemSpaceNeeded > curMemItemCap) {
        // copies base buffer plus old levels, adds space for new level
        mem_ = growCombinedMemBuffer(itemSpaceNeeded);
      }

      // sort base buffer via accessor which modifies the underlying base buffer,
      // then use as one of the inputs to propagate-carry
      final DoublesSketchAccessor bbAccessor = DoublesSketchAccessor.wrap(this, true);
      bbAccessor.sort();

      final long newBitPattern = DoublesUpdateImpl.inPlacePropagateCarry(
              0, // starting level
              null,
              bbAccessor,
              true,
              k_,
              DoublesSketchAccessor.wrap(this, true),
              getBitPattern()
      );

      assert newBitPattern == computeBitPattern(k_, newN); // internal consistency check
      //bit pattern on direct is always derived, no need to save it.
    }
    putN(newN);
    doublesSV = null;
  }

  @Override
  public void reset() {
    if (mem_.getCapacity() >= COMBINED_BUFFER) {
      mem_.putByte(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); //not compact, not ordered
      mem_.putLong(N_LONG, 0L);
      mem_.putDouble(MIN_DOUBLE, Double.NaN);
      mem_.putDouble(MAX_DOUBLE, Double.NaN);
    }
  }

  //Restricted overrides
  //Puts

  @Override
  void putMinItem(final double minQuantile) {
    assert (mem_.getCapacity() >= COMBINED_BUFFER);
    mem_.putDouble(MIN_DOUBLE, minQuantile);
  }

  @Override
  void putMaxItem(final double maxQuantile) {
    assert (mem_.getCapacity() >= COMBINED_BUFFER);
    mem_.putDouble(MAX_DOUBLE, maxQuantile);
  }

  @Override
  void putN(final long n) {
    assert (mem_.getCapacity() >= COMBINED_BUFFER);
    mem_.putLong(N_LONG, n);
  }

  @Override
  void putCombinedBuffer(final double[] combinedBuffer) {
    mem_.putDoubleArray(COMBINED_BUFFER, combinedBuffer, 0, combinedBuffer.length);
  }

  @Override
  void putBaseBufferCount(final int baseBufferCount) {
    //intentionally a no-op, not kept on-heap, always derived.
  }

  @Override
  void putBitPattern(final long bitPattern) {
    //intentionally a no-op, not kept on-heap, always derived.
  }

  @Override
  double[] growCombinedBuffer(final int curCombBufItemCap, final int itemSpaceNeeded) {
    mem_ = growCombinedMemBuffer(itemSpaceNeeded);
    // copy out any data that was there
    final double[] newCombBuf = new double[itemSpaceNeeded];
    mem_.getDoubleArray(COMBINED_BUFFER, newCombBuf, 0, curCombBufItemCap);
    return newCombBuf;
  }

  //Direct supporting methods

  private WritableMemory growCombinedMemBuffer(final int itemSpaceNeeded) {
    final long memBytes = mem_.getCapacity();
    final int needBytes = (itemSpaceNeeded << 3) + COMBINED_BUFFER; //+ preamble + min & max
    assert needBytes > memBytes;

    memReqSvr = (memReqSvr == null) ? mem_.getMemoryRequestServer() : memReqSvr;
    if (memReqSvr == null) {
      throw new SketchesArgumentException(
          "A request for more memory has been denied, "
          + "or a default MemoryRequestServer has not been provided. Must abort. ");
    }

    final WritableMemory newMem = memReqSvr.request(mem_, needBytes);

    mem_.copyTo(0, newMem, 0, memBytes);

    memReqSvr.requestClose(mem_, newMem);

    return newMem;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one
3	* or more contributor license agreements. See the NOTICE file
4	* distributed with this work for additional information
5	* regarding copyright ownership. The ASF licenses this file
6	* to you under the Apache License, Version 2.0 (the
7	* "License"); you may not use this file except in compliance
8	* with the License. You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing,
13	* software distributed under the License is distributed on an
14	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15	* KIND, either express or implied. See the License for the
16	* specific language governing permissions and limitations
17	* under the License.
18	*/
19
20	package org.apache.datasketches.quantiles;
21
22	import static org.apache.datasketches.quantiles.ClassicUtil.DOUBLES_SER_VER;
23	import static org.apache.datasketches.quantiles.ClassicUtil.checkFamilyID;
24	import static org.apache.datasketches.quantiles.ClassicUtil.checkK;
25	import static org.apache.datasketches.quantiles.ClassicUtil.computeBitPattern;
26	import static org.apache.datasketches.quantiles.PreambleUtil.COMBINED_BUFFER;
27	import static org.apache.datasketches.quantiles.PreambleUtil.EMPTY_FLAG_MASK;
28	import static org.apache.datasketches.quantiles.PreambleUtil.FLAGS_BYTE;
29	import static org.apache.datasketches.quantiles.PreambleUtil.MAX_DOUBLE;
30	import static org.apache.datasketches.quantiles.PreambleUtil.MIN_DOUBLE;
31	import static org.apache.datasketches.quantiles.PreambleUtil.N_LONG;
32	import static org.apache.datasketches.quantiles.PreambleUtil.extractFamilyID;
33	import static org.apache.datasketches.quantiles.PreambleUtil.extractFlags;
34	import static org.apache.datasketches.quantiles.PreambleUtil.extractK;
35	import static org.apache.datasketches.quantiles.PreambleUtil.extractN;
36	import static org.apache.datasketches.quantiles.PreambleUtil.extractPreLongs;
37	import static org.apache.datasketches.quantiles.PreambleUtil.extractSerVer;
38	import static org.apache.datasketches.quantiles.PreambleUtil.insertFamilyID;
39	import static org.apache.datasketches.quantiles.PreambleUtil.insertFlags;
40	import static org.apache.datasketches.quantiles.PreambleUtil.insertK;
41	import static org.apache.datasketches.quantiles.PreambleUtil.insertMaxDouble;
42	import static org.apache.datasketches.quantiles.PreambleUtil.insertMinDouble;
43	import static org.apache.datasketches.quantiles.PreambleUtil.insertN;
44	import static org.apache.datasketches.quantiles.PreambleUtil.insertPreLongs;
45	import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer;
46
47	import org.apache.datasketches.common.Family;
48	import org.apache.datasketches.common.SketchesArgumentException;
49	import org.apache.datasketches.memory.MemoryRequestServer;
50	import org.apache.datasketches.memory.WritableMemory;
51
52	/**
53	* Implements the DoublesSketch off-heap.
54	*
55	* @author Kevin Lang
56	* @author Lee Rhodes
57	*
58	*/
59	final class DirectUpdateDoublesSketch extends DirectUpdateDoublesSketchR {	1✔
60	MemoryRequestServer memReqSvr = null;	1✔
61
62	private DirectUpdateDoublesSketch(final int k) {
63	super(k); //Checks k	1✔
64	}	1✔
65
66	/**
67	* Obtains a new Direct instance of a DoublesSketch, which may be off-heap.
68	*
69	* @param k Parameter that controls space usage of sketch and accuracy of estimates.
70	* Must be greater than 1 and less than 65536 and a power of 2.
71	* @param dstMem the destination Memory that will be initialized to hold the data for this sketch.
72	* It must initially be at least (16 * MIN_K + 32) bytes, where MIN_K defaults to 2. As it grows
73	* it will request more memory using the MemoryRequest callback.
74	* @return a DirectUpdateDoublesSketch
75	*/
76	static DirectUpdateDoublesSketch newInstance(final int k, final WritableMemory dstMem) {
77	// must be able to hold at least an empty sketch
78	final long memCap = dstMem.getCapacity();	1✔
79	checkDirectMemCapacity(k, 0, memCap);	1✔
80
81	//initialize dstMem
82	dstMem.putLong(0, 0L); //clear pre0	1✔
83	insertPreLongs(dstMem, 2);	1✔
84	insertSerVer(dstMem, DOUBLES_SER_VER);	1✔
85	insertFamilyID(dstMem, Family.QUANTILES.getID());	1✔
86	insertFlags(dstMem, EMPTY_FLAG_MASK);	1✔
87	insertK(dstMem, k);	1✔
88
89	if (memCap >= COMBINED_BUFFER) {	1✔
90	insertN(dstMem, 0L);	1✔
91	insertMinDouble(dstMem, Double.NaN);	1✔
92	insertMaxDouble(dstMem, Double.NaN);	1✔
93	}
94
95	final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k);	1✔
96	dds.mem_ = dstMem;	1✔
97	return dds;	1✔
98	}
99
100	/**
101	* Wrap this sketch around the given non-compact Memory image of a DoublesSketch.
102	*
103	* @param srcMem the given non-compact Memory image of a DoublesSketch that may have data
104	* @return a sketch that wraps the given srcMem
105	*/
106	static DirectUpdateDoublesSketch wrapInstance(final WritableMemory srcMem) {
107	final long memCap = srcMem.getCapacity();	1✔
108
109	final int preLongs = extractPreLongs(srcMem);	1✔
110	final int serVer = extractSerVer(srcMem);	1✔
111	final int familyID = extractFamilyID(srcMem);	1✔
112	final int flags = extractFlags(srcMem);	1✔
113	final int k = extractK(srcMem);	1✔
114
115	final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; //Preamble flags empty state	1✔
116	final long n = empty ? 0 : extractN(srcMem);	1✔
117
118	//VALIDITY CHECKS
119	checkPreLongs(preLongs);	1✔
120	checkFamilyID(familyID);	1✔
121	DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER);	1✔
122	checkDirectFlags(flags); //Cannot be compact	1✔
123	checkK(k);	1✔
124	checkCompact(serVer, flags);	1✔
125	checkDirectMemCapacity(k, n, memCap);	1✔
126	checkEmptyAndN(empty, n);	1✔
127
128	final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k);	1✔
129	dds.mem_ = srcMem;	1✔
130	return dds;	1✔
131	}
132
133	@Override
134	public boolean isReadOnly() {
135	return false;	×
136	}
137
138	@Override
139	public void update(final double dataItem) {
140	if (Double.isNaN(dataItem)) { return; }	1✔
141
142	final int curBBCount = getBaseBufferCount();	1✔
143	final int newBBCount = curBBCount + 1; //derived, not stored	1✔
144
145	//must check memory capacity before we put anything in it
146	final int combBufItemCap = getCombinedBufferItemCapacity();	1✔
147	if (newBBCount > combBufItemCap) {	1✔
148	//only changes combinedBuffer when it is only a base buffer
149	mem_ = growCombinedMemBuffer(2 * getK());	1✔
150	}
151
152	final long curN = getN();	1✔
153	final long newN = curN + 1;	1✔
154
155	if (curN == 0) { //set min and max quantiles	1✔
156	putMaxItem(dataItem);	1✔
157	putMinItem(dataItem);	1✔
158	} else {
159	if (dataItem > getMaxItem()) { putMaxItem(dataItem); }	1✔
160	if (dataItem < getMinItem()) { putMinItem(dataItem); }	1✔
161	}
162
163	mem_.putDouble(COMBINED_BUFFER + ((long) curBBCount * Double.BYTES), dataItem); //put the item	1✔
164	mem_.putByte(FLAGS_BYTE, (byte) 0); //not compact, not ordered, not empty	1✔
165
166	if (newBBCount == (2 * k_)) { //Propagate	1✔
167	// make sure there will be enough levels for the propagation
168	final int curMemItemCap = getCombinedBufferItemCapacity();	1✔
169	final int itemSpaceNeeded = DoublesUpdateImpl.getRequiredItemCapacity(k_, newN);	1✔
170
171	//check mem has capacity to accommodate new level
172	if (itemSpaceNeeded > curMemItemCap) {	1✔
173	// copies base buffer plus old levels, adds space for new level
174	mem_ = growCombinedMemBuffer(itemSpaceNeeded);	1✔
175	}
176
177	// sort base buffer via accessor which modifies the underlying base buffer,
178	// then use as one of the inputs to propagate-carry
179	final DoublesSketchAccessor bbAccessor = DoublesSketchAccessor.wrap(this, true);	1✔
180	bbAccessor.sort();	1✔
181
182	final long newBitPattern = DoublesUpdateImpl.inPlacePropagateCarry(	1✔
183	0, // starting level
184	null,
185	bbAccessor,
186	true,
187	k_,
188	DoublesSketchAccessor.wrap(this, true),	1✔
189	getBitPattern()	1✔
190	);
191
192	assert newBitPattern == computeBitPattern(k_, newN); // internal consistency check	1✔
193	//bit pattern on direct is always derived, no need to save it.
194	}
195	putN(newN);	1✔
196	doublesSV = null;	1✔
197	}	1✔
198
199	@Override
200	public void reset() {
201	if (mem_.getCapacity() >= COMBINED_BUFFER) {	1✔
202	mem_.putByte(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); //not compact, not ordered	1✔
203	mem_.putLong(N_LONG, 0L);	1✔
204	mem_.putDouble(MIN_DOUBLE, Double.NaN);	1✔
205	mem_.putDouble(MAX_DOUBLE, Double.NaN);	1✔
206	}
207	}	1✔
208
209	//Restricted overrides
210	//Puts
211
212	@Override
213	void putMinItem(final double minQuantile) {
214	assert (mem_.getCapacity() >= COMBINED_BUFFER);	1✔
215	mem_.putDouble(MIN_DOUBLE, minQuantile);	1✔
216	}	1✔
217
218	@Override
219	void putMaxItem(final double maxQuantile) {
220	assert (mem_.getCapacity() >= COMBINED_BUFFER);	1✔
221	mem_.putDouble(MAX_DOUBLE, maxQuantile);	1✔
222	}	1✔
223
224	@Override
225	void putN(final long n) {
226	assert (mem_.getCapacity() >= COMBINED_BUFFER);	1✔
227	mem_.putLong(N_LONG, n);	1✔
228	}	1✔
229
230	@Override
231	void putCombinedBuffer(final double[] combinedBuffer) {
232	mem_.putDoubleArray(COMBINED_BUFFER, combinedBuffer, 0, combinedBuffer.length);	1✔
233	}	1✔
234
235	@Override
236	void putBaseBufferCount(final int baseBufferCount) {
237	//intentionally a no-op, not kept on-heap, always derived.
238	}	1✔
239
240	@Override
241	void putBitPattern(final long bitPattern) {
242	//intentionally a no-op, not kept on-heap, always derived.
243	}	1✔
244
245	@Override
246	double[] growCombinedBuffer(final int curCombBufItemCap, final int itemSpaceNeeded) {
247	mem_ = growCombinedMemBuffer(itemSpaceNeeded);	1✔
248	// copy out any data that was there
249	final double[] newCombBuf = new double[itemSpaceNeeded];	1✔
250	mem_.getDoubleArray(COMBINED_BUFFER, newCombBuf, 0, curCombBufItemCap);	1✔
251	return newCombBuf;	1✔
252	}
253
254	//Direct supporting methods
255
256	private WritableMemory growCombinedMemBuffer(final int itemSpaceNeeded) {
257	final long memBytes = mem_.getCapacity();	1✔
258	final int needBytes = (itemSpaceNeeded << 3) + COMBINED_BUFFER; //+ preamble + min & max	1✔
259	assert needBytes > memBytes;	1✔
260
261	memReqSvr = (memReqSvr == null) ? mem_.getMemoryRequestServer() : memReqSvr;	1✔
262	if (memReqSvr == null) {	1✔
263	throw new SketchesArgumentException(	×
264	"A request for more memory has been denied, "
265	+ "or a default MemoryRequestServer has not been provided. Must abort. ");
266	}
267
268	final WritableMemory newMem = memReqSvr.request(mem_, needBytes);	1✔
269
270	mem_.copyTo(0, newMem, 0, memBytes);	1✔
271
272	memReqSvr.requestClose(mem_, newMem);	1✔
273
274	return newMem;	1✔
275	}
276	}

apache / datasketches-java / #306

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous