#180

Build # #180

Build Type

push

Committed by

Commit Message

<a href="https://github.com/apache/datasketches-java/commit/<a class=hub.com/apache/datasketches-java/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/datasketches-java/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/datasketches-java/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/datasketches-java/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/datasketches-java/commit/3f5dc3804691e21d300c3a24da4bfc0edacc3c59">3f5dc3804">&lt;a href=&quot;https://github.com/apache/datasketches-java/commit/</a><a class="double-link" href="https://github.com/apache/datasketches-java/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/datasketches-java/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/datasketches-java/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/datasketches-java/commit/3f5dc3804691e21d300c3a24da4bfc0edacc3c59">3f5dc3804</a><a href="https://github.com/apache/datasketches-java/commit/3f5dc3804691e21d300c3a24da4bfc0edacc3c59">&lt;a href=&quot;https://github.com/apache/datasketches-java/commit/3f5dc3804691e21d300c3a24da4bfc0edacc3c59&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/datasketches-java/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/datasketches-java/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/apache/datasketches-java/commit/3f5dc3804691e21d300c3a24da4bfc0edacc3c59&amp;quot;&amp;gt;3f5dc3804&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/apache/datasketches-java/commit/3f5dc3804691e21d300c3a24da4bfc0edacc3c59&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/datasketches-java/commit/3f5dc3804691e21d300c3a24da4bfc0e&amp;lt;/a&amp;gt;dacc3c59&amp;quo</a>t;&gt;&amp;quot;&amp;gt;Merge &amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/datasketches-java/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/datasketches-java/commit/&lt;a class=&quot;double-link&quot; hre... (continued)

Run Details

1521 of 1521 new or added lines in 16 files covered. (100.0%)

19122 of 19448 relevant lines covered (98.32%)

0.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.92

/src/main/java/org/apache/datasketches/kll/MemoryValidate.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.datasketches.kll;

import static org.apache.datasketches.Family.idToFamily;
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_DOUBLE;
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_FLOAT;
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_SINGLE_ITEM;
import static org.apache.datasketches.kll.KllPreambleUtil.DEFAULT_M;
import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_DOUBLE;
import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_EMPTY_SINGLE;
import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_FLOAT;
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_EMPTY_FULL;
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_SINGLE;
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_UPDATABLE;
import static org.apache.datasketches.kll.KllPreambleUtil.extractDoubleSketchFlag;
import static org.apache.datasketches.kll.KllPreambleUtil.extractDyMinK;
import static org.apache.datasketches.kll.KllPreambleUtil.extractEmptyFlag;
import static org.apache.datasketches.kll.KllPreambleUtil.extractFamilyID;
import static org.apache.datasketches.kll.KllPreambleUtil.extractFlags;
import static org.apache.datasketches.kll.KllPreambleUtil.extractK;
import static org.apache.datasketches.kll.KllPreambleUtil.extractLevelZeroSortedFlag;
import static org.apache.datasketches.kll.KllPreambleUtil.extractM;
import static org.apache.datasketches.kll.KllPreambleUtil.extractN;
import static org.apache.datasketches.kll.KllPreambleUtil.extractNumLevels;
import static org.apache.datasketches.kll.KllPreambleUtil.extractPreInts;
import static org.apache.datasketches.kll.KllPreambleUtil.extractSerVer;
import static org.apache.datasketches.kll.KllPreambleUtil.extractSingleItemFlag;
import static org.apache.datasketches.kll.KllPreambleUtil.extractUpdatableFlag;

import org.apache.datasketches.Family;
import org.apache.datasketches.SketchesArgumentException;
import org.apache.datasketches.kll.KllPreambleUtil.Layout;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;

/**
 * This class performs all the error checking of an incoming Memory object and extracts the key fields in the process.
 * This is used by all sketches that read or import Memory objects.
 *
 * @author lrhodes
 *
 */
final class MemoryValidate {
  // first 8 bytes
  final int preInts; // = extractPreInts(srcMem);
  final int serVer;
  final int familyID;
  final String famName;
  final int flags;
  boolean empty;
  boolean singleItem;
  final boolean level0Sorted;
  final boolean doublesSketch;
  final boolean updatable;
  final int k;
  final int m;
  final int memCapacity;

  Layout layout;
  // depending on the layout, the next 8-16 bytes of the preamble, may be filled with assumed values.
  // For example, if the layout is compact & empty, n = 0, if compact and single, n = 1, etc.
  long n;
  // next 4 bytes
  int dyMinK;
  int numLevels;
  // derived
  int capacityItems; //capacity of Items array for exporting and for Updatable form
  int itemsRetained; //actual items retained in Compact form
  int itemsArrStart;
  int sketchBytes;
  Memory levelsArrCompact; //if sk = empty or single, this is derived
  Memory minMaxArrCompact; //if sk = empty or single, this is derived
  Memory itemsArrCompact;  //if sk = empty or single, this is derived
  WritableMemory levelsArrUpdatable;
  WritableMemory minMaxArrUpdatable;
  WritableMemory itemsArrUpdatable;

  MemoryValidate(final Memory srcMem) {
    memCapacity = (int) srcMem.getCapacity();
    preInts = extractPreInts(srcMem);
    serVer = extractSerVer(srcMem);

    familyID = extractFamilyID(srcMem);
    if (familyID != Family.KLL.getID()) { memoryValidateThrow(0, familyID); }
    famName = idToFamily(familyID).toString();
    flags = extractFlags(srcMem);
    empty = extractEmptyFlag(srcMem);
    level0Sorted  = extractLevelZeroSortedFlag(srcMem);
    singleItem    = extractSingleItemFlag(srcMem);
    doublesSketch = extractDoubleSketchFlag(srcMem);
    updatable    = extractUpdatableFlag(srcMem);
    k = extractK(srcMem);
    KllHelper.checkK(k);
    m = extractM(srcMem);
    if (m != DEFAULT_M) { memoryValidateThrow(7, m); }
    if ((serVer == SERIAL_VERSION_UPDATABLE) ^ updatable) { memoryValidateThrow(10, 0); }

    if (updatable) { updatableMemoryValidate((WritableMemory) srcMem); }
    else { compactMemoryValidate(srcMem); }
  }

  void compactMemoryValidate(final Memory srcMem) {
    if (empty && singleItem) { memoryValidateThrow(20, 0); }
    final int sw = (empty ? 1 : 0) | (singleItem ? 4 : 0) | (doublesSketch ? 8 : 0);
    switch (sw) {
      case 0: { //Float Compact FULL
        if (preInts != PREAMBLE_INTS_FLOAT) { memoryValidateThrow(6, preInts); }
        if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }

        layout = Layout.FLOAT_FULL_COMPACT;
        n = extractN(srcMem);
        dyMinK = extractDyMinK(srcMem);
        numLevels = extractNumLevels(srcMem);
        int offset = DATA_START_ADR_FLOAT;
        // LEVELS MEM
        final int[] myLevelsArr = new int[numLevels + 1];
        srcMem.getIntArray(offset, myLevelsArr, 0, numLevels); //copies all except the last one
        myLevelsArr[numLevels] = KllHelper.computeTotalItemCapacity(k, m, numLevels); //load the last one
        levelsArrCompact = Memory.wrap(myLevelsArr); //separate from srcMem,
        offset += (int)levelsArrCompact.getCapacity() - Integer.BYTES; // but one larger than srcMem
        // MIN/MAX MEM
        minMaxArrCompact = srcMem.region(offset, 2L * Float.BYTES);
        offset += (int)minMaxArrCompact.getCapacity();
        // ITEMS MEM
        itemsArrStart = offset;
        capacityItems = myLevelsArr[numLevels];
        itemsRetained = capacityItems - myLevelsArr[0];
        final float[] myItemsArr = new float[capacityItems];
        srcMem.getFloatArray(offset, myItemsArr, myLevelsArr[0], itemsRetained);
        itemsArrCompact = Memory.wrap(myItemsArr);
        sketchBytes = offset + itemsRetained * Float.BYTES;
        break;
      }
      case 1: { //Float Compact EMPTY
        if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }
        if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }
        layout = Layout.FLOAT_EMPTY_COMPACT;
        n = 0;           //assumed
        dyMinK = k;      //assumed
        numLevels = 1;   //assumed
        // LEVELS MEM
        levelsArrCompact = Memory.wrap(new int[] {k, k});
        // MIN/MAX MEM
        minMaxArrCompact = Memory.wrap(new float[] {Float.NaN, Float.NaN});
        // ITEMS MEM
        capacityItems = k;
        itemsRetained = 0;
        itemsArrCompact = Memory.wrap(new float[k]);
        sketchBytes = DATA_START_ADR_SINGLE_ITEM; //also used for empty
        itemsArrStart = DATA_START_ADR_SINGLE_ITEM;
        break;
      }
      case 4: { //Float Compact SINGLE
        if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }
        if (serVer != SERIAL_VERSION_SINGLE) { memoryValidateThrow(4, serVer); }
        layout = Layout.FLOAT_SINGLE_COMPACT;
        n = 1;
        dyMinK = k;
        numLevels = 1;
        // LEVELS MEM
        levelsArrCompact = Memory.wrap(new int[] {k - 1, k});
        final float minMax = srcMem.getFloat(DATA_START_ADR_SINGLE_ITEM);
        // MIN/MAX MEM
        minMaxArrCompact = Memory.wrap(new float[] {minMax, minMax});
        // ITEMS MEM
        capacityItems = k;
        itemsRetained = 1;
        final float[] myFloatItems = new float[k];
        myFloatItems[k - 1] = minMax;
        itemsArrCompact = Memory.wrap(myFloatItems);
        sketchBytes = DATA_START_ADR_SINGLE_ITEM + Float.BYTES;
        itemsArrStart = DATA_START_ADR_SINGLE_ITEM;
        break;
      }
      case 8: { //Double Compact FULL
        if (preInts != PREAMBLE_INTS_DOUBLE) { memoryValidateThrow(5, preInts); }
        if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }
        layout = Layout.DOUBLE_FULL_COMPACT;
        n = extractN(srcMem);
        dyMinK = extractDyMinK(srcMem);
        numLevels = extractNumLevels(srcMem);
        int offset = DATA_START_ADR_DOUBLE;
        // LEVELS MEM
        final int[] myLevelsArr = new int[numLevels + 1];
        srcMem.getIntArray(offset, myLevelsArr, 0, numLevels); //all except the last one
        myLevelsArr[numLevels] = KllHelper.computeTotalItemCapacity(k, m, numLevels); //load the last one
        levelsArrCompact = Memory.wrap(myLevelsArr); //separate from srcMem
        offset += (int)levelsArrCompact.getCapacity() - Integer.BYTES;
        // MIN/MAX MEM
        minMaxArrCompact = srcMem.region(offset, 2L * Double.BYTES);
        offset += (int)minMaxArrCompact.getCapacity();
        // ITEMS MEM
        itemsArrStart = offset;
        capacityItems = myLevelsArr[numLevels];
        itemsRetained = capacityItems - myLevelsArr[0];
        final double[] myItemsArr = new double[capacityItems];
        srcMem.getDoubleArray(offset, myItemsArr, myLevelsArr[0], itemsRetained);
        itemsArrCompact = Memory.wrap(myItemsArr);
        sketchBytes = offset + itemsRetained * Double.BYTES;
        break;
      }
      case 9: { //Double Compact EMPTY
        if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }
        if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }
        layout = Layout.DOUBLE_EMPTY_COMPACT;
        n = 0;
        dyMinK = k;
        numLevels = 1;

        // LEVELS MEM
        levelsArrCompact = Memory.wrap(new int[] {k, k});
        // MIN/MAX MEM
        minMaxArrCompact = Memory.wrap(new double[] {Double.NaN, Double.NaN});
        // ITEMS MEM
        capacityItems = k;
        itemsRetained = 0;
        itemsArrCompact = Memory.wrap(new double[k]);
        sketchBytes = DATA_START_ADR_SINGLE_ITEM; //also used for empty
        itemsArrStart = DATA_START_ADR_SINGLE_ITEM;
        break;
      }
      case 12: { //Double Compact SINGLE
        if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }
        if (serVer != SERIAL_VERSION_SINGLE) { memoryValidateThrow(4, serVer); }
        layout = Layout.DOUBLE_SINGLE_COMPACT;
        n = 1;
        dyMinK = k;
        numLevels = 1;

        // LEVELS MEM
        levelsArrCompact = Memory.wrap(new int[] {k - 1, k});
        final double minMax = srcMem.getDouble(DATA_START_ADR_SINGLE_ITEM);
        // MIN/MAX MEM
        minMaxArrCompact = Memory.wrap(new double[] {minMax, minMax});
        // ITEMS MEM
        capacityItems = k;
        itemsRetained = 1;
        final double[] myDoubleItems = new double[k];
        myDoubleItems[k - 1] = minMax;
        itemsArrCompact = Memory.wrap(myDoubleItems);
        sketchBytes = DATA_START_ADR_SINGLE_ITEM + Double.BYTES;
        itemsArrStart = DATA_START_ADR_SINGLE_ITEM;
        break;
      }
      default: break; //can't happen
    }
  }

  void updatableMemoryValidate(final WritableMemory wSrcMem) {
    if (doublesSketch) { //Double Updatable FULL
      if (preInts != PREAMBLE_INTS_DOUBLE) { memoryValidateThrow(5, preInts); }
      layout = Layout.DOUBLE_UPDATABLE;
      n = extractN(wSrcMem);
      empty = n == 0;       //empty & singleItem are set for convenience
      singleItem = n == 1;  // there is no error checking on these bits
      dyMinK = extractDyMinK(wSrcMem);
      numLevels = extractNumLevels(wSrcMem);

      int offset = DATA_START_ADR_DOUBLE;
      //LEVELS
      levelsArrUpdatable = wSrcMem.writableRegion(offset, (numLevels + 1L) * Integer.BYTES);
      offset += (int)levelsArrUpdatable.getCapacity();
      //MIN/MAX
      minMaxArrUpdatable = wSrcMem.writableRegion(offset, 2L * Double.BYTES);
      offset += (int)minMaxArrUpdatable.getCapacity();
      //ITEMS
      capacityItems = levelsArrUpdatable.getInt((long)numLevels * Integer.BYTES);
      final int itemsArrBytes = capacityItems * Double.BYTES;
      itemsArrStart = offset;
      itemsArrStart = memCapacity - itemsArrBytes;
      if (itemsArrStart < offset) { memoryValidateThrow(24, offset - itemsArrStart); }
      itemsArrUpdatable = wSrcMem.writableRegion(itemsArrStart, itemsArrBytes);
      sketchBytes = itemsArrStart + itemsArrBytes;
    }
    else { //Float Updatable FULL
      if (preInts != PREAMBLE_INTS_FLOAT) { memoryValidateThrow(6, preInts); }
      layout = Layout.FLOAT_UPDATABLE;
      n = extractN(wSrcMem);
      empty = n == 0;       //empty & singleItem are set for convenience
      singleItem = n == 1;  // there is no error checking on these bits
      dyMinK = extractDyMinK(wSrcMem);
      numLevels = extractNumLevels(wSrcMem);
      int offset = DATA_START_ADR_FLOAT;
      //LEVELS
      levelsArrUpdatable = wSrcMem.writableRegion(offset, (numLevels + 1L) * Integer.BYTES);
      offset += (int)levelsArrUpdatable.getCapacity();
      //MIN/MAX
      minMaxArrUpdatable = wSrcMem.writableRegion(offset, 2L * Float.BYTES);
      offset += (int)minMaxArrUpdatable.getCapacity();
      //ITEMS
      capacityItems = levelsArrUpdatable.getInt((long)numLevels * Integer.BYTES);
      final int itemsArrBytes = capacityItems * Float.BYTES;
      itemsArrStart = offset;
      itemsArrStart = memCapacity - itemsArrBytes;
      if (itemsArrStart < offset) { memoryValidateThrow(24, offset - itemsArrStart); }
      itemsArrUpdatable = wSrcMem.writableRegion(itemsArrStart, itemsArrBytes);
      sketchBytes = itemsArrStart + itemsArrBytes;
    }
  }

  private static void memoryValidateThrow(final int errNo, final int value) {
    String msg = "";
    switch (errNo) {
      case 0: msg = "FamilyID Field must be: " + Family.KLL.getID() + ", NOT: " + value; break;
      case 1: msg = "Empty Bit: 1 -> PreInts: " + PREAMBLE_INTS_EMPTY_SINGLE + ", NOT: " + value; break;
      case 2: msg = "Empty Bit: 1 -> SerVer: " + SERIAL_VERSION_EMPTY_FULL + ", NOT: " + value; break;
      //case 3: msg = "Single Item Bit: 1 -> PreInts: " + PREAMBLE_INTS_EMPTY_SINGLE + ", NOT: " + value; break;
      case 4: msg = "Single Item Bit: 1 -> SerVer: " + SERIAL_VERSION_SINGLE + ", NOT: " + value; break;
      case 5: msg = "Double Sketch Bit: 1 -> PreInts: " + PREAMBLE_INTS_DOUBLE + ", NOT: " + value; break;
      case 6: msg = "Double Sketch Bit: 0 -> PreInts: " + PREAMBLE_INTS_FLOAT + ", NOT: " + value; break;
      case 7: msg = "The M field must be set to " + DEFAULT_M + ", NOT: " + value; break;
      //case 8: msg = "The dynamic MinK must be equal to K, NOT: " + value; break;
      //case 9: msg = "numLevels must be one, NOT: " + value; break;
      case 10: msg = "((SerVer == 3) ^ (Updatable Bit)) must = 0."; break;
      case 20: msg = "Empty flag bit and SingleItem flag bit cannot both be set. Flags: " + value; break;
      //case 21: msg = "N != 0 and empty bit is set. N: " + value; break;
      //case 22: msg = "N != 1 and single item bit is set. N: " + value; break;
      //case 23: msg = "Family name is not KLL"; break;
      case 24: msg = "Given Memory has insufficient capacity. Need " + value + " bytes."; break;
      default: msg = "Unknown error: errNo: " + errNo; break;
    }
    throw new SketchesArgumentException(msg);
  }

}


1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one
3	* or more contributor license agreements. See the NOTICE file
4	* distributed with this work for additional information
5	* regarding copyright ownership. The ASF licenses this file
6	* to you under the Apache License, Version 2.0 (the
7	* "License"); you may not use this file except in compliance
8	* with the License. You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing,
13	* software distributed under the License is distributed on an
14	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15	* KIND, either express or implied. See the License for the
16	* specific language governing permissions and limitations
17	* under the License.
18	*/
19
20	package org.apache.datasketches.kll;
21
22	import static org.apache.datasketches.Family.idToFamily;
23	import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_DOUBLE;
24	import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_FLOAT;
25	import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_SINGLE_ITEM;
26	import static org.apache.datasketches.kll.KllPreambleUtil.DEFAULT_M;
27	import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_DOUBLE;
28	import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_EMPTY_SINGLE;
29	import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_FLOAT;
30	import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_EMPTY_FULL;
31	import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_SINGLE;
32	import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_UPDATABLE;
33	import static org.apache.datasketches.kll.KllPreambleUtil.extractDoubleSketchFlag;
34	import static org.apache.datasketches.kll.KllPreambleUtil.extractDyMinK;
35	import static org.apache.datasketches.kll.KllPreambleUtil.extractEmptyFlag;
36	import static org.apache.datasketches.kll.KllPreambleUtil.extractFamilyID;
37	import static org.apache.datasketches.kll.KllPreambleUtil.extractFlags;
38	import static org.apache.datasketches.kll.KllPreambleUtil.extractK;
39	import static org.apache.datasketches.kll.KllPreambleUtil.extractLevelZeroSortedFlag;
40	import static org.apache.datasketches.kll.KllPreambleUtil.extractM;
41	import static org.apache.datasketches.kll.KllPreambleUtil.extractN;
42	import static org.apache.datasketches.kll.KllPreambleUtil.extractNumLevels;
43	import static org.apache.datasketches.kll.KllPreambleUtil.extractPreInts;
44	import static org.apache.datasketches.kll.KllPreambleUtil.extractSerVer;
45	import static org.apache.datasketches.kll.KllPreambleUtil.extractSingleItemFlag;
46	import static org.apache.datasketches.kll.KllPreambleUtil.extractUpdatableFlag;
47
48	import org.apache.datasketches.Family;
49	import org.apache.datasketches.SketchesArgumentException;
50	import org.apache.datasketches.kll.KllPreambleUtil.Layout;
51	import org.apache.datasketches.memory.Memory;
52	import org.apache.datasketches.memory.WritableMemory;
53
54	/**
55	* This class performs all the error checking of an incoming Memory object and extracts the key fields in the process.
56	* This is used by all sketches that read or import Memory objects.
57	*
58	* @author lrhodes
59	*
60	*/
61	final class MemoryValidate {
62	// first 8 bytes
63	final int preInts; // = extractPreInts(srcMem);
64	final int serVer;
65	final int familyID;
66	final String famName;
67	final int flags;
68	boolean empty;
69	boolean singleItem;
70	final boolean level0Sorted;
71	final boolean doublesSketch;
72	final boolean updatable;
73	final int k;
74	final int m;
75	final int memCapacity;
76
77	Layout layout;
78	// depending on the layout, the next 8-16 bytes of the preamble, may be filled with assumed values.
79	// For example, if the layout is compact & empty, n = 0, if compact and single, n = 1, etc.
80	long n;
81	// next 4 bytes
82	int dyMinK;
83	int numLevels;
84	// derived
85	int capacityItems; //capacity of Items array for exporting and for Updatable form
86	int itemsRetained; //actual items retained in Compact form
87	int itemsArrStart;
88	int sketchBytes;
89	Memory levelsArrCompact; //if sk = empty or single, this is derived
90	Memory minMaxArrCompact; //if sk = empty or single, this is derived
91	Memory itemsArrCompact; //if sk = empty or single, this is derived
92	WritableMemory levelsArrUpdatable;
93	WritableMemory minMaxArrUpdatable;
94	WritableMemory itemsArrUpdatable;
95
96	MemoryValidate(final Memory srcMem) {	1✔
97	memCapacity = (int) srcMem.getCapacity();	1✔
98	preInts = extractPreInts(srcMem);	1✔
99	serVer = extractSerVer(srcMem);	1✔
100
101	familyID = extractFamilyID(srcMem);	1✔
102	if (familyID != Family.KLL.getID()) { memoryValidateThrow(0, familyID); }	1✔
103	famName = idToFamily(familyID).toString();	1✔
104	flags = extractFlags(srcMem);	1✔
105	empty = extractEmptyFlag(srcMem);	1✔
106	level0Sorted = extractLevelZeroSortedFlag(srcMem);	1✔
107	singleItem = extractSingleItemFlag(srcMem);	1✔
108	doublesSketch = extractDoubleSketchFlag(srcMem);	1✔
109	updatable = extractUpdatableFlag(srcMem);	1✔
110	k = extractK(srcMem);	1✔
111	KllHelper.checkK(k);	1✔
112	m = extractM(srcMem);	1✔
113	if (m != DEFAULT_M) { memoryValidateThrow(7, m); }	1✔
114	if ((serVer == SERIAL_VERSION_UPDATABLE) ^ updatable) { memoryValidateThrow(10, 0); }	1✔
115
116	if (updatable) { updatableMemoryValidate((WritableMemory) srcMem); }	1✔
117	else { compactMemoryValidate(srcMem); }	1✔
118	}	1✔
119
120	void compactMemoryValidate(final Memory srcMem) {
121	if (empty && singleItem) { memoryValidateThrow(20, 0); }	1✔
122	final int sw = (empty ? 1 : 0) \| (singleItem ? 4 : 0) \| (doublesSketch ? 8 : 0);	1✔
123	switch (sw) {	1✔
124	case 0: { //Float Compact FULL
125	if (preInts != PREAMBLE_INTS_FLOAT) { memoryValidateThrow(6, preInts); }	1✔
126	if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }	1✔
127
128	layout = Layout.FLOAT_FULL_COMPACT;	1✔
129	n = extractN(srcMem);	1✔
130	dyMinK = extractDyMinK(srcMem);	1✔
131	numLevels = extractNumLevels(srcMem);	1✔
132	int offset = DATA_START_ADR_FLOAT;	1✔
133	// LEVELS MEM
134	final int[] myLevelsArr = new int[numLevels + 1];	1✔
135	srcMem.getIntArray(offset, myLevelsArr, 0, numLevels); //copies all except the last one	1✔
136	myLevelsArr[numLevels] = KllHelper.computeTotalItemCapacity(k, m, numLevels); //load the last one	1✔
137	levelsArrCompact = Memory.wrap(myLevelsArr); //separate from srcMem,	1✔
138	offset += (int)levelsArrCompact.getCapacity() - Integer.BYTES; // but one larger than srcMem	1✔
139	// MIN/MAX MEM
140	minMaxArrCompact = srcMem.region(offset, 2L * Float.BYTES);	1✔
141	offset += (int)minMaxArrCompact.getCapacity();	1✔
142	// ITEMS MEM
143	itemsArrStart = offset;	1✔
144	capacityItems = myLevelsArr[numLevels];	1✔
145	itemsRetained = capacityItems - myLevelsArr[0];	1✔
146	final float[] myItemsArr = new float[capacityItems];	1✔
147	srcMem.getFloatArray(offset, myItemsArr, myLevelsArr[0], itemsRetained);	1✔
148	itemsArrCompact = Memory.wrap(myItemsArr);	1✔
149	sketchBytes = offset + itemsRetained * Float.BYTES;	1✔
150	break;	1✔
151	}
152	case 1: { //Float Compact EMPTY
153	if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }	1✔
154	if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }	1✔
155	layout = Layout.FLOAT_EMPTY_COMPACT;	1✔
156	n = 0; //assumed	1✔
157	dyMinK = k; //assumed	1✔
158	numLevels = 1; //assumed	1✔
159	// LEVELS MEM
160	levelsArrCompact = Memory.wrap(new int[] {k, k});	1✔
161	// MIN/MAX MEM
162	minMaxArrCompact = Memory.wrap(new float[] {Float.NaN, Float.NaN});	1✔
163	// ITEMS MEM
164	capacityItems = k;	1✔
165	itemsRetained = 0;	1✔
166	itemsArrCompact = Memory.wrap(new float[k]);	1✔
167	sketchBytes = DATA_START_ADR_SINGLE_ITEM; //also used for empty	1✔
168	itemsArrStart = DATA_START_ADR_SINGLE_ITEM;	1✔
169	break;	1✔
170	}
171	case 4: { //Float Compact SINGLE
172	if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }	1✔
173	if (serVer != SERIAL_VERSION_SINGLE) { memoryValidateThrow(4, serVer); }	1✔
174	layout = Layout.FLOAT_SINGLE_COMPACT;	1✔
175	n = 1;	1✔
176	dyMinK = k;	1✔
177	numLevels = 1;	1✔
178	// LEVELS MEM
179	levelsArrCompact = Memory.wrap(new int[] {k - 1, k});	1✔
180	final float minMax = srcMem.getFloat(DATA_START_ADR_SINGLE_ITEM);	1✔
181	// MIN/MAX MEM
182	minMaxArrCompact = Memory.wrap(new float[] {minMax, minMax});	1✔
183	// ITEMS MEM
184	capacityItems = k;	1✔
185	itemsRetained = 1;	1✔
186	final float[] myFloatItems = new float[k];	1✔
187	myFloatItems[k - 1] = minMax;	1✔
188	itemsArrCompact = Memory.wrap(myFloatItems);	1✔
189	sketchBytes = DATA_START_ADR_SINGLE_ITEM + Float.BYTES;	1✔
190	itemsArrStart = DATA_START_ADR_SINGLE_ITEM;	1✔
191	break;	1✔
192	}
193	case 8: { //Double Compact FULL
194	if (preInts != PREAMBLE_INTS_DOUBLE) { memoryValidateThrow(5, preInts); }	1✔
195	if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }	1✔
196	layout = Layout.DOUBLE_FULL_COMPACT;	1✔
197	n = extractN(srcMem);	1✔
198	dyMinK = extractDyMinK(srcMem);	1✔
199	numLevels = extractNumLevels(srcMem);	1✔
200	int offset = DATA_START_ADR_DOUBLE;	1✔
201	// LEVELS MEM
202	final int[] myLevelsArr = new int[numLevels + 1];	1✔
203	srcMem.getIntArray(offset, myLevelsArr, 0, numLevels); //all except the last one	1✔
204	myLevelsArr[numLevels] = KllHelper.computeTotalItemCapacity(k, m, numLevels); //load the last one	1✔
205	levelsArrCompact = Memory.wrap(myLevelsArr); //separate from srcMem	1✔
206	offset += (int)levelsArrCompact.getCapacity() - Integer.BYTES;	1✔
207	// MIN/MAX MEM
208	minMaxArrCompact = srcMem.region(offset, 2L * Double.BYTES);	1✔
209	offset += (int)minMaxArrCompact.getCapacity();	1✔
210	// ITEMS MEM
211	itemsArrStart = offset;	1✔
212	capacityItems = myLevelsArr[numLevels];	1✔
213	itemsRetained = capacityItems - myLevelsArr[0];	1✔
214	final double[] myItemsArr = new double[capacityItems];	1✔
215	srcMem.getDoubleArray(offset, myItemsArr, myLevelsArr[0], itemsRetained);	1✔
216	itemsArrCompact = Memory.wrap(myItemsArr);	1✔
217	sketchBytes = offset + itemsRetained * Double.BYTES;	1✔
218	break;	1✔
219	}
220	case 9: { //Double Compact EMPTY
221	if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }	1✔
222	if (serVer != SERIAL_VERSION_EMPTY_FULL) { memoryValidateThrow(2, serVer); }	1✔
223	layout = Layout.DOUBLE_EMPTY_COMPACT;	1✔
224	n = 0;	1✔
225	dyMinK = k;	1✔
226	numLevels = 1;	1✔
227
228	// LEVELS MEM
229	levelsArrCompact = Memory.wrap(new int[] {k, k});	1✔
230	// MIN/MAX MEM
231	minMaxArrCompact = Memory.wrap(new double[] {Double.NaN, Double.NaN});	1✔
232	// ITEMS MEM
233	capacityItems = k;	1✔
234	itemsRetained = 0;	1✔
235	itemsArrCompact = Memory.wrap(new double[k]);	1✔
236	sketchBytes = DATA_START_ADR_SINGLE_ITEM; //also used for empty	1✔
237	itemsArrStart = DATA_START_ADR_SINGLE_ITEM;	1✔
238	break;	1✔
239	}
240	case 12: { //Double Compact SINGLE
241	if (preInts != PREAMBLE_INTS_EMPTY_SINGLE) { memoryValidateThrow(1, preInts); }	1✔
242	if (serVer != SERIAL_VERSION_SINGLE) { memoryValidateThrow(4, serVer); }	1✔
243	layout = Layout.DOUBLE_SINGLE_COMPACT;	1✔
244	n = 1;	1✔
245	dyMinK = k;	1✔
246	numLevels = 1;	1✔
247
248	// LEVELS MEM
249	levelsArrCompact = Memory.wrap(new int[] {k - 1, k});	1✔
250	final double minMax = srcMem.getDouble(DATA_START_ADR_SINGLE_ITEM);	1✔
251	// MIN/MAX MEM
252	minMaxArrCompact = Memory.wrap(new double[] {minMax, minMax});	1✔
253	// ITEMS MEM
254	capacityItems = k;	1✔
255	itemsRetained = 1;	1✔
256	final double[] myDoubleItems = new double[k];	1✔
257	myDoubleItems[k - 1] = minMax;	1✔
258	itemsArrCompact = Memory.wrap(myDoubleItems);	1✔
259	sketchBytes = DATA_START_ADR_SINGLE_ITEM + Double.BYTES;	1✔
260	itemsArrStart = DATA_START_ADR_SINGLE_ITEM;	1✔
261	break;	1✔
262	}
263	default: break; //can't happen
264	}
265	}	1✔
266
267	void updatableMemoryValidate(final WritableMemory wSrcMem) {
268	if (doublesSketch) { //Double Updatable FULL	1✔
269	if (preInts != PREAMBLE_INTS_DOUBLE) { memoryValidateThrow(5, preInts); }	1✔
270	layout = Layout.DOUBLE_UPDATABLE;	1✔
271	n = extractN(wSrcMem);	1✔
272	empty = n == 0; //empty & singleItem are set for convenience	1✔
273	singleItem = n == 1; // there is no error checking on these bits	1✔
274	dyMinK = extractDyMinK(wSrcMem);	1✔
275	numLevels = extractNumLevels(wSrcMem);	1✔
276
277	int offset = DATA_START_ADR_DOUBLE;	1✔
278	//LEVELS
279	levelsArrUpdatable = wSrcMem.writableRegion(offset, (numLevels + 1L) * Integer.BYTES);	1✔
280	offset += (int)levelsArrUpdatable.getCapacity();	1✔
281	//MIN/MAX
282	minMaxArrUpdatable = wSrcMem.writableRegion(offset, 2L * Double.BYTES);	1✔
283	offset += (int)minMaxArrUpdatable.getCapacity();	1✔
284	//ITEMS
285	capacityItems = levelsArrUpdatable.getInt((long)numLevels * Integer.BYTES);	1✔
286	final int itemsArrBytes = capacityItems * Double.BYTES;	1✔
287	itemsArrStart = offset;	1✔
288	itemsArrStart = memCapacity - itemsArrBytes;	1✔
289	if (itemsArrStart < offset) { memoryValidateThrow(24, offset - itemsArrStart); }	1✔
290	itemsArrUpdatable = wSrcMem.writableRegion(itemsArrStart, itemsArrBytes);	1✔
291	sketchBytes = itemsArrStart + itemsArrBytes;	1✔
292	}	1✔
293	else { //Float Updatable FULL
294	if (preInts != PREAMBLE_INTS_FLOAT) { memoryValidateThrow(6, preInts); }	1✔
295	layout = Layout.FLOAT_UPDATABLE;	1✔
296	n = extractN(wSrcMem);	1✔
297	empty = n == 0; //empty & singleItem are set for convenience	1✔
298	singleItem = n == 1; // there is no error checking on these bits	1✔
299	dyMinK = extractDyMinK(wSrcMem);	1✔
300	numLevels = extractNumLevels(wSrcMem);	1✔
301	int offset = DATA_START_ADR_FLOAT;	1✔
302	//LEVELS
303	levelsArrUpdatable = wSrcMem.writableRegion(offset, (numLevels + 1L) * Integer.BYTES);	1✔
304	offset += (int)levelsArrUpdatable.getCapacity();	1✔
305	//MIN/MAX
306	minMaxArrUpdatable = wSrcMem.writableRegion(offset, 2L * Float.BYTES);	1✔
307	offset += (int)minMaxArrUpdatable.getCapacity();	1✔
308	//ITEMS
309	capacityItems = levelsArrUpdatable.getInt((long)numLevels * Integer.BYTES);	1✔
310	final int itemsArrBytes = capacityItems * Float.BYTES;	1✔
311	itemsArrStart = offset;	1✔
312	itemsArrStart = memCapacity - itemsArrBytes;	1✔
313	if (itemsArrStart < offset) { memoryValidateThrow(24, offset - itemsArrStart); }	1✔
314	itemsArrUpdatable = wSrcMem.writableRegion(itemsArrStart, itemsArrBytes);	1✔
315	sketchBytes = itemsArrStart + itemsArrBytes;	1✔
316	}
317	}	1✔
318
319	private static void memoryValidateThrow(final int errNo, final int value) {
320	String msg = "";	1✔
321	switch (errNo) {	1✔
322	case 0: msg = "FamilyID Field must be: " + Family.KLL.getID() + ", NOT: " + value; break;	1✔
323	case 1: msg = "Empty Bit: 1 -> PreInts: " + PREAMBLE_INTS_EMPTY_SINGLE + ", NOT: " + value; break;	1✔
324	case 2: msg = "Empty Bit: 1 -> SerVer: " + SERIAL_VERSION_EMPTY_FULL + ", NOT: " + value; break;	1✔
325	//case 3: msg = "Single Item Bit: 1 -> PreInts: " + PREAMBLE_INTS_EMPTY_SINGLE + ", NOT: " + value; break;
326	case 4: msg = "Single Item Bit: 1 -> SerVer: " + SERIAL_VERSION_SINGLE + ", NOT: " + value; break;	1✔
327	case 5: msg = "Double Sketch Bit: 1 -> PreInts: " + PREAMBLE_INTS_DOUBLE + ", NOT: " + value; break;	1✔
328	case 6: msg = "Double Sketch Bit: 0 -> PreInts: " + PREAMBLE_INTS_FLOAT + ", NOT: " + value; break;	1✔
329	case 7: msg = "The M field must be set to " + DEFAULT_M + ", NOT: " + value; break;	1✔
330	//case 8: msg = "The dynamic MinK must be equal to K, NOT: " + value; break;
331	//case 9: msg = "numLevels must be one, NOT: " + value; break;
332	case 10: msg = "((SerVer == 3) ^ (Updatable Bit)) must = 0."; break;	1✔
333	case 20: msg = "Empty flag bit and SingleItem flag bit cannot both be set. Flags: " + value; break;	1✔
334	//case 21: msg = "N != 0 and empty bit is set. N: " + value; break;
335	//case 22: msg = "N != 1 and single item bit is set. N: " + value; break;
336	//case 23: msg = "Family name is not KLL"; break;
337	case 24: msg = "Given Memory has insufficient capacity. Need " + value + " bytes."; break;	×
338	default: msg = "Unknown error: errNo: " + errNo; break;	×
339	}
340	throw new SketchesArgumentException(msg);	1✔
341	}
342
343	}
344

apache / datasketches-java / #180

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous