apache / carbondata / 11926

/*

 * Licensed to the Apache Software Foundation (ASF) under one or more

 * contributor license agreements.  See the NOTICE file distributed with

 * this work for additional information regarding copyright ownership.

 * The ASF licenses this file to You under the Apache License, Version 2.0

 * (the "License"); you may not use this file except in compliance with

 * the License.  You may obtain a copy of the License at

 *

 *    http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package org.apache.carbondata.processing.loading.sort.unsafe.sort;

import java.util.Comparator;

/**

 * A port of the Apache Spark's TimSort and they originally ported from Android TimSort class,

 * which utilizes a "stable, adaptive, iterative mergesort."

 * See the method comment on sort() for more details.

 *

 * This has been kept in Java with the original style in order to match very closely with the

 * Android source code, and thus be easy to verify correctness. The class is package private. We put

 * a simple Scala wrapper {@link org.apache.spark.util.collection.Sorter}, which is available to

 * package org.apache.spark.

 *

 * The purpose of the port is to generalize the interface to the sort to accept input data formats

 * besides simple arrays where every element is sorted individually. For instance, the AppendOnlyMap

 * uses this to sort an Array with alternating elements of the form [key, value, key, value].

 * This generalization comes with minimal overhead -- see SortDataFormat for more information.

 *

 * We allow key reuse to prevent creating many key objects -- see SortDataFormat.

 *

 * @see SortDataFormat

 * @see org.apache.spark.util.collection.Sorter

 */

  /**

   * This is the minimum sized sequence that will be merged.  Shorter

   * sequences will be lengthened by calling binarySort.  If the entire

   * array is less than this length, no merges will be performed.

   *

   * This constant should be a power of two.  It was 64 in Tim Peter's C

   * implementation, but 32 was empirically determined to work better in

   * this implementation.  In the unlikely event that you set this constant

   * to be a number that's not a power of two, you'll need to change the

   * minRunLength computation.

   *

   * If you decrease this constant, you must change the stackLen

   * computation in the TimSort constructor, or you risk an

   * ArrayOutOfBounds exception.  See listsort.txt for a discussion

   * of the minimum stack length required as a function of the length

   * of the array being sorted and the minimum merge sequence length.

   */

  private static final int MIN_MERGE = 32;

  private final SortDataFormat<K, Buffer> s;

  /**

   * A stable, adaptive, iterative mergesort that requires far fewer than

   * n lg(n) comparisons when running on partially sorted arrays, while

   * offering performance comparable to a traditional mergesort when run

   * on random arrays.  Like all proper mergesorts, this sort is stable and

   * runs O(n log n) time (worst case).  In the worst case, this sort requires

   * temporary storage space for n/2 object references; in the best case,

   * it requires only a small constant amount of space.

   *

   * This implementation was adapted from Tim Peters's list sort for

   * Python, which is described in detail here:

   *

   *   http://svn.python.org/projects/python/trunk/Objects/listsort.txt

   *

   * Tim's C code may be found here:

   *

   *   http://svn.python.org/projects/python/trunk/Objects/listobject.c

   *

   * The underlying techniques are described in this paper (and may have

   * even earlier origins):

   *

   *  "Optimistic Sorting and Information Theoretic Complexity"

   *  Peter McIlroy

   *  SODA (Fourth Annual ACM-SIAM Symposium on Discrete Algorithms),

   *  pp 467-474, Austin, Texas, 25-27 January 1993.

   *

   * While the API to this class consists solely of static methods, it is

   * (privately) instantiable; a TimSort instance holds the state of an ongoing

   * sort, assuming the input array is large enough to warrant the full-blown

   * TimSort. Small arrays are sorted in place, using a binary insertion sort.

   */

  public void sort(Buffer a, int lo, int hi, Comparator<? super K> c) {

    }

    // If array is small, do a "mini-TimSort" with no merges

    }

    /*

     * March over the array once, left to right, finding natural runs,

     * extending short natural runs to minRun elements, and merging runs

     * to maintain stack invariant.

     */

    do {

      // Identify next run

      // If run is short, extend to min(minRun, nRemaining)

      }

      // Push run onto pending-run stack, and maybe merge

      // Advance to find next run

    // Merge all remaining runs to complete sort

  /**

   * Sorts the specified portion of the specified array using a binary

   * insertion sort.  This is the best method for sorting small numbers

   * of elements.  It requires O(n log n) compares, but O(n^2) data

   * movement (worst case).

   *

   * If the initial part of the specified range is already sorted,

   * this method can take advantage of it: the method assumes that the

   * elements from index {@code lo}, inclusive, to {@code start},

   * exclusive are already sorted.

   *

   * @param a the array in which a range is to be sorted

   * @param lo the index of the first element in the range to be sorted

   * @param hi the index after the last element in the range to be sorted

   * @param start the index of the first element in the range that is

   *        not already known to be sorted ({@code lo <= start <= hi})

   * @param c comparator to used for the sort

   */

  @SuppressWarnings("fallthrough")

  private void binarySort(Buffer a, int lo, int hi, int start, Comparator<? super K> c) {

    }

      // Set left (and right) to the index where a[start] (pivot) belongs

      /*

       * Invariants:

       *   pivot >= all in [lo, left).

       *   pivot <  all in [right, start).

       */

        }

        else {

        }

      /*

       * The invariants still hold: pivot >= all in [lo, left) and

       * pivot < all in [left, start), so pivot belongs at left.  Note

       * that if there are elements equal to pivot, left points to the

       * first slot after them -- that's why this sort is stable.

       * Slide elements over to make room for pivot.

       */

      // Switch is just an optimization for arraycopy in default case

        case 2:  {

        }

        case 1:  {

        }

      }

    }

  /**

   * Returns the length of the run beginning at the specified position in

   * the specified array and reverses the run if it is descending (ensuring

   * that the run will always be ascending when the method returns).

   *

   * A run is the longest ascending sequence with:

   *

   *    a[lo] <= a[lo + 1] <= a[lo + 2] <= ...

   *

   * or the longest descending sequence with:

   *

   *    a[lo] >  a[lo + 1] >  a[lo + 2] >  ...

   *

   * For its intended use in a stable mergesort, the strictness of the

   * definition of "descending" is needed so that the call can safely

   * reverse a descending sequence without violating stability.

   *

   * @param a the array in which a run is to be counted and possibly reversed

   * @param lo index of the first element in the run

   * @param hi index after the last element that may be contained in the run.

  It is required that {@code lo < hi}.

   * @param c the comparator to used for the sort

   * @return  the length of the run beginning at the specified position in

   *          the specified array

   */

  private int countRunAndMakeAscending(Buffer a, int lo, int hi, Comparator<? super K> c) {

    }

    // Find end of run, and reverse range if descending

      }

    } else {                              // Ascending

      }

    }

  }

  /**

   * Reverse the specified range of the specified array.

   *

   * @param a the array in which a range is to be reversed

   * @param lo the index of the first element in the range to be reversed

   * @param hi the index after the last element in the range to be reversed

   */

  private void reverseRange(Buffer a, int lo, int hi) {

    }

  /**

   * Returns the minimum acceptable run length for an array of the specified

   * length. Natural runs shorter than this will be extended with

   * {@link #binarySort}.

   *

   * Roughly speaking, the computation is:

   *

   *  If n < MIN_MERGE, return n (it's too small to bother with fancy stuff).

   *  Else if n is an exact power of 2, return MIN_MERGE/2.

   *  Else return an int k, MIN_MERGE/2 <= k <= MIN_MERGE, such that n/k

   *   is close to, but strictly less than, an exact power of 2.

   *

   * For the rationale, see listsort.txt.

   *

   * @param n the length of the array to be sorted

   * @return the length of the minimum run to be merged

   */

  private int minRunLength(int n) {

    }

  }

    /**

     * The Buffer being sorted.

     */

    private final Buffer a;

    /**

     * Length of the sort Buffer.

     */

    private final int aLength;

    /**

     * The comparator for this sort.

     */

    private final Comparator<? super K> c;

    /**

     * When we get into galloping mode, we stay there until both runs win less

     * often than MIN_GALLOP consecutive times.

     */

    private static final int  MIN_GALLOP = 7;

    /**

     * This controls when we get *into* galloping mode.  It is initialized

     * to MIN_GALLOP.  The mergeLo and mergeHi methods nudge it higher for

     * random data, and lower for highly structured data.

     */

    /**

     * Maximum initial size of tmp array, which is used for merging.  The array

     * can grow to accommodate demand.

     *

     * Unlike Tim's original C version, we do not allocate this much storage

     * when sorting smaller arrays.  This change was required for performance.

     */

    private static final int INITIAL_TMP_STORAGE_LENGTH = 256;

    /**

     * Temp storage for merges.

     */

    private Buffer tmp; // Actual runtime type will be Object[], regardless of T

    /**

     * Length of the temp storage.

     */

    /**

     * A stack of pending runs yet to be merged.  Run i starts at

     * address base[i] and extends for len[i] elements.  It's always

     * true (so long as the indices are in bounds) that:

     *

     *     runBase[i] + runLen[i] == runBase[i + 1]

     *

     * so we could cut the storage for this, but it's a minor amount,

     * and keeping all the info explicit simplifies the code.

     */

    private final int[] runBase;

    private final int[] runLen;

    /**

     * Creates a TimSort instance to maintain the state of an ongoing sort.

     *

     * @param a the array to be sorted

     * @param c the comparator to determine the order of the sort

     */

      // Allocate temp storage (which may be increased later if necessary)

      /*

       * Allocate runs-to-be-merged stack (which cannot be expanded).  The

       * stack length requirements are described in listsort.txt.  The C

       * version always uses the same stack length (85), but this was

       * measured to be too expensive when sorting "mid-sized" arrays (e.g.,

       * 100 elements) in Java.  Therefore, we use smaller (but sufficiently

       * large) stack lengths for smaller arrays.  The "magic numbers" in the

       * computation below must be changed if MIN_MERGE is decreased.  See

       * the MIN_MERGE declaration above for more information.

       */

                      len <   1542  ? 10 :

                      len < 119151  ? 19 : 40);

    /**

     * Pushes the specified run onto the pending-run stack.

     *

     * @param runBase index of the first element in the run

     * @param runLen  the number of elements in the run

     */

    private void pushRun(int runBase, int runLen) {

    /**

     * Examines the stack of runs waiting to be merged and merges adjacent runs

     * until the stack invariants are reestablished:

     *

     *     1. runLen[i - 3] > runLen[i - 2] + runLen[i - 1]

     *     2. runLen[i - 2] > runLen[i - 1]

     *

     * This method is called each time a new run is pushed onto the stack,

     * so the invariants are guaranteed to hold for i < stackSize upon

     * entry to the method.

     */

    private void mergeCollapse() {

            || (n >= 2 && runLen[n - 2] <= runLen[n] + runLen[n - 1])) {

          }

        }

    /**

     * Merges all runs on the stack until only one remains.  This method is

     * called once, to complete the sort.

     */

    private void mergeForceCollapse() {

        }

    /**

     * Merges the two runs at stack indices i and i+1.  Run i must be

     * the penultimate or antepenultimate run on the stack.  In other words,

     * i must be equal to stackSize-2 or stackSize-3.

     *

     * @param i stack index of the first of the two runs to merge

     */

    private void mergeAt(int i) {

      /*

       * Record the length of the combined runs; if i is the 3rd-last

       * run now, also slide over the last run (which isn't involved

       * in this merge).  The current run (i+1) goes away in any case.

       */

      }

      /*

       * Find where the first element of run2 goes in run1. Prior elements

       * in run1 can be ignored (because they're already in place).

       */

      }

      /*

       * Find where the last element of run1 goes in run2. Subsequent elements

       * in run2 can be ignored (because they're already in place).

       */

      }

      // Merge remaining runs, using tmp array with min(len1, len2) elements

      }

      else {

      }

    /**

     * Locates the position at which to insert the specified key into the

     * specified sorted range; if the range contains an element equal to key,

     * returns the index of the leftmost equal element.

     *

     * @param key the key whose insertion point to search for

     * @param a the array in which to search

     * @param base the index of the first element in the range

     * @param len the length of the range; must be > 0

     * @param hint the index at which to begin the search, 0 <= hint < n.

     *     The closer hint is to the result, the faster this method will run.

     * @param c the comparator used to order the range, and to search

     * @return the int k,  0 <= k <= n such that a[b + k - 1] < key <= a[b + k],

     *    pretending that a[b - 1] is minus infinity and a[b + n] is infinity.

     *    In other words, key belongs at index b + k; or in other words,

     *    the first k elements of a should precede key, and the last n - k

     *    should follow it.

     */

    private int gallopLeft(K key, Buffer a, int base, int len, int hint, Comparator<? super K> c) {

        // Gallop right until a[base+hint+lastOfs] < key <= a[base+hint+ofs]

          }

        }

        }

        // Make offsets relative to base

        // Gallop left until a[base+hint-ofs] < key <= a[base+hint-lastOfs]

          }

        }

        }

        // Make offsets relative to base

      }

      /*

       * Now a[base+lastOfs] < key <= a[base+ofs], so key belongs somewhere

       * to the right of lastOfs but no farther right than ofs.  Do a binary

       * search, with invariant a[base + lastOfs - 1] < key <= a[base + ofs].

       */

        }

        else {

        }

    }

    /**

     * Like gallopLeft, except that if the range contains an element equal to

     * key, gallopRight returns the index after the rightmost equal element.

     *

     * @param key the key whose insertion point to search for

     * @param a the array in which to search

     * @param base the index of the first element in the range

     * @param len the length of the range; must be > 0

     * @param hint the index at which to begin the search, 0 <= hint < n.

     *     The closer hint is to the result, the faster this method will run.

     * @param c the comparator used to order the range, and to search

     * @return the int k,  0 <= k <= n such that a[b + k - 1] <= key < a[b + k]

     */

    private int gallopRight(K key, Buffer a, int base, int len, int hint, Comparator<? super K> c) {

        // Gallop left until a[b+hint - ofs] <= key < a[b+hint - lastOfs]

          }

        }

        }

        // Make offsets relative to b

        // Gallop right until a[b+hint + lastOfs] <= key < a[b+hint + ofs]

          }

        }

        }

        // Make offsets relative to b

      }

      /*

       * Now a[b + lastOfs] <= key < a[b + ofs], so key belongs somewhere to

       * the right of lastOfs but no farther right than ofs.  Do a binary

       * search, with invariant a[b + lastOfs - 1] <= key < a[b + ofs].

       */

        }

        else {

        }

    }

    /**

     * Merges two adjacent runs in place, in a stable fashion.  The first

     * element of the first run must be greater than the first element of the

     * second run (a[base1] > a[base2]), and the last element of the first run

     * (a[base1 + len1-1]) must be greater than all elements of the second run.

     *

     * For performance, this method should be called only when len1 <= len2;

     * its twin, mergeHi should be called if len1 >= len2.  (Either method

     * may be called if len1 == len2.)

     *

     * @param base1 index of first element in first run to be merged

     * @param len1  length of first run to be merged (must be > 0)

     * @param base2 index of first element in second run to be merged

     *        (must be aBase + aLen)

     * @param len2  length of second run to be merged (must be > 0)

     */

    private void mergeLo(int base1, int len1, int base2, int len2) {

      // Copy first run into temp array

      // Move first element of second run and deal with degenerate cases

      }

      }

      outer:

      while (true) {

        /*

         * Do the straightforward thing until (if ever) one run starts

         * winning consistently.

         */

        do {

            }

          } else {

            }

          }

        /*

         * One run is winning so consistently that galloping may be a

         * huge win. So try that, and continue galloping until (if ever)

         * neither run appears to be winning consistently anymore.

         */

        do {

            }

          }

          }

            }

          }

          }

        }

            "Comparison method violates its general contract!");

      } else {

      }

    /**

     * Like mergeLo, except that this method should be called only if

     * len1 >= len2; mergeLo should be called if len1 <= len2.  (Either method

     * may be called if len1 == len2.)

     *

     * @param base1 index of first element in first run to be merged

     * @param len1  length of first run to be merged (must be > 0)

     * @param base2 index of first element in second run to be merged

     *        (must be aBase + aLen)

     * @param len2  length of second run to be merged (must be > 0)

     */

    private void mergeHi(int base1, int len1, int base2, int len2) {

      // Copy second run into temp array

      // Move last element of first run and deal with degenerate cases

      }

      }

      outer:

      while (true) {

        /*

         * Do the straightforward thing until (if ever) one run

         * appears to win consistently.

         */

        do {

            }

          } else {

            }

          }

        /*

         * One run is winning so consistently that galloping may be a

         * huge win. So try that, and continue galloping until (if ever)

         * neither run appears to be winning consistently anymore.

         */

        do {

            }

          }

          }

            }

          }

          }

        }

            "Comparison method violates its general contract!");

      } else {

      }

    /**

     * Ensures that the external array tmp has at least the specified

     * number of elements, increasing its size if necessary.  The size

     * increases exponentially to ensure amortized linear time complexity.

     *

     * @param minCapacity the minimum required capacity of the tmp array

     * @return tmp, whether or not it grew

     */

    private Buffer ensureCapacity(int minCapacity) {

        // Compute smallest power of 2 > minCapacity

        }

        else {

        }