8694

Build # 8694

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/<a class="double-link" href="https://git"><a class=hub.com/apache/carbondata/commit/1e0b0ea96f90e72cf85b3428822a1dbb61fbe1d8">1e0b0ea96">&lt;a href=&quot;https://github.com/apache/carbondata/commit/</a><a class="double-link" href="https://github.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/apache/carbondata/commit/1e0b0ea96f90e72cf85b3428822a1dbb61fbe1d8">1e0b0ea96</a><a href="https://github.com/apache/carbondata/commit/1e0b0ea96f90e72cf85b3428822a1dbb61fbe1d8">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/apache/carbondata/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/apache/carbondata/commit/1e0b0ea96f90e72cf85b3428822a1dbb61fbe1d8&quot;&gt;1e0b0ea96&lt;/a&gt;&lt;a href=&quot;https://github.com/apache/carbondata/commit/1e0b0ea96f90e72cf85b3428822a1dbb61fbe1d8&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/apache/carbondata/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/apache/carbondata/commit/&amp;amp;lt;a ... (continued)

Run Details

16 of 16 new or added lines in 2 files covered. (100.0%)

59857 of 74252 relevant lines covered (80.61%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.65

/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.scan.filter.executer;

import java.io.IOException;
import java.util.BitSet;

import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.comparator.Comparator;
import org.apache.carbondata.core.util.comparator.SerializableComparator;

public class ExcludeFilterExecuterImpl implements FilterExecuter {

  private DimColumnResolvedFilterInfo dimColEvaluatorInfo;
  private DimColumnExecuterFilterInfo dimColumnExecuterInfo;
  private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
  private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
  protected SegmentProperties segmentProperties;
  private boolean isDimensionPresentInCurrentBlock = false;
  private boolean isMeasurePresentInCurrentBlock = false;
  private SerializableComparator comparator;
  /**
   * is dimension column data is natural sorted
   */
  private boolean isNaturalSorted = false;

  private byte[][] filterValues;

  public ExcludeFilterExecuterImpl(byte[][] filterValues, boolean isNaturalSorted) {
    this.filterValues = filterValues;
    this.isNaturalSorted = isNaturalSorted;
  }
  public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
      MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, SegmentProperties segmentProperties,
      boolean isMeasure) {
    this.segmentProperties = segmentProperties;
    if (!isMeasure) {
      this.dimColEvaluatorInfo = dimColEvaluatorInfo;
      dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();

      FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), segmentProperties,
          dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo, null, null);
      isDimensionPresentInCurrentBlock = true;
      isNaturalSorted =
          dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && dimColEvaluatorInfo
              .getDimension().isSortColumn();
    } else {
      this.msrColumnEvaluatorInfo = msrColumnEvaluatorInfo;
      msrColumnExecutorInfo = new MeasureColumnExecuterFilterInfo();
      FilterUtil
          .prepareKeysFromSurrogates(msrColumnEvaluatorInfo.getFilterValues(), segmentProperties,
              null, null, msrColumnEvaluatorInfo.getMeasure(), msrColumnExecutorInfo);
      isMeasurePresentInCurrentBlock = true;

      DataType msrType = FilterUtil.getMeasureDataType(msrColumnEvaluatorInfo);
      comparator = Comparator.getComparatorByDataTypeForMeasure(msrType);
    }

  }

  @Override
  public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
      boolean useBitsetPipeLine) throws IOException {
    if (isDimensionPresentInCurrentBlock) {
      int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
          .get(dimColEvaluatorInfo.getColumnIndex());
      if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
        rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
            rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
                rawBlockletColumnChunks.getFileReader(), chunkIndex);
      }
      DimensionRawColumnChunk dimensionRawColumnChunk =
          rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
      DimensionColumnPage[] dimensionColumnPages =
          dimensionRawColumnChunk.decodeAllColumnPages();
      filterValues = FilterUtil
          .getEncodedFilterValues(dimensionRawColumnChunk.getLocalDictionary(),
              dimColumnExecuterInfo.filterKeysForExclude);
      BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
      for (int i = 0; i < dimensionColumnPages.length; i++) {
        BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i],
            dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
            rawBlockletColumnChunks.getBitSetGroup(), i);
        bitSetGroup.setBitSet(bitSet, i);
      }

      return bitSetGroup;
    } else if (isMeasurePresentInCurrentBlock) {
      int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
          .get(msrColumnEvaluatorInfo.getColumnIndex());
      if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
        rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
            rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
                rawBlockletColumnChunks.getFileReader(), chunkIndex);
      }
      MeasureRawColumnChunk measureRawColumnChunk =
          rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
      ColumnPage[] ColumnPages =
          measureRawColumnChunk.decodeAllColumnPages();
      BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
      DataType msrType = FilterUtil.getMeasureDataType(msrColumnEvaluatorInfo);
      for (int i = 0; i < ColumnPages.length; i++) {
        BitSet bitSet =
            getFilteredIndexesForMeasure(
                measureRawColumnChunk.decodeColumnPage(i),
                measureRawColumnChunk.getRowCount()[i],
                useBitsetPipeLine,
                rawBlockletColumnChunks.getBitSetGroup(),
                i,
                msrType);
        bitSetGroup.setBitSet(bitSet, i);
      }
      return bitSetGroup;
    }
    return null;
  }

  @Override
  public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
    if (isDimensionPresentInCurrentBlock) {
      byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
      byte[] col = (byte[])value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal());
      for (int i = 0; i < filterValues.length; i++) {
        if (0 == ByteUtil.UnsafeComparer.INSTANCE.compareTo(col, 0, col.length,
            filterValues[i], 0, filterValues[i].length)) {
          return false;
        }
      }
    } else if (isMeasurePresentInCurrentBlock) {
      Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
      Object col = value.getVal(msrColumnEvaluatorInfo.getMeasure().getOrdinal() + dimOrdinalMax);
      for (int i = 0; i < filterValues.length; i++) {
        if (filterValues[i] == null) {
          if (null == col) {
            return false;
          }
          continue;
        }
        if (comparator.compare(col, filterValues[i]) == 0) {
          return false;
        }
      }
    }
    return true;
  }

  private BitSet getFilteredIndexes(ColumnPage columnPage, int numerOfRows, DataType msrType) {
    // Here the algorithm is
    // Get the measure values from the chunk. compare sequentially with the
    // the filter values. The one that matches sets it Bitset.
    BitSet bitSet = new BitSet(numerOfRows);
    bitSet.flip(0, numerOfRows);
    Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType);
    for (int i = 0; i < filterValues.length; i++) {
      BitSet nullBitSet = columnPage.getNullBits();
      if (filterValues[i] == null) {
        for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
          bitSet.flip(j);
        }
        continue;
      }
      for (int startIndex = 0; startIndex < numerOfRows; startIndex++) {
        if (!nullBitSet.get(startIndex)) {
          // Check if filterValue[i] matches with measure Values.
          Object msrValue = DataTypeUtil
              .getMeasureObjectBasedOnDataType(columnPage, startIndex,
                  msrType, msrColumnEvaluatorInfo.getMeasure());

          if (comparator.compare(msrValue, filterValues[i]) == 0) {
            // This is a match.
            bitSet.flip(startIndex);
          }
        }
      }
    }
    return bitSet;
  }

  /**
   * Below method will be used to apply filter on measure column
   * @param measureColumnPage
   * @param numberOfRows
   * @param useBitsetPipeLine
   * @param prvBitSetGroup
   * @param pageNumber
   * @param msrDataType
   * @return filtered indexes bitset
   */
  private BitSet getFilteredIndexesForMeasure(ColumnPage measureColumnPage, int numberOfRows,
      boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber, DataType msrDataType) {
    // check whether previous indexes can be optimal to apply filter on measure column
    if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
        msrColumnExecutorInfo.getFilterKeys().length)) {
      return getFilteredIndexesForMsrUsingPrvBitSet(measureColumnPage, prvBitSetGroup, pageNumber,
          numberOfRows, msrDataType);
    } else {
      return getFilteredIndexes(measureColumnPage, numberOfRows, msrDataType);
    }
  }
  /**
   * Below method will be used to apply filter on measure column based on previous filtered indexes
   * @param measureColumnPage
   * @param prvBitSetGroup
   * @param pageNumber
   * @param numberOfRows
   * @param msrDataType
   * @return filtred indexes bitset
   */
  private BitSet getFilteredIndexesForMsrUsingPrvBitSet(ColumnPage measureColumnPage,
      BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows, DataType msrDataType) {
    BitSet bitSet = new BitSet(numberOfRows);
    bitSet.flip(0, numberOfRows);
    Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
    BitSet nullBitSet = measureColumnPage.getNullBits();
    BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrDataType);
    for (int i = 0; i < filterValues.length; i++) {
      if (filterValues[i] == null) {
        for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
          bitSet.flip(j);
        }
        continue;
      }
      for (int index = prvPageBitSet.nextSetBit(0);
           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
        if (!nullBitSet.get(index)) {
          // Check if filterValue[i] matches with measure Values.
          Object msrValue = DataTypeUtil
              .getMeasureObjectBasedOnDataType(measureColumnPage, index,
                  msrDataType, msrColumnEvaluatorInfo.getMeasure());

          if (comparator.compare(msrValue, filterValues[i]) == 0) {
            // This is a match.
            bitSet.flip(index);
          }
        }
      }
    }
    return bitSet;
  }

  /**
   * Below method will be used to apply filter on dimension column
   * @param dimensionColumnPage
   * @param numberOfRows
   * @param useBitsetPipeLine
   * @param prvBitSetGroup
   * @param pageNumber
   * @return filtered indexes bitset
   */
  protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
      int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
    // check whether applying filtered based on previous bitset will be optimal
    if (filterValues.length > 0 && CarbonUtil
        .usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
            filterValues.length)) {
      return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber);
    } else {
      return getFilteredIndexes(dimensionColumnPage, numberOfRows);
    }
  }

  private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
      int numberOfRows) {
    if (dimensionColumnPage.isExplicitSorted()) {
      return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);
    }
    return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);
  }

  /**
   * Below method will be used to apply filter based on previous filtered bitset
   * @param dimensionColumnPage
   * @param prvBitSetGroup
   * @param pageNumber
   * @return filtered indexes bitset
   */
  private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
      BitSetGroup prvBitSetGroup, int pageNumber) {
    BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
    if (prvPageBitSet == null || prvPageBitSet.isEmpty()) {
      return prvPageBitSet;
    }
    BitSet bitSet = new BitSet();
    bitSet.or(prvPageBitSet);
    int compareResult = 0;
    // if dimension data was natural sorted then get the index from previous bitset
    // and use the same in next column data, otherwise use the inverted index reverse
    if (!dimensionColumnPage.isExplicitSorted()) {
      for (int index = prvPageBitSet.nextSetBit(0);
           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
        compareResult = CarbonUtil
            .isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1, index);
        if (compareResult != 0) {
          bitSet.set(index);
        } else {
          if (bitSet.get(index)) {
            bitSet.flip(index);
          }
        }
      }
    } else {
      for (int index = prvPageBitSet.nextSetBit(0);
           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
        compareResult = CarbonUtil
            .isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1,
                dimensionColumnPage.getInvertedReverseIndex(index));
        if (compareResult != 0) {
          bitSet.set(index);
        } else {
          if (bitSet.get(index)) {
            bitSet.flip(index);
          }
        }
      }
    }
    return bitSet;
  }

  private BitSet setFilterdIndexToBitSetWithColumnIndex(
      DimensionColumnPage dimensionColumnPage, int numerOfRows) {
    BitSet bitSet = new BitSet(numerOfRows);
    bitSet.flip(0, numerOfRows);
    if (filterValues.length == 0) {
      return bitSet;
    }
    int startIndex = 0;
    for (int i = 0; i < filterValues.length; i++) {
      if (startIndex >= numerOfRows) {
        break;
      }
      int[] rangeIndex = CarbonUtil
          .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
              filterValues[i]);
      for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
        bitSet.flip(dimensionColumnPage.getInvertedIndex(j));
      }
      if (rangeIndex[1] >= 0) {
        startIndex = rangeIndex[1] + 1;
      }
    }
    return bitSet;
  }

  private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
      int numerOfRows) {
    BitSet bitSet = new BitSet(numerOfRows);
    bitSet.flip(0, numerOfRows);
    // filterValues can be null when the dictionary chunk and surrogate size both are one
    if (filterValues.length == 0) {
      return bitSet;
    }
    // binary search can only be applied if column is sorted
    if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {
      int startIndex = 0;
      for (int i = 0; i < filterValues.length; i++) {
        if (startIndex >= numerOfRows) {
          break;
        }
        int[] rangeIndex = CarbonUtil
            .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
                filterValues[i]);
        for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
          bitSet.flip(j);
        }
        if (rangeIndex[1] >= 0) {
          startIndex = rangeIndex[1] + 1;
        }
      }
    } else {
      if (filterValues.length > 1) {
        for (int i = 0; i < numerOfRows; i++) {
          int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
              dimensionColumnPage, i);
          if (index >= 0) {
            bitSet.flip(i);
          }
        }
      } else {
        for (int j = 0; j < numerOfRows; j++) {
          if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {
            bitSet.flip(j);
          }
        }
      }
    }
    return bitSet;
  }

  @Override
  public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue,
      boolean[] isMinMaxSet) {
    BitSet bitSet = new BitSet(1);
    bitSet.flip(0, 1);
    return bitSet;
  }

  @Override
  public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
    if (isDimensionPresentInCurrentBlock) {
      int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
          .get(dimColEvaluatorInfo.getColumnIndex());
      if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
        rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
            rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
                rawBlockletColumnChunks.getFileReader(), chunkIndex);
      }
    } else if (isMeasurePresentInCurrentBlock) {
      int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
          .get(msrColumnEvaluatorInfo.getColumnIndex());
      if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
        rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
            rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
                rawBlockletColumnChunks.getFileReader(), chunkIndex);
      }
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.scan.filter.executer;
18
19	import java.io.IOException;
20	import java.util.BitSet;
21
22	import org.apache.carbondata.core.datastore.block.SegmentProperties;
23	import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
24	import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
25	import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
26	import org.apache.carbondata.core.datastore.page.ColumnPage;
27	import org.apache.carbondata.core.metadata.datatype.DataType;
28	import org.apache.carbondata.core.scan.filter.FilterUtil;
29	import org.apache.carbondata.core.scan.filter.intf.RowIntf;
30	import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
31	import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
32	import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
33	import org.apache.carbondata.core.util.BitSetGroup;
34	import org.apache.carbondata.core.util.ByteUtil;
35	import org.apache.carbondata.core.util.CarbonUtil;
36	import org.apache.carbondata.core.util.DataTypeUtil;
37	import org.apache.carbondata.core.util.comparator.Comparator;
38	import org.apache.carbondata.core.util.comparator.SerializableComparator;
39
40	public class ExcludeFilterExecuterImpl implements FilterExecuter {
41
42	private DimColumnResolvedFilterInfo dimColEvaluatorInfo;
43	private DimColumnExecuterFilterInfo dimColumnExecuterInfo;
44	private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
45	private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
46	protected SegmentProperties segmentProperties;
47	private boolean isDimensionPresentInCurrentBlock = false;	1✔
48	private boolean isMeasurePresentInCurrentBlock = false;	1✔
49	private SerializableComparator comparator;
50	/**
51	* is dimension column data is natural sorted
52	*/
53	private boolean isNaturalSorted = false;	1✔
54
55	private byte[][] filterValues;
56
57	public ExcludeFilterExecuterImpl(byte[][] filterValues, boolean isNaturalSorted) {	×
58	this.filterValues = filterValues;	×
59	this.isNaturalSorted = isNaturalSorted;	×
60	}	×
61	public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
62	MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo, SegmentProperties segmentProperties,
63	boolean isMeasure) {	1✔
64	this.segmentProperties = segmentProperties;	1✔
65	if (!isMeasure) {	1✔
66	this.dimColEvaluatorInfo = dimColEvaluatorInfo;	1✔
67	dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();	1✔
68
69	FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), segmentProperties,	1✔
70	dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo, null, null);	1✔
71	isDimensionPresentInCurrentBlock = true;	1✔
72	isNaturalSorted =	1✔
73	dimColEvaluatorInfo.getDimension().isUseInvertedIndex() && dimColEvaluatorInfo	1✔
74	.getDimension().isSortColumn();	1✔
75	} else {
76	this.msrColumnEvaluatorInfo = msrColumnEvaluatorInfo;	1✔
77	msrColumnExecutorInfo = new MeasureColumnExecuterFilterInfo();	1✔
78	FilterUtil	1✔
79	.prepareKeysFromSurrogates(msrColumnEvaluatorInfo.getFilterValues(), segmentProperties,	1✔
80	null, null, msrColumnEvaluatorInfo.getMeasure(), msrColumnExecutorInfo);	1✔
81	isMeasurePresentInCurrentBlock = true;	1✔
82
83	DataType msrType = FilterUtil.getMeasureDataType(msrColumnEvaluatorInfo);	1✔
84	comparator = Comparator.getComparatorByDataTypeForMeasure(msrType);	1✔
85	}
86
87	}	1✔
88
89	@Override
90	public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
91	boolean useBitsetPipeLine) throws IOException {
92	if (isDimensionPresentInCurrentBlock) {	1✔
93	int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()	1✔
94	.get(dimColEvaluatorInfo.getColumnIndex());	1✔
95	if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {	1✔
96	rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =	×
97	rawBlockletColumnChunks.getDataBlock().readDimensionChunk(	×
98	rawBlockletColumnChunks.getFileReader(), chunkIndex);	×
99	}
100	DimensionRawColumnChunk dimensionRawColumnChunk =	1✔
101	rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];	1✔
102	DimensionColumnPage[] dimensionColumnPages =	1✔
103	dimensionRawColumnChunk.decodeAllColumnPages();	1✔
104	filterValues = FilterUtil	1✔
105	.getEncodedFilterValues(dimensionRawColumnChunk.getLocalDictionary(),	1✔
106	dimColumnExecuterInfo.filterKeysForExclude);
107	BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());	1✔
108	for (int i = 0; i < dimensionColumnPages.length; i++) {	1✔
109	BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i],	1✔
110	dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,	1✔
111	rawBlockletColumnChunks.getBitSetGroup(), i);	1✔
112	bitSetGroup.setBitSet(bitSet, i);	1✔
113	}
114
115	return bitSetGroup;	1✔
116	} else if (isMeasurePresentInCurrentBlock) {	1✔
117	int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()	1✔
118	.get(msrColumnEvaluatorInfo.getColumnIndex());	1✔
119	if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {	1✔
120	rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =	×
121	rawBlockletColumnChunks.getDataBlock().readMeasureChunk(	×
122	rawBlockletColumnChunks.getFileReader(), chunkIndex);	×
123	}
124	MeasureRawColumnChunk measureRawColumnChunk =	1✔
125	rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];	1✔
126	ColumnPage[] ColumnPages =	1✔
127	measureRawColumnChunk.decodeAllColumnPages();	1✔
128	BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());	1✔
129	DataType msrType = FilterUtil.getMeasureDataType(msrColumnEvaluatorInfo);	1✔
130	for (int i = 0; i < ColumnPages.length; i++) {	1✔
131	BitSet bitSet =	1✔
132	getFilteredIndexesForMeasure(	1✔
133	measureRawColumnChunk.decodeColumnPage(i),	1✔
134	measureRawColumnChunk.getRowCount()[i],	1✔
135	useBitsetPipeLine,
136	rawBlockletColumnChunks.getBitSetGroup(),	1✔
137	i,
138	msrType);
139	bitSetGroup.setBitSet(bitSet, i);	1✔
140	}
141	return bitSetGroup;	1✔
142	}
143	return null;	×
144	}
145
146	@Override
147	public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
148	if (isDimensionPresentInCurrentBlock) {	1✔
149	byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();	1✔
150	byte[] col = (byte[])value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal());	1✔
151	for (int i = 0; i < filterValues.length; i++) {	1✔
152	if (0 == ByteUtil.UnsafeComparer.INSTANCE.compareTo(col, 0, col.length,	1✔
153	filterValues[i], 0, filterValues[i].length)) {
154	return false;	1✔
155	}
156	}
157	} else if (isMeasurePresentInCurrentBlock) {	1✔
158	Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();	1✔
159	Object col = value.getVal(msrColumnEvaluatorInfo.getMeasure().getOrdinal() + dimOrdinalMax);	1✔
160	for (int i = 0; i < filterValues.length; i++) {	1✔
161	if (filterValues[i] == null) {	1✔
162	if (null == col) {	1✔
163	return false;	1✔
164	}
165	continue;
166	}
167	if (comparator.compare(col, filterValues[i]) == 0) {	1✔
168	return false;	1✔
169	}
170	}
171	}
172	return true;	1✔
173	}
174
175	private BitSet getFilteredIndexes(ColumnPage columnPage, int numerOfRows, DataType msrType) {
176	// Here the algorithm is
177	// Get the measure values from the chunk. compare sequentially with the
178	// the filter values. The one that matches sets it Bitset.
179	BitSet bitSet = new BitSet(numerOfRows);	1✔
180	bitSet.flip(0, numerOfRows);	1✔
181	Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();	1✔
182	SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType);	1✔
183	for (int i = 0; i < filterValues.length; i++) {	1✔
184	BitSet nullBitSet = columnPage.getNullBits();	1✔
185	if (filterValues[i] == null) {	1✔
186	for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {	1✔
187	bitSet.flip(j);	1✔
188	}
189	continue;	1✔
190	}
191	for (int startIndex = 0; startIndex < numerOfRows; startIndex++) {	1✔
192	if (!nullBitSet.get(startIndex)) {	1✔
193	// Check if filterValue[i] matches with measure Values.
194	Object msrValue = DataTypeUtil	1✔
195	.getMeasureObjectBasedOnDataType(columnPage, startIndex,	1✔
196	msrType, msrColumnEvaluatorInfo.getMeasure());	1✔
197
198	if (comparator.compare(msrValue, filterValues[i]) == 0) {	1✔
199	// This is a match.
200	bitSet.flip(startIndex);	1✔
201	}
202	}
203	}
204	}
205	return bitSet;	1✔
206	}
207
208	/**
209	* Below method will be used to apply filter on measure column
210	* @param measureColumnPage
211	* @param numberOfRows
212	* @param useBitsetPipeLine
213	* @param prvBitSetGroup
214	* @param pageNumber
215	* @param msrDataType
216	* @return filtered indexes bitset
217	*/
218	private BitSet getFilteredIndexesForMeasure(ColumnPage measureColumnPage, int numberOfRows,
219	boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber, DataType msrDataType) {
220	// check whether previous indexes can be optimal to apply filter on measure column
221	if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,	1✔
222	msrColumnExecutorInfo.getFilterKeys().length)) {	1✔
223	return getFilteredIndexesForMsrUsingPrvBitSet(measureColumnPage, prvBitSetGroup, pageNumber,	×
224	numberOfRows, msrDataType);
225	} else {
226	return getFilteredIndexes(measureColumnPage, numberOfRows, msrDataType);	1✔
227	}
228	}
229	/**
230	* Below method will be used to apply filter on measure column based on previous filtered indexes
231	* @param measureColumnPage
232	* @param prvBitSetGroup
233	* @param pageNumber
234	* @param numberOfRows
235	* @param msrDataType
236	* @return filtred indexes bitset
237	*/
238	private BitSet getFilteredIndexesForMsrUsingPrvBitSet(ColumnPage measureColumnPage,
239	BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows, DataType msrDataType) {
240	BitSet bitSet = new BitSet(numberOfRows);	×
241	bitSet.flip(0, numberOfRows);	×
242	Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();	×
243	BitSet nullBitSet = measureColumnPage.getNullBits();	×
244	BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);	×
245	SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrDataType);	×
246	for (int i = 0; i < filterValues.length; i++) {	×
247	if (filterValues[i] == null) {	×
248	for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {	×
249	bitSet.flip(j);	×
250	}
251	continue;	×
252	}
253	for (int index = prvPageBitSet.nextSetBit(0);	×
254	index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {	×
255	if (!nullBitSet.get(index)) {	×
256	// Check if filterValue[i] matches with measure Values.
257	Object msrValue = DataTypeUtil	×
258	.getMeasureObjectBasedOnDataType(measureColumnPage, index,	×
259	msrDataType, msrColumnEvaluatorInfo.getMeasure());	×
260
261	if (comparator.compare(msrValue, filterValues[i]) == 0) {	×
262	// This is a match.
263	bitSet.flip(index);	×
264	}
265	}
266	}
267	}
268	return bitSet;	×
269	}
270
271	/**
272	* Below method will be used to apply filter on dimension column
273	* @param dimensionColumnPage
274	* @param numberOfRows
275	* @param useBitsetPipeLine
276	* @param prvBitSetGroup
277	* @param pageNumber
278	* @return filtered indexes bitset
279	*/
280	protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
281	int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
282	// check whether applying filtered based on previous bitset will be optimal
283	if (filterValues.length > 0 && CarbonUtil	1✔
284	.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,	1✔
285	filterValues.length)) {
286	return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber);	1✔
287	} else {
288	return getFilteredIndexes(dimensionColumnPage, numberOfRows);	1✔
289	}
290	}
291
292	private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
293	int numberOfRows) {
294	if (dimensionColumnPage.isExplicitSorted()) {	1✔
295	return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);	1✔
296	}
297	return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);	1✔
298	}
299
300	/**
301	* Below method will be used to apply filter based on previous filtered bitset
302	* @param dimensionColumnPage
303	* @param prvBitSetGroup
304	* @param pageNumber
305	* @return filtered indexes bitset
306	*/
307	private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
308	BitSetGroup prvBitSetGroup, int pageNumber) {
309	BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);	1✔
310	if (prvPageBitSet == null \|\| prvPageBitSet.isEmpty()) {	1✔
311	return prvPageBitSet;	×
312	}
313	BitSet bitSet = new BitSet();	1✔
314	bitSet.or(prvPageBitSet);	1✔
315	int compareResult = 0;	1✔
316	// if dimension data was natural sorted then get the index from previous bitset
317	// and use the same in next column data, otherwise use the inverted index reverse
318	if (!dimensionColumnPage.isExplicitSorted()) {	1✔
319	for (int index = prvPageBitSet.nextSetBit(0);	1✔
320	index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {	1✔
321	compareResult = CarbonUtil	1✔
322	.isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1, index);	1✔
323	if (compareResult != 0) {	1✔
324	bitSet.set(index);	×
325	} else {
326	if (bitSet.get(index)) {	1✔
327	bitSet.flip(index);	1✔
328	}
329	}
330	}
331	} else {
332	for (int index = prvPageBitSet.nextSetBit(0);	×
333	index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {	×
334	compareResult = CarbonUtil	×
335	.isFilterPresent(filterValues, dimensionColumnPage, 0, filterValues.length - 1,	×
336	dimensionColumnPage.getInvertedReverseIndex(index));	×
337	if (compareResult != 0) {	×
338	bitSet.set(index);	×
339	} else {
340	if (bitSet.get(index)) {	×
341	bitSet.flip(index);	×
342	}
343	}
344	}
345	}
346	return bitSet;	1✔
347	}
348
349	private BitSet setFilterdIndexToBitSetWithColumnIndex(
350	DimensionColumnPage dimensionColumnPage, int numerOfRows) {
351	BitSet bitSet = new BitSet(numerOfRows);	1✔
352	bitSet.flip(0, numerOfRows);	1✔
353	if (filterValues.length == 0) {	1✔
354	return bitSet;	1✔
355	}
356	int startIndex = 0;	1✔
357	for (int i = 0; i < filterValues.length; i++) {	1✔
358	if (startIndex >= numerOfRows) {	1✔
359	break;	×
360	}
361	int[] rangeIndex = CarbonUtil	1✔
362	.getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,	1✔
363	filterValues[i]);
364	for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {	1✔
365	bitSet.flip(dimensionColumnPage.getInvertedIndex(j));	1✔
366	}
367	if (rangeIndex[1] >= 0) {	1✔
368	startIndex = rangeIndex[1] + 1;	1✔
369	}
370	}
371	return bitSet;	1✔
372	}
373
374	private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
375	int numerOfRows) {
376	BitSet bitSet = new BitSet(numerOfRows);	1✔
377	bitSet.flip(0, numerOfRows);	1✔
378	// filterValues can be null when the dictionary chunk and surrogate size both are one
379	if (filterValues.length == 0) {	1✔
380	return bitSet;	1✔
381	}
382	// binary search can only be applied if column is sorted
383	if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {	1✔
384	int startIndex = 0;	×
385	for (int i = 0; i < filterValues.length; i++) {	×
386	if (startIndex >= numerOfRows) {	×
387	break;	×
388	}
389	int[] rangeIndex = CarbonUtil	×
390	.getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,	×
391	filterValues[i]);
392	for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {	×
393	bitSet.flip(j);	×
394	}
395	if (rangeIndex[1] >= 0) {	×
396	startIndex = rangeIndex[1] + 1;	×
397	}
398	}
399	} else {	×
400	if (filterValues.length > 1) {	1✔
401	for (int i = 0; i < numerOfRows; i++) {	1✔
402	int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,	1✔
403	dimensionColumnPage, i);
404	if (index >= 0) {	1✔
405	bitSet.flip(i);	1✔
406	}
407	}
408	} else {
409	for (int j = 0; j < numerOfRows; j++) {	1✔
410	if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {	1✔
411	bitSet.flip(j);	1✔
412	}
413	}
414	}
415	}
416	return bitSet;	1✔
417	}
418
419	@Override
420	public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue,
421	boolean[] isMinMaxSet) {
422	BitSet bitSet = new BitSet(1);	1✔
423	bitSet.flip(0, 1);	1✔
424	return bitSet;	1✔
425	}
426
427	@Override
428	public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
429	if (isDimensionPresentInCurrentBlock) {	1✔
430	int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()	1✔
431	.get(dimColEvaluatorInfo.getColumnIndex());	1✔
432	if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {	1✔
433	rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =	1✔
434	rawBlockletColumnChunks.getDataBlock().readDimensionChunk(	1✔
435	rawBlockletColumnChunks.getFileReader(), chunkIndex);	1✔
436	}
437	} else if (isMeasurePresentInCurrentBlock) {	1✔
438	int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()	1✔
439	.get(msrColumnEvaluatorInfo.getColumnIndex());	1✔
440	if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {	1✔
441	rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =	1✔
442	rawBlockletColumnChunks.getDataBlock().readMeasureChunk(	1✔
443	rawBlockletColumnChunks.getFileReader(), chunkIndex);	1✔
444	}
445	}
446	}	1✔
447	}

apache / carbondata / 8694

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous