2278

Build # 2278

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/4e258a702ca3cd8bf49c378d620979ca3dd6a3a7">4e258a702<a href="https://github.com/apache/carbondata/commit/4e258a702ca3cd8bf49c378d620979ca3dd6a3a7">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/e136d7a8433b99027b3e4338ae5ef8d5156788a1">e136d7a84</a>">e136d7a84</a><a href="https://github.com/apache/carbondata/commit/4e258a702ca3cd8bf49c378d620979ca3dd6a3a7"> into 7c4e79fca">7c4e79fca</a>

Run Details

61625 of 78030 relevant lines covered (78.98%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.66

/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.processing.sort.sortdata;

import java.io.Serializable;
import java.util.List;
import java.util.Objects;

import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;

/**
 * This class is used to hold field information for a table during data loading. These information
 * will be used to convert/construct/destruct row in sort process step. Because complex field is
 * processed the same as no-dict-no-sort-simple-dimension, so we treat them as the same and use
 * `no-dict-no-sort-dim` related variable to represent them in this class.
 */
public class TableFieldStat implements Serializable {
  private static final long serialVersionUID = 201712070950L;
  private int dictSortDimCnt = 0;
  private int dictNoSortDimCnt = 0;
  private int noDictSortDimCnt = 0;
  // for columns that are no_dict_dim and no_sort_dim, except complex/varchar dims
  private int noDictNoSortDimCnt = 0;
  // for columns that are complex data type
  private int complexDimCnt = 0;
  // for columns that are varchar data type
  private int varcharDimCnt = 0;
  // whether sort column is of dictionary type or not
  private boolean[] isSortColNoDictFlags;
  private boolean[] isVarcharDimFlags;
  private int measureCnt;
  private DataType[] measureDataType;
  private DataType[] noDictDataType;
  private DataType[] noDictSortDataType;
  private DataType[] noDictNoSortDataType;

  // indices for dict & sort dimension columns
  private int[] dictSortDimIdx;
  // indices for dict & no-sort dimension columns
  private int[] dictNoSortDimIdx;
  // indices for no-dict & sort dimension columns
  private int[] noDictSortDimIdx;
  // indices for no-dict & no-sort dimension columns, excluding complex/varchar columns
  private int[] noDictNoSortDimIdx;
  // indices for varchar dimension columns
  private int[] varcharDimIdx;
  // indices for varchar dimension columns
  private int [] complexDimIdx;
  // indices for measure columns
  private int[] measureIdx;

  public TableFieldStat(SortParameters sortParameters) {
    int noDictDimCnt = sortParameters.getNoDictionaryCount();
    int dictDimCnt = sortParameters.getDimColCount() - noDictDimCnt;
    this.complexDimCnt = sortParameters.getComplexDimColCount();
    this.isSortColNoDictFlags = sortParameters.getNoDictionarySortColumn();
    this.isVarcharDimFlags = sortParameters.getIsVarcharDimensionColumn();
    boolean[] isDimNoDictFlags = sortParameters.getNoDictionaryDimnesionColumn();
    boolean[] sortColumn = sortParameters.getSortColumn();
    for (int i = 0; i < isDimNoDictFlags.length; i++) {
      if (isDimNoDictFlags[i] && sortColumn[i]) {
        noDictSortDimCnt++;
      } else if (!isDimNoDictFlags[i] && sortColumn[i]) {
        dictSortDimCnt++;
      }
    }
    this.measureCnt = sortParameters.getMeasureColCount();
    this.measureDataType = sortParameters.getMeasureDataType();
    this.noDictDataType = sortParameters.getNoDictDataType();
    this.noDictSortDataType = sortParameters.getNoDictSortDataType();
    this.noDictNoSortDataType = sortParameters.getNoDictNoSortDataType();
    for (boolean flag : isVarcharDimFlags) {
      if (flag) {
        varcharDimCnt++;
      }
    }

    // be careful that the default value is 0
    this.dictSortDimIdx = new int[dictSortDimCnt];
    this.dictNoSortDimIdx = new int[dictDimCnt - dictSortDimCnt];
    this.noDictSortDimIdx = new int[noDictSortDimCnt];
    this.noDictNoSortDimIdx = new int[noDictDimCnt - noDictSortDimCnt - varcharDimCnt];
    this.complexDimIdx = new int[complexDimCnt];
    this.varcharDimIdx = new int[varcharDimCnt];
    this.measureIdx = new int[measureCnt];

    int tmpNoDictSortCnt = 0;
    int tmpNoDictNoSortCnt = 0;
    int tmpDictSortCnt = 0;
    int tmpDictNoSortCnt = 0;
    int tmpVarcharCnt = 0;
    int tmpComplexcount = 0;

    List<CarbonDimension> allDimensions =
        sortParameters.getCarbonTable().getDimensionByTableName(sortParameters.getTableName());

    for (int i = 0; i < allDimensions.size(); i++) {
      CarbonDimension carbonDimension = allDimensions.get(i);
      if (carbonDimension.hasEncoding(Encoding.DICTIONARY) && !carbonDimension.isComplex()) {
        if (carbonDimension.isSortColumn()) {
          dictSortDimIdx[tmpDictSortCnt++] = i;
        } else {
          dictNoSortDimIdx[tmpDictNoSortCnt++] = i;
        }
      } else if (!carbonDimension.isComplex()) {
        if (isVarcharDimFlags[i]) {
          varcharDimIdx[tmpVarcharCnt++] = i;
        } else if (carbonDimension.isSortColumn()) {
          noDictSortDimIdx[tmpNoDictSortCnt++] = i;
        } else {
          noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = i;
        }
      } else {
        complexDimIdx[tmpComplexcount++] = i;
      }
    }

    dictNoSortDimCnt = tmpDictNoSortCnt;
    noDictNoSortDimCnt = tmpNoDictNoSortCnt;

    int base = allDimensions.size();

    // indices for measure columns
    for (int i = 0; i < measureCnt; i++) {
      measureIdx[i] = base + i;
    }
  }

  public int getDictSortDimCnt() {
    return dictSortDimCnt;
  }

  public int getDictNoSortDimCnt() {
    return dictNoSortDimCnt;
  }

  public int getNoDictSortDimCnt() {
    return noDictSortDimCnt;
  }

  public int getNoDictNoSortDimCnt() {
    return noDictNoSortDimCnt;
  }

  public int getComplexDimCnt() {
    return complexDimCnt;
  }

  public int getVarcharDimCnt() {
    return varcharDimCnt;
  }

  public boolean[] getIsSortColNoDictFlags() {
    return isSortColNoDictFlags;
  }

  public boolean[] getIsVarcharDimFlags() {
    return isVarcharDimFlags;
  }

  public int getMeasureCnt() {
    return measureCnt;
  }

  public DataType[] getMeasureDataType() {
    return measureDataType;
  }

  public int[] getDictSortDimIdx() {
    return dictSortDimIdx;
  }

  public int[] getDictNoSortDimIdx() {
    return dictNoSortDimIdx;
  }

  public int[] getNoDictSortDimIdx() {
    return noDictSortDimIdx;
  }

  public int[] getNoDictNoSortDimIdx() {
    return noDictNoSortDimIdx;
  }

  public int[] getComplexDimIdx() {
    return complexDimIdx;
  }

  public int[] getVarcharDimIdx() {
    return varcharDimIdx;
  }

  public int[] getMeasureIdx() {
    return measureIdx;
  }

  @Override public boolean equals(Object o) {
    if (this == o) return true;
    if (!(o instanceof TableFieldStat)) return false;
    TableFieldStat that = (TableFieldStat) o;
    return dictSortDimCnt == that.dictSortDimCnt
        && dictNoSortDimCnt == that.dictNoSortDimCnt
        && noDictSortDimCnt == that.noDictSortDimCnt
        && noDictNoSortDimCnt == that.noDictNoSortDimCnt
        && complexDimCnt == that.complexDimCnt
        && varcharDimCnt == that.varcharDimCnt
        && measureCnt == that.measureCnt;
  }

  @Override public int hashCode() {
    return Objects.hash(dictSortDimCnt, dictNoSortDimCnt, noDictSortDimCnt,
        noDictNoSortDimCnt, complexDimCnt, varcharDimCnt, measureCnt);
  }

  public DataType[] getNoDictSortDataType() {
    return noDictSortDataType;
  }

  public DataType[] getNoDictNoSortDataType() {
    return noDictNoSortDataType;
  }


  public DataType[] getNoDictDataType() {
    return noDictDataType;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.processing.sort.sortdata;
19
20	import java.io.Serializable;
21	import java.util.List;
22	import java.util.Objects;
23
24	import org.apache.carbondata.core.metadata.datatype.DataType;
25	import org.apache.carbondata.core.metadata.encoder.Encoding;
26	import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
27
28	/**
29	* This class is used to hold field information for a table during data loading. These information
30	* will be used to convert/construct/destruct row in sort process step. Because complex field is
31	* processed the same as no-dict-no-sort-simple-dimension, so we treat them as the same and use
32	* `no-dict-no-sort-dim` related variable to represent them in this class.
33	*/
34	public class TableFieldStat implements Serializable {
35	private static final long serialVersionUID = 201712070950L;
36	private int dictSortDimCnt = 0;	1✔
37	private int dictNoSortDimCnt = 0;	1✔
38	private int noDictSortDimCnt = 0;	1✔
39	// for columns that are no_dict_dim and no_sort_dim, except complex/varchar dims
40	private int noDictNoSortDimCnt = 0;	1✔
41	// for columns that are complex data type
42	private int complexDimCnt = 0;	1✔
43	// for columns that are varchar data type
44	private int varcharDimCnt = 0;	1✔
45	// whether sort column is of dictionary type or not
46	private boolean[] isSortColNoDictFlags;
47	private boolean[] isVarcharDimFlags;
48	private int measureCnt;
49	private DataType[] measureDataType;
50	private DataType[] noDictDataType;
51	private DataType[] noDictSortDataType;
52	private DataType[] noDictNoSortDataType;
53
54	// indices for dict & sort dimension columns
55	private int[] dictSortDimIdx;
56	// indices for dict & no-sort dimension columns
57	private int[] dictNoSortDimIdx;
58	// indices for no-dict & sort dimension columns
59	private int[] noDictSortDimIdx;
60	// indices for no-dict & no-sort dimension columns, excluding complex/varchar columns
61	private int[] noDictNoSortDimIdx;
62	// indices for varchar dimension columns
63	private int[] varcharDimIdx;
64	// indices for varchar dimension columns
65	private int [] complexDimIdx;
66	// indices for measure columns
67	private int[] measureIdx;
68
69	public TableFieldStat(SortParameters sortParameters) {	1✔
70	int noDictDimCnt = sortParameters.getNoDictionaryCount();	1✔
71	int dictDimCnt = sortParameters.getDimColCount() - noDictDimCnt;	1✔
72	this.complexDimCnt = sortParameters.getComplexDimColCount();	1✔
73	this.isSortColNoDictFlags = sortParameters.getNoDictionarySortColumn();	1✔
74	this.isVarcharDimFlags = sortParameters.getIsVarcharDimensionColumn();	1✔
75	boolean[] isDimNoDictFlags = sortParameters.getNoDictionaryDimnesionColumn();	1✔
76	boolean[] sortColumn = sortParameters.getSortColumn();	1✔
77	for (int i = 0; i < isDimNoDictFlags.length; i++) {	1✔
78	if (isDimNoDictFlags[i] && sortColumn[i]) {	1✔
79	noDictSortDimCnt++;	1✔
80	} else if (!isDimNoDictFlags[i] && sortColumn[i]) {	1✔
81	dictSortDimCnt++;	1✔
82	}
83	}
84	this.measureCnt = sortParameters.getMeasureColCount();	1✔
85	this.measureDataType = sortParameters.getMeasureDataType();	1✔
86	this.noDictDataType = sortParameters.getNoDictDataType();	1✔
87	this.noDictSortDataType = sortParameters.getNoDictSortDataType();	1✔
88	this.noDictNoSortDataType = sortParameters.getNoDictNoSortDataType();	1✔
89	for (boolean flag : isVarcharDimFlags) {	1✔
90	if (flag) {	1✔
91	varcharDimCnt++;	×
92	}
93	}
94
95	// be careful that the default value is 0
96	this.dictSortDimIdx = new int[dictSortDimCnt];	1✔
97	this.dictNoSortDimIdx = new int[dictDimCnt - dictSortDimCnt];	1✔
98	this.noDictSortDimIdx = new int[noDictSortDimCnt];	1✔
99	this.noDictNoSortDimIdx = new int[noDictDimCnt - noDictSortDimCnt - varcharDimCnt];	1✔
100	this.complexDimIdx = new int[complexDimCnt];	1✔
101	this.varcharDimIdx = new int[varcharDimCnt];	1✔
102	this.measureIdx = new int[measureCnt];	1✔
103
104	int tmpNoDictSortCnt = 0;	1✔
105	int tmpNoDictNoSortCnt = 0;	1✔
106	int tmpDictSortCnt = 0;	1✔
107	int tmpDictNoSortCnt = 0;	1✔
108	int tmpVarcharCnt = 0;	1✔
109	int tmpComplexcount = 0;	1✔
110
111	List<CarbonDimension> allDimensions =	1✔
112	sortParameters.getCarbonTable().getDimensionByTableName(sortParameters.getTableName());	1✔
113
114	for (int i = 0; i < allDimensions.size(); i++) {	1✔
115	CarbonDimension carbonDimension = allDimensions.get(i);	1✔
116	if (carbonDimension.hasEncoding(Encoding.DICTIONARY) && !carbonDimension.isComplex()) {	1✔
117	if (carbonDimension.isSortColumn()) {	1✔
118	dictSortDimIdx[tmpDictSortCnt++] = i;	1✔
119	} else {
120	dictNoSortDimIdx[tmpDictNoSortCnt++] = i;	1✔
121	}
122	} else if (!carbonDimension.isComplex()) {	1✔
123	if (isVarcharDimFlags[i]) {	1✔
124	varcharDimIdx[tmpVarcharCnt++] = i;	×
125	} else if (carbonDimension.isSortColumn()) {	1✔
126	noDictSortDimIdx[tmpNoDictSortCnt++] = i;	1✔
127	} else {
128	noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = i;	1✔
129	}
130	} else {
131	complexDimIdx[tmpComplexcount++] = i;	1✔
132	}
133	}
134
135	dictNoSortDimCnt = tmpDictNoSortCnt;	1✔
136	noDictNoSortDimCnt = tmpNoDictNoSortCnt;	1✔
137
138	int base = allDimensions.size();	1✔
139
140	// indices for measure columns
141	for (int i = 0; i < measureCnt; i++) {	1✔
142	measureIdx[i] = base + i;	1✔
143	}
144	}	1✔
145
146	public int getDictSortDimCnt() {
147	return dictSortDimCnt;	1✔
148	}
149
150	public int getDictNoSortDimCnt() {
151	return dictNoSortDimCnt;	1✔
152	}
153
154	public int getNoDictSortDimCnt() {
155	return noDictSortDimCnt;	1✔
156	}
157
158	public int getNoDictNoSortDimCnt() {
159	return noDictNoSortDimCnt;	1✔
160	}
161
162	public int getComplexDimCnt() {
163	return complexDimCnt;	1✔
164	}
165
166	public int getVarcharDimCnt() {
167	return varcharDimCnt;	1✔
168	}
169
170	public boolean[] getIsSortColNoDictFlags() {
171	return isSortColNoDictFlags;	1✔
172	}
173
174	public boolean[] getIsVarcharDimFlags() {
175	return isVarcharDimFlags;	×
176	}
177
178	public int getMeasureCnt() {
179	return measureCnt;	1✔
180	}
181
182	public DataType[] getMeasureDataType() {
183	return measureDataType;	1✔
184	}
185
186	public int[] getDictSortDimIdx() {
187	return dictSortDimIdx;	1✔
188	}
189
190	public int[] getDictNoSortDimIdx() {
191	return dictNoSortDimIdx;	1✔
192	}
193
194	public int[] getNoDictSortDimIdx() {
195	return noDictSortDimIdx;	1✔
196	}
197
198	public int[] getNoDictNoSortDimIdx() {
199	return noDictNoSortDimIdx;	1✔
200	}
201
202	public int[] getComplexDimIdx() {
203	return complexDimIdx;	1✔
204	}
205
206	public int[] getVarcharDimIdx() {
207	return varcharDimIdx;	1✔
208	}
209
210	public int[] getMeasureIdx() {
211	return measureIdx;	1✔
212	}
213
214	@Override public boolean equals(Object o) {
215	if (this == o) return true;	×
216	if (!(o instanceof TableFieldStat)) return false;	×
217	TableFieldStat that = (TableFieldStat) o;	×
218	return dictSortDimCnt == that.dictSortDimCnt	×
219	&& dictNoSortDimCnt == that.dictNoSortDimCnt
220	&& noDictSortDimCnt == that.noDictSortDimCnt
221	&& noDictNoSortDimCnt == that.noDictNoSortDimCnt
222	&& complexDimCnt == that.complexDimCnt
223	&& varcharDimCnt == that.varcharDimCnt
224	&& measureCnt == that.measureCnt;
225	}
226
227	@Override public int hashCode() {
228	return Objects.hash(dictSortDimCnt, dictNoSortDimCnt, noDictSortDimCnt,	×
229	noDictNoSortDimCnt, complexDimCnt, varcharDimCnt, measureCnt);	×
230	}
231
232	public DataType[] getNoDictSortDataType() {
233	return noDictSortDataType;	1✔
234	}
235
236	public DataType[] getNoDictNoSortDataType() {
237	return noDictNoSortDataType;	1✔
238	}
239
240
241	public DataType[] getNoDictDataType() {
242	return noDictDataType;	1✔
243	}
244	}

apache / carbondata / 2278

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous