apache / carbondata / 586

Build # 586

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/c740cf6f817f3a81db1fb94d4311d7ba0752ead5">c740cf6f8<a href="https://github.com/apache/carbondata/commit/c740cf6f817f3a81db1fb94d4311d7ba0752ead5">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/671324dc22fffdaf4f804c6e5d54f1dd4fafb8b9">671324dc2</a>">671324dc2</a><a href="https://github.com/apache/carbondata/commit/c740cf6f817f3a81db1fb94d4311d7ba0752ead5"> into 8320918e5">8320918e5</a>

Run Details

59743 of 74247 relevant lines covered (80.47%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.73

/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.cache.dictionary;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.reader.CarbonDictionaryReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.DictionaryService;
import org.apache.carbondata.core.util.CarbonUtil;

/**
 * This class is responsible for loading the dictionary data for given columns
 */
public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {

  /**
   * carbon dictionary column identifier
   */
  private DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier;

  /**
   * @param dictionaryColumnUniqueIdentifier dictionary column identifier
   */
  DictionaryCacheLoaderImpl(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
    this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier;
  }

  /**
   * This method will load the dictionary data for a given columnIdentifier
   *
   * @param dictionaryInfo             dictionary info object which will hold the required data
   *                                   for a given column
   * @param dictionaryChunkStartOffset start offset from where dictionary file has to
   *                                   be read
   * @param dictionaryChunkEndOffset   end offset till where dictionary file has to
   *                                   be read
   * @param loadSortIndex              flag to indicate whether the sort index file has to be
   *                                   read in memory after dictionary loading
   * @throws IOException
   */
  @Override
  public void load(DictionaryInfo dictionaryInfo, long dictionaryChunkStartOffset,
      long dictionaryChunkEndOffset, boolean loadSortIndex) throws IOException {
    Iterator<byte[]> columnDictionaryChunkWrapper =
        load(dictionaryColumnUniqueIdentifier, dictionaryChunkStartOffset,
            dictionaryChunkEndOffset);
    if (loadSortIndex) {
      readSortIndexFile(dictionaryInfo, dictionaryColumnUniqueIdentifier);
    }
    fillDictionaryValuesAndAddToDictionaryChunks(dictionaryInfo, columnDictionaryChunkWrapper);
  }

  /**
   * This method will fill the dictionary values according to dictionary bucket size and
   * add to the dictionary chunk list
   *
   * @param dictionaryInfo
   * @param columnDictionaryChunkWrapper
   */
  private void fillDictionaryValuesAndAddToDictionaryChunks(DictionaryInfo dictionaryInfo,
      Iterator<byte[]> columnDictionaryChunkWrapper) {
    int dictionaryChunkSize = CarbonUtil.getDictionaryChunkSize();
    int sizeOfLastDictionaryChunk = dictionaryInfo.getSizeOfLastDictionaryChunk();
    int sizeOfOneDictionaryChunk = dictionaryChunkSize - sizeOfLastDictionaryChunk;
    if (sizeOfOneDictionaryChunk <= 0) {
      sizeOfOneDictionaryChunk = dictionaryChunkSize;
    }
    List<List<byte[]>> dictionaryChunks =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<byte[]> oneDictionaryChunk = new ArrayList<>(sizeOfOneDictionaryChunk);
    dictionaryChunks.add(oneDictionaryChunk);
    while (columnDictionaryChunkWrapper.hasNext()) {
      oneDictionaryChunk.add(columnDictionaryChunkWrapper.next());
      if (oneDictionaryChunk.size() >= sizeOfOneDictionaryChunk) {
        sizeOfOneDictionaryChunk = dictionaryChunkSize;
        oneDictionaryChunk = new ArrayList<>(sizeOfOneDictionaryChunk);
        dictionaryChunks.add(oneDictionaryChunk);
      }
    }
    for (List<byte[]> dictionaryChunk : dictionaryChunks) {
      if (!dictionaryChunk.isEmpty()) {
        dictionaryInfo.addDictionaryChunk(dictionaryChunk);
      }
    }
  }

  /**
   * This method will load the dictionary data between a given start and end offset
   *
   * @param dictionaryColumnUniqueIdentifier column unique identifier
   * @param startOffset      start offset of dictionary file
   * @param endOffset        end offset of dictionary file
   * @return iterator over dictionary values
   * @throws IOException
   */
  private Iterator<byte[]> load(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
      long startOffset, long endOffset) throws IOException {
    CarbonDictionaryReader dictionaryReader = getDictionaryReader(dictionaryColumnUniqueIdentifier);
    try {
      return dictionaryReader.read(startOffset, endOffset);
    } finally {
      dictionaryReader.close();
    }
  }

  /**
   * This method will read the sort index file and load into memory
   *
   * @param dictionaryInfo
   * @param dictionaryColumnUniqueIdentifier
   * @throws IOException
   */
  private void readSortIndexFile(DictionaryInfo dictionaryInfo,
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
    CarbonDictionarySortIndexReader sortIndexReader =
        getSortIndexReader(dictionaryColumnUniqueIdentifier);
    try {
      dictionaryInfo.setSortOrderIndex(sortIndexReader.readSortIndex());
      dictionaryInfo.setSortReverseOrderIndex(sortIndexReader.readInvertedSortIndex());
    } finally {
      sortIndexReader.close();
    }
  }

  /**
   * This method will create a dictionary reader instance to read the dictionary file
   *
   * @param dictionaryColumnUniqueIdentifier unique column identifier
   * @return carbon dictionary reader instance
   */
  private CarbonDictionaryReader getDictionaryReader(
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
    return dictService.getDictionaryReader(dictionaryColumnUniqueIdentifier);
  }

  /**
   * @param dictionaryColumnUniqueIdentifier unique column identifier
   * @return sort index reader instance
   */
  private CarbonDictionarySortIndexReader getSortIndexReader(
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
    return dictService
        .getDictionarySortIndexReader(dictionaryColumnUniqueIdentifier);
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.cache.dictionary;
19
20	import java.io.IOException;
21	import java.util.ArrayList;
22	import java.util.Iterator;
23	import java.util.List;
24
25	import org.apache.carbondata.core.constants.CarbonCommonConstants;
26	import org.apache.carbondata.core.reader.CarbonDictionaryReader;
27	import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
28	import org.apache.carbondata.core.service.CarbonCommonFactory;
29	import org.apache.carbondata.core.service.DictionaryService;
30	import org.apache.carbondata.core.util.CarbonUtil;
31
32	/**
33	* This class is responsible for loading the dictionary data for given columns
34	*/
35	public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
36
37	/**
38	* carbon dictionary column identifier
39	*/
40	private DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier;
41
42	/**
43	* @param dictionaryColumnUniqueIdentifier dictionary column identifier
44	*/
45	DictionaryCacheLoaderImpl(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {	2✔
46	this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier;	2✔
47	}	2✔
48
49	/**
50	* This method will load the dictionary data for a given columnIdentifier
51	*
52	* @param dictionaryInfo dictionary info object which will hold the required data
53	* for a given column
54	* @param dictionaryChunkStartOffset start offset from where dictionary file has to
55	* be read
56	* @param dictionaryChunkEndOffset end offset till where dictionary file has to
57	* be read
58	* @param loadSortIndex flag to indicate whether the sort index file has to be
59	* read in memory after dictionary loading
60	* @throws IOException
61	*/
62	@Override
63	public void load(DictionaryInfo dictionaryInfo, long dictionaryChunkStartOffset,
64	long dictionaryChunkEndOffset, boolean loadSortIndex) throws IOException {
65	Iterator<byte[]> columnDictionaryChunkWrapper =	2✔
66	load(dictionaryColumnUniqueIdentifier, dictionaryChunkStartOffset,	2✔
67	dictionaryChunkEndOffset);
68	if (loadSortIndex) {	2✔
69	readSortIndexFile(dictionaryInfo, dictionaryColumnUniqueIdentifier);	2✔
70	}
71	fillDictionaryValuesAndAddToDictionaryChunks(dictionaryInfo, columnDictionaryChunkWrapper);	2✔
72	}	2✔
73
74	/**
75	* This method will fill the dictionary values according to dictionary bucket size and
76	* add to the dictionary chunk list
77	*
78	* @param dictionaryInfo
79	* @param columnDictionaryChunkWrapper
80	*/
81	private void fillDictionaryValuesAndAddToDictionaryChunks(DictionaryInfo dictionaryInfo,
82	Iterator<byte[]> columnDictionaryChunkWrapper) {
83	int dictionaryChunkSize = CarbonUtil.getDictionaryChunkSize();	2✔
84	int sizeOfLastDictionaryChunk = dictionaryInfo.getSizeOfLastDictionaryChunk();	2✔
85	int sizeOfOneDictionaryChunk = dictionaryChunkSize - sizeOfLastDictionaryChunk;	2✔
86	if (sizeOfOneDictionaryChunk <= 0) {	2✔
87	sizeOfOneDictionaryChunk = dictionaryChunkSize;	2✔
88	}
89	List<List<byte[]>> dictionaryChunks =	2✔
90	new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
91	List<byte[]> oneDictionaryChunk = new ArrayList<>(sizeOfOneDictionaryChunk);	2✔
92	dictionaryChunks.add(oneDictionaryChunk);	2✔
93	while (columnDictionaryChunkWrapper.hasNext()) {	2✔
94	oneDictionaryChunk.add(columnDictionaryChunkWrapper.next());	2✔
95	if (oneDictionaryChunk.size() >= sizeOfOneDictionaryChunk) {	2✔
96	sizeOfOneDictionaryChunk = dictionaryChunkSize;	2✔
97	oneDictionaryChunk = new ArrayList<>(sizeOfOneDictionaryChunk);	2✔
98	dictionaryChunks.add(oneDictionaryChunk);	2✔
99	}
100	}
101	for (List<byte[]> dictionaryChunk : dictionaryChunks) {	2✔
102	if (!dictionaryChunk.isEmpty()) {	2✔
103	dictionaryInfo.addDictionaryChunk(dictionaryChunk);	2✔
104	}
105	}	2✔
106	}	2✔
107
108	/**
109	* This method will load the dictionary data between a given start and end offset
110	*
111	* @param dictionaryColumnUniqueIdentifier column unique identifier
112	* @param startOffset start offset of dictionary file
113	* @param endOffset end offset of dictionary file
114	* @return iterator over dictionary values
115	* @throws IOException
116	*/
117	private Iterator<byte[]> load(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
118	long startOffset, long endOffset) throws IOException {
119	CarbonDictionaryReader dictionaryReader = getDictionaryReader(dictionaryColumnUniqueIdentifier);	2✔
120	try {
121	return dictionaryReader.read(startOffset, endOffset);	2✔
122	} finally {
123	dictionaryReader.close();	2✔
124	}	×
125	}
126
127	/**
128	* This method will read the sort index file and load into memory
129	*
130	* @param dictionaryInfo
131	* @param dictionaryColumnUniqueIdentifier
132	* @throws IOException
133	*/
134	private void readSortIndexFile(DictionaryInfo dictionaryInfo,
135	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
136	CarbonDictionarySortIndexReader sortIndexReader =	2✔
137	getSortIndexReader(dictionaryColumnUniqueIdentifier);	2✔
138	try {
139	dictionaryInfo.setSortOrderIndex(sortIndexReader.readSortIndex());	2✔
140	dictionaryInfo.setSortReverseOrderIndex(sortIndexReader.readInvertedSortIndex());	2✔
141	} finally {
142	sortIndexReader.close();	2✔
143	}	2✔
144	}	2✔
145
146	/**
147	* This method will create a dictionary reader instance to read the dictionary file
148	*
149	* @param dictionaryColumnUniqueIdentifier unique column identifier
150	* @return carbon dictionary reader instance
151	*/
152	private CarbonDictionaryReader getDictionaryReader(
153	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
154	DictionaryService dictService = CarbonCommonFactory.getDictionaryService();	2✔
155	return dictService.getDictionaryReader(dictionaryColumnUniqueIdentifier);	2✔
156	}
157
158	/**
159	* @param dictionaryColumnUniqueIdentifier unique column identifier
160	* @return sort index reader instance
161	*/
162	private CarbonDictionarySortIndexReader getSortIndexReader(
163	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
164	DictionaryService dictService = CarbonCommonFactory.getDictionaryService();	2✔
165	return dictService	2✔
166	.getDictionarySortIndexReader(dictionaryColumnUniqueIdentifier);	2✔
167	}
168	}

apache / carbondata / 586

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous