8875

Build # 8875

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/652cc746dfb86a3510866942ed5b8a246778373c">652cc746d<a href="https://github.com/apache/carbondata/commit/652cc746dfb86a3510866942ed5b8a246778373c">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/9d7ae49394659abb1377b5524f8fe98e16529281">9d7ae4939</a>">9d7ae4939</a><a href="https://github.com/apache/carbondata/commit/652cc746dfb86a3510866942ed5b8a246778373c"> into 54bcf4963">54bcf4963</a>

Run Details

7 of 7 new or added lines in 4 files covered. (100.0%)

60481 of 75039 relevant lines covered (80.6%)

1.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.8

/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.cache.dictionary;

import java.io.IOException;
import java.util.List;

import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.CarbonLRUCache;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.DictionaryService;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.ObjectSizeCalculator;

/**
 * Abstract class which implements methods common to reverse and forward dictionary cache
 */
public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
    V extends Dictionary>
    implements Cache<DictionaryColumnUniqueIdentifier, Dictionary> {

  /**
   * thread pool size to be used for dictionary data reading
   */
  protected int thread_pool_size;

  /**
   * LRU cache variable
   */
  protected CarbonLRUCache carbonLRUCache;


  /**
   * @param carbonLRUCache
   */
  public AbstractDictionaryCache(CarbonLRUCache carbonLRUCache) {
    this.carbonLRUCache = carbonLRUCache;
    initThreadPoolSize();
  }

  @Override
  public void put(DictionaryColumnUniqueIdentifier key, Dictionary value) {
    throw new UnsupportedOperationException("Operation not supported");
  }


  /**
   * This method will initialize the thread pool size to be used for creating the
   * max number of threads for a job
   */
  private void initThreadPoolSize() {
    thread_pool_size = CarbonProperties.getInstance().getNumberOfCores();
  }

  /**
   * This method will read dictionary metadata file and return the dictionary meta chunks
   *
   * @param dictionaryColumnUniqueIdentifier
   * @return list of dictionary metadata chunks
   * @throws IOException read and close method throws IO exception
   */
  protected CarbonDictionaryColumnMetaChunk readLastChunkFromDictionaryMetadataFile(
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
    CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService
        .getDictionaryMetadataReader(dictionaryColumnUniqueIdentifier);

    CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;
    // read metadata file
    try {
      carbonDictionaryColumnMetaChunk =
          columnMetadataReaderImpl.readLastEntryOfDictionaryMetaChunk();
    } finally {
      // close the metadata reader
      columnMetadataReaderImpl.close();
    }
    return carbonDictionaryColumnMetaChunk;
  }

  /**
   * get the dictionary column meta chunk for object already read and stored in LRU cache
   * @param dictionaryColumnUniqueIdentifier
   * @param offsetRead
   * @return
   * @throws IOException
   */
  protected long getNumRecordsInCarbonDictionaryColumnMetaChunk(
          DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, long offsetRead)
          throws IOException {
    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
    CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService
            .getDictionaryMetadataReader(dictionaryColumnUniqueIdentifier);

    CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;
    // read metadata file
    try {
      carbonDictionaryColumnMetaChunk =
              columnMetadataReaderImpl.readEntryOfDictionaryMetaChunk(offsetRead);
    } finally {
      // close the metadata reader
      columnMetadataReaderImpl.close();
    }
    return carbonDictionaryColumnMetaChunk.getMax_surrogate_key();
  }

  /**
   * This method will validate dictionary metadata file for any modification
   *
   * @param carbonFile
   * @param fileTimeStamp
   * @param endOffset
   * @return
   */
  private boolean isDictionaryMetaFileModified(CarbonFile carbonFile, long fileTimeStamp,
      long endOffset) {
    return carbonFile.isFileModified(fileTimeStamp, endOffset);
  }

  /**
   * This method will return the carbon file objetc based on its type (local, HDFS)
   *
   * @param dictionaryColumnUniqueIdentifier
   * @return
   */
  private CarbonFile getDictionaryMetaCarbonFile(
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
    String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath();
    FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath);
    CarbonFile dictFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType);
    // When rename table triggered parallely with select query, dictionary files may not exist
    if (!dictFile.exists()) {
      throw new IOException("Dictionary file does not exist: " + dictionaryFilePath);
    }
    return dictFile;
  }

  protected long getSortIndexSize(long numOfRecords) {
    // sort index has sort index and reverse sort index,each is 4 byte integer.
    // 32 byte is the array header of both the integer arrays
    return numOfRecords * ObjectSizeCalculator.estimate(0, 16) * 2 + 32;
  }

  /**
   * This method will get the value for the given key. If value does not exist
   * for the given key, it will check and load the value.
   *
   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
   *                                         tableName and columnIdentifier
   * @param dictionaryInfo
   * @param lruCacheKey
   * @throws IOException                    in case memory is not sufficient to load dictionary
   *                                        into memory
   */
  protected void checkAndLoadDictionaryData(
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
      DictionaryInfo dictionaryInfo, String lruCacheKey, boolean loadSortIndex)
      throws IOException {
    // read last segm
    // ent dictionary meta chunk entry to get the end offset of file
    CarbonFile carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
    boolean dictionaryMetaFileModified =
        isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
            dictionaryInfo.getDictionaryMetaFileLength());
    // if dictionary metadata file is modified then only read the last entry from dictionary
    // meta file
    if (dictionaryMetaFileModified) {
      synchronized (dictionaryInfo) {
        carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
        dictionaryMetaFileModified =
            isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
                dictionaryInfo.getDictionaryMetaFileLength());
        // Double Check :
        // if dictionary metadata file is modified then only read the last entry from dictionary
        // meta file
        if (dictionaryMetaFileModified) {
          CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk =
              readLastChunkFromDictionaryMetadataFile(dictionaryColumnUniqueIdentifier);

          long requiredSize = getEstimatedDictionarySize(dictionaryInfo,
              carbonDictionaryColumnMetaChunk,
              dictionaryColumnUniqueIdentifier, loadSortIndex);

          if (requiredSize > 0) {
            dictionaryInfo.setMemorySize(requiredSize);
            boolean colCanBeAddedToLRUCache =
                    carbonLRUCache.tryPut(lruCacheKey, requiredSize);
            // if column can be added to lru cache then only load the
            // dictionary data
            if (colCanBeAddedToLRUCache) {
              // load dictionary data
              loadDictionaryData(dictionaryInfo, dictionaryColumnUniqueIdentifier,
                      dictionaryInfo.getOffsetTillFileIsRead(),
                      carbonDictionaryColumnMetaChunk.getEnd_offset(),
                      loadSortIndex);
              // set the end offset till where file is read
              dictionaryInfo
                      .setOffsetTillFileIsRead(carbonDictionaryColumnMetaChunk.getEnd_offset());
              long updateRequiredSize = ObjectSizeCalculator.estimate(dictionaryInfo, requiredSize);
              dictionaryInfo.setMemorySize(updateRequiredSize);
              if (!carbonLRUCache.put(lruCacheKey, dictionaryInfo, updateRequiredSize)) {
                throw new DictionaryBuilderException(
                        "Cannot load dictionary into memory. Not enough memory available");
              }
              dictionaryInfo.setFileTimeStamp(carbonFile.getLastModifiedTime());
              dictionaryInfo.setDictionaryMetaFileLength(carbonFile.getSize());
            } else {
              throw new DictionaryBuilderException(
                      "Cannot load dictionary into memory. Not enough memory available");
            }
          }
        }
      }
    }
    // increment the column access count
    incrementDictionaryAccessCount(dictionaryInfo);
  }

  /**
   * This method will prepare the lru cache key and return the same
   *
   * @param columnIdentifier
   * @return
   */
  protected String getLruCacheKey(String columnIdentifier, CacheType cacheType) {
    return columnIdentifier + CarbonCommonConstants.UNDERSCORE + cacheType.getCacheName();
  }

  /**
   * This method will check and load the dictionary file in memory for a given column
   *
   * @param dictionaryInfo                   holds dictionary information and data
   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
   *                                         tableName and columnIdentifier
   * @param dictionaryChunkStartOffset       start offset from where dictionary file has to
   *                                         be read
   * @param dictionaryChunkEndOffset         end offset till where dictionary file has to
   *                                         be read
   * @param loadSortIndex
   * @throws IOException
   */
  private void loadDictionaryData(DictionaryInfo dictionaryInfo,
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
      long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
      throws IOException {
    DictionaryCacheLoader dictionaryCacheLoader =
        new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier);
    dictionaryCacheLoader
        .load(dictionaryInfo, dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex);
  }

  /**
   * This method will increment the access count for a given dictionary column
   *
   * @param dictionaryInfo
   */
  protected void incrementDictionaryAccessCount(DictionaryInfo dictionaryInfo) {
    dictionaryInfo.incrementAccessCount();
  }

  /**
   * This method will update the dictionary acceess count which is required for its removal
   * from column LRU cache
   *
   * @param dictionaryList
   */
  protected void clearDictionary(List<Dictionary> dictionaryList) {
    for (Dictionary dictionary : dictionaryList) {
      dictionary.clear();
    }
  }

  @Override
  public void invalidateAll() {
    carbonLRUCache.clear();
  }

  /**
   * calculate the probable size of Dictionary in java heap
   * Use the value to check if can be added to lru cache
   * This helps to avoid unnecessary loading of dictionary files
   * if estimated size more than that can be fit into lru cache
   * Estimated size can be less or greater than the actual size
   * due to java optimizations
   * @param dictionaryInfo
   * @param carbonDictionaryColumnMetaChunk
   * @param dictionaryColumnUniqueIdentifier
   * @param readSortIndexSize
   * @return
   * @throws IOException
   */
  protected long getEstimatedDictionarySize(DictionaryInfo dictionaryInfo,
      CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk,
      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, boolean
      readSortIndexSize) throws IOException {
    return 0;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.cache.dictionary;
19
20	import java.io.IOException;
21	import java.util.List;
22
23	import org.apache.carbondata.core.cache.Cache;
24	import org.apache.carbondata.core.cache.CacheType;
25	import org.apache.carbondata.core.cache.CarbonLRUCache;
26	import org.apache.carbondata.core.constants.CarbonCommonConstants;
27	import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
28	import org.apache.carbondata.core.datastore.impl.FileFactory;
29	import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
30	import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
31	import org.apache.carbondata.core.service.CarbonCommonFactory;
32	import org.apache.carbondata.core.service.DictionaryService;
33	import org.apache.carbondata.core.util.CarbonProperties;
34	import org.apache.carbondata.core.util.ObjectSizeCalculator;
35
36	/**
37	* Abstract class which implements methods common to reverse and forward dictionary cache
38	*/
39	public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
40	V extends Dictionary>
41	implements Cache<DictionaryColumnUniqueIdentifier, Dictionary> {
42
43	/**
44	* thread pool size to be used for dictionary data reading
45	*/
46	protected int thread_pool_size;
47
48	/**
49	* LRU cache variable
50	*/
51	protected CarbonLRUCache carbonLRUCache;
52
53
54	/**
55	* @param carbonLRUCache
56	*/
57	public AbstractDictionaryCache(CarbonLRUCache carbonLRUCache) {	2✔
58	this.carbonLRUCache = carbonLRUCache;	2✔
59	initThreadPoolSize();	2✔
60	}	2✔
61
62	@Override
63	public void put(DictionaryColumnUniqueIdentifier key, Dictionary value) {
64	throw new UnsupportedOperationException("Operation not supported");	×
65	}
66
67
68	/**
69	* This method will initialize the thread pool size to be used for creating the
70	* max number of threads for a job
71	*/
72	private void initThreadPoolSize() {
73	thread_pool_size = CarbonProperties.getInstance().getNumberOfCores();	2✔
74	}	2✔
75
76	/**
77	* This method will read dictionary metadata file and return the dictionary meta chunks
78	*
79	* @param dictionaryColumnUniqueIdentifier
80	* @return list of dictionary metadata chunks
81	* @throws IOException read and close method throws IO exception
82	*/
83	protected CarbonDictionaryColumnMetaChunk readLastChunkFromDictionaryMetadataFile(
84	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
85	DictionaryService dictService = CarbonCommonFactory.getDictionaryService();	2✔
86	CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService	2✔
87	.getDictionaryMetadataReader(dictionaryColumnUniqueIdentifier);	2✔
88
89	CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;	2✔
90	// read metadata file
91	try {
92	carbonDictionaryColumnMetaChunk =	2✔
93	columnMetadataReaderImpl.readLastEntryOfDictionaryMetaChunk();	2✔
94	} finally {
95	// close the metadata reader
96	columnMetadataReaderImpl.close();	2✔
97	}	2✔
98	return carbonDictionaryColumnMetaChunk;	2✔
99	}
100
101	/**
102	* get the dictionary column meta chunk for object already read and stored in LRU cache
103	* @param dictionaryColumnUniqueIdentifier
104	* @param offsetRead
105	* @return
106	* @throws IOException
107	*/
108	protected long getNumRecordsInCarbonDictionaryColumnMetaChunk(
109	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, long offsetRead)
110	throws IOException {
111	DictionaryService dictService = CarbonCommonFactory.getDictionaryService();	2✔
112	CarbonDictionaryMetadataReader columnMetadataReaderImpl = dictService	2✔
113	.getDictionaryMetadataReader(dictionaryColumnUniqueIdentifier);	2✔
114
115	CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;	2✔
116	// read metadata file
117	try {
118	carbonDictionaryColumnMetaChunk =	2✔
119	columnMetadataReaderImpl.readEntryOfDictionaryMetaChunk(offsetRead);	2✔
120	} finally {
121	// close the metadata reader
122	columnMetadataReaderImpl.close();	2✔
123	}	2✔
124	return carbonDictionaryColumnMetaChunk.getMax_surrogate_key();	2✔
125	}
126
127	/**
128	* This method will validate dictionary metadata file for any modification
129	*
130	* @param carbonFile
131	* @param fileTimeStamp
132	* @param endOffset
133	* @return
134	*/
135	private boolean isDictionaryMetaFileModified(CarbonFile carbonFile, long fileTimeStamp,
136	long endOffset) {
137	return carbonFile.isFileModified(fileTimeStamp, endOffset);	2✔
138	}
139
140	/**
141	* This method will return the carbon file objetc based on its type (local, HDFS)
142	*
143	* @param dictionaryColumnUniqueIdentifier
144	* @return
145	*/
146	private CarbonFile getDictionaryMetaCarbonFile(
147	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
148	String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath();	2✔
149	FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath);	2✔
150	CarbonFile dictFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType);	2✔
151	// When rename table triggered parallely with select query, dictionary files may not exist
152	if (!dictFile.exists()) {	2✔
153	throw new IOException("Dictionary file does not exist: " + dictionaryFilePath);	×
154	}
155	return dictFile;	2✔
156	}
157
158	protected long getSortIndexSize(long numOfRecords) {
159	// sort index has sort index and reverse sort index,each is 4 byte integer.
160	// 32 byte is the array header of both the integer arrays
161	return numOfRecords * ObjectSizeCalculator.estimate(0, 16) * 2 + 32;	2✔
162	}
163
164	/**
165	* This method will get the value for the given key. If value does not exist
166	* for the given key, it will check and load the value.
167	*
168	* @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
169	* tableName and columnIdentifier
170	* @param dictionaryInfo
171	* @param lruCacheKey
172	* @throws IOException in case memory is not sufficient to load dictionary
173	* into memory
174	*/
175	protected void checkAndLoadDictionaryData(
176	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
177	DictionaryInfo dictionaryInfo, String lruCacheKey, boolean loadSortIndex)
178	throws IOException {
179	// read last segm
180	// ent dictionary meta chunk entry to get the end offset of file
181	CarbonFile carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);	2✔
182	boolean dictionaryMetaFileModified =	2✔
183	isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),	2✔
184	dictionaryInfo.getDictionaryMetaFileLength());	2✔
185	// if dictionary metadata file is modified then only read the last entry from dictionary
186	// meta file
187	if (dictionaryMetaFileModified) {	2✔
188	synchronized (dictionaryInfo) {	2✔
189	carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);	2✔
190	dictionaryMetaFileModified =	2✔
191	isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),	2✔
192	dictionaryInfo.getDictionaryMetaFileLength());	2✔
193	// Double Check :
194	// if dictionary metadata file is modified then only read the last entry from dictionary
195	// meta file
196	if (dictionaryMetaFileModified) {	2✔
197	CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk =	2✔
198	readLastChunkFromDictionaryMetadataFile(dictionaryColumnUniqueIdentifier);	2✔
199
200	long requiredSize = getEstimatedDictionarySize(dictionaryInfo,	2✔
201	carbonDictionaryColumnMetaChunk,
202	dictionaryColumnUniqueIdentifier, loadSortIndex);
203
204	if (requiredSize > 0) {	2✔
205	dictionaryInfo.setMemorySize(requiredSize);	2✔
206	boolean colCanBeAddedToLRUCache =	2✔
207	carbonLRUCache.tryPut(lruCacheKey, requiredSize);	2✔
208	// if column can be added to lru cache then only load the
209	// dictionary data
210	if (colCanBeAddedToLRUCache) {	2✔
211	// load dictionary data
212	loadDictionaryData(dictionaryInfo, dictionaryColumnUniqueIdentifier,	2✔
213	dictionaryInfo.getOffsetTillFileIsRead(),	2✔
214	carbonDictionaryColumnMetaChunk.getEnd_offset(),	2✔
215	loadSortIndex);
216	// set the end offset till where file is read
217	dictionaryInfo	2✔
218	.setOffsetTillFileIsRead(carbonDictionaryColumnMetaChunk.getEnd_offset());	2✔
219	long updateRequiredSize = ObjectSizeCalculator.estimate(dictionaryInfo, requiredSize);	2✔
220	dictionaryInfo.setMemorySize(updateRequiredSize);	2✔
221	if (!carbonLRUCache.put(lruCacheKey, dictionaryInfo, updateRequiredSize)) {	2✔
222	throw new DictionaryBuilderException(	×
223	"Cannot load dictionary into memory. Not enough memory available");
224	}
225	dictionaryInfo.setFileTimeStamp(carbonFile.getLastModifiedTime());	2✔
226	dictionaryInfo.setDictionaryMetaFileLength(carbonFile.getSize());	2✔
227	} else {	2✔
228	throw new DictionaryBuilderException(	2✔
229	"Cannot load dictionary into memory. Not enough memory available");
230	}
231	}
232	}
233	}	2✔
234	}
235	// increment the column access count
236	incrementDictionaryAccessCount(dictionaryInfo);	2✔
237	}	2✔
238
239	/**
240	* This method will prepare the lru cache key and return the same
241	*
242	* @param columnIdentifier
243	* @return
244	*/
245	protected String getLruCacheKey(String columnIdentifier, CacheType cacheType) {
246	return columnIdentifier + CarbonCommonConstants.UNDERSCORE + cacheType.getCacheName();	2✔
247	}
248
249	/**
250	* This method will check and load the dictionary file in memory for a given column
251	*
252	* @param dictionaryInfo holds dictionary information and data
253	* @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
254	* tableName and columnIdentifier
255	* @param dictionaryChunkStartOffset start offset from where dictionary file has to
256	* be read
257	* @param dictionaryChunkEndOffset end offset till where dictionary file has to
258	* be read
259	* @param loadSortIndex
260	* @throws IOException
261	*/
262	private void loadDictionaryData(DictionaryInfo dictionaryInfo,
263	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
264	long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
265	throws IOException {
266	DictionaryCacheLoader dictionaryCacheLoader =	2✔
267	new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier);
268	dictionaryCacheLoader	2✔
269	.load(dictionaryInfo, dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex);	2✔
270	}	2✔
271
272	/**
273	* This method will increment the access count for a given dictionary column
274	*
275	* @param dictionaryInfo
276	*/
277	protected void incrementDictionaryAccessCount(DictionaryInfo dictionaryInfo) {
278	dictionaryInfo.incrementAccessCount();	2✔
279	}	2✔
280
281	/**
282	* This method will update the dictionary acceess count which is required for its removal
283	* from column LRU cache
284	*
285	* @param dictionaryList
286	*/
287	protected void clearDictionary(List<Dictionary> dictionaryList) {
288	for (Dictionary dictionary : dictionaryList) {	×
289	dictionary.clear();	×
290	}	×
291	}	×
292
293	@Override
294	public void invalidateAll() {
295	carbonLRUCache.clear();	×
296	}	×
297
298	/**
299	* calculate the probable size of Dictionary in java heap
300	* Use the value to check if can be added to lru cache
301	* This helps to avoid unnecessary loading of dictionary files
302	* if estimated size more than that can be fit into lru cache
303	* Estimated size can be less or greater than the actual size
304	* due to java optimizations
305	* @param dictionaryInfo
306	* @param carbonDictionaryColumnMetaChunk
307	* @param dictionaryColumnUniqueIdentifier
308	* @param readSortIndexSize
309	* @return
310	* @throws IOException
311	*/
312	protected long getEstimatedDictionarySize(DictionaryInfo dictionaryInfo,
313	CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk,
314	DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, boolean
315	readSortIndexSize) throws IOException {
316	return 0;	×
317	}
318	}

apache / carbondata / 8875

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous