10916

Build # 10916

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/c83f9d854bc5c3a9389c39d2316ab9a5e3657c30">c83f9d854<a href="https://github.com/apache/carbondata/commit/c83f9d854bc5c3a9389c39d2316ab9a5e3657c30">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/acd9af1338d0b24e9bd12d95b30fc842990b2ba6">acd9af133</a>">acd9af133</a><a href="https://github.com/apache/carbondata/commit/c83f9d854bc5c3a9389c39d2316ab9a5e3657c30"> into 718be3729">718be3729</a>

Run Details

106 of 106 new or added lines in 7 files covered. (100.0%)

62759 of 79170 relevant lines covered (79.27%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.69

/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.cache;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;

import org.apache.log4j.Logger;

/**
 * class which manages the lru cache
 */
public final class CarbonLRUCache {
  /**
   * constant for converting MB into bytes
   */
  private static final int BYTE_CONVERSION_CONSTANT = 1024 * 1024;
  /**
   * Attribute for Carbon LOGGER
   */
  private static final Logger LOGGER =
      LogServiceFactory.getLogService(CarbonLRUCache.class.getName());
  /**
   * Map that will contain key as table unique name and value as cache Holder
   * object
   */
  private Map<String, Cacheable> lruCacheMap;
  /**
   * lruCacheSize
   */
  private long lruCacheMemorySize;
  /**
   * totalSize size of the cache
   */
  private long currentSize;

  /**
   * @param propertyName        property name to take the size configured
   * @param defaultPropertyName default property in case size is not configured
   */
  public CarbonLRUCache(String propertyName, String defaultPropertyName) {
    try {
      lruCacheMemorySize = Long
          .parseLong(CarbonProperties.getInstance().getProperty(propertyName, defaultPropertyName));
    } catch (NumberFormatException e) {
      LOGGER.error(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE
          + " is not in a valid format. Falling back to default value: "
          + CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT);
      lruCacheMemorySize = Long.parseLong(defaultPropertyName);
    }
    initCache();
    if (lruCacheMemorySize > 0) {
      LOGGER.info("Configured LRU cache size is " + lruCacheMemorySize + " MB");
      // convert in bytes
      lruCacheMemorySize = lruCacheMemorySize * BYTE_CONVERSION_CONSTANT;
    } else {
      LOGGER.info("LRU cache size not configured. Therefore default behavior will be "
              + "considered and no LRU based eviction of columns will be done");
    }
  }

  /**
   * initialize lru cache
   */
  private void initCache() {
    lruCacheMap =
        new LinkedHashMap<String, Cacheable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE, 1.0f,
            true);
  }

  /**
   * This method will give the list of all the keys that can be deleted from
   * the level LRU cache
   */
  private List<String> getKeysToBeRemoved(long size) {
    List<String> toBeDeletedKeys =
        new ArrayList<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    long removedSize = 0;
    for (Entry<String, Cacheable> entry : lruCacheMap.entrySet()) {
      String key = entry.getKey();
      Cacheable cacheInfo = entry.getValue();
      long memorySize = cacheInfo.getMemorySize();
      if (canBeRemoved(cacheInfo)) {
        removedSize = removedSize + memorySize;
        toBeDeletedKeys.add(key);
        // check if after removing the current file size, required
        // size when added to current size is sufficient to load a
        // level or not
        if (lruCacheMemorySize >= (currentSize - memorySize + size)) {
          toBeDeletedKeys.clear();
          toBeDeletedKeys.add(key);
          removedSize = memorySize;
          break;
        }
        // check if after removing the added size/removed size,
        // required size when added to current size is sufficient to
        // load a level or not
        else if (lruCacheMemorySize >= (currentSize - removedSize + size)) {
          break;
        }
      }
    }
    // this case will come when iteration is complete over the keys but
    // still size is not sufficient for level file to be loaded, then we
    // will not delete any of the keys
    if ((currentSize - removedSize + size) > lruCacheMemorySize) {
      toBeDeletedKeys.clear();
    }
    return toBeDeletedKeys;
  }

  /**
   * @param cacheInfo
   * @return
   */
  private boolean canBeRemoved(Cacheable cacheInfo) {
    if (cacheInfo.getAccessCount() > 0) {
      return false;
    }
    return true;
  }

  /**
   * @param key
   */
  public void remove(String key) {
    synchronized (lruCacheMap) {
      removeKey(key);
    }
  }

  /**
   * @param keys
   */
  public void removeAll(List<String> keys) {
    synchronized (lruCacheMap) {
      for (String key : keys) {
        removeKey(key);
      }
    }
  }

  /**
   * This method will remove the key from lru cache
   *
   * @param key
   */
  private void removeKey(String key) {
    Cacheable cacheable = lruCacheMap.get(key);
    if (null != cacheable) {
      long memorySize = cacheable.getMemorySize();
      cacheable.invalidate();
      lruCacheMap.remove(key);
      currentSize = currentSize - memorySize;
      LOGGER.info("Removed entry from InMemory lru cache :: " + key);
    }
  }

  /**
   * This method will check if required size is available in the memory and then add
   * the given cacheable to object to lru cache
   *
   * @param columnIdentifier
   * @param cacheInfo
   */
  public boolean put(String columnIdentifier, Cacheable cacheInfo, long requiredSize) {
    if (LOGGER.isDebugEnabled()) {
      LOGGER.debug("Required size for entry " + columnIdentifier + " :: " + requiredSize
          + " Current cache size :: " + currentSize);
    }
    boolean columnKeyAddedSuccessfully = false;
    if (isLRUCacheSizeConfigured()) {
      synchronized (lruCacheMap) {
        if (freeMemorySizeForAddingCache(requiredSize)) {
          currentSize = currentSize + requiredSize;
          addEntryToLRUCacheMap(columnIdentifier, cacheInfo);
          columnKeyAddedSuccessfully = true;
        } else {
          LOGGER.error(
              "Size not available. Entry cannot be added to lru cache :: " + columnIdentifier
                  + " .Required Size = " + requiredSize + " Size available " + (lruCacheMemorySize
                  - currentSize));
        }
      }
    } else {
      synchronized (lruCacheMap) {
        addEntryToLRUCacheMap(columnIdentifier, cacheInfo);
      }
      columnKeyAddedSuccessfully = true;
    }
    return columnKeyAddedSuccessfully;
  }

  /**
   * This method will check if required size is available in the memory
   * @param columnIdentifier
   * @param requiredSize
   * @return
   */
  public boolean tryPut(String columnIdentifier, long requiredSize) {
    if (LOGGER.isDebugEnabled()) {
      LOGGER.debug("checking Required size for entry " + columnIdentifier + " :: " + requiredSize
          + " Current cache size :: " + currentSize);
    }
    boolean columnKeyCanBeAdded = false;
    if (isLRUCacheSizeConfigured()) {
      synchronized (lruCacheMap) {
        if (freeMemorySizeForAddingCache(requiredSize)) {
          columnKeyCanBeAdded = true;
        } else {
          LOGGER.error(
              "Size check failed.Size not available. Entry cannot be added to lru cache :: "
                  + columnIdentifier + " .Required Size = " + requiredSize + " Size available " + (
                  lruCacheMemorySize - currentSize));
        }
      }
    } else {
      columnKeyCanBeAdded = true;
    }
    return columnKeyCanBeAdded;
  }

  /**
   * The method will add the cache entry to LRU cache map
   *
   * @param columnIdentifier
   * @param cacheInfo
   */
  private void addEntryToLRUCacheMap(String columnIdentifier, Cacheable cacheInfo) {
    if (null == lruCacheMap.get(columnIdentifier)) {
      lruCacheMap.put(columnIdentifier, cacheInfo);
    }
    if (LOGGER.isDebugEnabled()) {
      LOGGER.debug("Added entry to InMemory lru cache :: " + columnIdentifier);
    }
  }

  /**
   * this will check whether the LRU cache size is configured
   *
   * @return <Boolean> value
   */
  private boolean isLRUCacheSizeConfigured() {
    return lruCacheMemorySize > 0;
  }

  /**
   * This method will check a required column can be loaded into memory or not. If required
   * this method will call for eviction of existing data from memory
   *
   * @param requiredSize
   * @return
   */
  private boolean freeMemorySizeForAddingCache(long requiredSize) {
    boolean memoryAvailable = false;
    if (isSizeAvailableToLoadColumnDictionary(requiredSize)) {
      memoryAvailable = true;
    } else {
      // get the keys that can be removed from memory
      List<String> keysToBeRemoved = getKeysToBeRemoved(requiredSize);
      for (String cacheKey : keysToBeRemoved) {
        removeKey(cacheKey);
      }
      // after removing the keys check again if required size is available
      if (isSizeAvailableToLoadColumnDictionary(requiredSize)) {
        memoryAvailable = true;
      }
    }
    return memoryAvailable;
  }

  /**
   * This method will check if size is available to laod dictionary into memory
   *
   * @param requiredSize
   * @return
   */
  private boolean isSizeAvailableToLoadColumnDictionary(long requiredSize) {
    return lruCacheMemorySize >= (currentSize + requiredSize);
  }

  /**
   * @param key
   * @return
   */
  public Cacheable get(String key) {
    synchronized (lruCacheMap) {
      return lruCacheMap.get(key);
    }
  }

  /**
   * This method will empty the level cache
   */
  public void clear() {
    synchronized (lruCacheMap) {
      for (Cacheable cachebleObj : lruCacheMap.values()) {
        cachebleObj.invalidate();
      }
      lruCacheMap.clear();
    }
  }

  public Map<String, Cacheable> getCacheMap() {
    return lruCacheMap;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.core.cache;
19
20	import java.util.ArrayList;
21	import java.util.LinkedHashMap;
22	import java.util.List;
23	import java.util.Map;
24	import java.util.Map.Entry;
25
26	import org.apache.carbondata.common.logging.LogServiceFactory;
27	import org.apache.carbondata.core.constants.CarbonCommonConstants;
28	import org.apache.carbondata.core.util.CarbonProperties;
29
30	import org.apache.log4j.Logger;
31
32	/**
33	* class which manages the lru cache
34	*/
35	public final class CarbonLRUCache {
36	/**
37	* constant for converting MB into bytes
38	*/
39	private static final int BYTE_CONVERSION_CONSTANT = 1024 * 1024;
40	/**
41	* Attribute for Carbon LOGGER
42	*/
43	private static final Logger LOGGER =	2✔
44	LogServiceFactory.getLogService(CarbonLRUCache.class.getName());	2✔
45	/**
46	* Map that will contain key as table unique name and value as cache Holder
47	* object
48	*/
49	private Map<String, Cacheable> lruCacheMap;
50	/**
51	* lruCacheSize
52	*/
53	private long lruCacheMemorySize;
54	/**
55	* totalSize size of the cache
56	*/
57	private long currentSize;
58
59	/**
60	* @param propertyName property name to take the size configured
61	* @param defaultPropertyName default property in case size is not configured
62	*/
63	public CarbonLRUCache(String propertyName, String defaultPropertyName) {	2✔
64	try {
65	lruCacheMemorySize = Long	2✔
66	.parseLong(CarbonProperties.getInstance().getProperty(propertyName, defaultPropertyName));	2✔
67	} catch (NumberFormatException e) {	×
68	LOGGER.error(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE	×
69	+ " is not in a valid format. Falling back to default value: "
70	+ CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT);
71	lruCacheMemorySize = Long.parseLong(defaultPropertyName);	×
72	}	2✔
73	initCache();	2✔
74	if (lruCacheMemorySize > 0) {	2✔
75	LOGGER.info("Configured LRU cache size is " + lruCacheMemorySize + " MB");	2✔
76	// convert in bytes
77	lruCacheMemorySize = lruCacheMemorySize * BYTE_CONVERSION_CONSTANT;	2✔
78	} else {
79	LOGGER.info("LRU cache size not configured. Therefore default behavior will be "	2✔
80	+ "considered and no LRU based eviction of columns will be done");
81	}
82	}	2✔
83
84	/**
85	* initialize lru cache
86	*/
87	private void initCache() {
88	lruCacheMap =	2✔
89	new LinkedHashMap<String, Cacheable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE, 1.0f,
90	true);
91	}	2✔
92
93	/**
94	* This method will give the list of all the keys that can be deleted from
95	* the level LRU cache
96	*/
97	private List<String> getKeysToBeRemoved(long size) {
98	List<String> toBeDeletedKeys =	2✔
99	new ArrayList<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
100	long removedSize = 0;	2✔
101	for (Entry<String, Cacheable> entry : lruCacheMap.entrySet()) {	2✔
102	String key = entry.getKey();	2✔
103	Cacheable cacheInfo = entry.getValue();	2✔
104	long memorySize = cacheInfo.getMemorySize();	2✔
105	if (canBeRemoved(cacheInfo)) {	2✔
106	removedSize = removedSize + memorySize;	2✔
107	toBeDeletedKeys.add(key);	2✔
108	// check if after removing the current file size, required
109	// size when added to current size is sufficient to load a
110	// level or not
111	if (lruCacheMemorySize >= (currentSize - memorySize + size)) {	2✔
112	toBeDeletedKeys.clear();	2✔
113	toBeDeletedKeys.add(key);	2✔
114	removedSize = memorySize;	2✔
115	break;	2✔
116	}
117	// check if after removing the added size/removed size,
118	// required size when added to current size is sufficient to
119	// load a level or not
120	else if (lruCacheMemorySize >= (currentSize - removedSize + size)) {	2✔
121	break;	×
122	}
123	}
124	}	2✔
125	// this case will come when iteration is complete over the keys but
126	// still size is not sufficient for level file to be loaded, then we
127	// will not delete any of the keys
128	if ((currentSize - removedSize + size) > lruCacheMemorySize) {	2✔
129	toBeDeletedKeys.clear();	2✔
130	}
131	return toBeDeletedKeys;	2✔
132	}
133
134	/**
135	* @param cacheInfo
136	* @return
137	*/
138	private boolean canBeRemoved(Cacheable cacheInfo) {
139	if (cacheInfo.getAccessCount() > 0) {	2✔
140	return false;	×
141	}
142	return true;	2✔
143	}
144
145	/**
146	* @param key
147	*/
148	public void remove(String key) {
149	synchronized (lruCacheMap) {	2✔
150	removeKey(key);	2✔
151	}	2✔
152	}	2✔
153
154	/**
155	* @param keys
156	*/
157	public void removeAll(List<String> keys) {
158	synchronized (lruCacheMap) {	1✔
159	for (String key : keys) {	1✔
160	removeKey(key);	1✔
161	}	1✔
162	}	1✔
163	}	1✔
164
165	/**
166	* This method will remove the key from lru cache
167	*
168	* @param key
169	*/
170	private void removeKey(String key) {
171	Cacheable cacheable = lruCacheMap.get(key);	2✔
172	if (null != cacheable) {	2✔
173	long memorySize = cacheable.getMemorySize();	2✔
174	cacheable.invalidate();	2✔
175	lruCacheMap.remove(key);	2✔
176	currentSize = currentSize - memorySize;	2✔
177	LOGGER.info("Removed entry from InMemory lru cache :: " + key);	2✔
178	}
179	}	2✔
180
181	/**
182	* This method will check if required size is available in the memory and then add
183	* the given cacheable to object to lru cache
184	*
185	* @param columnIdentifier
186	* @param cacheInfo
187	*/
188	public boolean put(String columnIdentifier, Cacheable cacheInfo, long requiredSize) {
189	if (LOGGER.isDebugEnabled()) {	2✔
190	LOGGER.debug("Required size for entry " + columnIdentifier + " :: " + requiredSize	2✔
191	+ " Current cache size :: " + currentSize);
192	}
193	boolean columnKeyAddedSuccessfully = false;	2✔
194	if (isLRUCacheSizeConfigured()) {	2✔
195	synchronized (lruCacheMap) {	2✔
196	if (freeMemorySizeForAddingCache(requiredSize)) {	2✔
197	currentSize = currentSize + requiredSize;	2✔
198	addEntryToLRUCacheMap(columnIdentifier, cacheInfo);	2✔
199	columnKeyAddedSuccessfully = true;	2✔
200	} else {
201	LOGGER.error(	2✔
202	"Size not available. Entry cannot be added to lru cache :: " + columnIdentifier
203	+ " .Required Size = " + requiredSize + " Size available " + (lruCacheMemorySize
204	- currentSize));
205	}
206	}	2✔
207	} else {
208	synchronized (lruCacheMap) {	2✔
209	addEntryToLRUCacheMap(columnIdentifier, cacheInfo);	2✔
210	}	2✔
211	columnKeyAddedSuccessfully = true;	2✔
212	}
213	return columnKeyAddedSuccessfully;	2✔
214	}
215
216	/**
217	* This method will check if required size is available in the memory
218	* @param columnIdentifier
219	* @param requiredSize
220	* @return
221	*/
222	public boolean tryPut(String columnIdentifier, long requiredSize) {
223	if (LOGGER.isDebugEnabled()) {	2✔
224	LOGGER.debug("checking Required size for entry " + columnIdentifier + " :: " + requiredSize	2✔
225	+ " Current cache size :: " + currentSize);
226	}
227	boolean columnKeyCanBeAdded = false;	2✔
228	if (isLRUCacheSizeConfigured()) {	2✔
229	synchronized (lruCacheMap) {	2✔
230	if (freeMemorySizeForAddingCache(requiredSize)) {	2✔
231	columnKeyCanBeAdded = true;	2✔
232	} else {
233	LOGGER.error(	2✔
234	"Size check failed.Size not available. Entry cannot be added to lru cache :: "
235	+ columnIdentifier + " .Required Size = " + requiredSize + " Size available " + (
236	lruCacheMemorySize - currentSize));
237	}
238	}	2✔
239	} else {
240	columnKeyCanBeAdded = true;	1✔
241	}
242	return columnKeyCanBeAdded;	2✔
243	}
244
245	/**
246	* The method will add the cache entry to LRU cache map
247	*
248	* @param columnIdentifier
249	* @param cacheInfo
250	*/
251	private void addEntryToLRUCacheMap(String columnIdentifier, Cacheable cacheInfo) {
252	if (null == lruCacheMap.get(columnIdentifier)) {	2✔
253	lruCacheMap.put(columnIdentifier, cacheInfo);	2✔
254	}
255	if (LOGGER.isDebugEnabled()) {	2✔
256	LOGGER.debug("Added entry to InMemory lru cache :: " + columnIdentifier);	2✔
257	}
258	}	2✔
259
260	/**
261	* this will check whether the LRU cache size is configured
262	*
263	* @return <Boolean> value
264	*/
265	private boolean isLRUCacheSizeConfigured() {
266	return lruCacheMemorySize > 0;	2✔
267	}
268
269	/**
270	* This method will check a required column can be loaded into memory or not. If required
271	* this method will call for eviction of existing data from memory
272	*
273	* @param requiredSize
274	* @return
275	*/
276	private boolean freeMemorySizeForAddingCache(long requiredSize) {
277	boolean memoryAvailable = false;	2✔
278	if (isSizeAvailableToLoadColumnDictionary(requiredSize)) {	2✔
279	memoryAvailable = true;	2✔
280	} else {
281	// get the keys that can be removed from memory
282	List<String> keysToBeRemoved = getKeysToBeRemoved(requiredSize);	2✔
283	for (String cacheKey : keysToBeRemoved) {	2✔
284	removeKey(cacheKey);	2✔
285	}	2✔
286	// after removing the keys check again if required size is available
287	if (isSizeAvailableToLoadColumnDictionary(requiredSize)) {	2✔
288	memoryAvailable = true;	2✔
289	}
290	}
291	return memoryAvailable;	2✔
292	}
293
294	/**
295	* This method will check if size is available to laod dictionary into memory
296	*
297	* @param requiredSize
298	* @return
299	*/
300	private boolean isSizeAvailableToLoadColumnDictionary(long requiredSize) {
301	return lruCacheMemorySize >= (currentSize + requiredSize);	2✔
302	}
303
304	/**
305	* @param key
306	* @return
307	*/
308	public Cacheable get(String key) {
309	synchronized (lruCacheMap) {	2✔
310	return lruCacheMap.get(key);	2✔
311	}	×
312	}
313
314	/**
315	* This method will empty the level cache
316	*/
317	public void clear() {
318	synchronized (lruCacheMap) {	2✔
319	for (Cacheable cachebleObj : lruCacheMap.values()) {	2✔
320	cachebleObj.invalidate();	2✔
321	}	2✔
322	lruCacheMap.clear();	2✔
323	}	2✔
324	}	2✔
325
326	public Map<String, Cacheable> getCacheMap() {
327	return lruCacheMap;	1✔
328	}
329	}

apache / carbondata / 10916

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous