dc61d84f-3de3-467e-a0ac-cc3b31dc470f

Committed 10 Jan 2025 02:36AM UTC coverage: 93.091%. Remained the same

Build # dc61d84f-3de3-467e-a0ac-cc3b31dc470f

Build Type

push

circleci

Committed by

snarfed

Commit Message

memcache: add missing import for prod

Run Details

2 of 4 new or added lines in 1 file covered. (50.0%)

2 existing lines in 1 file now uncovered.

4460 of 4791 relevant lines covered (93.09%)

0.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.24

/memcache.py

"""Utilities for caching data in memcache."""
import functools
import logging
import os

from google.cloud.ndb.global_cache import _InProcessGlobalCache, MemcacheCache
from oauth_dropins.webutil import appengine_info

from pymemcache.client.base import PooledClient
from pymemcache.serde import PickleSerde
from pymemcache.test.utils import MockMemcacheClient

logger = logging.getLogger(__name__)

# https://github.com/memcached/memcached/wiki/Commands#standard-protocol
KEY_MAX_LEN = 250


if appengine_info.DEBUG or appengine_info.LOCAL_SERVER:
    logger.info('Using in memory mock memcache')
    memcache = MockMemcacheClient(allow_unicode_keys=True)
    pickle_memcache = MockMemcacheClient(allow_unicode_keys=True, serde=PickleSerde())
    global_cache = _InProcessGlobalCache()
else:
    logger.info('Using production Memorystore memcache')
    memcache = PooledClient(os.environ['MEMCACHE_HOST'], allow_unicode_keys=True,
                            timeout=10, connect_timeout=10) # seconds
    pickle_memcache = PooledClient(os.environ['MEMCACHE_HOST'],
                                   serde=PickleSerde(), allow_unicode_keys=True,
                                   timeout=10, connect_timeout=10)  # seconds
    global_cache = MemcacheCache(memcache)


def key(key):
    """Preprocesses a memcache key. Right now just truncates it to 250 chars.

    https://pymemcache.readthedocs.io/en/latest/apidoc/pymemcache.client.base.html
    https://github.com/memcached/memcached/wiki/Commands#standard-protocol

    TODO: truncate to 250 *UTF-8* chars, to handle Unicode chars in URLs. Related:
    pymemcache Client's allow_unicode_keys constructor kwarg.
    """
    return key[:KEY_MAX_LEN].replace(' ', '%20').encode()


def memoize_key(fn, *args, **kwargs):
    return key(f'{fn.__name__}-2-{repr(args)}-{repr(kwargs)}')


NONE = ()  # empty tuple

def memoize(expire=None, key=None):
    """Memoize function decorator that stores the cached value in memcache.

    Args:
      expire (timedelta): optional, expiration
      key (callable): function that takes the function's (*args, **kwargs) and
        returns the cache key to use
    """
    if expire:
        expire = int(expire.total_seconds())

    def decorator(fn):
        @functools.wraps(fn)
        def wrapped(*args, **kwargs):
            if key:
                cache_key = memoize_key(fn, key(*args, **kwargs))
            else:
                cache_key = memoize_key(fn, *args, **kwargs)

            val = pickle_memcache.get(cache_key)
            if val is not None:
                # logger.debug(f'cache hit {cache_key}')
                return None if val == NONE else val

            # logger.debug(f'cache miss {cache_key}')
            val = fn(*args, **kwargs)
            pickle_memcache.set(cache_key, NONE if val is None else val, expire=expire)
            return val

        return wrapped

    return decorator

1	"""Utilities for caching data in memcache."""
2	import functools	1✔
3	import logging	1✔
4	import os	1✔
5
6	from google.cloud.ndb.global_cache import _InProcessGlobalCache, MemcacheCache	1✔
7	from oauth_dropins.webutil import appengine_info	1✔
8
9	from pymemcache.client.base import PooledClient	1✔
10	from pymemcache.serde import PickleSerde	1✔
11	from pymemcache.test.utils import MockMemcacheClient	1✔
12
13	logger = logging.getLogger(__name__)	1✔
14
15	# https://github.com/memcached/memcached/wiki/Commands#standard-protocol
16	KEY_MAX_LEN = 250	1✔
17
18
19	if appengine_info.DEBUG or appengine_info.LOCAL_SERVER:	1✔
20	logger.info('Using in memory mock memcache')	1✔
21	memcache = MockMemcacheClient(allow_unicode_keys=True)	1✔
22	pickle_memcache = MockMemcacheClient(allow_unicode_keys=True, serde=PickleSerde())	1✔
23	global_cache = _InProcessGlobalCache()	1✔
24	else:
UNCOV 25	logger.info('Using production Memorystore memcache')	×
NEW 26	memcache = PooledClient(os.environ['MEMCACHE_HOST'], allow_unicode_keys=True,	×
27	timeout=10, connect_timeout=10) # seconds
NEW 28	pickle_memcache = PooledClient(os.environ['MEMCACHE_HOST'],	×
29	serde=PickleSerde(), allow_unicode_keys=True,
30	timeout=10, connect_timeout=10) # seconds
UNCOV 31	global_cache = MemcacheCache(memcache)	×
32
33
34	def key(key):	1✔
35	"""Preprocesses a memcache key. Right now just truncates it to 250 chars.
36
37	https://pymemcache.readthedocs.io/en/latest/apidoc/pymemcache.client.base.html
38	https://github.com/memcached/memcached/wiki/Commands#standard-protocol
39
40	TODO: truncate to 250 UTF-8 chars, to handle Unicode chars in URLs. Related:
41	pymemcache Client's allow_unicode_keys constructor kwarg.
42	"""
43	return key[:KEY_MAX_LEN].replace(' ', '%20').encode()	1✔
44
45
46	def memoize_key(fn, args, *kwargs):	1✔
47	return key(f'{fn.__name__}-2-{repr(args)}-{repr(kwargs)}')	1✔
48
49
50	NONE = () # empty tuple	1✔
51
52	def memoize(expire=None, key=None):	1✔
53	"""Memoize function decorator that stores the cached value in memcache.
54
55	Args:
56	expire (timedelta): optional, expiration
57	key (callable): function that takes the function's (args, *kwargs) and
58	returns the cache key to use
59	"""
60	if expire:	1✔
61	expire = int(expire.total_seconds())	1✔
62
63	def decorator(fn):	1✔
64	@functools.wraps(fn)	1✔
65	def wrapped(args, *kwargs):	1✔
66	if key:	1✔
67	cache_key = memoize_key(fn, key(args, *kwargs))	1✔
68	else:
69	cache_key = memoize_key(fn, args, *kwargs)	1✔
70
71	val = pickle_memcache.get(cache_key)	1✔
72	if val is not None:	1✔
73	# logger.debug(f'cache hit {cache_key}')
74	return None if val == NONE else val	1✔
75
76	# logger.debug(f'cache miss {cache_key}')
77	val = fn(args, *kwargs)	1✔
78	pickle_memcache.set(cache_key, NONE if val is None else val, expire=expire)	1✔
79	return val	1✔
80
81	return wrapped	1✔
82
83	return decorator	1✔

snarfed / bridgy-fed / dc61d84f-3de3-467e-a0ac-cc3b31dc470f

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous