3f16573b-ba55-4e0d-9e95-22da5729ff60

Committed 15 Jan 2025 05:02PM UTC coverage: 93.143% (+0.001%) from 93.142%

Build # 3f16573b-ba55-4e0d-9e95-22da5729ff60

Build Type

push

circleci

Committed by

snarfed

Commit Message

bump memcache memoize version for Protocol.for_id

when we started caching it in memcache in 2a9f24aee, we also cached Nones for valid web domains that just didn't happen to give us mf2. 2363a488f hopefully fixes that, but we also need to ignore the existing Nones in cache.

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

3 existing lines in 1 file now uncovered.

4510 of 4842 relevant lines covered (93.14%)

0.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.45

/memcache.py

"""Utilities for caching data in memcache."""
import functools
import logging
import os

from google.cloud.ndb.global_cache import _InProcessGlobalCache, MemcacheCache
from oauth_dropins.webutil import appengine_info

from pymemcache.client.base import PooledClient
from pymemcache.serde import PickleSerde
from pymemcache.test.utils import MockMemcacheClient

logger = logging.getLogger(__name__)

# https://github.com/memcached/memcached/wiki/Commands#standard-protocol
KEY_MAX_LEN = 250

MEMOIZE_VERSION = 2


if appengine_info.DEBUG or appengine_info.LOCAL_SERVER:
    logger.info('Using in memory mock memcache')
    memcache = MockMemcacheClient(allow_unicode_keys=True)
    pickle_memcache = MockMemcacheClient(allow_unicode_keys=True, serde=PickleSerde())
    global_cache = _InProcessGlobalCache()
else:
    logger.info('Using production Memorystore memcache')
    memcache = PooledClient(os.environ['MEMCACHE_HOST'], allow_unicode_keys=True,
                            timeout=10, connect_timeout=10) # seconds
    pickle_memcache = PooledClient(os.environ['MEMCACHE_HOST'],
                                   serde=PickleSerde(), allow_unicode_keys=True,
                                   timeout=10, connect_timeout=10)  # seconds
    global_cache = MemcacheCache(memcache)


def key(key):
    """Preprocesses a memcache key. Right now just truncates it to 250 chars.

    https://pymemcache.readthedocs.io/en/latest/apidoc/pymemcache.client.base.html
    https://github.com/memcached/memcached/wiki/Commands#standard-protocol

    TODO: truncate to 250 *UTF-8* chars, to handle Unicode chars in URLs. Related:
    pymemcache Client's allow_unicode_keys constructor kwarg.
    """
    assert isinstance(key, str), repr(key)
    return key[:KEY_MAX_LEN].replace(' ', '%20').encode()


def memoize_key(fn, *args, _version=MEMOIZE_VERSION, **kwargs):
    return key(f'{fn.__qualname__}-{_version}-{repr(args)}-{repr(kwargs)}')


NONE = ()  # empty tuple

def memoize(expire=None, key=None, write=True, version=MEMOIZE_VERSION):
    """Memoize function decorator that stores the cached value in memcache.

    Args:
      expire (timedelta): optional, expiration
      key (callable): function that takes the function's (*args, **kwargs) and
        returns the cache key to use. If it returns None, memcache won't be used.
      write (bool or callable): whether to write to memcache. If this is a
        callable, it will be called with the function's (*args, **kwargs) and should
        return True or False.
      version (int): overrides our default version number in the memcache key.
        Bumping this version can have the same effect as clearing the cache for
        just the affected function.
    """
    if expire:
        expire = int(expire.total_seconds())

    def decorator(fn):
        @functools.wraps(fn)
        def wrapped(*args, **kwargs):
            cache_key = None
            if key:
                key_val = key(*args, **kwargs)
                if key_val:
                    cache_key = memoize_key(fn, key_val, _version=version)
            else:
                cache_key = memoize_key(fn, *args, _version=version, **kwargs)

            if cache_key:
                val = pickle_memcache.get(cache_key)
                if val is not None:
                    logger.debug(f'cache hit {cache_key} {repr(val)[:100]}')
                    return None if val == NONE else val
                else:
                    logger.debug(f'cache miss {cache_key}')

            val = fn(*args, **kwargs)

            if cache_key:
                write_cache = (write if isinstance(write, bool)
                               else write(*args, **kwargs))
                if write_cache:
                    logger.debug(f'cache set {cache_key} {repr(val)[:100]}')
                    pickle_memcache.set(cache_key, NONE if val is None else val,
                                        expire=expire)

            return val

        return wrapped

    return decorator

1	"""Utilities for caching data in memcache."""
2	import functools	1✔
3	import logging	1✔
4	import os	1✔
5
6	from google.cloud.ndb.global_cache import _InProcessGlobalCache, MemcacheCache	1✔
7	from oauth_dropins.webutil import appengine_info	1✔
8
9	from pymemcache.client.base import PooledClient	1✔
10	from pymemcache.serde import PickleSerde	1✔
11	from pymemcache.test.utils import MockMemcacheClient	1✔
12
13	logger = logging.getLogger(__name__)	1✔
14
15	# https://github.com/memcached/memcached/wiki/Commands#standard-protocol
16	KEY_MAX_LEN = 250	1✔
17
18	MEMOIZE_VERSION = 2	1✔
19
20
21	if appengine_info.DEBUG or appengine_info.LOCAL_SERVER:	1✔
22	logger.info('Using in memory mock memcache')	1✔
23	memcache = MockMemcacheClient(allow_unicode_keys=True)	1✔
24	pickle_memcache = MockMemcacheClient(allow_unicode_keys=True, serde=PickleSerde())	1✔
25	global_cache = _InProcessGlobalCache()	1✔
26	else:
UNCOV 27	logger.info('Using production Memorystore memcache')	×
28	memcache = PooledClient(os.environ['MEMCACHE_HOST'], allow_unicode_keys=True,	×
29	timeout=10, connect_timeout=10) # seconds
UNCOV 30	pickle_memcache = PooledClient(os.environ['MEMCACHE_HOST'],	×
31	serde=PickleSerde(), allow_unicode_keys=True,
32	timeout=10, connect_timeout=10) # seconds
UNCOV 33	global_cache = MemcacheCache(memcache)	×
34
35
36	def key(key):	1✔
37	"""Preprocesses a memcache key. Right now just truncates it to 250 chars.
38
39	https://pymemcache.readthedocs.io/en/latest/apidoc/pymemcache.client.base.html
40	https://github.com/memcached/memcached/wiki/Commands#standard-protocol
41
42	TODO: truncate to 250 UTF-8 chars, to handle Unicode chars in URLs. Related:
43	pymemcache Client's allow_unicode_keys constructor kwarg.
44	"""
45	assert isinstance(key, str), repr(key)	1✔
46	return key[:KEY_MAX_LEN].replace(' ', '%20').encode()	1✔
47
48
49	def memoize_key(fn, args, _version=MEMOIZE_VERSION, *kwargs):	1✔
50	return key(f'{fn.__qualname__}-{_version}-{repr(args)}-{repr(kwargs)}')	1✔
51
52
53	NONE = () # empty tuple	1✔
54
55	def memoize(expire=None, key=None, write=True, version=MEMOIZE_VERSION):	1✔
56	"""Memoize function decorator that stores the cached value in memcache.
57
58	Args:
59	expire (timedelta): optional, expiration
60	key (callable): function that takes the function's (args, *kwargs) and
61	returns the cache key to use. If it returns None, memcache won't be used.
62	write (bool or callable): whether to write to memcache. If this is a
63	callable, it will be called with the function's (args, *kwargs) and should
64	return True or False.
65	version (int): overrides our default version number in the memcache key.
66	Bumping this version can have the same effect as clearing the cache for
67	just the affected function.
68	"""
69	if expire:	1✔
70	expire = int(expire.total_seconds())	1✔
71
72	def decorator(fn):	1✔
73	@functools.wraps(fn)	1✔
74	def wrapped(args, *kwargs):	1✔
75	cache_key = None	1✔
76	if key:	1✔
77	key_val = key(args, *kwargs)	1✔
78	if key_val:	1✔
79	cache_key = memoize_key(fn, key_val, _version=version)	1✔
80	else:
81	cache_key = memoize_key(fn, args, _version=version, *kwargs)	1✔
82
83	if cache_key:	1✔
84	val = pickle_memcache.get(cache_key)	1✔
85	if val is not None:	1✔
86	logger.debug(f'cache hit {cache_key} {repr(val)[:100]}')	1✔
87	return None if val == NONE else val	1✔
88	else:
89	logger.debug(f'cache miss {cache_key}')	1✔
90
91	val = fn(args, *kwargs)	1✔
92
93	if cache_key:	1✔
94	write_cache = (write if isinstance(write, bool)	1✔
95	else write(args, *kwargs))
96	if write_cache:	1✔
97	logger.debug(f'cache set {cache_key} {repr(val)[:100]}')	1✔
98	pickle_memcache.set(cache_key, NONE if val is None else val,	1✔
99	expire=expire)
100
101	return val	1✔
102
103	return wrapped	1✔
104
105	return decorator	1✔

snarfed / bridgy-fed / 3f16573b-ba55-4e0d-9e95-22da5729ff60

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous