#5214

Committed 16 Feb 2026 01:14AM UTC coverage: 16.352% (-73.5%) from 89.9%

Build # #5214

Build Type

Pull #3672

circleci

Committed by

renovate-bot

Commit Message

chore(deps): lock file maintenance (pep621)

Pull Request Pull Request #3672: chore(deps): lock file maintenance (pep621)

Run Details

267 of 2546 branches covered (10.49%)

Branch coverage included in aggregate %.

1169 of 6236 relevant lines covered (18.75%)

0.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.35

/src/auslib/util/cache.py

import pickle
import time
from copy import deepcopy

from repoze.lru import ExpiringLRUCache

from auslib.util.statsd import statsd

uncached_sentinel = object()


class MaybeCacher(object):
    """MaybeCacher is a very simple wrapper to work around the fact that we
    have two consumers of the auslib library (admin app, non-admin app) that
    require cache different things. Most notably, the non-admin app caches
    blobs and blob versions, while the admin app only caches blobs (because
    blob versions can change frequently). This class is intended to be
    instantiated as a global object, and then have caches created by consumers
    through calls to make_cache. Consumers that make changes (ie: the admin
    app) generally also set make_copies to True to get the cache to copy values
    on get/put to avoid the possibility of accidental cache pollution. For
    performance reasons, this should be disabled when not necessary.

    If the cache given to get/put/clear/invalidate doesn't exist, these methods
    are essentially no-ops. In a world where bug 1109295 is fixed, we might
    only need to handle the caching case."""

    def __init__(self):
        self.caches = {}
        self._make_copies = False
        # Ideally, we'd take in the cache class as an argument to this constructor.
        # Due to the way that everything in Balrog is initialized, this doesn't
        # work. We also can't use Flask in here for similar reasons, so we can't
        # pull it from application context. It's for these reasons we need to make
        # this customizable as a property. If that's all that mattered, we
        # could just set the property to a class. However, to support Redis we
        # _also_ need a Redis client passed in. The sanest way to do this is
        # to allow the caller to provide it in a closure, hence we end up
        # making this a callable instead of a simple class.
        self._factory = lambda _, maxsize, timeout: ExpiringLRUCache(maxsize, timeout)

    @property
    def factory(self):
        return self._factory

    @factory.setter
    def factory(self, value):
        if not callable(value):
            raise ValueError("factory must be callable!")
        self._factory = value

    @property
    def make_copies(self):
        return self._make_copies

    @make_copies.setter
    def make_copies(self, value):
        if value not in (True, False):
            raise TypeError("make_copies must be True or False")
        self._make_copies = value

    def make_cache(self, name, maxsize, timeout):
        if name in self.caches:
            raise Exception()

        self.caches[name] = self.factory(name, maxsize, timeout)

    def reset(self):
        self.caches.clear()

    def get(self, name, key, value_getter=None):
        """Returns the value of the specified key from the named cache.
        If value_getter is provided and no cache is found, or no value is
        found for the key, the return value of value_getter will be returned
        instead."""

        if name not in self.caches:
            if callable(value_getter):
                return value_getter()
            else:
                return None

        value = None
        cached_value = self.caches[name].get(key, uncached_sentinel)
        # If we got something other than a sentinel value, the key was in the cache, and we should return it
        if cached_value != uncached_sentinel:
            value = cached_value
            statsd.incr(f"cache.{name}.hits")
        else:
            # If we know how to look up the value, go do it, cache it, and return it
            if callable(value_getter):
                value = value_getter()
                self.put(name, key, value)
            statsd.incr(f"cache.{name}.misses")

        if self.make_copies:
            return deepcopy(value)
        else:
            return value

    def put(self, name, key, value):
        if name not in self.caches:
            return

        if self.make_copies:
            value = deepcopy(value)
        return self.caches[name].put(key, value)

    def clear(self, name=None):
        if name and name not in self.caches:
            return

        if not name:
            for c in self.caches.values():
                c.clear()
        else:
            self.caches[name].clear()

    def invalidate(self, name, key):
        if name not in self.caches:
            return

        self.caches[name].invalidate(key)


class RedisCache:
    """A thin wrapper around the redis client to expose a similar interface
    as ExpiringLRUCache. Unlike ExpiringLRUCache, redis does not support
    non-trivial objects, so objects are jsonified before storage and parsed
    upon retrieval.

    This cache can be used on its own, but ideally it is only used through a
    TwoLayerCache (see below).
    """

    def __init__(self, redis, name, timeout):
        self._name = name
        self._redis = redis
        # redis and repoze calculate expiry slightly differently; a timeout of
        # 5 seconds with repoze ends up being 6 seconds in redis. this really
        # doesn't matter...but it's better to be consistent than not, and redis'
        # behaviour is slightly confusing, so we make this small improvement
        # since we're wrapping it anyways.
        self._timeout = timeout - 1
        self.lookups = 0
        self.hits = 0
        self.misses = 0

    def fullkey(self, key):
        return f"{self._name}-{key}"

    def get(self, key, default=None):
        self.lookups += 1
        value = self._redis.get(self.fullkey(key))
        if value is not None:
            self.hits += 1
            return pickle.loads(value)

        self.misses += 1
        return default

    def put(self, key, value):
        # orjson dumps and loads faster than pickle, but we have a need to
        # support python objects (most notably: Blob instances). Pickle is
        # not too much slower than orjson, so we stick with that for now.
        self._redis.setex(self.fullkey(key), self._timeout, pickle.dumps(value))

    def clear(self):
        self._redis.delete(*self._redis.keys(self._name))

    def invalidate(self, key):
        self._redis.delete(self.fullkey(key))

    def remaining_timeout(self, key):
        absolute_timeout = self._redis.expiretime(self.fullkey(key))
        return absolute_timeout - time.time()


class TwoLayerCache:
    """A cache that wraps both a RedisCache and ExpiringLRUCache. The
    former is treated as authoritative, while the latter is used to minimize
    unnecessary fetches from Redis. This design allows caches to be shared
    across many pods while minimizing the perf impact of having an off-machine
    cache."""

    def __init__(self, redis, name, maxsize, timeout):
        self._redis_cache = RedisCache(redis, name, timeout)
        self._lru_cache = ExpiringLRUCache(maxsize, timeout)
        self.lookups = 0
        self.hits = 0
        self.misses = 0

    def get(self, key, default=None):
        self.lookups += 1
        value = self._lru_cache.get(key, default)
        if value == default:
            value = self._redis_cache.get(key, default)
            # ensure the LRU cache timeout matches the one in redis
            # this is important to ensure that when new pods spin up that
            # they will not cache keys for longer than redis
            # this ensures that we don't have to wait for multiple caches
            # to expire to refresh data.
            # in practice there will be a very small difference between the
            # timeouts, because the value we pass to put here is a relative
            # timeout in seconds, which the lru cache recalculates against
            # `time.time()`. this small (probably 1s in most cases) difference
            # is unlikely to be problematic in practice.
            self._lru_cache.put(key, value, self._redis_cache.remaining_timeout(key))
            if value == default:
                self.misses += 1
            else:
                self.hits += 1
        else:
            self.hits += 1

        return value

    def put(self, key, value):
        self._redis_cache.put(key, value)
        self._lru_cache.put(key, value)

    def clear(self):
        self._redis_cache.clear()
        self._lru_cache.clear()

    def invalidate(self, key):
        self._redis_cache.invalidate(key)
        self._lru_cache.invalidate(key)

1	import pickle	1✔
2	import time	1✔
3	from copy import deepcopy	1✔
4
5	from repoze.lru import ExpiringLRUCache	1✔
6
7	from auslib.util.statsd import statsd	1✔
8
9	uncached_sentinel = object()	1✔
10
11
12	class MaybeCacher(object):	1✔
13	"""MaybeCacher is a very simple wrapper to work around the fact that we
14	have two consumers of the auslib library (admin app, non-admin app) that
15	require cache different things. Most notably, the non-admin app caches
16	blobs and blob versions, while the admin app only caches blobs (because
17	blob versions can change frequently). This class is intended to be
18	instantiated as a global object, and then have caches created by consumers
19	through calls to make_cache. Consumers that make changes (ie: the admin
20	app) generally also set make_copies to True to get the cache to copy values
21	on get/put to avoid the possibility of accidental cache pollution. For
22	performance reasons, this should be disabled when not necessary.
23
24	If the cache given to get/put/clear/invalidate doesn't exist, these methods
25	are essentially no-ops. In a world where bug 1109295 is fixed, we might
26	only need to handle the caching case."""
27
28	def __init__(self):	1✔
29	self.caches = {}	1✔
30	self._make_copies = False	1✔
31	# Ideally, we'd take in the cache class as an argument to this constructor.
32	# Due to the way that everything in Balrog is initialized, this doesn't
33	# work. We also can't use Flask in here for similar reasons, so we can't
34	# pull it from application context. It's for these reasons we need to make
35	# this customizable as a property. If that's all that mattered, we
36	# could just set the property to a class. However, to support Redis we
37	# _also_ need a Redis client passed in. The sanest way to do this is
38	# to allow the caller to provide it in a closure, hence we end up
39	# making this a callable instead of a simple class.
40	self._factory = lambda _, maxsize, timeout: ExpiringLRUCache(maxsize, timeout)	1✔
41
42	@property	1✔
43	def factory(self):	1✔
44	return self._factory	1✔
45
46	@factory.setter	1✔
47	def factory(self, value):	1✔
48	if not callable(value):	×
49	raise ValueError("factory must be callable!")	×
50	self._factory = value	×
51
52	@property	1✔
53	def make_copies(self):	1✔
54	return self._make_copies	1✔
55
56	@make_copies.setter	1✔
57	def make_copies(self, value):	1✔
58	if value not in (True, False):	1!
59	raise TypeError("make_copies must be True or False")	×
60	self._make_copies = value	1✔
61
62	def make_cache(self, name, maxsize, timeout):	1✔
63	if name in self.caches:	1!
64	raise Exception()	×
65
66	self.caches[name] = self.factory(name, maxsize, timeout)	1✔
67
68	def reset(self):	1✔
69	self.caches.clear()	1✔
70
71	def get(self, name, key, value_getter=None):	1✔
72	"""Returns the value of the specified key from the named cache.
73	If value_getter is provided and no cache is found, or no value is
74	found for the key, the return value of value_getter will be returned
75	instead."""
76
77	if name not in self.caches:	1✔
78	if callable(value_getter):	1✔
79	return value_getter()	1✔
80	else:
81	return None	1✔
82
83	value = None	1✔
84	cached_value = self.caches[name].get(key, uncached_sentinel)	1✔
85	# If we got something other than a sentinel value, the key was in the cache, and we should return it
86	if cached_value != uncached_sentinel:	1✔
87	value = cached_value	1✔
88	statsd.incr(f"cache.{name}.hits")	1✔
89	else:
90	# If we know how to look up the value, go do it, cache it, and return it
91	if callable(value_getter):	1✔
92	value = value_getter()	1✔
93	self.put(name, key, value)	1✔
94	statsd.incr(f"cache.{name}.misses")	1✔
95
96	if self.make_copies:	1✔
97	return deepcopy(value)	1✔
98	else:
99	return value	1✔
100
101	def put(self, name, key, value):	1✔
102	if name not in self.caches:	1✔
103	return	1✔
104
105	if self.make_copies:	1✔
106	value = deepcopy(value)	1✔
107	return self.caches[name].put(key, value)	1✔
108
109	def clear(self, name=None):	1✔
110	if name and name not in self.caches:	×
111	return	×
112
113	if not name:	×
114	for c in self.caches.values():	×
115	c.clear()	×
116	else:
117	self.caches[name].clear()	×
118
119	def invalidate(self, name, key):	1✔
120	if name not in self.caches:	×
121	return	×
122
123	self.caches[name].invalidate(key)	×
124
125
126	class RedisCache:	1✔
127	"""A thin wrapper around the redis client to expose a similar interface
128	as ExpiringLRUCache. Unlike ExpiringLRUCache, redis does not support
129	non-trivial objects, so objects are jsonified before storage and parsed
130	upon retrieval.
131
132	This cache can be used on its own, but ideally it is only used through a
133	TwoLayerCache (see below).
134	"""
135
136	def __init__(self, redis, name, timeout):	1✔
137	self._name = name	1✔
138	self._redis = redis	1✔
139	# redis and repoze calculate expiry slightly differently; a timeout of
140	# 5 seconds with repoze ends up being 6 seconds in redis. this really
141	# doesn't matter...but it's better to be consistent than not, and redis'
142	# behaviour is slightly confusing, so we make this small improvement
143	# since we're wrapping it anyways.
144	self._timeout = timeout - 1	1✔
145	self.lookups = 0	1✔
146	self.hits = 0	1✔
147	self.misses = 0	1✔
148
149	def fullkey(self, key):	1✔
150	return f"{self._name}-{key}"	1✔
151
152	def get(self, key, default=None):	1✔
153	self.lookups += 1	1✔
154	value = self._redis.get(self.fullkey(key))	1✔
155	if value is not None:	1✔
156	self.hits += 1	1✔
157	return pickle.loads(value)	1✔
158
159	self.misses += 1	1✔
160	return default	1✔
161
162	def put(self, key, value):	1✔
163	# orjson dumps and loads faster than pickle, but we have a need to
164	# support python objects (most notably: Blob instances). Pickle is
165	# not too much slower than orjson, so we stick with that for now.
166	self._redis.setex(self.fullkey(key), self._timeout, pickle.dumps(value))	1✔
167
168	def clear(self):	1✔
169	self._redis.delete(*self._redis.keys(self._name))	×
170
171	def invalidate(self, key):	1✔
172	self._redis.delete(self.fullkey(key))	×
173
174	def remaining_timeout(self, key):	1✔
175	absolute_timeout = self._redis.expiretime(self.fullkey(key))	1✔
176	return absolute_timeout - time.time()	1✔
177
178
179	class TwoLayerCache:	1✔
180	"""A cache that wraps both a RedisCache and ExpiringLRUCache. The
181	former is treated as authoritative, while the latter is used to minimize
182	unnecessary fetches from Redis. This design allows caches to be shared
183	across many pods while minimizing the perf impact of having an off-machine
184	cache."""
185
186	def __init__(self, redis, name, maxsize, timeout):	1✔
187	self._redis_cache = RedisCache(redis, name, timeout)	1✔
188	self._lru_cache = ExpiringLRUCache(maxsize, timeout)	1✔
189	self.lookups = 0	1✔
190	self.hits = 0	1✔
191	self.misses = 0	1✔
192
193	def get(self, key, default=None):	1✔
194	self.lookups += 1	1✔
195	value = self._lru_cache.get(key, default)	1✔
196	if value == default:	1✔
197	value = self._redis_cache.get(key, default)	1✔
198	# ensure the LRU cache timeout matches the one in redis
199	# this is important to ensure that when new pods spin up that
200	# they will not cache keys for longer than redis
201	# this ensures that we don't have to wait for multiple caches
202	# to expire to refresh data.
203	# in practice there will be a very small difference between the
204	# timeouts, because the value we pass to put here is a relative
205	# timeout in seconds, which the lru cache recalculates against
206	# `time.time()`. this small (probably 1s in most cases) difference
207	# is unlikely to be problematic in practice.
208	self._lru_cache.put(key, value, self._redis_cache.remaining_timeout(key))	1✔
209	if value == default:	1✔
210	self.misses += 1	1✔
211	else:
212	self.hits += 1	1✔
213	else:
214	self.hits += 1	1✔
215
216	return value	1✔
217
218	def put(self, key, value):	1✔
219	self._redis_cache.put(key, value)	1✔
220	self._lru_cache.put(key, value)	1✔
221
222	def clear(self):	1✔
223	self._redis_cache.clear()	×
224	self._lru_cache.clear()	×
225
226	def invalidate(self, key):	1✔
227	self._redis_cache.invalidate(key)	×
228	self._lru_cache.invalidate(key)	×

mozilla-releng / balrog / #5214

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous