9028295268

Committed 10 May 2024 06:31AM UTC coverage: 96.962% (+0.1%) from 96.825%

Build # 9028295268

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #27 from xhdnoah/master

feat: add trace_provider arg

Run Details

17 of 17 new or added lines in 3 files covered. (100.0%)

3 existing lines in 1 file now uncovered.

383 of 395 relevant lines covered (96.96%)

2.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.13

/cachext/backends.py

import abc
import pickle
import time
from threading import RLock, Lock
from .otel import otel_instrument


class BaseBackend(metaclass=abc.ABCMeta):

    def trans_key(self, key):
        if self.prefix is None:
            return key
        return '{}.{}'.format(self.prefix, key)

    @abc.abstractmethod
    def get(self, key):
        raise NotImplementedError

    @abc.abstractmethod
    def get_many(self, keys):
        raise NotImplementedError

    @abc.abstractmethod
    def set(self, key, value, ttl=None):
        raise NotImplementedError

    @abc.abstractmethod
    def set_many(self, mapping, ttl=None):
        raise NotImplementedError

    @abc.abstractmethod
    def delete(self, key):
        raise NotImplementedError

    @abc.abstractmethod
    def delete_many(self, keys):
        raise NotImplementedError

    @abc.abstractmethod
    def incr(self, key, delta=1):
        raise NotImplementedError

    @abc.abstractmethod
    def decr(self, key, delta=1):
        raise NotImplementedError

    @abc.abstractmethod
    def expire(self, key, seconds):
        raise NotImplementedError

    @abc.abstractmethod
    def expireat(self, key, timestamp):
        raise NotImplementedError

    @abc.abstractmethod
    def ttl(self, key):
        raise NotImplementedError

    @abc.abstractmethod
    def exists(self, key):
        raise NotImplementedError

    @abc.abstractmethod
    def clear(self):
        raise NotImplementedError


class Redis(BaseBackend):

    def __init__(self, prefix=None, default_ttl=600, **kwargs):
        import redis
        otel_instrument()
        self._client = redis.StrictRedis(**kwargs)
        self.prefix = prefix
        self.default_ttl = default_ttl

    def get(self, key):
        key = self.trans_key(key)
        v = self._client.get(key)
        return v if v is None else pickle.loads(v)

    def get_many(self, keys):
        keys = [self.trans_key(k) for k in keys]
        values = self._client.mget(keys)
        return [v if v is None else pickle.loads(v) for v in values]

    def set(self, key, value, ttl=None):
        key = self.trans_key(key)
        if ttl is None:
            ttl = self.default_ttl
        else:
            ttl = int(ttl)
        return self._client.set(
            key, pickle.dumps(value),
            ex=ttl)

    def set_many(self, mapping, ttl=None):
        mapping = {self.trans_key(k): pickle.dumps(v)
                   for k, v in mapping.items()}
        rv = self._client.mset(mapping)
        if ttl is None:
            ttl = self.default_ttl
        else:
            ttl = int(ttl)
        for k in mapping.keys():
            self._client.expire(k, ttl)
        return rv

    def delete(self, key):
        key = self.trans_key(key)
        return self._client.delete(key)

    def delete_many(self, keys):
        keys = [self.trans_key(k) for k in keys]
        return self._client.delete(*keys)

    def incr(self, key, delta=1):
        key = self.trans_key(key)
        return self._client.incr(key, delta)

    def decr(self, key, delta=1):
        key = self.trans_key(key)
        return self._client.decr(key, delta)

    def expire(self, key, seconds):
        key = self.trans_key(key)
        return self._client.expire(key, int(seconds))

    def expireat(self, key, timestamp):
        key = self.trans_key(key)
        return self._client.expireat(key, int(timestamp))

    def ttl(self, key):
        key = self.trans_key(key)
        return self._client.ttl(key)

    def exists(self, key):
        key = self.trans_key(key)
        return self._client.exists(key) > 0

    def clear(self):
        self._client.flushdb()
        return True


class Memcached(BaseBackend):

    def __init__(self, prefix=None, default_ttl=600, **kwargs):
        import pylibmc
        self._client = pylibmc.Client(**kwargs)
        self.prefix = prefix
        self.default_ttl = default_ttl

    def get(self, key):
        key = self.trans_key(key)
        return self._client.get(key)

    def get_many(self, keys):
        keys = [self.trans_key(k) for k in keys]
        maps = self._client.get_multi(keys)
        return [maps.get(k, None) for k in keys]

    def set(self, key, value, ttl=None):
        key = self.trans_key(key)
        if ttl is None:
            ttl = self.default_ttl
        else:
            ttl = int(ttl)
        return self._client.set(key, value, time=ttl)

    def set_many(self, mapping, ttl=None):
        mapping = {self.trans_key(k): v
                   for k, v in mapping.items()}
        if ttl is None:
            ttl = self.default_ttl
        else:
            ttl = int(ttl)
        return self._client.set_multi(mapping, time=ttl)

    def delete(self, key):
        key = self.trans_key(key)
        return self._client.delete(key)

    def delete_many(self, keys):
        keys = [self.trans_key(k) for k in keys]
        return self._client.delete_multi(keys)

    def incr(self, key, delta=1):
        key = self.trans_key(key)
        return self._client.incr(key, delta)

    def decr(self, key, delta=1):
        key = self.trans_key(key)
        return self._client.decr(key, delta)

    def expire(self, key, seconds):
        key = self.trans_key(key)
        return self._client.touch(key, int(seconds))

    def expireat(self, key, timestamp):
        key = self.trans_key(key)
        value = self._client.get(key)
        now = time.time()
        delta = int(timestamp - now)
        return self._client.replace(key, value, delta)

    def ttl(self, key):
        raise NotImplementedError

    def exists(self, key):
        key = self.trans_key(key)
        return self._client.get(key) is not None

    def clear(self):
        self._client.flush_all()
        return True


class Simple(BaseBackend):

    def __init__(self, prefix=None, threshold=500, default_ttl=600):
        self._cache = {}
        self.threshold = threshold
        self.default_ttl = default_ttl
        self.prefix = prefix
        self.rlock = RLock()
        self.lock = Lock()

    def _ttl2expire(self, ttl):
        if ttl is None:
            ttl = self.default_ttl
        now = time.time()
        return int(now + ttl)

    def _expired(self, ts):
        now = time.time()
        return now > ts

    def _prune(self):
        toremove = []
        for k, (exp, v) in self._cache.items():
            if self._expired(exp):
                toremove.append(k)
        for k in toremove:
            self._cache.pop(k, None)
        return len(self._cache)

    def get(self, key):
        key = self.trans_key(key)
        exp, v = self._cache.get(key, (None, None))
        if exp is None:
            return None
        if self._expired(exp):
            self._cache.pop(key)
            return None
        return v

    def get_many(self, keys):
        return [self.get(k) for k in keys]

    def set(self, key, value, ttl=None):
        key = self.trans_key(key)
        with self.rlock:
            if len(self._cache) >= self.threshold \
                    and self._prune() >= self.threshold:
                return False
            self._cache[key] = (
                self._ttl2expire(ttl), value)
        return True

    def set_many(self, mapping, ttl=None):
        count = len(mapping.keys())
        with self.rlock:
            if len(self._cache) + count >= self.threshold \
                    and self._prune() + count >= self.threshold:
                return False
            for k, v in mapping.items():
                self._cache[self.trans_key(k)] = (
                    self._ttl2expire(ttl), v)
        return True

    def delete(self, key):
        key = self.trans_key(key)
        try:
            self._cache.pop(key)
            return 1
        except KeyError:
            pass
        return 0

    def delete_many(self, keys):
        with self.rlock:
            return sum([self.delete(k) for k in keys])

    def incr(self, key, delta=1):
        key = self.trans_key(key)
        with self.lock:
            exp, v = self._cache.get(key)
            v += delta
            self._cache[key] = (
                exp, v)
        return v

    def decr(self, key, delta=1):
        key = self.trans_key(key)
        with self.lock:
            exp, v = self._cache.get(key)
            v -= delta
            self._cache[key] = (
                exp, v)
        return v

    def expire(self, key, seconds):
        key = self.trans_key(key)
        try:
            exp, v = self._cache[key]
        except KeyError:
            return 0
        self._cache[key] = (self._ttl2expire(seconds), v)
        return 1

    def expireat(self, key, timestamp):
        key = self.trans_key(key)
        try:
            exp, v = self._cache[key]
        except KeyError:
            return 0
        self._cache[key] = (timestamp, v)
        return 1

    def ttl(self, key):
        key = self.trans_key(key)
        try:
            exp, v = self._cache[key]
        except KeyError:
            return -2
        ttl = exp - time.time()
        if ttl < 0:
            self._cache.pop(key)
            return -2
        return int(ttl)

    def exists(self, key):
        key = self.trans_key(key)
        return key in self._cache

    def clear(self):
        self._cache.clear()
        return True

1	import abc	3✔
2	import pickle	3✔
3	import time	3✔
4	from threading import RLock, Lock	3✔
5	from .otel import otel_instrument	3✔
6
7
8	class BaseBackend(metaclass=abc.ABCMeta):	3✔
9
10	def trans_key(self, key):	3✔
11	if self.prefix is None:	3✔
12	return key	3✔
13	return '{}.{}'.format(self.prefix, key)	3✔
14
15	@abc.abstractmethod	3✔
16	def get(self, key):	2✔
17	raise NotImplementedError	3✔
18
19	@abc.abstractmethod	3✔
20	def get_many(self, keys):	2✔
21	raise NotImplementedError	3✔
22
23	@abc.abstractmethod	3✔
24	def set(self, key, value, ttl=None):	3✔
25	raise NotImplementedError	3✔
26
27	@abc.abstractmethod	3✔
28	def set_many(self, mapping, ttl=None):	3✔
29	raise NotImplementedError	3✔
30
31	@abc.abstractmethod	3✔
32	def delete(self, key):	2✔
33	raise NotImplementedError	3✔
34
35	@abc.abstractmethod	3✔
36	def delete_many(self, keys):	2✔
37	raise NotImplementedError	3✔
38
39	@abc.abstractmethod	3✔
40	def incr(self, key, delta=1):	3✔
41	raise NotImplementedError	×
42
43	@abc.abstractmethod	3✔
44	def decr(self, key, delta=1):	3✔
45	raise NotImplementedError	×
46
47	@abc.abstractmethod	3✔
48	def expire(self, key, seconds):	2✔
49	raise NotImplementedError	3✔
50
51	@abc.abstractmethod	3✔
52	def expireat(self, key, timestamp):	2✔
53	raise NotImplementedError	3✔
54
55	@abc.abstractmethod	3✔
56	def ttl(self, key):	2✔
57	raise NotImplementedError	3✔
58
59	@abc.abstractmethod	3✔
60	def exists(self, key):	2✔
61	raise NotImplementedError	3✔
62
63	@abc.abstractmethod	3✔
64	def clear(self):	2✔
65	raise NotImplementedError	3✔
66
67
68	class Redis(BaseBackend):	3✔
69
70	def __init__(self, prefix=None, default_ttl=600, **kwargs):	3✔
71	import redis	3✔
72	otel_instrument()	3✔
73	self._client = redis.StrictRedis(**kwargs)	3✔
74	self.prefix = prefix	3✔
75	self.default_ttl = default_ttl	3✔
76
77	def get(self, key):	3✔
78	key = self.trans_key(key)	3✔
79	v = self._client.get(key)	3✔
80	return v if v is None else pickle.loads(v)	3✔
81
82	def get_many(self, keys):	3✔
83	keys = [self.trans_key(k) for k in keys]	3✔
84	values = self._client.mget(keys)	3✔
85	return [v if v is None else pickle.loads(v) for v in values]	3✔
86
87	def set(self, key, value, ttl=None):	3✔
88	key = self.trans_key(key)	3✔
89	if ttl is None:	3✔
90	ttl = self.default_ttl	3✔
91	else:
92	ttl = int(ttl)	3✔
93	return self._client.set(	3✔
94	key, pickle.dumps(value),
95	ex=ttl)
96
97	def set_many(self, mapping, ttl=None):	3✔
98	mapping = {self.trans_key(k): pickle.dumps(v)	3✔
99	for k, v in mapping.items()}
100	rv = self._client.mset(mapping)	3✔
101	if ttl is None:	3✔
UNCOV 102	ttl = self.default_ttl	×
103	else:
104	ttl = int(ttl)	3✔
105	for k in mapping.keys():	3✔
106	self._client.expire(k, ttl)	3✔
107	return rv	3✔
108
109	def delete(self, key):	3✔
110	key = self.trans_key(key)	3✔
111	return self._client.delete(key)	3✔
112
113	def delete_many(self, keys):	3✔
114	keys = [self.trans_key(k) for k in keys]	3✔
115	return self._client.delete(*keys)	3✔
116
117	def incr(self, key, delta=1):	3✔
118	key = self.trans_key(key)	3✔
119	return self._client.incr(key, delta)	3✔
120
121	def decr(self, key, delta=1):	3✔
122	key = self.trans_key(key)	3✔
123	return self._client.decr(key, delta)	3✔
124
125	def expire(self, key, seconds):	3✔
126	key = self.trans_key(key)	3✔
127	return self._client.expire(key, int(seconds))	3✔
128
129	def expireat(self, key, timestamp):	3✔
130	key = self.trans_key(key)	3✔
131	return self._client.expireat(key, int(timestamp))	3✔
132
133	def ttl(self, key):	3✔
134	key = self.trans_key(key)	3✔
135	return self._client.ttl(key)	3✔
136
137	def exists(self, key):	3✔
138	key = self.trans_key(key)	3✔
139	return self._client.exists(key) > 0	3✔
140
141	def clear(self):	3✔
142	self._client.flushdb()	3✔
143	return True	3✔
144
145
146	class Memcached(BaseBackend):	3✔
147
148	def __init__(self, prefix=None, default_ttl=600, **kwargs):	3✔
149	import pylibmc	3✔
150	self._client = pylibmc.Client(**kwargs)	3✔
151	self.prefix = prefix	3✔
152	self.default_ttl = default_ttl	3✔
153
154	def get(self, key):	3✔
155	key = self.trans_key(key)	3✔
156	return self._client.get(key)	3✔
157
158	def get_many(self, keys):	3✔
159	keys = [self.trans_key(k) for k in keys]	3✔
160	maps = self._client.get_multi(keys)	3✔
161	return [maps.get(k, None) for k in keys]	3✔
162
163	def set(self, key, value, ttl=None):	3✔
164	key = self.trans_key(key)	3✔
165	if ttl is None:	3✔
166	ttl = self.default_ttl	3✔
167	else:
168	ttl = int(ttl)	3✔
169	return self._client.set(key, value, time=ttl)	3✔
170
171	def set_many(self, mapping, ttl=None):	3✔
172	mapping = {self.trans_key(k): v	3✔
173	for k, v in mapping.items()}
174	if ttl is None:	3✔
UNCOV 175	ttl = self.default_ttl	×
176	else:
177	ttl = int(ttl)	3✔
178	return self._client.set_multi(mapping, time=ttl)	3✔
179
180	def delete(self, key):	3✔
181	key = self.trans_key(key)	3✔
182	return self._client.delete(key)	3✔
183
184	def delete_many(self, keys):	3✔
185	keys = [self.trans_key(k) for k in keys]	3✔
186	return self._client.delete_multi(keys)	3✔
187
188	def incr(self, key, delta=1):	3✔
189	key = self.trans_key(key)	3✔
190	return self._client.incr(key, delta)	3✔
191
192	def decr(self, key, delta=1):	3✔
193	key = self.trans_key(key)	3✔
194	return self._client.decr(key, delta)	3✔
195
196	def expire(self, key, seconds):	3✔
197	key = self.trans_key(key)	3✔
198	return self._client.touch(key, int(seconds))	3✔
199
200	def expireat(self, key, timestamp):	3✔
201	key = self.trans_key(key)	3✔
202	value = self._client.get(key)	3✔
203	now = time.time()	3✔
204	delta = int(timestamp - now)	3✔
205	return self._client.replace(key, value, delta)	3✔
206
207	def ttl(self, key):	3✔
208	raise NotImplementedError	3✔
209
210	def exists(self, key):	3✔
211	key = self.trans_key(key)	3✔
212	return self._client.get(key) is not None	3✔
213
214	def clear(self):	3✔
215	self._client.flush_all()	3✔
216	return True	3✔
217
218
219	class Simple(BaseBackend):	3✔
220
221	def __init__(self, prefix=None, threshold=500, default_ttl=600):	3✔
222	self._cache = {}	3✔
223	self.threshold = threshold	3✔
224	self.default_ttl = default_ttl	3✔
225	self.prefix = prefix	3✔
226	self.rlock = RLock()	3✔
227	self.lock = Lock()	3✔
228
229	def _ttl2expire(self, ttl):	3✔
230	if ttl is None:	3✔
231	ttl = self.default_ttl	3✔
232	now = time.time()	3✔
233	return int(now + ttl)	3✔
234
235	def _expired(self, ts):	3✔
236	now = time.time()	3✔
237	return now > ts	3✔
238
239	def _prune(self):	3✔
240	toremove = []	3✔
241	for k, (exp, v) in self._cache.items():	3✔
242	if self._expired(exp):	3✔
243	toremove.append(k)	3✔
244	for k in toremove:	3✔
245	self._cache.pop(k, None)	3✔
246	return len(self._cache)	3✔
247
248	def get(self, key):	3✔
249	key = self.trans_key(key)	3✔
250	exp, v = self._cache.get(key, (None, None))	3✔
251	if exp is None:	3✔
252	return None	3✔
253	if self._expired(exp):	3✔
254	self._cache.pop(key)	3✔
255	return None	3✔
256	return v	3✔
257
258	def get_many(self, keys):	3✔
259	return [self.get(k) for k in keys]	3✔
260
261	def set(self, key, value, ttl=None):	3✔
262	key = self.trans_key(key)	3✔
263	with self.rlock:	3✔
264	if len(self._cache) >= self.threshold \	3✔
265	and self._prune() >= self.threshold:
266	return False	3✔
267	self._cache[key] = (	3✔
268	self._ttl2expire(ttl), value)
269	return True	3✔
270
271	def set_many(self, mapping, ttl=None):	3✔
272	count = len(mapping.keys())	3✔
273	with self.rlock:	3✔
274	if len(self._cache) + count >= self.threshold \	3✔
275	and self._prune() + count >= self.threshold:
UNCOV 276	return False	×
277	for k, v in mapping.items():	3✔
278	self._cache[self.trans_key(k)] = (	3✔
279	self._ttl2expire(ttl), v)
280	return True	3✔
281
282	def delete(self, key):	3✔
283	key = self.trans_key(key)	3✔
284	try:	3✔
285	self._cache.pop(key)	3✔
286	return 1	3✔
287	except KeyError:	3✔
288	pass	3✔
289	return 0	3✔
290
291	def delete_many(self, keys):	3✔
292	with self.rlock:	3✔
293	return sum([self.delete(k) for k in keys])	3✔
294
295	def incr(self, key, delta=1):	3✔
296	key = self.trans_key(key)	3✔
297	with self.lock:	3✔
298	exp, v = self._cache.get(key)	3✔
299	v += delta	3✔
300	self._cache[key] = (	3✔
301	exp, v)
302	return v	3✔
303
304	def decr(self, key, delta=1):	3✔
305	key = self.trans_key(key)	3✔
306	with self.lock:	3✔
307	exp, v = self._cache.get(key)	3✔
308	v -= delta	3✔
309	self._cache[key] = (	3✔
310	exp, v)
311	return v	3✔
312
313	def expire(self, key, seconds):	3✔
314	key = self.trans_key(key)	3✔
315	try:	3✔
316	exp, v = self._cache[key]	3✔
317	except KeyError:	3✔
318	return 0	3✔
319	self._cache[key] = (self._ttl2expire(seconds), v)	3✔
320	return 1	3✔
321
322	def expireat(self, key, timestamp):	3✔
323	key = self.trans_key(key)	3✔
324	try:	3✔
325	exp, v = self._cache[key]	3✔
326	except KeyError:	3✔
327	return 0	3✔
328	self._cache[key] = (timestamp, v)	3✔
329	return 1	3✔
330
331	def ttl(self, key):	3✔
332	key = self.trans_key(key)	3✔
333	try:	3✔
334	exp, v = self._cache[key]	3✔
335	except KeyError:	3✔
336	return -2	3✔
337	ttl = exp - time.time()	3✔
338	if ttl < 0:	3✔
339	self._cache.pop(key)	3✔
340	return -2	3✔
341	return int(ttl)	3✔
342
343	def exists(self, key):	3✔
344	key = self.trans_key(key)	3✔
345	return key in self._cache	3✔
346
347	def clear(self):	3✔
348	self._cache.clear()	3✔
349	return True	3✔

shanbay / cachext / 9028295268

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous