4532

Committed 23 Sep 2019 - 19:39 coverage decreased (-49.7%) to 17.596%

Build # 4532

Build Type

buildkite

Committed by Richard van der Hoff

Commit Message

update changelog

Pull Request Pull Request #6079: Add submit_url response parameter to msisdn /requestToken

Run Details

359 of 12986 branches covered (2.76%)

Branch coverage included in aggregate %.

0 of 7 new or added lines in 1 file covered. (0.0%)

18869 existing lines in 281 files now uncovered.

8809 of 39116 relevant lines covered (22.52%)

0.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

24.64

/synapse/util/caches/dictionary_cache.py

# -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import threading
from collections import namedtuple

from synapse.util.caches.lrucache import LruCache

from . import register_cache

logger = logging.getLogger(__name__)


class DictionaryEntry(namedtuple("DictionaryEntry", ("full", "known_absent", "value"))):
    """Returned when getting an entry from the cache

    Attributes:
        full (bool): Whether the cache has the full or dict or just some keys.
            If not full then not all requested keys will necessarily be present
            in `value`
        known_absent (set): Keys that were looked up in the dict and were not
            there.
        value (dict): The full or partial dict value
    """

    def __len__(self):
        return len(self.value)


class DictionaryCache(object):
    """Caches key -> dictionary lookups, supporting caching partial dicts, i.e.
    fetching a subset of dictionary keys for a particular key.
    """

    def __init__(self, name, max_entries=1000):
        self.cache = LruCache(max_size=max_entries, size_callback=len)

        self.name = name
        self.sequence = 0
        self.thread = None
        # caches_by_name[name] = self.cache

        class Sentinel(object):
            __slots__ = []

        self.sentinel = Sentinel()
        self.metrics = register_cache("dictionary", name, self.cache)

    def check_thread(self):
        expected_thread = self.thread
        if expected_thread is None:
            self.thread = threading.current_thread()
        else:
            if expected_thread is not threading.current_thread():
                raise ValueError(
                    "Cache objects can only be accessed from the main thread"
                )

    def get(self, key, dict_keys=None):
        """Fetch an entry out of the cache

        Args:
            key
            dict_key(list): If given a set of keys then return only those keys
                that exist in the cache.

        Returns:
            DictionaryEntry
        """
        entry = self.cache.get(key, self.sentinel)
        if entry is not self.sentinel:
            self.metrics.inc_hits()

            if dict_keys is None:
                return DictionaryEntry(
                    entry.full, entry.known_absent, dict(entry.value)
                )
            else:
                return DictionaryEntry(
                    entry.full,
                    entry.known_absent,
                    {k: entry.value[k] for k in dict_keys if k in entry.value},
                )

        self.metrics.inc_misses()
        return DictionaryEntry(False, set(), {})

    def invalidate(self, key):
        self.check_thread()

        # Increment the sequence number so that any SELECT statements that
        # raced with the INSERT don't update the cache (SYN-369)
        self.sequence += 1
        self.cache.pop(key, None)

    def invalidate_all(self):
        self.check_thread()
        self.sequence += 1
        self.cache.clear()

    def update(self, sequence, key, value, fetched_keys=None):
        """Updates the entry in the cache

        Args:
            sequence
            key (K)
            value (dict[X,Y]): The value to update the cache with.
            fetched_keys (None|set[X]): All of the dictionary keys which were
                fetched from the database.

                If None, this is the complete value for key K. Otherwise, it
                is used to infer a list of keys which we know don't exist in
                the full dict.
        """
        self.check_thread()
        if self.sequence == sequence:
            # Only update the cache if the caches sequence number matches the
            # number that the cache had before the SELECT was started (SYN-369)
            if fetched_keys is None:
                self._insert(key, value, set())
            else:
                self._update_or_insert(key, value, fetched_keys)

    def _update_or_insert(self, key, value, known_absent):
        # We pop and reinsert as we need to tell the cache the size may have
        # changed

        entry = self.cache.pop(key, DictionaryEntry(False, set(), {}))
        entry.value.update(value)
        entry.known_absent.update(known_absent)
        self.cache[key] = entry

    def _insert(self, key, value, known_absent):
        self.cache[key] = DictionaryEntry(True, known_absent, value)

1	# -- coding: utf-8 --
2	# Copyright 2015, 2016 OpenMarket Ltd
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15
16	import logging	1×
17	import threading	1×
18	from collections import namedtuple	1×
19
20	from synapse.util.caches.lrucache import LruCache	1×
21
22	from . import register_cache	1×
23
24	logger = logging.getLogger(__name__)	1×
25
26
27	class DictionaryEntry(namedtuple("DictionaryEntry", ("full", "known_absent", "value"))):	1×
28	"""Returned when getting an entry from the cache
29
30	Attributes:
31	full (bool): Whether the cache has the full or dict or just some keys.
32	If not full then not all requested keys will necessarily be present
33	in `value`
34	known_absent (set): Keys that were looked up in the dict and were not
35	there.
36	value (dict): The full or partial dict value
37	"""
38
39	def __len__(self):	1×
UNCOV 40	return len(self.value)	!
41
42
43	class DictionaryCache(object):	1×
44	"""Caches key -> dictionary lookups, supporting caching partial dicts, i.e.
45	fetching a subset of dictionary keys for a particular key.
46	"""
47
48	def __init__(self, name, max_entries=1000):	1×
UNCOV 49	self.cache = LruCache(max_size=max_entries, size_callback=len)	!
50
UNCOV 51	self.name = name	!
UNCOV 52	self.sequence = 0	!
UNCOV 53	self.thread = None	!
54	# caches_by_name[name] = self.cache
55
UNCOV 56	class Sentinel(object):	!
UNCOV 57	__slots__ = []	!
58
UNCOV 59	self.sentinel = Sentinel()	!
UNCOV 60	self.metrics = register_cache("dictionary", name, self.cache)	!
61
62	def check_thread(self):	1×
UNCOV 63	expected_thread = self.thread	!
UNCOV 64	if expected_thread is None:	Branches [[0, 65], [0, 67]] missed. !
UNCOV 65	self.thread = threading.current_thread()	!
66	else:
UNCOV 67	if expected_thread is not threading.current_thread():	Branches [[0, 62], [0, 68]] missed. !
68	raise ValueError(	!
69	"Cache objects can only be accessed from the main thread"
70	)
71
72	def get(self, key, dict_keys=None):	1×
73	"""Fetch an entry out of the cache
74
75	Args:
76	key
77	dict_key(list): If given a set of keys then return only those keys
78	that exist in the cache.
79
80	Returns:
81	DictionaryEntry
82	"""
UNCOV 83	entry = self.cache.get(key, self.sentinel)	!
UNCOV 84	if entry is not self.sentinel:	Branches [[0, 85], [0, 98]] missed. !
UNCOV 85	self.metrics.inc_hits()	!
86
UNCOV 87	if dict_keys is None:	Branches [[0, 88], [0, 92]] missed. !
UNCOV 88	return DictionaryEntry(	!
89	entry.full, entry.known_absent, dict(entry.value)
90	)
91	else:
92	return DictionaryEntry(	Branches [[0, 95], [0, 72]] missed. !
93	entry.full,
94	entry.known_absent,
95	{k: entry.value[k] for k in dict_keys if k in entry.value},
96	)
97
UNCOV 98	self.metrics.inc_misses()	!
UNCOV 99	return DictionaryEntry(False, set(), {})	!
100
101	def invalidate(self, key):	1×
102	self.check_thread()	!
103
104	# Increment the sequence number so that any SELECT statements that
105	# raced with the INSERT don't update the cache (SYN-369)
106	self.sequence += 1	!
107	self.cache.pop(key, None)	!
108
109	def invalidate_all(self):	1×
110	self.check_thread()	!
111	self.sequence += 1	!
112	self.cache.clear()	!
113
114	def update(self, sequence, key, value, fetched_keys=None):	1×
115	"""Updates the entry in the cache
116
117	Args:
118	sequence
119	key (K)
120	value (dict[X,Y]): The value to update the cache with.
121	fetched_keys (None\|set[X]): All of the dictionary keys which were
122	fetched from the database.
123
124	If None, this is the complete value for key K. Otherwise, it
125	is used to infer a list of keys which we know don't exist in
126	the full dict.
127	"""
UNCOV 128	self.check_thread()	!
UNCOV 129	if self.sequence == sequence:	Branches [[0, 114], [0, 132]] missed. !
130	# Only update the cache if the caches sequence number matches the
131	# number that the cache had before the SELECT was started (SYN-369)
UNCOV 132	if fetched_keys is None:	Branches [[0, 133], [0, 135]] missed. !
UNCOV 133	self._insert(key, value, set())	!
134	else:
135	self._update_or_insert(key, value, fetched_keys)	!
136
137	def _update_or_insert(self, key, value, known_absent):	1×
138	# We pop and reinsert as we need to tell the cache the size may have
139	# changed
140
141	entry = self.cache.pop(key, DictionaryEntry(False, set(), {}))	!
142	entry.value.update(value)	!
143	entry.known_absent.update(known_absent)	!
144	self.cache[key] = entry	!
145
146	def _insert(self, key, value, known_absent):	1×
UNCOV 147	self.cache[key] = DictionaryEntry(True, known_absent, value)	!

matrix-org / synapse / 4532

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous