4532

Committed 23 Sep 2019 - 19:39 coverage decreased (-49.7%) to 17.596%

Build # 4532

Build Type

buildkite

Committed by Richard van der Hoff

Commit Message

update changelog

Pull Request Pull Request #6079: Add submit_url response parameter to msisdn /requestToken

Run Details

359 of 12986 branches covered (2.76%)

Branch coverage included in aggregate %.

0 of 7 new or added lines in 1 file covered. (0.0%)

18869 existing lines in 281 files now uncovered.

8809 of 39116 relevant lines covered (22.52%)

0.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

28.57

/synapse/storage/keys.py

# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2019 New Vector Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import logging

import six

import attr
from signedjson.key import decode_verify_key_bytes

from synapse.util import batch_iter
from synapse.util.caches.descriptors import cached, cachedList

from ._base import SQLBaseStore

logger = logging.getLogger(__name__)

# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
# despite being deprecated and removed in favor of memoryview
if six.PY2:
    db_binary_type = six.moves.builtins.buffer
else:
    db_binary_type = memoryview


@attr.s(slots=True, frozen=True)
class FetchKeyResult(object):
    verify_key = attr.ib()  # VerifyKey: the key itself
    valid_until_ts = attr.ib()  # int: how long we can use this key for


class KeyStore(SQLBaseStore):
    """Persistence for signature verification keys
    """

    @cached()
    def _get_server_verify_key(self, server_name_and_key_id):
        raise NotImplementedError()

    @cachedList(
        cached_method_name="_get_server_verify_key", list_name="server_name_and_key_ids"
    )
    def get_server_verify_keys(self, server_name_and_key_ids):
        """
        Args:
            server_name_and_key_ids (iterable[Tuple[str, str]]):
                iterable of (server_name, key-id) tuples to fetch keys for

        Returns:
            Deferred: resolves to dict[Tuple[str, str], FetchKeyResult|None]:
                map from (server_name, key_id) -> FetchKeyResult, or None if the key is
                unknown
        """
        keys = {}

        def _get_keys(txn, batch):
            """Processes a batch of keys to fetch, and adds the result to `keys`."""

            # batch_iter always returns tuples so it's safe to do len(batch)
            sql = (
                "SELECT server_name, key_id, verify_key, ts_valid_until_ms "
                "FROM server_signature_keys WHERE 1=0"
            ) + " OR (server_name=? AND key_id=?)" * len(batch)

            txn.execute(sql, tuple(itertools.chain.from_iterable(batch)))

            for row in txn:
                server_name, key_id, key_bytes, ts_valid_until_ms = row

                if ts_valid_until_ms is None:
                    # Old keys may be stored with a ts_valid_until_ms of null,
                    # in which case we treat this as if it was set to `0`, i.e.
                    # it won't match key requests that define a minimum
                    # `ts_valid_until_ms`.
                    ts_valid_until_ms = 0

                res = FetchKeyResult(
                    verify_key=decode_verify_key_bytes(key_id, bytes(key_bytes)),
                    valid_until_ts=ts_valid_until_ms,
                )
                keys[(server_name, key_id)] = res

        def _txn(txn):
            for batch in batch_iter(server_name_and_key_ids, 50):
                _get_keys(txn, batch)
            return keys

        return self.runInteraction("get_server_verify_keys", _txn)

    def store_server_verify_keys(self, from_server, ts_added_ms, verify_keys):
        """Stores NACL verification keys for remote servers.
        Args:
            from_server (str): Where the verification keys were looked up
            ts_added_ms (int): The time to record that the key was added
            verify_keys (iterable[tuple[str, str, FetchKeyResult]]):
                keys to be stored. Each entry is a triplet of
                (server_name, key_id, key).
        """
        key_values = []
        value_values = []
        invalidations = []
        for server_name, key_id, fetch_result in verify_keys:
            key_values.append((server_name, key_id))
            value_values.append(
                (
                    from_server,
                    ts_added_ms,
                    fetch_result.valid_until_ts,
                    db_binary_type(fetch_result.verify_key.encode()),
                )
            )
            # invalidate takes a tuple corresponding to the params of
            # _get_server_verify_key. _get_server_verify_key only takes one
            # param, which is itself the 2-tuple (server_name, key_id).
            invalidations.append((server_name, key_id))

        def _invalidate(res):
            f = self._get_server_verify_key.invalidate
            for i in invalidations:
                f((i,))
            return res

        return self.runInteraction(
            "store_server_verify_keys",
            self._simple_upsert_many_txn,
            table="server_signature_keys",
            key_names=("server_name", "key_id"),
            key_values=key_values,
            value_names=(
                "from_server",
                "ts_added_ms",
                "ts_valid_until_ms",
                "verify_key",
            ),
            value_values=value_values,
        ).addCallback(_invalidate)

    def store_server_keys_json(
        self, server_name, key_id, from_server, ts_now_ms, ts_expires_ms, key_json_bytes
    ):
        """Stores the JSON bytes for a set of keys from a server
        The JSON should be signed by the originating server, the intermediate
        server, and by this server. Updates the value for the
        (server_name, key_id, from_server) triplet if one already existed.
        Args:
            server_name (str): The name of the server.
            key_id (str): The identifer of the key this JSON is for.
            from_server (str): The server this JSON was fetched from.
            ts_now_ms (int): The time now in milliseconds.
            ts_valid_until_ms (int): The time when this json stops being valid.
            key_json (bytes): The encoded JSON.
        """
        return self._simple_upsert(
            table="server_keys_json",
            keyvalues={
                "server_name": server_name,
                "key_id": key_id,
                "from_server": from_server,
            },
            values={
                "server_name": server_name,
                "key_id": key_id,
                "from_server": from_server,
                "ts_added_ms": ts_now_ms,
                "ts_valid_until_ms": ts_expires_ms,
                "key_json": db_binary_type(key_json_bytes),
            },
            desc="store_server_keys_json",
        )

    def get_server_keys_json(self, server_keys):
        """Retrive the key json for a list of server_keys and key ids.
        If no keys are found for a given server, key_id and source then
        that server, key_id, and source triplet entry will be an empty list.
        The JSON is returned as a byte array so that it can be efficiently
        used in an HTTP response.
        Args:
            server_keys (list): List of (server_name, key_id, source) triplets.
        Returns:
            Deferred[dict[Tuple[str, str, str|None], list[dict]]]:
                Dict mapping (server_name, key_id, source) triplets to lists of dicts
        """

        def _get_server_keys_json_txn(txn):
            results = {}
            for server_name, key_id, from_server in server_keys:
                keyvalues = {"server_name": server_name}
                if key_id is not None:
                    keyvalues["key_id"] = key_id
                if from_server is not None:
                    keyvalues["from_server"] = from_server
                rows = self._simple_select_list_txn(
                    txn,
                    "server_keys_json",
                    keyvalues=keyvalues,
                    retcols=(
                        "key_id",
                        "from_server",
                        "ts_added_ms",
                        "ts_valid_until_ms",
                        "key_json",
                    ),
                )
                results[(server_name, key_id, from_server)] = rows
            return results

        return self.runInteraction("get_server_keys_json", _get_server_keys_json_txn)

1	# -- coding: utf-8 --
2	# Copyright 2014-2016 OpenMarket Ltd
3	# Copyright 2019 New Vector Ltd.
4	#
5	# Licensed under the Apache License, Version 2.0 (the "License");
6	# you may not use this file except in compliance with the License.
7	# You may obtain a copy of the License at
8	#
9	# http://www.apache.org/licenses/LICENSE-2.0
10	#
11	# Unless required by applicable law or agreed to in writing, software
12	# distributed under the License is distributed on an "AS IS" BASIS,
13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	# See the License for the specific language governing permissions and
15	# limitations under the License.
16
17	import itertools	1×
18	import logging	1×
19
20	import six	1×
21
22	import attr	1×
23	from signedjson.key import decode_verify_key_bytes	1×
24
25	from synapse.util import batch_iter	1×
26	from synapse.util.caches.descriptors import cached, cachedList	1×
27
28	from ._base import SQLBaseStore	1×
29
30	logger = logging.getLogger(__name__)	1×
31
32	# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
33	# despite being deprecated and removed in favor of memoryview
34	if six.PY2:	Branches [[0, 35]] missed. 1×
35	db_binary_type = six.moves.builtins.buffer	!
36	else:
37	db_binary_type = memoryview	1×
38
39
40	@attr.s(slots=True, frozen=True)	1×
41	class FetchKeyResult(object):	1×
42	verify_key = attr.ib() # VerifyKey: the key itself	1×
43	valid_until_ts = attr.ib() # int: how long we can use this key for	1×
44
45
46	class KeyStore(SQLBaseStore):	1×
47	"""Persistence for signature verification keys
48	"""
49
50	@cached()	1×
51	def _get_server_verify_key(self, server_name_and_key_id):
52	raise NotImplementedError()	!
53
54	@cachedList(	1×
55	cached_method_name="_get_server_verify_key", list_name="server_name_and_key_ids"
56	)
57	def get_server_verify_keys(self, server_name_and_key_ids):
58	"""
59	Args:
60	server_name_and_key_ids (iterable[Tuple[str, str]]):
61	iterable of (server_name, key-id) tuples to fetch keys for
62
63	Returns:
64	Deferred: resolves to dict[Tuple[str, str], FetchKeyResult\|None]:
65	map from (server_name, key_id) -> FetchKeyResult, or None if the key is
66	unknown
67	"""
UNCOV 68	keys = {}	!
69
UNCOV 70	def _get_keys(txn, batch):	!
71	"""Processes a batch of keys to fetch, and adds the result to `keys`."""
72
73	# batch_iter always returns tuples so it's safe to do len(batch)
UNCOV 74	sql = (	!
75	"SELECT server_name, key_id, verify_key, ts_valid_until_ms "
76	"FROM server_signature_keys WHERE 1=0"
77	) + " OR (server_name=? AND key_id=?)" * len(batch)
78
UNCOV 79	txn.execute(sql, tuple(itertools.chain.from_iterable(batch)))	!
80
UNCOV 81	for row in txn:	Branches [[0, 70], [0, 82]] missed. !
UNCOV 82	server_name, key_id, key_bytes, ts_valid_until_ms = row	!
83
UNCOV 84	if ts_valid_until_ms is None:	Branches [[0, 89], [0, 91]] missed. !
85	# Old keys may be stored with a ts_valid_until_ms of null,
86	# in which case we treat this as if it was set to `0`, i.e.
87	# it won't match key requests that define a minimum
88	# `ts_valid_until_ms`.
89	ts_valid_until_ms = 0	!
90
UNCOV 91	res = FetchKeyResult(	!
92	verify_key=decode_verify_key_bytes(key_id, bytes(key_bytes)),
93	valid_until_ts=ts_valid_until_ms,
94	)
UNCOV 95	keys[(server_name, key_id)] = res	!
96
UNCOV 97	def _txn(txn):	!
UNCOV 98	for batch in batch_iter(server_name_and_key_ids, 50):	Branches [[0, 99], [0, 100]] missed. !
UNCOV 99	_get_keys(txn, batch)	!
UNCOV 100	return keys	!
101
UNCOV 102	return self.runInteraction("get_server_verify_keys", _txn)	!
103
104	def store_server_verify_keys(self, from_server, ts_added_ms, verify_keys):	1×
105	"""Stores NACL verification keys for remote servers.
106	Args:
107	from_server (str): Where the verification keys were looked up
108	ts_added_ms (int): The time to record that the key was added
109	verify_keys (iterable[tuple[str, str, FetchKeyResult]]):
110	keys to be stored. Each entry is a triplet of
111	(server_name, key_id, key).
112	"""
UNCOV 113	key_values = []	!
UNCOV 114	value_values = []	!
UNCOV 115	invalidations = []	!
UNCOV 116	for server_name, key_id, fetch_result in verify_keys:	Branches [[0, 117], [0, 131]] missed. !
UNCOV 117	key_values.append((server_name, key_id))	!
UNCOV 118	value_values.append(	!
119	(
120	from_server,
121	ts_added_ms,
122	fetch_result.valid_until_ts,
123	db_binary_type(fetch_result.verify_key.encode()),
124	)
125	)
126	# invalidate takes a tuple corresponding to the params of
127	# _get_server_verify_key. _get_server_verify_key only takes one
128	# param, which is itself the 2-tuple (server_name, key_id).
UNCOV 129	invalidations.append((server_name, key_id))	!
130
UNCOV 131	def _invalidate(res):	!
UNCOV 132	f = self._get_server_verify_key.invalidate	!
UNCOV 133	for i in invalidations:	Branches [[0, 134], [0, 135]] missed. !
UNCOV 134	f((i,))	!
UNCOV 135	return res	!
136
UNCOV 137	return self.runInteraction(	!
138	"store_server_verify_keys",
139	self._simple_upsert_many_txn,
140	table="server_signature_keys",
141	key_names=("server_name", "key_id"),
142	key_values=key_values,
143	value_names=(
144	"from_server",
145	"ts_added_ms",
146	"ts_valid_until_ms",
147	"verify_key",
148	),
149	value_values=value_values,
150	).addCallback(_invalidate)
151
152	def store_server_keys_json(	1×
153	self, server_name, key_id, from_server, ts_now_ms, ts_expires_ms, key_json_bytes
154	):
155	"""Stores the JSON bytes for a set of keys from a server
156	The JSON should be signed by the originating server, the intermediate
157	server, and by this server. Updates the value for the
158	(server_name, key_id, from_server) triplet if one already existed.
159	Args:
160	server_name (str): The name of the server.
161	key_id (str): The identifer of the key this JSON is for.
162	from_server (str): The server this JSON was fetched from.
163	ts_now_ms (int): The time now in milliseconds.
164	ts_valid_until_ms (int): The time when this json stops being valid.
165	key_json (bytes): The encoded JSON.
166	"""
UNCOV 167	return self._simple_upsert(	!
168	table="server_keys_json",
169	keyvalues={
170	"server_name": server_name,
171	"key_id": key_id,
172	"from_server": from_server,
173	},
174	values={
175	"server_name": server_name,
176	"key_id": key_id,
177	"from_server": from_server,
178	"ts_added_ms": ts_now_ms,
179	"ts_valid_until_ms": ts_expires_ms,
180	"key_json": db_binary_type(key_json_bytes),
181	},
182	desc="store_server_keys_json",
183	)
184
185	def get_server_keys_json(self, server_keys):	1×
186	"""Retrive the key json for a list of server_keys and key ids.
187	If no keys are found for a given server, key_id and source then
188	that server, key_id, and source triplet entry will be an empty list.
189	The JSON is returned as a byte array so that it can be efficiently
190	used in an HTTP response.
191	Args:
192	server_keys (list): List of (server_name, key_id, source) triplets.
193	Returns:
194	Deferred[dict[Tuple[str, str, str\|None], list[dict]]]:
195	Dict mapping (server_name, key_id, source) triplets to lists of dicts
196	"""
197
UNCOV 198	def _get_server_keys_json_txn(txn):	!
UNCOV 199	results = {}	!
UNCOV 200	for server_name, key_id, from_server in server_keys:	Branches [[0, 201], [0, 219]] missed. !
UNCOV 201	keyvalues = {"server_name": server_name}	!
UNCOV 202	if key_id is not None:	Branches [[0, 203], [0, 204]] missed. !
UNCOV 203	keyvalues["key_id"] = key_id	!
UNCOV 204	if from_server is not None:	Branches [[0, 205], [0, 206]] missed. !
205	keyvalues["from_server"] = from_server	!
UNCOV 206	rows = self._simple_select_list_txn(	!
207	txn,
208	"server_keys_json",
209	keyvalues=keyvalues,
210	retcols=(
211	"key_id",
212	"from_server",
213	"ts_added_ms",
214	"ts_valid_until_ms",
215	"key_json",
216	),
217	)
UNCOV 218	results[(server_name, key_id, from_server)] = rows	!
UNCOV 219	return results	!
220
UNCOV 221	return self.runInteraction("get_server_keys_json", _get_server_keys_json_txn)	!

matrix-org / synapse / 4532

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous