4532

Committed 23 Sep 2019 - 19:39 coverage decreased (-49.7%) to 17.596%

Build # 4532

Build Type

buildkite

Committed by Richard van der Hoff

Commit Message

update changelog

Pull Request Pull Request #6079: Add submit_url response parameter to msisdn /requestToken

Run Details

359 of 12986 branches covered (2.76%)

Branch coverage included in aggregate %.

0 of 7 new or added lines in 1 file covered. (0.0%)

18869 existing lines in 281 files now uncovered.

8809 of 39116 relevant lines covered (22.52%)

0.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

16.85

/synapse/util/ratelimitutils.py

# -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import contextlib
import logging

from twisted.internet import defer

from synapse.api.errors import LimitExceededError
from synapse.logging.context import (
    PreserveLoggingContext,
    make_deferred_yieldable,
    run_in_background,
)

logger = logging.getLogger(__name__)


class FederationRateLimiter(object):
    def __init__(self, clock, config):
        """
        Args:
            clock (Clock)
            config (FederationRateLimitConfig)
        """

        def new_limiter():
            return _PerHostRatelimiter(clock=clock, config=config)

        self.ratelimiters = collections.defaultdict(new_limiter)

    def ratelimit(self, host):
        """Used to ratelimit an incoming request from given host

        Example usage:

            with rate_limiter.ratelimit(origin) as wait_deferred:
                yield wait_deferred
                # Handle request ...

        Args:
            host (str): Origin of incoming request.

        Returns:
            context manager which returns a deferred.
        """
        return self.ratelimiters[host].ratelimit()


class _PerHostRatelimiter(object):
    def __init__(self, clock, config):
        """
        Args:
            clock (Clock)
            config (FederationRateLimitConfig)
        """
        self.clock = clock

        self.window_size = config.window_size
        self.sleep_limit = config.sleep_limit
        self.sleep_sec = config.sleep_delay / 1000.0
        self.reject_limit = config.reject_limit
        self.concurrent_requests = config.concurrent

        # request_id objects for requests which have been slept
        self.sleeping_requests = set()

        # map from request_id object to Deferred for requests which are ready
        # for processing but have been queued
        self.ready_request_queue = collections.OrderedDict()

        # request id objects for requests which are in progress
        self.current_processing = set()

        # times at which we have recently (within the last window_size ms)
        # received requests.
        self.request_times = []

    @contextlib.contextmanager
    def ratelimit(self):
        # `contextlib.contextmanager` takes a generator and turns it into a
        # context manager. The generator should only yield once with a value
        # to be returned by manager.
        # Exceptions will be reraised at the yield.

        request_id = object()
        ret = self._on_enter(request_id)
        try:
            yield ret
        finally:
            self._on_exit(request_id)

    def _on_enter(self, request_id):
        time_now = self.clock.time_msec()

        # remove any entries from request_times which aren't within the window
        self.request_times[:] = [
            r for r in self.request_times if time_now - r < self.window_size
        ]

        # reject the request if we already have too many queued up (either
        # sleeping or in the ready queue).
        queue_size = len(self.ready_request_queue) + len(self.sleeping_requests)
        if queue_size > self.reject_limit:
            raise LimitExceededError(
                retry_after_ms=int(self.window_size / self.sleep_limit)
            )

        self.request_times.append(time_now)

        def queue_request():
            if len(self.current_processing) >= self.concurrent_requests:
                queue_defer = defer.Deferred()
                self.ready_request_queue[request_id] = queue_defer
                logger.info(
                    "Ratelimiter: queueing request (queue now %i items)",
                    len(self.ready_request_queue),
                )

                return queue_defer
            else:
                return defer.succeed(None)

        logger.debug(
            "Ratelimit [%s]: len(self.request_times)=%d",
            id(request_id),
            len(self.request_times),
        )

        if len(self.request_times) > self.sleep_limit:
            logger.debug("Ratelimiter: sleeping request for %f sec", self.sleep_sec)
            ret_defer = run_in_background(self.clock.sleep, self.sleep_sec)

            self.sleeping_requests.add(request_id)

            def on_wait_finished(_):
                logger.debug("Ratelimit [%s]: Finished sleeping", id(request_id))
                self.sleeping_requests.discard(request_id)
                queue_defer = queue_request()
                return queue_defer

            ret_defer.addBoth(on_wait_finished)
        else:
            ret_defer = queue_request()

        def on_start(r):
            logger.debug("Ratelimit [%s]: Processing req", id(request_id))
            self.current_processing.add(request_id)
            return r

        def on_err(r):
            # XXX: why is this necessary? this is called before we start
            # processing the request so why would the request be in
            # current_processing?
            self.current_processing.discard(request_id)
            return r

        def on_both(r):
            # Ensure that we've properly cleaned up.
            self.sleeping_requests.discard(request_id)
            self.ready_request_queue.pop(request_id, None)
            return r

        ret_defer.addCallbacks(on_start, on_err)
        ret_defer.addBoth(on_both)
        return make_deferred_yieldable(ret_defer)

    def _on_exit(self, request_id):
        logger.debug("Ratelimit [%s]: Processed req", id(request_id))
        self.current_processing.discard(request_id)
        try:
            # start processing the next item on the queue.
            _, deferred = self.ready_request_queue.popitem(last=False)

            with PreserveLoggingContext():
                deferred.callback(None)
        except KeyError:
            pass

1	# -- coding: utf-8 --
2	# Copyright 2015, 2016 OpenMarket Ltd
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15
16	import collections	1×
17	import contextlib	1×
18	import logging	1×
19
20	from twisted.internet import defer	1×
21
22	from synapse.api.errors import LimitExceededError	1×
23	from synapse.logging.context import (	1×
24	PreserveLoggingContext,
25	make_deferred_yieldable,
26	run_in_background,
27	)
28
29	logger = logging.getLogger(__name__)	1×
30
31
32	class FederationRateLimiter(object):	1×
33	def __init__(self, clock, config):	1×
34	"""
35	Args:
36	clock (Clock)
37	config (FederationRateLimitConfig)
38	"""
39
UNCOV 40	def new_limiter():	!
UNCOV 41	return _PerHostRatelimiter(clock=clock, config=config)	!
42
UNCOV 43	self.ratelimiters = collections.defaultdict(new_limiter)	!
44
45	def ratelimit(self, host):	1×
46	"""Used to ratelimit an incoming request from given host
47
48	Example usage:
49
50	with rate_limiter.ratelimit(origin) as wait_deferred:
51	yield wait_deferred
52	# Handle request ...
53
54	Args:
55	host (str): Origin of incoming request.
56
57	Returns:
58	context manager which returns a deferred.
59	"""
UNCOV 60	return self.ratelimiters[host].ratelimit()	!
61
62
63	class _PerHostRatelimiter(object):	1×
64	def __init__(self, clock, config):	1×
65	"""
66	Args:
67	clock (Clock)
68	config (FederationRateLimitConfig)
69	"""
UNCOV 70	self.clock = clock	!
71
UNCOV 72	self.window_size = config.window_size	!
UNCOV 73	self.sleep_limit = config.sleep_limit	!
UNCOV 74	self.sleep_sec = config.sleep_delay / 1000.0	!
UNCOV 75	self.reject_limit = config.reject_limit	!
UNCOV 76	self.concurrent_requests = config.concurrent	!
77
78	# request_id objects for requests which have been slept
UNCOV 79	self.sleeping_requests = set()	!
80
81	# map from request_id object to Deferred for requests which are ready
82	# for processing but have been queued
UNCOV 83	self.ready_request_queue = collections.OrderedDict()	!
84
85	# request id objects for requests which are in progress
UNCOV 86	self.current_processing = set()	!
87
88	# times at which we have recently (within the last window_size ms)
89	# received requests.
UNCOV 90	self.request_times = []	!
91
92	@contextlib.contextmanager	1×
93	def ratelimit(self):
94	# `contextlib.contextmanager` takes a generator and turns it into a
95	# context manager. The generator should only yield once with a value
96	# to be returned by manager.
97	# Exceptions will be reraised at the yield.
98
UNCOV 99	request_id = object()	!
UNCOV 100	ret = self._on_enter(request_id)	!
UNCOV 101	try:	!
UNCOV 102	yield ret	!
103	finally:
UNCOV 104	self._on_exit(request_id)	!
105
106	def _on_enter(self, request_id):	1×
UNCOV 107	time_now = self.clock.time_msec()	!
108
109	# remove any entries from request_times which aren't within the window
UNCOV 110	self.request_times[:] = [	Branches [[0, 111], [0, 116]] missed. !
111	r for r in self.request_times if time_now - r < self.window_size
112	]
113
114	# reject the request if we already have too many queued up (either
115	# sleeping or in the ready queue).
UNCOV 116	queue_size = len(self.ready_request_queue) + len(self.sleeping_requests)	!
UNCOV 117	if queue_size > self.reject_limit:	Branches [[0, 118], [0, 122]] missed. !
118	raise LimitExceededError(	!
119	retry_after_ms=int(self.window_size / self.sleep_limit)
120	)
121
UNCOV 122	self.request_times.append(time_now)	!
123
UNCOV 124	def queue_request():	!
UNCOV 125	if len(self.current_processing) >= self.concurrent_requests:	Branches [[0, 126], [0, 135]] missed. !
126	queue_defer = defer.Deferred()	!
127	self.ready_request_queue[request_id] = queue_defer	!
128	logger.info(	!
129	"Ratelimiter: queueing request (queue now %i items)",
130	len(self.ready_request_queue),
131	)
132
133	return queue_defer	!
134	else:
UNCOV 135	return defer.succeed(None)	!
136
UNCOV 137	logger.debug(	!
138	"Ratelimit [%s]: len(self.request_times)=%d",
139	id(request_id),
140	len(self.request_times),
141	)
142
UNCOV 143	if len(self.request_times) > self.sleep_limit:	Branches [[0, 144], [0, 157]] missed. !
144	logger.debug("Ratelimiter: sleeping request for %f sec", self.sleep_sec)	!
145	ret_defer = run_in_background(self.clock.sleep, self.sleep_sec)	!
146
147	self.sleeping_requests.add(request_id)	!
148
149	def on_wait_finished(_):	!
150	logger.debug("Ratelimit [%s]: Finished sleeping", id(request_id))	!
151	self.sleeping_requests.discard(request_id)	!
152	queue_defer = queue_request()	!
153	return queue_defer	!
154
155	ret_defer.addBoth(on_wait_finished)	!
156	else:
UNCOV 157	ret_defer = queue_request()	!
158
UNCOV 159	def on_start(r):	!
UNCOV 160	logger.debug("Ratelimit [%s]: Processing req", id(request_id))	!
UNCOV 161	self.current_processing.add(request_id)	!
UNCOV 162	return r	!
163
UNCOV 164	def on_err(r):	!
165	# XXX: why is this necessary? this is called before we start
166	# processing the request so why would the request be in
167	# current_processing?
168	self.current_processing.discard(request_id)	!
169	return r	!
170
UNCOV 171	def on_both(r):	!
172	# Ensure that we've properly cleaned up.
UNCOV 173	self.sleeping_requests.discard(request_id)	!
UNCOV 174	self.ready_request_queue.pop(request_id, None)	!
UNCOV 175	return r	!
176
UNCOV 177	ret_defer.addCallbacks(on_start, on_err)	!
UNCOV 178	ret_defer.addBoth(on_both)	!
UNCOV 179	return make_deferred_yieldable(ret_defer)	!
180
181	def _on_exit(self, request_id):	1×
UNCOV 182	logger.debug("Ratelimit [%s]: Processed req", id(request_id))	!
UNCOV 183	self.current_processing.discard(request_id)	!
UNCOV 184	try:	!
185	# start processing the next item on the queue.
UNCOV 186	_, deferred = self.ready_request_queue.popitem(last=False)	!
187
188	with PreserveLoggingContext():	!
189	deferred.callback(None)	!
UNCOV 190	except KeyError:	!
UNCOV 191	pass	!

matrix-org / synapse / 4532

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous