287381919

Build # 287381919

Build Type

push

github

Committed by Unrud

Commit Message

Improve log message

Run Details

1212 of 1561 branches covered (77.64%)

Branch coverage included in aggregate %.

0 of 1 new or added line in 1 file covered. (0.0%)

2 existing lines in 1 file now uncovered.

2900 of 3439 relevant lines covered (84.33%)

28.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.82

/radicale/server.py

# This file is part of Radicale Server - Calendar Server
# Copyright © 2008 Nicolas Kandel
# Copyright © 2008 Pascal Halter
# Copyright © 2008-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Radicale.  If not, see <http://www.gnu.org/licenses/>.

"""
Built-in WSGI server.

"""

import errno
import os
import select
import socket
import socketserver
import ssl
import sys
import wsgiref.simple_server
from urllib.parse import unquote

from radicale import Application, config
from radicale.log import logger

if hasattr(socket, "EAI_ADDRFAMILY"):
    COMPAT_EAI_ADDRFAMILY = socket.EAI_ADDRFAMILY
elif hasattr(socket, "EAI_NONAME"):
    # Windows and BSD don't have a special error code for this
    COMPAT_EAI_ADDRFAMILY = socket.EAI_NONAME
if hasattr(socket, "EAI_NODATA"):
    COMPAT_EAI_NODATA = socket.EAI_NODATA
elif hasattr(socket, "EAI_NONAME"):
    # Windows and BSD don't have a special error code for this
    COMPAT_EAI_NODATA = socket.EAI_NONAME
if hasattr(socket, "IPPROTO_IPV6"):
    COMPAT_IPPROTO_IPV6 = socket.IPPROTO_IPV6
elif os.name == "nt":
    # Workaround: https://bugs.python.org/issue29515
    COMPAT_IPPROTO_IPV6 = 41


def format_address(address):
    return "[%s]:%d" % address[:2]


class ParallelHTTPServer(socketserver.ThreadingMixIn,
                         wsgiref.simple_server.WSGIServer):

    # We wait for child threads ourself
    block_on_close = False

    def __init__(self, configuration, family, address, RequestHandlerClass):
        self.configuration = configuration
        self.address_family = family
        super().__init__(address, RequestHandlerClass)
        self.client_sockets = set()

    def server_bind(self):
        if self.address_family == socket.AF_INET6:
            # Only allow IPv6 connections to the IPv6 socket
            self.socket.setsockopt(COMPAT_IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
        super().server_bind()

    def get_request(self):
        # Set timeout for client
        request, client_address = super().get_request()
        timeout = self.configuration.get("server", "timeout")
        if timeout:
            request.settimeout(timeout)
        client_socket, client_socket_out = socket.socketpair()
        self.client_sockets.add(client_socket_out)
        return request, (*client_address, client_socket)

    def finish_request_locked(self, request, client_address):
        return super().finish_request(request, client_address)

    def finish_request(self, request, client_address):
        *client_address, client_socket = client_address
        client_address = tuple(client_address)
        try:
            return self.finish_request_locked(request, client_address)
        finally:
            client_socket.close()

    def handle_error(self, request, client_address):
        if issubclass(sys.exc_info()[0], socket.timeout):
            logger.info("Client timed out", exc_info=True)
        else:
            logger.error("An exception occurred during request: %s",
                         sys.exc_info()[1], exc_info=True)


class ParallelHTTPSServer(ParallelHTTPServer):

    def server_bind(self):
        super().server_bind()
        # Wrap the TCP socket in an SSL socket
        certfile = self.configuration.get("server", "certificate")
        keyfile = self.configuration.get("server", "key")
        cafile = self.configuration.get("server", "certificate_authority")
        # Test if the files can be read
        for name, filename in [("certificate", certfile), ("key", keyfile),
                               ("certificate_authority", cafile)]:
            type_name = config.DEFAULT_CONFIG_SCHEMA["server"][name][
                "type"].__name__
            source = self.configuration.get_source("server", name)
            if name == "certificate_authority" and not filename:
                continue
            try:
                open(filename, "r").close()
            except OSError as e:
                raise RuntimeError(
                    "Invalid %s value for option %r in section %r in %s: %r "
                    "(%s)" % (type_name, name, "server", source, filename,
                              e)) from e
        context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
        context.load_cert_chain(certfile=certfile, keyfile=keyfile)
        if cafile:
            context.load_verify_locations(cafile=cafile)
            context.verify_mode = ssl.CERT_REQUIRED
        self.socket = context.wrap_socket(
            self.socket, server_side=True, do_handshake_on_connect=False)

    def finish_request_locked(self, request, client_address):
        try:
            try:
                request.do_handshake()
            except socket.timeout:
                raise
            except Exception as e:
                raise RuntimeError("SSL handshake failed: %s" % e) from e
        except Exception:
            try:
                self.handle_error(request, client_address)
            finally:
                self.shutdown_request(request)
            return
        return super().finish_request_locked(request, client_address)


class ServerHandler(wsgiref.simple_server.ServerHandler):

    # Don't pollute WSGI environ with OS environment
    os_environ = {}

    def log_exception(self, exc_info):
        logger.error("An exception occurred during request: %s",
                     exc_info[1], exc_info=exc_info)


class RequestHandler(wsgiref.simple_server.WSGIRequestHandler):
    """HTTP requests handler."""

    def log_request(self, code="-", size="-"):
        pass  # Disable request logging.

    def log_error(self, format_, *args):
        logger.error("An error occurred during request: %s", format_ % args)

    def get_environ(self):
        env = super().get_environ()
        if hasattr(self.connection, "getpeercert"):
            # The certificate can be evaluated by the auth module
            env["REMOTE_CERTIFICATE"] = self.connection.getpeercert()
        # Parent class only tries latin1 encoding
        env["PATH_INFO"] = unquote(self.path.split("?", 1)[0])
        return env

    def handle(self):
        """Copy of WSGIRequestHandler.handle with different ServerHandler"""

        self.raw_requestline = self.rfile.readline(65537)
        if len(self.raw_requestline) > 65536:
            self.requestline = ""
            self.request_version = ""
            self.command = ""
            self.send_error(414)
            return

        if not self.parse_request():
            return

        handler = ServerHandler(
            self.rfile, self.wfile, self.get_stderr(), self.get_environ()
        )
        handler.request_handler = self
        handler.run(self.server.get_app())


def serve(configuration, shutdown_socket=None):
    """Serve radicale from configuration.

    `shutdown_socket` can be used to gracefully shutdown the server.
    The socket can be created with `socket.socketpair()`, when the other socket
    gets closed the server stops accepting new requests by clients and the
    function returns after all active requests are finished.

    """

    logger.info("Starting Radicale")
    # Copy configuration before modifying
    configuration = configuration.copy()
    configuration.update({"server": {"_internal_server": "True"}}, "server",
                         privileged=True)

    use_ssl = configuration.get("server", "ssl")
    server_class = ParallelHTTPSServer if use_ssl else ParallelHTTPServer
    application = Application(configuration)
    servers = {}
    try:
        for address in configuration.get("server", "hosts"):
            # Try to bind sockets for IPv4 and IPv6
            possible_families = (socket.AF_INET, socket.AF_INET6)
            bind_ok = False
            for i, family in enumerate(possible_families):
                is_last = i == len(possible_families) - 1
                try:
                    server = server_class(configuration, family, address,
                                          RequestHandler)
                except OSError as e:
                    # Ignore unsupported families (only one must work)
                    if ((bind_ok or not is_last) and (
                            isinstance(e, socket.gaierror) and (
                                # Hostname does not exist or doesn't have
                                # address for address family
                                # macOS: IPv6 address for INET address family
                                e.errno == socket.EAI_NONAME or
                                # Address not for address family
                                e.errno == COMPAT_EAI_ADDRFAMILY or
                                e.errno == COMPAT_EAI_NODATA) or
                            # Workaround for PyPy
                            str(e) == "address family mismatched" or
                            # Address family not available (e.g. IPv6 disabled)
                            # macOS: IPv4 address for INET6 address family with
                            #        IPV6_V6ONLY set
                            e.errno == errno.EADDRNOTAVAIL or
                            # Address family not supported
                            e.errno == errno.EAFNOSUPPORT or
                            # Protocol not supported
                            e.errno == errno.EPROTONOSUPPORT)):
                        continue
                    raise RuntimeError("Failed to start server %r: %s" % (
                                           format_address(address), e)) from e
                servers[server.socket] = server
                bind_ok = True
                server.set_app(application)
                logger.info("Listening on %r%s",
                            format_address(server.server_address),
                            " with SSL" if use_ssl else "")
        if not servers:
            raise RuntimeError("No servers started")

        # Mainloop
        select_timeout = None
        if os.name == "nt":
            # Fallback to busy waiting. (select(...) blocks SIGINT on Windows.)
            select_timeout = 1.0
        max_connections = configuration.get("server", "max_connections")
        logger.info("Radicale server ready")
        while True:
            rlist = []
            # Wait for finished clients
            for server in servers.values():
                rlist.extend(server.client_sockets)
            # Accept new connections if max_connections is not reached
            if max_connections <= 0 or len(rlist) < max_connections:
                rlist.extend(servers)
            # Use socket to get notified of program shutdown
            if shutdown_socket is not None:
                rlist.append(shutdown_socket)
            rlist, _, _ = select.select(rlist, [], [], select_timeout)
            rlist = set(rlist)
            if shutdown_socket in rlist:
                logger.info("Stopping Radicale")
                break
            for server in servers.values():
                finished_sockets = server.client_sockets.intersection(rlist)
                for s in finished_sockets:
                    s.close()
                    server.client_sockets.remove(s)
                    rlist.remove(s)
                if finished_sockets:
                    server.service_actions()
            if rlist:
                server = servers.get(rlist.pop())
                if server:
                    server.handle_request()
    finally:
        # Wait for clients to finish and close servers
        for server in servers.values():
            for s in server.client_sockets:
                s.recv(1)
                s.close()
            server.server_close()

1	# This file is part of Radicale Server - Calendar Server
2	# Copyright © 2008 Nicolas Kandel
3	# Copyright © 2008 Pascal Halter
4	# Copyright © 2008-2017 Guillaume Ayoub
5	# Copyright © 2017-2019 Unrud <unrud@outlook.com>
6	#
7	# This library is free software: you can redistribute it and/or modify
8	# it under the terms of the GNU General Public License as published by
9	# the Free Software Foundation, either version 3 of the License, or
10	# (at your option) any later version.
11	#
12	# This library is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	# GNU General Public License for more details.
16	#
17	# You should have received a copy of the GNU General Public License
18	# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
19
20	"""	34✔
21	Built-in WSGI server.
22
23	"""
24
25	import errno	34✔
26	import os	34✔
27	import select	34✔
28	import socket	34✔
29	import socketserver	34✔
30	import ssl	34✔
31	import sys	34✔
32	import wsgiref.simple_server	34✔
33	from urllib.parse import unquote	34✔
34
35	from radicale import Application, config	34✔
36	from radicale.log import logger	34✔
37
38	if hasattr(socket, "EAI_ADDRFAMILY"):	34!
39	COMPAT_EAI_ADDRFAMILY = socket.EAI_ADDRFAMILY	34✔
40	elif hasattr(socket, "EAI_NONAME"):	×
41	# Windows and BSD don't have a special error code for this
42	COMPAT_EAI_ADDRFAMILY = socket.EAI_NONAME	×
43	if hasattr(socket, "EAI_NODATA"):	34!
44	COMPAT_EAI_NODATA = socket.EAI_NODATA	34✔
45	elif hasattr(socket, "EAI_NONAME"):	×
46	# Windows and BSD don't have a special error code for this
47	COMPAT_EAI_NODATA = socket.EAI_NONAME	×
48	if hasattr(socket, "IPPROTO_IPV6"):	34!
49	COMPAT_IPPROTO_IPV6 = socket.IPPROTO_IPV6	34✔
50	elif os.name == "nt":	×
51	# Workaround: https://bugs.python.org/issue29515
52	COMPAT_IPPROTO_IPV6 = 41	×
53
54
55	def format_address(address):	34✔
56	return "[%s]:%d" % address[:2]	34✔
57
58
59	class ParallelHTTPServer(socketserver.ThreadingMixIn,	34✔
60	wsgiref.simple_server.WSGIServer):
61
62	# We wait for child threads ourself
63	block_on_close = False	34✔
64
65	def __init__(self, configuration, family, address, RequestHandlerClass):	34✔
66	self.configuration = configuration	34✔
67	self.address_family = family	34✔
68	super().__init__(address, RequestHandlerClass)	34✔
69	self.client_sockets = set()	34✔
70
71	def server_bind(self):	34✔
72	if self.address_family == socket.AF_INET6:	34✔
73	# Only allow IPv6 connections to the IPv6 socket
74	self.socket.setsockopt(COMPAT_IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)	34✔
75	super().server_bind()	34✔
76
77	def get_request(self):	34✔
78	# Set timeout for client
79	request, client_address = super().get_request()	34✔
80	timeout = self.configuration.get("server", "timeout")	34✔
81	if timeout:	34!
82	request.settimeout(timeout)	34✔
83	client_socket, client_socket_out = socket.socketpair()	34✔
84	self.client_sockets.add(client_socket_out)	34✔
85	return request, (*client_address, client_socket)	34✔
86
87	def finish_request_locked(self, request, client_address):	34✔
88	return super().finish_request(request, client_address)	34✔
89
90	def finish_request(self, request, client_address):	34✔
91	*client_address, client_socket = client_address	34✔
92	client_address = tuple(client_address)	34✔
93	try:	34✔
94	return self.finish_request_locked(request, client_address)	34✔
95	finally:
96	client_socket.close()	34✔
97
98	def handle_error(self, request, client_address):	34✔
99	if issubclass(sys.exc_info()[0], socket.timeout):	×
NEW 100	logger.info("Client timed out", exc_info=True)	×
101	else:
102	logger.error("An exception occurred during request: %s",	×
103	sys.exc_info()[1], exc_info=True)
104
105
106	class ParallelHTTPSServer(ParallelHTTPServer):	34✔
107
108	def server_bind(self):	34✔
109	super().server_bind()	34✔
110	# Wrap the TCP socket in an SSL socket
111	certfile = self.configuration.get("server", "certificate")	34✔
112	keyfile = self.configuration.get("server", "key")	34✔
113	cafile = self.configuration.get("server", "certificate_authority")	34✔
114	# Test if the files can be read
115	for name, filename in [("certificate", certfile), ("key", keyfile),	34✔
116	("certificate_authority", cafile)]:
117	type_name = config.DEFAULT_CONFIG_SCHEMA["server"][name][	34✔
118	"type"].__name__
119	source = self.configuration.get_source("server", name)	34✔
120	if name == "certificate_authority" and not filename:	34✔
121	continue	34✔
122	try:	34✔
123	open(filename, "r").close()	34✔
124	except OSError as e:	×
125	raise RuntimeError(	×
126	"Invalid %s value for option %r in section %r in %s: %r "
127	"(%s)" % (type_name, name, "server", source, filename,
128	e)) from e
129	context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)	34✔
130	context.load_cert_chain(certfile=certfile, keyfile=keyfile)	34✔
131	if cafile:	34!
132	context.load_verify_locations(cafile=cafile)	×
133	context.verify_mode = ssl.CERT_REQUIRED	×
134	self.socket = context.wrap_socket(	34✔
135	self.socket, server_side=True, do_handshake_on_connect=False)
136
137	def finish_request_locked(self, request, client_address):	34✔
138	try:	34✔
139	try:	34✔
140	request.do_handshake()	34✔
141	except socket.timeout:	×
142	raise	×
143	except Exception as e:	×
144	raise RuntimeError("SSL handshake failed: %s" % e) from e	×
145	except Exception:	×
146	try:	×
147	self.handle_error(request, client_address)	×
148	finally:
149	self.shutdown_request(request)	×
150	return	×
151	return super().finish_request_locked(request, client_address)	34✔
152
153
154	class ServerHandler(wsgiref.simple_server.ServerHandler):	34✔
155
156	# Don't pollute WSGI environ with OS environment
157	os_environ = {}	34✔
158
159	def log_exception(self, exc_info):	34✔
160	logger.error("An exception occurred during request: %s",	×
161	exc_info[1], exc_info=exc_info)
162
163
164	class RequestHandler(wsgiref.simple_server.WSGIRequestHandler):	34✔
165	"""HTTP requests handler."""
166
167	def log_request(self, code="-", size="-"):	34✔
168	pass # Disable request logging.	34✔
169
170	def log_error(self, format_, *args):	34✔
171	logger.error("An error occurred during request: %s", format_ % args)	×
172
173	def get_environ(self):	34✔
174	env = super().get_environ()	34✔
175	if hasattr(self.connection, "getpeercert"):	34✔
176	# The certificate can be evaluated by the auth module
177	env["REMOTE_CERTIFICATE"] = self.connection.getpeercert()	34✔
178	# Parent class only tries latin1 encoding
179	env["PATH_INFO"] = unquote(self.path.split("?", 1)[0])	34✔
180	return env	34✔
181
182	def handle(self):	34✔
183	"""Copy of WSGIRequestHandler.handle with different ServerHandler"""
184
185	self.raw_requestline = self.rfile.readline(65537)	34✔
186	if len(self.raw_requestline) > 65536:	34!
187	self.requestline = ""	×
188	self.request_version = ""	×
189	self.command = ""	×
190	self.send_error(414)	×
191	return	×
192
193	if not self.parse_request():	34!
194	return	×
195
196	handler = ServerHandler(	34✔
197	self.rfile, self.wfile, self.get_stderr(), self.get_environ()
198	)
199	handler.request_handler = self	34✔
200	handler.run(self.server.get_app())	34✔
201
202
203	def serve(configuration, shutdown_socket=None):	34✔
204	"""Serve radicale from configuration.
205
206	`shutdown_socket` can be used to gracefully shutdown the server.
207	The socket can be created with `socket.socketpair()`, when the other socket
208	gets closed the server stops accepting new requests by clients and the
209	function returns after all active requests are finished.
210
211	"""
212
213	logger.info("Starting Radicale")	34✔
214	# Copy configuration before modifying
215	configuration = configuration.copy()	34✔
216	configuration.update({"server": {"_internal_server": "True"}}, "server",	34✔
217	privileged=True)
218
219	use_ssl = configuration.get("server", "ssl")	34✔
220	server_class = ParallelHTTPSServer if use_ssl else ParallelHTTPServer	34✔
221	application = Application(configuration)	34✔
222	servers = {}	34✔
223	try:	34✔
224	for address in configuration.get("server", "hosts"):	34✔
225	# Try to bind sockets for IPv4 and IPv6
226	possible_families = (socket.AF_INET, socket.AF_INET6)	34✔
227	bind_ok = False	34✔
228	for i, family in enumerate(possible_families):	34✔
229	is_last = i == len(possible_families) - 1	34✔
230	try:	34✔
231	server = server_class(configuration, family, address,	34✔
232	RequestHandler)
233	except OSError as e:	34✔
234	# Ignore unsupported families (only one must work)
235	if ((bind_ok or not is_last) and (	34!
236	isinstance(e, socket.gaierror) and (
237	# Hostname does not exist or doesn't have
238	# address for address family
239	# macOS: IPv6 address for INET address family
240	e.errno == socket.EAI_NONAME or
241	# Address not for address family
242	e.errno == COMPAT_EAI_ADDRFAMILY or
243	e.errno == COMPAT_EAI_NODATA) or
244	# Workaround for PyPy
245	str(e) == "address family mismatched" or
246	# Address family not available (e.g. IPv6 disabled)
247	# macOS: IPv4 address for INET6 address family with
248	# IPV6_V6ONLY set
249	e.errno == errno.EADDRNOTAVAIL or
250	# Address family not supported
251	e.errno == errno.EAFNOSUPPORT or
252	# Protocol not supported
253	e.errno == errno.EPROTONOSUPPORT)):
254	continue	34✔
255	raise RuntimeError("Failed to start server %r: %s" % (	×
256	format_address(address), e)) from e
257	servers[server.socket] = server	34✔
258	bind_ok = True	34✔
259	server.set_app(application)	34✔
260	logger.info("Listening on %r%s",	34✔
261	format_address(server.server_address),
262	" with SSL" if use_ssl else "")
263	if not servers:	34!
UNCOV 264	raise RuntimeError("No servers started")	×
265
266	# Mainloop
267	select_timeout = None	34✔
268	if os.name == "nt":	34!
269	# Fallback to busy waiting. (select(...) blocks SIGINT on Windows.)
UNCOV 270	select_timeout = 1.0	×
271	max_connections = configuration.get("server", "max_connections")	34✔
272	logger.info("Radicale server ready")	34✔
273	while True:	23✔
274	rlist = []	34✔
275	# Wait for finished clients
276	for server in servers.values():	34✔
277	rlist.extend(server.client_sockets)	34✔
278	# Accept new connections if max_connections is not reached
279	if max_connections <= 0 or len(rlist) < max_connections:	34!
280	rlist.extend(servers)	34✔
281	# Use socket to get notified of program shutdown
282	if shutdown_socket is not None:	34✔
283	rlist.append(shutdown_socket)	34✔
284	rlist, _, _ = select.select(rlist, [], [], select_timeout)	34✔
285	rlist = set(rlist)	34✔
286	if shutdown_socket in rlist:	34✔
287	logger.info("Stopping Radicale")	34✔
288	break	34✔
289	for server in servers.values():	34✔
290	finished_sockets = server.client_sockets.intersection(rlist)	34✔
291	for s in finished_sockets:	34✔
292	s.close()	34✔
293	server.client_sockets.remove(s)	34✔
294	rlist.remove(s)	34✔
295	if finished_sockets:	34✔
296	server.service_actions()	34✔
297	if rlist:	34✔
298	server = servers.get(rlist.pop())	34✔
299	if server:	34!
300	server.handle_request()	34✔
301	finally:
302	# Wait for clients to finish and close servers
303	for server in servers.values():	34✔
304	for s in server.client_sockets:	34✔
305	s.recv(1)	2✔
306	s.close()	2✔
307	server.server_close()	34✔

Kozea / Radicale / 287381919

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous