4908056526

Build Type

push

github

Committed by Andreas Wicenec

Commit Message

Fixed small issues with existing graphs

Run Details

20 of 25 new or added lines in 2 files covered. (80.0%)

88 existing lines in 5 files now uncovered.

15342 of 19053 relevant lines covered (80.52%)

1.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.04

/daliuge-common/dlg/restutils.py

#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2016
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
import codecs
import http.client
import io
import json
import logging
import socketserver
import urllib.parse
import wsgiref.simple_server

from . import common
from . import exceptions
from .exceptions import DaliugeException, SubManagerException


logger = logging.getLogger(__name__)


class ThreadingWSGIServer(
    socketserver.ThreadingMixIn, wsgiref.simple_server.WSGIServer
):
    daemon_threads = True
    allow_reuse_address = True


class LoggingWSGIRequestHandler(wsgiref.simple_server.WSGIRequestHandler):
    def log_message(self, fmt, *args):
        pass
        # logger.debug(fmt, *args)


class RestServerWSGIServer:
    def __init__(self, wsgi_app, listen="localhost", port=8080):
        self.wsgi_app = wsgi_app
        self.listen = listen
        self.port = port
        self.server = wsgiref.simple_server.make_server(
            self.listen,
            self.port,
            self.wsgi_app,
            server_class=ThreadingWSGIServer,
            handler_class=LoggingWSGIRequestHandler,
        )

    def serve_forever(self):
        self.server.serve_forever()

    def server_close(self):
        self.server.shutdown()
        self.server.server_close()


class RestClientException(DaliugeException):
    """
    Exception thrown by the RestClient
    """


def hexdigits(n):
    digits = 0
    while n:
        digits += 1
        n //= 16
    return digits


def chunk(data):
    return ("%x" % len(data)).encode("ascii") + b"\r\n" + data + b"\r\n"


class chunked(object):
    """
    A reader that returns chunked HTTP content
    """

    def __init__(self, content):
        self.content = content
        self.finished = False

    def read(self, n):
        if self.finished:
            return b""
        n = n - hexdigits(n) - 4
        data = self.content.read(n)
        if not data:
            self.finished = True
            return b"0\r\n\r\n"
        return chunk(data)


class RestClient(object):
    """
    The base class for our REST clients
    """

    def __init__(self, host, port, url_prefix="", timeout=None):
        self.host = host
        self.port = port
        self.url_prefix = url_prefix
        self.timeout = timeout
        self._conn = None
        self._resp = None

    def _close(self):
        if self._resp:
            self._resp.close()
        if self._conn:
            self._conn.close()

    __del__ = _close

    def __enter__(self):
        return self

    def __exit__(self, typ, value, traceback):
        self._close()
        if typ:
            raise value

    def _get_json(self, url):
        ret = self._GET(url)
        return json.load(ret) if ret else None

    def _post_form(self, url, content=None):
        if content is not None:
            content = urllib.parse.urlencode(content)
        ret = self._POST(
            url, content, content_type="application/x-www-form-urlencoded"
        )
        return json.load(ret) if ret else None

    def _post_json(self, url, content, compress=False):
        if not isinstance(content, (str, bytes)):
            content = common.JSONStream(content)
        ret = self._POST(
            url, content, content_type="application/json", compress=compress
        )
        return json.load(ret) if ret else None

    def _GET(self, url):
        stream, _ = self._request(url, "GET")
        return stream

    def _POST(self, url, content=None, content_type=None, compress=False):
        headers = {}
        if content_type:
            headers["Content-Type"] = content_type
        if compress and content:
            headers["Content-Encoding"] = "gzip"
            if isinstance(content, str):
                content = codecs.getencoder("utf8")(content)[0]
            if not hasattr(content, "read"):
                content = io.BytesIO(content)
            content = common.ZlibCompressedStream(content)
        stream, _ = self._request(url, "POST", content, headers)
        return stream

    def _DELETE(self, url):
        stream, _ = self._request(url, "DELETE")
        return stream

    def _request(self, url, method, content=None, headers={}):
        # Do the HTTP stuff...
        url = self.url_prefix + url
        logger.debug(
            "Sending %s request to %s:%d%s", method, self.host, self.port, url
        )

        if not common.portIsOpen(self.host, self.port, self.timeout):
            raise RestClientException(
                "Cannot connect to %s:%d after %.2f [s]"
                % (self.host, self.port, self.timeout)
            )

        if content and hasattr(content, "read"):
            headers["Transfer-Encoding"] = "chunked"
            content = chunked(content)

        self._conn = http.client.HTTPConnection(self.host, self.port)
        self._conn.request(method, url, content, headers)
        self._resp = self._conn.getresponse()

        # Server errors are encoded in the body as json content
        if self._resp.status != http.HTTPStatus.OK:
            msg = "Error on remote %s@%s:%s%s (status %d): " % (
                method,
                self.host,
                self.port,
                url,
                self._resp.status,
            )

            try:
                error = json.loads(self._resp.read().decode("utf-8"))
                etype = getattr(exceptions, error["type"])
                eargs = error["args"]

                if etype == SubManagerException:
                    for host, args in eargs.items():
                        subetype = getattr(exceptions, args["type"])
                        subargs = args["args"]
                        eargs[host] = subetype(*subargs)
                    ex = etype(eargs)
                else:
                    ex = etype(*eargs)
                if hasattr(ex, "msg"):
                    ex.msg = msg + ex.msg
            except Exception:
                ex = RestClientException(msg + "Unknown")

            raise ex

        if not self._resp.length:
            return None, None
        return codecs.getreader("utf-8")(self._resp), self._resp

1	#
2	# ICRAR - International Centre for Radio Astronomy Research
3	# (c) UWA - The University of Western Australia, 2016
4	# Copyright by UWA (in the framework of the ICRAR)
5	# All rights reserved
6	#
7	# This library is free software; you can redistribute it and/or
8	# modify it under the terms of the GNU Lesser General Public
9	# License as published by the Free Software Foundation; either
10	# version 2.1 of the License, or (at your option) any later version.
11	#
12	# This library is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15	# Lesser General Public License for more details.
16	#
17	# You should have received a copy of the GNU Lesser General Public
18	# License along with this library; if not, write to the Free Software
19	# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20	# MA 02111-1307 USA
21	#
22	import codecs	3✔
23	import http.client	3✔
24	import io	3✔
25	import json	3✔
26	import logging	3✔
27	import socketserver	3✔
28	import urllib.parse	3✔
29	import wsgiref.simple_server	3✔
30
31	from . import common	3✔
32	from . import exceptions	3✔
33	from .exceptions import DaliugeException, SubManagerException	3✔
34
35
36	logger = logging.getLogger(__name__)	3✔
37
38
39	class ThreadingWSGIServer(	3✔
40	socketserver.ThreadingMixIn, wsgiref.simple_server.WSGIServer
41	):
42	daemon_threads = True	3✔
43	allow_reuse_address = True	3✔
44
45
46	class LoggingWSGIRequestHandler(wsgiref.simple_server.WSGIRequestHandler):	3✔
47	def log_message(self, fmt, *args):	3✔
48	pass	2✔
49	# logger.debug(fmt, *args)
50
51
52	class RestServerWSGIServer:	3✔
53	def __init__(self, wsgi_app, listen="localhost", port=8080):	3✔
54	self.wsgi_app = wsgi_app	2✔
55	self.listen = listen	2✔
56	self.port = port	2✔
57	self.server = wsgiref.simple_server.make_server(	2✔
58	self.listen,
59	self.port,
60	self.wsgi_app,
61	server_class=ThreadingWSGIServer,
62	handler_class=LoggingWSGIRequestHandler,
63	)
64
65	def serve_forever(self):	3✔
66	self.server.serve_forever()	2✔
67
68	def server_close(self):	3✔
69	self.server.shutdown()	2✔
70	self.server.server_close()	2✔
71
72
73	class RestClientException(DaliugeException):	3✔
74	"""
75	Exception thrown by the RestClient
76	"""
77
78
79	def hexdigits(n):	3✔
80	digits = 0	2✔
81	while n:	2✔
82	digits += 1	2✔
83	n //= 16	2✔
84	return digits	2✔
85
86
87	def chunk(data):	3✔
88	return ("%x" % len(data)).encode("ascii") + b"\r\n" + data + b"\r\n"	2✔
89
90
91	class chunked(object):	3✔
92	"""
93	A reader that returns chunked HTTP content
94	"""
95
96	def __init__(self, content):	3✔
97	self.content = content	2✔
98	self.finished = False	2✔
99
100	def read(self, n):	3✔
101	if self.finished:	2✔
102	return b""	2✔
103	n = n - hexdigits(n) - 4	2✔
104	data = self.content.read(n)	2✔
105	if not data:	2✔
106	self.finished = True	2✔
107	return b"0\r\n\r\n"	2✔
108	return chunk(data)	2✔
109
110
111	class RestClient(object):	3✔
112	"""
113	The base class for our REST clients
114	"""
115
116	def __init__(self, host, port, url_prefix="", timeout=None):	3✔
117	self.host = host	3✔
118	self.port = port	3✔
119	self.url_prefix = url_prefix	3✔
120	self.timeout = timeout	3✔
121	self._conn = None	3✔
122	self._resp = None	3✔
123
124	def _close(self):	3✔
125	if self._resp:	3✔
126	self._resp.close()	3✔
127	if self._conn:	3✔
128	self._conn.close()	3✔
129
130	__del__ = _close	3✔
131
132	def __enter__(self):	3✔
133	return self	2✔
134
135	def __exit__(self, typ, value, traceback):	3✔
136	self._close()	2✔
137	if typ:	2✔
138	raise value	2✔
139
140	def _get_json(self, url):	3✔
141	ret = self._GET(url)	3✔
142	return json.load(ret) if ret else None	3✔
143
144	def _post_form(self, url, content=None):	3✔
145	if content is not None:	3✔
146	content = urllib.parse.urlencode(content)	3✔
147	ret = self._POST(	3✔
148	url, content, content_type="application/x-www-form-urlencoded"
149	)
150	return json.load(ret) if ret else None	3✔
151
152	def _post_json(self, url, content, compress=False):	3✔
153	if not isinstance(content, (str, bytes)):	2✔
154	content = common.JSONStream(content)	2✔
155	ret = self._POST(	2✔
156	url, content, content_type="application/json", compress=compress
157	)
158	return json.load(ret) if ret else None	2✔
159
160	def _GET(self, url):	3✔
161	stream, _ = self._request(url, "GET")	3✔
162	return stream	3✔
163
164	def _POST(self, url, content=None, content_type=None, compress=False):	3✔
165	headers = {}	3✔
166	if content_type:	3✔
167	headers["Content-Type"] = content_type	3✔
168	if compress and content:	3✔
169	headers["Content-Encoding"] = "gzip"	2✔
170	if isinstance(content, str):	2✔
UNCOV 171	content = codecs.getencoder("utf8")(content)[0]	×
172	if not hasattr(content, "read"):	2✔
UNCOV 173	content = io.BytesIO(content)	×
174	content = common.ZlibCompressedStream(content)	2✔
175	stream, _ = self._request(url, "POST", content, headers)	3✔
176	return stream	3✔
177
178	def _DELETE(self, url):	3✔
179	stream, _ = self._request(url, "DELETE")	2✔
180	return stream	2✔
181
182	def _request(self, url, method, content=None, headers={}):	3✔
183	# Do the HTTP stuff...
184	url = self.url_prefix + url	3✔
185	logger.debug(	3✔
186	"Sending %s request to %s:%d%s", method, self.host, self.port, url
187	)
188
189	if not common.portIsOpen(self.host, self.port, self.timeout):	3✔
UNCOV 190	raise RestClientException(	×
191	"Cannot connect to %s:%d after %.2f [s]"
192	% (self.host, self.port, self.timeout)
193	)
194
195	if content and hasattr(content, "read"):	3✔
196	headers["Transfer-Encoding"] = "chunked"	2✔
197	content = chunked(content)	2✔
198
199	self._conn = http.client.HTTPConnection(self.host, self.port)	3✔
200	self._conn.request(method, url, content, headers)	3✔
201	self._resp = self._conn.getresponse()	3✔
202
203	# Server errors are encoded in the body as json content
204	if self._resp.status != http.HTTPStatus.OK:	3✔
205	msg = "Error on remote %s@%s:%s%s (status %d): " % (	3✔
206	method,
207	self.host,
208	self.port,
209	url,
210	self._resp.status,
211	)
212
213	try:	3✔
214	error = json.loads(self._resp.read().decode("utf-8"))	3✔
215	etype = getattr(exceptions, error["type"])	3✔
216	eargs = error["args"]	2✔
217
218	if etype == SubManagerException:	2✔
219	for host, args in eargs.items():	2✔
220	subetype = getattr(exceptions, args["type"])	2✔
221	subargs = args["args"]	2✔
222	eargs[host] = subetype(*subargs)	2✔
223	ex = etype(eargs)	2✔
224	else:
225	ex = etype(*eargs)	2✔
226	if hasattr(ex, "msg"):	2✔
227	ex.msg = msg + ex.msg	2✔
228	except Exception:	3✔
229	ex = RestClientException(msg + "Unknown")	3✔
230
231	raise ex	3✔
232
233	if not self._resp.length:	3✔
UNCOV 234	return None, None	×
235	return codecs.getreader("utf-8")(self._resp), self._resp	3✔

ICRAR / daliuge / 4908056526

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous