72squared / redpipe / 215

pending completion

Build # 215

Build Type

Pull #8

travis-ci-com

Committed by

web-flow

Commit Message

<a href="https://github.com/72squared/redpipe/commit/<a class=hub.com/72squared/redpipe/commit/3ab82f35654c6da0065e9171a918be7db67ccd25">3ab82f356<a href="https://github.com/72squared/redpipe/commit/3ab82f35654c6da0065e9171a918be7db67ccd25">">Merge </a><a class="double-link" href="https://github.com/72squared/redpipe/commit/<a class="double-link" href="https://github.com/72squared/redpipe/commit/cee5ef37eb9500f9b88dd52d87759f3f37ef5156">cee5ef37e</a>">cee5ef37e</a><a href="https://github.com/72squared/redpipe/commit/3ab82f35654c6da0065e9171a918be7db67ccd25"> into ec1577c9c">ec1577c9c</a>

Pull Request Pull Request #8: Make compatible with python3.7

Run Details

416 of 416 new or added lines in 8 files covered. (100.0%)

1602 of 1611 relevant lines covered (99.44%)

4.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.63

/redpipe/fields.py

# -*- coding: utf-8 -*-

"""
A module for marshalling data in and out of redis and back into the python
data type we expect.

Used extensively in the `redpipe.keyspaces` module for type-casting keys and
values.

"""

import json
import re
import typing

# python 3.7 compatibility change
from typing import (TypeVar, Generic, Optional, Union)
try:
    from typing import Protocol
except ImportError:
    from typing_extensions import Protocol  # type: ignore


from .exceptions import InvalidValue

__all__ = [
    'IntegerField',
    'FloatField',
    'TextField',
    'AsciiField',
    'BinaryField',
    'BooleanField',
    'ListField',
    'DictField',
    'StringListField',
    'Field'
]

T = TypeVar('T')


class Field(Protocol, Generic[T]):
    @classmethod
    def encode(cls, value: T) -> bytes: ...

    @classmethod
    def decode(cls, value: Optional[bytes]) -> Optional[T]: ...


class BooleanField(object):
    """
    Used for boolean fields.
    """
    @classmethod
    def is_true(cls, val):
        if val is True:
            return True
        if val is False:
            return False

        strval = str(val).lower()
        if strval in ['true', '1']:
            return True

        if strval in ['false', '0', 'none', '']:
            return False

        return True if val else False

    @classmethod
    def encode(cls, value: bool) -> bytes:
        """
        convert a boolean value into something we can persist to redis.
        An empty string is the representation for False.

        :param value: bool
        :return: bytes
        """
        return b'1' if cls.is_true(value) else b''

    @classmethod
    def decode(cls, value: Optional[bytes]) -> Optional[bool]:
        """
        convert from redis bytes into a boolean value

        :param value: bytes
        :return: bool
        """
        return None if value is None else bool(value)


class FloatField(object):
    """
    Numeric field that supports integers and floats (values are turned into
    floats on load from persistence).
    """

    @classmethod
    def decode(cls, value: Optional[bytes]) -> Optional[float]:
        """
        decode the bytes from redis back into a float

        :param value: bytes
        :return: float
        """
        return None if value is None else float(value)

    @classmethod
    def encode(cls, value: float) -> bytes:
        """
        encode a floating point number to bytes in redis

        :param value: float
        :return: bytes
        """
        try:
            coerced = float(value)
        except (TypeError, ValueError):
            raise InvalidValue('not a float')
        response = repr(coerced)
        if response.endswith('.0'):
            response = response[:-2]
        return response.encode()


class IntegerField(object):
    """
    Used for integer numeric fields.
    """

    @classmethod
    def decode(cls, value: Optional[bytes]) -> Optional[int]:
        """
        read bytes from redis and turn it back into an integer.

        :param value: bytes
        :return: int
        """
        return None if value is None else int(float(value))

    @classmethod
    def encode(cls, value: int) -> bytes:
        """
        take an integer and turn it into a string representation
        to write into redis.

        :param value: int
        :return: str
        """
        try:
            return repr(int(float(value))).encode()
        except (TypeError, ValueError):
            raise InvalidValue('not an int')


class TextField(object):
    """
    A unicode string field.

    Encoded via utf-8 before writing to persistence.
    """
    _encoding = 'utf-8'

    @classmethod
    def encode(cls, value: str) -> bytes:
        """
        take a valid unicode string and turn it into utf-8 bytes

        :param value: unicode, str
        :return: bytes
        """
        coerced = str(value)
        if coerced == value:
            return coerced.encode(cls._encoding)

        raise InvalidValue('not text')

    @classmethod
    def decode(cls, value: Optional[bytes]) -> Optional[str]:
        """
        take bytes from redis and turn them into unicode string

        :param value:
        :return:
        """
        return None if value is None else str(value.decode(cls._encoding))


class AsciiField(TextField):
    """
    Used for ascii-only text
    """
    PATTERN = re.compile('^([ -~]+)?$')

    @classmethod
    def encode(cls, value: str) -> bytes:
        """
        take a list of strings and turn it into utf-8 byte-string

        :param value:
        :return:
        """
        coerced = str(value)
        if coerced == value and cls.PATTERN.match(coerced):
            return coerced.encode(cls._encoding)

        raise InvalidValue('not ascii')


class BinaryField(object):
    """
    A bytes field. Not encoded.
    """

    @classmethod
    def encode(cls, value: bytes) -> bytes:
        """
        write binary data into redis without encoding it.

        :param value: bytes
        :return: bytes
        """
        try:
            coerced = bytes(value)
            if coerced == value:
                return coerced
        except (TypeError, UnicodeError):
            pass

        raise InvalidValue('not binary')

    @classmethod
    def decode(cls, value: Optional[bytes]) -> Optional[bytes]:
        """
        read binary data from redis and pass it on through.

        :param value: bytes
        :return: bytes
        """
        return None if value is None else bytes(value)


class ListField(object):
    """
    A list field. Marshalled in and out of redis via json.
    Values of the list can be any arbitrary data.
    """
    _encoding = 'utf-8'

    @classmethod
    def encode(cls, value: list) -> bytes:
        """
        take a list and turn it into a utf-8 encoded byte-string for redis.

        :param value: list
        :return: bytes
        """
        try:
            coerced = list(value)
            if coerced == value:
                return json.dumps(coerced).encode(cls._encoding)
        except TypeError:
            pass

        raise InvalidValue('not a list')

    @classmethod
    def decode(cls, value: Union[bytes, None, list]) -> Optional[list]:
        """
        take a utf-8 encoded byte-string from redis and
        turn it back into a list

        :param value: bytes
        :return: list
        """
        try:
            return None if value is None else \
                list(json.loads(value.decode(cls._encoding)))  # type: ignore
        except (TypeError, AttributeError):
            return list(value)  # type: ignore


class DictField(object):
    _encoding = 'utf-8'

    @classmethod
    def encode(cls, value: dict) -> bytes:
        """
        encode the dict as a json string to be written into redis.

        :param value: dict
        :return: bytes
        """
        try:
            coerced = dict(value)
            if coerced == value:
                return json.dumps(coerced).encode(cls._encoding)
        except (TypeError, ValueError):
            pass
        raise InvalidValue('not a dict')

    @classmethod
    def decode(cls, value: Union[bytes, None, dict]) -> Optional[dict]:
        """
        decode the data from a json string in redis back into a dict object.

        :param value: bytes
        :return: dict
        """
        try:
            return None if value is None else \
                dict(json.loads(value.decode(cls._encoding)))  # type: ignore
        except (TypeError, AttributeError):
            return dict(value)  # type: ignore


class StringListField(object):
    """
    Used for storing a list of strings, serialized as a comma-separated list.
    """
    _encoding = 'utf-8'

    @classmethod
    def decode(cls,
               value: Union[bytes, None, typing.List[str]]
               ) -> Optional[typing.List[str]]:
        """
        decode the data from redis.
        :param value: bytes
        :return: list
        """
        if value is None or isinstance(value, list):
            return value

        try:
            data = [v for v in value.decode(cls._encoding).split(',') if
                    v != '']
            return data if data else None
        except AttributeError:
            return None

    @classmethod
    def encode(cls, value: typing.List[str]) -> bytes:
        """
        encode the list it so it can be stored in redis.

        :param value: list
        :return: bytes
        """
        try:
            coerced = [str(v) for v in value]
            if coerced == value:
                return ",".join(coerced).encode(cls._encoding) if len(
                    value) > 0 else b''
        except TypeError:
            pass

        raise InvalidValue('not a list of strings')

1	# -- coding: utf-8 --
2
3	"""	5✔
4	A module for marshalling data in and out of redis and back into the python
5	data type we expect.
6
7	Used extensively in the `redpipe.keyspaces` module for type-casting keys and
8	values.
9
10	"""
11
12	import json	5✔
13	import re	5✔
14	import typing	5✔
15
16	# python 3.7 compatibility change
17	from typing import (TypeVar, Generic, Optional, Union)	5✔
18	try:	5✔
19	from typing import Protocol	5✔
20	except ImportError:	×
21	from typing_extensions import Protocol # type: ignore	×
22
23
24	from .exceptions import InvalidValue	5✔
25
26	__all__ = [	5✔
27	'IntegerField',
28	'FloatField',
29	'TextField',
30	'AsciiField',
31	'BinaryField',
32	'BooleanField',
33	'ListField',
34	'DictField',
35	'StringListField',
36	'Field'
37	]
38
39	T = TypeVar('T')	5✔
40
41
42	class Field(Protocol, Generic[T]):	5✔
43	@classmethod	5✔
44	def encode(cls, value: T) -> bytes: ...	5✔
45
46	@classmethod	5✔
47	def decode(cls, value: Optional[bytes]) -> Optional[T]: ...	5✔
48
49
50	class BooleanField(object):	5✔
51	"""
52	Used for boolean fields.
53	"""
54	@classmethod	5✔
55	def is_true(cls, val):	5✔
56	if val is True:	5✔
57	return True	5✔
58	if val is False:	5✔
59	return False	5✔
60
61	strval = str(val).lower()	5✔
62	if strval in ['true', '1']:	5✔
63	return True	5✔
64
65	if strval in ['false', '0', 'none', '']:	5✔
66	return False	5✔
67
68	return True if val else False	5✔
69
70	@classmethod	5✔
71	def encode(cls, value: bool) -> bytes:	5✔
72	"""
73	convert a boolean value into something we can persist to redis.
74	An empty string is the representation for False.
75
76	:param value: bool
77	:return: bytes
78	"""
79	return b'1' if cls.is_true(value) else b''	5✔
80
81	@classmethod	5✔
82	def decode(cls, value: Optional[bytes]) -> Optional[bool]:	5✔
83	"""
84	convert from redis bytes into a boolean value
85
86	:param value: bytes
87	:return: bool
88	"""
89	return None if value is None else bool(value)	5✔
90
91
92	class FloatField(object):	5✔
93	"""
94	Numeric field that supports integers and floats (values are turned into
95	floats on load from persistence).
96	"""
97
98	@classmethod	5✔
99	def decode(cls, value: Optional[bytes]) -> Optional[float]:	5✔
100	"""
101	decode the bytes from redis back into a float
102
103	:param value: bytes
104	:return: float
105	"""
106	return None if value is None else float(value)	5✔
107
108	@classmethod	5✔
109	def encode(cls, value: float) -> bytes:	5✔
110	"""
111	encode a floating point number to bytes in redis
112
113	:param value: float
114	:return: bytes
115	"""
116	try:	5✔
117	coerced = float(value)	5✔
118	except (TypeError, ValueError):	5✔
119	raise InvalidValue('not a float')	5✔
120	response = repr(coerced)	5✔
121	if response.endswith('.0'):	5✔
122	response = response[:-2]	5✔
123	return response.encode()	5✔
124
125
126	class IntegerField(object):	5✔
127	"""
128	Used for integer numeric fields.
129	"""
130
131	@classmethod	5✔
132	def decode(cls, value: Optional[bytes]) -> Optional[int]:	5✔
133	"""
134	read bytes from redis and turn it back into an integer.
135
136	:param value: bytes
137	:return: int
138	"""
139	return None if value is None else int(float(value))	5✔
140
141	@classmethod	5✔
142	def encode(cls, value: int) -> bytes:	5✔
143	"""
144	take an integer and turn it into a string representation
145	to write into redis.
146
147	:param value: int
148	:return: str
149	"""
150	try:	5✔
151	return repr(int(float(value))).encode()	5✔
152	except (TypeError, ValueError):	5✔
153	raise InvalidValue('not an int')	5✔
154
155
156	class TextField(object):	5✔
157	"""
158	A unicode string field.
159
160	Encoded via utf-8 before writing to persistence.
161	"""
162	_encoding = 'utf-8'	5✔
163
164	@classmethod	5✔
165	def encode(cls, value: str) -> bytes:	5✔
166	"""
167	take a valid unicode string and turn it into utf-8 bytes
168
169	:param value: unicode, str
170	:return: bytes
171	"""
172	coerced = str(value)	5✔
173	if coerced == value:	5✔
174	return coerced.encode(cls._encoding)	5✔
175
176	raise InvalidValue('not text')	5✔
177
178	@classmethod	5✔
179	def decode(cls, value: Optional[bytes]) -> Optional[str]:	5✔
180	"""
181	take bytes from redis and turn them into unicode string
182
183	:param value:
184	:return:
185	"""
186	return None if value is None else str(value.decode(cls._encoding))	5✔
187
188
189	class AsciiField(TextField):	5✔
190	"""
191	Used for ascii-only text
192	"""
193	PATTERN = re.compile('^([ -~]+)?$')	5✔
194
195	@classmethod	5✔
196	def encode(cls, value: str) -> bytes:	5✔
197	"""
198	take a list of strings and turn it into utf-8 byte-string
199
200	:param value:
201	:return:
202	"""
203	coerced = str(value)	5✔
204	if coerced == value and cls.PATTERN.match(coerced):	5✔
205	return coerced.encode(cls._encoding)	5✔
206
207	raise InvalidValue('not ascii')	5✔
208
209
210	class BinaryField(object):	5✔
211	"""
212	A bytes field. Not encoded.
213	"""
214
215	@classmethod	5✔
216	def encode(cls, value: bytes) -> bytes:	5✔
217	"""
218	write binary data into redis without encoding it.
219
220	:param value: bytes
221	:return: bytes
222	"""
223	try:	5✔
224	coerced = bytes(value)	5✔
225	if coerced == value:	5✔
226	return coerced	5✔
227	except (TypeError, UnicodeError):	5✔
228	pass	5✔
229
230	raise InvalidValue('not binary')	5✔
231
232	@classmethod	5✔
233	def decode(cls, value: Optional[bytes]) -> Optional[bytes]:	5✔
234	"""
235	read binary data from redis and pass it on through.
236
237	:param value: bytes
238	:return: bytes
239	"""
240	return None if value is None else bytes(value)	5✔
241
242
243	class ListField(object):	5✔
244	"""
245	A list field. Marshalled in and out of redis via json.
246	Values of the list can be any arbitrary data.
247	"""
248	_encoding = 'utf-8'	5✔
249
250	@classmethod	5✔
251	def encode(cls, value: list) -> bytes:	5✔
252	"""
253	take a list and turn it into a utf-8 encoded byte-string for redis.
254
255	:param value: list
256	:return: bytes
257	"""
258	try:	5✔
259	coerced = list(value)	5✔
260	if coerced == value:	5✔
261	return json.dumps(coerced).encode(cls._encoding)	5✔
262	except TypeError:	5✔
263	pass	5✔
264
265	raise InvalidValue('not a list')	5✔
266
267	@classmethod	5✔
268	def decode(cls, value: Union[bytes, None, list]) -> Optional[list]:	5✔
269	"""
270	take a utf-8 encoded byte-string from redis and
271	turn it back into a list
272
273	:param value: bytes
274	:return: list
275	"""
276	try:	5✔
277	return None if value is None else \	5✔
278	list(json.loads(value.decode(cls._encoding))) # type: ignore
279	except (TypeError, AttributeError):	5✔
280	return list(value) # type: ignore	5✔
281
282
283	class DictField(object):	5✔
284	_encoding = 'utf-8'	5✔
285
286	@classmethod	5✔
287	def encode(cls, value: dict) -> bytes:	5✔
288	"""
289	encode the dict as a json string to be written into redis.
290
291	:param value: dict
292	:return: bytes
293	"""
294	try:	5✔
295	coerced = dict(value)	5✔
296	if coerced == value:	5✔
297	return json.dumps(coerced).encode(cls._encoding)	5✔
298	except (TypeError, ValueError):	5✔
299	pass	5✔
300	raise InvalidValue('not a dict')	5✔
301
302	@classmethod	5✔
303	def decode(cls, value: Union[bytes, None, dict]) -> Optional[dict]:	5✔
304	"""
305	decode the data from a json string in redis back into a dict object.
306
307	:param value: bytes
308	:return: dict
309	"""
310	try:	5✔
311	return None if value is None else \	5✔
312	dict(json.loads(value.decode(cls._encoding))) # type: ignore
313	except (TypeError, AttributeError):	5✔
314	return dict(value) # type: ignore	5✔
315
316
317	class StringListField(object):	5✔
318	"""
319	Used for storing a list of strings, serialized as a comma-separated list.
320	"""
321	_encoding = 'utf-8'	5✔
322
323	@classmethod	5✔
324	def decode(cls,	5✔
325	value: Union[bytes, None, typing.List[str]]
326	) -> Optional[typing.List[str]]:
327	"""
328	decode the data from redis.
329	:param value: bytes
330	:return: list
331	"""
332	if value is None or isinstance(value, list):	5✔
333	return value	5✔
334
335	try:	5✔
336	data = [v for v in value.decode(cls._encoding).split(',') if	5✔
337	v != '']
338	return data if data else None	5✔
339	except AttributeError:	5✔
340	return None	5✔
341
342	@classmethod	5✔
343	def encode(cls, value: typing.List[str]) -> bytes:	5✔
344	"""
345	encode the list it so it can be stored in redis.
346
347	:param value: list
348	:return: bytes
349	"""
350	try:	5✔
351	coerced = [str(v) for v in value]	5✔
352	if coerced == value:	5✔
353	return ",".join(coerced).encode(cls._encoding) if len(	5✔
354	value) > 0 else b''
355	except TypeError:	5✔
356	pass	5✔
357
358	raise InvalidValue('not a list of strings')	5✔

72squared / redpipe / 215

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous