461876467

Build Type

push

github-actions

Committed by James McKinney

Commit Message

<a href="https://github.com/open-contracting/ocdskit/commit/<a class=hub.com/open-contracting/ocdskit/commit/<a class="double-link" href="https://git"><a class=hub.com/open-contracting/ocdskit/commit/<a class="double-link" href="https://git"><a class=hub.com/open-contracting/ocdskit/commit/f0d09835f5b1e159c9662d8b706c5b3d0439cf47">f0d09835f<a href="https://github.com/open-contracting/ocdskit/commit/f0d09835f5b1e159c9662d8b706c5b3d0439cf47">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/open-contracting/ocdskit/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/open-contracting/ocdskit/commit/f0d09835f5b1e159c9662d8b706c5b3d0439cf47&quot;&gt;f0d09835f&lt;/a&gt;&lt;a href=&quot;https://github.com/open-contracting/ocdskit/commit/f0d09835f5b1e159c9662d8b706c5b3d0439cf47&quot;&gt;&amp;quot;&amp;gt;json: Fix &amp;lt;/a&amp;gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/open-contracting/ocdskit/commit/</a><a class="double-link" href="https://github.com/open-contracting/ocdskit/commit/<a class="double-link" href="https://github.com/open-contracting/ocdskit/commit/ce1da46f93c024cca94f6a4fa374b82925a95177">ce1da46f9</a>">ce1da46f9</a><a href="https://github.com/open-contracting/ocdskit/commit/f0d09835f5b1e159c9662d8b706c5b3d0439cf47">">ce1da46f9</a>

Run Details

2 of 2 new or added lines in 1 file covered. (100.0%)

1969 of 2037 relevant lines covered (96.66%)

23.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.52

/ocdskit/util.py

import itertools
import json
from decimal import Decimal
import ijson

from ocdskit.exceptions import UnknownFormatError

try:
    import orjson

    jsonlib = orjson
    USING_ORJSON = True
except ImportError:
    jsonlib = json
    USING_ORJSON = False


# See `grouper` recipe: https://docs.python.org/3.8/library/itertools.html#recipes
def grouper(iterable, n, fillvalue=None):
    args = [iter(iterable)] * n
    return itertools.zip_longest(*args, fillvalue=fillvalue)


# https://stackoverflow.com/questions/21663800/python-make-a-list-generator-json-serializable/46841935#46841935
class SerializableGenerator(list):
    def __init__(self, iterable):
        iterator = iter(iterable)
        try:
            # If `iter()` is omitted, then `__iter__` won't exhaust `head`.
            self.head = iter([next(iterator)])
            # Adding an item to the list ensures `__bool__` and `__len__` work.
            self.append(iterator)
        except StopIteration:
            # `__iter__` requires `head` to be set.
            self.head = []

    def __iter__(self):
        # `*self[:1]` is used, because `self[0]` raises IndexError when `iterable` is empty.
        return itertools.chain(self.head, *self[:1])


class JSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Decimal):
            return float(obj)
        # https://docs.python.org/3/library/json.html#json.JSONEncoder.default
        try:
            iter(obj)
        except TypeError:
            pass
        else:
            return SerializableGenerator(obj)
        return json.JSONEncoder.default(self, obj)


def iterencode(data, ensure_ascii=False, **kwargs):
    """
    Returns a generator that yields each string representation as available.
    """
    if 'indent' not in kwargs:
        kwargs['separators'] = (',', ':')
    return JSONEncoder(ensure_ascii=ensure_ascii, **kwargs).iterencode(data)


def json_dump(data, io, ensure_ascii=False, **kwargs):
    """
    Dumps JSON to a file-like object.
    """
    if 'indent' not in kwargs:
        kwargs['separators'] = (',', ':')
    json.dump(data, io, ensure_ascii=ensure_ascii, cls=JSONEncoder, **kwargs)


def json_dumps(data, ensure_ascii=False, indent=None, sort_keys=False, **kwargs):
    """
    Dumps JSON to a string, and returns it.
    """
    # orjson doesn't support `ensure_ascii` if `True`, `indent` if not `2` or other arguments except for `sort_keys`.
    if not USING_ORJSON or ensure_ascii or indent and indent != 2 or kwargs:
        if not indent:
            kwargs['separators'] = (',', ':')
        return json.dumps(data, cls=JSONEncoder, ensure_ascii=ensure_ascii, indent=indent, sort_keys=sort_keys,
                          **kwargs)

    option = 0
    if indent:
        option |= orjson.OPT_INDENT_2
    if sort_keys:
        option |= orjson.OPT_SORT_KEYS

    # orjson dumps to bytes.
    return orjson.dumps(data, default=JSONEncoder().default, option=option).decode()


def get_ocds_minor_version(data):
    """
    Returns the OCDS minor version of the record package, release package, record or release.
    """
    if is_package(data):
        if 'version' in data:
            return data['version']
        return '1.0'
    if is_record(data):
        if any('parties' in release for release in data['releases']):
            return '1.1'
        return '1.0'
    # release
    if 'parties' in data:
        return '1.1'
    return '1.0'


def is_package(data):
    """
    Returns whether the data is a record package or release package.
    """
    return is_record_package(data) or is_release_package(data)


def is_record_package(data):
    """
    Returns whether the data is a record package.

    A record package has a required ``records`` field. Its other required fields are shared with release packages.
    """
    return 'records' in data


def is_record(data):
    """
    Returns whether the data is a record.

    A record has required ``releases`` and ``ocid`` fields.
    """
    return 'releases' in data and 'ocid' in data


def is_release_package(data):
    """
    Returns whether the data is a release package.

    A release package has a required ``releases`` field. Its other required fields are shared with record packages.
    To distinguish a release package from a record, we test for the absence of the ``ocid`` field.
    """
    return 'releases' in data and 'ocid' not in data


def is_release(data):
    """
    Returns whether the data is a release (embedded or linked, individual or compiled).
    """
    return 'date' in data


def is_compiled_release(data):
    """
    Returns whether the data is a compiled release (embedded or linked).
    """
    return 'tag' in data and isinstance(data['tag'], list) and 'compiled' in data['tag']


def is_linked_release(data):
    """
    Returns whether the data is a linked release.

    A linked release has required ``url`` and ``date`` fields and an optional ``tag`` field. An embedded release has
    required ``date`` and ``tag`` fields (among others), and it can have a ``url`` field as an additional field.

    To distinguish a linked release from an embedded release, we test for the presence of the required ``url`` field
    and test whether the number of fields is fewer than three.
    """
    return 'url' in data and len(data) <= 3


def _empty_package(uri, publisher, published_date, version):
    if publisher is None:
        publisher = {}

    return {
        'uri': uri,
        'publisher': publisher,
        'publishedDate': published_date,
        'license': None,
        'publicationPolicy': None,
        'version': version,
        'extensions': {},
    }


def detect_format(path, root_path=''):
    """
    Returns the format of OCDS data, and whether the OCDS data is concatenated or in an array.

    If the OCDS data is concatenated or in an array, assumes that all items have the same format as the first item.

    :param str path: the path to a file
    :param str root_path: the path to the OCDS data within the file
    :returns: the format, whether data is concatenated, and whether data is in an array
    :rtype: tuple
    :raises UnknownFormatError: if the format cannot be detected
    """
    with open(path, 'rb') as f:
        events = iter(ijson.parse(f, multiple_values=True))

        while True:
            prefix, event, value = next(events)
            if prefix == root_path:
                break

        if prefix:
            prefix += '.'

        if event == 'start_array':
            prefix += 'item.'
        elif event != 'start_map':
            raise UnknownFormatError('top-level JSON value is a {}'.format(event))

        records_prefix = '{}records'.format(prefix)
        releases_prefix = '{}releases'.format(prefix)
        ocid_prefix = '{}ocid'.format(prefix)
        tag_item_prefix = '{}tag.item'.format(prefix)

        has_records = False
        has_releases = False
        has_ocid = False
        has_tag = False
        is_compiled = False
        is_array = event == 'start_array'

        for prefix, event, value in events:
            if prefix == records_prefix:
                has_records = True
            elif prefix == releases_prefix:
                has_releases = True
            elif prefix == ocid_prefix:
                has_ocid = True
            elif prefix == tag_item_prefix:
                has_tag = True
                if value == 'compiled':
                    is_compiled = True
            if not prefix and event not in ('end_array', 'end_map', 'map_key'):
                return _detect_format_result(True, is_array, has_records, has_releases, has_ocid, has_tag, is_compiled)

        return _detect_format_result(False, is_array, has_records, has_releases, has_ocid, has_tag, is_compiled)


def _detect_format_result(is_concatenated, is_array, has_records, has_releases, has_ocid, has_tag, is_compiled):
    if has_records:
        detected_format = 'record package'
    elif has_releases and has_ocid:
        detected_format = 'record'
    elif has_releases:
        detected_format = 'release package'
    elif is_compiled:
        detected_format = 'compiled release'
    elif has_tag:
        detected_format = 'release'
    elif has_ocid:
        detected_format = 'versioned release'
    else:
        if is_array:
            infix = 'array'
        else:
            infix = 'object'
        raise UnknownFormatError('top-level JSON value is a non-OCDS {}'.format(infix))

    return (detected_format, is_concatenated, is_array)


def _empty_record_package(uri='', publisher=None, published_date='', version=None):
    package = _empty_package(uri, publisher, published_date, version)
    package['packages'] = []
    package['records'] = []
    return package


def _empty_release_package(uri='', publisher=None, published_date='', version=None):
    package = _empty_package(uri, publisher, published_date, version)
    package['releases'] = []
    return package


def _update_package_metadata(output, package):
    for field in ('publisher', 'license', 'publicationPolicy'):
        if field in package:
            output[field] = package[field]

    # We use an insertion-ordered dict to keep extensions in order without duplication.
    if 'extensions' in package:
        output['extensions'].update(dict.fromkeys(package['extensions']))


def _resolve_metadata(output, field):
    if output[field]:
        output[field] = list(output[field])
    else:
        del output[field]


def _remove_empty_optional_metadata(output):
    for field in ('license', 'publicationPolicy', 'version'):
        if output[field] is None:
            del output[field]

1	import itertools	24✔
2	import json	24✔
3	from decimal import Decimal	24✔
4	import ijson	24✔
5
6	from ocdskit.exceptions import UnknownFormatError	24✔
7
8	try:	24✔
9	import orjson	24✔
10
11	jsonlib = orjson	24✔
12	USING_ORJSON = True	24✔
13	except ImportError:	24✔
14	jsonlib = json	24✔
15	USING_ORJSON = False	24✔
16
17
18	# See `grouper` recipe: https://docs.python.org/3.8/library/itertools.html#recipes
19	def grouper(iterable, n, fillvalue=None):	24✔
20	args = [iter(iterable)] * n	24✔
21	return itertools.zip_longest(*args, fillvalue=fillvalue)	24✔
22
23
24	# https://stackoverflow.com/questions/21663800/python-make-a-list-generator-json-serializable/46841935#46841935
25	class SerializableGenerator(list):	24✔
26	def __init__(self, iterable):	24✔
27	iterator = iter(iterable)	24✔
28	try:	24✔
29	# If `iter()` is omitted, then `__iter__` won't exhaust `head`.
30	self.head = iter([next(iterator)])	24✔
31	# Adding an item to the list ensures `__bool__` and `__len__` work.
32	self.append(iterator)	24✔
33	except StopIteration:	24✔
34	# `__iter__` requires `head` to be set.
35	self.head = []	24✔
36
37	def __iter__(self):	24✔
38	# `*self[:1]` is used, because `self[0]` raises IndexError when `iterable` is empty.
39	return itertools.chain(self.head, *self[:1])	24✔
40
41
42	class JSONEncoder(json.JSONEncoder):	24✔
43	def default(self, obj):	24✔
44	if isinstance(obj, Decimal):	24✔
45	return float(obj)	24✔
46	# https://docs.python.org/3/library/json.html#json.JSONEncoder.default
47	try:	24✔
48	iter(obj)	24✔
49	except TypeError:	×
50	pass	×
51	else:
52	return SerializableGenerator(obj)	24✔
53	return json.JSONEncoder.default(self, obj)	×
54
55
56	def iterencode(data, ensure_ascii=False, **kwargs):	24✔
57	"""
58	Returns a generator that yields each string representation as available.
59	"""
60	if 'indent' not in kwargs:	24✔
61	kwargs['separators'] = (',', ':')	24✔
62	return JSONEncoder(ensure_ascii=ensure_ascii, **kwargs).iterencode(data)	24✔
63
64
65	def json_dump(data, io, ensure_ascii=False, **kwargs):	24✔
66	"""
67	Dumps JSON to a file-like object.
68	"""
69	if 'indent' not in kwargs:	24✔
70	kwargs['separators'] = (',', ':')	24✔
71	json.dump(data, io, ensure_ascii=ensure_ascii, cls=JSONEncoder, **kwargs)	24✔
72
73
74	def json_dumps(data, ensure_ascii=False, indent=None, sort_keys=False, **kwargs):	24✔
75	"""
76	Dumps JSON to a string, and returns it.
77	"""
78	# orjson doesn't support `ensure_ascii` if `True`, `indent` if not `2` or other arguments except for `sort_keys`.
79	if not USING_ORJSON or ensure_ascii or indent and indent != 2 or kwargs:	24✔
80	if not indent:	24✔
81	kwargs['separators'] = (',', ':')	24✔
82	return json.dumps(data, cls=JSONEncoder, ensure_ascii=ensure_ascii, indent=indent, sort_keys=sort_keys,	24✔
83	**kwargs)
84
85	option = 0	24✔
86	if indent:	24✔
87	option \|= orjson.OPT_INDENT_2	24✔
88	if sort_keys:	24✔
89	option \|= orjson.OPT_SORT_KEYS	×
90
91	# orjson dumps to bytes.
92	return orjson.dumps(data, default=JSONEncoder().default, option=option).decode()	24✔
93
94
95	def get_ocds_minor_version(data):	24✔
96	"""
97	Returns the OCDS minor version of the record package, release package, record or release.
98	"""
99	if is_package(data):	24✔
100	if 'version' in data:	24✔
101	return data['version']	24✔
102	return '1.0'	24✔
103	if is_record(data):	24✔
104	if any('parties' in release for release in data['releases']):	24✔
105	return '1.1'	24✔
106	return '1.0'	24✔
107	# release
108	if 'parties' in data:	24✔
109	return '1.1'	24✔
110	return '1.0'	24✔
111
112
113	def is_package(data):	24✔
114	"""
115	Returns whether the data is a record package or release package.
116	"""
117	return is_record_package(data) or is_release_package(data)	24✔
118
119
120	def is_record_package(data):	24✔
121	"""
122	Returns whether the data is a record package.
123
124	A record package has a required ``records`` field. Its other required fields are shared with release packages.
125	"""
126	return 'records' in data	24✔
127
128
129	def is_record(data):	24✔
130	"""
131	Returns whether the data is a record.
132
133	A record has required ``releases`` and ``ocid`` fields.
134	"""
135	return 'releases' in data and 'ocid' in data	24✔
136
137
138	def is_release_package(data):	24✔
139	"""
140	Returns whether the data is a release package.
141
142	A release package has a required ``releases`` field. Its other required fields are shared with record packages.
143	To distinguish a release package from a record, we test for the absence of the ``ocid`` field.
144	"""
145	return 'releases' in data and 'ocid' not in data	24✔
146
147
148	def is_release(data):	24✔
149	"""
150	Returns whether the data is a release (embedded or linked, individual or compiled).
151	"""
152	return 'date' in data	24✔
153
154
155	def is_compiled_release(data):	24✔
156	"""
157	Returns whether the data is a compiled release (embedded or linked).
158	"""
159	return 'tag' in data and isinstance(data['tag'], list) and 'compiled' in data['tag']	24✔
160
161
162	def is_linked_release(data):	24✔
163	"""
164	Returns whether the data is a linked release.
165
166	A linked release has required ``url`` and ``date`` fields and an optional ``tag`` field. An embedded release has
167	required ``date`` and ``tag`` fields (among others), and it can have a ``url`` field as an additional field.
168
169	To distinguish a linked release from an embedded release, we test for the presence of the required ``url`` field
170	and test whether the number of fields is fewer than three.
171	"""
172	return 'url' in data and len(data) <= 3	24✔
173
174
175	def _empty_package(uri, publisher, published_date, version):	24✔
176	if publisher is None:	24✔
177	publisher = {}	24✔
178
179	return {	24✔
180	'uri': uri,
181	'publisher': publisher,
182	'publishedDate': published_date,
183	'license': None,
184	'publicationPolicy': None,
185	'version': version,
186	'extensions': {},
187	}
188
189
190	def detect_format(path, root_path=''):	24✔
191	"""
192	Returns the format of OCDS data, and whether the OCDS data is concatenated or in an array.
193
194	If the OCDS data is concatenated or in an array, assumes that all items have the same format as the first item.
195
196	:param str path: the path to a file
197	:param str root_path: the path to the OCDS data within the file
198	:returns: the format, whether data is concatenated, and whether data is in an array
199	:rtype: tuple
200	:raises UnknownFormatError: if the format cannot be detected
201	"""
202	with open(path, 'rb') as f:	24✔
203	events = iter(ijson.parse(f, multiple_values=True))	24✔
204
205	while True:	12✔
206	prefix, event, value = next(events)	24✔
207	if prefix == root_path:	24✔
208	break	24✔
209
210	if prefix:	24✔
211	prefix += '.'	24✔
212
213	if event == 'start_array':	24✔
214	prefix += 'item.'	24✔
215	elif event != 'start_map':	24✔
216	raise UnknownFormatError('top-level JSON value is a {}'.format(event))	24✔
217
218	records_prefix = '{}records'.format(prefix)	24✔
219	releases_prefix = '{}releases'.format(prefix)	24✔
220	ocid_prefix = '{}ocid'.format(prefix)	24✔
221	tag_item_prefix = '{}tag.item'.format(prefix)	24✔
222
223	has_records = False	24✔
224	has_releases = False	24✔
225	has_ocid = False	24✔
226	has_tag = False	24✔
227	is_compiled = False	24✔
228	is_array = event == 'start_array'	24✔
229
230	for prefix, event, value in events:	24✔
231	if prefix == records_prefix:	24✔
232	has_records = True	24✔
233	elif prefix == releases_prefix:	24✔
234	has_releases = True	24✔
235	elif prefix == ocid_prefix:	24✔
236	has_ocid = True	24✔
237	elif prefix == tag_item_prefix:	24✔
238	has_tag = True	24✔
239	if value == 'compiled':	24✔
240	is_compiled = True	24✔
241	if not prefix and event not in ('end_array', 'end_map', 'map_key'):	24✔
242	return _detect_format_result(True, is_array, has_records, has_releases, has_ocid, has_tag, is_compiled)	24✔
243
244	return _detect_format_result(False, is_array, has_records, has_releases, has_ocid, has_tag, is_compiled)	24✔
245
246
247	def _detect_format_result(is_concatenated, is_array, has_records, has_releases, has_ocid, has_tag, is_compiled):	24✔
248	if has_records:	24✔
249	detected_format = 'record package'	24✔
250	elif has_releases and has_ocid:	24✔
251	detected_format = 'record'	24✔
252	elif has_releases:	24✔
253	detected_format = 'release package'	24✔
254	elif is_compiled:	24✔
255	detected_format = 'compiled release'	24✔
256	elif has_tag:	24✔
257	detected_format = 'release'	24✔
258	elif has_ocid:	24✔
259	detected_format = 'versioned release'	24✔
260	else:
261	if is_array:	24✔
262	infix = 'array'	24✔
263	else:
264	infix = 'object'	24✔
265	raise UnknownFormatError('top-level JSON value is a non-OCDS {}'.format(infix))	24✔
266
267	return (detected_format, is_concatenated, is_array)	24✔
268
269
270	def _empty_record_package(uri='', publisher=None, published_date='', version=None):	24✔
271	package = _empty_package(uri, publisher, published_date, version)	24✔
272	package['packages'] = []	24✔
273	package['records'] = []	24✔
274	return package	24✔
275
276
277	def _empty_release_package(uri='', publisher=None, published_date='', version=None):	24✔
278	package = _empty_package(uri, publisher, published_date, version)	24✔
279	package['releases'] = []	24✔
280	return package	24✔
281
282
283	def _update_package_metadata(output, package):	24✔
284	for field in ('publisher', 'license', 'publicationPolicy'):	24✔
285	if field in package:	24✔
286	output[field] = package[field]	24✔
287
288	# We use an insertion-ordered dict to keep extensions in order without duplication.
289	if 'extensions' in package:	24✔
290	output['extensions'].update(dict.fromkeys(package['extensions']))	24✔
291
292
293	def _resolve_metadata(output, field):	24✔
294	if output[field]:	24✔
295	output[field] = list(output[field])	24✔
296	else:
297	del output[field]	24✔
298
299
300	def _remove_empty_optional_metadata(output):	24✔
301	for field in ('license', 'publicationPolicy', 'version'):	24✔
302	if output[field] is None:	24✔
303	del output[field]	24✔

open-contracting / ocdskit / 461876467

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous