10217049583

Committed 02 Aug 2024 02:12PM UTC coverage: 89.878% (+0.3%) from 89.605%

Build # 10217049583

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #822 from aaxelb/fix/up-celery

[ENG-6072] fix ci

Run Details

22642 of 25192 relevant lines covered (89.88%)

1.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

24.03

/trove/render/jsonapi.py

import contextlib
import datetime
import hashlib
import json
from typing import Iterable, Union

from primitive_metadata import primitive_rdf

from trove import exceptions as trove_exceptions
from trove.vocab.jsonapi import (
    JSONAPI_MEMBERNAME,
    JSONAPI_RELATIONSHIP,
    JSONAPI_ATTRIBUTE,
    JSONAPI_LINK_OBJECT,
)
from trove.vocab import mediatypes
from trove.vocab.namespaces import (
    OSFMAP,
    OWL,
    RDF,
    TROVE,
)
from trove.vocab.trove import (
    TROVE_API_THESAURUS,
    trove_indexcard_namespace,
)
from ._base import BaseRenderer


# a jsonapi resource may pull rdf data using an iri or blank node
# (using conventions from py for rdf as python primitives)
_IriOrBlanknode = Union[str, frozenset]


class RdfJsonapiRenderer(BaseRenderer):
    '''render rdf data into jsonapi resources, guided by a given rdf vocabulary

    the given vocab describes how rdf predicates and classes in the data should
    map to jsonapi fields and resource objects in the rendered output, using
    `prefix jsonapi: <https://jsonapi.org/format/1.1/#>` and linked anchors in
    the jsonapi spec to represent jsonapi concepts:
      - jsonapi member name:
          `<iri> jsonapi:document-member-names "foo"@en`
      - jsonapi attribute:
          `<predicate_iri> rdf:type jsonapi:document-resource-object-attributes`
      - jsonapi relationship:
          `<predicate_iri> rdf:type jsonapi:document-resource-object-relationships`
      - to-one relationship or single-value attribute:
          `<predicate_iri> rdf:type owl:FunctionalProperty`

    note: does not support relationship links (or many other jsonapi features)
    '''
    MEDIATYPE = mediatypes.JSONAPI
    INDEXCARD_DERIVER_IRI = TROVE['derive/osfmap_json']

    __to_include: set[primitive_rdf.RdfObject] | None = None

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self._vocab = primitive_rdf.RdfGraph(TROVE_API_THESAURUS)
        self._identifier_object_cache = {}
        # TODO: move "id namespace" to vocab (property on each type)
        self._id_namespace_set = [trove_indexcard_namespace()]

    def render_document(self, data: primitive_rdf.RdfGraph, focus_iri: str) -> str:
        self._data = data
        return json.dumps(
            self.render_dict(focus_iri),
            indent=2,  # TODO: pretty-print query param?
        )

    def render_dict(self, primary_iris: Union[str, Iterable[str]]) -> dict:
        _primary_data: dict | list | None = None
        _included_data = []
        with self._contained__to_include() as _to_include:
            if isinstance(primary_iris, str):
                _already_included = {primary_iris}
                _primary_data = self.render_resource_object(primary_iris)
            else:
                _already_included = set(primary_iris)
                _primary_data = [
                    self.render_resource_object(_iri)
                    for _iri in primary_iris
                ]
            while _to_include:
                _next = _to_include.pop()
                if _next not in _already_included:
                    _already_included.add(_next)
                    _included_data.append(self.render_resource_object(_next))
        _document = {'data': _primary_data}
        if _included_data:
            _document['included'] = _included_data
        return _document

    def render_resource_object(self, iri_or_blanknode: _IriOrBlanknode) -> dict:
        _resource_object = {**self.render_identifier_object(iri_or_blanknode)}
        _twopledict = (
            (self._data.tripledict.get(iri_or_blanknode) or {})
            if isinstance(iri_or_blanknode, str)
            else primitive_rdf.twopledict_from_twopleset(iri_or_blanknode)
        )
        for _pred, _obj_set in _twopledict.items():
            if _pred != RDF.type:
                self._render_field(_pred, _obj_set, into=_resource_object)
        if isinstance(iri_or_blanknode, str):
            _resource_object.setdefault('links', {})['self'] = iri_or_blanknode
        return _resource_object

    def render_identifier_object(self, iri_or_blanknode: _IriOrBlanknode):
        try:
            return self._identifier_object_cache[iri_or_blanknode]
        except KeyError:
            if isinstance(iri_or_blanknode, str):
                _type_iris = list(self._data.q(iri_or_blanknode, RDF.type))
                _id_obj = {
                    'id': self._resource_id_for_iri(iri_or_blanknode),
                    'type': self._single_typename(_type_iris),
                }
            elif isinstance(iri_or_blanknode, frozenset):
                _type_iris = [
                    _obj
                    for _pred, _obj in iri_or_blanknode
                    if _pred == RDF.type
                ]
                _id_obj = {
                    'id': self._resource_id_for_blanknode(iri_or_blanknode),
                    'type': self._single_typename(_type_iris),
                }
            else:
                raise trove_exceptions.ExpectedIriOrBlanknode(f'expected str or frozenset (got {iri_or_blanknode})')
            self._identifier_object_cache[iri_or_blanknode] = _id_obj
            return _id_obj

    def _single_typename(self, type_iris: list[str]):
        if not type_iris:
            raise trove_exceptions.MissingRdfType
        if len(type_iris) == 1:
            return self._membername_for_iri(type_iris[0])
        # choose one predictably, preferring osfmap and trove
        for _namespace in (OSFMAP, TROVE):
            _type_iris = sorted(_iri for _iri in type_iris if _iri in _namespace)
            if _type_iris:
                return self._membername_for_iri(_type_iris[0])
        return self._membername_for_iri(sorted(type_iris)[0])

    def _membername_for_iri(self, iri: str):
        try:
            _membername = next(self._vocab.q(iri, JSONAPI_MEMBERNAME))
        except StopIteration:
            pass
        else:
            if isinstance(_membername, primitive_rdf.Literal):
                return _membername.unicode_value
            raise trove_exceptions.ExpectedLiteralObject((iri, JSONAPI_MEMBERNAME, _membername))
        return self.iri_shorthand.compact_iri(iri)

    def _resource_id_for_blanknode(self, blanknode: frozenset):
        # content-addressed blanknode id (maybe-TODO: care about hash stability,
        # tho don't need it with cached render_identifier_object implementation)
        return hashlib.sha256(str(blanknode).encode()).hexdigest()

    def _resource_id_for_iri(self, iri: str):
        for _iri_namespace in self._id_namespace_set:
            if iri in _iri_namespace:
                return primitive_rdf.iri_minus_namespace(iri, namespace=_iri_namespace)
        # as fallback, hash the iri for a valid jsonapi member name
        return hashlib.sha256(iri.encode()).hexdigest()

    def _render_field(self, predicate_iri, object_set, *, into: dict):
        _is_relationship = (predicate_iri, RDF.type, JSONAPI_RELATIONSHIP) in self._vocab
        _is_attribute = (predicate_iri, RDF.type, JSONAPI_ATTRIBUTE) in self._vocab
        _field_key = self._membername_for_iri(predicate_iri)
        _doc_key = 'meta'  # unless configured for jsonapi, default to unstructured 'meta'
        if ':' not in _field_key:
            if _is_relationship:
                _doc_key = 'relationships'
            elif _is_attribute:
                _doc_key = 'attributes'
        if _is_relationship:
            _fieldvalue = self._render_relationship_object(predicate_iri, object_set)
        else:
            _fieldvalue = self._one_or_many(predicate_iri, self._attribute_datalist(object_set))
        # update the given `into` resource object
        into.setdefault(_doc_key, {})[_field_key] = _fieldvalue

    def _one_or_many(self, predicate_iri: str, datalist: list):
        _only_one = (predicate_iri, RDF.type, OWL.FunctionalProperty) in self._vocab
        if _only_one:
            if len(datalist) > 1:
                raise trove_exceptions.OwlObjection(f'multiple objects for to-one relation <{predicate_iri}>: {datalist}')
            return (datalist[0] if datalist else None)
        return datalist

    def _attribute_datalist(self, object_set):
        return [
            self._render_attribute_datum(_obj)
            for _obj in object_set
        ]

    def _render_relationship_object(self, predicate_iri, object_set):
        _data = []
        _links = {}
        for _obj in object_set:
            if isinstance(_obj, frozenset):
                if (RDF.type, RDF.Seq) in _obj:
                    for _seq_obj in primitive_rdf.sequence_objects_in_order(_obj):
                        _data.append(self.render_identifier_object(_seq_obj))
                        self._pls_include(_seq_obj)
                elif (RDF.type, JSONAPI_LINK_OBJECT) in _obj:
                    _key, _link_obj = self._render_link_object(_obj)
                    _links[_key] = _link_obj
                else:
                    _data.append(self.render_identifier_object(_obj))
                    self._pls_include(_obj)
            else:
                assert isinstance(_obj, str)
                _data.append(self.render_identifier_object(_obj))
                self._pls_include(_obj)
        _relationship_obj = {
            'data': self._one_or_many(predicate_iri, _data),
        }
        if _links:
            _relationship_obj['links'] = _links
        return _relationship_obj

    def _render_link_object(self, link_obj: frozenset):
        _membername = next(
            _obj.unicode_value
            for _pred, _obj in link_obj
            if _pred == JSONAPI_MEMBERNAME
        )
        _rendered_link = {
            'href': next(
                _obj
                for _pred, _obj in link_obj
                if _pred == RDF.value
            ),
            # TODO:
            # 'rel':
            # 'describedby':
            # 'title':
            # 'type':
            # 'hreflang':
            # 'meta':
        }
        return _membername, _rendered_link

    def _make_object_gen(self, object_set):
        for _obj in object_set:
            if isinstance(_obj, frozenset) and ((RDF.type, RDF.Seq) in _obj):
                yield from primitive_rdf.sequence_objects_in_order(_obj)
            else:
                yield _obj

    @contextlib.contextmanager
    def _contained__to_include(self):
        assert self.__to_include is None
        self.__to_include = set()
        try:
            yield self.__to_include
        finally:
            self.__to_include = None

    def _pls_include(self, item):
        if self.__to_include is not None:
            self.__to_include.add(item)

    def _render_attribute_datum(self, rdfobject: primitive_rdf.RdfObject) -> dict | list | str | float | int:
        if isinstance(rdfobject, frozenset):
            if (RDF.type, RDF.Seq) in rdfobject:
                return [
                    self._render_attribute_datum(_seq_obj)
                    for _seq_obj in primitive_rdf.sequence_objects_in_order(rdfobject)
                ]
            _json_blanknode = {}
            for _pred, _obj_set in primitive_rdf.twopledict_from_twopleset(rdfobject).items():
                _key = self._membername_for_iri(_pred)
                _json_blanknode[_key] = self._one_or_many(_pred, self._attribute_datalist(_obj_set))
            return _json_blanknode
        if isinstance(rdfobject, primitive_rdf.Literal):
            if RDF.JSON in rdfobject.datatype_iris:
                return json.loads(rdfobject.unicode_value)
            return rdfobject.unicode_value  # TODO: decide how to represent language
        elif isinstance(rdfobject, str):
            try:  # maybe it's a jsonapi resource
                return self.render_identifier_object(rdfobject)
            except Exception:
                return rdfobject
        elif isinstance(rdfobject, (float, int)):
            return rdfobject
        elif isinstance(rdfobject, datetime.date):
            # just "YYYY-MM-DD"
            return datetime.date.isoformat(rdfobject)
        raise trove_exceptions.UnsupportedRdfObject(rdfobject)

1	import contextlib	2✔
2	import datetime	2✔
3	import hashlib	2✔
4	import json	2✔
5	from typing import Iterable, Union	2✔
6
7	from primitive_metadata import primitive_rdf	2✔
8
9	from trove import exceptions as trove_exceptions	2✔
10	from trove.vocab.jsonapi import (	2✔
11	JSONAPI_MEMBERNAME,
12	JSONAPI_RELATIONSHIP,
13	JSONAPI_ATTRIBUTE,
14	JSONAPI_LINK_OBJECT,
15	)
16	from trove.vocab import mediatypes	2✔
17	from trove.vocab.namespaces import (	2✔
18	OSFMAP,
19	OWL,
20	RDF,
21	TROVE,
22	)
23	from trove.vocab.trove import (	2✔
24	TROVE_API_THESAURUS,
25	trove_indexcard_namespace,
26	)
27	from ._base import BaseRenderer	2✔
28
29
30	# a jsonapi resource may pull rdf data using an iri or blank node
31	# (using conventions from py for rdf as python primitives)
32	_IriOrBlanknode = Union[str, frozenset]	2✔
33
34
35	class RdfJsonapiRenderer(BaseRenderer):	2✔
36	'''render rdf data into jsonapi resources, guided by a given rdf vocabulary	2✔
37
38	the given vocab describes how rdf predicates and classes in the data should	×
39	map to jsonapi fields and resource objects in the rendered output, using	×
40	`prefix jsonapi: <https://jsonapi.org/format/1.1/#>` and linked anchors in	×
41	the jsonapi spec to represent jsonapi concepts:	×
42	- jsonapi member name:	×
43	`<iri> jsonapi:document-member-names "foo"@en`	×
44	- jsonapi attribute:	×
45	`<predicate_iri> rdf:type jsonapi:document-resource-object-attributes`	×
46	- jsonapi relationship:	×
47	`<predicate_iri> rdf:type jsonapi:document-resource-object-relationships`	×
48	- to-one relationship or single-value attribute:	×
49	`<predicate_iri> rdf:type owl:FunctionalProperty`	×
50
51	note: does not support relationship links (or many other jsonapi features)	×
52	'''	×
53	MEDIATYPE = mediatypes.JSONAPI	2✔
54	INDEXCARD_DERIVER_IRI = TROVE['derive/osfmap_json']	2✔
55
56	__to_include: set[primitive_rdf.RdfObject] \| None = None	2✔
57
58	def __init__(self, **kwargs):	2✔
59	super().__init__(**kwargs)	×
60	self._vocab = primitive_rdf.RdfGraph(TROVE_API_THESAURUS)	×
61	self._identifier_object_cache = {}	×
62	# TODO: move "id namespace" to vocab (property on each type)
63	self._id_namespace_set = [trove_indexcard_namespace()]	×
64
65	def render_document(self, data: primitive_rdf.RdfGraph, focus_iri: str) -> str:	2✔
66	self._data = data	×
67	return json.dumps(
68	self.render_dict(focus_iri),
69	indent=2, # TODO: pretty-print query param?
70	)
71
72	def render_dict(self, primary_iris: Union[str, Iterable[str]]) -> dict:	2✔
73	_primary_data: dict \| list \| None = None	×
74	_included_data = []	×
75	with self._contained__to_include() as _to_include:	×
76	if isinstance(primary_iris, str):	×
77	_already_included = {primary_iris}	×
78	_primary_data = self.render_resource_object(primary_iris)	×
79	else:	×
80	_already_included = set(primary_iris)	×
81	_primary_data = [
82	self.render_resource_object(_iri)
83	for _iri in primary_iris
84	]
85	while _to_include:	×
86	_next = _to_include.pop()	×
87	if _next not in _already_included:	×
88	_already_included.add(_next)	×
89	_included_data.append(self.render_resource_object(_next))	×
90	_document = {'data': _primary_data}	×
91	if _included_data:	×
92	_document['included'] = _included_data	×
93	return _document	×
94
95	def render_resource_object(self, iri_or_blanknode: _IriOrBlanknode) -> dict:	2✔
96	_resource_object = {**self.render_identifier_object(iri_or_blanknode)}	×
97	_twopledict = (
98	(self._data.tripledict.get(iri_or_blanknode) or {})
99	if isinstance(iri_or_blanknode, str)
100	else primitive_rdf.twopledict_from_twopleset(iri_or_blanknode)
101	)
102	for _pred, _obj_set in _twopledict.items():	×
103	if _pred != RDF.type:	×
104	self._render_field(_pred, _obj_set, into=_resource_object)	×
105	if isinstance(iri_or_blanknode, str):	×
106	_resource_object.setdefault('links', {})['self'] = iri_or_blanknode	×
107	return _resource_object	×
108
109	def render_identifier_object(self, iri_or_blanknode: _IriOrBlanknode):	2✔
110	try:	×
111	return self._identifier_object_cache[iri_or_blanknode]	×
112	except KeyError:	×
113	if isinstance(iri_or_blanknode, str):	×
114	_type_iris = list(self._data.q(iri_or_blanknode, RDF.type))	×
115	_id_obj = {
116	'id': self._resource_id_for_iri(iri_or_blanknode),
117	'type': self._single_typename(_type_iris),
118	}
119	elif isinstance(iri_or_blanknode, frozenset):	×
120	_type_iris = [
121	_obj
122	for _pred, _obj in iri_or_blanknode
123	if _pred == RDF.type
124	]
125	_id_obj = {
126	'id': self._resource_id_for_blanknode(iri_or_blanknode),
127	'type': self._single_typename(_type_iris),
128	}
129	else:	×
130	raise trove_exceptions.ExpectedIriOrBlanknode(f'expected str or frozenset (got {iri_or_blanknode})')	×
131	self._identifier_object_cache[iri_or_blanknode] = _id_obj	×
132	return _id_obj	×
133
134	def _single_typename(self, type_iris: list[str]):	2✔
135	if not type_iris:	×
136	raise trove_exceptions.MissingRdfType	×
137	if len(type_iris) == 1:	×
138	return self._membername_for_iri(type_iris[0])	×
139	# choose one predictably, preferring osfmap and trove
140	for _namespace in (OSFMAP, TROVE):	×
141	_type_iris = sorted(_iri for _iri in type_iris if _iri in _namespace)	×
142	if _type_iris:	×
143	return self._membername_for_iri(_type_iris[0])	×
144	return self._membername_for_iri(sorted(type_iris)[0])	×
145
146	def _membername_for_iri(self, iri: str):	2✔
147	try:	×
148	_membername = next(self._vocab.q(iri, JSONAPI_MEMBERNAME))	×
149	except StopIteration:	×
150	pass	×
151	else:	×
152	if isinstance(_membername, primitive_rdf.Literal):	×
153	return _membername.unicode_value	×
154	raise trove_exceptions.ExpectedLiteralObject((iri, JSONAPI_MEMBERNAME, _membername))	×
155	return self.iri_shorthand.compact_iri(iri)	×
156
157	def _resource_id_for_blanknode(self, blanknode: frozenset):	2✔
158	# content-addressed blanknode id (maybe-TODO: care about hash stability,
159	# tho don't need it with cached render_identifier_object implementation)
160	return hashlib.sha256(str(blanknode).encode()).hexdigest()	×
161
162	def _resource_id_for_iri(self, iri: str):	2✔
163	for _iri_namespace in self._id_namespace_set:	×
164	if iri in _iri_namespace:	×
165	return primitive_rdf.iri_minus_namespace(iri, namespace=_iri_namespace)	×
166	# as fallback, hash the iri for a valid jsonapi member name
167	return hashlib.sha256(iri.encode()).hexdigest()	×
168
169	def _render_field(self, predicate_iri, object_set, *, into: dict):	2✔
170	_is_relationship = (predicate_iri, RDF.type, JSONAPI_RELATIONSHIP) in self._vocab	×
171	_is_attribute = (predicate_iri, RDF.type, JSONAPI_ATTRIBUTE) in self._vocab	×
172	_field_key = self._membername_for_iri(predicate_iri)	×
173	_doc_key = 'meta' # unless configured for jsonapi, default to unstructured 'meta'	×
174	if ':' not in _field_key:	×
175	if _is_relationship:	×
176	_doc_key = 'relationships'	×
177	elif _is_attribute:	×
178	_doc_key = 'attributes'	×
179	if _is_relationship:	×
180	_fieldvalue = self._render_relationship_object(predicate_iri, object_set)	×
181	else:	×
182	_fieldvalue = self._one_or_many(predicate_iri, self._attribute_datalist(object_set))	×
183	# update the given `into` resource object
184	into.setdefault(_doc_key, {})[_field_key] = _fieldvalue	×
185
186	def _one_or_many(self, predicate_iri: str, datalist: list):	2✔
187	_only_one = (predicate_iri, RDF.type, OWL.FunctionalProperty) in self._vocab	×
188	if _only_one:	×
189	if len(datalist) > 1:	×
190	raise trove_exceptions.OwlObjection(f'multiple objects for to-one relation <{predicate_iri}>: {datalist}')	×
191	return (datalist[0] if datalist else None)	×
192	return datalist	×
193
194	def _attribute_datalist(self, object_set):	2✔
195	return [
196	self._render_attribute_datum(_obj)
197	for _obj in object_set
198	]
199
200	def _render_relationship_object(self, predicate_iri, object_set):	2✔
201	_data = []	×
202	_links = {}	×
203	for _obj in object_set:	×
204	if isinstance(_obj, frozenset):	×
205	if (RDF.type, RDF.Seq) in _obj:	×
206	for _seq_obj in primitive_rdf.sequence_objects_in_order(_obj):	×
207	_data.append(self.render_identifier_object(_seq_obj))	×
208	self._pls_include(_seq_obj)	×
209	elif (RDF.type, JSONAPI_LINK_OBJECT) in _obj:	×
210	_key, _link_obj = self._render_link_object(_obj)	×
211	_links[_key] = _link_obj	×
212	else:	×
213	_data.append(self.render_identifier_object(_obj))	×
214	self._pls_include(_obj)	×
215	else:	×
216	assert isinstance(_obj, str)	×
217	_data.append(self.render_identifier_object(_obj))	×
218	self._pls_include(_obj)	×
219	_relationship_obj = {
220	'data': self._one_or_many(predicate_iri, _data),
221	}
222	if _links:	×
223	_relationship_obj['links'] = _links	×
224	return _relationship_obj	×
225
226	def _render_link_object(self, link_obj: frozenset):	2✔
227	_membername = next(
228	_obj.unicode_value
229	for _pred, _obj in link_obj
230	if _pred == JSONAPI_MEMBERNAME
231	)
232	_rendered_link = {
233	'href': next(
234	_obj
235	for _pred, _obj in link_obj
236	if _pred == RDF.value
237	),
238	# TODO:
239	# 'rel':
240	# 'describedby':
241	# 'title':
242	# 'type':
243	# 'hreflang':
244	# 'meta':
245	}
246	return _membername, _rendered_link
247
248	def _make_object_gen(self, object_set):	2✔
249	for _obj in object_set:
250	if isinstance(_obj, frozenset) and ((RDF.type, RDF.Seq) in _obj):
251	yield from primitive_rdf.sequence_objects_in_order(_obj)
252	else:
253	yield _obj
254
255	@contextlib.contextmanager	2✔
256	def _contained__to_include(self):	2✔
257	assert self.__to_include is None
258	self.__to_include = set()
259	try:
260	yield self.__to_include
261	finally:
262	self.__to_include = None
263
264	def _pls_include(self, item):	2✔
265	if self.__to_include is not None:
266	self.__to_include.add(item)
267
268	def _render_attribute_datum(self, rdfobject: primitive_rdf.RdfObject) -> dict \| list \| str \| float \| int:	2✔
269	if isinstance(rdfobject, frozenset):
270	if (RDF.type, RDF.Seq) in rdfobject:
271	return [
272	self._render_attribute_datum(_seq_obj)
273	for _seq_obj in primitive_rdf.sequence_objects_in_order(rdfobject)
274	]
275	_json_blanknode = {}
276	for _pred, _obj_set in primitive_rdf.twopledict_from_twopleset(rdfobject).items():
277	_key = self._membername_for_iri(_pred)
278	_json_blanknode[_key] = self._one_or_many(_pred, self._attribute_datalist(_obj_set))
279	return _json_blanknode
280	if isinstance(rdfobject, primitive_rdf.Literal):
281	if RDF.JSON in rdfobject.datatype_iris:
282	return json.loads(rdfobject.unicode_value)
283	return rdfobject.unicode_value # TODO: decide how to represent language
284	elif isinstance(rdfobject, str):
285	try: # maybe it's a jsonapi resource
286	return self.render_identifier_object(rdfobject)
287	except Exception:
288	return rdfobject
289	elif isinstance(rdfobject, (float, int)):
290	return rdfobject
291	elif isinstance(rdfobject, datetime.date):
292	# just "YYYY-MM-DD"
293	return datetime.date.isoformat(rdfobject)
294	raise trove_exceptions.UnsupportedRdfObject(rdfobject)

CenterForOpenScience / SHARE / 10217049583

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous