8143972779

Committed 04 Mar 2024 04:54PM UTC coverage: 83.219%. Remained the same

Build # 8143972779

Build Type

Pull #449

github

Committed by

web-flow

Commit Message

Merge 6d2bc1d30 into 91939bc97

Pull Request Pull Request #449: Bump xml2js and parse-bmfont-xml

Run Details

6283 of 7550 relevant lines covered (83.22%)

0.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

41.43

/askomics/libaskomics/RdfFile.py

from askomics.libaskomics.File import File
from askomics.libaskomics.RdfGraph import RdfGraph
from askomics.libaskomics.SparqlQueryLauncher import SparqlQueryLauncher
from askomics.libaskomics.TriplestoreExplorer import TriplestoreExplorer
from askomics.libaskomics.Utils import Utils


class RdfFile(File):
    """RDF (turtle) File

    Attributes
    ----------
    public : bool
        Public or private dataset
    """

    def __init__(self, app, session, file_info, host_url=None, external_endpoint=None, custom_uri=None, external_graph=None):
        """init

        Parameters
        ----------
        app : Flask
            Flask app
        session :
            AskOmics session
        file_info : dict
            file info
        host_url : None, optional
            AskOmics url
        """
        File.__init__(self, app, session, file_info, host_url, external_endpoint=external_endpoint, custom_uri=custom_uri, external_graph=external_graph)

        self.type_dict = {
            "rdf/ttl": "turtle",
            "rdf/xml": "xml",
            "rdf/nt": "nt"
        }

    def set_preview(self):
        """Summary"""
        pass

    def save_preview(self):
        """Save location and endpoint in preview"""
        data = None
        error = None
        try:
            location, remote_graph = self.get_location_and_remote_graph()
            data = {"location": location, "remote_graph": remote_graph}
        except Exception as e:
            error = str(e)
        self.save_preview_in_db(data, error)

    def get_location_and_remote_graph(self):
        """Get location of data if specified

        Returns
        -------
        str
            Location
        """

        if self.settings.get('askomics', 'skip_rdf_preview', fallback=False):
            return "", ""

        graph = RdfGraph(self.app, self.session)
        graph.parse(self.path, format=self.type_dict[self.type])
        triple_loc = (None, self.prov.atLocation, None)
        triple_graph = (None, self.dcat.Dataset, None)
        loc = None
        remote_graph = None
        for s, p, o in graph.graph.triples(triple_loc):
            loc = str(o)
            break

        for s, p, o in graph.graph.triples(triple_graph):
            remote_graph = str(o)
            break

        return loc, remote_graph

    def get_preview(self):
        """Get a preview of the frist 100 lines of a ttl file

        Returns
        -------
        TYPE
            Description
        """
        with open(self.path) as ttl_file:
            # Read 100 lines
            head = ''
            for x in range(1, 100):
                head += ttl_file.readline()

        location = None
        remote_graph = None
        try:
            if self.preview:
                location = self.preview['location']
                remote_graph = self.preview['remote_graph']
            else:
                location, remote_graph = self.get_location_and_remote_graph()
        except Exception as e:
            self.error_message = str(e)
            # Todo: Better error management
            raise e

        return {
            'type': self.type,
            'id': self.id,
            'name': self.human_name,
            'error': self.error,
            'error_message': self.error_message,
            'data': {
                'preview': head,
                'location': location,
                'remote_graph': remote_graph
            }
        }

    def delete_metadata_location(self):
        """Delete metadata from data"""
        self.graph_chunk.remove((None, self.prov.atLocation, None))
        self.graph_chunk.remove((None, self.dcat.Dataset, None))

    def integrate(self, public=False):
        """Integrate the file into the triplestore

        Parameters
        ----------
        public : bool, optional
            Integrate in private or public graph
        """
        sparql = SparqlQueryLauncher(self.app, self.session)
        tse = TriplestoreExplorer(self.app, self.session)

        self.public = public

        method = self.settings.get('triplestore', 'upload_method')

        # Load file into a RDF graph
        self.graph_chunk.parse(self.path, format=self.type_dict[self.type])

        # get metadata
        self.set_metadata()

        # Remove metadata from data
        self.delete_metadata_location()

        # insert metadata
        sparql.insert_data(self.graph_metadata, self.file_graph, metadata=True)

        if method == "load":
            # write rdf into a tmpfile and load it
            temp_file_name = 'tmp_{}_{}.{}'.format(
                Utils.get_random_string(5),
                self.name,
                self.rdf_extention
            )

            # Try to load data. if failure, wait 5 sec and retry 5 time
            Utils.redo_if_failure(self.log, 5, 5, self.load_graph, self.graph_chunk, temp_file_name)

        else:
            # Insert
            # Try to insert data. if failure, wait 5 sec and retry 5 time
            Utils.redo_if_failure(self.log, 5, 5, sparql.insert_data, self.graph_chunk, self.file_graph)

        # Remove chached abstraction
        tse.uncache_abstraction(public=self.public)

        self.set_triples_number()

1	from askomics.libaskomics.File import File	1✔
2	from askomics.libaskomics.RdfGraph import RdfGraph	1✔
3	from askomics.libaskomics.SparqlQueryLauncher import SparqlQueryLauncher	1✔
4	from askomics.libaskomics.TriplestoreExplorer import TriplestoreExplorer	1✔
5	from askomics.libaskomics.Utils import Utils	1✔
6
7
8	class RdfFile(File):	1✔
9	"""RDF (turtle) File
10
11	Attributes
12	----------
13	public : bool
14	Public or private dataset
15	"""
16
17	def __init__(self, app, session, file_info, host_url=None, external_endpoint=None, custom_uri=None, external_graph=None):	1✔
18	"""init
19
20	Parameters
21	----------
22	app : Flask
23	Flask app
24	session :
25	AskOmics session
26	file_info : dict
27	file info
28	host_url : None, optional
29	AskOmics url
30	"""
31	File.__init__(self, app, session, file_info, host_url, external_endpoint=external_endpoint, custom_uri=custom_uri, external_graph=external_graph)	1✔
32
33	self.type_dict = {	1✔
34	"rdf/ttl": "turtle",
35	"rdf/xml": "xml",
36	"rdf/nt": "nt"
37	}
38
39	def set_preview(self):	1✔
40	"""Summary"""
41	pass	×
42
43	def save_preview(self):	1✔
44	"""Save location and endpoint in preview"""
45	data = None	×
46	error = None	×
47	try:	×
48	location, remote_graph = self.get_location_and_remote_graph()	×
49	data = {"location": location, "remote_graph": remote_graph}	×
50	except Exception as e:	×
51	error = str(e)	×
52	self.save_preview_in_db(data, error)	×
53
54	def get_location_and_remote_graph(self):	1✔
55	"""Get location of data if specified
56
57	Returns
58	-------
59	str
60	Location
61	"""
62
63	if self.settings.get('askomics', 'skip_rdf_preview', fallback=False):	×
64	return "", ""	×
65
66	graph = RdfGraph(self.app, self.session)	×
67	graph.parse(self.path, format=self.type_dict[self.type])	×
68	triple_loc = (None, self.prov.atLocation, None)	×
69	triple_graph = (None, self.dcat.Dataset, None)	×
70	loc = None	×
71	remote_graph = None	×
72	for s, p, o in graph.graph.triples(triple_loc):	×
73	loc = str(o)	×
74	break	×
75
76	for s, p, o in graph.graph.triples(triple_graph):	×
77	remote_graph = str(o)	×
78	break	×
79
80	return loc, remote_graph	×
81
82	def get_preview(self):	1✔
83	"""Get a preview of the frist 100 lines of a ttl file
84
85	Returns
86	-------
87	TYPE
88	Description
89	"""
90	with open(self.path) as ttl_file:	×
91	# Read 100 lines
92	head = ''	×
93	for x in range(1, 100):	×
94	head += ttl_file.readline()	×
95
96	location = None	×
97	remote_graph = None	×
98	try:	×
99	if self.preview:	×
100	location = self.preview['location']	×
101	remote_graph = self.preview['remote_graph']	×
102	else:
103	location, remote_graph = self.get_location_and_remote_graph()	×
104	except Exception as e:	×
105	self.error_message = str(e)	×
106	# Todo: Better error management
107	raise e	×
108
109	return {	×
110	'type': self.type,
111	'id': self.id,
112	'name': self.human_name,
113	'error': self.error,
114	'error_message': self.error_message,
115	'data': {
116	'preview': head,
117	'location': location,
118	'remote_graph': remote_graph
119	}
120	}
121
122	def delete_metadata_location(self):	1✔
123	"""Delete metadata from data"""
124	self.graph_chunk.remove((None, self.prov.atLocation, None))	1✔
125	self.graph_chunk.remove((None, self.dcat.Dataset, None))	1✔
126
127	def integrate(self, public=False):	1✔
128	"""Integrate the file into the triplestore
129
130	Parameters
131	----------
132	public : bool, optional
133	Integrate in private or public graph
134	"""
135	sparql = SparqlQueryLauncher(self.app, self.session)	1✔
136	tse = TriplestoreExplorer(self.app, self.session)	1✔
137
138	self.public = public	1✔
139
140	method = self.settings.get('triplestore', 'upload_method')	1✔
141
142	# Load file into a RDF graph
143	self.graph_chunk.parse(self.path, format=self.type_dict[self.type])	1✔
144
145	# get metadata
146	self.set_metadata()	1✔
147
148	# Remove metadata from data
149	self.delete_metadata_location()	1✔
150
151	# insert metadata
152	sparql.insert_data(self.graph_metadata, self.file_graph, metadata=True)	1✔
153
154	if method == "load":	1✔
155	# write rdf into a tmpfile and load it
156	temp_file_name = 'tmp_{}_{}.{}'.format(	×
157	Utils.get_random_string(5),
158	self.name,
159	self.rdf_extention
160	)
161
162	# Try to load data. if failure, wait 5 sec and retry 5 time
163	Utils.redo_if_failure(self.log, 5, 5, self.load_graph, self.graph_chunk, temp_file_name)	×
164
165	else:
166	# Insert
167	# Try to insert data. if failure, wait 5 sec and retry 5 time
168	Utils.redo_if_failure(self.log, 5, 5, sparql.insert_data, self.graph_chunk, self.file_graph)	1✔
169
170	# Remove chached abstraction
171	tse.uncache_abstraction(public=self.public)	1✔
172
173	self.set_triples_number()	1✔

askomics / flaskomics / 8143972779

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous