6590008757

Committed 20 Oct 2023 03:58PM UTC coverage: 83.758% (+0.4%) from 83.31%

Build # 6590008757

Build Type

push

github-actions

Committed by

web-flow

Commit Message

Merge pull request #420 from askomics/dev

Release 4.5.0

Run Details

633 of 633 new or added lines in 29 files covered. (100.0%)

6240 of 7450 relevant lines covered (83.76%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/askomics/libaskomics/RdfFile.py

from askomics.libaskomics.File import File
from askomics.libaskomics.RdfGraph import RdfGraph
from askomics.libaskomics.SparqlQueryLauncher import SparqlQueryLauncher
from askomics.libaskomics.TriplestoreExplorer import TriplestoreExplorer
from askomics.libaskomics.Utils import Utils


class RdfFile(File):
    """RDF (turtle) File

    Attributes
    ----------
    public : bool
        Public or private dataset
    """

    def __init__(self, app, session, file_info, host_url=None, external_endpoint=None, custom_uri=None, external_graph=None):
        """init

        Parameters
        ----------
        app : Flask
            Flask app
        session :
            AskOmics session
        file_info : dict
            file info
        host_url : None, optional
            AskOmics url
        """
        File.__init__(self, app, session, file_info, host_url, external_endpoint=external_endpoint, custom_uri=custom_uri, external_graph=external_graph)

        self.type_dict = {
            "rdf/ttl": "turtle",
            "rdf/xml": "xml",
            "rdf/nt": "nt"
        }

    def set_preview(self):
        """Summary"""
        pass

    def get_location_and_remote_graph(self):
        """Get location of data if specified

        Returns
        -------
        str
            Location
        """
        graph = RdfGraph(self.app, self.session)
        graph.parse(self.path, format=self.type_dict[self.type])
        triple_loc = (None, self.prov.atLocation, None)
        triple_graph = (None, self.dcat.Dataset, None)
        loc = None
        remote_graph = None
        for s, p, o in graph.graph.triples(triple_loc):
            loc = str(o)
            break

        for s, p, o in graph.graph.triples(triple_graph):
            remote_graph = str(o)
            break

        return loc, remote_graph

    def get_preview(self):
        """Get a preview of the frist 100 lines of a ttl file

        Returns
        -------
        TYPE
            Description
        """
        with open(self.path) as ttl_file:
            # Read 100 lines
            head = ''
            for x in range(1, 100):
                head += ttl_file.readline()

        location = None
        remote_graph = None
        try:
            location, remote_graph = self.get_location_and_remote_graph()
        except Exception as e:
            self.error_message = str(e)
            # Todo: Better error management
            raise e

        return {
            'type': self.type,
            'id': self.id,
            'name': self.human_name,
            'error': self.error,
            'error_message': self.error_message,
            'data': {
                'preview': head,
                'location': location,
                'remote_graph': remote_graph
            }
        }

    def delete_metadata_location(self):
        """Delete metadata from data"""
        self.graph_chunk.remove((None, self.prov.atLocation, None))
        self.graph_chunk.remove((None, self.dcat.Dataset, None))

    def integrate(self, public=False):
        """Integrate the file into the triplestore

        Parameters
        ----------
        public : bool, optional
            Integrate in private or public graph
        """
        sparql = SparqlQueryLauncher(self.app, self.session)
        tse = TriplestoreExplorer(self.app, self.session)

        self.public = public

        method = self.settings.get('triplestore', 'upload_method')

        # Load file into a RDF graph
        self.graph_chunk.parse(self.path, format=self.type_dict[self.type])

        # get metadata
        self.set_metadata()

        # Remove metadata from data
        self.delete_metadata_location()

        # insert metadata
        sparql.insert_data(self.graph_metadata, self.file_graph, metadata=True)

        if method == "load":
            # write rdf into a tmpfile and load it
            temp_file_name = 'tmp_{}_{}.{}'.format(
                Utils.get_random_string(5),
                self.name,
                self.rdf_extention
            )

            # Try to load data. if failure, wait 5 sec and retry 5 time
            Utils.redo_if_failure(self.log, 5, 5, self.load_graph, self.graph_chunk, temp_file_name)

        else:
            # Insert
            # Try to insert data. if failure, wait 5 sec and retry 5 time
            Utils.redo_if_failure(self.log, 5, 5, sparql.insert_data, self.graph_chunk, self.file_graph)

        # Remove chached abstraction
        tse.uncache_abstraction(public=self.public)

        self.set_triples_number()

1	from askomics.libaskomics.File import File	1✔
2	from askomics.libaskomics.RdfGraph import RdfGraph	1✔
3	from askomics.libaskomics.SparqlQueryLauncher import SparqlQueryLauncher	1✔
4	from askomics.libaskomics.TriplestoreExplorer import TriplestoreExplorer	1✔
5	from askomics.libaskomics.Utils import Utils	1✔
6
7
8	class RdfFile(File):	1✔
9	"""RDF (turtle) File
10
11	Attributes
12	----------
13	public : bool
14	Public or private dataset
15	"""
16
17	def __init__(self, app, session, file_info, host_url=None, external_endpoint=None, custom_uri=None, external_graph=None):	1✔
18	"""init
19
20	Parameters
21	----------
22	app : Flask
23	Flask app
24	session :
25	AskOmics session
26	file_info : dict
27	file info
28	host_url : None, optional
29	AskOmics url
30	"""
31	File.__init__(self, app, session, file_info, host_url, external_endpoint=external_endpoint, custom_uri=custom_uri, external_graph=external_graph)	1✔
32
33	self.type_dict = {	1✔
34	"rdf/ttl": "turtle",
35	"rdf/xml": "xml",
36	"rdf/nt": "nt"
37	}
38
39	def set_preview(self):	1✔
40	"""Summary"""
41	pass	×
42
43	def get_location_and_remote_graph(self):	1✔
44	"""Get location of data if specified
45
46	Returns
47	-------
48	str
49	Location
50	"""
51	graph = RdfGraph(self.app, self.session)	×
52	graph.parse(self.path, format=self.type_dict[self.type])	×
53	triple_loc = (None, self.prov.atLocation, None)	×
54	triple_graph = (None, self.dcat.Dataset, None)	×
55	loc = None	×
56	remote_graph = None	×
57	for s, p, o in graph.graph.triples(triple_loc):	×
58	loc = str(o)	×
59	break	×
60
61	for s, p, o in graph.graph.triples(triple_graph):	×
62	remote_graph = str(o)	×
63	break	×
64
65	return loc, remote_graph	×
66
67	def get_preview(self):	1✔
68	"""Get a preview of the frist 100 lines of a ttl file
69
70	Returns
71	-------
72	TYPE
73	Description
74	"""
75	with open(self.path) as ttl_file:	×
76	# Read 100 lines
77	head = ''	×
78	for x in range(1, 100):	×
79	head += ttl_file.readline()	×
80
81	location = None	×
82	remote_graph = None	×
83	try:	×
84	location, remote_graph = self.get_location_and_remote_graph()	×
85	except Exception as e:	×
86	self.error_message = str(e)	×
87	# Todo: Better error management
88	raise e	×
89
90	return {	×
91	'type': self.type,
92	'id': self.id,
93	'name': self.human_name,
94	'error': self.error,
95	'error_message': self.error_message,
96	'data': {
97	'preview': head,
98	'location': location,
99	'remote_graph': remote_graph
100	}
101	}
102
103	def delete_metadata_location(self):	1✔
104	"""Delete metadata from data"""
105	self.graph_chunk.remove((None, self.prov.atLocation, None))	1✔
106	self.graph_chunk.remove((None, self.dcat.Dataset, None))	1✔
107
108	def integrate(self, public=False):	1✔
109	"""Integrate the file into the triplestore
110
111	Parameters
112	----------
113	public : bool, optional
114	Integrate in private or public graph
115	"""
116	sparql = SparqlQueryLauncher(self.app, self.session)	1✔
117	tse = TriplestoreExplorer(self.app, self.session)	1✔
118
119	self.public = public	1✔
120
121	method = self.settings.get('triplestore', 'upload_method')	1✔
122
123	# Load file into a RDF graph
124	self.graph_chunk.parse(self.path, format=self.type_dict[self.type])	1✔
125
126	# get metadata
127	self.set_metadata()	1✔
128
129	# Remove metadata from data
130	self.delete_metadata_location()	1✔
131
132	# insert metadata
133	sparql.insert_data(self.graph_metadata, self.file_graph, metadata=True)	1✔
134
135	if method == "load":	1✔
136	# write rdf into a tmpfile and load it
137	temp_file_name = 'tmp_{}_{}.{}'.format(	×
138	Utils.get_random_string(5),
139	self.name,
140	self.rdf_extention
141	)
142
143	# Try to load data. if failure, wait 5 sec and retry 5 time
144	Utils.redo_if_failure(self.log, 5, 5, self.load_graph, self.graph_chunk, temp_file_name)	×
145
146	else:
147	# Insert
148	# Try to insert data. if failure, wait 5 sec and retry 5 time
149	Utils.redo_if_failure(self.log, 5, 5, sparql.insert_data, self.graph_chunk, self.file_graph)	1✔
150
151	# Remove chached abstraction
152	tse.uncache_abstraction(public=self.public)	1✔
153
154	self.set_triples_number()	1✔

askomics / flaskomics / 6590008757

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous