6710529529

Committed 31 Oct 2023 05:51PM UTC coverage: 83.28% (-0.02%) from 83.302%

Build # 6710529529

Build Type

push

github-actions

Committed by

mboudet

Commit Message

Status

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

6276 of 7536 relevant lines covered (83.28%)

0.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.31

/askomics/api/file.py

"""Api routes"""
import requests
import sys
import traceback
import urllib

from askomics.api.auth import login_required, api_auth
from askomics.libaskomics.FilesHandler import FilesHandler
from askomics.libaskomics.FilesUtils import FilesUtils
from askomics.libaskomics.Dataset import Dataset
from askomics.libaskomics.RdfFile import RdfFile

from flask import (Blueprint, current_app, jsonify, request, send_from_directory, session)

file_bp = Blueprint('file', __name__, url_prefix='/')


@file_bp.route('/api/files', methods=['GET', 'POST'])
@api_auth
@login_required
def get_files():
    """Get files info of the logged user

    Returns
    -------
    json
        files: list of all files of current user
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    files_id = None
    if request.method == 'POST':
        data = request.get_json()
        if data:
            files_id = data.get('filesId')

    try:
        files_handler = FilesHandler(current_app, session)
        files = files_handler.get_files_infos(files_id=files_id)
        disk_space = files_handler.get_size_occupied_by_user()
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'files': [],
            'diskSpace': 0,
            'error': True,
            'errorMessage': str(e)
        }), 500

    return jsonify({
        'files': files,
        'diskSpace': disk_space,
        'error': False,
        'errorMessage': ''
    })


@file_bp.route('/api/files/editname', methods=['POST'])
@api_auth
@login_required
def edit_file():
    """Edit file name

    Returns
    -------
    json
        files: list of all files of current user
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    data = request.get_json()
    current_app.logger.debug(data)
    if not (data and data.get("id") and data.get("newName")):
        return jsonify({
            'files': [],
            'diskSpace': 0,
            'error': True,
            'errorMessage': "Missing parameters"
        }), 400

    files_id = [data["id"]]
    new_name = data["newName"]

    try:
        files_handler = FilesHandler(current_app, session)
        files_handler.handle_files(files_id)

        for file in files_handler.files:
            file.edit_name_in_db(new_name)
        files = files_handler.get_files_infos()

    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'files': [],
            'diskSpace': 0,
            'error': True,
            'errorMessage': str(e)
        }), 500

    return jsonify({
        'files': files,
        'diskSpace': files_handler.get_size_occupied_by_user(),
        'error': False,
        'errorMessage': ''
    })


@file_bp.route('/api/files/upload_chunk', methods=['POST'])
@api_auth
@login_required
def upload_chunk():
    """Upload a file chunk

    Returns
    -------
    json
        path: name of the local file. To append the next chunk into it
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    files_utils = FilesUtils(current_app, session)
    disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None

    if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
        return jsonify({
            'errorMessage': "Exceeded quota",
            "path": '',
            "error": True
        }), 400

    data = request.get_json()

    skip_preview = data.get('skip_preview', False)
    if not (data and all([key in data for key in ["first", "last", "size", "name", "type", "size", "chunk"]])):
        return jsonify({
            "path": '',
            "error": True,
            "errorMessage": "Missing parameters"
        }), 400

    if not (data["first"] or data.get("path")):
        return jsonify({
            "path": '',
            "error": True,
            "errorMessage": "Missing path parameter"
        }), 400

    try:
        files = FilesHandler(current_app, session)
        path = files.persist_chunk(data, skip_preview)
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            "path": '',
            "error": True,
            "errorMessage": str(e)
        }), 500
    return jsonify({
        "path": path,
        "error": False,
        "errorMessage": ""
    })


@file_bp.route('/api/files/upload_url', methods=["POST"])
@api_auth
@login_required
def upload_url():
    """Upload a distant file with an URL

    Returns
    -------
    json
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    files_utils = FilesUtils(current_app, session)
    disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None

    if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
        return jsonify({
            'errorMessage': "Exceeded quota",
            "error": True
        }), 400

    data = request.get_json()
    if not (data and data.get("url")):
        return jsonify({
            "error": True,
            "errorMessage": "Missing url parameter"
        }), 400

    try:
        if session["user"]["quota"] > 0:
            with requests.get(data["url"], stream=True) as r:
                # Check header for total size, and check quota.
                if r.headers.get('Content-length'):
                    total_size = int(r.headers.get('Content-length')) + disk_space
                    if total_size >= session["user"]["quota"]:
                        return jsonify({
                            'errorMessage': "File will exceed quota",
                            "error": True
                        }), 400

        session_dict = {'user': session['user']}
        current_app.celery.send_task('download_file', (session_dict, data["url"]))
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            "error": True,
            "errorMessage": str(e)
        }), 500
    return jsonify({
        "error": False,
        "errorMessage": ""
    })


@file_bp.route('/api/files/preview', methods=['POST'])
@api_auth
@login_required
def get_preview():
    """Get files preview

    Returns
    -------
    json
        previewFiles: preview of selected files
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    data = request.get_json()
    if not (data and data.get('filesId')):
        return jsonify({
            'previewFiles': [],
            'error': True,
            'errorMessage': "Missing filesId parameter"
        }), 400

    try:

        files_handler = FilesHandler(current_app, session)
        files_handler.handle_files(data['filesId'])

        results = []
        for file in files_handler.files:
            if file.status == "error":
                continue

            file.set_preview()
            res = file.get_preview()
            results.append(res)
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'previewFiles': [],
            'error': True,
            'errorMessage': str(e)
        }), 500

    errorMessage = ''
    error = False
    if not results:
        errorMessage = "None of the selected files are in an integrable state"
        error = True

    return jsonify({
        'previewFiles': results,
        'error': error,
        'errorMessage': errorMessage
    })


@file_bp.route('/api/files/delete', methods=['POST'])
@api_auth
@login_required
def delete_files():
    """Delete files
    Returns
    -------
    json
        files: list of all files of current user
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """

    data = request.get_json()
    if not (data and data.get('filesIdToDelete')):
        return jsonify({
            'files': [],
            'error': True,
            'errorMessage': "Missing filesIdToDelete parameter"
        }), 400

    try:
        files = FilesHandler(current_app, session)
        remaining_files = files.delete_files(data.get('filesIdToDelete', []))
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'files': [],
            'error': True,
            'errorMessage': str(e)
        }), 500

    return jsonify({
        'files': remaining_files,
        'error': False,
        'errorMessage': ''
    })


@file_bp.route('/api/files/integrate', methods=['POST'])
@api_auth
@login_required
def integrate():
    """Integrate a file

    Returns
    -------
    json
        datasets_id: dataset ids
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    data = request.get_json()
    if not (data and data.get("fileId")):
        return jsonify({
            'error': True,
            'errorMessage': "Missing fileId parameter",
            'dataset_ids': None
        }), 400

    session_dict = {'user': session['user']}
    task = None
    dataset_ids = []

    try:

        files_handler = FilesHandler(current_app, session, host_url=request.host_url)
        files_handler.handle_files([data["fileId"], ])

        for file in files_handler.files:

            if file.status == "error":
                continue

            data["externalEndpoint"] = data["externalEndpoint"] if (data.get("externalEndpoint") and isinstance(file, RdfFile)) else None
            data["externalGraph"] = data["externalGraph"] if (data.get("externalGraph") and isinstance(file, RdfFile)) else None
            data["customUri"] = data["customUri"] if (data.get("customUri") and not isinstance(file, RdfFile)) else None

            dataset_info = {
                "celery_id": None,
                "file_id": file.id,
                "name": file.human_name,
                "graph_name": file.file_graph,
                "public": (data.get("public", False) if session["user"]["admin"] else False) or current_app.iniconfig.getboolean("askomics", "single_tenant", fallback=False)
            }

            endpoint = data["externalEndpoint"] or current_app.iniconfig.get('triplestore', 'endpoint')

            dataset = Dataset(current_app, session, dataset_info)
            dataset.save_in_db(endpoint, data["externalGraph"])
            data["dataset_id"] = dataset.id
            dataset_ids.append(dataset.id)
            task = current_app.celery.send_task('integrate', (session_dict, data, request.host_url))

            dataset.update_celery(task.id)

    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'error': True,
            'errorMessage': str(e),
            'dataset_ids': None
        }), 500

    return jsonify({
        'error': False,
        'errorMessage': '',
        'dataset_ids': dataset_ids
    })


@file_bp.route('/api/files/ttl/<path:user_id>/<path:username>/<path:path>', methods=['GET'])
@api_auth
def serve_file(path, user_id, username):
    """Serve a static ttl file of a user

    Parameters
    ----------
    path : string
        The file path to serve
    user_id : int
        user id
    username : string
        username

    Returns
    -------
    file
        the file
    """
    # Re-encode the path because we stored the encoded file name
    path = urllib.parse.quote(path)

    dir_path = "{}/{}_{}/ttl".format(
        current_app.iniconfig.get('askomics', 'data_directory'),
        user_id,
        username
    )

    return send_from_directory(dir_path, path)


@file_bp.route('/api/files/columns', methods=['GET'])
def get_column_types():
    """Give the list of available column types

    Returns
    -------
    json
        types: list of available column types
    """

    data = ["numeric", "text", "category", "boolean", "date", "reference", "strand", "start", "end", "general_relation", "symetric_relation", "indirect_relation", "label"]

    return jsonify({
        "types": data
    })

1	"""Api routes"""
2	import requests	1✔
3	import sys	1✔
4	import traceback	1✔
5	import urllib	1✔
6
7	from askomics.api.auth import login_required, api_auth	1✔
8	from askomics.libaskomics.FilesHandler import FilesHandler	1✔
9	from askomics.libaskomics.FilesUtils import FilesUtils	1✔
10	from askomics.libaskomics.Dataset import Dataset	1✔
11	from askomics.libaskomics.RdfFile import RdfFile	1✔
12
13	from flask import (Blueprint, current_app, jsonify, request, send_from_directory, session)	1✔
14
15	file_bp = Blueprint('file', __name__, url_prefix='/')	1✔
16
17
18	@file_bp.route('/api/files', methods=['GET', 'POST'])	1✔
19	@api_auth	1✔
20	@login_required	1✔
21	def get_files():	1✔
22	"""Get files info of the logged user
23
24	Returns
25	-------
26	json
27	files: list of all files of current user
28	error: True if error, else False
29	errorMessage: the error message of error, else an empty string
30	"""
31	files_id = None	1✔
32	if request.method == 'POST':	1✔
33	data = request.get_json()	1✔
34	if data:	1✔
35	files_id = data.get('filesId')	1✔
36
37	try:	1✔
38	files_handler = FilesHandler(current_app, session)	1✔
39	files = files_handler.get_files_infos(files_id=files_id)	1✔
40	disk_space = files_handler.get_size_occupied_by_user()	1✔
41	except Exception as e:	×
42	traceback.print_exc(file=sys.stdout)	×
43	return jsonify({	×
44	'files': [],
45	'diskSpace': 0,
46	'error': True,
47	'errorMessage': str(e)
48	}), 500
49
50	return jsonify({	1✔
51	'files': files,
52	'diskSpace': disk_space,
53	'error': False,
54	'errorMessage': ''
55	})
56
57
58	@file_bp.route('/api/files/editname', methods=['POST'])	1✔
59	@api_auth	1✔
60	@login_required	1✔
61	def edit_file():	1✔
62	"""Edit file name
63
64	Returns
65	-------
66	json
67	files: list of all files of current user
68	error: True if error, else False
69	errorMessage: the error message of error, else an empty string
70	"""
71	data = request.get_json()	1✔
72	current_app.logger.debug(data)	1✔
73	if not (data and data.get("id") and data.get("newName")):	1✔
74	return jsonify({	×
75	'files': [],
76	'diskSpace': 0,
77	'error': True,
78	'errorMessage': "Missing parameters"
79	}), 400
80
81	files_id = [data["id"]]	1✔
82	new_name = data["newName"]	1✔
83
84	try:	1✔
85	files_handler = FilesHandler(current_app, session)	1✔
86	files_handler.handle_files(files_id)	1✔
87
88	for file in files_handler.files:	1✔
89	file.edit_name_in_db(new_name)	1✔
90	files = files_handler.get_files_infos()	1✔
91
92	except Exception as e:	×
93	traceback.print_exc(file=sys.stdout)	×
94	return jsonify({	×
95	'files': [],
96	'diskSpace': 0,
97	'error': True,
98	'errorMessage': str(e)
99	}), 500
100
101	return jsonify({	1✔
102	'files': files,
103	'diskSpace': files_handler.get_size_occupied_by_user(),
104	'error': False,
105	'errorMessage': ''
106	})
107
108
109	@file_bp.route('/api/files/upload_chunk', methods=['POST'])	1✔
110	@api_auth	1✔
111	@login_required	1✔
112	def upload_chunk():	1✔
113	"""Upload a file chunk
114
115	Returns
116	-------
117	json
118	path: name of the local file. To append the next chunk into it
119	error: True if error, else False
120	errorMessage: the error message of error, else an empty string
121	"""
122	files_utils = FilesUtils(current_app, session)	1✔
123	disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None	1✔
124
125	if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:	1✔
126	return jsonify({	×
127	'errorMessage': "Exceeded quota",
128	"path": '',
129	"error": True
130	}), 400
131
132	data = request.get_json()	1✔
133
134	skip_preview = data.get('skip_preview', False)	1✔
135	if not (data and all([key in data for key in ["first", "last", "size", "name", "type", "size", "chunk"]])):	1✔
136	return jsonify({	×
137	"path": '',
138	"error": True,
139	"errorMessage": "Missing parameters"
140	}), 400
141
142	if not (data["first"] or data.get("path")):	1✔
143	return jsonify({	×
144	"path": '',
145	"error": True,
146	"errorMessage": "Missing path parameter"
147	}), 400
148
149	try:	1✔
150	files = FilesHandler(current_app, session)	1✔
151	path = files.persist_chunk(data, skip_preview)	1✔
152	except Exception as e:	×
153	traceback.print_exc(file=sys.stdout)	×
154	return jsonify({	×
155	"path": '',
156	"error": True,
157	"errorMessage": str(e)
158	}), 500
159	return jsonify({	1✔
160	"path": path,
161	"error": False,
162	"errorMessage": ""
163	})
164
165
166	@file_bp.route('/api/files/upload_url', methods=["POST"])	1✔
167	@api_auth	1✔
168	@login_required	1✔
169	def upload_url():	1✔
170	"""Upload a distant file with an URL
171
172	Returns
173	-------
174	json
175	error: True if error, else False
176	errorMessage: the error message of error, else an empty string
177	"""
178	files_utils = FilesUtils(current_app, session)	1✔
179	disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None	1✔
180
181	if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:	1✔
182	return jsonify({	×
183	'errorMessage': "Exceeded quota",
184	"error": True
185	}), 400
186
187	data = request.get_json()	1✔
188	if not (data and data.get("url")):	1✔
189	return jsonify({	×
190	"error": True,
191	"errorMessage": "Missing url parameter"
192	}), 400
193
194	try:	1✔
195	if session["user"]["quota"] > 0:	1✔
196	with requests.get(data["url"], stream=True) as r:	×
197	# Check header for total size, and check quota.
198	if r.headers.get('Content-length'):	×
199	total_size = int(r.headers.get('Content-length')) + disk_space	×
200	if total_size >= session["user"]["quota"]:	×
201	return jsonify({	×
202	'errorMessage': "File will exceed quota",
203	"error": True
204	}), 400
205
206	session_dict = {'user': session['user']}	1✔
207	current_app.celery.send_task('download_file', (session_dict, data["url"]))	1✔
208	except Exception as e:	×
209	traceback.print_exc(file=sys.stdout)	×
210	return jsonify({	×
211	"error": True,
212	"errorMessage": str(e)
213	}), 500
214	return jsonify({	1✔
215	"error": False,
216	"errorMessage": ""
217	})
218
219
220	@file_bp.route('/api/files/preview', methods=['POST'])	1✔
221	@api_auth	1✔
222	@login_required	1✔
223	def get_preview():	1✔
224	"""Get files preview
225
226	Returns
227	-------
228	json
229	previewFiles: preview of selected files
230	error: True if error, else False
231	errorMessage: the error message of error, else an empty string
232	"""
233	data = request.get_json()	1✔
234	if not (data and data.get('filesId')):	1✔
235	return jsonify({	×
236	'previewFiles': [],
237	'error': True,
238	'errorMessage': "Missing filesId parameter"
239	}), 400
240
241	try:	1✔
242
243	files_handler = FilesHandler(current_app, session)	1✔
244	files_handler.handle_files(data['filesId'])	1✔
245
246	results = []	1✔
247	for file in files_handler.files:	1✔
248	if file.status == "error":	1✔
249	continue	×
250
251	file.set_preview()	1✔
252	res = file.get_preview()	1✔
253	results.append(res)	1✔
254	except Exception as e:	×
255	traceback.print_exc(file=sys.stdout)	×
256	return jsonify({	×
257	'previewFiles': [],
258	'error': True,
259	'errorMessage': str(e)
260	}), 500
261
262	errorMessage = ''	1✔
263	error = False	1✔
264	if not results:	1✔
265	errorMessage = "None of the selected files are in an integrable state"	1✔
266	error = True	1✔
267
268	return jsonify({	1✔
269	'previewFiles': results,
270	'error': error,
271	'errorMessage': errorMessage
272	})
273
274
275	@file_bp.route('/api/files/delete', methods=['POST'])	1✔
276	@api_auth	1✔
277	@login_required	1✔
278	def delete_files():	1✔
279	"""Delete files
280	Returns
281	-------
282	json
283	files: list of all files of current user
284	error: True if error, else False
285	errorMessage: the error message of error, else an empty string
286	"""
287
288	data = request.get_json()	1✔
289	if not (data and data.get('filesIdToDelete')):	1✔
290	return jsonify({	×
291	'files': [],
292	'error': True,
293	'errorMessage': "Missing filesIdToDelete parameter"
294	}), 400
295
296	try:	1✔
297	files = FilesHandler(current_app, session)	1✔
298	remaining_files = files.delete_files(data.get('filesIdToDelete', []))	1✔
299	except Exception as e:	1✔
300	traceback.print_exc(file=sys.stdout)	1✔
301	return jsonify({	1✔
302	'files': [],
303	'error': True,
304	'errorMessage': str(e)
305	}), 500
306
307	return jsonify({	1✔
308	'files': remaining_files,
309	'error': False,
310	'errorMessage': ''
311	})
312
313
314	@file_bp.route('/api/files/integrate', methods=['POST'])	1✔
315	@api_auth	1✔
316	@login_required	1✔
317	def integrate():	1✔
318	"""Integrate a file
319
320	Returns
321	-------
322	json
323	datasets_id: dataset ids
324	error: True if error, else False
325	errorMessage: the error message of error, else an empty string
326	"""
327	data = request.get_json()	1✔
328	if not (data and data.get("fileId")):	1✔
329	return jsonify({	×
330	'error': True,
331	'errorMessage': "Missing fileId parameter",
332	'dataset_ids': None
333	}), 400
334
335	session_dict = {'user': session['user']}	1✔
336	task = None	1✔
337	dataset_ids = []	1✔
338
339	try:	1✔
340
341	files_handler = FilesHandler(current_app, session, host_url=request.host_url)	1✔
342	files_handler.handle_files([data["fileId"], ])	1✔
343
344	for file in files_handler.files:	1✔
345
346	if file.status == "error":	1✔
347	continue	×
348
349	data["externalEndpoint"] = data["externalEndpoint"] if (data.get("externalEndpoint") and isinstance(file, RdfFile)) else None	1✔
350	data["externalGraph"] = data["externalGraph"] if (data.get("externalGraph") and isinstance(file, RdfFile)) else None	1✔
351	data["customUri"] = data["customUri"] if (data.get("customUri") and not isinstance(file, RdfFile)) else None	1✔
352
353	dataset_info = {	1✔
354	"celery_id": None,
355	"file_id": file.id,
356	"name": file.human_name,
357	"graph_name": file.file_graph,
358	"public": (data.get("public", False) if session["user"]["admin"] else False) or current_app.iniconfig.getboolean("askomics", "single_tenant", fallback=False)
359	}
360
361	endpoint = data["externalEndpoint"] or current_app.iniconfig.get('triplestore', 'endpoint')	1✔
362
363	dataset = Dataset(current_app, session, dataset_info)	1✔
364	dataset.save_in_db(endpoint, data["externalGraph"])	1✔
365	data["dataset_id"] = dataset.id	1✔
366	dataset_ids.append(dataset.id)	1✔
367	task = current_app.celery.send_task('integrate', (session_dict, data, request.host_url))	1✔
368
369	dataset.update_celery(task.id)	1✔
370
371	except Exception as e:	×
372	traceback.print_exc(file=sys.stdout)	×
373	return jsonify({	×
374	'error': True,
375	'errorMessage': str(e),
376	'dataset_ids': None
377	}), 500
378
379	return jsonify({	1✔
380	'error': False,
381	'errorMessage': '',
382	'dataset_ids': dataset_ids
383	})
384
385
386	@file_bp.route('/api/files/ttl/<path:user_id>/<path:username>/<path:path>', methods=['GET'])	1✔
387	@api_auth	1✔
388	def serve_file(path, user_id, username):	1✔
389	"""Serve a static ttl file of a user
390
391	Parameters
392	----------
393	path : string
394	The file path to serve
395	user_id : int
396	user id
397	username : string
398	username
399
400	Returns
401	-------
402	file
403	the file
404	"""
405	# Re-encode the path because we stored the encoded file name
406	path = urllib.parse.quote(path)	1✔
407
408	dir_path = "{}/{}_{}/ttl".format(	1✔
409	current_app.iniconfig.get('askomics', 'data_directory'),
410	user_id,
411	username
412	)
413
414	return send_from_directory(dir_path, path)	1✔
415
416
417	@file_bp.route('/api/files/columns', methods=['GET'])	1✔
418	def get_column_types():	1✔
419	"""Give the list of available column types
420
421	Returns
422	-------
423	json
424	types: list of available column types
425	"""
426
427	data = ["numeric", "text", "category", "boolean", "date", "reference", "strand", "start", "end", "general_relation", "symetric_relation", "indirect_relation", "label"]	×
428
429	return jsonify({	×
430	"types": data
431	})

askomics / flaskomics / 6710529529

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous