6590008757

Committed 20 Oct 2023 03:58PM UTC coverage: 83.758% (+0.4%) from 83.31%

Build # 6590008757

Build Type

push

github-actions

Committed by

web-flow

Commit Message

Merge pull request #420 from askomics/dev

Release 4.5.0

Run Details

633 of 633 new or added lines in 29 files covered. (100.0%)

6240 of 7450 relevant lines covered (83.76%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.27

/askomics/api/file.py

"""Api routes"""
import requests
import sys
import traceback
import urllib

from askomics.api.auth import login_required, api_auth
from askomics.libaskomics.FilesHandler import FilesHandler
from askomics.libaskomics.FilesUtils import FilesUtils
from askomics.libaskomics.Dataset import Dataset
from askomics.libaskomics.RdfFile import RdfFile

from flask import (Blueprint, current_app, jsonify, request, send_from_directory, session)

file_bp = Blueprint('file', __name__, url_prefix='/')


@file_bp.route('/api/files', methods=['GET', 'POST'])
@api_auth
@login_required
def get_files():
    """Get files info of the logged user

    Returns
    -------
    json
        files: list of all files of current user
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    files_id = None
    if request.method == 'POST':
        data = request.get_json()
        if data:
            files_id = data.get('filesId')

    try:
        files_handler = FilesHandler(current_app, session)
        files = files_handler.get_files_infos(files_id=files_id)
        disk_space = files_handler.get_size_occupied_by_user()
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'files': [],
            'diskSpace': 0,
            'error': True,
            'errorMessage': str(e)
        }), 500

    return jsonify({
        'files': files,
        'diskSpace': disk_space,
        'error': False,
        'errorMessage': ''
    })


@file_bp.route('/api/files/editname', methods=['POST'])
@api_auth
@login_required
def edit_file():
    """Edit file name

    Returns
    -------
    json
        files: list of all files of current user
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    data = request.get_json()
    current_app.logger.debug(data)
    if not (data and data.get("id") and data.get("newName")):
        return jsonify({
            'files': [],
            'diskSpace': 0,
            'error': True,
            'errorMessage': "Missing parameters"
        }), 400

    files_id = [data["id"]]
    new_name = data["newName"]

    try:
        files_handler = FilesHandler(current_app, session)
        files_handler.handle_files(files_id)

        for file in files_handler.files:
            file.edit_name_in_db(new_name)
        files = files_handler.get_files_infos()

    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'files': [],
            'diskSpace': 0,
            'error': True,
            'errorMessage': str(e)
        }), 500

    return jsonify({
        'files': files,
        'diskSpace': files_handler.get_size_occupied_by_user(),
        'error': False,
        'errorMessage': ''
    })


@file_bp.route('/api/files/upload_chunk', methods=['POST'])
@api_auth
@login_required
def upload_chunk():
    """Upload a file chunk

    Returns
    -------
    json
        path: name of the local file. To append the next chunk into it
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    files_utils = FilesUtils(current_app, session)
    disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None

    if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
        return jsonify({
            'errorMessage': "Exceeded quota",
            "path": '',
            "error": True
        }), 400

    data = request.get_json()
    if not (data and all([key in data for key in ["first", "last", "size", "name", "type", "size", "chunk"]])):
        return jsonify({
            "path": '',
            "error": True,
            "errorMessage": "Missing parameters"
        }), 400

    if not (data["first"] or data.get("path")):
        return jsonify({
            "path": '',
            "error": True,
            "errorMessage": "Missing path parameter"
        }), 400

    try:
        files = FilesHandler(current_app, session)
        path = files.persist_chunk(data)
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            "path": '',
            "error": True,
            "errorMessage": str(e)
        }), 500
    return jsonify({
        "path": path,
        "error": False,
        "errorMessage": ""
    })


@file_bp.route('/api/files/upload_url', methods=["POST"])
@api_auth
@login_required
def upload_url():
    """Upload a distant file with an URL

    Returns
    -------
    json
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    files_utils = FilesUtils(current_app, session)
    disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None

    if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
        return jsonify({
            'errorMessage': "Exceeded quota",
            "error": True
        }), 400

    data = request.get_json()
    if not (data and data.get("url")):
        return jsonify({
            "error": True,
            "errorMessage": "Missing url parameter"
        }), 400

    try:
        if session["user"]["quota"] > 0:
            with requests.get(data["url"], stream=True) as r:
                # Check header for total size, and check quota.
                if r.headers.get('Content-length'):
                    total_size = int(r.headers.get('Content-length')) + disk_space
                    if total_size >= session["user"]["quota"]:
                        return jsonify({
                            'errorMessage': "File will exceed quota",
                            "error": True
                        }), 400

        session_dict = {'user': session['user']}
        current_app.celery.send_task('download_file', (session_dict, data["url"]))
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            "error": True,
            "errorMessage": str(e)
        }), 500
    return jsonify({
        "error": False,
        "errorMessage": ""
    })


@file_bp.route('/api/files/preview', methods=['POST'])
@api_auth
@login_required
def get_preview():
    """Get files preview

    Returns
    -------
    json
        previewFiles: preview of selected files
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    data = request.get_json()
    if not (data and data.get('filesId')):
        return jsonify({
            'previewFiles': [],
            'error': True,
            'errorMessage': "Missing filesId parameter"
        }), 400

    try:

        files_handler = FilesHandler(current_app, session)
        files_handler.handle_files(data['filesId'])

        results = []
        for file in files_handler.files:
            file.set_preview()
            res = file.get_preview()
            results.append(res)
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'previewFiles': [],
            'error': True,
            'errorMessage': str(e)
        }), 500

    return jsonify({
        'previewFiles': results,
        'error': False,
        'errorMessage': ''
    })


@file_bp.route('/api/files/delete', methods=['POST'])
@api_auth
@login_required
def delete_files():
    """Delete files
    Returns
    -------
    json
        files: list of all files of current user
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """

    data = request.get_json()
    if not (data and data.get('filesIdToDelete')):
        return jsonify({
            'files': [],
            'error': True,
            'errorMessage': "Missing filesIdToDelete parameter"
        }), 400

    try:
        files = FilesHandler(current_app, session)
        remaining_files = files.delete_files(data.get('filesIdToDelete', []))
    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'files': [],
            'error': True,
            'errorMessage': str(e)
        }), 500

    return jsonify({
        'files': remaining_files,
        'error': False,
        'errorMessage': ''
    })


@file_bp.route('/api/files/integrate', methods=['POST'])
@api_auth
@login_required
def integrate():
    """Integrate a file

    Returns
    -------
    json
        datasets_id: dataset ids
        error: True if error, else False
        errorMessage: the error message of error, else an empty string
    """
    data = request.get_json()
    if not (data and data.get("fileId")):
        return jsonify({
            'error': True,
            'errorMessage': "Missing fileId parameter",
            'dataset_ids': None
        }), 400

    session_dict = {'user': session['user']}
    task = None
    dataset_ids = []

    try:

        files_handler = FilesHandler(current_app, session, host_url=request.host_url)
        files_handler.handle_files([data["fileId"], ])

        for file in files_handler.files:

            data["externalEndpoint"] = data["externalEndpoint"] if (data.get("externalEndpoint") and isinstance(file, RdfFile)) else None
            data["externalGraph"] = data["externalGraph"] if (data.get("externalGraph") and isinstance(file, RdfFile)) else None
            data["customUri"] = data["customUri"] if (data.get("customUri") and not isinstance(file, RdfFile)) else None

            dataset_info = {
                "celery_id": None,
                "file_id": file.id,
                "name": file.human_name,
                "graph_name": file.file_graph,
                "public": (data.get("public", False) if session["user"]["admin"] else False) or current_app.iniconfig.getboolean("askomics", "single_tenant", fallback=False)
            }

            endpoint = data["externalEndpoint"] or current_app.iniconfig.get('triplestore', 'endpoint')

            dataset = Dataset(current_app, session, dataset_info)
            dataset.save_in_db(endpoint, data["externalGraph"])
            data["dataset_id"] = dataset.id
            dataset_ids.append(dataset.id)
            task = current_app.celery.send_task('integrate', (session_dict, data, request.host_url))

            dataset.update_celery(task.id)

    except Exception as e:
        traceback.print_exc(file=sys.stdout)
        return jsonify({
            'error': True,
            'errorMessage': str(e),
            'dataset_ids': None
        }), 500

    return jsonify({
        'error': False,
        'errorMessage': '',
        'dataset_ids': dataset_ids
    })


@file_bp.route('/api/files/ttl/<path:user_id>/<path:username>/<path:path>', methods=['GET'])
@api_auth
def serve_file(path, user_id, username):
    """Serve a static ttl file of a user

    Parameters
    ----------
    path : string
        The file path to serve
    user_id : int
        user id
    username : string
        username

    Returns
    -------
    file
        the file
    """
    # Re-encode the path because we stored the encoded file name
    path = urllib.parse.quote(path)

    dir_path = "{}/{}_{}/ttl".format(
        current_app.iniconfig.get('askomics', 'data_directory'),
        user_id,
        username
    )

    return send_from_directory(dir_path, path)


@file_bp.route('/api/files/columns', methods=['GET'])
def get_column_types():
    """Give the list of available column types

    Returns
    -------
    json
        types: list of available column types
    """

    data = ["numeric", "text", "category", "boolean", "date", "reference", "strand", "start", "end", "general_relation", "symetric_relation", "indirect_relation", "label"]

    return jsonify({
        "types": data
    })

1	"""Api routes"""
2	import requests	1✔
3	import sys	1✔
4	import traceback	1✔
5	import urllib	1✔
6
7	from askomics.api.auth import login_required, api_auth	1✔
8	from askomics.libaskomics.FilesHandler import FilesHandler	1✔
9	from askomics.libaskomics.FilesUtils import FilesUtils	1✔
10	from askomics.libaskomics.Dataset import Dataset	1✔
11	from askomics.libaskomics.RdfFile import RdfFile	1✔
12
13	from flask import (Blueprint, current_app, jsonify, request, send_from_directory, session)	1✔
14
15	file_bp = Blueprint('file', __name__, url_prefix='/')	1✔
16
17
18	@file_bp.route('/api/files', methods=['GET', 'POST'])	1✔
19	@api_auth	1✔
20	@login_required	1✔
21	def get_files():	1✔
22	"""Get files info of the logged user
23
24	Returns
25	-------
26	json
27	files: list of all files of current user
28	error: True if error, else False
29	errorMessage: the error message of error, else an empty string
30	"""
31	files_id = None	1✔
32	if request.method == 'POST':	1✔
33	data = request.get_json()	1✔
34	if data:	1✔
35	files_id = data.get('filesId')	1✔
36
37	try:	1✔
38	files_handler = FilesHandler(current_app, session)	1✔
39	files = files_handler.get_files_infos(files_id=files_id)	1✔
40	disk_space = files_handler.get_size_occupied_by_user()	1✔
41	except Exception as e:	×
42	traceback.print_exc(file=sys.stdout)	×
43	return jsonify({	×
44	'files': [],
45	'diskSpace': 0,
46	'error': True,
47	'errorMessage': str(e)
48	}), 500
49
50	return jsonify({	1✔
51	'files': files,
52	'diskSpace': disk_space,
53	'error': False,
54	'errorMessage': ''
55	})
56
57
58	@file_bp.route('/api/files/editname', methods=['POST'])	1✔
59	@api_auth	1✔
60	@login_required	1✔
61	def edit_file():	1✔
62	"""Edit file name
63
64	Returns
65	-------
66	json
67	files: list of all files of current user
68	error: True if error, else False
69	errorMessage: the error message of error, else an empty string
70	"""
71	data = request.get_json()	1✔
72	current_app.logger.debug(data)	1✔
73	if not (data and data.get("id") and data.get("newName")):	1✔
74	return jsonify({	×
75	'files': [],
76	'diskSpace': 0,
77	'error': True,
78	'errorMessage': "Missing parameters"
79	}), 400
80
81	files_id = [data["id"]]	1✔
82	new_name = data["newName"]	1✔
83
84	try:	1✔
85	files_handler = FilesHandler(current_app, session)	1✔
86	files_handler.handle_files(files_id)	1✔
87
88	for file in files_handler.files:	1✔
89	file.edit_name_in_db(new_name)	1✔
90	files = files_handler.get_files_infos()	1✔
91
92	except Exception as e:	×
93	traceback.print_exc(file=sys.stdout)	×
94	return jsonify({	×
95	'files': [],
96	'diskSpace': 0,
97	'error': True,
98	'errorMessage': str(e)
99	}), 500
100
101	return jsonify({	1✔
102	'files': files,
103	'diskSpace': files_handler.get_size_occupied_by_user(),
104	'error': False,
105	'errorMessage': ''
106	})
107
108
109	@file_bp.route('/api/files/upload_chunk', methods=['POST'])	1✔
110	@api_auth	1✔
111	@login_required	1✔
112	def upload_chunk():	1✔
113	"""Upload a file chunk
114
115	Returns
116	-------
117	json
118	path: name of the local file. To append the next chunk into it
119	error: True if error, else False
120	errorMessage: the error message of error, else an empty string
121	"""
122	files_utils = FilesUtils(current_app, session)	1✔
123	disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None	1✔
124
125	if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:	1✔
126	return jsonify({	×
127	'errorMessage': "Exceeded quota",
128	"path": '',
129	"error": True
130	}), 400
131
132	data = request.get_json()	1✔
133	if not (data and all([key in data for key in ["first", "last", "size", "name", "type", "size", "chunk"]])):	1✔
134	return jsonify({	×
135	"path": '',
136	"error": True,
137	"errorMessage": "Missing parameters"
138	}), 400
139
140	if not (data["first"] or data.get("path")):	1✔
141	return jsonify({	×
142	"path": '',
143	"error": True,
144	"errorMessage": "Missing path parameter"
145	}), 400
146
147	try:	1✔
148	files = FilesHandler(current_app, session)	1✔
149	path = files.persist_chunk(data)	1✔
150	except Exception as e:	×
151	traceback.print_exc(file=sys.stdout)	×
152	return jsonify({	×
153	"path": '',
154	"error": True,
155	"errorMessage": str(e)
156	}), 500
157	return jsonify({	1✔
158	"path": path,
159	"error": False,
160	"errorMessage": ""
161	})
162
163
164	@file_bp.route('/api/files/upload_url', methods=["POST"])	1✔
165	@api_auth	1✔
166	@login_required	1✔
167	def upload_url():	1✔
168	"""Upload a distant file with an URL
169
170	Returns
171	-------
172	json
173	error: True if error, else False
174	errorMessage: the error message of error, else an empty string
175	"""
176	files_utils = FilesUtils(current_app, session)	1✔
177	disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None	1✔
178
179	if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:	1✔
180	return jsonify({	×
181	'errorMessage': "Exceeded quota",
182	"error": True
183	}), 400
184
185	data = request.get_json()	1✔
186	if not (data and data.get("url")):	1✔
187	return jsonify({	×
188	"error": True,
189	"errorMessage": "Missing url parameter"
190	}), 400
191
192	try:	1✔
193	if session["user"]["quota"] > 0:	1✔
194	with requests.get(data["url"], stream=True) as r:	×
195	# Check header for total size, and check quota.
196	if r.headers.get('Content-length'):	×
197	total_size = int(r.headers.get('Content-length')) + disk_space	×
198	if total_size >= session["user"]["quota"]:	×
199	return jsonify({	×
200	'errorMessage': "File will exceed quota",
201	"error": True
202	}), 400
203
204	session_dict = {'user': session['user']}	1✔
205	current_app.celery.send_task('download_file', (session_dict, data["url"]))	1✔
206	except Exception as e:	×
207	traceback.print_exc(file=sys.stdout)	×
208	return jsonify({	×
209	"error": True,
210	"errorMessage": str(e)
211	}), 500
212	return jsonify({	1✔
213	"error": False,
214	"errorMessage": ""
215	})
216
217
218	@file_bp.route('/api/files/preview', methods=['POST'])	1✔
219	@api_auth	1✔
220	@login_required	1✔
221	def get_preview():	1✔
222	"""Get files preview
223
224	Returns
225	-------
226	json
227	previewFiles: preview of selected files
228	error: True if error, else False
229	errorMessage: the error message of error, else an empty string
230	"""
231	data = request.get_json()	1✔
232	if not (data and data.get('filesId')):	1✔
233	return jsonify({	×
234	'previewFiles': [],
235	'error': True,
236	'errorMessage': "Missing filesId parameter"
237	}), 400
238
239	try:	1✔
240
241	files_handler = FilesHandler(current_app, session)	1✔
242	files_handler.handle_files(data['filesId'])	1✔
243
244	results = []	1✔
245	for file in files_handler.files:	1✔
246	file.set_preview()	1✔
247	res = file.get_preview()	1✔
248	results.append(res)	1✔
249	except Exception as e:	×
250	traceback.print_exc(file=sys.stdout)	×
251	return jsonify({	×
252	'previewFiles': [],
253	'error': True,
254	'errorMessage': str(e)
255	}), 500
256
257	return jsonify({	1✔
258	'previewFiles': results,
259	'error': False,
260	'errorMessage': ''
261	})
262
263
264	@file_bp.route('/api/files/delete', methods=['POST'])	1✔
265	@api_auth	1✔
266	@login_required	1✔
267	def delete_files():	1✔
268	"""Delete files
269	Returns
270	-------
271	json
272	files: list of all files of current user
273	error: True if error, else False
274	errorMessage: the error message of error, else an empty string
275	"""
276
277	data = request.get_json()	1✔
278	if not (data and data.get('filesIdToDelete')):	1✔
279	return jsonify({	×
280	'files': [],
281	'error': True,
282	'errorMessage': "Missing filesIdToDelete parameter"
283	}), 400
284
285	try:	1✔
286	files = FilesHandler(current_app, session)	1✔
287	remaining_files = files.delete_files(data.get('filesIdToDelete', []))	1✔
288	except Exception as e:	1✔
289	traceback.print_exc(file=sys.stdout)	1✔
290	return jsonify({	1✔
291	'files': [],
292	'error': True,
293	'errorMessage': str(e)
294	}), 500
295
296	return jsonify({	1✔
297	'files': remaining_files,
298	'error': False,
299	'errorMessage': ''
300	})
301
302
303	@file_bp.route('/api/files/integrate', methods=['POST'])	1✔
304	@api_auth	1✔
305	@login_required	1✔
306	def integrate():	1✔
307	"""Integrate a file
308
309	Returns
310	-------
311	json
312	datasets_id: dataset ids
313	error: True if error, else False
314	errorMessage: the error message of error, else an empty string
315	"""
316	data = request.get_json()	1✔
317	if not (data and data.get("fileId")):	1✔
318	return jsonify({	×
319	'error': True,
320	'errorMessage': "Missing fileId parameter",
321	'dataset_ids': None
322	}), 400
323
324	session_dict = {'user': session['user']}	1✔
325	task = None	1✔
326	dataset_ids = []	1✔
327
328	try:	1✔
329
330	files_handler = FilesHandler(current_app, session, host_url=request.host_url)	1✔
331	files_handler.handle_files([data["fileId"], ])	1✔
332
333	for file in files_handler.files:	1✔
334
335	data["externalEndpoint"] = data["externalEndpoint"] if (data.get("externalEndpoint") and isinstance(file, RdfFile)) else None	1✔
336	data["externalGraph"] = data["externalGraph"] if (data.get("externalGraph") and isinstance(file, RdfFile)) else None	1✔
337	data["customUri"] = data["customUri"] if (data.get("customUri") and not isinstance(file, RdfFile)) else None	1✔
338
339	dataset_info = {	1✔
340	"celery_id": None,
341	"file_id": file.id,
342	"name": file.human_name,
343	"graph_name": file.file_graph,
344	"public": (data.get("public", False) if session["user"]["admin"] else False) or current_app.iniconfig.getboolean("askomics", "single_tenant", fallback=False)
345	}
346
347	endpoint = data["externalEndpoint"] or current_app.iniconfig.get('triplestore', 'endpoint')	1✔
348
349	dataset = Dataset(current_app, session, dataset_info)	1✔
350	dataset.save_in_db(endpoint, data["externalGraph"])	1✔
351	data["dataset_id"] = dataset.id	1✔
352	dataset_ids.append(dataset.id)	1✔
353	task = current_app.celery.send_task('integrate', (session_dict, data, request.host_url))	1✔
354
355	dataset.update_celery(task.id)	1✔
356
357	except Exception as e:	×
358	traceback.print_exc(file=sys.stdout)	×
359	return jsonify({	×
360	'error': True,
361	'errorMessage': str(e),
362	'dataset_ids': None
363	}), 500
364
365	return jsonify({	1✔
366	'error': False,
367	'errorMessage': '',
368	'dataset_ids': dataset_ids
369	})
370
371
372	@file_bp.route('/api/files/ttl/<path:user_id>/<path:username>/<path:path>', methods=['GET'])	1✔
373	@api_auth	1✔
374	def serve_file(path, user_id, username):	1✔
375	"""Serve a static ttl file of a user
376
377	Parameters
378	----------
379	path : string
380	The file path to serve
381	user_id : int
382	user id
383	username : string
384	username
385
386	Returns
387	-------
388	file
389	the file
390	"""
391	# Re-encode the path because we stored the encoded file name
392	path = urllib.parse.quote(path)	1✔
393
394	dir_path = "{}/{}_{}/ttl".format(	1✔
395	current_app.iniconfig.get('askomics', 'data_directory'),
396	user_id,
397	username
398	)
399
400	return send_from_directory(dir_path, path)	1✔
401
402
403	@file_bp.route('/api/files/columns', methods=['GET'])	1✔
404	def get_column_types():	1✔
405	"""Give the list of available column types
406
407	Returns
408	-------
409	json
410	types: list of available column types
411	"""
412
413	data = ["numeric", "text", "category", "boolean", "date", "reference", "strand", "start", "end", "general_relation", "symetric_relation", "indirect_relation", "label"]	×
414
415	return jsonify({	×
416	"types": data
417	})

askomics / flaskomics / 6590008757

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous