19114618549

Committed 05 Nov 2025 07:59PM UTC coverage: 94.093% (+0.03%) from 94.065%

Build # 19114618549

Build Type

Pull #1069

github

Committed by

peterkle

Commit Message

add more tests

Pull Request Pull Request #1069: RDISCROWD-8392 upgrade to boto3

Run Details

214 of 223 new or added lines in 7 files covered. (95.96%)

152 existing lines in 8 files now uncovered.

17920 of 19045 relevant lines covered (94.09%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.04

/pybossa/view/fileproxy.py

# -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2018 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.

from urllib.parse import urlparse, parse_qs
from functools import wraps
from flask import Blueprint, current_app, Response, request
from flask_login import current_user, login_required

import six
import requests
import json
from werkzeug.exceptions import Forbidden, BadRequest, InternalServerError, NotFound

from pybossa.cache.projects import get_project_data
from pybossa.contributions_guard import ContributionsGuard
from pybossa.core import task_repo, signer
from pybossa.encryption import AESWithGCM
# from pybossa.pybhdfs.client import HDFSKerberos
from pybossa.sched import has_lock
from pybossa.task_creator_helper import get_encryption_key, read_encrypted_file


blueprint = Blueprint('fileproxy', __name__)

TASK_SIGNATURE_MAX_SIZE = 128


def no_cache(view_func):
    @wraps(view_func)
    def decorated(*args, **kwargs):
        response = view_func(*args, **kwargs)
        response.headers.add('Cache-Control', 'no-store')
        response.headers.add('Pragma', 'no-cache')
        return response
    return decorated


def check_allowed(user_id, task_id, project, is_valid_url):
    task = task_repo.get_task(task_id)

    if not task or task.project_id != project['id']:
        raise BadRequest('Task does not exist')

    if not any(is_valid_url(v) for v in task.info.values()):
        raise Forbidden('Invalid task content')

    if current_user.admin:
        return True

    if has_lock(task_id, user_id,
                project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
        return True

    if user_id in project['owners_ids']:
        return True

    raise Forbidden('FORBIDDEN')


def read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature):
    if not signature:
        current_app.logger.exception(
            'Project id {} no signature {}'.format(project_id, key_name))
        raise Forbidden('No signature')
    size_signature = len(signature)
    if size_signature > TASK_SIGNATURE_MAX_SIZE:
        current_app.logger.exception(
            'Project id {}, path {} invalid task signature. Signature length {} exceeds max allowed length {}.'
            .format(project_id, key_name, size_signature, TASK_SIGNATURE_MAX_SIZE))
        raise Forbidden('Invalid signature')

    project = get_project_data(project_id)
    timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)

    payload = signer.loads(signature, max_age=timeout)
    task_id = payload['task_id']

    check_allowed(current_user.id, task_id, project,
                  lambda v: v == request.path)
    decrypted, key = read_encrypted_file(store, project, bucket, key_name)

    response = Response(decrypted, content_type=key.content_type)
    if hasattr(key, "content_encoding") and key.content_encoding:
        response.headers.add('Content-Encoding', key.content_encoding)
    if hasattr(key, "content_disposition") and key.content_disposition:
        response.headers.add('Content-Disposition', key.content_disposition)
    return response


@blueprint.route('/encrypted/<string:store>/<string:bucket>/workflow_request/<string:workflow_uid>/<int:project_id>/<path:path>')
@no_cache
@login_required
def encrypted_workflow_file(store, bucket, workflow_uid, project_id, path):
    """Proxy encrypted task file in a cloud storage for workflow"""
    key_name = '/workflow_request/{}/{}/{}'.format(
        workflow_uid, project_id, path)
    signature = request.args.get('task-signature')
    current_app.logger.info(
        'Project id {} decrypt workflow file. {}'.format(project_id, path))
    return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)


@blueprint.route('/encrypted/<string:store>/<string:bucket>/<int:project_id>/<path:path>')
@no_cache
@login_required
def encrypted_file(store, bucket, project_id, path):
    """Proxy encrypted task file in a cloud storage"""
    key_name = '/{}/{}'.format(project_id, path)
    signature = request.args.get('task-signature')
    current_app.logger.info(
        'Project id {} decrypt file. {}'.format(project_id, path))
    current_app.logger.info(
        "store %s, bucket %s, project_id %s, path %s", store, bucket, str(project_id), path)
    return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)


def encrypt_task_response_data(task_id, project_id, data):
    content = None
    task = task_repo.get_task(task_id)
    if not (task and isinstance(task.info, dict) and 'private_json__encrypted_payload' in task.info):
        return content

    project = get_project_data(project_id)
    secret = get_encryption_key(project)
    cipher = AESWithGCM(secret)
    content = json.dumps(data)
    content = cipher.encrypt(content.encode('utf8'))
    return content


@blueprint.route('/hdfs/<string:cluster>/<int:project_id>/<path:path>')
@no_cache
@login_required
def hdfs_file(project_id, cluster, path):
    raise BadRequest("Invalid task. HDFS is not supported")


def validate_task(project, task_id, user_id):
    """Confirm task payload is valid and user is authorized to access task."""
    task = task_repo.get_task(task_id)

    if not task or task.project_id != project['id']:
        raise BadRequest('Task does not exist')

    if current_user.admin:
        return True

    if has_lock(task_id, user_id,
                project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
        return True

    if user_id in project['owners_ids']:
        return True

    raise Forbidden('FORBIDDEN')


@blueprint.route('/encrypted/taskpayload/<int:project_id>/<int:task_id>')
@no_cache
@login_required
def encrypted_task_payload(project_id, task_id):
    """Proxy to decrypt encrypted task payload"""
    current_app.logger.info(
        'Project id {}, task id {}, decrypt task payload.'.format(project_id, task_id))
    signature = request.args.get('task-signature')
    if not signature:
        current_app.logger.exception(
            'Project id {}, task id {} has no signature.'.format(project_id, task_id))
        raise Forbidden('No signature')

    size_signature = len(signature)
    if size_signature > TASK_SIGNATURE_MAX_SIZE:
        current_app.logger.exception(
            'Project id {}, task id {} invalid task signature. Signature length {} exceeds max allowed length {}.'
            .format(project_id, task_id, size_signature, TASK_SIGNATURE_MAX_SIZE))
        raise Forbidden('Invalid signature')

    project = get_project_data(project_id)
    if not project:
        current_app.logger.exception(
            'Invalid project id {}.'.format(project_id, task_id))
        raise BadRequest('Invalid Project')

    timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)

    payload = signer.loads(signature, max_age=timeout)
    task_id = payload.get('task_id', 0)

    validate_task(project, task_id, current_user.id)

    # decrypt encrypted task data under private_json__encrypted_payload
    try:
        secret = get_encryption_key(project)
        task = task_repo.get_task(task_id)
        content = task.info.get('private_json__encrypted_payload')
        if content:
            cipher = AESWithGCM(secret)
            content = cipher.decrypt(content)
        else:
            content = ''
    except Exception as e:
        current_app.logger.exception(
            'Project id {} task {} decrypt encrypted data {}'.format(project_id, task_id, e))
        raise InternalServerError('An Error Occurred')

    response = Response(content, content_type='application/json')
    return response

1	# -- coding: utf8 --
2	# This file is part of PYBOSSA.
3	#
4	# Copyright (C) 2018 Scifabric LTD.
5	#
6	# PYBOSSA is free software: you can redistribute it and/or modify
7	# it under the terms of the GNU Affero General Public License as published by
8	# the Free Software Foundation, either version 3 of the License, or
9	# (at your option) any later version.
10	#
11	# PYBOSSA is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU Affero General Public License for more details.
15	#
16	# You should have received a copy of the GNU Affero General Public License
17	# along with PYBOSSA. If not, see <http://www.gnu.org/licenses/>.
18
19	from urllib.parse import urlparse, parse_qs	1✔
20	from functools import wraps	1✔
21	from flask import Blueprint, current_app, Response, request	1✔
22	from flask_login import current_user, login_required	1✔
23
24	import six	1✔
25	import requests	1✔
26	import json	1✔
27	from werkzeug.exceptions import Forbidden, BadRequest, InternalServerError, NotFound	1✔
28
29	from pybossa.cache.projects import get_project_data	1✔
30	from pybossa.contributions_guard import ContributionsGuard	1✔
31	from pybossa.core import task_repo, signer	1✔
32	from pybossa.encryption import AESWithGCM	1✔
33	# from pybossa.pybhdfs.client import HDFSKerberos
34	from pybossa.sched import has_lock	1✔
35	from pybossa.task_creator_helper import get_encryption_key, read_encrypted_file	1✔
36
37
38	blueprint = Blueprint('fileproxy', __name__)	1✔
39
40	TASK_SIGNATURE_MAX_SIZE = 128	1✔
41
42
43	def no_cache(view_func):	1✔
44	@wraps(view_func)	1✔
45	def decorated(args, *kwargs):	1✔
46	response = view_func(args, *kwargs)	1✔
47	response.headers.add('Cache-Control', 'no-store')	1✔
48	response.headers.add('Pragma', 'no-cache')	1✔
49	return response	1✔
50	return decorated	1✔
51
52
53	def check_allowed(user_id, task_id, project, is_valid_url):	1✔
54	task = task_repo.get_task(task_id)	1✔
55
56	if not task or task.project_id != project['id']:	1✔
57	raise BadRequest('Task does not exist')	1✔
58
59	if not any(is_valid_url(v) for v in task.info.values()):	1✔
60	raise Forbidden('Invalid task content')	1✔
61
62	if current_user.admin:	1✔
63	return True	1✔
64
65	if has_lock(task_id, user_id,	1✔
66	project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
67	return True	1✔
68
69	if user_id in project['owners_ids']:	×
70	return True	×
71
72	raise Forbidden('FORBIDDEN')	×
73
74
75	def read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature):	1✔
76	if not signature:	1✔
77	current_app.logger.exception(	1✔
78	'Project id {} no signature {}'.format(project_id, key_name))
79	raise Forbidden('No signature')	1✔
80	size_signature = len(signature)	1✔
81	if size_signature > TASK_SIGNATURE_MAX_SIZE:	1✔
82	current_app.logger.exception(	1✔
83	'Project id {}, path {} invalid task signature. Signature length {} exceeds max allowed length {}.'
84	.format(project_id, key_name, size_signature, TASK_SIGNATURE_MAX_SIZE))
85	raise Forbidden('Invalid signature')	1✔
86
87	project = get_project_data(project_id)	1✔
88	timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)	1✔
89
90	payload = signer.loads(signature, max_age=timeout)	1✔
91	task_id = payload['task_id']	1✔
92
93	check_allowed(current_user.id, task_id, project,	1✔
94	lambda v: v == request.path)
95	decrypted, key = read_encrypted_file(store, project, bucket, key_name)	1✔
96
97	response = Response(decrypted, content_type=key.content_type)	1✔
98	if hasattr(key, "content_encoding") and key.content_encoding:	1✔
99	response.headers.add('Content-Encoding', key.content_encoding)	1✔
100	if hasattr(key, "content_disposition") and key.content_disposition:	1✔
101	response.headers.add('Content-Disposition', key.content_disposition)	1✔
102	return response	1✔
103
104
105	@blueprint.route('/encrypted/<string:store>/<string:bucket>/workflow_request/<string:workflow_uid>/<int:project_id>/<path:path>')	1✔
106	@no_cache	1✔
107	@login_required	1✔
108	def encrypted_workflow_file(store, bucket, workflow_uid, project_id, path):	1✔
109	"""Proxy encrypted task file in a cloud storage for workflow"""
110	key_name = '/workflow_request/{}/{}/{}'.format(	1✔
111	workflow_uid, project_id, path)
112	signature = request.args.get('task-signature')	1✔
113	current_app.logger.info(	1✔
114	'Project id {} decrypt workflow file. {}'.format(project_id, path))
115	return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)	1✔
116
117
118	@blueprint.route('/encrypted/<string:store>/<string:bucket>/<int:project_id>/<path:path>')	1✔
119	@no_cache	1✔
120	@login_required	1✔
121	def encrypted_file(store, bucket, project_id, path):	1✔
122	"""Proxy encrypted task file in a cloud storage"""
123	key_name = '/{}/{}'.format(project_id, path)	1✔
124	signature = request.args.get('task-signature')	1✔
125	current_app.logger.info(	1✔
126	'Project id {} decrypt file. {}'.format(project_id, path))
127	current_app.logger.info(	1✔
128	"store %s, bucket %s, project_id %s, path %s", store, bucket, str(project_id), path)
129	return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)	1✔
130
131
132	def encrypt_task_response_data(task_id, project_id, data):	1✔
133	content = None	1✔
134	task = task_repo.get_task(task_id)	1✔
135	if not (task and isinstance(task.info, dict) and 'private_json__encrypted_payload' in task.info):	1✔
136	return content	1✔
137
138	project = get_project_data(project_id)	1✔
139	secret = get_encryption_key(project)	1✔
140	cipher = AESWithGCM(secret)	1✔
141	content = json.dumps(data)	1✔
142	content = cipher.encrypt(content.encode('utf8'))	1✔
143	return content	1✔
144
145
146	@blueprint.route('/hdfs/<string:cluster>/<int:project_id>/<path:path>')	1✔
147	@no_cache	1✔
148	@login_required	1✔
149	def hdfs_file(project_id, cluster, path):	1✔
UNCOV 150	raise BadRequest("Invalid task. HDFS is not supported")	×
151
152
153	def validate_task(project, task_id, user_id):	1✔
154	"""Confirm task payload is valid and user is authorized to access task."""
155	task = task_repo.get_task(task_id)	1✔
156
157	if not task or task.project_id != project['id']:	1✔
158	raise BadRequest('Task does not exist')	1✔
159
160	if current_user.admin:	1✔
161	return True	1✔
162
163	if has_lock(task_id, user_id,	1✔
164	project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
165	return True	1✔
166
167	if user_id in project['owners_ids']:	1✔
168	return True	1✔
169
170	raise Forbidden('FORBIDDEN')	1✔
171
172
173	@blueprint.route('/encrypted/taskpayload/<int:project_id>/<int:task_id>')	1✔
174	@no_cache	1✔
175	@login_required	1✔
176	def encrypted_task_payload(project_id, task_id):	1✔
177	"""Proxy to decrypt encrypted task payload"""
178	current_app.logger.info(	1✔
179	'Project id {}, task id {}, decrypt task payload.'.format(project_id, task_id))
180	signature = request.args.get('task-signature')	1✔
181	if not signature:	1✔
182	current_app.logger.exception(	1✔
183	'Project id {}, task id {} has no signature.'.format(project_id, task_id))
184	raise Forbidden('No signature')	1✔
185
186	size_signature = len(signature)	1✔
187	if size_signature > TASK_SIGNATURE_MAX_SIZE:	1✔
188	current_app.logger.exception(	1✔
189	'Project id {}, task id {} invalid task signature. Signature length {} exceeds max allowed length {}.'
190	.format(project_id, task_id, size_signature, TASK_SIGNATURE_MAX_SIZE))
191	raise Forbidden('Invalid signature')	1✔
192
193	project = get_project_data(project_id)	1✔
194	if not project:	1✔
195	current_app.logger.exception(	1✔
196	'Invalid project id {}.'.format(project_id, task_id))
197	raise BadRequest('Invalid Project')	1✔
198
199	timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)	1✔
200
201	payload = signer.loads(signature, max_age=timeout)	1✔
202	task_id = payload.get('task_id', 0)	1✔
203
204	validate_task(project, task_id, current_user.id)	1✔
205
206	# decrypt encrypted task data under private_json__encrypted_payload
207	try:	1✔
208	secret = get_encryption_key(project)	1✔
209	task = task_repo.get_task(task_id)	1✔
210	content = task.info.get('private_json__encrypted_payload')	1✔
211	if content:	1✔
212	cipher = AESWithGCM(secret)	1✔
213	content = cipher.decrypt(content)	1✔
214	else:
215	content = ''	1✔
216	except Exception as e:	1✔
217	current_app.logger.exception(	1✔
218	'Project id {} task {} decrypt encrypted data {}'.format(project_id, task_id, e))
219	raise InternalServerError('An Error Occurred')	1✔
220
221	response = Response(content, content_type='application/json')	1✔
222	return response	1✔

bloomberg / pybossa / 19114618549

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous