19482063855

Committed 18 Nov 2025 09:34PM UTC coverage: 93.533% (-0.5%) from 94.065%

Build # 19482063855

Build Type

Pull #1075

github

Committed by

dchhabda

Commit Message

modified boto2-3 migration

Pull Request Pull Request #1075: RDISCROWD-8392: deprecate old boto. use boto3 only (Updated)

Run Details

10 of 19 new or added lines in 3 files covered. (52.63%)

87 existing lines in 5 files now uncovered.

17703 of 18927 relevant lines covered (93.53%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.86

/pybossa/view/fileproxy.py

# -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2018 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.

from urllib.parse import urlparse, parse_qs
from functools import wraps
from flask import Blueprint, current_app, Response, request
from flask_login import current_user, login_required

import six
import requests
import json
from werkzeug.exceptions import Forbidden, BadRequest, InternalServerError, NotFound

from pybossa.cache.projects import get_project_data
from pybossa.contributions_guard import ContributionsGuard
from pybossa.core import task_repo, signer
from pybossa.encryption import AESWithGCM
# from pybossa.pybhdfs.client import HDFSKerberos
from pybossa.sched import has_lock
from pybossa.task_creator_helper import get_encryption_key, read_encrypted_file


blueprint = Blueprint('fileproxy', __name__)

TASK_SIGNATURE_MAX_SIZE = 128

def no_cache(view_func):
    @wraps(view_func)
    def decorated(*args, **kwargs):
        response = view_func(*args, **kwargs)
        response.headers.add('Cache-Control', 'no-store')
        response.headers.add('Pragma', 'no-cache')
        return response
    return decorated


def check_allowed(user_id, task_id, project, is_valid_url):
    task = task_repo.get_task(task_id)

    if not task or task.project_id != project['id']:
        raise BadRequest('Task does not exist')

    if not any(is_valid_url(v) for v in task.info.values()):
        raise Forbidden('Invalid task content')

    if current_user.admin:
        return True

    if has_lock(task_id, user_id,
                project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
        return True

    if user_id in project['owners_ids']:
        return True

    raise Forbidden('FORBIDDEN')

def read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature):
    if not signature:
        current_app.logger.exception('Project id {} no signature {}'.format(project_id, key_name))
        raise Forbidden('No signature')
    size_signature = len(signature)
    if size_signature > TASK_SIGNATURE_MAX_SIZE:
        current_app.logger.exception(
            'Project id {}, path {} invalid task signature. Signature length {} exceeds max allowed length {}.' \
                .format(project_id, key_name, size_signature, TASK_SIGNATURE_MAX_SIZE))
        raise Forbidden('Invalid signature')

    project = get_project_data(project_id)
    if not project:
        current_app.logger.exception('Invalid project id {}.'.format(project_id))
        raise BadRequest('Invalid Project')

    timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)

    payload = signer.loads(signature, max_age=timeout)
    task_id = payload['task_id']

    check_allowed(current_user.id, task_id, project, lambda v: v == request.path)
    decrypted, key = read_encrypted_file(store, project, bucket, key_name)

    response = Response(decrypted, content_type=key.content_type)
    if hasattr(key, "content_encoding") and key.content_encoding:
        response.headers.add('Content-Encoding', key.content_encoding)
    if hasattr(key, "content_disposition") and key.content_disposition:
        response.headers.add('Content-Disposition', key.content_disposition)
    return response


@blueprint.route('/encrypted/<string:store>/<string:bucket>/workflow_request/<string:workflow_uid>/<int:project_id>/<path:path>')
@no_cache
@login_required
def encrypted_workflow_file(store, bucket, workflow_uid, project_id, path):
    """Proxy encrypted task file in a cloud storage for workflow"""
    key_name = '/workflow_request/{}/{}/{}'.format(workflow_uid, project_id, path)
    signature = request.args.get('task-signature')
    current_app.logger.info('Project id {} decrypt workflow file. {}'.format(project_id, path))
    return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)


@blueprint.route('/encrypted/<string:store>/<string:bucket>/<int:project_id>/<path:path>')
@no_cache
@login_required
def encrypted_file(store, bucket, project_id, path):
    """Proxy encrypted task file in a cloud storage"""
    key_name = '/{}/{}'.format(project_id, path)
    signature = request.args.get('task-signature')
    current_app.logger.info('Project id {} decrypt file. {}'.format(project_id, path))
    current_app.logger.info("store %s, bucket %s, project_id %s, path %s", store, bucket, str(project_id), path)
    return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)


def encrypt_task_response_data(task_id, project_id, data):
    content = None
    task = task_repo.get_task(task_id)
    if not (task and isinstance(task.info, dict) and 'private_json__encrypted_payload' in task.info):
        return content

    project = get_project_data(project_id)
    secret = get_encryption_key(project)
    cipher = AESWithGCM(secret)
    content = json.dumps(data)
    content = cipher.encrypt(content.encode('utf8'))
    return content


@blueprint.route('/hdfs/<string:cluster>/<int:project_id>/<path:path>')
@no_cache
@login_required
def hdfs_file(project_id, cluster, path):
    raise BadRequest("Invalid task. HDFS is not supported")


def validate_task(project, task_id, user_id):
    """Confirm task payload is valid and user is authorized to access task."""
    task = task_repo.get_task(task_id)

    if not task or task.project_id != project['id']:
        raise BadRequest('Task does not exist')

    if current_user.admin:
        return True

    if has_lock(task_id, user_id,
                project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
        return True

    if user_id in project['owners_ids']:
        return True

    raise Forbidden('FORBIDDEN')


@blueprint.route('/encrypted/taskpayload/<int:project_id>/<int:task_id>')
@no_cache
@login_required
def encrypted_task_payload(project_id, task_id):
    """Proxy to decrypt encrypted task payload"""
    current_app.logger.info('Project id {}, task id {}, decrypt task payload.'.format(project_id, task_id))
    signature = request.args.get('task-signature')
    if not signature:
        current_app.logger.exception('Project id {}, task id {} has no signature.'.format(project_id, task_id))
        raise Forbidden('No signature')

    size_signature = len(signature)
    if size_signature > TASK_SIGNATURE_MAX_SIZE:
        current_app.logger.exception(
            'Project id {}, task id {} invalid task signature. Signature length {} exceeds max allowed length {}.' \
                .format(project_id, task_id, size_signature, TASK_SIGNATURE_MAX_SIZE))
        raise Forbidden('Invalid signature')

    project = get_project_data(project_id)
    if not project:
        current_app.logger.exception('Invalid project id {}.'.format(project_id))
        raise BadRequest('Invalid Project')

    timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)

    payload = signer.loads(signature, max_age=timeout)
    task_id = payload.get('task_id', 0)

    validate_task(project, task_id, current_user.id)

    ## decrypt encrypted task data under private_json__encrypted_payload
    try:
        secret = get_encryption_key(project)
        task = task_repo.get_task(task_id)
        content = task.info.get('private_json__encrypted_payload')
        if content:
            cipher = AESWithGCM(secret)
            content = cipher.decrypt(content)
        else:
            content = ''
    except Exception as e:
        current_app.logger.exception('Project id {} task {} decrypt encrypted data {}'.format(project_id, task_id, e))
        raise InternalServerError('An Error Occurred')

    response = Response(content, content_type='application/json')
    return response

1	# -- coding: utf8 --
2	# This file is part of PYBOSSA.
3	#
4	# Copyright (C) 2018 Scifabric LTD.
5	#
6	# PYBOSSA is free software: you can redistribute it and/or modify
7	# it under the terms of the GNU Affero General Public License as published by
8	# the Free Software Foundation, either version 3 of the License, or
9	# (at your option) any later version.
10	#
11	# PYBOSSA is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU Affero General Public License for more details.
15	#
16	# You should have received a copy of the GNU Affero General Public License
17	# along with PYBOSSA. If not, see <http://www.gnu.org/licenses/>.
18
19	from urllib.parse import urlparse, parse_qs	1✔
20	from functools import wraps	1✔
21	from flask import Blueprint, current_app, Response, request	1✔
22	from flask_login import current_user, login_required	1✔
23
24	import six	1✔
25	import requests	1✔
26	import json	1✔
27	from werkzeug.exceptions import Forbidden, BadRequest, InternalServerError, NotFound	1✔
28
29	from pybossa.cache.projects import get_project_data	1✔
30	from pybossa.contributions_guard import ContributionsGuard	1✔
31	from pybossa.core import task_repo, signer	1✔
32	from pybossa.encryption import AESWithGCM	1✔
33	# from pybossa.pybhdfs.client import HDFSKerberos
34	from pybossa.sched import has_lock	1✔
35	from pybossa.task_creator_helper import get_encryption_key, read_encrypted_file	1✔
36
37
38	blueprint = Blueprint('fileproxy', __name__)	1✔
39
40	TASK_SIGNATURE_MAX_SIZE = 128	1✔
41
42	def no_cache(view_func):	1✔
43	@wraps(view_func)	1✔
44	def decorated(args, *kwargs):	1✔
45	response = view_func(args, *kwargs)	1✔
46	response.headers.add('Cache-Control', 'no-store')	1✔
47	response.headers.add('Pragma', 'no-cache')	1✔
48	return response	1✔
49	return decorated	1✔
50
51
52	def check_allowed(user_id, task_id, project, is_valid_url):	1✔
53	task = task_repo.get_task(task_id)	1✔
54
55	if not task or task.project_id != project['id']:	1✔
56	raise BadRequest('Task does not exist')	1✔
57
58	if not any(is_valid_url(v) for v in task.info.values()):	1✔
59	raise Forbidden('Invalid task content')	1✔
60
61	if current_user.admin:	1✔
62	return True	1✔
63
64	if has_lock(task_id, user_id,	1✔
65	project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
66	return True	1✔
67
68	if user_id in project['owners_ids']:	×
69	return True	×
70
71	raise Forbidden('FORBIDDEN')	×
72
73	def read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature):	1✔
74	if not signature:	1✔
75	current_app.logger.exception('Project id {} no signature {}'.format(project_id, key_name))	1✔
76	raise Forbidden('No signature')	1✔
77	size_signature = len(signature)	1✔
78	if size_signature > TASK_SIGNATURE_MAX_SIZE:	1✔
79	current_app.logger.exception(	1✔
80	'Project id {}, path {} invalid task signature. Signature length {} exceeds max allowed length {}.' \
81	.format(project_id, key_name, size_signature, TASK_SIGNATURE_MAX_SIZE))
82	raise Forbidden('Invalid signature')	1✔
83
84	project = get_project_data(project_id)	1✔
85	if not project:	1✔
86	current_app.logger.exception('Invalid project id {}.'.format(project_id))	1✔
87	raise BadRequest('Invalid Project')	1✔
88
89	timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)	1✔
90
91	payload = signer.loads(signature, max_age=timeout)	1✔
92	task_id = payload['task_id']	1✔
93
94	check_allowed(current_user.id, task_id, project, lambda v: v == request.path)	1✔
95	decrypted, key = read_encrypted_file(store, project, bucket, key_name)	1✔
96
97	response = Response(decrypted, content_type=key.content_type)	1✔
98	if hasattr(key, "content_encoding") and key.content_encoding:	1✔
99	response.headers.add('Content-Encoding', key.content_encoding)	1✔
100	if hasattr(key, "content_disposition") and key.content_disposition:	1✔
101	response.headers.add('Content-Disposition', key.content_disposition)	1✔
102	return response	1✔
103
104
105	@blueprint.route('/encrypted/<string:store>/<string:bucket>/workflow_request/<string:workflow_uid>/<int:project_id>/<path:path>')	1✔
106	@no_cache	1✔
107	@login_required	1✔
108	def encrypted_workflow_file(store, bucket, workflow_uid, project_id, path):	1✔
109	"""Proxy encrypted task file in a cloud storage for workflow"""
110	key_name = '/workflow_request/{}/{}/{}'.format(workflow_uid, project_id, path)	1✔
111	signature = request.args.get('task-signature')	1✔
112	current_app.logger.info('Project id {} decrypt workflow file. {}'.format(project_id, path))	1✔
113	return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)	1✔
114
115
116	@blueprint.route('/encrypted/<string:store>/<string:bucket>/<int:project_id>/<path:path>')	1✔
117	@no_cache	1✔
118	@login_required	1✔
119	def encrypted_file(store, bucket, project_id, path):	1✔
120	"""Proxy encrypted task file in a cloud storage"""
121	key_name = '/{}/{}'.format(project_id, path)	1✔
122	signature = request.args.get('task-signature')	1✔
123	current_app.logger.info('Project id {} decrypt file. {}'.format(project_id, path))	1✔
124	current_app.logger.info("store %s, bucket %s, project_id %s, path %s", store, bucket, str(project_id), path)	1✔
125	return read_encrypted_file_with_signature(store, project_id, bucket, key_name, signature)	1✔
126
127
128	def encrypt_task_response_data(task_id, project_id, data):	1✔
UNCOV 129	content = None	×
UNCOV 130	task = task_repo.get_task(task_id)	×
UNCOV 131	if not (task and isinstance(task.info, dict) and 'private_json__encrypted_payload' in task.info):	×
UNCOV 132	return content	×
133
UNCOV 134	project = get_project_data(project_id)	×
UNCOV 135	secret = get_encryption_key(project)	×
UNCOV 136	cipher = AESWithGCM(secret)	×
UNCOV 137	content = json.dumps(data)	×
UNCOV 138	content = cipher.encrypt(content.encode('utf8'))	×
UNCOV 139	return content	×
140
141
142	@blueprint.route('/hdfs/<string:cluster>/<int:project_id>/<path:path>')	1✔
143	@no_cache	1✔
144	@login_required	1✔
145	def hdfs_file(project_id, cluster, path):	1✔
146	raise BadRequest("Invalid task. HDFS is not supported")	×
147
148
149	def validate_task(project, task_id, user_id):	1✔
150	"""Confirm task payload is valid and user is authorized to access task."""
151	task = task_repo.get_task(task_id)	1✔
152
153	if not task or task.project_id != project['id']:	1✔
154	raise BadRequest('Task does not exist')	1✔
155
156	if current_user.admin:	1✔
157	return True	1✔
158
159	if has_lock(task_id, user_id,	1✔
160	project['info'].get('timeout', ContributionsGuard.STAMP_TTL)):
161	return True	1✔
162
163	if user_id in project['owners_ids']:	1✔
164	return True	1✔
165
166	raise Forbidden('FORBIDDEN')	1✔
167
168
169	@blueprint.route('/encrypted/taskpayload/<int:project_id>/<int:task_id>')	1✔
170	@no_cache	1✔
171	@login_required	1✔
172	def encrypted_task_payload(project_id, task_id):	1✔
173	"""Proxy to decrypt encrypted task payload"""
174	current_app.logger.info('Project id {}, task id {}, decrypt task payload.'.format(project_id, task_id))	1✔
175	signature = request.args.get('task-signature')	1✔
176	if not signature:	1✔
177	current_app.logger.exception('Project id {}, task id {} has no signature.'.format(project_id, task_id))	1✔
178	raise Forbidden('No signature')	1✔
179
180	size_signature = len(signature)	1✔
181	if size_signature > TASK_SIGNATURE_MAX_SIZE:	1✔
182	current_app.logger.exception(	1✔
183	'Project id {}, task id {} invalid task signature. Signature length {} exceeds max allowed length {}.' \
184	.format(project_id, task_id, size_signature, TASK_SIGNATURE_MAX_SIZE))
185	raise Forbidden('Invalid signature')	1✔
186
187	project = get_project_data(project_id)	1✔
188	if not project:	1✔
189	current_app.logger.exception('Invalid project id {}.'.format(project_id))	1✔
190	raise BadRequest('Invalid Project')	1✔
191
192	timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)	1✔
193
194	payload = signer.loads(signature, max_age=timeout)	1✔
195	task_id = payload.get('task_id', 0)	1✔
196
197	validate_task(project, task_id, current_user.id)	1✔
198
199	## decrypt encrypted task data under private_json__encrypted_payload
200	try:	1✔
201	secret = get_encryption_key(project)	1✔
202	task = task_repo.get_task(task_id)	1✔
203	content = task.info.get('private_json__encrypted_payload')	1✔
204	if content:	1✔
205	cipher = AESWithGCM(secret)	1✔
206	content = cipher.decrypt(content)	1✔
207	else:
208	content = ''	1✔
209	except Exception as e:	1✔
210	current_app.logger.exception('Project id {} task {} decrypt encrypted data {}'.format(project_id, task_id, e))	1✔
211	raise InternalServerError('An Error Occurred')	1✔
212
213	response = Response(content, content_type='application/json')	1✔
214	return response	1✔

bloomberg / pybossa / 19482063855

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous