13020205265

Committed 28 Jan 2025 09:50PM UTC coverage: 94.132% (-0.001%) from 94.133%

Build # 13020205265

Build Type

Pull #1025

github

Committed by

dchhabda

Commit Message

perform duplicate check using checksum

Pull Request Pull Request #1025: perform duplicate check using checksum

Run Details

25 of 27 new or added lines in 5 files covered. (92.59%)

9 existing lines in 2 files now uncovered.

17500 of 18591 relevant lines covered (94.13%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.29

/pybossa/api/task.py

# -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.
"""
PYBOSSA api module for exposing domain object Task via an API.

This package adds GET, POST, PUT and DELETE methods for:
    * tasks

"""
from flask import abort, current_app
from flask_login import current_user
from werkzeug.exceptions import BadRequest, Conflict, NotFound
from pybossa.model.task import Task
from pybossa.model.project import Project
from pybossa.core import result_repo
from pybossa.util import sign_task
from .api_base import APIBase
from pybossa.api.pwd_manager import get_pwd_manager
from pybossa.util import get_user_id_or_ip, validate_required_fields
from pybossa.core import task_repo, project_repo
from pybossa.cache.projects import get_project_data
from pybossa.data_access import when_data_access
import hashlib
from flask import url_for
from pybossa.cloud_store_api.s3 import upload_json_data
from pybossa.auth.task import TaskAuth
from pybossa.cache import delete_memoized
from pybossa.cache.task_browse_helpers import get_searchable_columns
import json
import copy
from pybossa.task_creator_helper import get_task_expiration
from pybossa.model import make_timestamp
from pybossa.task_creator_helper import generate_checksum
from pybossa.cache.projects import get_project_data


class TaskAPI(APIBase):

    """Class for domain object Task."""

    __class__ = Task
    reserved_keys = set(['id', 'created', 'state', 'fav_user_ids',
        'calibration'])

    immutable_keys = set(['project_id'])

    def _forbidden_attributes(self, data):
        for key in data.keys():
            if key in self.reserved_keys:
                raise BadRequest("Reserved keys in payload")

    def _update_attribute(self, new, old):
        for key, value in old.info.items():
            new.info.setdefault(key, value)

        gold_task = bool(new.gold_answers)
        n_taskruns = len(new.task_runs)
        if new.state == 'completed':
            if gold_task or (old.n_answers < new.n_answers and
                n_taskruns < new.n_answers):
                new.state = 'ongoing'
        if new.state == 'ongoing':
            if not gold_task and (n_taskruns >= new.n_answers):
                new.state = 'completed'
        new.calibration = int(gold_task)
        current_app.logger.info("Updating task %d, old state: %s, new state: %s, "
                                "old exported: %s, new exported: %s",
                                new.id, old.state, new.state,
                                str(old.exported), str(new.exported))
        if new.expiration is not None:
            new.expiration = get_task_expiration(new.expiration, old.created)

    def _preprocess_post_data(self, data):
        project_id = data["project_id"]
        info = data["info"]
        if isinstance(info, dict):
            hdfs_task = any([val.startswith("/fileproxy/hdfs/") for val in info.values() if isinstance(val, str)])
            if hdfs_task:
                raise BadRequest("Invalid task payload. HDFS is not supported")
        project = get_project_data(project_id)
        dup_checksum = generate_checksum(project=project, task=data)
        data["dup_checksum"] = dup_checksum
        duplicate = task_repo.find_duplicate(project_id=project_id, info=info, dup_checksum=dup_checksum)
        if duplicate:
            message = {
                'reason': 'DUPLICATE_TASK',
                'task_id': duplicate
            }
            raise Conflict(json.dumps(message))


        if 'n_answers' not in data:
            project = project_repo.get(project_id)
            if not project:
                raise NotFound(f'Non existing project id {project_id}')
            data['n_answers'] = project.get_default_n_answers()
        user_pref = data.get('user_pref', {})
        if user_pref.get('languages'):
            user_pref['languages'] = [s.lower() for s in user_pref.get('languages', [])]
        if user_pref.get('locations'):
            user_pref['locations'] = [s.lower() for s in user_pref.get('locations', [])]
        if user_pref.get('assign_user'):
            user_pref['assign_user'] = [s.lower() for s in user_pref.get('assign_user', [])]
        invalid_fields = validate_required_fields(info)
        if invalid_fields:
            raise BadRequest('Missing or incorrect required fields: {}'
                            .format(','.join(invalid_fields)))
        if data.get('gold_answers'):
            try:
                gold_answers = data['gold_answers']
                if type(gold_answers) is dict:
                    data['calibration'] = 1
                    data['exported'] = True
            except Exception as e:
                raise BadRequest('Invalid gold_answers')
        create_time = data.get("created") or make_timestamp()
        data["expiration"] = get_task_expiration(data.get('expiration'), create_time)

    def _verify_auth(self, item):
        if not current_user.is_authenticated:
            return False
        if current_user.admin or current_user.subadmin:
            return True
        project = Project(**get_project_data(item.project_id))
        pwd_manager = get_pwd_manager(project)
        return not pwd_manager.password_needed(project, get_user_id_or_ip())

    def _sign_item(self, item):
        project_id = item['project_id']
        if current_user.admin or \
           current_user.id in get_project_data(project_id)['owners_ids']:
            sign_task(item)

    def _select_attributes(self, data):
        return TaskAuth.apply_access_control(data, user=current_user, project_data=get_project_data(data['project_id']))

    def put(self, oid):
        # reset cache / memoized
        delete_memoized(get_searchable_columns)
        return super(TaskAPI, self).put(oid)

1	# -- coding: utf8 --
2	# This file is part of PYBOSSA.
3	#
4	# Copyright (C) 2015 Scifabric LTD.
5	#
6	# PYBOSSA is free software: you can redistribute it and/or modify
7	# it under the terms of the GNU Affero General Public License as published by
8	# the Free Software Foundation, either version 3 of the License, or
9	# (at your option) any later version.
10	#
11	# PYBOSSA is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU Affero General Public License for more details.
15	#
16	# You should have received a copy of the GNU Affero General Public License
17	# along with PYBOSSA. If not, see <http://www.gnu.org/licenses/>.
18	"""	1✔
19	PYBOSSA api module for exposing domain object Task via an API.
20
21	This package adds GET, POST, PUT and DELETE methods for:
22	* tasks
23
24	"""
25	from flask import abort, current_app	1✔
26	from flask_login import current_user	1✔
27	from werkzeug.exceptions import BadRequest, Conflict, NotFound	1✔
28	from pybossa.model.task import Task	1✔
29	from pybossa.model.project import Project	1✔
30	from pybossa.core import result_repo	1✔
31	from pybossa.util import sign_task	1✔
32	from .api_base import APIBase	1✔
33	from pybossa.api.pwd_manager import get_pwd_manager	1✔
34	from pybossa.util import get_user_id_or_ip, validate_required_fields	1✔
35	from pybossa.core import task_repo, project_repo	1✔
36	from pybossa.cache.projects import get_project_data	1✔
37	from pybossa.data_access import when_data_access	1✔
38	import hashlib	1✔
39	from flask import url_for	1✔
40	from pybossa.cloud_store_api.s3 import upload_json_data	1✔
41	from pybossa.auth.task import TaskAuth	1✔
42	from pybossa.cache import delete_memoized	1✔
43	from pybossa.cache.task_browse_helpers import get_searchable_columns	1✔
44	import json	1✔
45	import copy	1✔
46	from pybossa.task_creator_helper import get_task_expiration	1✔
47	from pybossa.model import make_timestamp	1✔
48	from pybossa.task_creator_helper import generate_checksum	1✔
49	from pybossa.cache.projects import get_project_data	1✔
50
51
52	class TaskAPI(APIBase):	1✔
53
54	"""Class for domain object Task."""
55
56	__class__ = Task	1✔
57	reserved_keys = set(['id', 'created', 'state', 'fav_user_ids',	1✔
58	'calibration'])
59
60	immutable_keys = set(['project_id'])	1✔
61
62	def _forbidden_attributes(self, data):	1✔
63	for key in data.keys():	1✔
64	if key in self.reserved_keys:	1✔
65	raise BadRequest("Reserved keys in payload")	1✔
66
67	def _update_attribute(self, new, old):	1✔
68	for key, value in old.info.items():	1✔
69	new.info.setdefault(key, value)	1✔
70
71	gold_task = bool(new.gold_answers)	1✔
72	n_taskruns = len(new.task_runs)	1✔
73	if new.state == 'completed':	1✔
74	if gold_task or (old.n_answers < new.n_answers and	1✔
75	n_taskruns < new.n_answers):
76	new.state = 'ongoing'	1✔
77	if new.state == 'ongoing':	1✔
78	if not gold_task and (n_taskruns >= new.n_answers):	1✔
79	new.state = 'completed'	1✔
80	new.calibration = int(gold_task)	1✔
81	current_app.logger.info("Updating task %d, old state: %s, new state: %s, "	1✔
82	"old exported: %s, new exported: %s",
83	new.id, old.state, new.state,
84	str(old.exported), str(new.exported))
85	if new.expiration is not None:	1✔
86	new.expiration = get_task_expiration(new.expiration, old.created)	1✔
87
88	def _preprocess_post_data(self, data):	1✔
89	project_id = data["project_id"]	1✔
90	info = data["info"]	1✔
91	if isinstance(info, dict):	1✔
92	hdfs_task = any([val.startswith("/fileproxy/hdfs/") for val in info.values() if isinstance(val, str)])	1✔
93	if hdfs_task:	1✔
94	raise BadRequest("Invalid task payload. HDFS is not supported")	1✔
95	project = get_project_data(project_id)	1✔
96	dup_checksum = generate_checksum(project=project, task=data)	1✔
97	data["dup_checksum"] = dup_checksum	1✔
98	duplicate = task_repo.find_duplicate(project_id=project_id, info=info, dup_checksum=dup_checksum)	1✔
99	if duplicate:	1✔
100	message = {	1✔
101	'reason': 'DUPLICATE_TASK',
102	'task_id': duplicate
103	}
104	raise Conflict(json.dumps(message))	1✔
105
106
107	if 'n_answers' not in data:	1✔
108	project = project_repo.get(project_id)	1✔
109	if not project:	1✔
110	raise NotFound(f'Non existing project id {project_id}')	1✔
111	data['n_answers'] = project.get_default_n_answers()	1✔
112	user_pref = data.get('user_pref', {})	1✔
113	if user_pref.get('languages'):	1✔
UNCOV 114	user_pref['languages'] = [s.lower() for s in user_pref.get('languages', [])]	×
115	if user_pref.get('locations'):	1✔
UNCOV 116	user_pref['locations'] = [s.lower() for s in user_pref.get('locations', [])]	×
117	if user_pref.get('assign_user'):	1✔
UNCOV 118	user_pref['assign_user'] = [s.lower() for s in user_pref.get('assign_user', [])]	×
119	invalid_fields = validate_required_fields(info)	1✔
120	if invalid_fields:	1✔
121	raise BadRequest('Missing or incorrect required fields: {}'	1✔
122	.format(','.join(invalid_fields)))
123	if data.get('gold_answers'):	1✔
124	try:	1✔
125	gold_answers = data['gold_answers']	1✔
126	if type(gold_answers) is dict:	1✔
127	data['calibration'] = 1	1✔
128	data['exported'] = True	1✔
UNCOV 129	except Exception as e:	×
130	raise BadRequest('Invalid gold_answers')	×
131	create_time = data.get("created") or make_timestamp()	1✔
132	data["expiration"] = get_task_expiration(data.get('expiration'), create_time)	1✔
133
134	def _verify_auth(self, item):	1✔
135	if not current_user.is_authenticated:	1✔
UNCOV 136	return False	×
137	if current_user.admin or current_user.subadmin:	1✔
138	return True	1✔
139	project = Project(**get_project_data(item.project_id))	1✔
140	pwd_manager = get_pwd_manager(project)	1✔
141	return not pwd_manager.password_needed(project, get_user_id_or_ip())	1✔
142
143	def _sign_item(self, item):	1✔
144	project_id = item['project_id']	1✔
145	if current_user.admin or \	1✔
146	current_user.id in get_project_data(project_id)['owners_ids']:
147	sign_task(item)	1✔
148
149	def _select_attributes(self, data):	1✔
150	return TaskAuth.apply_access_control(data, user=current_user, project_data=get_project_data(data['project_id']))	1✔
151
152	def put(self, oid):	1✔
153	# reset cache / memoized
154	delete_memoized(get_searchable_columns)	1✔
155	return super(TaskAPI, self).put(oid)	1✔

bloomberg / pybossa / 13020205265

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous