12163548050

Committed 04 Dec 2024 03:58PM UTC coverage: 83.629% (+0.2%) from 83.45%

Build # 12163548050

Build Type

push

github

Committed by

ItIsJordan

Commit Message

Fix bug in search_test.py

Fixes a bug where testing for MadAnalysis was checking against the database entry and not the document dictionary (as it should have been)

Run Details

4572 of 5467 relevant lines covered (83.63%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.03

/hepdata/modules/records/utils/analyses.py

# -*- coding: utf-8 -*-
#
# This file is part of HEPData.
# Copyright (C) 2021 CERN.
#
# HEPData is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# HEPData is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with HEPData; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

import logging

from celery import shared_task
from flask import current_app
from invenio_db import db
import requests

from hepdata.ext.opensearch.api import index_record_ids
from hepdata.modules.submission.api import get_latest_hepsubmission, is_resource_added_to_submission
from hepdata.modules.submission.models import DataResource, HEPSubmission, data_reference_link
from hepdata.utils.users import get_user_from_id
from hepdata.modules.records.subscribers.rest import subscribe
from hepdata.modules.records.subscribers.api import is_current_user_subscribed_to_record

logging.basicConfig()
log = logging.getLogger(__name__)


@shared_task
def update_analyses(endpoint=None):
    """
    Update (Rivet, MadAnalysis 5 and SModelS) analyses and remove outdated resources.
    Allow bulk subscription to record update notifications if "subscribe_user_id" in endpoint.

    :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or None (default) for both
    """
    endpoints = current_app.config["ANALYSES_ENDPOINTS"]
    for analysis_endpoint in endpoints:

        if endpoint and endpoint != analysis_endpoint:
            continue

        if "endpoint_url" in endpoints[analysis_endpoint]:

            log.info("Updating analyses from {0}...".format(analysis_endpoint))

            response = requests.get(endpoints[analysis_endpoint]["endpoint_url"])

            if response and response.status_code == 200:

                analyses = response.json()

                analysis_resources = DataResource.query.filter_by(file_type=analysis_endpoint).all()

                # Check for missing analyses.
                for record in analyses:
                    submission = get_latest_hepsubmission(inspire_id=record, overall_status='finished')

                    if submission:
                        num_new_resources = 0

                        for analysis in analyses[record]:
                            _resource_url = endpoints[analysis_endpoint]["url_template"].format(analysis)

                            if not is_resource_added_to_submission(submission.publication_recid, submission.version,
                                                                   _resource_url):

                                log.info('Adding {} analysis to ins{} with URL {}'.format(
                                    analysis_endpoint, record, _resource_url)
                                )
                                new_resource = DataResource(
                                    file_location=_resource_url,
                                    file_type=analysis_endpoint)

                                submission.resources.append(new_resource)
                                num_new_resources += 1

                            else:

                                # Remove resources from 'analysis_resources' list.
                                resources = list(filter(lambda a: a.file_location == _resource_url, analysis_resources))
                                for resource in resources:
                                    analysis_resources.remove(resource)

                        if num_new_resources:

                            try:
                                db.session.add(submission)
                                db.session.commit()
                                latest_submission = get_latest_hepsubmission(inspire_id=record)
                                if submission.version == latest_submission.version:
                                    index_record_ids([submission.publication_recid])
                            except Exception as e:
                                db.session.rollback()
                                log.error(e)

                    else:
                        log.debug("An analysis is available in {0} but with no equivalent in HEPData (ins{1}).".format(
                            analysis_endpoint, record))

                if analysis_resources:
                    # Extra resources that were not found in the analyses JSON file.
                    # Need to delete extra resources then reindex affected submissions.
                    # Only take action if latest version is finished (most important case).
                    try:
                        recids_to_reindex = []
                        for extra_analysis_resource in analysis_resources:
                            query = db.select([data_reference_link.columns.submission_id]).where(
                                data_reference_link.columns.dataresource_id == extra_analysis_resource.id)
                            results = db.session.execute(query)
                            for result in results:
                                submission_id = result[0]
                            submission = HEPSubmission.query.filter_by(id=submission_id).first()
                            latest_submission = get_latest_hepsubmission(
                                publication_recid=submission.publication_recid, overall_status='finished'
                            )
                            if submission and latest_submission and submission.version == latest_submission.version:
                                log.info('Removing {} analysis with URL {} from submission {} version {}'
                                         .format(analysis_endpoint, extra_analysis_resource.file_location,
                                                 submission.publication_recid, submission.version))
                                db.session.delete(extra_analysis_resource)
                                recids_to_reindex.append(submission.publication_recid)
                        db.session.commit()
                        if recids_to_reindex:
                            index_record_ids(list(set(recids_to_reindex)))  # remove duplicates before indexing
                    except Exception as e:
                        db.session.rollback()
                        log.error(e)

                # Allow bulk subscription to record update notifications.
                if "subscribe_user_id" in endpoints[analysis_endpoint]:
                    user = get_user_from_id(endpoints[analysis_endpoint]["subscribe_user_id"])
                    if user:
                        for record in analyses:
                            submission = get_latest_hepsubmission(inspire_id=record, overall_status='finished')
                            if submission and not is_current_user_subscribed_to_record(submission.publication_recid, user):
                                subscribe(submission.publication_recid, user)

        else:
            log.debug("No endpoint url configured for {0}".format(analysis_endpoint))

1	# -- coding: utf-8 --
2	#
3	# This file is part of HEPData.
4	# Copyright (C) 2021 CERN.
5	#
6	# HEPData is free software; you can redistribute it
7	# and/or modify it under the terms of the GNU General Public License as
8	# published by the Free Software Foundation; either version 2 of the
9	# License, or (at your option) any later version.
10	#
11	# HEPData is distributed in the hope that it will be
12	# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	# General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with HEPData; if not, write to the
18	# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
19	# MA 02111-1307, USA.
20	#
21	# In applying this license, CERN does not
22	# waive the privileges and immunities granted to it by virtue of its status
23	# as an Intergovernmental Organization or submit itself to any jurisdiction.
24
25	import logging	1✔
26
27	from celery import shared_task	1✔
28	from flask import current_app	1✔
29	from invenio_db import db	1✔
30	import requests	1✔
31
32	from hepdata.ext.opensearch.api import index_record_ids	1✔
33	from hepdata.modules.submission.api import get_latest_hepsubmission, is_resource_added_to_submission	1✔
34	from hepdata.modules.submission.models import DataResource, HEPSubmission, data_reference_link	1✔
35	from hepdata.utils.users import get_user_from_id	1✔
36	from hepdata.modules.records.subscribers.rest import subscribe	1✔
37	from hepdata.modules.records.subscribers.api import is_current_user_subscribed_to_record	1✔
38
39	logging.basicConfig()	1✔
40	log = logging.getLogger(__name__)	1✔
41
42
43	@shared_task	1✔
44	def update_analyses(endpoint=None):	1✔
45	"""
46	Update (Rivet, MadAnalysis 5 and SModelS) analyses and remove outdated resources.
47	Allow bulk subscription to record update notifications if "subscribe_user_id" in endpoint.
48
49	:param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or None (default) for both
50	"""
51	endpoints = current_app.config["ANALYSES_ENDPOINTS"]	1✔
52	for analysis_endpoint in endpoints:	1✔
53
54	if endpoint and endpoint != analysis_endpoint:	1✔
55	continue	1✔
56
57	if "endpoint_url" in endpoints[analysis_endpoint]:	1✔
58
59	log.info("Updating analyses from {0}...".format(analysis_endpoint))	1✔
60
61	response = requests.get(endpoints[analysis_endpoint]["endpoint_url"])	1✔
62
63	if response and response.status_code == 200:	1✔
64
65	analyses = response.json()	1✔
66
67	analysis_resources = DataResource.query.filter_by(file_type=analysis_endpoint).all()	1✔
68
69	# Check for missing analyses.
70	for record in analyses:	1✔
71	submission = get_latest_hepsubmission(inspire_id=record, overall_status='finished')	1✔
72
73	if submission:	1✔
74	num_new_resources = 0	1✔
75
76	for analysis in analyses[record]:	1✔
77	_resource_url = endpoints[analysis_endpoint]["url_template"].format(analysis)	1✔
78
79	if not is_resource_added_to_submission(submission.publication_recid, submission.version,	1✔
80	_resource_url):
81
82	log.info('Adding {} analysis to ins{} with URL {}'.format(	1✔
83	analysis_endpoint, record, _resource_url)
84	)
85	new_resource = DataResource(	1✔
86	file_location=_resource_url,
87	file_type=analysis_endpoint)
88
89	submission.resources.append(new_resource)	1✔
90	num_new_resources += 1	1✔
91
92	else:
93
94	# Remove resources from 'analysis_resources' list.
95	resources = list(filter(lambda a: a.file_location == _resource_url, analysis_resources))	1✔
96	for resource in resources:	1✔
97	analysis_resources.remove(resource)	1✔
98
99	if num_new_resources:	1✔
100
101	try:	1✔
102	db.session.add(submission)	1✔
103	db.session.commit()	1✔
104	latest_submission = get_latest_hepsubmission(inspire_id=record)	1✔
105	if submission.version == latest_submission.version:	1✔
106	index_record_ids([submission.publication_recid])	1✔
107	except Exception as e:	×
108	db.session.rollback()	×
109	log.error(e)	×
110
111	else:
112	log.debug("An analysis is available in {0} but with no equivalent in HEPData (ins{1}).".format(	1✔
113	analysis_endpoint, record))
114
115	if analysis_resources:	1✔
116	# Extra resources that were not found in the analyses JSON file.
117	# Need to delete extra resources then reindex affected submissions.
118	# Only take action if latest version is finished (most important case).
119	try:	1✔
120	recids_to_reindex = []	1✔
121	for extra_analysis_resource in analysis_resources:	1✔
122	query = db.select([data_reference_link.columns.submission_id]).where(	1✔
123	data_reference_link.columns.dataresource_id == extra_analysis_resource.id)
124	results = db.session.execute(query)	1✔
125	for result in results:	1✔
126	submission_id = result[0]	1✔
127	submission = HEPSubmission.query.filter_by(id=submission_id).first()	1✔
128	latest_submission = get_latest_hepsubmission(	1✔
129	publication_recid=submission.publication_recid, overall_status='finished'
130	)
131	if submission and latest_submission and submission.version == latest_submission.version:	1✔
132	log.info('Removing {} analysis with URL {} from submission {} version {}'	1✔
133	.format(analysis_endpoint, extra_analysis_resource.file_location,
134	submission.publication_recid, submission.version))
135	db.session.delete(extra_analysis_resource)	1✔
136	recids_to_reindex.append(submission.publication_recid)	1✔
137	db.session.commit()	1✔
138	if recids_to_reindex:	1✔
139	index_record_ids(list(set(recids_to_reindex))) # remove duplicates before indexing	1✔
140	except Exception as e:	×
141	db.session.rollback()	×
142	log.error(e)	×
143
144	# Allow bulk subscription to record update notifications.
145	if "subscribe_user_id" in endpoints[analysis_endpoint]:	1✔
146	user = get_user_from_id(endpoints[analysis_endpoint]["subscribe_user_id"])	1✔
147	if user:	1✔
148	for record in analyses:	1✔
149	submission = get_latest_hepsubmission(inspire_id=record, overall_status='finished')	1✔
150	if submission and not is_current_user_subscribed_to_record(submission.publication_recid, user):	1✔
151	subscribe(submission.publication_recid, user)	1✔
152
153	else:
154	log.debug("No endpoint url configured for {0}".format(analysis_endpoint))	×

HEPData / hepdata / 12163548050

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous