3251121369

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/adsabs/ADSOrcid/commit/<a class=hub.com/adsabs/ADSOrcid/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSOrcid/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSOrcid/commit/402a4990311a918b505063f43f83ccfff4cfee5b">402a49903<a href="https://github.com/adsabs/ADSOrcid/commit/402a4990311a918b505063f43f83ccfff4cfee5b">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSOrcid/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSOrcid/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/adsabs/ADSOrcid/commit/402a4990311a918b505063f43f83ccfff4cfee5b&quot;&gt;402a49903&lt;/a&gt;&lt;a href=&quot;https://github.com/adsabs/ADSOrcid/commit/402a4990311a918b505063f43f83ccfff4cfee5b&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSOrcid/commit/402a4990311a918b505063f43f83ccff&lt;/a&gt;f4cfee5b">&quot;&gt;Merge &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSOrcid/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSOrcid/commit/<a class="double-link" href="https://github.com/adsabs/ADSOrcid/commit/b24659c762160f21c9be389673d5b5a80961ccf3">b24659c76</a><a href="https://github.com/adsabs/ADSOrcid/commit/402a4990311a918b505063f43f83ccfff4cfee5b">&amp;quot;&amp;gt;b24659c76&amp;lt;/a&amp;gt;&amp;quot;&amp;gt;b24659c76&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSOrcid/commit/402a4990311a918b505063f43f83ccfff4cfe&lt;/a&gt;e5b&quot;&gt; into &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSOrcid/commit/<a class="double-link" href="https://github.com/adsabs/ADSOrcid/commit/7281227d1">7281227d1">7281227d1</a>

Pull Request Pull Request #103: Make sure prior claim is removed if blacklisted

Run Details

4 of 4 new or added lines in 1 file covered. (100.0%)

799 of 991 relevant lines covered (80.63%)

0.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.5

/ADSOrcid/updater.py


"""
Library for updating papers (db claims/records).
"""

from builtins import str
from ADSOrcid import names
from ADSOrcid.models import ClaimsLog, Records
from adsputils import get_date, setup_logging, u2asc
from datetime import timedelta
from sqlalchemy.sql.expression import and_
import Levenshtein
import json
import os
import sys


# ============================= INITIALIZATION ==================================== #
# - Use app logger:
#import logging
#logger = logging.getLogger('orcid-pipeline')
# - Or individual logger for this file:
from adsputils import setup_logging, load_config
proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
config = load_config(proj_home=proj_home)
logger = setup_logging(__name__, proj_home=proj_home,
                        level=config.get('LOGGING_LEVEL', 'INFO'),
                        attach_stdout=config.get('LOG_STDOUT', False))


# =============================== FUNCTIONS ======================================= #

def update_record(rec, claim, min_levenshtein):
    """
    update the ADS Record; we'll add ORCID information into it
    (at the correct position)

    :param: rec - JSON structure, it contains metadata; we expect
            it to have 'authors' field, and 'claims' field

    :param: claim - JSON structure, it contains claim data,
            especially:
                orcidid
                author
                author_norm
            We use those field to find out which author made the
            claim.

    :return: tuple(clain_category, position) or None if no record
        was updated
    """
    assert(isinstance(rec, dict))
    assert(isinstance(claim, dict))
    assert('authors' in rec)
    assert('claims' in rec)
    assert(isinstance(rec['authors'], list))

    claims = rec.get('claims', {})
    rec['claims'] = claims
    authors = rec.get('authors', [])

    # make sure the claims have the necessary structure
    fld_name = u'unverified'
    if 'account_id' in claim and claim['account_id']: # the claim was made by ADS verified user
        fld_name = u'verified'

    num_authors = len(authors)

    if fld_name not in claims or claims[fld_name] is None:
        claims[fld_name] = ['-'] * num_authors
    elif len(claims[fld_name]) < num_authors: # check the length is correct
        claims[fld_name] += ['-'] * (num_authors - len(claims[fld_name]))
    elif len(claims[fld_name]) > num_authors:
        claims[fld_name] = claims[fld_name][0:num_authors]

    # always remove the orcidid
    modified = False
    orcidid = claim['orcidid']
    for v in list(claims.values()):
        while orcidid in v:
            v[v.index(orcidid)] = '-'
            modified = True

    # check to see if claim (ORCID ID + bibcode) is blacklisted
    if rec.get('status') \
            and rec.get('status').get('blacklisted') \
            and claim['orcidid'] in rec.get('status').get('blacklisted'):
        if modified:
            return ('removed', -1)
        else:
            return None

    variant_keys = ('author', 'orcid_name', 'author_norm', 'short_name', 'ascii_name')

    # first check to see if there's an exact name match on the appropriate keys
    claims_clean = set()
    for key in variant_keys:
        for variant in claim.get(key, []):
            if bool(variant.strip()):
                try:
                    claims_clean.add(names.cleanup_name(variant).lower().encode('utf-8'))
                except RuntimeError:
                    # don't add a blank variant to the set
                    continue

    aidx = 0
    for author in rec['authors']:
        try:
            author_clean = names.cleanup_name(author).lower().encode('utf8')
        except RuntimeError:
            # don't add a blank name to the set
            continue
        if author_clean in claims_clean:
            claims[fld_name][aidx] = claim.get('status', 'created') == 'removed' and '-' or orcidid
            return (fld_name, aidx)
        # also try the transliterated/ascii form of the author name
        elif u2asc(author_clean) in claims_clean:
            claims[fld_name][aidx] = claim.get('status', 'created') == 'removed' and '-' or orcidid
            return (fld_name, aidx)
        aidx += 1

    # if there is no exact match, try on Levenshtein distance, searching using descending priority
    for fx in variant_keys:
        if fx in claim and claim[fx]:
            #c = [x for x in claim[fx] if bool(x.strip())]
            assert(isinstance(claim[fx], list))
            idx = find_orcid_position(rec['authors'], claim[fx], min_levenshtein=min_levenshtein)
            if idx > -1:
                if idx >= num_authors:
                    logger.error(u'Index is beyond list boundary: \n' +
                                     u'Field {fx}, author {author}, len(authors)={la}, len({fx})=lfx'
                                     .format(
                                       fx=fx, author=claim[fx], la=num_authors, lfx=len(claim[fx])
                                       )
                                     )
                    continue

                claims[fld_name][idx] = claim.get('status', 'created') == 'removed' and '-' or orcidid
                return (fld_name, idx)

    if modified:
        return ('removed', -1)

def find_orcid_position(authors_list, name_variants,
                        min_levenshtein=0.9):
    """
    Find the position of ORCID in the list of other strings

    :param authors_list - array of names that will be searched
    :param name_variants - array of names of a single author

    :return list of positions that match
    """
    try:
        al = [names.cleanup_name(x).lower().encode('utf8') for x in authors_list]
    except RuntimeError:
        logger.error('Blank author present in author list: %s' % authors_list)
        return -1
    # compute similarity between all authors (and the supplied variants)
    # this is not very efficient, however the lists should be small
    # and short, so 3000 operations take less than 1s)
    res = []
    res_asc = []
    aidx = vidx = 0
    nv = []
    for name in name_variants:
        try:
            variant = names.cleanup_name(name).lower().encode('utf8')
            nv.append(variant)
        except RuntimeError:
            # don't accept a blank name
            continue
        if bool(variant.strip()):
            aidx = 0
            for author in al:
                res.append((Levenshtein.ratio(author, variant), aidx, vidx))
                # check transliterated/ascii form of names in author list if name is different from ascii version
                if u2asc(author) != author:
                    if sys.version_info > (3,):
                        res_asc.append((Levenshtein.ratio(u2asc(author).encode(), variant), aidx, vidx))
                    else:
                        res_asc.append((Levenshtein.ratio(u2asc(author), variant), aidx, vidx))
                else:
                    res_asc.append(res[-1])
                aidx += 1
        vidx += 1

    # sort results from the highest match
    res = sorted(res, key=lambda x: x[0], reverse=True)
    res_asc = sorted(res_asc, key=lambda x: x[0], reverse=True)

    if len(res) == 0:
        return -1

    # if transliterated forms have a higher Lev ratio, accept the transliterated form
    if res_asc[0][0] > res[0][0]:
        res = res_asc

    if res[0][0] < min_levenshtein:
        # test submatch (0.6470588235294118, 19, 0) (required:0.69) closest: vernetto, s, variant: vernetto, silvia teresa
        author_name = al[res[0][1]]
        variant_name = nv[res[0][2]]
        if author_name in variant_name or variant_name in author_name:
            if sys.version_info < (3,):
                logger.debug(u'Using submatch for: %s (required:%s) closest: %s, variant: %s' \
                             % (res[0], min_levenshtein,
                                unicode(author_name, 'utf-8'),
                                unicode(variant_name, 'utf-8')))
            else:
                logger.debug('Using submatch for: %s (required:%s) closest: %s, variant: %s' \
                             % (res[0], min_levenshtein,
                                author_name,
                                variant_name))
            return res[0][1]

        if sys.version_info < (3,):
            logger.debug(u'No match found: the closest is: %s (required:%s) closest: %s, variant: %s' \
                            % (res[0], min_levenshtein,
                               unicode(author_name, 'utf-8'),
                               unicode(variant_name, 'utf-8')))
        else:
            logger.debug('No match found: the closest is: %s (required:%s) closest: %s, variant: %s' \
                         % (res[0], min_levenshtein,
                            author_name,
                            variant_name))
        return -1

    logger.debug('Found match: %s (min_levenstein=%s), authors=%s', authors_list[res[0][1]], min_levenshtein, authors_list)
    return res[0][1]


def _remove_orcid(rec, orcidid):
    """Finds and removes the orcidid from the list of claims.

    :return: True/False if the rec was modified
    """
    modified = False
    claims = rec.get('claims', {})
    for data in list(claims.values()):
        if orcidid in data:
            data[data.index(orcidid)] = '-'
            modified = True
    return modified

def reindex_all_claims(app, orcidid, since=None, ignore_errors=False):
    """
    Procedure that will re-play all claims
    that were modified since a given starting point.
    """

    last_check = get_date(since or '1974-11-09T22:56:52.518001Z')
    recs_modified = set()

    with app.session_scope() as session:
        author = app.retrieve_orcid(orcidid)
        claimed = set()
        removed = set()
        for claim in session.query(ClaimsLog).filter(
                        and_(ClaimsLog.orcidid == orcidid, ClaimsLog.created > last_check)
                        ).all():
            if claim.status in ('claimed', 'updated', 'forced'):
                claimed.add(claim.bibcode)
            elif claim.status == 'removed':
                removed.add(claim.bibcode)

        with app.session_scope() as session:
            for bibcode in removed:
                r = session.query(Records).filter_by(bibcode=bibcode).first()
                if r is None:
                    continue
                rec = r.toJSON()
                if _remove_orcid(rec, orcidid):
                    r.claims = json.dumps(rec.get('claims'))
                    r.updated = get_date()
                    recs_modified.add(bibcode)

            for bibcode in claimed:
                r = session.query(Records).filter_by(bibcode=bibcode).first()
                if r is None:
                    continue
                rec = r.toJSON()

                claim = {'bibcode': bibcode, 'orcidid': orcidid}
                claim.update(author.get('facts', {}))
                try:
                    _claims = update_record(rec, claim, app.conf.get('MIN_LEVENSHTEIN_RATIO', 0.9))
                    if _claims:
                        r.claims = json.dumps(rec.get('claims', {}))
                        r.updated = get_date()
                        recs_modified.add(bibcode)
                except Exception as e:
                    if ignore_errors:
                        app.logger.error(u'Error processing {0} {1}'.format(bibcode, orcidid))
                    else:
                        raise e


            session.commit()

        return list(recs_modified)


def get_all_touched_profiles(app, since='1974-11-09T22:56:52.518001Z', max_failures=5, max_cons_failures=2):
    """Queries the orcid-service for all new/updated
    orcid profiles"""

    orcid_ids = set()
    latest_point = get_date(since) # RFC 3339 format
    failures = cons_failures = 0

    while True:
        # increase the timestamp by one microsec and get new updates
        latest_point = latest_point + timedelta(microseconds=1)
        r = app.client.get(app.conf.get('API_ORCID_UPDATES_ENDPOINT') % latest_point.isoformat(),
                           params={'fields': ['orcid_id', 'updated', 'created']},
                           headers={'Authorization': 'Bearer {0}'.format(app.conf.get('API_TOKEN'))})

        if r.status_code != 200:
            cons_failures += 1
            failures += 1
            if cons_failures < max_cons_failures and failures < max_failures:
                logger.warn('Error querying API enpoint, will retry: failures=%s, cons_failures=%s' % (failures, cons_failures))
                latest_point = latest_point - timedelta(microseconds=1)
                continue
            raise Exception(r.text)

        if r.text.strip() == "":
            break

        # we received the data, immediately update the databaes (so that other processes don't
        # ask for the same starting date)
        data = r.json()

        if len(data) == 0:
            break

        cons_failures = 0 # reset

        # data should be ordered by date update (but to be sure, let's check it); we'll save it
        # as latest 'check point'
        dates = [get_date(x['updated']) for x in data]
        dates = sorted(dates, reverse=True)
        latest_point = dates[0]
        for rec in data:
            orcid_ids.add(rec['orcid_id'])

    return list(orcid_ids)


1
2	"""	1✔
3	Library for updating papers (db claims/records).
4	"""
5
6	from builtins import str	1✔
7	from ADSOrcid import names	1✔
8	from ADSOrcid.models import ClaimsLog, Records	1✔
9	from adsputils import get_date, setup_logging, u2asc	1✔
10	from datetime import timedelta	1✔
11	from sqlalchemy.sql.expression import and_	1✔
12	import Levenshtein	1✔
13	import json	1✔
14	import os	1✔
15	import sys	1✔
16
17
18	# ============================= INITIALIZATION ==================================== #
19	# - Use app logger:
20	#import logging
21	#logger = logging.getLogger('orcid-pipeline')
22	# - Or individual logger for this file:
23	from adsputils import setup_logging, load_config	1✔
24	proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))	1✔
25	config = load_config(proj_home=proj_home)	1✔
26	logger = setup_logging(__name__, proj_home=proj_home,	1✔
27	level=config.get('LOGGING_LEVEL', 'INFO'),
28	attach_stdout=config.get('LOG_STDOUT', False))
29
30
31	# =============================== FUNCTIONS ======================================= #
32
33	def update_record(rec, claim, min_levenshtein):	1✔
34	"""
35	update the ADS Record; we'll add ORCID information into it
36	(at the correct position)
37
38	:param: rec - JSON structure, it contains metadata; we expect
39	it to have 'authors' field, and 'claims' field
40
41	:param: claim - JSON structure, it contains claim data,
42	especially:
43	orcidid
44	author
45	author_norm
46	We use those field to find out which author made the
47	claim.
48
49	:return: tuple(clain_category, position) or None if no record
50	was updated
51	"""
52	assert(isinstance(rec, dict))	1✔
53	assert(isinstance(claim, dict))	1✔
54	assert('authors' in rec)	1✔
55	assert('claims' in rec)	1✔
56	assert(isinstance(rec['authors'], list))	1✔
57
58	claims = rec.get('claims', {})	1✔
59	rec['claims'] = claims	1✔
60	authors = rec.get('authors', [])	1✔
61
62	# make sure the claims have the necessary structure
63	fld_name = u'unverified'	1✔
64	if 'account_id' in claim and claim['account_id']: # the claim was made by ADS verified user	1✔
65	fld_name = u'verified'	1✔
66
67	num_authors = len(authors)	1✔
68
69	if fld_name not in claims or claims[fld_name] is None:	1✔
70	claims[fld_name] = ['-'] * num_authors	1✔
71	elif len(claims[fld_name]) < num_authors: # check the length is correct	1✔
72	claims[fld_name] += ['-'] * (num_authors - len(claims[fld_name]))	1✔
73	elif len(claims[fld_name]) > num_authors:	1✔
74	claims[fld_name] = claims[fld_name][0:num_authors]	×
75
76	# always remove the orcidid
77	modified = False	1✔
78	orcidid = claim['orcidid']	1✔
79	for v in list(claims.values()):	1✔
80	while orcidid in v:	1✔
81	v[v.index(orcidid)] = '-'	1✔
82	modified = True	1✔
83
84	# check to see if claim (ORCID ID + bibcode) is blacklisted
85	if rec.get('status') \	1✔
86	and rec.get('status').get('blacklisted') \
87	and claim['orcidid'] in rec.get('status').get('blacklisted'):
88	if modified:	1✔
89	return ('removed', -1)	×
90	else:
91	return None	1✔
92
93	variant_keys = ('author', 'orcid_name', 'author_norm', 'short_name', 'ascii_name')	1✔
94
95	# first check to see if there's an exact name match on the appropriate keys
96	claims_clean = set()	1✔
97	for key in variant_keys:	1✔
98	for variant in claim.get(key, []):	1✔
99	if bool(variant.strip()):	1✔
100	try:	1✔
101	claims_clean.add(names.cleanup_name(variant).lower().encode('utf-8'))	1✔
102	except RuntimeError:	×
103	# don't add a blank variant to the set
104	continue	×
105
106	aidx = 0	1✔
107	for author in rec['authors']:	1✔
108	try:	1✔
109	author_clean = names.cleanup_name(author).lower().encode('utf8')	1✔
110	except RuntimeError:	×
111	# don't add a blank name to the set
112	continue	×
113	if author_clean in claims_clean:	1✔
114	claims[fld_name][aidx] = claim.get('status', 'created') == 'removed' and '-' or orcidid	1✔
115	return (fld_name, aidx)	1✔
116	# also try the transliterated/ascii form of the author name
117	elif u2asc(author_clean) in claims_clean:	1✔
118	claims[fld_name][aidx] = claim.get('status', 'created') == 'removed' and '-' or orcidid	1✔
119	return (fld_name, aidx)	1✔
120	aidx += 1	1✔
121
122	# if there is no exact match, try on Levenshtein distance, searching using descending priority
123	for fx in variant_keys:	1✔
124	if fx in claim and claim[fx]:	1✔
125	#c = [x for x in claim[fx] if bool(x.strip())]
126	assert(isinstance(claim[fx], list))	1✔
127	idx = find_orcid_position(rec['authors'], claim[fx], min_levenshtein=min_levenshtein)	1✔
128	if idx > -1:	1✔
129	if idx >= num_authors:	1✔
130	logger.error(u'Index is beyond list boundary: \n' +	×
131	u'Field {fx}, author {author}, len(authors)={la}, len({fx})=lfx'
132	.format(
133	fx=fx, author=claim[fx], la=num_authors, lfx=len(claim[fx])
134	)
135	)
136	continue	×
137
138	claims[fld_name][idx] = claim.get('status', 'created') == 'removed' and '-' or orcidid	1✔
139	return (fld_name, idx)	1✔
140
141	if modified:	×
142	return ('removed', -1)	×
143
144	def find_orcid_position(authors_list, name_variants,	1✔
145	min_levenshtein=0.9):
146	"""
147	Find the position of ORCID in the list of other strings
148
149	:param authors_list - array of names that will be searched
150	:param name_variants - array of names of a single author
151
152	:return list of positions that match
153	"""
154	try:	1✔
155	al = [names.cleanup_name(x).lower().encode('utf8') for x in authors_list]	1✔
156	except RuntimeError:	×
157	logger.error('Blank author present in author list: %s' % authors_list)	×
158	return -1	×
159	# compute similarity between all authors (and the supplied variants)
160	# this is not very efficient, however the lists should be small
161	# and short, so 3000 operations take less than 1s)
162	res = []	1✔
163	res_asc = []	1✔
164	aidx = vidx = 0	1✔
165	nv = []	1✔
166	for name in name_variants:	1✔
167	try:	1✔
168	variant = names.cleanup_name(name).lower().encode('utf8')	1✔
169	nv.append(variant)	1✔
170	except RuntimeError:	1✔
171	# don't accept a blank name
172	continue	1✔
173	if bool(variant.strip()):	1✔
174	aidx = 0	1✔
175	for author in al:	1✔
176	res.append((Levenshtein.ratio(author, variant), aidx, vidx))	1✔
177	# check transliterated/ascii form of names in author list if name is different from ascii version
178	if u2asc(author) != author:	1✔
179	if sys.version_info > (3,):	1✔
180	res_asc.append((Levenshtein.ratio(u2asc(author).encode(), variant), aidx, vidx))	×
181	else:
182	res_asc.append((Levenshtein.ratio(u2asc(author), variant), aidx, vidx))	1✔
183	else:
184	res_asc.append(res[-1])	1✔
185	aidx += 1	1✔
186	vidx += 1	1✔
187
188	# sort results from the highest match
189	res = sorted(res, key=lambda x: x[0], reverse=True)	1✔
190	res_asc = sorted(res_asc, key=lambda x: x[0], reverse=True)	1✔
191
192	if len(res) == 0:	1✔
193	return -1	1✔
194
195	# if transliterated forms have a higher Lev ratio, accept the transliterated form
196	if res_asc[0][0] > res[0][0]:	1✔
197	res = res_asc	1✔
198
199	if res[0][0] < min_levenshtein:	1✔
200	# test submatch (0.6470588235294118, 19, 0) (required:0.69) closest: vernetto, s, variant: vernetto, silvia teresa
201	author_name = al[res[0][1]]	1✔
202	variant_name = nv[res[0][2]]	1✔
203	if author_name in variant_name or variant_name in author_name:	1✔
204	if sys.version_info < (3,):	1✔
205	logger.debug(u'Using submatch for: %s (required:%s) closest: %s, variant: %s' \	1✔
206	% (res[0], min_levenshtein,
207	unicode(author_name, 'utf-8'),
208	unicode(variant_name, 'utf-8')))
209	else:
210	logger.debug('Using submatch for: %s (required:%s) closest: %s, variant: %s' \	×
211	% (res[0], min_levenshtein,
212	author_name,
213	variant_name))
214	return res[0][1]	1✔
215
216	if sys.version_info < (3,):	1✔
217	logger.debug(u'No match found: the closest is: %s (required:%s) closest: %s, variant: %s' \	1✔
218	% (res[0], min_levenshtein,
219	unicode(author_name, 'utf-8'),
220	unicode(variant_name, 'utf-8')))
221	else:
222	logger.debug('No match found: the closest is: %s (required:%s) closest: %s, variant: %s' \	×
223	% (res[0], min_levenshtein,
224	author_name,
225	variant_name))
226	return -1	1✔
227
228	logger.debug('Found match: %s (min_levenstein=%s), authors=%s', authors_list[res[0][1]], min_levenshtein, authors_list)	1✔
229	return res[0][1]	1✔
230
231
232	def _remove_orcid(rec, orcidid):	1✔
233	"""Finds and removes the orcidid from the list of claims.
234
235	:return: True/False if the rec was modified
236	"""
237	modified = False	1✔
238	claims = rec.get('claims', {})	1✔
239	for data in list(claims.values()):	1✔
240	if orcidid in data:	1✔
241	data[data.index(orcidid)] = '-'	1✔
242	modified = True	1✔
243	return modified	1✔
244
245	def reindex_all_claims(app, orcidid, since=None, ignore_errors=False):	1✔
246	"""
247	Procedure that will re-play all claims
248	that were modified since a given starting point.
249	"""
250
251	last_check = get_date(since or '1974-11-09T22:56:52.518001Z')	1✔
252	recs_modified = set()	1✔
253
254	with app.session_scope() as session:	1✔
255	author = app.retrieve_orcid(orcidid)	1✔
256	claimed = set()	1✔
257	removed = set()	1✔
258	for claim in session.query(ClaimsLog).filter(	1✔
259	and_(ClaimsLog.orcidid == orcidid, ClaimsLog.created > last_check)
260	).all():
261	if claim.status in ('claimed', 'updated', 'forced'):	1✔
262	claimed.add(claim.bibcode)	1✔
263	elif claim.status == 'removed':	1✔
264	removed.add(claim.bibcode)	1✔
265
266	with app.session_scope() as session:	1✔
267	for bibcode in removed:	1✔
268	r = session.query(Records).filter_by(bibcode=bibcode).first()	1✔
269	if r is None:	1✔
270	continue	×
271	rec = r.toJSON()	1✔
272	if _remove_orcid(rec, orcidid):	1✔
273	r.claims = json.dumps(rec.get('claims'))	1✔
274	r.updated = get_date()	1✔
275	recs_modified.add(bibcode)	1✔
276
277	for bibcode in claimed:	1✔
278	r = session.query(Records).filter_by(bibcode=bibcode).first()	1✔
279	if r is None:	1✔
280	continue	×
281	rec = r.toJSON()	1✔
282
283	claim = {'bibcode': bibcode, 'orcidid': orcidid}	1✔
284	claim.update(author.get('facts', {}))	1✔
285	try:	1✔
286	_claims = update_record(rec, claim, app.conf.get('MIN_LEVENSHTEIN_RATIO', 0.9))	1✔
287	if _claims:	1✔
288	r.claims = json.dumps(rec.get('claims', {}))	1✔
289	r.updated = get_date()	1✔
290	recs_modified.add(bibcode)	1✔
291	except Exception as e:	×
292	if ignore_errors:	×
293	app.logger.error(u'Error processing {0} {1}'.format(bibcode, orcidid))	×
294	else:
295	raise e	×
296
297
298	session.commit()	1✔
299
300	return list(recs_modified)	1✔
301
302
303	def get_all_touched_profiles(app, since='1974-11-09T22:56:52.518001Z', max_failures=5, max_cons_failures=2):	1✔
304	"""Queries the orcid-service for all new/updated
305	orcid profiles"""
306
307	orcid_ids = set()	×
308	latest_point = get_date(since) # RFC 3339 format	×
309	failures = cons_failures = 0	×
310
311	while True:
312	# increase the timestamp by one microsec and get new updates
313	latest_point = latest_point + timedelta(microseconds=1)	×
314	r = app.client.get(app.conf.get('API_ORCID_UPDATES_ENDPOINT') % latest_point.isoformat(),	×
315	params={'fields': ['orcid_id', 'updated', 'created']},
316	headers={'Authorization': 'Bearer {0}'.format(app.conf.get('API_TOKEN'))})
317
318	if r.status_code != 200:	×
319	cons_failures += 1	×
320	failures += 1	×
321	if cons_failures < max_cons_failures and failures < max_failures:	×
322	logger.warn('Error querying API enpoint, will retry: failures=%s, cons_failures=%s' % (failures, cons_failures))	×
323	latest_point = latest_point - timedelta(microseconds=1)	×
324	continue	×
325	raise Exception(r.text)	×
326
327	if r.text.strip() == "":	×
328	break	×
329
330	# we received the data, immediately update the databaes (so that other processes don't
331	# ask for the same starting date)
332	data = r.json()	×
333
334	if len(data) == 0:	×
335	break	×
336
337	cons_failures = 0 # reset	×
338
339	# data should be ordered by date update (but to be sure, let's check it); we'll save it
340	# as latest 'check point'
341	dates = [get_date(x['updated']) for x in data]	×
342	dates = sorted(dates, reverse=True)	×
343	latest_point = dates[0]	×
344	for rec in data:	×
345	orcid_ids.add(rec['orcid_id'])	×
346
347	return list(orcid_ids)	×
348

adsabs / ADSOrcid / 3251121369

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous