1031008916

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/adsabs/ADSCitationCapture/commit/<a class=hub.com/adsabs/ADSCitationCapture/commit/9ce5eb4c8af9a7aa8b9937fa1523d691b94862c0">9ce5eb4c8<a href="https://github.com/adsabs/ADSCitationCapture/commit/9ce5eb4c8af9a7aa8b9937fa1523d691b94862c0">">Merge </a><a class="double-link" href="https://github.com/adsabs/ADSCitationCapture/commit/<a class="double-link" href="https://github.com/adsabs/ADSCitationCapture/commit/43c0dc82543b99fcab80158a3fef7cc320e32fb1">43c0dc825</a>">43c0dc825</a><a href="https://github.com/adsabs/ADSCitationCapture/commit/9ce5eb4c8af9a7aa8b9937fa1523d691b94862c0"> into 1be5ae594">1be5ae594</a>

Pull Request Pull Request #37: changes for python 3

Run Details

51 of 51 new or added lines in 8 files covered. (100.0%)

834 of 1213 relevant lines covered (68.76%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

13.46

/ADSCitationCapture/api.py


import os
import requests
import urllib.request, urllib.parse, urllib.error
import math
from adsputils import setup_logging

# ============================= INITIALIZATION ==================================== #
# - Use app logger:
#import logging
#logger = logging.getLogger('ads-citation-capture')
# - Or individual logger for this file:
from adsputils import setup_logging, load_config
proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
config = load_config(proj_home=proj_home)
logger = setup_logging(__name__, proj_home=proj_home,
                        level=config.get('LOGGING_LEVEL', 'INFO'),
                        attach_stdout=config.get('LOG_STDOUT', False))


# =============================== FUNCTIONS ======================================= #
def _request_citations_page(app, bibcode, start, rows):
    params = urllib.parse.urlencode({
                'fl': 'bibcode',
                'q': 'citations(bibcode:{0})'.format(bibcode),
                'start': start,
                'rows': rows,
                'sort': 'date desc, bibcode desc',
            })
    headers = {}
    headers["Authorization"] = "Bearer:{}".format(app.conf['ADS_API_TOKEN'])
    url = app.conf['ADS_API_URL']+"search/query?"+params
    r_json = {}
    try:
        r = requests.get(url, headers=headers)
    except:
        logger.error("Search API request failed for citations (start: %i): %s", start, bibcode)
        raise
    if not r.ok:
        msg = "Search API request with error code '{}' for bibcode (start: {}): {}".format(r.status_code, start, bibcode)
        logger.error(msg)
        raise Exception(msg)
    else:
        try:
            r_json = r.json()
        except ValueError:
            msg = "No JSON object could be decoded from Search API response when searching canonical bibcodes (start: {}) for: {}".format(start, bibcode)
            logger.error(msg)
            raise Exception(msg)
        else:
            return r_json
    return r_json

def request_existing_citations(app, bibcode):
    start = 0
    rows = 25
    existing_citation_bibcodes = []
    n_existing_citations = None
    while True:
        retries = 0
        while True:
            try:
                answer = _request_citations_page(app, bibcode, start, rows)
            except:
                if retries < 3:
                    logger.info("Retrying Search API request for citations (start: %i): %s", start, bibcode)
                    retries += 1
                else:
                    logger.exception("Failed Search API request for citations (start: %i): %s", start, bibcode)
                    raise
            else:
                break
        existing_citation_bibcodes += answer['response']['docs']
        if n_existing_citations is None:
            n_existing_citations = answer['response']['numFound']
        start += rows
        if start > int(n_existing_citations):
            break
    # Transform from list of dict to list of bibcodes:
    existing_citation_bibcodes = [b['bibcode'] for b in existing_citation_bibcodes]
    return existing_citation_bibcodes

def get_canonical_bibcodes(app, bibcodes, timeout=30):
    """
    Convert input bibcodes into their canonical form if they exist, hence
    the returned list can be smaller than the input bibcode list
    """
    chunk_size = 2000 # Max number of records supported by bigquery
    bibcodes_chunks = [bibcodes[i * chunk_size:(i + 1) * chunk_size] for i in range(int(round(((len(bibcodes) + chunk_size - 1))) / chunk_size ))]
    canonical_bibcodes = []
    total_n_chunks = len(bibcodes_chunks)
    # Execute multiple requests to bigquery if the list of bibcodes is longer than the accepted maximum
    for n_chunk, bibcodes_chunk in enumerate(bibcodes_chunks):
        retries = 0
        while True:
            try:
                canonical_bibcodes += _get_canonical_bibcodes(app, n_chunk, total_n_chunks, bibcodes_chunk, timeout)
            except:
                if retries < 3:
                    logger.info("Retrying BigQuery API request for bibcodes (chunk: %i/%i): %s", n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))
                    retries += 1
                else:
                    logger.exception("Failed BigQuery API request for bibcodes (chunk: %i/%i): %s", n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))
                    raise
            else:
                break
    return canonical_bibcodes

def _get_canonical_bibcodes(app, n_chunk, total_n_chunks, bibcodes_chunk, timeout):
    canonical_bibcodes = []
    params = urllib.parse.urlencode({
                'fl': 'bibcode',
                'q': '*:*',
                'wt': 'json',
                'fq':'{!bitset}',
                'rows': len(bibcodes_chunk),
            })
    headers = {}
    headers["Authorization"] = "Bearer:{}".format(app.conf['ADS_API_TOKEN'])
    headers["Content-Type"] = "big-query/csv"
    url = app.conf['ADS_API_URL']+"search/bigquery?"+params
    r_json = {}
    data = "bibcode\n" + "\n".join(bibcodes_chunk)
    try:
        r = requests.post(url, headers=headers, data=data, timeout=timeout)
    except:
        logger.error("BigQuery API request failed for bibcodes (chunk: %i/%i): %s", n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))
        raise
    if not r.ok:
        msg = "BigQuery API request with error code '{}' for bibcodes (chunk: {}/{}): {}".format(r.status_code, n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))
        logger.error(msg)
        raise Exception(msg)
    else:
        try:
            r_json = r.json()
        except ValueError:
            msg = "No JSON object could be decoded from BigQuery API response when searching canonical bibcodes (chunk: {}/{}) for: {}".format(n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))
            logger.error(msg)
            raise Exception(msg)
        else:
            for paper in r_json.get('response', {}).get('docs', []):
                canonical_bibcodes.append(paper['bibcode'])
    return canonical_bibcodes

def get_canonical_bibcode(app, bibcode, timeout=30):
    """
    Convert input bibcodes into their canonical form if they exist
    """
    canonical = get_canonical_bibcodes(app, [bibcode], timeout=timeout)
    if len(canonical) == 0:
        return None
    else:
        return canonical[0]

1
2	import os	1✔
3	import requests	1✔
4	import urllib.request, urllib.parse, urllib.error	1✔
5	import math	1✔
6	from adsputils import setup_logging	1✔
7
8	# ============================= INITIALIZATION ==================================== #
9	# - Use app logger:
10	#import logging
11	#logger = logging.getLogger('ads-citation-capture')
12	# - Or individual logger for this file:
13	from adsputils import setup_logging, load_config	1✔
14	proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))	1✔
15	config = load_config(proj_home=proj_home)	1✔
16	logger = setup_logging(__name__, proj_home=proj_home,	1✔
17	level=config.get('LOGGING_LEVEL', 'INFO'),
18	attach_stdout=config.get('LOG_STDOUT', False))
19
20
21	# =============================== FUNCTIONS ======================================= #
22	def _request_citations_page(app, bibcode, start, rows):	1✔
23	params = urllib.parse.urlencode({	×
24	'fl': 'bibcode',
25	'q': 'citations(bibcode:{0})'.format(bibcode),
26	'start': start,
27	'rows': rows,
28	'sort': 'date desc, bibcode desc',
29	})
30	headers = {}	×
31	headers["Authorization"] = "Bearer:{}".format(app.conf['ADS_API_TOKEN'])	×
32	url = app.conf['ADS_API_URL']+"search/query?"+params	×
33	r_json = {}	×
34	try:	×
35	r = requests.get(url, headers=headers)	×
36	except:	×
37	logger.error("Search API request failed for citations (start: %i): %s", start, bibcode)	×
38	raise	×
39	if not r.ok:	×
40	msg = "Search API request with error code '{}' for bibcode (start: {}): {}".format(r.status_code, start, bibcode)	×
41	logger.error(msg)	×
42	raise Exception(msg)	×
43	else:
44	try:	×
45	r_json = r.json()	×
46	except ValueError:	×
47	msg = "No JSON object could be decoded from Search API response when searching canonical bibcodes (start: {}) for: {}".format(start, bibcode)	×
48	logger.error(msg)	×
49	raise Exception(msg)	×
50	else:
51	return r_json	×
52	return r_json	×
53
54	def request_existing_citations(app, bibcode):	1✔
55	start = 0	×
56	rows = 25	×
57	existing_citation_bibcodes = []	×
58	n_existing_citations = None	×
59	while True:
60	retries = 0	×
61	while True:
62	try:	×
63	answer = _request_citations_page(app, bibcode, start, rows)	×
64	except:	×
65	if retries < 3:	×
66	logger.info("Retrying Search API request for citations (start: %i): %s", start, bibcode)	×
67	retries += 1	×
68	else:
69	logger.exception("Failed Search API request for citations (start: %i): %s", start, bibcode)	×
70	raise	×
71	else:
72	break	×
73	existing_citation_bibcodes += answer['response']['docs']	×
74	if n_existing_citations is None:	×
75	n_existing_citations = answer['response']['numFound']	×
76	start += rows	×
77	if start > int(n_existing_citations):	×
78	break	×
79	# Transform from list of dict to list of bibcodes:
80	existing_citation_bibcodes = [b['bibcode'] for b in existing_citation_bibcodes]	×
81	return existing_citation_bibcodes	×
82
83	def get_canonical_bibcodes(app, bibcodes, timeout=30):	1✔
84	"""
85	Convert input bibcodes into their canonical form if they exist, hence
86	the returned list can be smaller than the input bibcode list
87	"""
88	chunk_size = 2000 # Max number of records supported by bigquery	×
89	bibcodes_chunks = [bibcodes[i * chunk_size:(i + 1) * chunk_size] for i in range(int(round(((len(bibcodes) + chunk_size - 1))) / chunk_size ))]	×
90	canonical_bibcodes = []	×
91	total_n_chunks = len(bibcodes_chunks)	×
92	# Execute multiple requests to bigquery if the list of bibcodes is longer than the accepted maximum
93	for n_chunk, bibcodes_chunk in enumerate(bibcodes_chunks):	×
94	retries = 0	×
95	while True:
96	try:	×
97	canonical_bibcodes += _get_canonical_bibcodes(app, n_chunk, total_n_chunks, bibcodes_chunk, timeout)	×
98	except:	×
99	if retries < 3:	×
100	logger.info("Retrying BigQuery API request for bibcodes (chunk: %i/%i): %s", n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))	×
101	retries += 1	×
102	else:
103	logger.exception("Failed BigQuery API request for bibcodes (chunk: %i/%i): %s", n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))	×
104	raise	×
105	else:
106	break	×
107	return canonical_bibcodes	×
108
109	def _get_canonical_bibcodes(app, n_chunk, total_n_chunks, bibcodes_chunk, timeout):	1✔
110	canonical_bibcodes = []	×
111	params = urllib.parse.urlencode({	×
112	'fl': 'bibcode',
113	'q': ':',
114	'wt': 'json',
115	'fq':'{!bitset}',
116	'rows': len(bibcodes_chunk),
117	})
118	headers = {}	×
119	headers["Authorization"] = "Bearer:{}".format(app.conf['ADS_API_TOKEN'])	×
120	headers["Content-Type"] = "big-query/csv"	×
121	url = app.conf['ADS_API_URL']+"search/bigquery?"+params	×
122	r_json = {}	×
123	data = "bibcode\n" + "\n".join(bibcodes_chunk)	×
124	try:	×
125	r = requests.post(url, headers=headers, data=data, timeout=timeout)	×
126	except:	×
127	logger.error("BigQuery API request failed for bibcodes (chunk: %i/%i): %s", n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))	×
128	raise	×
129	if not r.ok:	×
130	msg = "BigQuery API request with error code '{}' for bibcodes (chunk: {}/{}): {}".format(r.status_code, n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))	×
131	logger.error(msg)	×
132	raise Exception(msg)	×
133	else:
134	try:	×
135	r_json = r.json()	×
136	except ValueError:	×
137	msg = "No JSON object could be decoded from BigQuery API response when searching canonical bibcodes (chunk: {}/{}) for: {}".format(n_chunk+1, total_n_chunks, " ".join(bibcodes_chunk))	×
138	logger.error(msg)	×
139	raise Exception(msg)	×
140	else:
141	for paper in r_json.get('response', {}).get('docs', []):	×
142	canonical_bibcodes.append(paper['bibcode'])	×
143	return canonical_bibcodes	×
144
145	def get_canonical_bibcode(app, bibcode, timeout=30):	1✔
146	"""
147	Convert input bibcodes into their canonical form if they exist
148	"""
149	canonical = get_canonical_bibcodes(app, [bibcode], timeout=timeout)	×
150	if len(canonical) == 0:	×
151	return None	×
152	else:
153	return canonical[0]	×

adsabs / ADSCitationCapture / 1031008916

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous