1288102057

Build # 1288102057

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/adsabs/ADSReferencePipeline/commit/<a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/236f4a11628dc59e457de260d02aae53bb970638">236f4a116">&lt;a href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/</a><a class="double-link" href="https://github.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/236f4a11628dc59e457de260d02aae53bb970638">236f4a116</a><a href="https://github.com/adsabs/ADSReferencePipeline/commit/236f4a11628dc59e457de260d02aae53bb970638">&lt;a href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/236f4a11628dc59e457de260d02aae53bb970638&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/adsabs/ADSReferencePipeline/commit/236f4a11628dc59e457de260d02aae53bb970638&amp;quot;&amp;gt;236f4a116&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/236f4a11628dc59e457de260d02aae53bb970638&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/236f4a11628dc59e457de260d02aae53&amp;lt;/a&amp;gt;bb970638&amp;quot;&amp;</a>gt;&amp;quot;&amp;gt;Merge &amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/adsabs/ADSReferencePipeli... (continued)

Pull Request Pull Request #1: the first

Run Details

3139 of 3139 new or added lines in 25 files covered. (100.0%)

2662 of 3139 relevant lines covered (84.8%)

0.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.88

/adsrefpipe/utils.py


from builtins import zip
from builtins import str
from os import path
import time
import re
import json
import requests

from adsputils import setup_logging, load_config

logger = setup_logging('reference-pipeline')
config = {}
config.update(load_config())


DATE_FORMAT = "%d-%02d-%02d %02d:%02d:%02d"


class ReprocessQueryType:
    score, bibstem, year, failed = range(4)


def get_date_created(filename):
    """

    :param filename:
    :return: file's created date in the format YYYY/MM/DD HH:MM:SS
    """
    return DATE_FORMAT%(time.localtime(path.getctime(filename))[:-3])


def get_date_modified(filename):
    """

    :param filename:
    :return: file's modified date in the format YYYY/MM/DD HH:MM:SS
    """
    return DATE_FORMAT%(time.localtime(path.getmtime(filename))[:-3])


def get_date_modified_struct_time(filename):
    """

    :param filename:
    :return: file's modified date in the time.struct_time format
    """
    return time.localtime(path.getmtime(filename))

def get_date_now():
    """

    :return: current time in the format YYYY/MM/DD HH:MM:SS
    """
    return DATE_FORMAT%(time.localtime(time.time())[:-3])

def get_resolved_filename(source_filename):
    """

    :param source_filename:
    :return: resolved file name to save the result of service resolved references
    """
    return source_filename.replace('sources','retrieve') + '.result'


def get_resolved_references(references, service_url):
    """
    send a request to reference service

    :param references: list of references
    :param service_url
    :return:
    """
    if service_url.endswith('text'):
        str_references,ids = map(list, zip(*[ref.values() for ref in references]))
        payload = {'reference': str_references, 'id': ids}
    elif service_url.endswith('xml'):
        payload = {'parsed_reference': references}
    else:
        logger.error('Unrecognizable service url `%s`.'%service_url)
        return None

    headers = {'Content-type': 'application/json',
               'Accept': 'application/json',
               'Authorization': 'Bearer ' + config['REFERENCE_PIPELINE_ADSWS_API_TOKEN']}
    try:
        r = requests.post(url=service_url, data=json.dumps(payload), headers=headers)
        if (r.status_code == 200):
            resolved = json.loads(r.content)['resolved']
            logger.debug('Resolved %d references successfully.' % (len(resolved)))
            return resolved
        logger.error('Attempt at resolving %d %s references failed with status code %s.' % (len(references), type, r.status_code))
        return None
    except requests.exceptions.RequestException as e:
        logger.error('Unable to connect to the service: %s'%str(e))
        return None


def read_classic_resolved_file(source_bibcode, filename):
    """
    read classic resolved file

    :param source_bibcode:
    :param filename:
    :return:
    """
    try:
        resolved = []
        found = False
        with open(filename, 'r', encoding='utf-8') as f:
            for line in f:
                if found:
                    if line.startswith('---<'):
                        break
                    resolved.append(line)
                else:
                    if line.startswith('---<') and line[4:-5] == source_bibcode:
                        found = True
        logger.debug('Read %d references from classic resolved file %s successfully.' % (len(resolved), filename))
        return resolved
    except:
        logger.error('Unable to read references from classic resolved file %s.' % (filename))
        return None

def get_compare_state(service_bibcode, classic_bibcode, classic_score):
    """
    compare service and classic resolved bibcodes and return descriptive state
    :param service_bibcode:
    :param classic_bibcode:
    :param classic_score:
    :return:
    """
    not_found = '.' * 19

    if classic_score == u'5':
        if service_bibcode == not_found:
            return 'UNVER'
        # if service_bibcode != not_found:
        return 'NEWU'
    if classic_score == u'1':
        if classic_bibcode.upper() == service_bibcode.upper():
            return 'MATCH'
        if service_bibcode != not_found:
            return 'DIFF'
        if service_bibcode == not_found:
            return 'MISS'
    if classic_score == u'0':
        if service_bibcode != not_found:
            return 'NEW'
        return 'NONE'

def compare_classic_and_service(service, source_bibcode, classic_filename):
    """
    compare the result of service and classic resolved references

    :param service: resolved references from service, in dict structure
    :param source_bibcode:
    :param classic: resolved references from classic, string format
    :return:
    """
    classic = read_classic_resolved_file(source_bibcode, classic_filename)
    if not classic:
        return None

    compare = []
    if len(service) == len(classic):
        for i, (s, c) in enumerate(zip(service, classic)):
            service_bibcode = s.get('bibcode', None)
            classic_bibcode = c[2:21].strip()
            classic_score = c[0]

            state = get_compare_state(service_bibcode, classic_bibcode, classic_score)
            compare.append((i+1, classic_bibcode, classic_score, state))
    else:
        for i, s in enumerate(service):
            service_reference = s.get('refstring', '')
            # just in case reference is enumerated, remove the enumeration
            service_reference = service_reference[service_reference.find(']') + 1:] if service_reference.startswith(' [') else service_reference
            service_bibcode = s.get('bibcode', '')
            for c in classic:
                classic_reference = c[24:-1].strip()
                classic_bibcode = c[2:21]
                classic_score = c[0]
                if (service_reference == classic_reference) or (service_bibcode == classic_bibcode and service_bibcode != '.' * 19):
                    state = get_compare_state(service_bibcode, classic_bibcode, classic_score)
                    compare.append((i+1, classic_bibcode, classic_score, state))
                    classic.remove(c)
                    break
    return compare

1
2	from builtins import zip	1✔
3	from builtins import str	1✔
4	from os import path	1✔
5	import time	1✔
6	import re	1✔
7	import json	1✔
8	import requests	1✔
9
10	from adsputils import setup_logging, load_config	1✔
11
12	logger = setup_logging('reference-pipeline')	1✔
13	config = {}	1✔
14	config.update(load_config())	1✔
15
16
17	DATE_FORMAT = "%d-%02d-%02d %02d:%02d:%02d"	1✔
18
19
20	class ReprocessQueryType:	1✔
21	score, bibstem, year, failed = range(4)	1✔
22
23
24	def get_date_created(filename):	1✔
25	"""
26
27	:param filename:
28	:return: file's created date in the format YYYY/MM/DD HH:MM:SS
29	"""
30	return DATE_FORMAT%(time.localtime(path.getctime(filename))[:-3])	×
31
32
33	def get_date_modified(filename):	1✔
34	"""
35
36	:param filename:
37	:return: file's modified date in the format YYYY/MM/DD HH:MM:SS
38	"""
39	return DATE_FORMAT%(time.localtime(path.getmtime(filename))[:-3])	1✔
40
41
42	def get_date_modified_struct_time(filename):	1✔
43	"""
44
45	:param filename:
46	:return: file's modified date in the time.struct_time format
47	"""
48	return time.localtime(path.getmtime(filename))	×
49
50	def get_date_now():	1✔
51	"""
52
53	:return: current time in the format YYYY/MM/DD HH:MM:SS
54	"""
55	return DATE_FORMAT%(time.localtime(time.time())[:-3])	1✔
56
57	def get_resolved_filename(source_filename):	1✔
58	"""
59
60	:param source_filename:
61	:return: resolved file name to save the result of service resolved references
62	"""
63	return source_filename.replace('sources','retrieve') + '.result'	1✔
64
65
66	def get_resolved_references(references, service_url):	1✔
67	"""
68	send a request to reference service
69
70	:param references: list of references
71	:param service_url
72	:return:
73	"""
74	if service_url.endswith('text'):	×
75	str_references,ids = map(list, zip(*[ref.values() for ref in references]))	×
76	payload = {'reference': str_references, 'id': ids}	×
77	elif service_url.endswith('xml'):	×
78	payload = {'parsed_reference': references}	×
79	else:
80	logger.error('Unrecognizable service url `%s`.'%service_url)	×
81	return None	×
82
83	headers = {'Content-type': 'application/json',	×
84	'Accept': 'application/json',
85	'Authorization': 'Bearer ' + config['REFERENCE_PIPELINE_ADSWS_API_TOKEN']}
86	try:	×
87	r = requests.post(url=service_url, data=json.dumps(payload), headers=headers)	×
88	if (r.status_code == 200):	×
89	resolved = json.loads(r.content)['resolved']	×
90	logger.debug('Resolved %d references successfully.' % (len(resolved)))	×
91	return resolved	×
92	logger.error('Attempt at resolving %d %s references failed with status code %s.' % (len(references), type, r.status_code))	×
93	return None	×
94	except requests.exceptions.RequestException as e:	×
95	logger.error('Unable to connect to the service: %s'%str(e))	×
96	return None	×
97
98
99	def read_classic_resolved_file(source_bibcode, filename):	1✔
100	"""
101	read classic resolved file
102
103	:param source_bibcode:
104	:param filename:
105	:return:
106	"""
107	try:	1✔
108	resolved = []	1✔
109	found = False	1✔
110	with open(filename, 'r', encoding='utf-8') as f:	1✔
111	for line in f:	1✔
112	if found:	1✔
113	if line.startswith('---<'):	1✔
114	break	×
115	resolved.append(line)	1✔
116	else:
117	if line.startswith('---<') and line[4:-5] == source_bibcode:	1✔
118	found = True	1✔
119	logger.debug('Read %d references from classic resolved file %s successfully.' % (len(resolved), filename))	1✔
120	return resolved	1✔
121	except:	×
122	logger.error('Unable to read references from classic resolved file %s.' % (filename))	×
123	return None	×
124
125	def get_compare_state(service_bibcode, classic_bibcode, classic_score):	1✔
126	"""
127	compare service and classic resolved bibcodes and return descriptive state
128	:param service_bibcode:
129	:param classic_bibcode:
130	:param classic_score:
131	:return:
132	"""
133	not_found = '.' * 19	1✔
134
135	if classic_score == u'5':	1✔
136	if service_bibcode == not_found:	×
137	return 'UNVER'	×
138	# if service_bibcode != not_found:
139	return 'NEWU'	×
140	if classic_score == u'1':	1✔
141	if classic_bibcode.upper() == service_bibcode.upper():	1✔
142	return 'MATCH'	×
143	if service_bibcode != not_found:	1✔
144	return 'DIFF'	1✔
145	if service_bibcode == not_found:	×
146	return 'MISS'	×
147	if classic_score == u'0':	×
148	if service_bibcode != not_found:	×
149	return 'NEW'	×
150	return 'NONE'	×
151
152	def compare_classic_and_service(service, source_bibcode, classic_filename):	1✔
153	"""
154	compare the result of service and classic resolved references
155
156	:param service: resolved references from service, in dict structure
157	:param source_bibcode:
158	:param classic: resolved references from classic, string format
159	:return:
160	"""
161	classic = read_classic_resolved_file(source_bibcode, classic_filename)	1✔
162	if not classic:	1✔
163	return None	×
164
165	compare = []	1✔
166	if len(service) == len(classic):	1✔
167	for i, (s, c) in enumerate(zip(service, classic)):	1✔
168	service_bibcode = s.get('bibcode', None)	1✔
169	classic_bibcode = c[2:21].strip()	1✔
170	classic_score = c[0]	1✔
171
172	state = get_compare_state(service_bibcode, classic_bibcode, classic_score)	1✔
173	compare.append((i+1, classic_bibcode, classic_score, state))	1✔
174	else:
175	for i, s in enumerate(service):	×
176	service_reference = s.get('refstring', '')	×
177	# just in case reference is enumerated, remove the enumeration
178	service_reference = service_reference[service_reference.find(']') + 1:] if service_reference.startswith(' [') else service_reference	×
179	service_bibcode = s.get('bibcode', '')	×
180	for c in classic:	×
181	classic_reference = c[24:-1].strip()	×
182	classic_bibcode = c[2:21]	×
183	classic_score = c[0]	×
184	if (service_reference == classic_reference) or (service_bibcode == classic_bibcode and service_bibcode != '.' * 19):	×
185	state = get_compare_state(service_bibcode, classic_bibcode, classic_score)	×
186	compare.append((i+1, classic_bibcode, classic_score, state))	×
187	classic.remove(c)	×
188	break	×
189	return compare	1✔

adsabs / ADSReferencePipeline / 1288102057

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous