1136300255

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/adsabs/ADSReferencePipeline/commit/<a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/a0c0077548543560380efc8c49cb6b67ca6f27c3">a0c007754<a href="https://github.com/adsabs/ADSReferencePipeline/commit/a0c0077548543560380efc8c49cb6b67ca6f27c3">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/adsabs/ADSReferencePipeline/commit/a0c0077548543560380efc8c49cb6b67ca6f27c3&quot;&gt;a0c007754&lt;/a&gt;&lt;a href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/a0c0077548543560380efc8c49cb6b67ca6f27c3&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/a0c0077548543560380efc8c49cb6b67&lt;/a&gt;ca6f27c3&quot;&gt;&quot;&gt;Merge &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://github.com/adsabs/ADSReferencePipeline/commit/71792face29a3b491467e16a2bc82a0629d7f5a0">71792face</a><a href="https://github.com/adsabs/ADSReferencePipeline/commit/a0c0077548543560380efc8c49cb6b67ca6f27c3">&amp;quot;&amp;gt;71792face&amp;lt;/a&amp;gt;&amp;quot;&amp;gt;71792face&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/a0c0077548543560380efc8c49cb6b67ca6f2&lt;/a&gt;7c3&quot;&gt; into &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://github.com/adsabs/ADSReferencePipeline/commit/5c76c5adf">5c76c5adf">5c76c5adf</a>

Pull Request Pull Request #1: the first

Run Details

3020 of 3020 new or added lines in 24 files covered. (100.0%)

2510 of 3020 relevant lines covered (83.11%)

0.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.81

/adsrefpipe/xmlparsers/AGUxml.py

import sys, os
import re
import argparse

from adsputils import setup_logging, load_config

logger = setup_logging('reference-xml')
config = {}
config.update(load_config())

from adsrefpipe.xmlparsers.reference import XMLreference, ReferenceError
from adsrefpipe.xmlparsers.common import get_references, get_xml_block, extract_tag, match_int, match_doi

class AGUreference(XMLreference):

    re_issue = re.compile(r'^([ABCDE])\d+', re.DOTALL | re.VERBOSE)
    re_cleanup_doi = [
        (re.compile(r'__amp__lt;'), '<'),
        (re.compile(r'__amp__gt;'), '>'),
    ]
    re_xml_to_text = re.compile(r'<([A-Za-z_]*)\b[^>]*>(.*?)</\1>')

    def parse(self, prevref=None):
        """

        :param prevref:
        :return:
        """
        self.parsed = 0

        qualifier = ''
        self['year'] = match_int(self.xmlnode_nodecontents('year'))
        self['volume'] = match_int(self.xmlnode_nodecontents('volume'))
        self['issue'] = self.xmlnode_nodecontents('issue')
        self['pages'] = self.xmlnode_nodecontents('firstPage')
        if self['pages'] == 'null':
            if self.xmlnode_nodecontents('CitationNumber') and self.xmlnode_nodecontents('CitationNumber') != 'null':
                qualifier = self.xmlnode_nodecontents('CitationNumber')[0]
                self['pages'] = self.xmlnode_nodecontents('CitationNumber')
                self['CitationNumber'] = self.xmlnode_nodecontents('CitationNumber')
            else:
                qualifier = ''
                self['pages'] = ''
        self['pages'] = self.combine_page_qualifier(self['pages'], qualifier)

        self['jrlstr'] = self.xmlnode_nodecontents('journal_title')
        if self['jrlstr'] == 'null':
            self['jrlstr'] = self.xmlnode_nodecontents('reftitle')
        self['ttlstr'] = self.xmlnode_nodecontents('article_title')

        match = self.re_issue.search(self['issue'])
        if match:
            self['jrlstr'] += " %s" % match.group(1)
        if qualifier:
            self['jrlstr'] += " %s" % qualifier

        doi = self.xmlnode_nodecontents('DOI').strip()
        for one_set in self.re_cleanup_doi:
            doi = one_set[0].sub(one_set[1], doi)
        if len(doi) > 0:
            self['doi'] = doi
        else:
            # attempt to extract it from refstr
            doi = match_doi(self.reference_str.toxml())
            if doi:
                self['doi'] = doi

        self['authors'] = self.xmlnode_nodecontents('first_author')

        self['refstr'] = self.get_reference_str()
        if not self['refstr']:
            self['refplaintext'] = self.parse_refplaintext()

        self.parsed = 1

    def parse_refplaintext(self):
        """

        :return:
        """
        # 8/25/2020 searched all source files and was not able to find a
        # reference file with this tag put keeping it anyway
        refplaintext = self.xmlnode_nodecontents('unstructured_citation')
        if refplaintext:
            return refplaintext
        nodes = self.re_xml_to_text.findall(self.xmlnode_nodecontents('citation', keepxml=1))
        refplaintext = [node[1] for node in nodes if node[1].lower() != 'null']
        if len(refplaintext) > 0:
            return ', '.join(refplaintext)
        return ''


def AGUtoREFs(filename=None, buffer=None, unicode=None):
    """

    :param filename:
    :param buffer:
    :param unicode:
    :return:
    """
    references = []
    pairs = get_references(filename=filename, buffer=buffer)

    for pair in pairs:
        bibcode = pair[0]
        buffer = pair[1]

        references_bibcode = {'bibcode':bibcode, 'references':[]}

        block_references = get_xml_block(buffer, 'citation')

        for reference in block_references:
            logger.debug("AGUxml: parsing %s" % reference)
            try:
                agu_reference = AGUreference(reference)
                references_bibcode['references'].append({**agu_reference.get_parsed_reference(), 'xml_reference':reference})
            except ReferenceError as error_desc:
                logger.error("AGUxml: error parsing reference: %s" %error_desc)

        references.append(references_bibcode)
        logger.debug("%s: parsed %d references" % (bibcode, len(references)))

    return references


if __name__ == '__main__':      # pragma: no cover
    parser = argparse.ArgumentParser(description='Parse AGU references')
    parser.add_argument('-f', '--filename', help='the path to source file')
    parser.add_argument('-b', '--buffer', help='xml reference(s)')
    args = parser.parse_args()
    if args.filename:
        print(AGUtoREFs(filename=args.filename))
    if args.buffer:
        print(AGUtoREFs(buffer=args.buffer))
    # if no reference source is provided, just run the source test file
    if not args.filename and not args.buffer:
        print(AGUtoREFs(os.path.abspath(os.path.dirname(__file__) + '/../tests/unittests/stubdata/test.agu.xml')))
    sys.exit(0)

1	import sys, os	1✔
2	import re	1✔
3	import argparse	1✔
4
5	from adsputils import setup_logging, load_config	1✔
6
7	logger = setup_logging('reference-xml')	1✔
8	config = {}	1✔
9	config.update(load_config())	1✔
10
11	from adsrefpipe.xmlparsers.reference import XMLreference, ReferenceError	1✔
12	from adsrefpipe.xmlparsers.common import get_references, get_xml_block, extract_tag, match_int, match_doi	1✔
13
14	class AGUreference(XMLreference):	1✔
15
16	re_issue = re.compile(r'^([ABCDE])\d+', re.DOTALL \| re.VERBOSE)	1✔
17	re_cleanup_doi = [	1✔
18	(re.compile(r'__amp__lt;'), '<'),
19	(re.compile(r'__amp__gt;'), '>'),
20	]
21	re_xml_to_text = re.compile(r'<([A-Za-z_])\b[^>]>(.*?)</\1>')	1✔
22
23	def parse(self, prevref=None):	1✔
24	"""
25
26	:param prevref:
27	:return:
28	"""
29	self.parsed = 0	1✔
30
31	qualifier = ''	1✔
32	self['year'] = match_int(self.xmlnode_nodecontents('year'))	1✔
33	self['volume'] = match_int(self.xmlnode_nodecontents('volume'))	1✔
34	self['issue'] = self.xmlnode_nodecontents('issue')	1✔
35	self['pages'] = self.xmlnode_nodecontents('firstPage')	1✔
36	if self['pages'] == 'null':	1✔
37	if self.xmlnode_nodecontents('CitationNumber') and self.xmlnode_nodecontents('CitationNumber') != 'null':	1✔
38	qualifier = self.xmlnode_nodecontents('CitationNumber')[0]	1✔
39	self['pages'] = self.xmlnode_nodecontents('CitationNumber')	1✔
40	self['CitationNumber'] = self.xmlnode_nodecontents('CitationNumber')	1✔
41	else:
42	qualifier = ''	1✔
43	self['pages'] = ''	1✔
44	self['pages'] = self.combine_page_qualifier(self['pages'], qualifier)	1✔
45
46	self['jrlstr'] = self.xmlnode_nodecontents('journal_title')	1✔
47	if self['jrlstr'] == 'null':	1✔
48	self['jrlstr'] = self.xmlnode_nodecontents('reftitle')	1✔
49	self['ttlstr'] = self.xmlnode_nodecontents('article_title')	1✔
50
51	match = self.re_issue.search(self['issue'])	1✔
52	if match:	1✔
53	self['jrlstr'] += " %s" % match.group(1)	1✔
54	if qualifier:	1✔
55	self['jrlstr'] += " %s" % qualifier	1✔
56
57	doi = self.xmlnode_nodecontents('DOI').strip()	1✔
58	for one_set in self.re_cleanup_doi:	1✔
59	doi = one_set[0].sub(one_set[1], doi)	1✔
60	if len(doi) > 0:	1✔
61	self['doi'] = doi	1✔
62	else:
63	# attempt to extract it from refstr
64	doi = match_doi(self.reference_str.toxml())	1✔
65	if doi:	1✔
66	self['doi'] = doi	×
67
68	self['authors'] = self.xmlnode_nodecontents('first_author')	1✔
69
70	self['refstr'] = self.get_reference_str()	1✔
71	if not self['refstr']:	1✔
72	self['refplaintext'] = self.parse_refplaintext()	1✔
73
74	self.parsed = 1	1✔
75
76	def parse_refplaintext(self):	1✔
77	"""
78
79	:return:
80	"""
81	# 8/25/2020 searched all source files and was not able to find a
82	# reference file with this tag put keeping it anyway
83	refplaintext = self.xmlnode_nodecontents('unstructured_citation')	1✔
84	if refplaintext:	1✔
85	return refplaintext	×
86	nodes = self.re_xml_to_text.findall(self.xmlnode_nodecontents('citation', keepxml=1))	1✔
87	refplaintext = [node[1] for node in nodes if node[1].lower() != 'null']	1✔
88	if len(refplaintext) > 0:	1✔
89	return ', '.join(refplaintext)	1✔
90	return ''	1✔
91
92
93	def AGUtoREFs(filename=None, buffer=None, unicode=None):	1✔
94	"""
95
96	:param filename:
97	:param buffer:
98	:param unicode:
99	:return:
100	"""
101	references = []	1✔
102	pairs = get_references(filename=filename, buffer=buffer)	1✔
103
104	for pair in pairs:	1✔
105	bibcode = pair[0]	1✔
106	buffer = pair[1]	1✔
107
108	references_bibcode = {'bibcode':bibcode, 'references':[]}	1✔
109
110	block_references = get_xml_block(buffer, 'citation')	1✔
111
112	for reference in block_references:	1✔
113	logger.debug("AGUxml: parsing %s" % reference)	1✔
114	try:	1✔
115	agu_reference = AGUreference(reference)	1✔
116	references_bibcode['references'].append({**agu_reference.get_parsed_reference(), 'xml_reference':reference})	1✔
117	except ReferenceError as error_desc:	×
118	logger.error("AGUxml: error parsing reference: %s" %error_desc)	×
119
120	references.append(references_bibcode)	1✔
121	logger.debug("%s: parsed %d references" % (bibcode, len(references)))	1✔
122
123	return references	1✔
124
125
126	if __name__ == '__main__': # pragma: no cover
127	parser = argparse.ArgumentParser(description='Parse AGU references')
128	parser.add_argument('-f', '--filename', help='the path to source file')
129	parser.add_argument('-b', '--buffer', help='xml reference(s)')
130	args = parser.parse_args()
131	if args.filename:
132	print(AGUtoREFs(filename=args.filename))
133	if args.buffer:
134	print(AGUtoREFs(buffer=args.buffer))
135	# if no reference source is provided, just run the source test file
136	if not args.filename and not args.buffer:
137	print(AGUtoREFs(os.path.abspath(os.path.dirname(__file__) + '/../tests/unittests/stubdata/test.agu.xml')))
138	sys.exit(0)

adsabs / ADSReferencePipeline / 1136300255

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous