1106086475

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/adsabs/ADSReferencePipeline/commit/<a class=hub.com/adsabs/ADSReferencePipeline/commit/6562b4f24f5ae0f5e34affd60690cbeac0b9a7fe">6562b4f24<a href="https://github.com/adsabs/ADSReferencePipeline/commit/6562b4f24f5ae0f5e34affd60690cbeac0b9a7fe">">Merge </a><a class="double-link" href="https://github.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://github.com/adsabs/ADSReferencePipeline/commit/c6373792a3562b738b5c62e7d03c1eea800b3677">c6373792a</a>">c6373792a</a><a href="https://github.com/adsabs/ADSReferencePipeline/commit/6562b4f24f5ae0f5e34affd60690cbeac0b9a7fe"> into 5c76c5adf">5c76c5adf</a>

Pull Request Pull Request #1: the first

Run Details

2770 of 2770 new or added lines in 23 files covered. (100.0%)

2222 of 2770 relevant lines covered (80.22%)

0.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.21

/adsrefpipe/xmlparsers/xmlFile.py

#!/usr/bin/env python
#
#   File:  XmlFile.py
#

import xml.dom.minidom as dom
from xml.parsers.expat import ExpatError
import re
from collections import UserList


class XmlList(dom.Element, UserList):
    def __init__(self, elements=None, name=None):
        """

        :param elements:
        :param name:
        """
        if not elements:
            elements = []

        UserList.__init__(self, elements)

        if not name:
            self.noname = 1
            name = 'XmlList'

        dom.Element.__init__(self, name)

        self.childNodes = elements
        self.__name = name

    def toxml(self):
        """

        :return:
        """
        if not self:
            return ''
        elif self.noname:
            return self.childNodes[0].toxml()
        else:
            return dom.Element.toxml(self)

    def __str__(self):
        """

        :return:
        """
        if not self and self.__name == 'XmlList':
            return ''
        elif self.noname:
            return self.childNodes[0].toprettyxml(indent='  ')
        else:
            return self.toprettyxml(indent='  ')


class XmlString(XmlList):
    re_cleanup = [
        (re.compile(r'\s\s+'), r' '),
        (re.compile(r'> <'), r'><'),
        (re.compile(r'&'), '__amp__')
    ]
    re_remove_all_tags = re.compile(r'<[^<]+>')
    re_match_open_tag = re.compile(r'<(?!.*<)')
    re_match_text_between_tags = re.compile(r'[^<>]*')

    def __init__(self, buffer=None, doctype=None):
        """

        :param buffer:
        :param doctype:
        """
        # use dummy string if nothing no input is specified
        if not buffer: buffer = '<xmldoc />'

        buffer = buffer.replace('\n', ' ')

        for one_set in self.re_cleanup:
            buffer = one_set[0].sub(one_set[1], buffer)

        # up to three attempt to fix the reference, remove untag tags (ie, <883::AID-MASY883>)
        # but there is also less than and equal that at this point I can not do anything about
        # unless at the end, turn xml into text and replacing them
        # not sure why range does not work here!!
        for _ in [0,1,2,3]:
            try:
                xml = dom.parseString(buffer)
                self.__doctype = doctype
                XmlList.__init__(self, elements=xml.childNodes, name=doctype)
                return
            except ExpatError as e:
                match = re.findall(r'(\d+)', str(e))
                if len(match) == 2:
                    start = int(match[1])
                    range = [self.re_match_open_tag.search(buffer[:start]).span()[0],
                             self.re_match_text_between_tags.search(buffer[start:]).span()[1]+start+1]
                    remove_text = buffer[range[0]:range[1]]
                    if remove_text.count('<') == 1 and not (remove_text.startswith('</') or remove_text.startswith('< ')):
                        buffer = buffer.replace(remove_text,'')
                continue
        # no success, so turn xml into text, remove < and > if any, and then put one tag around it
        # to be able to extract it as text from this structure
        top_tag = buffer.split(' ',1)[0][1:]
        the_buffer = self.re_remove_all_tags.sub(' ', buffer).replace('<','&lt;').replace('>','&gt;')
        buffer_transform = "<%s> %s </%s>"%(top_tag, the_buffer, top_tag)
        xml = dom.parseString(buffer_transform)
        self.__doctype = doctype
        XmlList.__init__(self, elements=xml.childNodes, name=doctype)

1	#!/usr/bin/env python
2	#
3	# File: XmlFile.py
4	#
5
6	import xml.dom.minidom as dom	1✔
7	from xml.parsers.expat import ExpatError	1✔
8	import re	1✔
9	from collections import UserList	1✔
10
11
12	class XmlList(dom.Element, UserList):	1✔
13	def __init__(self, elements=None, name=None):	1✔
14	"""
15
16	:param elements:
17	:param name:
18	"""
19	if not elements:	1✔
20	elements = []	×
21
22	UserList.__init__(self, elements)	1✔
23
24	if not name:	1✔
25	self.noname = 1	1✔
26	name = 'XmlList'	1✔
27
28	dom.Element.__init__(self, name)	1✔
29
30	self.childNodes = elements	1✔
31	self.__name = name	1✔
32
33	def toxml(self):	1✔
34	"""
35
36	:return:
37	"""
38	if not self:	1✔
39	return ''	×
40	elif self.noname:	1✔
41	return self.childNodes[0].toxml()	1✔
42	else:
43	return dom.Element.toxml(self)	×
44
45	def __str__(self):	1✔
46	"""
47
48	:return:
49	"""
50	if not self and self.__name == 'XmlList':	1✔
51	return ''	×
52	elif self.noname:	1✔
53	return self.childNodes[0].toprettyxml(indent=' ')	1✔
54	else:
55	return self.toprettyxml(indent=' ')	×
56
57
58	class XmlString(XmlList):	1✔
59	re_cleanup = [	1✔
60	(re.compile(r'\s\s+'), r' '),
61	(re.compile(r'> <'), r'><'),
62	(re.compile(r'&'), '__amp__')
63	]
64	re_remove_all_tags = re.compile(r'<[^<]+>')	1✔
65	re_match_open_tag = re.compile(r'<(?!.*<)')	1✔
66	re_match_text_between_tags = re.compile(r'[^<>]*')	1✔
67
68	def __init__(self, buffer=None, doctype=None):	1✔
69	"""
70
71	:param buffer:
72	:param doctype:
73	"""
74	# use dummy string if nothing no input is specified
75	if not buffer: buffer = '<xmldoc />'	1✔
76
77	buffer = buffer.replace('\n', ' ')	1✔
78
79	for one_set in self.re_cleanup:	1✔
80	buffer = one_set[0].sub(one_set[1], buffer)	1✔
81
82	# up to three attempt to fix the reference, remove untag tags (ie, <883::AID-MASY883>)
83	# but there is also less than and equal that at this point I can not do anything about
84	# unless at the end, turn xml into text and replacing them
85	# not sure why range does not work here!!
86	for _ in [0,1,2,3]:	1✔
87	try:	1✔
88	xml = dom.parseString(buffer)	1✔
89	self.__doctype = doctype	1✔
90	XmlList.__init__(self, elements=xml.childNodes, name=doctype)	1✔
91	return	1✔
92	except ExpatError as e:	1✔
93	match = re.findall(r'(\d+)', str(e))	1✔
94	if len(match) == 2:	1✔
95	start = int(match[1])	1✔
96	range = [self.re_match_open_tag.search(buffer[:start]).span()[0],	1✔
97	self.re_match_text_between_tags.search(buffer[start:]).span()[1]+start+1]
98	remove_text = buffer[range[0]:range[1]]	1✔
99	if remove_text.count('<') == 1 and not (remove_text.startswith('</') or remove_text.startswith('< ')):	1✔
100	buffer = buffer.replace(remove_text,'')	×
101	continue	1✔
102	# no success, so turn xml into text, remove < and > if any, and then put one tag around it
103	# to be able to extract it as text from this structure
104	top_tag = buffer.split(' ',1)[0][1:]	1✔
105	the_buffer = self.re_remove_all_tags.sub(' ', buffer).replace('<','<').replace('>','>')	1✔
106	buffer_transform = "<%s> %s </%s>"%(top_tag, the_buffer, top_tag)	1✔
107	xml = dom.parseString(buffer_transform)	1✔
108	self.__doctype = doctype	×
109	XmlList.__init__(self, elements=xml.childNodes, name=doctype)	×

adsabs / ADSReferencePipeline / 1106086475

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous