381269281

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/LeapBeyond/scrubadub/commit/<a class=hub.com/LeapBeyond/scrubadub/commit/213805057dcdb692dfa6884d8fac23ad2596fafc">213805057<a href="https://github.com/LeapBeyond/scrubadub/commit/213805057dcdb692dfa6884d8fac23ad2596fafc">">Merge </a><a class="double-link" href="https://github.com/LeapBeyond/scrubadub/commit/<a class="double-link" href="https://github.com/LeapBeyond/scrubadub/commit/d74fa7d0d3ea05d2ef0e224bd1614f078c171774">d74fa7d0d</a>">d74fa7d0d</a><a href="https://github.com/LeapBeyond/scrubadub/commit/213805057dcdb692dfa6884d8fac23ad2596fafc"> into 81bd15912">81bd15912</a>

Pull Request Pull Request #79: Adding localisation, improving docs

Run Details

203 of 203 new or added lines in 25 files covered. (100.0%)

1016 of 1091 relevant lines covered (93.13%)

1.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.11

/scrubadub/filth/base.py

import warnings
from faker import Faker
from typing import Optional, ClassVar, Pattern, List, Match

from .. import exceptions
from .. import utils


class Filth(object):
    """This is the base class for all ``Filth`` that is detected in dirty dirty
    text.
    """

    # this allows people to customize the output, especially for placeholder
    # text and identifier replacements
    prefix = u'{{'  # type: ClassVar[str]
    suffix = u'}}'  # type: ClassVar[str]

    # the `type` is used when filths are merged to come up with a sane label
    type = ''  # type: ClassVar[str]

    # the `lookup` is used to keep track of all of the different types of filth
    # that are encountered across all `Filth` types.
    lookup = utils.Lookup()

    # For backwards compatibility, but this is deprecated.
    regex = None  # type: Optional[Pattern[str]]

    def __init__(self, beg: Optional[int] = None, end: Optional[int] = None, text: Optional[str] = None,
                 match: Optional[Match] = None, detector_name: Optional[str] = None,
                 document_name: Optional[str] = None, replacement_string: Optional[str] = None,
                 locale: Optional[str] = None, **kwargs):

        self.beg = 0  # type: int
        self.end = 0  # type: int
        self.text = ''  # type: str
        self.match = None  # type: Optional[Match]

        if match is not None and isinstance(match, Match):
            self.beg = match.start()
            self.end = match.end()
            self.text = match.string[match.start():match.end()]
            self.match = match

        if beg is not None:
            self.beg = beg
        if end is not None:
            self.end = end
        if text is not None:
            self.text = text

        self.detector_name = detector_name  # type: Optional[str]
        self.document_name = document_name  # type: Optional[str]
        self.replacement_string = replacement_string  # type: Optional[str]
        self.locale = locale  # type: Optional[str]

    @property
    def placeholder(self) -> str:
        return self.type.upper()

    @property
    def identifier(self) -> str:
        # NOTE: this is not an efficient way to store this in memory. could
        # alternatively hash the type and text and do away with the overhead
        # bits of storing the tuple in the lookup
        i = self.lookup[(self.type, self.text.lower())]
        return u'%s-%d' % (self.placeholder, i)

    def replace_with(self, replace_with: str = 'placeholder', **kwargs) -> str:
        warnings.warn(
            "Filth.replace_with() will be removed in favour of using the more general PostProcessors",
            DeprecationWarning
        )
        if self.prefix != '{{' or self.suffix != '}}':
            warnings.warn(
                "Setting prefixes and suffixes with scrubadub.filth.Filth.prefix or scrubadub.filth.Filth.suffix "
                "is depreciated in favour of using the PrefixSuffixReplacer",
                DeprecationWarning
            )

        if replace_with == 'placeholder':
            return self.prefix + self.placeholder + self.suffix
        # elif replace_with == 'surrogate':
        #     raise NotImplementedError
        elif replace_with == 'identifier':
            return self.prefix + self.identifier + self.suffix
        else:
            raise exceptions.InvalidReplaceWith(replace_with)

    def merge(self, other_filth: 'Filth') -> 'MergedFilth':
        return MergedFilth(self, other_filth)

    def __repr__(self) -> str:
        return self._to_string()

    def _to_string(self, attributes: Optional[List[str]] = None) -> str:
        if attributes is None:
            attributes = ['text', 'document_name', 'beg', 'end', 'comparison_type', 'detector_name', 'locale']

        item_attributes = [
            "{}={}".format(item, getattr(self, item, None).__repr__())
            for item in attributes
            if getattr(self, item, None) is not None
        ]
        return "<{} {}>".format(self.__class__.__name__, " ".join(item_attributes))

    def __eq__(self, other) -> bool:
        """Only test equality on a subset of class attributes and some are optional"""
        match = True

        if not hasattr(other, 'beg') or not hasattr(other, 'end') or not hasattr(other, 'text'):
            raise TypeError("Unsupported comparison with a Filth and {}".format(type(other)))

        match &= (self.beg == other.beg)
        match &= (self.end == other.end)
        match &= (self.text == other.text)

        if hasattr(self, 'document_name') or hasattr(other, 'document_name'):
            match &= (self.document_name == other.document_name)
        if hasattr(self, 'detector_name') or hasattr(other, 'detector_name'):
            match &= (self.detector_name == other.detector_name)

        return match

    @staticmethod
    def generate(faker: Faker) -> str:
        """Generates an example of this ``Filth`` type, usually using the faker python library.

        :param faker: The ``Faker`` class from the ``faker`` library
        :type faker: Faker
        :return: An example of this ``Filth``
        :rtype: str
        """
        raise NotImplementedError("A generate() function has not been implemented for this Filth")


class MergedFilth(Filth):
    """This class takes care of merging different types of filth"""

    def __init__(self, a_filth: Filth, b_filth: Filth):
        super(MergedFilth, self).__init__(
            beg=a_filth.beg,
            end=a_filth.end,
            text=a_filth.text,
            document_name=a_filth.document_name,
        )
        self.filths = [a_filth]
        self._update_content(b_filth)

    def _update_content(self, other_filth: Filth):
        """this updates the bounds, text and placeholder for the merged
        filth
        """
        if self.end < other_filth.beg or other_filth.end < self.beg:
            raise exceptions.FilthMergeError(
                "a_filth goes from [%s, %s) and b_filth goes from [%s, %s)" % (
                    self.beg, self.end, other_filth.beg, other_filth.end
                ))

        if self.document_name != other_filth.document_name:
            raise exceptions.FilthMergeError(
                "This MergedFilth is in document {}, but the Filth that is being merged is in another document {}"
                "".format(self.document_name.__repr__(), other_filth.document_name.__repr__())
            )

        # get the text over lap correct
        if self.beg < other_filth.beg:
            first = self  # type: Filth
            second = other_filth  # type: Filth
        else:
            second = self
            first = other_filth
        end_offset = second.end - first.end
        if end_offset > 0:
            self.text = first.text + second.text[-end_offset:]

        # update the beg/end strings
        self.beg = min(self.beg, other_filth.beg)
        self.end = max(self.end, other_filth.end)
        if self.end - self.beg != len(self.text):
            raise exceptions.FilthMergeError("text length isn't consistent")

        # update the placeholder
        self.filths.append(other_filth)
        self._placeholder = '+'.join([filth.type for filth in self.filths])

    @property
    def placeholder(self):
        return self._placeholder.upper()

    def merge(self, other_filth: Filth) -> 'MergedFilth':
        """Be smart about merging filth in this case to avoid nesting merged
        filths.
        """
        self._update_content(other_filth)
        return self

    def __repr__(self) -> str:
        return self._to_string(['filths'])


class RegexFilth(Filth):
    def __init__(self, *args, **kwargs):
        warnings.warn("Use of RegexFilth is depreciated, use Filth directly instead.", DeprecationWarning)
        super(RegexFilth, self).__init__(*args, **kwargs)

1	import warnings	2✔
2	from faker import Faker	2✔
3	from typing import Optional, ClassVar, Pattern, List, Match	2✔
4
5	from .. import exceptions	2✔
6	from .. import utils	2✔
7
8
9	class Filth(object):	2✔
10	"""This is the base class for all ``Filth`` that is detected in dirty dirty
11	text.
12	"""
13
14	# this allows people to customize the output, especially for placeholder
15	# text and identifier replacements
16	prefix = u'{{' # type: ClassVar[str]	2✔
17	suffix = u'}}' # type: ClassVar[str]	2✔
18
19	# the `type` is used when filths are merged to come up with a sane label
20	type = '' # type: ClassVar[str]	2✔
21
22	# the `lookup` is used to keep track of all of the different types of filth
23	# that are encountered across all `Filth` types.
24	lookup = utils.Lookup()	2✔
25
26	# For backwards compatibility, but this is deprecated.
27	regex = None # type: Optional[Pattern[str]]	2✔
28
29	def __init__(self, beg: Optional[int] = None, end: Optional[int] = None, text: Optional[str] = None,	2✔
30	match: Optional[Match] = None, detector_name: Optional[str] = None,
31	document_name: Optional[str] = None, replacement_string: Optional[str] = None,
32	locale: Optional[str] = None, **kwargs):
33
34	self.beg = 0 # type: int	2✔
35	self.end = 0 # type: int	2✔
36	self.text = '' # type: str	2✔
37	self.match = None # type: Optional[Match]	2✔
38
39	if match is not None and isinstance(match, Match):	2✔
40	self.beg = match.start()	2✔
41	self.end = match.end()	2✔
42	self.text = match.string[match.start():match.end()]	2✔
43	self.match = match	2✔
44
45	if beg is not None:	2✔
46	self.beg = beg	2✔
47	if end is not None:	2✔
48	self.end = end	2✔
49	if text is not None:	2✔
50	self.text = text	2✔
51
52	self.detector_name = detector_name # type: Optional[str]	2✔
53	self.document_name = document_name # type: Optional[str]	2✔
54	self.replacement_string = replacement_string # type: Optional[str]	2✔
55	self.locale = locale # type: Optional[str]	2✔
56
57	@property	2✔
58	def placeholder(self) -> str:	2✔
59	return self.type.upper()	2✔
60
61	@property	2✔
62	def identifier(self) -> str:	2✔
63	# NOTE: this is not an efficient way to store this in memory. could
64	# alternatively hash the type and text and do away with the overhead
65	# bits of storing the tuple in the lookup
66	i = self.lookup[(self.type, self.text.lower())]	2✔
67	return u'%s-%d' % (self.placeholder, i)	2✔
68
69	def replace_with(self, replace_with: str = 'placeholder', **kwargs) -> str:	2✔
70	warnings.warn(	2✔
71	"Filth.replace_with() will be removed in favour of using the more general PostProcessors",
72	DeprecationWarning
73	)
74	if self.prefix != '{{' or self.suffix != '}}':	2✔
75	warnings.warn(	2✔
76	"Setting prefixes and suffixes with scrubadub.filth.Filth.prefix or scrubadub.filth.Filth.suffix "
77	"is depreciated in favour of using the PrefixSuffixReplacer",
78	DeprecationWarning
79	)
80
81	if replace_with == 'placeholder':	2✔
82	return self.prefix + self.placeholder + self.suffix	2✔
83	# elif replace_with == 'surrogate':
84	# raise NotImplementedError
85	elif replace_with == 'identifier':	2✔
86	return self.prefix + self.identifier + self.suffix	2✔
87	else:
88	raise exceptions.InvalidReplaceWith(replace_with)	2✔
89
90	def merge(self, other_filth: 'Filth') -> 'MergedFilth':	2✔
91	return MergedFilth(self, other_filth)	2✔
92
93	def __repr__(self) -> str:	2✔
94	return self._to_string()	2✔
95
96	def _to_string(self, attributes: Optional[List[str]] = None) -> str:	2✔
97	if attributes is None:	2✔
98	attributes = ['text', 'document_name', 'beg', 'end', 'comparison_type', 'detector_name', 'locale']	2✔
99
100	item_attributes = [	2✔
101	"{}={}".format(item, getattr(self, item, None).__repr__())
102	for item in attributes
103	if getattr(self, item, None) is not None
104	]
105	return "<{} {}>".format(self.__class__.__name__, " ".join(item_attributes))	2✔
106
107	def __eq__(self, other) -> bool:	2✔
108	"""Only test equality on a subset of class attributes and some are optional"""
109	match = True	2✔
110
111	if not hasattr(other, 'beg') or not hasattr(other, 'end') or not hasattr(other, 'text'):	2✔
112	raise TypeError("Unsupported comparison with a Filth and {}".format(type(other)))	×
113
114	match &= (self.beg == other.beg)	2✔
115	match &= (self.end == other.end)	2✔
116	match &= (self.text == other.text)	2✔
117
118	if hasattr(self, 'document_name') or hasattr(other, 'document_name'):	2✔
119	match &= (self.document_name == other.document_name)	2✔
120	if hasattr(self, 'detector_name') or hasattr(other, 'detector_name'):	2✔
121	match &= (self.detector_name == other.detector_name)	2✔
122
123	return match	2✔
124
125	@staticmethod	2✔
126	def generate(faker: Faker) -> str:	2✔
127	"""Generates an example of this ``Filth`` type, usually using the faker python library.
128
129	:param faker: The ``Faker`` class from the ``faker`` library
130	:type faker: Faker
131	:return: An example of this ``Filth``
132	:rtype: str
133	"""
134	raise NotImplementedError("A generate() function has not been implemented for this Filth")	×
135
136
137	class MergedFilth(Filth):	2✔
138	"""This class takes care of merging different types of filth"""
139
140	def __init__(self, a_filth: Filth, b_filth: Filth):	2✔
141	super(MergedFilth, self).__init__(	2✔
142	beg=a_filth.beg,
143	end=a_filth.end,
144	text=a_filth.text,
145	document_name=a_filth.document_name,
146	)
147	self.filths = [a_filth]	2✔
148	self._update_content(b_filth)	2✔
149
150	def _update_content(self, other_filth: Filth):	2✔
151	"""this updates the bounds, text and placeholder for the merged
152	filth
153	"""
154	if self.end < other_filth.beg or other_filth.end < self.beg:	2✔
155	raise exceptions.FilthMergeError(	2✔
156	"a_filth goes from [%s, %s) and b_filth goes from [%s, %s)" % (
157	self.beg, self.end, other_filth.beg, other_filth.end
158	))
159
160	if self.document_name != other_filth.document_name:	2✔
161	raise exceptions.FilthMergeError(	2✔
162	"This MergedFilth is in document {}, but the Filth that is being merged is in another document {}"
163	"".format(self.document_name.__repr__(), other_filth.document_name.__repr__())
164	)
165
166	# get the text over lap correct
167	if self.beg < other_filth.beg:	2✔
168	first = self # type: Filth	2✔
169	second = other_filth # type: Filth	2✔
170	else:
171	second = self	2✔
172	first = other_filth	2✔
173	end_offset = second.end - first.end	2✔
174	if end_offset > 0:	2✔
175	self.text = first.text + second.text[-end_offset:]	2✔
176
177	# update the beg/end strings
178	self.beg = min(self.beg, other_filth.beg)	2✔
179	self.end = max(self.end, other_filth.end)	2✔
180	if self.end - self.beg != len(self.text):	2✔
181	raise exceptions.FilthMergeError("text length isn't consistent")	2✔
182
183	# update the placeholder
184	self.filths.append(other_filth)	2✔
185	self._placeholder = '+'.join([filth.type for filth in self.filths])	2✔
186
187	@property	2✔
188	def placeholder(self):	1✔
189	return self._placeholder.upper()	2✔
190
191	def merge(self, other_filth: Filth) -> 'MergedFilth':	2✔
192	"""Be smart about merging filth in this case to avoid nesting merged
193	filths.
194	"""
195	self._update_content(other_filth)	2✔
196	return self	2✔
197
198	def __repr__(self) -> str:	2✔
199	return self._to_string(['filths'])	2✔
200
201
202	class RegexFilth(Filth):	2✔
203	def __init__(self, args, *kwargs):	2✔
204	warnings.warn("Use of RegexFilth is depreciated, use Filth directly instead.", DeprecationWarning)	2✔
205	super(RegexFilth, self).__init__(args, *kwargs)	2✔

LeapBeyond / scrubadub / 381269281

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous