436180776

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/LeapBeyond/scrubadub/commit/<a class=hub.com/LeapBeyond/scrubadub/commit/<a class="double-link" href="https://git"><a class=hub.com/LeapBeyond/scrubadub/commit/<a class="double-link" href="https://git"><a class=hub.com/LeapBeyond/scrubadub/commit/<a class="double-link" href="https://git"><a class=hub.com/LeapBeyond/scrubadub/commit/<a class="double-link" href="https://git"><a class=hub.com/LeapBeyond/scrubadub/commit/1c8ecfc37c4cc2997d538012f0bf0a17375c5fe0">1c8ecfc37">&lt;a href=&quot;https://github.com/LeapBeyond/scrubadub/commit/</a><a class="double-link" href="https://github.com/LeapBeyond/scrubadub/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LeapBeyond/scrubadub/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LeapBeyond/scrubadub/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LeapBeyond/scrubadub/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LeapBeyond/scrubadub/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LeapBeyond/scrubadub/commit/1c8ecfc37c4cc2997d538012f0bf0a17375c5fe0">1c8ecfc37</a><a href="https://github.com/LeapBeyond/scrubadub/commit/1c8ecfc37c4cc2997d538012f0bf0a17375c5fe0">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/LeapBeyond/scrubadub/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/LeapBeyond/scrubadub/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LeapBeyond/scrubadub/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LeapBeyond/scrubadub/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LeapBeyond/scrubadub/commit/1c8ecfc37c4cc2997d538012f0bf0a17375c5fe0&quot;&gt;1c8ecfc37&lt;/a&gt;&lt;a href=&quot;https://github.com/LeapBeyond/scrubadub/commit/1c8ecfc37c4cc2997d538012f0bf0a17375c5fe0&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/LeapBeyond/scrubadub/commit/1c8ecfc37c4cc2997d538012f0bf0a17375c5fe0&amp;quot;&amp;gt;&amp;amp;quot;&amp;amp;gt;&amp;amp;amp;lt;a href=&amp;amp;amp;quot;https://github.com/LeapBey... (continued)

Pull Request Pull Request #86: improved DOB detector with a new nuclear filter option

Run Details

22 of 22 new or added lines in 1 file covered. (100.0%)

1187 of 1324 relevant lines covered (89.65%)

1.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.98

/scrubadub/detectors/date_of_birth.py

"""
This modules provides a detector for date of births.

It is simply looking for any detected dates that are more than 18 years and date of birth has been
mentioned within 2 previous lines.

Using 18+ years may not be suitable for all use cases; use with caution.
"""
import re
from dateparser.search import search_dates
from datetime import datetime

from typing import Optional


from scrubadub.detectors.base import Detector
from scrubadub.filth import DoBFilth


class DoBDetector(Detector):
    filth_cls = DoBFilth
    name = 'dob'

    def __init__(self, context_before: Optional[int] = None, context_after: Optional[int] = None,
                 min_age_years: Optional[int] = None, filter_type: Optional[str] = None,
                 **kwargs):

        if context_before is None:
            self.context_before = int(2)
        if context_after is None:
            self.context_after = int(1)
        if min_age_years is None:
            self.min_age_years = int(18)
        if filter_type is None:
            self.filter_type = 'context_dob'

        self.trigger_words = ['birth', 'born', 'dob', 'd.o.b.']
        super(DoBDetector, self).__init__(**kwargs)

    def context_dob(self, text: str, document_name: Optional[str] = None):
        """ Contextual Date of Birth filter -
        only look for dates that are in close proximity with the words similar to birth
        """
        if isinstance(text, list):
            lines = text
        else:
            lines = text.split('\n')
        for count, line in enumerate(lines):
            # using the dateparser lib - locale can be set here
            date_picker = search_dates(line, languages=['en'])
            if date_picker is not None:
                for identified_date in date_picker:
                    # calculate if at least 18 years has lapsed
                    # find if anything related to date of birth has been mentioned in the previous lines
                    if datetime.now().year - identified_date[1].year >= self.min_age_years and \
                            any(trigger_word in ' '.join(lines[count - self.context_before:count + self.context_after]
                                                         ).lower() for trigger_word in self.trigger_words):
                        # date_picker doesn't return the start end, so using re.finditer for the start & end locations
                        found_dates = re.finditer(re.escape(r"{0}".format(identified_date[0])), text)

                        for instance in found_dates:
                            yield self.filth_cls(
                                beg=instance.start(),
                                end=instance.end(),
                                text=instance.group(),
                                detector_name=self.name,
                                document_name=document_name,
                                locale=self.locale,
                            )

    def nuke_dob(self, text: str, document_name: Optional[str] = None):
        """ Nuclear option - remove all dates in the text that has lapsed x years
        default x = 18
        """
        date_picker = search_dates(text, languages=['en'])
        for identified_date in date_picker:
            if datetime.now().year - identified_date[1].year >= self.min_age_years:
                found_dates = re.finditer(re.escape(r"{0}".format(identified_date[0])), text)

                for instance in found_dates:
                    yield self.filth_cls(
                        beg=instance.start(),
                        end=instance.end(),
                        text=instance.group(),
                        detector_name=self.name,
                        document_name=document_name,
                        locale=self.locale,
                    )

    def iter_filth(self, text: str, document_name: Optional[str] = None):

        if self.filter_type == 'context_dob':
            yield from self.context_dob(text, document_name)
        else:
            yield from self.nuke_dob(text, document_name)

    @classmethod
    def supported_locale(cls, locale: str) -> bool:
        """Returns true if this ``Detector`` supports the given locale.

        :param locale: The locale of the documents in the format: 2 letter lower-case language code followed by an
                       underscore and the two letter upper-case country code, eg "en_GB" or "de_CH".
        :type locale: str
        :return: ``True`` if the locale is supported, otherwise ``False``
        :rtype: bool
        """
        language, region = cls.locale_split(locale)
        return language in ['en']

1	"""
2	This modules provides a detector for date of births.
3
4	It is simply looking for any detected dates that are more than 18 years and date of birth has been
5	mentioned within 2 previous lines.
6
7	Using 18+ years may not be suitable for all use cases; use with caution.
8	"""
9	import re	2✔
10	from dateparser.search import search_dates	2✔
11	from datetime import datetime	2✔
12
13	from typing import Optional	2✔
14
15
16	from scrubadub.detectors.base import Detector	2✔
17	from scrubadub.filth import DoBFilth	2✔
18
19
20	class DoBDetector(Detector):	2✔
21	filth_cls = DoBFilth	2✔
22	name = 'dob'	2✔
23
24	def __init__(self, context_before: Optional[int] = None, context_after: Optional[int] = None,	2✔
25	min_age_years: Optional[int] = None, filter_type: Optional[str] = None,
26	**kwargs):
27
28	if context_before is None:	2✔
29	self.context_before = int(2)	2✔
30	if context_after is None:	2✔
31	self.context_after = int(1)	2✔
32	if min_age_years is None:	2✔
33	self.min_age_years = int(18)	2✔
34	if filter_type is None:	2✔
35	self.filter_type = 'context_dob'	2✔
36
37	self.trigger_words = ['birth', 'born', 'dob', 'd.o.b.']	2✔
38	super(DoBDetector, self).__init__(**kwargs)	2✔
39
40	def context_dob(self, text: str, document_name: Optional[str] = None):	2✔
41	""" Contextual Date of Birth filter -
42	only look for dates that are in close proximity with the words similar to birth
43	"""
44	if isinstance(text, list):	2✔
45	lines = text	×
46	else:
47	lines = text.split('\n')	2✔
48	for count, line in enumerate(lines):	2✔
49	# using the dateparser lib - locale can be set here
50	date_picker = search_dates(line, languages=['en'])	2✔
51	if date_picker is not None:	2✔
52	for identified_date in date_picker:	2✔
53	# calculate if at least 18 years has lapsed
54	# find if anything related to date of birth has been mentioned in the previous lines
55	if datetime.now().year - identified_date[1].year >= self.min_age_years and \	2✔
56	any(trigger_word in ' '.join(lines[count - self.context_before:count + self.context_after]
57	).lower() for trigger_word in self.trigger_words):
58	# date_picker doesn't return the start end, so using re.finditer for the start & end locations
59	found_dates = re.finditer(re.escape(r"{0}".format(identified_date[0])), text)	2✔
60
61	for instance in found_dates:	2✔
62	yield self.filth_cls(	2✔
63	beg=instance.start(),
64	end=instance.end(),
65	text=instance.group(),
66	detector_name=self.name,
67	document_name=document_name,
68	locale=self.locale,
69	)
70
71	def nuke_dob(self, text: str, document_name: Optional[str] = None):	2✔
72	""" Nuclear option - remove all dates in the text that has lapsed x years
73	default x = 18
74	"""
75	date_picker = search_dates(text, languages=['en'])	×
76	for identified_date in date_picker:	×
77	if datetime.now().year - identified_date[1].year >= self.min_age_years:	×
78	found_dates = re.finditer(re.escape(r"{0}".format(identified_date[0])), text)	×
79
80	for instance in found_dates:	×
81	yield self.filth_cls(	×
82	beg=instance.start(),
83	end=instance.end(),
84	text=instance.group(),
85	detector_name=self.name,
86	document_name=document_name,
87	locale=self.locale,
88	)
89
90	def iter_filth(self, text: str, document_name: Optional[str] = None):	2✔
91
92	if self.filter_type == 'context_dob':	2✔
93	yield from self.context_dob(text, document_name)	2✔
94	else:
95	yield from self.nuke_dob(text, document_name)	×
96
97	@classmethod	2✔
98	def supported_locale(cls, locale: str) -> bool:	2✔
99	"""Returns true if this ``Detector`` supports the given locale.
100
101	:param locale: The locale of the documents in the format: 2 letter lower-case language code followed by an
102	underscore and the two letter upper-case country code, eg "en_GB" or "de_CH".
103	:type locale: str
104	:return: ``True`` if the locale is supported, otherwise ``False``
105	:rtype: bool
106	"""
107	language, region = cls.locale_split(locale)	2✔
108	return language in ['en']	2✔

LeapBeyond / scrubadub / 436180776

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous