1146

pending completion

Build # 1146

Build Type

push

travis-ci

Committed by

havok2063

Commit Message

<a href="https://github.com/sdss/marvin/commit/08c0ced24">reverting submodule brain back to </a><a class="double-link" href="https://github.com/sdss/marvin/commit/95afcdb71">95afcdb71</a>

Run Details

1331 of 3521 branches covered (37.8%)

Branch coverage included in aggregate %.

6515 of 11325 relevant lines covered (57.53%)

1.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.9

/python/marvin/utils/general/structs.py

#!/usr/bin/env python
# encoding: utf-8
#
# structs.py
#
# Created by José Sánchez-Gallego on 17 Aug 2017.


from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

from collections import OrderedDict

import six

from fuzzywuzzy import fuzz as fuzz_fuzz
from fuzzywuzzy import process as fuzz_proc


__ALL__ = ['FuzzyDict', 'Dotable', 'DotableCaseInsensitive', 'get_best_fuzzy',
           'FuzzyList', 'string_folding_wrapper']


class Dotable(dict):
    """A custom dict class that allows dot access to nested dictionaries.

    Copied from http://hayd.github.io/2013/dotable-dictionaries/. Note that
    this allows you to use dots to get dictionary values, but not to set them.

    """

    def __getattr__(self, value):
        if '__' in value:
            return dict.__getattr__(self, value)
        else:
            return self.__getitem__(value)

    # def __init__(self, d):
    #     dict.__init__(self, ((k, self.parse(v)) for k, v in d.iteritems()))

    @classmethod
    def parse(cls, v):
        if isinstance(v, dict):
            return cls(v)
        elif isinstance(v, list):
            return [cls.parse(i) for i in v]
        else:
            return v


class DotableCaseInsensitive(Dotable):
    """Like dotable but access to attributes and keys is case insensitive."""

    def _match(self, list_of_keys, value):

        lower_values = [str(xx).lower() for xx in list_of_keys]
        if value.lower() in lower_values:
            return list_of_keys[lower_values.index(value.lower())]
        else:
            return False

    def __getattr__(self, value):
        if '__' in value:
            return super(DotableCaseInsensitive, self).__getattr__(value)
        return self.__getitem__(value)

    def __getitem__(self, value):
        key = self._match(list(self.keys()), value)
        if key is False:
            raise KeyError('{0} key or attribute not found'.format(value))
        return dict.__getitem__(self, key)


def get_best_fuzzy(value, choices, min_score=75, scorer=fuzz_fuzz.WRatio, return_score=False):
    """Returns the best match in a list of choices using fuzzywuzzy."""

    if not isinstance(value, six.string_types):
        raise ValueError('invalid value. Must be a string.')

    if len(value) < 3:
        raise ValueError('your fuzzy search value must be at least three characters long.')

    # If the value contains _ivar or _mask this is probably and incorrect use
    # of the fuzzy feature. We raise an error.
    if '_ivar' in value:
        raise ValueError('_ivar not allowd in search value.')
    elif '_mask' in value:
        raise ValueError('_mask not allowd in search value.')

    bests = fuzz_proc.extractBests(value, choices, scorer=scorer, score_cutoff=min_score)

    if len(bests) == 0:
        best = None
    elif len(bests) == 1:
        best = bests[0]
    else:
        if bests[0][1] == bests[1][1]:
            best = None
        else:
            best = bests[0]

    if best is None:
        raise ValueError('cannot find a good match for {0!r}. '
                         'Your input value is too ambiguous.'.format(value))

    return best if return_score else best[0]


class FuzzyDict(OrderedDict):
    """A dotable dictionary that uses fuzzywuzzy to select the key."""

    def __getattr__(self, value):
        if '__' in value:
            return super(FuzzyDict, self).__getattr__(value)
        return self.__getitem__(value)

    def __getitem__(self, value):

        if not isinstance(value, six.string_types):
            return self.values()[value]

        if value in self.keys():
            return dict.__getitem__(self, value)

        best = get_best_fuzzy(value, self.keys())

        return dict.__getitem__(self, best)

    def __dir__(self):

        return list(self.keys())


class FuzzyList(list):
    """A list that uses fuzzywuzzy to select the item.

    Parameters:
        the_list (list):
            The list on which we will do fuzzy searching.
        use_fuzzy (function):
            A function that will be used to perform the fuzzy selection
    """

    def __init__(self, the_list, use_fuzzy=None):

        self.use_fuzzy = use_fuzzy if use_fuzzy else get_best_fuzzy

        list.__init__(self, the_list)

    def mapper(self, item):
        """The function that maps each item to the querable string."""

        return str(item)

    def __eq__(self, value):

        self_values = [self.mapper(item) for item in self]

        try:
            best = self.use_fuzzy(value, self_values)
        except ValueError:
            # Second pass, using underscores.
            best = self.use_fuzzy(value.replace(' ', '_'), self_values)

        return self[self_values.index(best)]

    def __contains__(self, value):

        if not isinstance(value, six.string_types):
            return super(FuzzyList, self).__contains__(value)

        try:
            self.__eq__(value)
            return True
        except ValueError:
            return False

    def __getitem__(self, value):

        if isinstance(value, six.string_types):
            return self == value
        else:
            return list.__getitem__(self, value)

    def __getattr__(self, value):

        self_values = [super(FuzzyList, self).__getattribute__('mapper')(item)
                       for item in self]

        if value in self_values:
            return self[value]

        return super(FuzzyList, self).__getattribute__(value)

    def __dir__(self):

        return [self.mapper(item) for item in self]


class OrderedDefaultDict(FuzzyDict):

    def __init__(self, default_factory=None, *args, **kwargs):
        OrderedDict.__init__(self, *args, **kwargs)
        self.default_factory = default_factory

    def __missing__(self, key):
        result = self[key] = self.default_factory()
        return result


class StringFolder(object):
    """
    Class that will fold strings. See 'fold_string'.
    This object may be safely deleted or go out of scope when
    strings have been folded.
    """
    def __init__(self):
        self.unicode_map = {}

    def fold_string(self, s):
        """
        Given a string (or unicode) parameter s, return a string object
        that has the same value as s (and may be s). For all objects
        with a given value, the same object will be returned. For unicode
        objects that can be coerced to a string with the same value, a
        string object will be returned.
        If s is not a string or unicode object, it is returned unchanged.
        :param s: a string or unicode object.
        :return: a string or unicode object.
        """
        # If s is not a string or unicode object, return it unchanged
        if not isinstance(s, basestring):
            return s

        # If s is already a string, then str() has no effect.
        # If s is Unicode, try and encode as a string and use intern.
        # If s is Unicode and can't be encoded as a string, this try
        # will raise a UnicodeEncodeError.
        try:
            return intern(str(s))
        except UnicodeEncodeError:
            # Fall through and handle s as Unicode
            pass

        # Look up the unicode value in the map and return
        # the object from the map. If there is no matching entry,
        # store this unicode object in the map and return it.
        t = self.unicode_map.get(s, None)
        if t is None:
            # Put s in the map
            t = self.unicode_map[s] = s
        return t


def string_folding_wrapper(results, keys=None):
    """
    This generator yields rows from the results as tuples,
    with all string values folded.
    """
    # Get the list of keys so that we build tuples with all
    # the values in key order.

    if keys is None:
        try:
            keys = results.keys()
        except AttributeError as e:
            print('No keys are accessible.  Cannot fold strings!: {0}'.format(e))
            yield results

    folder = StringFolder()
    for row in results:
        yield tuple(
            folder.fold_string(row.__getattribute__(key))
            for key in keys
        )

1	#!/usr/bin/env python
2	# encoding: utf-8
3	#
4	# structs.py
5	#
6	# Created by José Sánchez-Gallego on 17 Aug 2017.
7
8
9	from __future__ import division	2✔
10	from __future__ import print_function	2✔
11	from __future__ import absolute_import	2✔
12
13	from collections import OrderedDict	2✔
14
15	import six	2✔
16
17	from fuzzywuzzy import fuzz as fuzz_fuzz	2✔
18	from fuzzywuzzy import process as fuzz_proc	2✔
19
20
21	__ALL__ = ['FuzzyDict', 'Dotable', 'DotableCaseInsensitive', 'get_best_fuzzy',	2✔
22	'FuzzyList', 'string_folding_wrapper']
23
24
25	class Dotable(dict):	2✔
26	"""A custom dict class that allows dot access to nested dictionaries.
27
28	Copied from http://hayd.github.io/2013/dotable-dictionaries/. Note that
29	this allows you to use dots to get dictionary values, but not to set them.
30
31	"""
32
33	def __getattr__(self, value):	2✔
34	if '__' in value:	2!
35	return dict.__getattr__(self, value)	×
36	else:
37	return self.__getitem__(value)	2✔
38
39	# def __init__(self, d):
40	# dict.__init__(self, ((k, self.parse(v)) for k, v in d.iteritems()))
41
42	@classmethod	2✔
43	def parse(cls, v):
44	if isinstance(v, dict):	×
45	return cls(v)	×
46	elif isinstance(v, list):	×
47	return [cls.parse(i) for i in v]	×
48	else:
49	return v	×
50
51
52	class DotableCaseInsensitive(Dotable):	2✔
53	"""Like dotable but access to attributes and keys is case insensitive."""
54
55	def _match(self, list_of_keys, value):	2✔
56
57	lower_values = [str(xx).lower() for xx in list_of_keys]	2✔
58	if value.lower() in lower_values:	2!
59	return list_of_keys[lower_values.index(value.lower())]	2✔
60	else:
61	return False	×
62
63	def __getattr__(self, value):	2✔
64	if '__' in value:	2!
65	return super(DotableCaseInsensitive, self).__getattr__(value)	×
66	return self.__getitem__(value)	2✔
67
68	def __getitem__(self, value):	2✔
69	key = self._match(list(self.keys()), value)	2✔
70	if key is False:	2!
71	raise KeyError('{0} key or attribute not found'.format(value))	×
72	return dict.__getitem__(self, key)	2✔
73
74
75	def get_best_fuzzy(value, choices, min_score=75, scorer=fuzz_fuzz.WRatio, return_score=False):	2✔
76	"""Returns the best match in a list of choices using fuzzywuzzy."""
77
78	if not isinstance(value, six.string_types):	2!
79	raise ValueError('invalid value. Must be a string.')	×
80
81	if len(value) < 3:	2!
82	raise ValueError('your fuzzy search value must be at least three characters long.')	×
83
84	# If the value contains _ivar or _mask this is probably and incorrect use
85	# of the fuzzy feature. We raise an error.
86	if '_ivar' in value:	2!
87	raise ValueError('_ivar not allowd in search value.')	×
88	elif '_mask' in value:	2!
89	raise ValueError('_mask not allowd in search value.')	×
90
91	bests = fuzz_proc.extractBests(value, choices, scorer=scorer, score_cutoff=min_score)	2✔
92
93	if len(bests) == 0:	2✔
94	best = None	2✔
95	elif len(bests) == 1:	2✔
96	best = bests[0]	2✔
97	else:
98	if bests[0][1] == bests[1][1]:	2✔
99	best = None	2✔
100	else:
101	best = bests[0]	2✔
102
103	if best is None:	2✔
104	raise ValueError('cannot find a good match for {0!r}. '	2✔
105	'Your input value is too ambiguous.'.format(value))
106
107	return best if return_score else best[0]	2✔
108
109
110	class FuzzyDict(OrderedDict):	2✔
111	"""A dotable dictionary that uses fuzzywuzzy to select the key."""
112
113	def __getattr__(self, value):	2✔
114	if '__' in value:	×
115	return super(FuzzyDict, self).__getattr__(value)	×
116	return self.__getitem__(value)	×
117
118	def __getitem__(self, value):	2✔
119
120	if not isinstance(value, six.string_types):	2!
121	return self.values()[value]	×
122
123	if value in self.keys():	2!
124	return dict.__getitem__(self, value)	2✔
125
126	best = get_best_fuzzy(value, self.keys())	×
127
128	return dict.__getitem__(self, best)	×
129
130	def __dir__(self):	2✔
131
132	return list(self.keys())	2✔
133
134
135	class FuzzyList(list):	2✔
136	"""A list that uses fuzzywuzzy to select the item.
137
138	Parameters:
139	the_list (list):
140	The list on which we will do fuzzy searching.
141	use_fuzzy (function):
142	A function that will be used to perform the fuzzy selection
143	"""
144
145	def __init__(self, the_list, use_fuzzy=None):	2✔
146
147	self.use_fuzzy = use_fuzzy if use_fuzzy else get_best_fuzzy	2✔
148
149	list.__init__(self, the_list)	2✔
150
151	def mapper(self, item):	2✔
152	"""The function that maps each item to the querable string."""
153
154	return str(item)	×
155
156	def __eq__(self, value):	2✔
157
158	self_values = [self.mapper(item) for item in self]	2✔
159
160	try:	2✔
161	best = self.use_fuzzy(value, self_values)	2✔
162	except ValueError:	2✔
163	# Second pass, using underscores.
164	best = self.use_fuzzy(value.replace(' ', '_'), self_values)	2✔
165
166	return self[self_values.index(best)]	2✔
167
168	def __contains__(self, value):	2✔
169
170	if not isinstance(value, six.string_types):	2✔
171	return super(FuzzyList, self).__contains__(value)	2✔
172
173	try:	2✔
174	self.__eq__(value)	2✔
175	return True	2✔
176	except ValueError:	2✔
177	return False	2✔
178
179	def __getitem__(self, value):	2✔
180
181	if isinstance(value, six.string_types):	2✔
182	return self == value	2✔
183	else:
184	return list.__getitem__(self, value)	2✔
185
186	def __getattr__(self, value):	2✔
187
188	self_values = [super(FuzzyList, self).__getattribute__('mapper')(item)	2✔
189	for item in self]
190
191	if value in self_values:	2!
192	return self[value]	×
193
194	return super(FuzzyList, self).__getattribute__(value)	2✔
195
196	def __dir__(self):	2✔
197
198	return [self.mapper(item) for item in self]	×
199
200
201	class OrderedDefaultDict(FuzzyDict):	2✔
202
203	def __init__(self, default_factory=None, args, *kwargs):	2✔
204	OrderedDict.__init__(self, args, *kwargs)	×
205	self.default_factory = default_factory	×
206
207	def __missing__(self, key):	2✔
208	result = self[key] = self.default_factory()	×
209	return result	×
210
211
212	class StringFolder(object):	2✔
213	"""
214	Class that will fold strings. See 'fold_string'.
215	This object may be safely deleted or go out of scope when
216	strings have been folded.
217	"""
218	def __init__(self):	2✔
219	self.unicode_map = {}	×
220
221	def fold_string(self, s):	2✔
222	"""
223	Given a string (or unicode) parameter s, return a string object
224	that has the same value as s (and may be s). For all objects
225	with a given value, the same object will be returned. For unicode
226	objects that can be coerced to a string with the same value, a
227	string object will be returned.
228	If s is not a string or unicode object, it is returned unchanged.
229	:param s: a string or unicode object.
230	:return: a string or unicode object.
231	"""
232	# If s is not a string or unicode object, return it unchanged
233	if not isinstance(s, basestring):	×
234	return s	×
235
236	# If s is already a string, then str() has no effect.
237	# If s is Unicode, try and encode as a string and use intern.
238	# If s is Unicode and can't be encoded as a string, this try
239	# will raise a UnicodeEncodeError.
240	try:	×
241	return intern(str(s))	×
242	except UnicodeEncodeError:	×
243	# Fall through and handle s as Unicode
244	pass	×
245
246	# Look up the unicode value in the map and return
247	# the object from the map. If there is no matching entry,
248	# store this unicode object in the map and return it.
249	t = self.unicode_map.get(s, None)	×
250	if t is None:	×
251	# Put s in the map
252	t = self.unicode_map[s] = s	×
253	return t	×
254
255
256	def string_folding_wrapper(results, keys=None):	2✔
257	"""
258	This generator yields rows from the results as tuples,
259	with all string values folded.
260	"""
261	# Get the list of keys so that we build tuples with all
262	# the values in key order.
263
264	if keys is None:	×
265	try:	×
266	keys = results.keys()	×
267	except AttributeError as e:	×
268	print('No keys are accessible. Cannot fold strings!: {0}'.format(e))	×
269	yield results	×
270
271	folder = StringFolder()	×
272	for row in results:	×
273	yield tuple(	×
274	folder.fold_string(row.__getattribute__(key))
275	for key in keys
276	)

sdss / marvin / 1146

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous