#2246

pending completion

Build # #2246

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/3e6cc6df5cea45a7ab709d14330b36ad445517d6">3e6cc6df5<a href="https://github.com/datajoint/datajoint-python/commit/3e6cc6df5cea45a7ab709d14330b36ad445517d6">">Merge </a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/f05c50d1ac6666cf0afe8781b0b9bc48193e3ad0">f05c50d1a</a>">f05c50d1a</a><a href="https://github.com/datajoint/datajoint-python/commit/3e6cc6df5cea45a7ab709d14330b36ad445517d6"> into bff58baac">bff58baac</a>

Run Details

211 of 211 new or added lines in 8 files covered. (100.0%)

2627 of 2879 relevant lines covered (91.25%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.33

/datajoint/fetch.py

from functools import partial
from pathlib import Path
import warnings
import pandas
import itertools
import re
import numpy as np
import uuid
import numbers
from . import blob, hash
from .errors import DataJointError
from .settings import config
from .utils import OrderedDict, safe_write

class key:
    """
    object that allows requesting the primary key as an argument in expression.fetch()
    The string "KEY" can be used instead of the class key
    """
    pass


def is_key(attr):
    return attr is key or attr == 'KEY'


def to_dicts(recarray):
    """convert record array to a dictionaries"""
    for rec in recarray:
        yield OrderedDict(zip(recarray.dtype.names, rec.tolist()))


def _get(connection, attr, data, squeeze, download_path):
    """
    This function is called for every attribute 

    :param connection: a dj.Connection object
    :param attr: attribute name from the table's heading
    :param data: literal value fetched from the table
    :param squeeze: if True squeeze blobs
    :param download_path: for fetches that download data, e.g. attachments
    :return: unpacked data
    """
    if data is None:
        return

    extern = connection.schemas[attr.database].external[attr.store] if attr.is_external else None

    # apply attribute adapter if present
    adapt = attr.adapter.get if attr.adapter else lambda x: x

    if attr.is_filepath:
        return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0])

    if attr.is_attachment:
        # Steps:
        # 1. get the attachment filename
        # 2. check if the file already exists at download_path, verify checksum
        # 3. if exists and checksum passes then return the local filepath
        # 4. Otherwise, download the remote file and return the new filepath
        _uuid = uuid.UUID(bytes=data) if attr.is_external else None
        attachment_name = (extern.get_attachment_name(_uuid) if attr.is_external
                    else data.split(b"\0", 1)[0].decode())
        local_filepath = Path(download_path) / attachment_name
        if local_filepath.is_file():
            attachment_checksum = _uuid if attr.is_external else hash.uuid_from_buffer(data)
            if attachment_checksum == hash.uuid_from_file(local_filepath, init_string=attachment_name + '\0'):
                return adapt(local_filepath)  # checksum passed, no need to download again
            # generate the next available alias filename
            for n in itertools.count():
                f = local_filepath.parent / (local_filepath.stem + '_%04x' % n + local_filepath.suffix)
                if not f.is_file():
                    local_filepath = f
                    break
                if attachment_checksum == hash.uuid_from_file(f, init_string=attachment_name + '\0'):
                    return adapt(f)  # checksum passed, no need to download again
        # Save attachment
        if attr.is_external:
            extern.download_attachment(_uuid, attachment_name, local_filepath)
        else:
            # write from buffer
            safe_write(local_filepath, data.split(b"\0", 1)[1])
        return adapt(local_filepath)  # download file from remote store

    return adapt(uuid.UUID(bytes=data) if attr.uuid else (
            blob.unpack(extern.get(uuid.UUID(bytes=data)) if attr.is_external else data, squeeze=squeeze) 
            if attr.is_blob else data))


def _flatten_attribute_list(primary_key, attrs):
    """
    :param primary_key: list of attributes in primary key
    :param attrs: list of attribute names, which may include "KEY", "KEY DESC" or "KEY ASC"
    :return: generator of attributes where "KEY" is replaces with its component attributes
    """
    for a in attrs:
        if re.match(r'^\s*KEY(\s+[aA][Ss][Cc])?\s*$', a):
            yield from primary_key
        elif re.match(r'^\s*KEY\s+[Dd][Ee][Ss][Cc]\s*$', a):
            yield from (q + ' DESC' for q in primary_key)
        else:
            yield a


class Fetch:
    """
    A fetch object that handles retrieving elements from the table expression.
    :param expression: the QueryExpression object to fetch from.
    """

    def __init__(self, expression):
        self._expression = expression

    def __call__(self, *attrs, offset=None, limit=None, order_by=None, format=None, as_dict=None,
                 squeeze=False, download_path='.'):
        """
        Fetches the expression results from the database into an np.array or list of dictionaries and
        unpacks blob attributes.
        :param attrs: zero or more attributes to fetch. If not provided, the call will return
        all attributes of this relation. If provided, returns tuples with an entry for each attribute.
        :param offset: the number of tuples to skip in the returned result
        :param limit: the maximum number of tuples to return
        :param order_by: a single attribute or the list of attributes to order the results.
                No ordering should be assumed if order_by=None.
                To reverse the order, add DESC to the attribute name or names: e.g. ("age DESC", "frequency")
                To order by primary key, use "KEY" or "KEY DESC"
        :param format: Effective when as_dict=None and when attrs is empty
                None: default from config['fetch_format'] or 'array' if not configured
                "array": use numpy.key_array
                "frame": output pandas.DataFrame. .
        :param as_dict: returns a list of dictionaries instead of a record array.
                Defaults to False for .fetch() and to True for .fetch('KEY')
        :param squeeze:  if True, remove extra dimensions from arrays
        :param download_path: for fetches that download data, e.g. attachments
        :return: the contents of the relation in the form of a structured numpy.array or a dict list
        """
        if order_by is not None:
            # if 'order_by' passed in a string, make into list
            if isinstance(order_by, str):
                order_by = [order_by]
            # expand "KEY" or "KEY DESC"
            order_by = list(_flatten_attribute_list(self._expression.primary_key, order_by))

        attrs_as_dict = as_dict and attrs
        if attrs_as_dict:
            # absorb KEY into attrs and prepare to return attributes as dict (issue #595)
            if any(is_key(k) for k in attrs):
                attrs = list(self._expression.primary_key) + [
                    a for a in attrs if a not in self._expression.primary_key]
        if as_dict is None:
            as_dict = bool(attrs)  # default to True for "KEY" and False when fetching entire result
        # format should not be specified with attrs or is_dict=True
        if format is not None and (as_dict or attrs):
            raise DataJointError('Cannot specify output format when as_dict=True or '
                                 'when attributes are selected to be fetched separately.')
        if format not in {None, "array", "frame"}:
            raise DataJointError('Fetch output format must be in {{"array", "frame"}} but "{}" was given'.format(format))

        if not (attrs or as_dict) and format is None:
            format = config['fetch_format']  # default to array
            if format not in {"array", "frame"}:
                raise DataJointError('Invalid entry "{}" in datajoint.config["fetch_format"]: use "array" or "frame"'.format(format))

        if limit is None and offset is not None:
            warnings.warn('Offset set, but no limit. Setting limit to a large number. '
                          'Consider setting a limit explicitly.')
            limit = 8000000000  # just a very large number to effect no limit

        get = partial(_get, self._expression.connection, squeeze=squeeze, download_path=download_path)
        if attrs:  # a list of attributes provided
            attributes = [a for a in attrs if not is_key(a)]
            ret = self._expression.proj(*attributes).fetch(
                offset=offset, limit=limit, order_by=order_by,
                as_dict=False, squeeze=squeeze, download_path=download_path)
            if attrs_as_dict:
                ret = [{k: v for k, v in zip(ret.dtype.names, x) if k in attrs} for x in ret]
            else:
                return_values = [
                    list((to_dicts if as_dict else lambda x: x)(ret[self._expression.primary_key])) if is_key(attribute)
                    else ret[attribute] for attribute in attrs]
                ret = return_values[0] if len(attrs) == 1 else return_values
        else:  # fetch all attributes as a numpy.record_array or pandas.DataFrame
            cur = self._expression.cursor(as_dict=as_dict, limit=limit, offset=offset, order_by=order_by)
            heading = self._expression.heading
            if as_dict:
                ret = [OrderedDict((name, get(heading[name], d[name])) for name in heading.names) for d in cur]
            else:
                ret = list(cur.fetchall())
                record_type = (heading.as_dtype if not ret else np.dtype(
                    [(name, type(value))   # use the first element to determine the type for blobs
                        if heading[name].is_blob and isinstance(value, numbers.Number)
                        else (name, heading.as_dtype[name])
                        for value, name in zip(ret[0], heading.as_dtype.names)]))
                try:
                    ret = np.array(ret, dtype=record_type)
                except Exception as e:
                    raise
                for name in heading:
                    ret[name] = list(map(partial(get, heading[name]), ret[name]))
                if format == "frame":
                    ret = pandas.DataFrame(ret).set_index(heading.primary_key)
        return ret


class Fetch1:
    """
    Fetch object for fetching exactly one row.
    :param relation: relation the fetch object fetches data from
    """
    def __init__(self, relation):
        self._expression = relation

    def __call__(self, *attrs, squeeze=False, download_path='.'):
        """
        Fetches the expression results from the database when the expression is known to yield only one entry.

        If no attributes are specified, returns the result as a dict.
        If attributes are specified returns the corresponding results as a tuple.

        Examples:
        d = rel.fetch1()   # as a dictionary
        a, b = rel.fetch1('a', 'b')   # as a tuple

        :params *attrs: attributes to return when expanding into a tuple. If empty, the return result is a dict
        :param squeeze:  When true, remove extra dimensions from arrays in attributes
        :param download_path: for fetches that download data, e.g. attachments
        :return: the one tuple in the relation in the form of a dict
        """
        heading = self._expression.heading

        if not attrs:  # fetch all attributes, return as ordered dict
            cur = self._expression.cursor(as_dict=True)
            ret = cur.fetchone()
            if not ret or cur.fetchone():
                raise DataJointError('fetch1 should only be used for relations with exactly one tuple')
            ret = OrderedDict((name, _get(self._expression.connection, heading[name], ret[name],
                                          squeeze=squeeze, download_path=download_path))
                              for name in heading.names)
        else:  # fetch some attributes, return as tuple
            attributes = [a for a in attrs if not is_key(a)]
            result = self._expression.proj(*attributes).fetch(squeeze=squeeze, download_path=download_path)
            if len(result) != 1:
                raise DataJointError('fetch1 should only return one tuple. %d tuples were found' % len(result))
            return_values = tuple(
                next(to_dicts(result[self._expression.primary_key])) if is_key(attribute) else result[attribute][0]
                for attribute in attrs)
            ret = return_values[0] if len(attrs) == 1 else return_values
        return ret

1	from functools import partial	1✔
2	from pathlib import Path	1✔
3	import warnings	1✔
4	import pandas	1✔
5	import itertools	1✔
6	import re	1✔
7	import numpy as np	1✔
8	import uuid	1✔
9	import numbers	1✔
10	from . import blob, hash	1✔
11	from .errors import DataJointError	1✔
12	from .settings import config	1✔
13	from .utils import OrderedDict, safe_write	1✔
14
15	class key:	1✔
16	"""
17	object that allows requesting the primary key as an argument in expression.fetch()
18	The string "KEY" can be used instead of the class key
19	"""
20	pass	1✔
21
22
23	def is_key(attr):	1✔
24	return attr is key or attr == 'KEY'	1✔
25
26
27	def to_dicts(recarray):	1✔
28	"""convert record array to a dictionaries"""
29	for rec in recarray:	1✔
30	yield OrderedDict(zip(recarray.dtype.names, rec.tolist()))	1✔
31
32
33	def _get(connection, attr, data, squeeze, download_path):	1✔
34	"""
35	This function is called for every attribute
36
37	:param connection: a dj.Connection object
38	:param attr: attribute name from the table's heading
39	:param data: literal value fetched from the table
40	:param squeeze: if True squeeze blobs
41	:param download_path: for fetches that download data, e.g. attachments
42	:return: unpacked data
43	"""
44	if data is None:	1✔
45	return	1✔
46
47	extern = connection.schemas[attr.database].external[attr.store] if attr.is_external else None	1✔
48
49	# apply attribute adapter if present
50	adapt = attr.adapter.get if attr.adapter else lambda x: x	1✔
51
52	if attr.is_filepath:	1✔
53	return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0])	1✔
54
55	if attr.is_attachment:	1✔
56	# Steps:
57	# 1. get the attachment filename
58	# 2. check if the file already exists at download_path, verify checksum
59	# 3. if exists and checksum passes then return the local filepath
60	# 4. Otherwise, download the remote file and return the new filepath
61	_uuid = uuid.UUID(bytes=data) if attr.is_external else None	1✔
62	attachment_name = (extern.get_attachment_name(_uuid) if attr.is_external	1✔
63	else data.split(b"\0", 1)[0].decode())
64	local_filepath = Path(download_path) / attachment_name	1✔
65	if local_filepath.is_file():	1✔
66	attachment_checksum = _uuid if attr.is_external else hash.uuid_from_buffer(data)	1✔
67	if attachment_checksum == hash.uuid_from_file(local_filepath, init_string=attachment_name + '\0'):	1✔
68	return adapt(local_filepath) # checksum passed, no need to download again	1✔
69	# generate the next available alias filename
70	for n in itertools.count():	1✔
71	f = local_filepath.parent / (local_filepath.stem + '_%04x' % n + local_filepath.suffix)	1✔
72	if not f.is_file():	1✔
73	local_filepath = f	1✔
74	break	1✔
75	if attachment_checksum == hash.uuid_from_file(f, init_string=attachment_name + '\0'):	×
76	return adapt(f) # checksum passed, no need to download again	×
77	# Save attachment
78	if attr.is_external:	1✔
79	extern.download_attachment(_uuid, attachment_name, local_filepath)	1✔
80	else:
81	# write from buffer
82	safe_write(local_filepath, data.split(b"\0", 1)[1])	1✔
83	return adapt(local_filepath) # download file from remote store	1✔
84
85	return adapt(uuid.UUID(bytes=data) if attr.uuid else (	1✔
86	blob.unpack(extern.get(uuid.UUID(bytes=data)) if attr.is_external else data, squeeze=squeeze)
87	if attr.is_blob else data))
88
89
90	def _flatten_attribute_list(primary_key, attrs):	1✔
91	"""
92	:param primary_key: list of attributes in primary key
93	:param attrs: list of attribute names, which may include "KEY", "KEY DESC" or "KEY ASC"
94	:return: generator of attributes where "KEY" is replaces with its component attributes
95	"""
96	for a in attrs:	1✔
97	if re.match(r'^\sKEY(\s+[aA][Ss][Cc])?\s$', a):	1✔
98	yield from primary_key	1✔
99	elif re.match(r'^\sKEY\s+[Dd][Ee][Ss][Cc]\s$', a):	1✔
100	yield from (q + ' DESC' for q in primary_key)	1✔
101	else:
102	yield a	1✔
103
104
105	class Fetch:	1✔
106	"""
107	A fetch object that handles retrieving elements from the table expression.
108	:param expression: the QueryExpression object to fetch from.
109	"""
110
111	def __init__(self, expression):	1✔
112	self._expression = expression	1✔
113
114	def __call__(self, *attrs, offset=None, limit=None, order_by=None, format=None, as_dict=None,	1✔
115	squeeze=False, download_path='.'):
116	"""
117	Fetches the expression results from the database into an np.array or list of dictionaries and
118	unpacks blob attributes.
119	:param attrs: zero or more attributes to fetch. If not provided, the call will return
120	all attributes of this relation. If provided, returns tuples with an entry for each attribute.
121	:param offset: the number of tuples to skip in the returned result
122	:param limit: the maximum number of tuples to return
123	:param order_by: a single attribute or the list of attributes to order the results.
124	No ordering should be assumed if order_by=None.
125	To reverse the order, add DESC to the attribute name or names: e.g. ("age DESC", "frequency")
126	To order by primary key, use "KEY" or "KEY DESC"
127	:param format: Effective when as_dict=None and when attrs is empty
128	None: default from config['fetch_format'] or 'array' if not configured
129	"array": use numpy.key_array
130	"frame": output pandas.DataFrame. .
131	:param as_dict: returns a list of dictionaries instead of a record array.
132	Defaults to False for .fetch() and to True for .fetch('KEY')
133	:param squeeze: if True, remove extra dimensions from arrays
134	:param download_path: for fetches that download data, e.g. attachments
135	:return: the contents of the relation in the form of a structured numpy.array or a dict list
136	"""
137	if order_by is not None:	1✔
138	# if 'order_by' passed in a string, make into list
139	if isinstance(order_by, str):	1✔
140	order_by = [order_by]	1✔
141	# expand "KEY" or "KEY DESC"
142	order_by = list(_flatten_attribute_list(self._expression.primary_key, order_by))	1✔
143
144	attrs_as_dict = as_dict and attrs	1✔
145	if attrs_as_dict:	1✔
146	# absorb KEY into attrs and prepare to return attributes as dict (issue #595)
147	if any(is_key(k) for k in attrs):	1✔
148	attrs = list(self._expression.primary_key) + [	×
149	a for a in attrs if a not in self._expression.primary_key]
150	if as_dict is None:	1✔
151	as_dict = bool(attrs) # default to True for "KEY" and False when fetching entire result	1✔
152	# format should not be specified with attrs or is_dict=True
153	if format is not None and (as_dict or attrs):	1✔
154	raise DataJointError('Cannot specify output format when as_dict=True or '	×
155	'when attributes are selected to be fetched separately.')
156	if format not in {None, "array", "frame"}:	1✔
157	raise DataJointError('Fetch output format must be in {{"array", "frame"}} but "{}" was given'.format(format))	×
158
159	if not (attrs or as_dict) and format is None:	1✔
160	format = config['fetch_format'] # default to array	1✔
161	if format not in {"array", "frame"}:	1✔
162	raise DataJointError('Invalid entry "{}" in datajoint.config["fetch_format"]: use "array" or "frame"'.format(format))	×
163
164	if limit is None and offset is not None:	1✔
165	warnings.warn('Offset set, but no limit. Setting limit to a large number. '	1✔
166	'Consider setting a limit explicitly.')
167	limit = 8000000000 # just a very large number to effect no limit	1✔
168
169	get = partial(_get, self._expression.connection, squeeze=squeeze, download_path=download_path)	1✔
170	if attrs: # a list of attributes provided	1✔
171	attributes = [a for a in attrs if not is_key(a)]	1✔
172	ret = self._expression.proj(*attributes).fetch(	1✔
173	offset=offset, limit=limit, order_by=order_by,
174	as_dict=False, squeeze=squeeze, download_path=download_path)
175	if attrs_as_dict:	1✔
176	ret = [{k: v for k, v in zip(ret.dtype.names, x) if k in attrs} for x in ret]	1✔
177	else:
178	return_values = [	1✔
179	list((to_dicts if as_dict else lambda x: x)(ret[self._expression.primary_key])) if is_key(attribute)
180	else ret[attribute] for attribute in attrs]
181	ret = return_values[0] if len(attrs) == 1 else return_values	1✔
182	else: # fetch all attributes as a numpy.record_array or pandas.DataFrame
183	cur = self._expression.cursor(as_dict=as_dict, limit=limit, offset=offset, order_by=order_by)	1✔
184	heading = self._expression.heading	1✔
185	if as_dict:	1✔
186	ret = [OrderedDict((name, get(heading[name], d[name])) for name in heading.names) for d in cur]	1✔
187	else:
188	ret = list(cur.fetchall())	1✔
189	record_type = (heading.as_dtype if not ret else np.dtype(	1✔
190	[(name, type(value)) # use the first element to determine the type for blobs
191	if heading[name].is_blob and isinstance(value, numbers.Number)
192	else (name, heading.as_dtype[name])
193	for value, name in zip(ret[0], heading.as_dtype.names)]))
194	try:	1✔
195	ret = np.array(ret, dtype=record_type)	1✔
196	except Exception as e:	×
197	raise	×
198	for name in heading:	1✔
199	ret[name] = list(map(partial(get, heading[name]), ret[name]))	1✔
200	if format == "frame":	1✔
201	ret = pandas.DataFrame(ret).set_index(heading.primary_key)	1✔
202	return ret	1✔
203
204
205	class Fetch1:	1✔
206	"""
207	Fetch object for fetching exactly one row.
208	:param relation: relation the fetch object fetches data from
209	"""
210	def __init__(self, relation):	1✔
211	self._expression = relation	1✔
212
213	def __call__(self, *attrs, squeeze=False, download_path='.'):	1✔
214	"""
215	Fetches the expression results from the database when the expression is known to yield only one entry.
216
217	If no attributes are specified, returns the result as a dict.
218	If attributes are specified returns the corresponding results as a tuple.
219
220	Examples:
221	d = rel.fetch1() # as a dictionary
222	a, b = rel.fetch1('a', 'b') # as a tuple
223
224	:params *attrs: attributes to return when expanding into a tuple. If empty, the return result is a dict
225	:param squeeze: When true, remove extra dimensions from arrays in attributes
226	:param download_path: for fetches that download data, e.g. attachments
227	:return: the one tuple in the relation in the form of a dict
228	"""
229	heading = self._expression.heading	1✔
230
231	if not attrs: # fetch all attributes, return as ordered dict	1✔
232	cur = self._expression.cursor(as_dict=True)	1✔
233	ret = cur.fetchone()	1✔
234	if not ret or cur.fetchone():	1✔
235	raise DataJointError('fetch1 should only be used for relations with exactly one tuple')	1✔
236	ret = OrderedDict((name, _get(self._expression.connection, heading[name], ret[name],	1✔
237	squeeze=squeeze, download_path=download_path))
238	for name in heading.names)
239	else: # fetch some attributes, return as tuple
240	attributes = [a for a in attrs if not is_key(a)]	1✔
241	result = self._expression.proj(*attributes).fetch(squeeze=squeeze, download_path=download_path)	1✔
242	if len(result) != 1:	1✔
243	raise DataJointError('fetch1 should only return one tuple. %d tuples were found' % len(result))	1✔
244	return_values = tuple(	1✔
245	next(to_dicts(result[self._expression.primary_key])) if is_key(attribute) else result[attribute][0]
246	for attribute in attrs)
247	ret = return_values[0] if len(attrs) == 1 else return_values	1✔
248	return ret	1✔

datajoint / datajoint-python / #2246

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous