#12901

pending completion

Build # #12901

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/864be0ccca479b08973e3dc4531e096bf97088fa">864be0ccc<a href="https://github.com/datajoint/datajoint-python/commit/864be0ccca479b08973e3dc4531e096bf97088fa">">Merge </a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/d3b1af13150e5e3a26410b98f2dc2a19ec2b5368">d3b1af131</a>">d3b1af131</a><a href="https://github.com/datajoint/datajoint-python/commit/864be0ccca479b08973e3dc4531e096bf97088fa"> into 3b6e84588">3b6e84588</a>

Run Details

81 of 81 new or added lines in 10 files covered. (100.0%)

3134 of 3489 relevant lines covered (89.83%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.21

/datajoint/user_tables.py

"""
Hosts the table tiers, user tables should be derived from.
"""
import collections
import pandas as pd
import numpy as np
from .table import Table
from .autopopulate import AutoPopulate
from .logging import logger
from .utils import from_camel_case, ClassProperty, dict_to_uuid
from .errors import DataJointError, DuplicateError
from .expression import QueryExpression

_base_regexp = r"[a-z][a-z0-9]*(_[a-z][a-z0-9]*)*"

# attributes that trigger instantiation of user classes


supported_class_attrs = {
    "key_source",
    "describe",
    "alter",
    "heading",
    "populate",
    "progress",
    "primary_key",
    "proj",
    "aggr",
    "join",
    "fetch",
    "fetch1",
    "head",
    "tail",
    "descendants",
    "ancestors",
    "parts",
    "parents",
    "children",
    "insert",
    "insert1",
    "update1",
    "drop",
    "drop_quick",
    "delete",
    "delete_quick",
}


class TableMeta(type):
    """
    TableMeta subclasses allow applying some instance methods and properties directly
    at class level. For example, this allows Table.fetch() instead of Table().fetch().
    """

    def __getattribute__(cls, name):
        # trigger instantiation for supported class attrs
        return (
            cls().__getattribute__(name)
            if name in supported_class_attrs
            else super().__getattribute__(name)
        )

    def __and__(cls, arg):
        return cls() & arg

    def __xor__(cls, arg):
        return cls() ^ arg

    def __sub__(cls, arg):
        return cls() - arg

    def __neg__(cls):
        return -cls()

    def __mul__(cls, arg):
        return cls() * arg

    def __matmul__(cls, arg):
        return cls() @ arg

    def __add__(cls, arg):
        return cls() + arg

    def __iter__(cls):
        return iter(cls())


class UserTable(Table, metaclass=TableMeta):
    """
    A subclass of UserTable is a dedicated class interfacing a base table.
    UserTable is initialized by the decorator generated by schema().
    """

    # set by @schema
    _connection = None
    _heading = None
    _support = None

    # set by subclass
    tier_regexp = None
    _prefix = None

    @property
    def definition(self):
        """
        :return: a string containing the table definition using the DataJoint DDL.
        """
        raise NotImplementedError(
            'Subclasses of Table must implement the property "definition"'
        )

    @ClassProperty
    def connection(cls):
        return cls._connection

    @ClassProperty
    def table_name(cls):
        """
        :return: the table name of the table formatted for mysql.
        """
        if cls._prefix is None:
            raise AttributeError("Class prefix is not defined!")
        return cls._prefix + from_camel_case(cls.__name__)

    @ClassProperty
    def full_table_name(cls):
        if cls not in {Manual, Imported, Lookup, Computed, Part, Params, UserTable}:
            # for derived classes only
            if cls.database is None:
                raise DataJointError(
                    "Class %s is not properly declared (schema decorator not applied?)"
                    % cls.__name__
                )
            return r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name)


class Manual(UserTable):
    """
    Inherit from this class if the table's values are entered manually.
    """

    _prefix = r""
    tier_regexp = r"(?P<manual>" + _prefix + _base_regexp + ")"


class Lookup(UserTable):
    """
    Inherit from this class if the table's values are for lookup. This is
    currently equivalent to defining the table as Manual and serves semantic
    purposes only.
    """

    _prefix = "#"
    tier_regexp = (
        r"(?P<lookup>" + _prefix + _base_regexp.replace("TIER", "lookup") + ")"
    )


class Params(UserTable):
    """
    Inherit from this class if the table's values are paramsets. Adds hashing of fields
    that include 'param' and stores as additional unique index 'paramset hash'
    """

    _prefix = "##"
    tier_regexp = (
        r"(?P<params>" + _prefix + _base_regexp.replace("TIER", "params") + ")"
    )

    def __init__(self):
        super().__init__()
        if "params" not in self.definition:
            raise DataJointError("Params tables must have at least one 'params' field")

        # NOTE: Ideally, this would be a UUID, not char(36), but I was having trouble
        # fetching by UUID - what's the syntax? (self * f"hash='{UUID}'").fetch()
        self.definition += (
            "hide__paramset_hash: char(36)\nunique index (hide__paramset_hash)\n"
        )

    def insert(self, rows, **kwargs):
        if isinstance(rows, QueryExpression):
            raise NotImplementedError(
                "Params tables do not yet support insertion via QueryExpression"
            )

        rows = self._normalize_insert_data(rows)

        new_rows = []
        for row in rows:
            if isinstance(row, np.record):
                row_values = tuple(row)
            elif isinstance(row, collections.abc.Mapping):
                row_values = row.values()
            else:
                row_values = row

            paramset_hash = str(
                dict_to_uuid(
                    {
                        field: value
                        for field, value in zip(self.heading, row_values)
                        if "param" in field and "hash" not in field
                    }
                )
            )

            existing_key = (self & f'hide__paramset_hash="{paramset_hash}"').fetch(
                "KEY"
            )
            if existing_key:
                logger.warning(
                    "Paramset already included with the following primary "
                    + f"key: {existing_key[0]}"
                )
                continue

            if isinstance(row, collections.abc.Mapping):
                new_rows.append(dict(**row, hide__paramset_hash=paramset_hash))
            else:
                new_rows.append([*row, paramset_hash])

        super().insert(new_rows, **kwargs)


class Imported(UserTable, AutoPopulate):
    """
    Inherit from this class if the table's values are imported from external data sources.
    The inherited class must at least provide the function `_make_tuples`.
    """

    _prefix = "_"
    tier_regexp = r"(?P<imported>" + _prefix + _base_regexp + ")"


class Computed(UserTable, AutoPopulate):
    """
    Inherit from this class if the table's values are computed from other tables in the schema.
    The inherited class must at least provide the function `_make_tuples`.
    """

    _prefix = "__"
    tier_regexp = r"(?P<computed>" + _prefix + _base_regexp + ")"


class Part(UserTable):
    """
    Inherit from this class if the table's values are details of an entry in another table
    and if this table is populated by the other table. For example, the entries inheriting from
    dj.Part could be single entries of a matrix, while the parent table refers to the entire matrix.
    Part tables are implemented as classes inside classes.
    """

    _connection = None
    _master = None

    tier_regexp = (
        r"(?P<master>"
        + "|".join(
            [c.tier_regexp for c in (Manual, Lookup, Imported, Computed, Params)]
        )
        + r"){1,1}"
        + "__"
        + r"(?P<part>"
        + _base_regexp
        + ")"
    )

    @ClassProperty
    def connection(cls):
        return cls._connection

    @ClassProperty
    def full_table_name(cls):
        return (
            None
            if cls.database is None or cls.table_name is None
            else r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name)
        )

    @ClassProperty
    def master(cls):
        return cls._master

    @ClassProperty
    def table_name(cls):
        return (
            None
            if cls.master is None
            else cls.master.table_name + "__" + from_camel_case(cls.__name__)
        )

    def delete(self, force=False):
        """
        unless force is True, prohibits direct deletes from parts.
        """
        if force:
            super().delete(force_parts=True)
        else:
            raise DataJointError(
                "Cannot delete from a Part directly. Delete from master instead"
            )

    def drop(self, force=False):
        """
        unless force is True, prohibits direct deletes from parts.
        """
        if force:
            super().drop()
        else:
            raise DataJointError(
                "Cannot drop a Part directly.  Delete from master instead"
            )

1	"""
2	Hosts the table tiers, user tables should be derived from.
3	"""
4	import collections	1✔
5	import pandas as pd	1✔
6	import numpy as np	1✔
7	from .table import Table	1✔
8	from .autopopulate import AutoPopulate	1✔
9	from .logging import logger	1✔
10	from .utils import from_camel_case, ClassProperty, dict_to_uuid	1✔
11	from .errors import DataJointError, DuplicateError	1✔
12	from .expression import QueryExpression	1✔
13
14	_base_regexp = r"[a-z][a-z0-9](_[a-z][a-z0-9])*"	1✔
15
16	# attributes that trigger instantiation of user classes
17
18
19	supported_class_attrs = {	1✔
20	"key_source",
21	"describe",
22	"alter",
23	"heading",
24	"populate",
25	"progress",
26	"primary_key",
27	"proj",
28	"aggr",
29	"join",
30	"fetch",
31	"fetch1",
32	"head",
33	"tail",
34	"descendants",
35	"ancestors",
36	"parts",
37	"parents",
38	"children",
39	"insert",
40	"insert1",
41	"update1",
42	"drop",
43	"drop_quick",
44	"delete",
45	"delete_quick",
46	}
47
48
49	class TableMeta(type):	1✔
50	"""
51	TableMeta subclasses allow applying some instance methods and properties directly
52	at class level. For example, this allows Table.fetch() instead of Table().fetch().
53	"""
54
55	def __getattribute__(cls, name):	1✔
56	# trigger instantiation for supported class attrs
57	return (	1✔
58	cls().__getattribute__(name)
59	if name in supported_class_attrs
60	else super().__getattribute__(name)
61	)
62
63	def __and__(cls, arg):	1✔
64	return cls() & arg	1✔
65
66	def __xor__(cls, arg):	1✔
67	return cls() ^ arg	1✔
68
69	def __sub__(cls, arg):	1✔
70	return cls() - arg	1✔
71
72	def __neg__(cls):	1✔
73	return -cls()	×
74
75	def __mul__(cls, arg):	1✔
76	return cls() * arg	1✔
77
78	def __matmul__(cls, arg):	1✔
79	return cls() @ arg	1✔
80
81	def __add__(cls, arg):	1✔
82	return cls() + arg	1✔
83
84	def __iter__(cls):	1✔
85	return iter(cls())	×
86
87
88	class UserTable(Table, metaclass=TableMeta):	1✔
89	"""
90	A subclass of UserTable is a dedicated class interfacing a base table.
91	UserTable is initialized by the decorator generated by schema().
92	"""
93
94	# set by @schema
95	_connection = None	1✔
96	_heading = None	1✔
97	_support = None	1✔
98
99	# set by subclass
100	tier_regexp = None	1✔
101	_prefix = None	1✔
102
103	@property	1✔
104	def definition(self):	1✔
105	"""
106	:return: a string containing the table definition using the DataJoint DDL.
107	"""
108	raise NotImplementedError(	1✔
109	'Subclasses of Table must implement the property "definition"'
110	)
111
112	@ClassProperty	1✔
113	def connection(cls):	1✔
114	return cls._connection	1✔
115
116	@ClassProperty	1✔
117	def table_name(cls):	1✔
118	"""
119	:return: the table name of the table formatted for mysql.
120	"""
121	if cls._prefix is None:	1✔
122	raise AttributeError("Class prefix is not defined!")	1✔
123	return cls._prefix + from_camel_case(cls.__name__)	1✔
124
125	@ClassProperty	1✔
126	def full_table_name(cls):	1✔
127	if cls not in {Manual, Imported, Lookup, Computed, Part, Params, UserTable}:	1✔
128	# for derived classes only
129	if cls.database is None:	1✔
130	raise DataJointError(	1✔
131	"Class %s is not properly declared (schema decorator not applied?)"
132	% cls.__name__
133	)
134	return r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name)	1✔
135
136
137	class Manual(UserTable):	1✔
138	"""
139	Inherit from this class if the table's values are entered manually.
140	"""
141
142	_prefix = r""	1✔
143	tier_regexp = r"(?P<manual>" + _prefix + _base_regexp + ")"	1✔
144
145
146	class Lookup(UserTable):	1✔
147	"""
148	Inherit from this class if the table's values are for lookup. This is
149	currently equivalent to defining the table as Manual and serves semantic
150	purposes only.
151	"""
152
153	_prefix = "#"	1✔
154	tier_regexp = (	1✔
155	r"(?P<lookup>" + _prefix + _base_regexp.replace("TIER", "lookup") + ")"
156	)
157
158
159	class Params(UserTable):	1✔
160	"""
161	Inherit from this class if the table's values are paramsets. Adds hashing of fields
162	that include 'param' and stores as additional unique index 'paramset hash'
163	"""
164
165	_prefix = "##"	1✔
166	tier_regexp = (	1✔
167	r"(?P<params>" + _prefix + _base_regexp.replace("TIER", "params") + ")"
168	)
169
170	def __init__(self):	1✔
171	super().__init__()	×
172	if "params" not in self.definition:	×
173	raise DataJointError("Params tables must have at least one 'params' field")	×
174
175	# NOTE: Ideally, this would be a UUID, not char(36), but I was having trouble
176	# fetching by UUID - what's the syntax? (self * f"hash='{UUID}'").fetch()
177	self.definition += (	×
178	"hide__paramset_hash: char(36)\nunique index (hide__paramset_hash)\n"
179	)
180
181	def insert(self, rows, **kwargs):	1✔
182	if isinstance(rows, QueryExpression):	×
183	raise NotImplementedError(	×
184	"Params tables do not yet support insertion via QueryExpression"
185	)
186
187	rows = self._normalize_insert_data(rows)	×
188
189	new_rows = []	×
190	for row in rows:	×
191	if isinstance(row, np.record):	×
192	row_values = tuple(row)	×
193	elif isinstance(row, collections.abc.Mapping):	×
194	row_values = row.values()	×
195	else:
196	row_values = row	×
197
198	paramset_hash = str(	×
199	dict_to_uuid(
200	{
201	field: value
202	for field, value in zip(self.heading, row_values)
203	if "param" in field and "hash" not in field
204	}
205	)
206	)
207
208	existing_key = (self & f'hide__paramset_hash="{paramset_hash}"').fetch(	×
209	"KEY"
210	)
211	if existing_key:	×
212	logger.warning(	×
213	"Paramset already included with the following primary "
214	+ f"key: {existing_key[0]}"
215	)
216	continue	×
217
218	if isinstance(row, collections.abc.Mapping):	×
219	new_rows.append(dict(**row, hide__paramset_hash=paramset_hash))	×
220	else:
221	new_rows.append([*row, paramset_hash])	×
222
223	super().insert(new_rows, **kwargs)	×
224
225
226	class Imported(UserTable, AutoPopulate):	1✔
227	"""
228	Inherit from this class if the table's values are imported from external data sources.
229	The inherited class must at least provide the function `_make_tuples`.
230	"""
231
232	_prefix = "_"	1✔
233	tier_regexp = r"(?P<imported>" + _prefix + _base_regexp + ")"	1✔
234
235
236	class Computed(UserTable, AutoPopulate):	1✔
237	"""
238	Inherit from this class if the table's values are computed from other tables in the schema.
239	The inherited class must at least provide the function `_make_tuples`.
240	"""
241
242	_prefix = "__"	1✔
243	tier_regexp = r"(?P<computed>" + _prefix + _base_regexp + ")"	1✔
244
245
246	class Part(UserTable):	1✔
247	"""
248	Inherit from this class if the table's values are details of an entry in another table
249	and if this table is populated by the other table. For example, the entries inheriting from
250	dj.Part could be single entries of a matrix, while the parent table refers to the entire matrix.
251	Part tables are implemented as classes inside classes.
252	"""
253
254	_connection = None	1✔
255	_master = None	1✔
256
257	tier_regexp = (	1✔
258	r"(?P<master>"
259	+ "\|".join(
260	[c.tier_regexp for c in (Manual, Lookup, Imported, Computed, Params)]
261	)
262	+ r"){1,1}"
263	+ "__"
264	+ r"(?P<part>"
265	+ _base_regexp
266	+ ")"
267	)
268
269	@ClassProperty	1✔
270	def connection(cls):	1✔
271	return cls._connection	1✔
272
273	@ClassProperty	1✔
274	def full_table_name(cls):	1✔
275	return (	1✔
276	None
277	if cls.database is None or cls.table_name is None
278	else r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name)
279	)
280
281	@ClassProperty	1✔
282	def master(cls):	1✔
283	return cls._master	1✔
284
285	@ClassProperty	1✔
286	def table_name(cls):	1✔
287	return (	1✔
288	None
289	if cls.master is None
290	else cls.master.table_name + "__" + from_camel_case(cls.__name__)
291	)
292
293	def delete(self, force=False):	1✔
294	"""
295	unless force is True, prohibits direct deletes from parts.
296	"""
297	if force:	1✔
298	super().delete(force_parts=True)	1✔
299	else:
300	raise DataJointError(	×
301	"Cannot delete from a Part directly. Delete from master instead"
302	)
303
304	def drop(self, force=False):	1✔
305	"""
306	unless force is True, prohibits direct deletes from parts.
307	"""
308	if force:	×
309	super().drop()	×
310	else:
311	raise DataJointError(	×
312	"Cannot drop a Part directly. Delete from master instead"
313	)

datajoint / datajoint-python / #12901

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous