#12751

pending completion

Build # #12751

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/c19908ca737275b9f86001e13c2fe0b7517677b8">c19908ca7<a href="https://github.com/datajoint/datajoint-python/commit/c19908ca737275b9f86001e13c2fe0b7517677b8">">Merge </a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/79237c24cb68f456e78b847d0705fe8971c3a9be">79237c24c</a>">79237c24c</a><a href="https://github.com/datajoint/datajoint-python/commit/c19908ca737275b9f86001e13c2fe0b7517677b8"> into 03dfbaf8a">03dfbaf8a</a>

Run Details

2 of 2 new or added lines in 2 files covered. (100.0%)

3020 of 3343 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.94

/datajoint/schemas.py

import warnings
import logging
import inspect
import re
import itertools
import collections
from .connection import conn
from .diagram import Diagram, _get_tier
from .settings import config
from .errors import DataJointError, AccessError
from .jobs import JobTable
from .external import ExternalMapping
from .heading import Heading
from .utils import user_choice, to_camel_case
from .user_tables import Part, Computed, Imported, Manual, Lookup
from .table import lookup_class_name, Log, FreeTable
import types

logger = logging.getLogger(__name__.split(".")[0])


def ordered_dir(class_):
    """
    List (most) attributes of the class including inherited ones, similar to `dir` build-in function,
    but respects order of attribute declaration as much as possible.

    :param class_: class to list members for
    :return: a list of attributes declared in class_ and its superclasses
    """
    attr_list = list()
    for c in reversed(class_.mro()):
        attr_list.extend(e for e in c.__dict__ if e not in attr_list)
    return attr_list


class Schema:
    """
    A schema object is a decorator for UserTable classes that binds them to their database.
    It also specifies the namespace `context` in which other UserTable classes are defined.
    """

    def __init__(
        self,
        schema_name=None,
        context=None,
        *,
        connection=None,
        create_schema=True,
        create_tables=True,
        add_objects=None,
    ):
        """
        Associate database schema `schema_name`. If the schema does not exist, attempt to
        create it on the server.

        If the schema_name is omitted, then schema.activate(..) must be called later
        to associate with the database.

        :param schema_name: the database schema to associate.
        :param context: dictionary for looking up foreign key references, leave None to use local context.
        :param connection: Connection object. Defaults to datajoint.conn().
        :param create_schema: When False, do not create the schema and raise an error if missing.
        :param create_tables: When False, do not create tables and raise errors when accessing missing tables.
        :param add_objects: a mapping with additional objects to make available to the context in which table classes
        are declared.
        """
        self._log = None
        self.connection = connection
        self.database = None
        self.context = context
        self.create_schema = create_schema
        self.create_tables = create_tables
        self._jobs = None
        self.external = ExternalMapping(self)
        self.add_objects = add_objects
        self.declare_list = []
        if schema_name:
            self.activate(schema_name)

    def is_activated(self):
        return self.database is not None

    def activate(
        self,
        schema_name=None,
        *,
        connection=None,
        create_schema=None,
        create_tables=None,
        add_objects=None,
    ):
        """
        Associate database schema `schema_name`. If the schema does not exist, attempt to
        create it on the server.

        :param schema_name: the database schema to associate.
            schema_name=None is used to assert that the schema has already been activated.
        :param connection: Connection object. Defaults to datajoint.conn().
        :param create_schema: If False, do not create the schema and raise an error if missing.
        :param create_tables: If False, do not create tables and raise errors when attempting
            to access missing tables.
        :param add_objects: a mapping with additional objects to make available to the context
            in which table classes are declared.
        """
        if schema_name is None:
            if self.exists:
                return
            raise DataJointError("Please provide a schema_name to activate the schema.")
        if self.database is not None and self.exists:
            if self.database == schema_name:  # already activated
                return
            raise DataJointError(
                "The schema is already activated for schema {db}.".format(
                    db=self.database
                )
            )
        if connection is not None:
            self.connection = connection
        if self.connection is None:
            self.connection = conn()
        self.database = schema_name
        if create_schema is not None:
            self.create_schema = create_schema
        if create_tables is not None:
            self.create_tables = create_tables
        if add_objects:
            self.add_objects = add_objects
        if not self.exists:
            if not self.create_schema or not self.database:
                raise DataJointError(
                    "Database `{name}` has not yet been declared. "
                    "Set argument create_schema=True to create it.".format(
                        name=schema_name
                    )
                )
            # create database
            logger.debug("Creating schema `{name}`.".format(name=schema_name))
            try:
                self.connection.query(
                    "CREATE DATABASE `{name}`".format(name=schema_name)
                )
            except AccessError:
                raise DataJointError(
                    "Schema `{name}` does not exist and could not be created. "
                    "Check permissions.".format(name=schema_name)
                )
            else:
                self.log("created")
        self.connection.register(self)

        # decorate all tables already decorated
        for cls, context in self.declare_list:
            if self.add_objects:
                context = dict(context, **self.add_objects)
            self._decorate_master(cls, context)

    def _assert_exists(self, message=None):
        if not self.exists:
            raise DataJointError(
                message
                or "Schema `{db}` has not been created.".format(db=self.database)
            )

    def __call__(self, cls, *, context=None):
        """
        Binds the supplied class to a schema. This is intended to be used as a decorator.

        :param cls: class to decorate.
        :param context: supplied when called from spawn_missing_classes
        """
        context = context or self.context or inspect.currentframe().f_back.f_locals
        if issubclass(cls, Part):
            raise DataJointError(
                "The schema decorator should not be applied to Part relations"
            )
        if self.is_activated():
            self._decorate_master(cls, context)
        else:
            self.declare_list.append((cls, context))
        return cls

    def _decorate_master(self, cls, context):
        """

        :param cls: the master class to process
        :param context: the class' declaration context
        """
        self._decorate_table(
            cls, context=dict(context, self=cls, **{cls.__name__: cls})
        )
        # Process part tables
        for part in ordered_dir(cls):
            if part[0].isupper():
                part = getattr(cls, part)
                if inspect.isclass(part) and issubclass(part, Part):
                    part._master = cls
                    # allow addressing master by name or keyword 'master'
                    self._decorate_table(
                        part,
                        context=dict(
                            context, master=cls, self=part, **{cls.__name__: cls}
                        ),
                    )

    def _decorate_table(self, table_class, context, assert_declared=False):
        """
        assign schema properties to the table class and declare the table
        """
        table_class.database = self.database
        table_class._connection = self.connection
        table_class._heading = Heading(
            table_info=dict(
                conn=self.connection,
                database=self.database,
                table_name=table_class.table_name,
                context=context,
            )
        )
        table_class._support = [table_class.full_table_name]
        table_class.declaration_context = context

        # instantiate the class, declare the table if not already
        instance = table_class()
        is_declared = instance.is_declared
        if not is_declared and not assert_declared and self.create_tables:
            instance.declare(context)
            self.connection.dependencies.clear()
        is_declared = is_declared or instance.is_declared

        # add table definition to the doc string
        if isinstance(table_class.definition, str):
            table_class.__doc__ = (
                (table_class.__doc__ or "")
                + "\nTable definition:\n\n"
                + table_class.definition
            )

        # fill values in Lookup tables from their contents property
        if (
            isinstance(instance, Lookup)
            and hasattr(instance, "contents")
            and is_declared
        ):
            contents = list(instance.contents)
            if len(contents) > len(instance):
                if instance.heading.has_autoincrement:
                    warnings.warn(
                        (
                            "Contents has changed but cannot be inserted because "
                            "{table} has autoincrement."
                        ).format(table=instance.__class__.__name__)
                    )
                else:
                    instance.insert(contents, skip_duplicates=True)

    @property
    def log(self):
        self._assert_exists()
        if self._log is None:
            self._log = Log(self.connection, self.database)
        return self._log

    def __repr__(self):
        return "Schema `{name}`\n".format(name=self.database)

    @property
    def size_on_disk(self):
        """
        :return: size of the entire schema in bytes
        """
        self._assert_exists()
        return int(
            self.connection.query(
                """
            SELECT SUM(data_length + index_length)
            FROM information_schema.tables WHERE table_schema='{db}'
            """.format(
                    db=self.database
                )
            ).fetchone()[0]
        )

    def spawn_missing_classes(self, context=None):
        """
        Creates the appropriate python user relation classes from tables in the schema and places them
        in the context.

        :param context: alternative context to place the missing classes into, e.g. locals()
        """
        self._assert_exists()
        if context is None:
            if self.context is not None:
                context = self.context
            else:
                # if context is missing, use the calling namespace
                frame = inspect.currentframe().f_back
                context = frame.f_locals
                del frame
        tables = [
            row[0]
            for row in self.connection.query("SHOW TABLES in `%s`" % self.database)
            if lookup_class_name(
                "`{db}`.`{tab}`".format(db=self.database, tab=row[0]), context, 0
            )
            is None
        ]
        master_classes = (Lookup, Manual, Imported, Computed)
        part_tables = []
        for table_name in tables:
            class_name = to_camel_case(table_name)
            if class_name not in context:
                try:
                    cls = next(
                        cls
                        for cls in master_classes
                        if re.fullmatch(cls.tier_regexp, table_name)
                    )
                except StopIteration:
                    if re.fullmatch(Part.tier_regexp, table_name):
                        part_tables.append(table_name)
                else:
                    # declare and decorate master relation classes
                    context[class_name] = self(
                        type(class_name, (cls,), dict()), context=context
                    )

        # attach parts to masters
        for table_name in part_tables:
            groups = re.fullmatch(Part.tier_regexp, table_name).groupdict()
            class_name = to_camel_case(groups["part"])
            try:
                master_class = context[to_camel_case(groups["master"])]
            except KeyError:
                raise DataJointError(
                    "The table %s does not follow DataJoint naming conventions"
                    % table_name
                )
            part_class = type(class_name, (Part,), dict(definition=...))
            part_class._master = master_class
            self._decorate_table(part_class, context=context, assert_declared=True)
            setattr(master_class, class_name, part_class)

    def drop(self, force=False):
        """
        Drop the associated schema if it exists
        """
        if not self.exists:
            logger.info(
                "Schema named `{database}` does not exist. Doing nothing.".format(
                    database=self.database
                )
            )
        elif (
            not config["safemode"]
            or force
            or user_choice(
                "Proceed to delete entire schema `%s`?" % self.database, default="no"
            )
            == "yes"
        ):
            logger.debug("Dropping `{database}`.".format(database=self.database))
            try:
                self.connection.query(
                    "DROP DATABASE `{database}`".format(database=self.database)
                )
                logger.debug(
                    "Schema `{database}` was dropped successfully.".format(
                        database=self.database
                    )
                )
            except AccessError:
                raise AccessError(
                    "An attempt to drop schema `{database}` "
                    "has failed. Check permissions.".format(database=self.database)
                )

    @property
    def exists(self):
        """
        :return: true if the associated schema exists on the server
        """
        if self.database is None:
            raise DataJointError("Schema must be activated first.")
        return bool(
            self.connection.query(
                "SELECT schema_name "
                "FROM information_schema.schemata "
                "WHERE schema_name = '{database}'".format(database=self.database)
            ).rowcount
        )

    @property
    def jobs(self):
        """
        schema.jobs provides a view of the job reservation table for the schema

        :return: jobs table
        """
        self._assert_exists()
        if self._jobs is None:
            self._jobs = JobTable(self.connection, self.database)
        return self._jobs

    @property
    def code(self):
        self._assert_exists()
        return self.save()

    def save(self, python_filename=None):
        """
        Generate the code for a module that recreates the schema.
        This method is in preparation for a future release and is not officially supported.

        :return: a string containing the body of a complete Python module defining this schema.
        """
        self._assert_exists()
        module_count = itertools.count()
        # add virtual modules for referenced modules with names vmod0, vmod1, ...
        module_lookup = collections.defaultdict(
            lambda: "vmod" + str(next(module_count))
        )
        db = self.database

        def make_class_definition(table):
            tier = _get_tier(table).__name__
            class_name = table.split(".")[1].strip("`")
            indent = ""
            if tier == "Part":
                class_name = class_name.split("__")[-1]
                indent += "    "
            class_name = to_camel_case(class_name)

            def replace(s):
                d, tabs = s.group(1), s.group(2)
                return ("" if d == db else (module_lookup[d] + ".")) + ".".join(
                    to_camel_case(tab) for tab in tabs.lstrip("__").split("__")
                )

            return ("" if tier == "Part" else "\n@schema\n") + (
                "{indent}class {class_name}(dj.{tier}):\n"
                '{indent}    definition = """\n'
                '{indent}    {defi}"""'
            ).format(
                class_name=class_name,
                indent=indent,
                tier=tier,
                defi=re.sub(
                    r"`([^`]+)`.`([^`]+)`",
                    replace,
                    FreeTable(self.connection, table).describe(printout=False),
                ).replace("\n", "\n    " + indent),
            )

        diagram = Diagram(self)
        body = "\n\n".join(
            make_class_definition(table) for table in diagram.topological_sort()
        )
        python_code = "\n\n".join(
            (
                '"""This module was auto-generated by datajoint from an existing schema"""',
                "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
                "\n".join(
                    "{module} = dj.VirtualModule('{module}', '{schema_name}')".format(
                        module=v, schema_name=k
                    )
                    for k, v in module_lookup.items()
                ),
                body,
            )
        )
        if python_filename is None:
            return python_code
        with open(python_filename, "wt") as f:
            f.write(python_code)

    def list_tables(self):
        """
        Return a list of all tables in the schema except tables with ~ in first character such
        as ~logs and ~job

        :return: A list of table names from the database schema.
        """
        return [
            t
            for d, t in (
                full_t.replace("`", "").split(".")
                for full_t in Diagram(self).topological_sort()
            )
            if d == self.database
        ]


class VirtualModule(types.ModuleType):
    """
    A virtual module imitates a Python module representing a DataJoint schema from table definitions in the database.
    It declares the schema objects and a class for each table.
    """

    def __init__(
        self,
        module_name,
        schema_name,
        *,
        create_schema=False,
        create_tables=False,
        connection=None,
        add_objects=None,
    ):
        """
        Creates a python module with the given name from the name of a schema on the server and
        automatically adds classes to it corresponding to the tables in the schema.

        :param module_name: displayed module name
        :param schema_name: name of the database in mysql
        :param create_schema: if True, create the schema on the database server
        :param create_tables: if True, module.schema can be used as the decorator for declaring new
        :param connection: a dj.Connection object to pass into the schema
        :param add_objects: additional objects to add to the module
        :return: the python module containing classes from the schema object and the table classes
        """
        super(VirtualModule, self).__init__(name=module_name)
        _schema = Schema(
            schema_name,
            create_schema=create_schema,
            create_tables=create_tables,
            connection=connection,
        )
        if add_objects:
            self.__dict__.update(add_objects)
        self.__dict__["schema"] = _schema
        _schema.spawn_missing_classes(context=self.__dict__)


def list_schemas(connection=None):
    """

    :param connection: a dj.Connection object
    :return: list of all accessible schemas on the server
    """
    return [
        r[0]
        for r in (connection or conn()).query(
            "SELECT schema_name "
            "FROM information_schema.schemata "
            'WHERE schema_name <> "information_schema"'
        )
    ]

1	import warnings	1✔
2	import logging	1✔
3	import inspect	1✔
4	import re	1✔
5	import itertools	1✔
6	import collections	1✔
7	from .connection import conn	1✔
8	from .diagram import Diagram, _get_tier	1✔
9	from .settings import config	1✔
10	from .errors import DataJointError, AccessError	1✔
11	from .jobs import JobTable	1✔
12	from .external import ExternalMapping	1✔
13	from .heading import Heading	1✔
14	from .utils import user_choice, to_camel_case	1✔
15	from .user_tables import Part, Computed, Imported, Manual, Lookup	1✔
16	from .table import lookup_class_name, Log, FreeTable	1✔
17	import types	1✔
18
19	logger = logging.getLogger(__name__.split(".")[0])	1✔
20
21
22	def ordered_dir(class_):	1✔
23	"""
24	List (most) attributes of the class including inherited ones, similar to `dir` build-in function,
25	but respects order of attribute declaration as much as possible.
26
27	:param class_: class to list members for
28	:return: a list of attributes declared in class_ and its superclasses
29	"""
30	attr_list = list()	1✔
31	for c in reversed(class_.mro()):	1✔
32	attr_list.extend(e for e in c.__dict__ if e not in attr_list)	1✔
33	return attr_list	1✔
34
35
36	class Schema:	1✔
37	"""
38	A schema object is a decorator for UserTable classes that binds them to their database.
39	It also specifies the namespace `context` in which other UserTable classes are defined.
40	"""
41
42	def __init__(	1✔
43	self,
44	schema_name=None,
45	context=None,
46	*,
47	connection=None,
48	create_schema=True,
49	create_tables=True,
50	add_objects=None,
51	):
52	"""
53	Associate database schema `schema_name`. If the schema does not exist, attempt to
54	create it on the server.
55
56	If the schema_name is omitted, then schema.activate(..) must be called later
57	to associate with the database.
58
59	:param schema_name: the database schema to associate.
60	:param context: dictionary for looking up foreign key references, leave None to use local context.
61	:param connection: Connection object. Defaults to datajoint.conn().
62	:param create_schema: When False, do not create the schema and raise an error if missing.
63	:param create_tables: When False, do not create tables and raise errors when accessing missing tables.
64	:param add_objects: a mapping with additional objects to make available to the context in which table classes
65	are declared.
66	"""
67	self._log = None	1✔
68	self.connection = connection	1✔
69	self.database = None	1✔
70	self.context = context	1✔
71	self.create_schema = create_schema	1✔
72	self.create_tables = create_tables	1✔
73	self._jobs = None	1✔
74	self.external = ExternalMapping(self)	1✔
75	self.add_objects = add_objects	1✔
76	self.declare_list = []	1✔
77	if schema_name:	1✔
78	self.activate(schema_name)	1✔
79
80	def is_activated(self):	1✔
81	return self.database is not None	1✔
82
83	def activate(	1✔
84	self,
85	schema_name=None,
86	*,
87	connection=None,
88	create_schema=None,
89	create_tables=None,
90	add_objects=None,
91	):
92	"""
93	Associate database schema `schema_name`. If the schema does not exist, attempt to
94	create it on the server.
95
96	:param schema_name: the database schema to associate.
97	schema_name=None is used to assert that the schema has already been activated.
98	:param connection: Connection object. Defaults to datajoint.conn().
99	:param create_schema: If False, do not create the schema and raise an error if missing.
100	:param create_tables: If False, do not create tables and raise errors when attempting
101	to access missing tables.
102	:param add_objects: a mapping with additional objects to make available to the context
103	in which table classes are declared.
104	"""
105	if schema_name is None:	1✔
106	if self.exists:	×
107	return	×
108	raise DataJointError("Please provide a schema_name to activate the schema.")	×
109	if self.database is not None and self.exists:	1✔
110	if self.database == schema_name: # already activated	×
111	return	×
112	raise DataJointError(	×
113	"The schema is already activated for schema {db}.".format(
114	db=self.database
115	)
116	)
117	if connection is not None:	1✔
118	self.connection = connection	1✔
119	if self.connection is None:	1✔
120	self.connection = conn()	1✔
121	self.database = schema_name	1✔
122	if create_schema is not None:	1✔
123	self.create_schema = create_schema	1✔
124	if create_tables is not None:	1✔
125	self.create_tables = create_tables	1✔
126	if add_objects:	1✔
127	self.add_objects = add_objects	×
128	if not self.exists:	1✔
129	if not self.create_schema or not self.database:	1✔
130	raise DataJointError(	×
131	"Database `{name}` has not yet been declared. "
132	"Set argument create_schema=True to create it.".format(
133	name=schema_name
134	)
135	)
136	# create database
137	logger.debug("Creating schema `{name}`.".format(name=schema_name))	1✔
138	try:	1✔
139	self.connection.query(	1✔
140	"CREATE DATABASE `{name}`".format(name=schema_name)
141	)
142	except AccessError:	1✔
143	raise DataJointError(	1✔
144	"Schema `{name}` does not exist and could not be created. "
145	"Check permissions.".format(name=schema_name)
146	)
147	else:
148	self.log("created")	1✔
149	self.connection.register(self)	1✔
150
151	# decorate all tables already decorated
152	for cls, context in self.declare_list:	1✔
153	if self.add_objects:	1✔
154	context = dict(context, **self.add_objects)	×
155	self._decorate_master(cls, context)	1✔
156
157	def _assert_exists(self, message=None):	1✔
158	if not self.exists:	1✔
159	raise DataJointError(	×
160	message
161	or "Schema `{db}` has not been created.".format(db=self.database)
162	)
163
164	def __call__(self, cls, *, context=None):	1✔
165	"""
166	Binds the supplied class to a schema. This is intended to be used as a decorator.
167
168	:param cls: class to decorate.
169	:param context: supplied when called from spawn_missing_classes
170	"""
171	context = context or self.context or inspect.currentframe().f_back.f_locals	1✔
172	if issubclass(cls, Part):	1✔
173	raise DataJointError(	1✔
174	"The schema decorator should not be applied to Part relations"
175	)
176	if self.is_activated():	1✔
177	self._decorate_master(cls, context)	1✔
178	else:
179	self.declare_list.append((cls, context))	1✔
180	return cls	1✔
181
182	def _decorate_master(self, cls, context):	1✔
183	"""
184
185	:param cls: the master class to process
186	:param context: the class' declaration context
187	"""
188	self._decorate_table(	1✔
189	cls, context=dict(context, self=cls, **{cls.__name__: cls})
190	)
191	# Process part tables
192	for part in ordered_dir(cls):	1✔
193	if part[0].isupper():	1✔
194	part = getattr(cls, part)	1✔
195	if inspect.isclass(part) and issubclass(part, Part):	1✔
196	part._master = cls	1✔
197	# allow addressing master by name or keyword 'master'
198	self._decorate_table(	1✔
199	part,
200	context=dict(
201	context, master=cls, self=part, **{cls.__name__: cls}
202	),
203	)
204
205	def _decorate_table(self, table_class, context, assert_declared=False):	1✔
206	"""
207	assign schema properties to the table class and declare the table
208	"""
209	table_class.database = self.database	1✔
210	table_class._connection = self.connection	1✔
211	table_class._heading = Heading(	1✔
212	table_info=dict(
213	conn=self.connection,
214	database=self.database,
215	table_name=table_class.table_name,
216	context=context,
217	)
218	)
219	table_class._support = [table_class.full_table_name]	1✔
220	table_class.declaration_context = context	1✔
221
222	# instantiate the class, declare the table if not already
223	instance = table_class()	1✔
224	is_declared = instance.is_declared	1✔
225	if not is_declared and not assert_declared and self.create_tables:	1✔
226	instance.declare(context)	1✔
227	self.connection.dependencies.clear()	1✔
228	is_declared = is_declared or instance.is_declared	1✔
229
230	# add table definition to the doc string
231	if isinstance(table_class.definition, str):	1✔
232	table_class.__doc__ = (	1✔
233	(table_class.__doc__ or "")
234	+ "\nTable definition:\n\n"
235	+ table_class.definition
236	)
237
238	# fill values in Lookup tables from their contents property
239	if (	1✔
240	isinstance(instance, Lookup)
241	and hasattr(instance, "contents")
242	and is_declared
243	):
244	contents = list(instance.contents)	1✔
245	if len(contents) > len(instance):	1✔
246	if instance.heading.has_autoincrement:	1✔
247	warnings.warn(	×
248	(
249	"Contents has changed but cannot be inserted because "
250	"{table} has autoincrement."
251	).format(table=instance.__class__.__name__)
252	)
253	else:
254	instance.insert(contents, skip_duplicates=True)	1✔
255
256	@property	1✔
257	def log(self):
258	self._assert_exists()	1✔
259	if self._log is None:	1✔
260	self._log = Log(self.connection, self.database)	1✔
261	return self._log	1✔
262
263	def __repr__(self):	1✔
264	return "Schema `{name}`\n".format(name=self.database)	×
265
266	@property	1✔
267	def size_on_disk(self):
268	"""
269	:return: size of the entire schema in bytes
270	"""
271	self._assert_exists()	1✔
272	return int(	1✔
273	self.connection.query(
274	"""
275	SELECT SUM(data_length + index_length)
276	FROM information_schema.tables WHERE table_schema='{db}'
277	""".format(
278	db=self.database
279	)
280	).fetchone()[0]
281	)
282
283	def spawn_missing_classes(self, context=None):	1✔
284	"""
285	Creates the appropriate python user relation classes from tables in the schema and places them
286	in the context.
287
288	:param context: alternative context to place the missing classes into, e.g. locals()
289	"""
290	self._assert_exists()	1✔
291	if context is None:	1✔
292	if self.context is not None:	1✔
293	context = self.context	1✔
294	else:
295	# if context is missing, use the calling namespace
296	frame = inspect.currentframe().f_back	1✔
297	context = frame.f_locals	1✔
298	del frame	1✔
299	tables = [	1✔
300	row[0]
301	for row in self.connection.query("SHOW TABLES in `%s`" % self.database)
302	if lookup_class_name(
303	"`{db}`.`{tab}`".format(db=self.database, tab=row[0]), context, 0
304	)
305	is None
306	]
307	master_classes = (Lookup, Manual, Imported, Computed)	1✔
308	part_tables = []	1✔
309	for table_name in tables:	1✔
310	class_name = to_camel_case(table_name)	1✔
311	if class_name not in context:	1✔
312	try:	1✔
313	cls = next(	1✔
314	cls
315	for cls in master_classes
316	if re.fullmatch(cls.tier_regexp, table_name)
317	)
318	except StopIteration:	1✔
319	if re.fullmatch(Part.tier_regexp, table_name):	1✔
320	part_tables.append(table_name)	1✔
321	else:
322	# declare and decorate master relation classes
323	context[class_name] = self(	1✔
324	type(class_name, (cls,), dict()), context=context
325	)
326
327	# attach parts to masters
328	for table_name in part_tables:	1✔
329	groups = re.fullmatch(Part.tier_regexp, table_name).groupdict()	1✔
330	class_name = to_camel_case(groups["part"])	1✔
331	try:	1✔
332	master_class = context[to_camel_case(groups["master"])]	1✔
333	except KeyError:	×
334	raise DataJointError(	×
335	"The table %s does not follow DataJoint naming conventions"
336	% table_name
337	)
338	part_class = type(class_name, (Part,), dict(definition=...))	1✔
339	part_class._master = master_class	1✔
340	self._decorate_table(part_class, context=context, assert_declared=True)	1✔
341	setattr(master_class, class_name, part_class)	1✔
342
343	def drop(self, force=False):	1✔
344	"""
345	Drop the associated schema if it exists
346	"""
347	if not self.exists:	1✔
348	logger.info(	1✔
349	"Schema named `{database}` does not exist. Doing nothing.".format(
350	database=self.database
351	)
352	)
353	elif (	1✔
354	not config["safemode"]
355	or force
356	or user_choice(
357	"Proceed to delete entire schema `%s`?" % self.database, default="no"
358	)
359	== "yes"
360	):
361	logger.debug("Dropping `{database}`.".format(database=self.database))	1✔
362	try:	1✔
363	self.connection.query(	1✔
364	"DROP DATABASE `{database}`".format(database=self.database)
365	)
366	logger.debug(	1✔
367	"Schema `{database}` was dropped successfully.".format(
368	database=self.database
369	)
370	)
371	except AccessError:	1✔
372	raise AccessError(	1✔
373	"An attempt to drop schema `{database}` "
374	"has failed. Check permissions.".format(database=self.database)
375	)
376
377	@property	1✔
378	def exists(self):
379	"""
380	:return: true if the associated schema exists on the server
381	"""
382	if self.database is None:	1✔
383	raise DataJointError("Schema must be activated first.")	×
384	return bool(	1✔
385	self.connection.query(
386	"SELECT schema_name "
387	"FROM information_schema.schemata "
388	"WHERE schema_name = '{database}'".format(database=self.database)
389	).rowcount
390	)
391
392	@property	1✔
393	def jobs(self):
394	"""
395	schema.jobs provides a view of the job reservation table for the schema
396
397	:return: jobs table
398	"""
399	self._assert_exists()	1✔
400	if self._jobs is None:	1✔
401	self._jobs = JobTable(self.connection, self.database)	1✔
402	return self._jobs	1✔
403
404	@property	1✔
405	def code(self):
406	self._assert_exists()	1✔
407	return self.save()	1✔
408
409	def save(self, python_filename=None):	1✔
410	"""
411	Generate the code for a module that recreates the schema.
412	This method is in preparation for a future release and is not officially supported.
413
414	:return: a string containing the body of a complete Python module defining this schema.
415	"""
416	self._assert_exists()	1✔
417	module_count = itertools.count()	1✔
418	# add virtual modules for referenced modules with names vmod0, vmod1, ...
419	module_lookup = collections.defaultdict(	1✔
420	lambda: "vmod" + str(next(module_count))
421	)
422	db = self.database	1✔
423
424	def make_class_definition(table):	1✔
425	tier = _get_tier(table).__name__	1✔
426	class_name = table.split(".")[1].strip("`")	1✔
427	indent = ""	1✔
428	if tier == "Part":	1✔
429	class_name = class_name.split("__")[-1]	1✔
430	indent += " "	1✔
431	class_name = to_camel_case(class_name)	1✔
432
433	def replace(s):	1✔
434	d, tabs = s.group(1), s.group(2)	1✔
435	return ("" if d == db else (module_lookup[d] + ".")) + ".".join(	1✔
436	to_camel_case(tab) for tab in tabs.lstrip("__").split("__")
437	)
438
439	return ("" if tier == "Part" else "\n@schema\n") + (	1✔
440	"{indent}class {class_name}(dj.{tier}):\n"
441	'{indent} definition = """\n'
442	'{indent} {defi}"""'
443	).format(
444	class_name=class_name,
445	indent=indent,
446	tier=tier,
447	defi=re.sub(
448	r"`([^`]+)`.`([^`]+)`",
449	replace,
450	FreeTable(self.connection, table).describe(printout=False),
451	).replace("\n", "\n " + indent),
452	)
453
454	diagram = Diagram(self)	1✔
455	body = "\n\n".join(	1✔
456	make_class_definition(table) for table in diagram.topological_sort()
457	)
458	python_code = "\n\n".join(	1✔
459	(
460	'"""This module was auto-generated by datajoint from an existing schema"""',
461	"import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
462	"\n".join(
463	"{module} = dj.VirtualModule('{module}', '{schema_name}')".format(
464	module=v, schema_name=k
465	)
466	for k, v in module_lookup.items()
467	),
468	body,
469	)
470	)
471	if python_filename is None:	1✔
472	return python_code	1✔
473	with open(python_filename, "wt") as f:	×
474	f.write(python_code)	×
475
476	def list_tables(self):	1✔
477	"""
478	Return a list of all tables in the schema except tables with ~ in first character such
479	as ~logs and ~job
480
481	:return: A list of table names from the database schema.
482	"""
483	return [	1✔
484	t
485	for d, t in (
486	full_t.replace("`", "").split(".")
487	for full_t in Diagram(self).topological_sort()
488	)
489	if d == self.database
490	]
491
492
493	class VirtualModule(types.ModuleType):	1✔
494	"""
495	A virtual module imitates a Python module representing a DataJoint schema from table definitions in the database.
496	It declares the schema objects and a class for each table.
497	"""
498
499	def __init__(	1✔
500	self,
501	module_name,
502	schema_name,
503	*,
504	create_schema=False,
505	create_tables=False,
506	connection=None,
507	add_objects=None,
508	):
509	"""
510	Creates a python module with the given name from the name of a schema on the server and
511	automatically adds classes to it corresponding to the tables in the schema.
512
513	:param module_name: displayed module name
514	:param schema_name: name of the database in mysql
515	:param create_schema: if True, create the schema on the database server
516	:param create_tables: if True, module.schema can be used as the decorator for declaring new
517	:param connection: a dj.Connection object to pass into the schema
518	:param add_objects: additional objects to add to the module
519	:return: the python module containing classes from the schema object and the table classes
520	"""
521	super(VirtualModule, self).__init__(name=module_name)	1✔
522	_schema = Schema(	1✔
523	schema_name,
524	create_schema=create_schema,
525	create_tables=create_tables,
526	connection=connection,
527	)
528	if add_objects:	1✔
529	self.__dict__.update(add_objects)	1✔
530	self.__dict__["schema"] = _schema	1✔
531	_schema.spawn_missing_classes(context=self.__dict__)	1✔
532
533
534	def list_schemas(connection=None):	1✔
535	"""
536
537	:param connection: a dj.Connection object
538	:return: list of all accessible schemas on the server
539	"""
540	return [	1✔
541	r[0]
542	for r in (connection or conn()).query(
543	"SELECT schema_name "
544	"FROM information_schema.schemata "
545	'WHERE schema_name <> "information_schema"'
546	)
547	]

datajoint / datajoint-python / #12751

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous