#12028

pending completion

Build # #12028

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/599f971831c8e0755c73fce673617d3a0b2b586f">599f97183<a href="https://github.com/datajoint/datajoint-python/commit/599f971831c8e0755c73fce673617d3a0b2b586f">">Merge </a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/17f5fe4db7e2994cd470282a7bc1bcb7505dbbdd">17f5fe4db</a>">17f5fe4db</a><a href="https://github.com/datajoint/datajoint-python/commit/599f971831c8e0755c73fce673617d3a0b2b586f"> into f5fc2f68d">f5fc2f68d</a>

Run Details

3061 of 3386 relevant lines covered (90.4%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.83

/datajoint/schemas.py

import warnings
import logging
import inspect
import re
import itertools
import collections
from .connection import conn
from .diagram import Diagram, _get_tier
from .settings import config
from .errors import DataJointError, AccessError
from .jobs import JobTable
from .external import ExternalMapping
from .heading import Heading
from .utils import user_choice, to_camel_case
from .user_tables import Part, Computed, Imported, Manual, Lookup
from .table import lookup_class_name, Log, FreeTable
import types

logger = logging.getLogger(__name__)


def ordered_dir(class_):
    """
    List (most) attributes of the class including inherited ones, similar to `dir` build-in function,
    but respects order of attribute declaration as much as possible.

    :param class_: class to list members for
    :return: a list of attributes declared in class_ and its superclasses
    """
    attr_list = list()
    for c in reversed(class_.mro()):
        attr_list.extend(e for e in c.__dict__ if e not in attr_list)
    return attr_list


class Schema:
    """
    A schema object is a decorator for UserTable classes that binds them to their database.
    It also specifies the namespace `context` in which other UserTable classes are defined.
    """

    def __init__(
        self,
        schema_name=None,
        context=None,
        *,
        connection=None,
        create_schema=True,
        create_tables=True,
        add_objects=None
    ):
        """
        Associate database schema `schema_name`. If the schema does not exist, attempt to
        create it on the server.

        If the schema_name is omitted, then schema.activate(..) must be called later
        to associate with the database.

        :param schema_name: the database schema to associate.
        :param context: dictionary for looking up foreign key references, leave None to use local context.
        :param connection: Connection object. Defaults to datajoint.conn().
        :param create_schema: When False, do not create the schema and raise an error if missing.
        :param create_tables: When False, do not create tables and raise errors when accessing missing tables.
        :param add_objects: a mapping with additional objects to make available to the context in which table classes
        are declared.
        """
        self._log = None
        self.connection = connection
        self.database = None
        self.context = context
        self.create_schema = create_schema
        self.create_tables = create_tables
        self._jobs = None
        self.external = ExternalMapping(self)
        self.add_objects = add_objects
        self.declare_list = []
        if schema_name:
            self.activate(schema_name)

    def is_activated(self):
        return self.database is not None

    def activate(
        self,
        schema_name=None,
        *,
        connection=None,
        create_schema=None,
        create_tables=None,
        add_objects=None
    ):
        """
        Associate database schema `schema_name`. If the schema does not exist, attempt to
        create it on the server.

        :param schema_name: the database schema to associate.
            schema_name=None is used to assert that the schema has already been activated.
        :param connection: Connection object. Defaults to datajoint.conn().
        :param create_schema: If False, do not create the schema and raise an error if missing.
        :param create_tables: If False, do not create tables and raise errors when attempting
            to access missing tables.
        :param add_objects: a mapping with additional objects to make available to the context
            in which table classes are declared.
        """
        if schema_name is None:
            if self.exists:
                return
            raise DataJointError("Please provide a schema_name to activate the schema.")
        if self.database is not None and self.exists:
            if self.database == schema_name:  # already activated
                return
            raise DataJointError(
                "The schema is already activated for schema {db}.".format(
                    db=self.database
                )
            )
        if connection is not None:
            self.connection = connection
        if self.connection is None:
            self.connection = conn()
        self.database = schema_name
        if create_schema is not None:
            self.create_schema = create_schema
        if create_tables is not None:
            self.create_tables = create_tables
        if add_objects:
            self.add_objects = add_objects
        if not self.exists:
            if not self.create_schema or not self.database:
                raise DataJointError(
                    "Database `{name}` has not yet been declared. "
                    "Set argument create_schema=True to create it.".format(
                        name=schema_name
                    )
                )
            # create database
            logger.info("Creating schema `{name}`.".format(name=schema_name))
            try:
                self.connection.query(
                    "CREATE DATABASE `{name}`".format(name=schema_name)
                )
            except AccessError:
                raise DataJointError(
                    "Schema `{name}` does not exist and could not be created. "
                    "Check permissions.".format(name=schema_name)
                )
            else:
                self.log("created")
        self.connection.register(self)

        # decorate all tables already decorated
        for cls, context in self.declare_list:
            if self.add_objects:
                context = dict(context, **self.add_objects)
            self._decorate_master(cls, context)

    def _assert_exists(self, message=None):
        if not self.exists:
            raise DataJointError(
                message
                or "Schema `{db}` has not been created.".format(db=self.database)
            )

    def __call__(self, cls, *, context=None):
        """
        Binds the supplied class to a schema. This is intended to be used as a decorator.

        :param cls: class to decorate.
        :param context: supplied when called from spawn_missing_classes
        """
        context = context or self.context or inspect.currentframe().f_back.f_locals
        if issubclass(cls, Part):
            raise DataJointError(
                "The schema decorator should not be applied to Part relations"
            )
        if self.is_activated():
            self._decorate_master(cls, context)
        else:
            self.declare_list.append((cls, context))
        return cls

    def _decorate_master(self, cls, context):
        """

        :param cls: the master class to process
        :param context: the class' declaration context
        """
        self._decorate_table(
            cls, context=dict(context, self=cls, **{cls.__name__: cls})
        )
        # Process part tables
        for part in ordered_dir(cls):
            if part[0].isupper():
                part = getattr(cls, part)
                if inspect.isclass(part) and issubclass(part, Part):
                    part._master = cls
                    # allow addressing master by name or keyword 'master'
                    self._decorate_table(
                        part,
                        context=dict(
                            context, master=cls, self=part, **{cls.__name__: cls}
                        ),
                    )

    def _decorate_table(self, table_class, context, assert_declared=False):
        """
        assign schema properties to the table class and declare the table
        """
        table_class.database = self.database
        table_class._connection = self.connection
        table_class._heading = Heading(
            table_info=dict(
                conn=self.connection,
                database=self.database,
                table_name=table_class.table_name,
                context=context,
            )
        )
        table_class._support = [table_class.full_table_name]
        table_class.declaration_context = context

        # instantiate the class, declare the table if not already
        instance = table_class()
        is_declared = instance.is_declared
        if not is_declared:
            if not self.create_tables or assert_declared:
                raise DataJointError("Table `%s` not declared" % instance.table_name)
            instance.declare(context)
            self.connection.dependencies.clear()
        is_declared = is_declared or instance.is_declared

        # add table definition to the doc string
        if isinstance(table_class.definition, str):
            table_class.__doc__ = (
                (table_class.__doc__ or "")
                + "\nTable definition:\n\n"
                + table_class.definition
            )

        # fill values in Lookup tables from their contents property
        if (
            isinstance(instance, Lookup)
            and hasattr(instance, "contents")
            and is_declared
        ):
            contents = list(instance.contents)
            if len(contents) > len(instance):
                if instance.heading.has_autoincrement:
                    warnings.warn(
                        (
                            "Contents has changed but cannot be inserted because "
                            "{table} has autoincrement."
                        ).format(table=instance.__class__.__name__)
                    )
                else:
                    instance.insert(contents, skip_duplicates=True)

    @property
    def log(self):
        self._assert_exists()
        if self._log is None:
            self._log = Log(self.connection, self.database)
        return self._log

    def __repr__(self):
        return "Schema `{name}`\n".format(name=self.database)

    @property
    def size_on_disk(self):
        """
        :return: size of the entire schema in bytes
        """
        self._assert_exists()
        return int(
            self.connection.query(
                """
            SELECT SUM(data_length + index_length)
            FROM information_schema.tables WHERE table_schema='{db}'
            """.format(
                    db=self.database
                )
            ).fetchone()[0]
        )

    def spawn_missing_classes(self, context=None):
        """
        Creates the appropriate python user relation classes from tables in the schema and places them
        in the context.

        :param context: alternative context to place the missing classes into, e.g. locals()
        """
        self._assert_exists()
        if context is None:
            if self.context is not None:
                context = self.context
            else:
                # if context is missing, use the calling namespace
                frame = inspect.currentframe().f_back
                context = frame.f_locals
                del frame
        tables = [
            row[0]
            for row in self.connection.query("SHOW TABLES in `%s`" % self.database)
            if lookup_class_name(
                "`{db}`.`{tab}`".format(db=self.database, tab=row[0]), context, 0
            )
            is None
        ]
        master_classes = (Lookup, Manual, Imported, Computed)
        part_tables = []
        for table_name in tables:
            class_name = to_camel_case(table_name)
            if class_name not in context:
                try:
                    cls = next(
                        cls
                        for cls in master_classes
                        if re.fullmatch(cls.tier_regexp, table_name)
                    )
                except StopIteration:
                    if re.fullmatch(Part.tier_regexp, table_name):
                        part_tables.append(table_name)
                else:
                    # declare and decorate master relation classes
                    context[class_name] = self(
                        type(class_name, (cls,), dict()), context=context
                    )

        # attach parts to masters
        for table_name in part_tables:
            groups = re.fullmatch(Part.tier_regexp, table_name).groupdict()
            class_name = to_camel_case(groups["part"])
            try:
                master_class = context[to_camel_case(groups["master"])]
            except KeyError:
                raise DataJointError(
                    "The table %s does not follow DataJoint naming conventions"
                    % table_name
                )
            part_class = type(class_name, (Part,), dict(definition=...))
            part_class._master = master_class
            self._decorate_table(part_class, context=context, assert_declared=True)
            setattr(master_class, class_name, part_class)

    def drop(self, force=False):
        """
        Drop the associated schema if it exists
        """
        if not self.exists:
            logger.info(
                "Schema named `{database}` does not exist. Doing nothing.".format(
                    database=self.database
                )
            )
        elif (
            not config["safemode"]
            or force
            or user_choice(
                "Proceed to delete entire schema `%s`?" % self.database, default="no"
            )
            == "yes"
        ):
            logger.info("Dropping `{database}`.".format(database=self.database))
            try:
                self.connection.query(
                    "DROP DATABASE `{database}`".format(database=self.database)
                )
                logger.info(
                    "Schema `{database}` was dropped successfully.".format(
                        database=self.database
                    )
                )
            except AccessError:
                raise AccessError(
                    "An attempt to drop schema `{database}` "
                    "has failed. Check permissions.".format(database=self.database)
                )

    @property
    def exists(self):
        """
        :return: true if the associated schema exists on the server
        """
        if self.database is None:
            raise DataJointError("Schema must be activated first.")
        return bool(
            self.connection.query(
                "SELECT schema_name "
                "FROM information_schema.schemata "
                "WHERE schema_name = '{database}'".format(database=self.database)
            ).rowcount
        )

    @property
    def jobs(self):
        """
        schema.jobs provides a view of the job reservation table for the schema

        :return: jobs table
        """
        self._assert_exists()
        if self._jobs is None:
            self._jobs = JobTable(self.connection, self.database)
        return self._jobs

    @property
    def code(self):
        self._assert_exists()
        return self.save()

    def save(self, python_filename=None):
        """
        Generate the code for a module that recreates the schema.
        This method is in preparation for a future release and is not officially supported.

        :return: a string containing the body of a complete Python module defining this schema.
        """
        self._assert_exists()
        module_count = itertools.count()
        # add virtual modules for referenced modules with names vmod0, vmod1, ...
        module_lookup = collections.defaultdict(
            lambda: "vmod" + str(next(module_count))
        )
        db = self.database

        def make_class_definition(table):
            tier = _get_tier(table).__name__
            class_name = table.split(".")[1].strip("`")
            indent = ""
            if tier == "Part":
                class_name = class_name.split("__")[-1]
                indent += "    "
            class_name = to_camel_case(class_name)

            def replace(s):
                d, tabs = s.group(1), s.group(2)
                return ("" if d == db else (module_lookup[d] + ".")) + ".".join(
                    to_camel_case(tab) for tab in tabs.lstrip("__").split("__")
                )

            return ("" if tier == "Part" else "\n@schema\n") + (
                "{indent}class {class_name}(dj.{tier}):\n"
                '{indent}    definition = """\n'
                '{indent}    {defi}"""'
            ).format(
                class_name=class_name,
                indent=indent,
                tier=tier,
                defi=re.sub(
                    r"`([^`]+)`.`([^`]+)`",
                    replace,
                    FreeTable(self.connection, table).describe(printout=False),
                ).replace("\n", "\n    " + indent),
            )

        diagram = Diagram(self)
        body = "\n\n".join(
            make_class_definition(table) for table in diagram.topological_sort()
        )
        python_code = "\n\n".join(
            (
                '"""This module was auto-generated by datajoint from an existing schema"""',
                "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
                "\n".join(
                    "{module} = dj.VirtualModule('{module}', '{schema_name}')".format(
                        module=v, schema_name=k
                    )
                    for k, v in module_lookup.items()
                ),
                body,
            )
        )
        if python_filename is None:
            return python_code
        with open(python_filename, "wt") as f:
            f.write(python_code)

    def list_tables(self):
        """
        Return a list of all tables in the schema except tables with ~ in first character such
        as ~logs and ~job

        :return: A list of table names from the database schema.
        """
        return [
            t
            for d, t in (
                full_t.replace("`", "").split(".")
                for full_t in Diagram(self).topological_sort()
            )
            if d == self.database
        ]


class VirtualModule(types.ModuleType):
    """
    A virtual module imitates a Python module representing a DataJoint schema from table definitions in the database.
    It declares the schema objects and a class for each table.
    """

    def __init__(
        self,
        module_name,
        schema_name,
        *,
        create_schema=False,
        create_tables=False,
        connection=None,
        add_objects=None
    ):
        """
        Creates a python module with the given name from the name of a schema on the server and
        automatically adds classes to it corresponding to the tables in the schema.

        :param module_name: displayed module name
        :param schema_name: name of the database in mysql
        :param create_schema: if True, create the schema on the database server
        :param create_tables: if True, module.schema can be used as the decorator for declaring new
        :param connection: a dj.Connection object to pass into the schema
        :param add_objects: additional objects to add to the module
        :return: the python module containing classes from the schema object and the table classes
        """
        super(VirtualModule, self).__init__(name=module_name)
        _schema = Schema(
            schema_name,
            create_schema=create_schema,
            create_tables=create_tables,
            connection=connection,
        )
        if add_objects:
            self.__dict__.update(add_objects)
        self.__dict__["schema"] = _schema
        _schema.spawn_missing_classes(context=self.__dict__)


def list_schemas(connection=None):
    """

    :param connection: a dj.Connection object
    :return: list of all accessible schemas on the server
    """
    return [
        r[0]
        for r in (connection or conn()).query(
            "SELECT schema_name "
            "FROM information_schema.schemata "
            'WHERE schema_name <> "information_schema"'
        )
    ]

1	import warnings	1✔
2	import logging	1✔
3	import inspect	1✔
4	import re	1✔
5	import itertools	1✔
6	import collections	1✔
7	from .connection import conn	1✔
8	from .diagram import Diagram, _get_tier	1✔
9	from .settings import config	1✔
10	from .errors import DataJointError, AccessError	1✔
11	from .jobs import JobTable	1✔
12	from .external import ExternalMapping	1✔
13	from .heading import Heading	1✔
14	from .utils import user_choice, to_camel_case	1✔
15	from .user_tables import Part, Computed, Imported, Manual, Lookup	1✔
16	from .table import lookup_class_name, Log, FreeTable	1✔
17	import types	1✔
18
19	logger = logging.getLogger(__name__)	1✔
20
21
22	def ordered_dir(class_):	1✔
23	"""
24	List (most) attributes of the class including inherited ones, similar to `dir` build-in function,
25	but respects order of attribute declaration as much as possible.
26
27	:param class_: class to list members for
28	:return: a list of attributes declared in class_ and its superclasses
29	"""
30	attr_list = list()	1✔
31	for c in reversed(class_.mro()):	1✔
32	attr_list.extend(e for e in c.__dict__ if e not in attr_list)	1✔
33	return attr_list	1✔
34
35
36	class Schema:	1✔
37	"""
38	A schema object is a decorator for UserTable classes that binds them to their database.
39	It also specifies the namespace `context` in which other UserTable classes are defined.
40	"""
41
42	def __init__(	1✔
43	self,
44	schema_name=None,
45	context=None,
46	*,
47	connection=None,
48	create_schema=True,
49	create_tables=True,
50	add_objects=None
51	):
52	"""
53	Associate database schema `schema_name`. If the schema does not exist, attempt to
54	create it on the server.
55
56	If the schema_name is omitted, then schema.activate(..) must be called later
57	to associate with the database.
58
59	:param schema_name: the database schema to associate.
60	:param context: dictionary for looking up foreign key references, leave None to use local context.
61	:param connection: Connection object. Defaults to datajoint.conn().
62	:param create_schema: When False, do not create the schema and raise an error if missing.
63	:param create_tables: When False, do not create tables and raise errors when accessing missing tables.
64	:param add_objects: a mapping with additional objects to make available to the context in which table classes
65	are declared.
66	"""
67	self._log = None	1✔
68	self.connection = connection	1✔
69	self.database = None	1✔
70	self.context = context	1✔
71	self.create_schema = create_schema	1✔
72	self.create_tables = create_tables	1✔
73	self._jobs = None	1✔
74	self.external = ExternalMapping(self)	1✔
75	self.add_objects = add_objects	1✔
76	self.declare_list = []	1✔
77	if schema_name:	1✔
78	self.activate(schema_name)	1✔
79
80	def is_activated(self):	1✔
81	return self.database is not None	1✔
82
83	def activate(	1✔
84	self,
85	schema_name=None,
86	*,
87	connection=None,
88	create_schema=None,
89	create_tables=None,
90	add_objects=None
91	):
92	"""
93	Associate database schema `schema_name`. If the schema does not exist, attempt to
94	create it on the server.
95
96	:param schema_name: the database schema to associate.
97	schema_name=None is used to assert that the schema has already been activated.
98	:param connection: Connection object. Defaults to datajoint.conn().
99	:param create_schema: If False, do not create the schema and raise an error if missing.
100	:param create_tables: If False, do not create tables and raise errors when attempting
101	to access missing tables.
102	:param add_objects: a mapping with additional objects to make available to the context
103	in which table classes are declared.
104	"""
105	if schema_name is None:	1✔
106	if self.exists:	×
107	return	×
108	raise DataJointError("Please provide a schema_name to activate the schema.")	×
109	if self.database is not None and self.exists:	1✔
110	if self.database == schema_name: # already activated	×
111	return	×
112	raise DataJointError(	×
113	"The schema is already activated for schema {db}.".format(
114	db=self.database
115	)
116	)
117	if connection is not None:	1✔
118	self.connection = connection	1✔
119	if self.connection is None:	1✔
120	self.connection = conn()	1✔
121	self.database = schema_name	1✔
122	if create_schema is not None:	1✔
123	self.create_schema = create_schema	×
124	if create_tables is not None:	1✔
125	self.create_tables = create_tables	×
126	if add_objects:	1✔
127	self.add_objects = add_objects	×
128	if not self.exists:	1✔
129	if not self.create_schema or not self.database:	1✔
130	raise DataJointError(	×
131	"Database `{name}` has not yet been declared. "
132	"Set argument create_schema=True to create it.".format(
133	name=schema_name
134	)
135	)
136	# create database
137	logger.info("Creating schema `{name}`.".format(name=schema_name))	1✔
138	try:	1✔
139	self.connection.query(	1✔
140	"CREATE DATABASE `{name}`".format(name=schema_name)
141	)
142	except AccessError:	1✔
143	raise DataJointError(	1✔
144	"Schema `{name}` does not exist and could not be created. "
145	"Check permissions.".format(name=schema_name)
146	)
147	else:
148	self.log("created")	1✔
149	self.connection.register(self)	1✔
150
151	# decorate all tables already decorated
152	for cls, context in self.declare_list:	1✔
153	if self.add_objects:	1✔
154	context = dict(context, **self.add_objects)	×
155	self._decorate_master(cls, context)	1✔
156
157	def _assert_exists(self, message=None):	1✔
158	if not self.exists:	1✔
159	raise DataJointError(	×
160	message
161	or "Schema `{db}` has not been created.".format(db=self.database)
162	)
163
164	def __call__(self, cls, *, context=None):	1✔
165	"""
166	Binds the supplied class to a schema. This is intended to be used as a decorator.
167
168	:param cls: class to decorate.
169	:param context: supplied when called from spawn_missing_classes
170	"""
171	context = context or self.context or inspect.currentframe().f_back.f_locals	1✔
172	if issubclass(cls, Part):	1✔
173	raise DataJointError(	1✔
174	"The schema decorator should not be applied to Part relations"
175	)
176	if self.is_activated():	1✔
177	self._decorate_master(cls, context)	1✔
178	else:
179	self.declare_list.append((cls, context))	1✔
180	return cls	1✔
181
182	def _decorate_master(self, cls, context):	1✔
183	"""
184
185	:param cls: the master class to process
186	:param context: the class' declaration context
187	"""
188	self._decorate_table(	1✔
189	cls, context=dict(context, self=cls, **{cls.__name__: cls})
190	)
191	# Process part tables
192	for part in ordered_dir(cls):	1✔
193	if part[0].isupper():	1✔
194	part = getattr(cls, part)	1✔
195	if inspect.isclass(part) and issubclass(part, Part):	1✔
196	part._master = cls	1✔
197	# allow addressing master by name or keyword 'master'
198	self._decorate_table(	1✔
199	part,
200	context=dict(
201	context, master=cls, self=part, **{cls.__name__: cls}
202	),
203	)
204
205	def _decorate_table(self, table_class, context, assert_declared=False):	1✔
206	"""
207	assign schema properties to the table class and declare the table
208	"""
209	table_class.database = self.database	1✔
210	table_class._connection = self.connection	1✔
211	table_class._heading = Heading(	1✔
212	table_info=dict(
213	conn=self.connection,
214	database=self.database,
215	table_name=table_class.table_name,
216	context=context,
217	)
218	)
219	table_class._support = [table_class.full_table_name]	1✔
220	table_class.declaration_context = context	1✔
221
222	# instantiate the class, declare the table if not already
223	instance = table_class()	1✔
224	is_declared = instance.is_declared	1✔
225	if not is_declared:	1✔
226	if not self.create_tables or assert_declared:	1✔
227	raise DataJointError("Table `%s` not declared" % instance.table_name)	×
228	instance.declare(context)	1✔
229	self.connection.dependencies.clear()	1✔
230	is_declared = is_declared or instance.is_declared	1✔
231
232	# add table definition to the doc string
233	if isinstance(table_class.definition, str):	1✔
234	table_class.__doc__ = (	1✔
235	(table_class.__doc__ or "")
236	+ "\nTable definition:\n\n"
237	+ table_class.definition
238	)
239
240	# fill values in Lookup tables from their contents property
241	if (	1✔
242	isinstance(instance, Lookup)
243	and hasattr(instance, "contents")
244	and is_declared
245	):
246	contents = list(instance.contents)	1✔
247	if len(contents) > len(instance):	1✔
248	if instance.heading.has_autoincrement:	1✔
249	warnings.warn(	×
250	(
251	"Contents has changed but cannot be inserted because "
252	"{table} has autoincrement."
253	).format(table=instance.__class__.__name__)
254	)
255	else:
256	instance.insert(contents, skip_duplicates=True)	1✔
257
258	@property	1✔
259	def log(self):	1✔
260	self._assert_exists()	1✔
261	if self._log is None:	1✔
262	self._log = Log(self.connection, self.database)	1✔
263	return self._log	1✔
264
265	def __repr__(self):	1✔
266	return "Schema `{name}`\n".format(name=self.database)	×
267
268	@property	1✔
269	def size_on_disk(self):	1✔
270	"""
271	:return: size of the entire schema in bytes
272	"""
273	self._assert_exists()	1✔
274	return int(	1✔
275	self.connection.query(
276	"""
277	SELECT SUM(data_length + index_length)
278	FROM information_schema.tables WHERE table_schema='{db}'
279	""".format(
280	db=self.database
281	)
282	).fetchone()[0]
283	)
284
285	def spawn_missing_classes(self, context=None):	1✔
286	"""
287	Creates the appropriate python user relation classes from tables in the schema and places them
288	in the context.
289
290	:param context: alternative context to place the missing classes into, e.g. locals()
291	"""
292	self._assert_exists()	1✔
293	if context is None:	1✔
294	if self.context is not None:	1✔
295	context = self.context	1✔
296	else:
297	# if context is missing, use the calling namespace
298	frame = inspect.currentframe().f_back	1✔
299	context = frame.f_locals	1✔
300	del frame	1✔
301	tables = [	1✔
302	row[0]
303	for row in self.connection.query("SHOW TABLES in `%s`" % self.database)
304	if lookup_class_name(
305	"`{db}`.`{tab}`".format(db=self.database, tab=row[0]), context, 0
306	)
307	is None
308	]
309	master_classes = (Lookup, Manual, Imported, Computed)	1✔
310	part_tables = []	1✔
311	for table_name in tables:	1✔
312	class_name = to_camel_case(table_name)	1✔
313	if class_name not in context:	1✔
314	try:	1✔
315	cls = next(	1✔
316	cls
317	for cls in master_classes
318	if re.fullmatch(cls.tier_regexp, table_name)
319	)
320	except StopIteration:	1✔
321	if re.fullmatch(Part.tier_regexp, table_name):	1✔
322	part_tables.append(table_name)	1✔
323	else:
324	# declare and decorate master relation classes
325	context[class_name] = self(	1✔
326	type(class_name, (cls,), dict()), context=context
327	)
328
329	# attach parts to masters
330	for table_name in part_tables:	1✔
331	groups = re.fullmatch(Part.tier_regexp, table_name).groupdict()	1✔
332	class_name = to_camel_case(groups["part"])	1✔
333	try:	1✔
334	master_class = context[to_camel_case(groups["master"])]	1✔
335	except KeyError:	×
336	raise DataJointError(	×
337	"The table %s does not follow DataJoint naming conventions"
338	% table_name
339	)
340	part_class = type(class_name, (Part,), dict(definition=...))	1✔
341	part_class._master = master_class	1✔
342	self._decorate_table(part_class, context=context, assert_declared=True)	1✔
343	setattr(master_class, class_name, part_class)	1✔
344
345	def drop(self, force=False):	1✔
346	"""
347	Drop the associated schema if it exists
348	"""
349	if not self.exists:	1✔
350	logger.info(	1✔
351	"Schema named `{database}` does not exist. Doing nothing.".format(
352	database=self.database
353	)
354	)
355	elif (	1✔
356	not config["safemode"]
357	or force
358	or user_choice(
359	"Proceed to delete entire schema `%s`?" % self.database, default="no"
360	)
361	== "yes"
362	):
363	logger.info("Dropping `{database}`.".format(database=self.database))	1✔
364	try:	1✔
365	self.connection.query(	1✔
366	"DROP DATABASE `{database}`".format(database=self.database)
367	)
368	logger.info(	1✔
369	"Schema `{database}` was dropped successfully.".format(
370	database=self.database
371	)
372	)
373	except AccessError:	1✔
374	raise AccessError(	1✔
375	"An attempt to drop schema `{database}` "
376	"has failed. Check permissions.".format(database=self.database)
377	)
378
379	@property	1✔
380	def exists(self):	1✔
381	"""
382	:return: true if the associated schema exists on the server
383	"""
384	if self.database is None:	1✔
385	raise DataJointError("Schema must be activated first.")	×
386	return bool(	1✔
387	self.connection.query(
388	"SELECT schema_name "
389	"FROM information_schema.schemata "
390	"WHERE schema_name = '{database}'".format(database=self.database)
391	).rowcount
392	)
393
394	@property	1✔
395	def jobs(self):	1✔
396	"""
397	schema.jobs provides a view of the job reservation table for the schema
398
399	:return: jobs table
400	"""
401	self._assert_exists()	1✔
402	if self._jobs is None:	1✔
403	self._jobs = JobTable(self.connection, self.database)	1✔
404	return self._jobs	1✔
405
406	@property	1✔
407	def code(self):	1✔
408	self._assert_exists()	1✔
409	return self.save()	1✔
410
411	def save(self, python_filename=None):	1✔
412	"""
413	Generate the code for a module that recreates the schema.
414	This method is in preparation for a future release and is not officially supported.
415
416	:return: a string containing the body of a complete Python module defining this schema.
417	"""
418	self._assert_exists()	1✔
419	module_count = itertools.count()	1✔
420	# add virtual modules for referenced modules with names vmod0, vmod1, ...
421	module_lookup = collections.defaultdict(	1✔
422	lambda: "vmod" + str(next(module_count))
423	)
424	db = self.database	1✔
425
426	def make_class_definition(table):	1✔
427	tier = _get_tier(table).__name__	1✔
428	class_name = table.split(".")[1].strip("`")	1✔
429	indent = ""	1✔
430	if tier == "Part":	1✔
431	class_name = class_name.split("__")[-1]	1✔
432	indent += " "	1✔
433	class_name = to_camel_case(class_name)	1✔
434
435	def replace(s):	1✔
436	d, tabs = s.group(1), s.group(2)	1✔
437	return ("" if d == db else (module_lookup[d] + ".")) + ".".join(	1✔
438	to_camel_case(tab) for tab in tabs.lstrip("__").split("__")
439	)
440
441	return ("" if tier == "Part" else "\n@schema\n") + (	1✔
442	"{indent}class {class_name}(dj.{tier}):\n"
443	'{indent} definition = """\n'
444	'{indent} {defi}"""'
445	).format(
446	class_name=class_name,
447	indent=indent,
448	tier=tier,
449	defi=re.sub(
450	r"`([^`]+)`.`([^`]+)`",
451	replace,
452	FreeTable(self.connection, table).describe(printout=False),
453	).replace("\n", "\n " + indent),
454	)
455
456	diagram = Diagram(self)	1✔
457	body = "\n\n".join(	1✔
458	make_class_definition(table) for table in diagram.topological_sort()
459	)
460	python_code = "\n\n".join(	1✔
461	(
462	'"""This module was auto-generated by datajoint from an existing schema"""',
463	"import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
464	"\n".join(
465	"{module} = dj.VirtualModule('{module}', '{schema_name}')".format(
466	module=v, schema_name=k
467	)
468	for k, v in module_lookup.items()
469	),
470	body,
471	)
472	)
473	if python_filename is None:	1✔
474	return python_code	1✔
475	with open(python_filename, "wt") as f:	×
476	f.write(python_code)	×
477
478	def list_tables(self):	1✔
479	"""
480	Return a list of all tables in the schema except tables with ~ in first character such
481	as ~logs and ~job
482
483	:return: A list of table names from the database schema.
484	"""
485	return [	1✔
486	t
487	for d, t in (
488	full_t.replace("`", "").split(".")
489	for full_t in Diagram(self).topological_sort()
490	)
491	if d == self.database
492	]
493
494
495	class VirtualModule(types.ModuleType):	1✔
496	"""
497	A virtual module imitates a Python module representing a DataJoint schema from table definitions in the database.
498	It declares the schema objects and a class for each table.
499	"""
500
501	def __init__(	1✔
502	self,
503	module_name,
504	schema_name,
505	*,
506	create_schema=False,
507	create_tables=False,
508	connection=None,
509	add_objects=None
510	):
511	"""
512	Creates a python module with the given name from the name of a schema on the server and
513	automatically adds classes to it corresponding to the tables in the schema.
514
515	:param module_name: displayed module name
516	:param schema_name: name of the database in mysql
517	:param create_schema: if True, create the schema on the database server
518	:param create_tables: if True, module.schema can be used as the decorator for declaring new
519	:param connection: a dj.Connection object to pass into the schema
520	:param add_objects: additional objects to add to the module
521	:return: the python module containing classes from the schema object and the table classes
522	"""
523	super(VirtualModule, self).__init__(name=module_name)	1✔
524	_schema = Schema(	1✔
525	schema_name,
526	create_schema=create_schema,
527	create_tables=create_tables,
528	connection=connection,
529	)
530	if add_objects:	1✔
531	self.__dict__.update(add_objects)	1✔
532	self.__dict__["schema"] = _schema	1✔
533	_schema.spawn_missing_classes(context=self.__dict__)	1✔
534
535
536	def list_schemas(connection=None):	1✔
537	"""
538
539	:param connection: a dj.Connection object
540	:return: list of all accessible schemas on the server
541	"""
542	return [	1✔
543	r[0]
544	for r in (connection or conn()).query(
545	"SELECT schema_name "
546	"FROM information_schema.schemata "
547	'WHERE schema_name <> "information_schema"'
548	)
549	]

datajoint / datajoint-python / #12028

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous