#12454

pending completion

Build # #12454

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://git"><a class=hub.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://git"><a class=hub.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://git"><a class=hub.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://git"><a class=hub.com/datajoint/datajoint-python/commit/a066e18e52c140decf592a764552e48c53495f72">a066e18e5">&lt;a href=&quot;https://github.com/datajoint/datajoint-python/commit/</a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datajoint/datajoint-python/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datajoint/datajoint-python/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datajoint/datajoint-python/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datajoint/datajoint-python/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datajoint/datajoint-python/commit/a066e18e52c140decf592a764552e48c53495f72">a066e18e5</a><a href="https://github.com/datajoint/datajoint-python/commit/a066e18e52c140decf592a764552e48c53495f72">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/datajoint/datajoint-python/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/datajoint/datajoint-python/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datajoint/datajoint-python/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datajoint/datajoint-python/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datajoint/datajoint-python/commit/a066e18e52c140decf592a764552e48c53495f72&quot;&gt;a066e18e5&lt;/a&gt;&lt;a href=&quot;https://github.com/datajoint/datajoint-python/commit/a066e18e52c140decf592a764552e48c53495f72&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/datajoint/datajoint-python/commit/a066e18e52c140decf592a764552e48c53495f72&amp;quot;&amp;... (continued)

Run Details

66 of 66 new or added lines in 14 files covered. (100.0%)

3012 of 3348 relevant lines covered (89.96%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.52

/datajoint/settings.py

"""
Settings for DataJoint.
"""
from contextlib import contextmanager
import json
import os
import pprint
import logging
import collections
from enum import Enum
from .errors import DataJointError

LOCALCONFIG = "dj_local_conf.json"
GLOBALCONFIG = ".datajoint_config.json"
# subfolding for external storage in filesystem.
# 2, 2 means that file abcdef is stored as /ab/cd/abcdef
DEFAULT_SUBFOLDING = (2, 2)

validators = collections.defaultdict(lambda: lambda value: True)
validators["database.port"] = lambda a: isinstance(a, int)

Role = Enum("Role", "manual lookup imported computed job")
role_to_prefix = {
    Role.manual: "",
    Role.lookup: "#",
    Role.imported: "_",
    Role.computed: "__",
    Role.job: "~",
}
prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix))

default = dict(
    {
        "database.host": "localhost",
        "database.password": None,
        "database.user": None,
        "database.port": 3306,
        "database.reconnect": True,
        "connection.init_function": None,
        "connection.charset": "",  # pymysql uses '' as default
        "loglevel": "INFO",
        "safemode": True,
        "fetch_format": "array",
        "display.limit": 12,
        "display.width": 14,
        "display.show_tuple_count": True,
        "database.use_tls": None,
        "enable_python_native_blobs": True,  # python-native/dj0 encoding support
    }
)

logger = logging.getLogger(__name__.split(".")[0])
log_levels = {
    "INFO": logging.INFO,
    "WARNING": logging.WARNING,
    "CRITICAL": logging.CRITICAL,
    "DEBUG": logging.DEBUG,
    "ERROR": logging.ERROR,
    None: logging.NOTSET,
}


class Config(collections.abc.MutableMapping):

    instance = None

    def __init__(self, *args, **kwargs):
        if not Config.instance:
            Config.instance = Config.__Config(*args, **kwargs)
        else:
            Config.instance._conf.update(dict(*args, **kwargs))

    def __getattr__(self, name):
        return getattr(self.instance, name)

    def __getitem__(self, item):
        return self.instance.__getitem__(item)

    def __setitem__(self, item, value):
        self.instance.__setitem__(item, value)

    def __str__(self):
        return pprint.pformat(self.instance._conf, indent=4)

    def __repr__(self):
        return self.__str__()

    def __delitem__(self, key):
        del self.instance._conf[key]

    def __iter__(self):
        return iter(self.instance._conf)

    def __len__(self):
        return len(self.instance._conf)

    def save(self, filename, verbose=False):
        """
        Saves the settings in JSON format to the given file path.

        :param filename: filename of the local JSON settings file.
        :param verbose: report having saved the settings file
        """
        with open(filename, "w") as fid:
            json.dump(self._conf, fid, indent=4)
        if verbose:
            logger.info("Saved settings in " + filename)

    def load(self, filename):
        """
        Updates the setting from config file in JSON format.

        :param filename: filename of the local JSON settings file. If None, the local config file is used.
        """
        if filename is None:
            filename = LOCALCONFIG
        with open(filename, "r") as fid:
            self._conf.update(json.load(fid))

    def save_local(self, verbose=False):
        """
        saves the settings in the local config file
        """
        self.save(LOCALCONFIG, verbose)

    def save_global(self, verbose=False):
        """
        saves the settings in the global config file
        """
        self.save(os.path.expanduser(os.path.join("~", GLOBALCONFIG)), verbose)

    def get_store_spec(self, store):
        """
        find configuration of external stores for blobs and attachments
        """
        try:
            spec = self["stores"][store]
        except KeyError:
            raise DataJointError(
                "Storage {store} is requested but not configured".format(store=store)
            )

        spec["subfolding"] = spec.get("subfolding", DEFAULT_SUBFOLDING)
        spec_keys = {  # REQUIRED in uppercase and allowed in lowercase
            "file": ("PROTOCOL", "LOCATION", "subfolding", "stage"),
            "s3": (
                "PROTOCOL",
                "ENDPOINT",
                "BUCKET",
                "ACCESS_KEY",
                "SECRET_KEY",
                "LOCATION",
                "secure",
                "subfolding",
                "stage",
                "proxy_server",
            ),
        }

        try:
            spec_keys = spec_keys[spec.get("protocol", "").lower()]
        except KeyError:
            raise DataJointError(
                'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(
                    store=store
                )
            )

        # check that all required keys are present in spec
        try:
            raise DataJointError(
                'dj.config["stores"]["{store}"] is missing "{k}"'.format(
                    store=store,
                    k=next(
                        k.lower()
                        for k in spec_keys
                        if k.isupper() and k.lower() not in spec
                    ),
                )
            )
        except StopIteration:
            pass

        # check that only allowed keys are present in spec
        try:
            raise DataJointError(
                'Invalid key "{k}" in dj.config["stores"]["{store}"]'.format(
                    store=store,
                    k=next(
                        k
                        for k in spec
                        if k.upper() not in spec_keys and k.lower() not in spec_keys
                    ),
                )
            )
        except StopIteration:
            pass  # no invalid keys

        return spec

    @contextmanager
    def __call__(self, **kwargs):
        """
        The config object can also be used in a with statement to change the state of the configuration
        temporarily. kwargs to the context manager are the keys into config, where '.' is replaced by a
        double underscore '__'. The context manager yields the changed config object.

        Example:
        >>> import datajoint as dj
        >>> with dj.config(safemode=False, database__host="localhost") as cfg:
        >>>     # do dangerous stuff here
        """

        try:
            backup = self.instance
            self.instance = Config.__Config(self.instance._conf)
            new = {k.replace("__", "."): v for k, v in kwargs.items()}
            self.instance._conf.update(new)
            yield self
        except:
            self.instance = backup
            raise
        else:
            self.instance = backup

    class __Config:
        """
        Stores datajoint settings. Behaves like a dictionary, but applies validator functions
        when certain keys are set.

        The default parameters are stored in datajoint.settings.default . If a local config file
        exists, the settings specified in this file override the default settings.
        """

        def __init__(self, *args, **kwargs):
            self._conf = dict(default)
            self._conf.update(dict(*args, **kwargs))  # use the free update to set keys

        def __getitem__(self, key):
            return self._conf[key]

        def __setitem__(self, key, value):
            logger.log(
                logging.INFO, "Setting {0:s} to {1:s}".format(str(key), str(value))
            )
            if validators[key](value):
                self._conf[key] = value
            else:
                raise DataJointError("Validator for {0:s} did not pass".format(key))


# Load configuration from file
config = Config()
config_files = (
    os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join("~", GLOBALCONFIG))
)
try:
    config_file = next(n for n in config_files if os.path.exists(n))
except StopIteration:
    pass
else:
    config.load(config_file)

# override login credentials with environment variables
mapping = {
    k: v
    for k, v in zip(
        (
            "database.host",
            "database.user",
            "database.password",
            "external.aws_access_key_id",
            "external.aws_secret_access_key",
        ),
        map(
            os.getenv,
            (
                "DJ_HOST",
                "DJ_USER",
                "DJ_PASS",
                "DJ_AWS_ACCESS_KEY_ID",
                "DJ_AWS_SECRET_ACCESS_KEY",
            ),
        ),
    )
    if v is not None
}
config.update(mapping)

logger.setLevel(log_levels[config["loglevel"]])

1	"""
2	Settings for DataJoint.
3	"""
4	from contextlib import contextmanager	1✔
5	import json	1✔
6	import os	1✔
7	import pprint	1✔
8	import logging	1✔
9	import collections	1✔
10	from enum import Enum	1✔
11	from .errors import DataJointError	1✔
12
13	LOCALCONFIG = "dj_local_conf.json"	1✔
14	GLOBALCONFIG = ".datajoint_config.json"	1✔
15	# subfolding for external storage in filesystem.
16	# 2, 2 means that file abcdef is stored as /ab/cd/abcdef
17	DEFAULT_SUBFOLDING = (2, 2)	1✔
18
19	validators = collections.defaultdict(lambda: lambda value: True)	1✔
20	validators["database.port"] = lambda a: isinstance(a, int)	1✔
21
22	Role = Enum("Role", "manual lookup imported computed job")	1✔
23	role_to_prefix = {	1✔
24	Role.manual: "",
25	Role.lookup: "#",
26	Role.imported: "_",
27	Role.computed: "__",
28	Role.job: "~",
29	}
30	prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix))	1✔
31
32	default = dict(	1✔
33	{
34	"database.host": "localhost",
35	"database.password": None,
36	"database.user": None,
37	"database.port": 3306,
38	"database.reconnect": True,
39	"connection.init_function": None,
40	"connection.charset": "", # pymysql uses '' as default
41	"loglevel": "INFO",
42	"safemode": True,
43	"fetch_format": "array",
44	"display.limit": 12,
45	"display.width": 14,
46	"display.show_tuple_count": True,
47	"database.use_tls": None,
48	"enable_python_native_blobs": True, # python-native/dj0 encoding support
49	}
50	)
51
52	logger = logging.getLogger(__name__.split(".")[0])	1✔
53	log_levels = {	1✔
54	"INFO": logging.INFO,
55	"WARNING": logging.WARNING,
56	"CRITICAL": logging.CRITICAL,
57	"DEBUG": logging.DEBUG,
58	"ERROR": logging.ERROR,
59	None: logging.NOTSET,
60	}
61
62
63	class Config(collections.abc.MutableMapping):	1✔
64
65	instance = None	1✔
66
67	def __init__(self, args, *kwargs):	1✔
68	if not Config.instance:	1✔
69	Config.instance = Config.__Config(args, *kwargs)	1✔
70	else:
71	Config.instance._conf.update(dict(args, *kwargs))	1✔
72
73	def __getattr__(self, name):	1✔
74	return getattr(self.instance, name)	1✔
75
76	def __getitem__(self, item):	1✔
77	return self.instance.__getitem__(item)	1✔
78
79	def __setitem__(self, item, value):	1✔
80	self.instance.__setitem__(item, value)	1✔
81
82	def __str__(self):	1✔
83	return pprint.pformat(self.instance._conf, indent=4)	1✔
84
85	def __repr__(self):	1✔
86	return self.__str__()	1✔
87
88	def __delitem__(self, key):	1✔
89	del self.instance._conf[key]	1✔
90
91	def __iter__(self):	1✔
92	return iter(self.instance._conf)	1✔
93
94	def __len__(self):	1✔
95	return len(self.instance._conf)	1✔
96
97	def save(self, filename, verbose=False):	1✔
98	"""
99	Saves the settings in JSON format to the given file path.
100
101	:param filename: filename of the local JSON settings file.
102	:param verbose: report having saved the settings file
103	"""
104	with open(filename, "w") as fid:	1✔
105	json.dump(self._conf, fid, indent=4)	1✔
106	if verbose:	1✔
107	logger.info("Saved settings in " + filename)	×
108
109	def load(self, filename):	1✔
110	"""
111	Updates the setting from config file in JSON format.
112
113	:param filename: filename of the local JSON settings file. If None, the local config file is used.
114	"""
115	if filename is None:	1✔
116	filename = LOCALCONFIG	×
117	with open(filename, "r") as fid:	1✔
118	self._conf.update(json.load(fid))	1✔
119
120	def save_local(self, verbose=False):	1✔
121	"""
122	saves the settings in the local config file
123	"""
124	self.save(LOCALCONFIG, verbose)	1✔
125
126	def save_global(self, verbose=False):	1✔
127	"""
128	saves the settings in the global config file
129	"""
130	self.save(os.path.expanduser(os.path.join("~", GLOBALCONFIG)), verbose)	×
131
132	def get_store_spec(self, store):	1✔
133	"""
134	find configuration of external stores for blobs and attachments
135	"""
136	try:	1✔
137	spec = self["stores"][store]	1✔
138	except KeyError:	×
139	raise DataJointError(	×
140	"Storage {store} is requested but not configured".format(store=store)
141	)
142
143	spec["subfolding"] = spec.get("subfolding", DEFAULT_SUBFOLDING)	1✔
144	spec_keys = { # REQUIRED in uppercase and allowed in lowercase	1✔
145	"file": ("PROTOCOL", "LOCATION", "subfolding", "stage"),
146	"s3": (
147	"PROTOCOL",
148	"ENDPOINT",
149	"BUCKET",
150	"ACCESS_KEY",
151	"SECRET_KEY",
152	"LOCATION",
153	"secure",
154	"subfolding",
155	"stage",
156	"proxy_server",
157	),
158	}
159
160	try:	1✔
161	spec_keys = spec_keys[spec.get("protocol", "").lower()]	1✔
162	except KeyError:	×
163	raise DataJointError(	×
164	'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(
165	store=store
166	)
167	)
168
169	# check that all required keys are present in spec
170	try:	1✔
171	raise DataJointError(	1✔
172	'dj.config["stores"]["{store}"] is missing "{k}"'.format(
173	store=store,
174	k=next(
175	k.lower()
176	for k in spec_keys
177	if k.isupper() and k.lower() not in spec
178	),
179	)
180	)
181	except StopIteration:	1✔
182	pass	1✔
183
184	# check that only allowed keys are present in spec
185	try:	1✔
186	raise DataJointError(	1✔
187	'Invalid key "{k}" in dj.config["stores"]["{store}"]'.format(
188	store=store,
189	k=next(
190	k
191	for k in spec
192	if k.upper() not in spec_keys and k.lower() not in spec_keys
193	),
194	)
195	)
196	except StopIteration:	1✔
197	pass # no invalid keys	1✔
198
199	return spec	1✔
200
201	@contextmanager	1✔
202	def __call__(self, **kwargs):
203	"""
204	The config object can also be used in a with statement to change the state of the configuration
205	temporarily. kwargs to the context manager are the keys into config, where '.' is replaced by a
206	double underscore '__'. The context manager yields the changed config object.
207
208	Example:
209	>>> import datajoint as dj
210	>>> with dj.config(safemode=False, database__host="localhost") as cfg:
211	>>> # do dangerous stuff here
212	"""
213
214	try:	1✔
215	backup = self.instance	1✔
216	self.instance = Config.__Config(self.instance._conf)	1✔
217	new = {k.replace("__", "."): v for k, v in kwargs.items()}	1✔
218	self.instance._conf.update(new)	1✔
219	yield self	1✔
220	except:	1✔
221	self.instance = backup	1✔
222	raise	1✔
223	else:
224	self.instance = backup	1✔
225
226	class __Config:	1✔
227	"""
228	Stores datajoint settings. Behaves like a dictionary, but applies validator functions
229	when certain keys are set.
230
231	The default parameters are stored in datajoint.settings.default . If a local config file
232	exists, the settings specified in this file override the default settings.
233	"""
234
235	def __init__(self, args, *kwargs):	1✔
236	self._conf = dict(default)	1✔
237	self._conf.update(dict(args, *kwargs)) # use the free update to set keys	1✔
238
239	def __getitem__(self, key):	1✔
240	return self._conf[key]	1✔
241
242	def __setitem__(self, key, value):	1✔
243	logger.log(	1✔
244	logging.INFO, "Setting {0:s} to {1:s}".format(str(key), str(value))
245	)
246	if validators[key](value):	1✔
247	self._conf[key] = value	1✔
248	else:
249	raise DataJointError("Validator for {0:s} did not pass".format(key))	1✔
250
251
252	# Load configuration from file
253	config = Config()	1✔
254	config_files = (	1✔
255	os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join("~", GLOBALCONFIG))
256	)
257	try:	1✔
258	config_file = next(n for n in config_files if os.path.exists(n))	1✔
259	except StopIteration:	1✔
260	pass	1✔
261	else:
262	config.load(config_file)	×
263
264	# override login credentials with environment variables
265	mapping = {	1✔
266	k: v
267	for k, v in zip(
268	(
269	"database.host",
270	"database.user",
271	"database.password",
272	"external.aws_access_key_id",
273	"external.aws_secret_access_key",
274	),
275	map(
276	os.getenv,
277	(
278	"DJ_HOST",
279	"DJ_USER",
280	"DJ_PASS",
281	"DJ_AWS_ACCESS_KEY_ID",
282	"DJ_AWS_SECRET_ACCESS_KEY",
283	),
284	),
285	)
286	if v is not None
287	}
288	config.update(mapping)	1✔
289
290	logger.setLevel(log_levels[config["loglevel"]])	1✔

datajoint / datajoint-python / #12454

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous