#12396

pending completion

Build # #12396

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/4898b9994b57dd36b083eb82370d4a6b4be1bd1a">4898b9994<a href="https://github.com/datajoint/datajoint-python/commit/4898b9994b57dd36b083eb82370d4a6b4be1bd1a">">Merge </a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/267095f2263810d368f9f7fc4ef2424334c1b65d">267095f22</a>">267095f22</a><a href="https://github.com/datajoint/datajoint-python/commit/4898b9994b57dd36b083eb82370d4a6b4be1bd1a"> into 0ff34f283">0ff34f283</a>

Run Details

11 of 11 new or added lines in 2 files covered. (100.0%)

3067 of 3392 relevant lines covered (90.42%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.59

/datajoint/settings.py

"""
Settings for DataJoint.
"""
from contextlib import contextmanager
import json
import os
import pprint
import logging
import collections
from enum import Enum
from .errors import DataJointError

LOCALCONFIG = "dj_local_conf.json"
GLOBALCONFIG = ".datajoint_config.json"
# subfolding for external storage in filesystem.
# 2, 2 means that file abcdef is stored as /ab/cd/abcdef
DEFAULT_SUBFOLDING = (2, 2)

validators = collections.defaultdict(lambda: lambda value: True)
validators["database.port"] = lambda a: isinstance(a, int)

Role = Enum("Role", "manual lookup imported computed job")
role_to_prefix = {
    Role.manual: "",
    Role.lookup: "#",
    Role.imported: "_",
    Role.computed: "__",
    Role.job: "~",
}
prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix))

default = dict(
    {
        "database.host": "localhost",
        "database.password": None,
        "database.user": None,
        "database.port": 3306,
        "database.reconnect": True,
        "connection.init_function": None,
        "connection.charset": "",  # pymysql uses '' as default
        "loglevel": "INFO",
        "safemode": True,
        "fetch_format": "array",
        "display.limit": 12,
        "display.width": 14,
        "display.show_tuple_count": True,
        "database.use_tls": None,
        "enable_python_native_blobs": True,  # python-native/dj0 encoding support
        "filepath_checksum_size_limit": None,  # file size limit for when to disable checksums
    }
)

logger = logging.getLogger(__name__)
log_levels = {
    "INFO": logging.INFO,
    "WARNING": logging.WARNING,
    "CRITICAL": logging.CRITICAL,
    "DEBUG": logging.DEBUG,
    "ERROR": logging.ERROR,
    None: logging.NOTSET,
}


class Config(collections.abc.MutableMapping):

    instance = None

    def __init__(self, *args, **kwargs):
        if not Config.instance:
            Config.instance = Config.__Config(*args, **kwargs)
        else:
            Config.instance._conf.update(dict(*args, **kwargs))

    def __getattr__(self, name):
        return getattr(self.instance, name)

    def __getitem__(self, item):
        return self.instance.__getitem__(item)

    def __setitem__(self, item, value):
        self.instance.__setitem__(item, value)

    def __str__(self):
        return pprint.pformat(self.instance._conf, indent=4)

    def __repr__(self):
        return self.__str__()

    def __delitem__(self, key):
        del self.instance._conf[key]

    def __iter__(self):
        return iter(self.instance._conf)

    def __len__(self):
        return len(self.instance._conf)

    def save(self, filename, verbose=False):
        """
        Saves the settings in JSON format to the given file path.

        :param filename: filename of the local JSON settings file.
        :param verbose: report having saved the settings file
        """
        with open(filename, "w") as fid:
            json.dump(self._conf, fid, indent=4)
        if verbose:
            print("Saved settings in " + filename)

    def load(self, filename):
        """
        Updates the setting from config file in JSON format.

        :param filename: filename of the local JSON settings file. If None, the local config file is used.
        """
        if filename is None:
            filename = LOCALCONFIG
        with open(filename, "r") as fid:
            self._conf.update(json.load(fid))

    def save_local(self, verbose=False):
        """
        saves the settings in the local config file
        """
        self.save(LOCALCONFIG, verbose)

    def save_global(self, verbose=False):
        """
        saves the settings in the global config file
        """
        self.save(os.path.expanduser(os.path.join("~", GLOBALCONFIG)), verbose)

    def get_store_spec(self, store):
        """
        find configuration of external stores for blobs and attachments
        """
        try:
            spec = self["stores"][store]
        except KeyError:
            raise DataJointError(
                "Storage {store} is requested but not configured".format(store=store)
            )

        spec["subfolding"] = spec.get("subfolding", DEFAULT_SUBFOLDING)
        spec_keys = {  # REQUIRED in uppercase and allowed in lowercase
            "file": ("PROTOCOL", "LOCATION", "subfolding", "stage"),
            "s3": (
                "PROTOCOL",
                "ENDPOINT",
                "BUCKET",
                "ACCESS_KEY",
                "SECRET_KEY",
                "LOCATION",
                "secure",
                "subfolding",
                "stage",
                "proxy_server",
            ),
        }

        try:
            spec_keys = spec_keys[spec.get("protocol", "").lower()]
        except KeyError:
            raise DataJointError(
                'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(
                    store=store
                )
            )

        # check that all required keys are present in spec
        try:
            raise DataJointError(
                'dj.config["stores"]["{store}"] is missing "{k}"'.format(
                    store=store,
                    k=next(
                        k.lower()
                        for k in spec_keys
                        if k.isupper() and k.lower() not in spec
                    ),
                )
            )
        except StopIteration:
            pass

        # check that only allowed keys are present in spec
        try:
            raise DataJointError(
                'Invalid key "{k}" in dj.config["stores"]["{store}"]'.format(
                    store=store,
                    k=next(
                        k
                        for k in spec
                        if k.upper() not in spec_keys and k.lower() not in spec_keys
                    ),
                )
            )
        except StopIteration:
            pass  # no invalid keys

        return spec

    @contextmanager
    def __call__(self, **kwargs):
        """
        The config object can also be used in a with statement to change the state of the configuration
        temporarily. kwargs to the context manager are the keys into config, where '.' is replaced by a
        double underscore '__'. The context manager yields the changed config object.

        Example:
        >>> import datajoint as dj
        >>> with dj.config(safemode=False, database__host="localhost") as cfg:
        >>>     # do dangerous stuff here
        """

        try:
            backup = self.instance
            self.instance = Config.__Config(self.instance._conf)
            new = {k.replace("__", "."): v for k, v in kwargs.items()}
            self.instance._conf.update(new)
            yield self
        except:
            self.instance = backup
            raise
        else:
            self.instance = backup

    class __Config:
        """
        Stores datajoint settings. Behaves like a dictionary, but applies validator functions
        when certain keys are set.

        The default parameters are stored in datajoint.settings.default . If a local config file
        exists, the settings specified in this file override the default settings.
        """

        def __init__(self, *args, **kwargs):
            self._conf = dict(default)
            self._conf.update(dict(*args, **kwargs))  # use the free update to set keys

        def __getitem__(self, key):
            return self._conf[key]

        def __setitem__(self, key, value):
            logger.log(
                logging.INFO, "Setting {0:s} to {1:s}".format(str(key), str(value))
            )
            if validators[key](value):
                self._conf[key] = value
            else:
                raise DataJointError("Validator for {0:s} did not pass".format(key))


# Load configuration from file
config = Config()
config_files = (
    os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join("~", GLOBALCONFIG))
)
try:
    config_file = next(n for n in config_files if os.path.exists(n))
except StopIteration:
    pass
else:
    config.load(config_file)

# override login credentials with environment variables
mapping = {
    k: v
    for k, v in zip(
        (
            "database.host",
            "database.user",
            "database.password",
            "external.aws_access_key_id",
            "external.aws_secret_access_key",
        ),
        map(
            os.getenv,
            (
                "DJ_HOST",
                "DJ_USER",
                "DJ_PASS",
                "DJ_AWS_ACCESS_KEY_ID",
                "DJ_AWS_SECRET_ACCESS_KEY",
            ),
        ),
    )
    if v is not None
}
config.update(mapping)

logger.setLevel(log_levels[config["loglevel"]])

1	"""
2	Settings for DataJoint.
3	"""
4	from contextlib import contextmanager	1✔
5	import json	1✔
6	import os	1✔
7	import pprint	1✔
8	import logging	1✔
9	import collections	1✔
10	from enum import Enum	1✔
11	from .errors import DataJointError	1✔
12
13	LOCALCONFIG = "dj_local_conf.json"	1✔
14	GLOBALCONFIG = ".datajoint_config.json"	1✔
15	# subfolding for external storage in filesystem.
16	# 2, 2 means that file abcdef is stored as /ab/cd/abcdef
17	DEFAULT_SUBFOLDING = (2, 2)	1✔
18
19	validators = collections.defaultdict(lambda: lambda value: True)	1✔
20	validators["database.port"] = lambda a: isinstance(a, int)	1✔
21
22	Role = Enum("Role", "manual lookup imported computed job")	1✔
23	role_to_prefix = {	1✔
24	Role.manual: "",
25	Role.lookup: "#",
26	Role.imported: "_",
27	Role.computed: "__",
28	Role.job: "~",
29	}
30	prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix))	1✔
31
32	default = dict(	1✔
33	{
34	"database.host": "localhost",
35	"database.password": None,
36	"database.user": None,
37	"database.port": 3306,
38	"database.reconnect": True,
39	"connection.init_function": None,
40	"connection.charset": "", # pymysql uses '' as default
41	"loglevel": "INFO",
42	"safemode": True,
43	"fetch_format": "array",
44	"display.limit": 12,
45	"display.width": 14,
46	"display.show_tuple_count": True,
47	"database.use_tls": None,
48	"enable_python_native_blobs": True, # python-native/dj0 encoding support
49	"filepath_checksum_size_limit": None, # file size limit for when to disable checksums
50	}
51	)
52
53	logger = logging.getLogger(__name__)	1✔
54	log_levels = {	1✔
55	"INFO": logging.INFO,
56	"WARNING": logging.WARNING,
57	"CRITICAL": logging.CRITICAL,
58	"DEBUG": logging.DEBUG,
59	"ERROR": logging.ERROR,
60	None: logging.NOTSET,
61	}
62
63
64	class Config(collections.abc.MutableMapping):	1✔
65
66	instance = None	1✔
67
68	def __init__(self, args, *kwargs):	1✔
69	if not Config.instance:	1✔
70	Config.instance = Config.__Config(args, *kwargs)	1✔
71	else:
72	Config.instance._conf.update(dict(args, *kwargs))	1✔
73
74	def __getattr__(self, name):	1✔
75	return getattr(self.instance, name)	1✔
76
77	def __getitem__(self, item):	1✔
78	return self.instance.__getitem__(item)	1✔
79
80	def __setitem__(self, item, value):	1✔
81	self.instance.__setitem__(item, value)	1✔
82
83	def __str__(self):	1✔
84	return pprint.pformat(self.instance._conf, indent=4)	1✔
85
86	def __repr__(self):	1✔
87	return self.__str__()	1✔
88
89	def __delitem__(self, key):	1✔
90	del self.instance._conf[key]	1✔
91
92	def __iter__(self):	1✔
93	return iter(self.instance._conf)	1✔
94
95	def __len__(self):	1✔
96	return len(self.instance._conf)	1✔
97
98	def save(self, filename, verbose=False):	1✔
99	"""
100	Saves the settings in JSON format to the given file path.
101
102	:param filename: filename of the local JSON settings file.
103	:param verbose: report having saved the settings file
104	"""
105	with open(filename, "w") as fid:	1✔
106	json.dump(self._conf, fid, indent=4)	1✔
107	if verbose:	1✔
108	print("Saved settings in " + filename)	×
109
110	def load(self, filename):	1✔
111	"""
112	Updates the setting from config file in JSON format.
113
114	:param filename: filename of the local JSON settings file. If None, the local config file is used.
115	"""
116	if filename is None:	1✔
117	filename = LOCALCONFIG	×
118	with open(filename, "r") as fid:	1✔
119	self._conf.update(json.load(fid))	1✔
120
121	def save_local(self, verbose=False):	1✔
122	"""
123	saves the settings in the local config file
124	"""
125	self.save(LOCALCONFIG, verbose)	1✔
126
127	def save_global(self, verbose=False):	1✔
128	"""
129	saves the settings in the global config file
130	"""
131	self.save(os.path.expanduser(os.path.join("~", GLOBALCONFIG)), verbose)	×
132
133	def get_store_spec(self, store):	1✔
134	"""
135	find configuration of external stores for blobs and attachments
136	"""
137	try:	1✔
138	spec = self["stores"][store]	1✔
139	except KeyError:	×
140	raise DataJointError(	×
141	"Storage {store} is requested but not configured".format(store=store)
142	)
143
144	spec["subfolding"] = spec.get("subfolding", DEFAULT_SUBFOLDING)	1✔
145	spec_keys = { # REQUIRED in uppercase and allowed in lowercase	1✔
146	"file": ("PROTOCOL", "LOCATION", "subfolding", "stage"),
147	"s3": (
148	"PROTOCOL",
149	"ENDPOINT",
150	"BUCKET",
151	"ACCESS_KEY",
152	"SECRET_KEY",
153	"LOCATION",
154	"secure",
155	"subfolding",
156	"stage",
157	"proxy_server",
158	),
159	}
160
161	try:	1✔
162	spec_keys = spec_keys[spec.get("protocol", "").lower()]	1✔
163	except KeyError:	×
164	raise DataJointError(	×
165	'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(
166	store=store
167	)
168	)
169
170	# check that all required keys are present in spec
171	try:	1✔
172	raise DataJointError(	1✔
173	'dj.config["stores"]["{store}"] is missing "{k}"'.format(
174	store=store,
175	k=next(
176	k.lower()
177	for k in spec_keys
178	if k.isupper() and k.lower() not in spec
179	),
180	)
181	)
182	except StopIteration:	1✔
183	pass	1✔
184
185	# check that only allowed keys are present in spec
186	try:	1✔
187	raise DataJointError(	1✔
188	'Invalid key "{k}" in dj.config["stores"]["{store}"]'.format(
189	store=store,
190	k=next(
191	k
192	for k in spec
193	if k.upper() not in spec_keys and k.lower() not in spec_keys
194	),
195	)
196	)
197	except StopIteration:	1✔
198	pass # no invalid keys	1✔
199
200	return spec	1✔
201
202	@contextmanager	1✔
203	def __call__(self, **kwargs):	1✔
204	"""
205	The config object can also be used in a with statement to change the state of the configuration
206	temporarily. kwargs to the context manager are the keys into config, where '.' is replaced by a
207	double underscore '__'. The context manager yields the changed config object.
208
209	Example:
210	>>> import datajoint as dj
211	>>> with dj.config(safemode=False, database__host="localhost") as cfg:
212	>>> # do dangerous stuff here
213	"""
214
215	try:	1✔
216	backup = self.instance	1✔
217	self.instance = Config.__Config(self.instance._conf)	1✔
218	new = {k.replace("__", "."): v for k, v in kwargs.items()}	1✔
219	self.instance._conf.update(new)	1✔
220	yield self	1✔
221	except:	1✔
222	self.instance = backup	1✔
223	raise	1✔
224	else:
225	self.instance = backup	1✔
226
227	class __Config:	1✔
228	"""
229	Stores datajoint settings. Behaves like a dictionary, but applies validator functions
230	when certain keys are set.
231
232	The default parameters are stored in datajoint.settings.default . If a local config file
233	exists, the settings specified in this file override the default settings.
234	"""
235
236	def __init__(self, args, *kwargs):	1✔
237	self._conf = dict(default)	1✔
238	self._conf.update(dict(args, *kwargs)) # use the free update to set keys	1✔
239
240	def __getitem__(self, key):	1✔
241	return self._conf[key]	1✔
242
243	def __setitem__(self, key, value):	1✔
244	logger.log(	1✔
245	logging.INFO, "Setting {0:s} to {1:s}".format(str(key), str(value))
246	)
247	if validators[key](value):	1✔
248	self._conf[key] = value	1✔
249	else:
250	raise DataJointError("Validator for {0:s} did not pass".format(key))	1✔
251
252
253	# Load configuration from file
254	config = Config()	1✔
255	config_files = (	1✔
256	os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join("~", GLOBALCONFIG))
257	)
258	try:	1✔
259	config_file = next(n for n in config_files if os.path.exists(n))	1✔
260	except StopIteration:	1✔
261	pass	1✔
262	else:
263	config.load(config_file)	×
264
265	# override login credentials with environment variables
266	mapping = {	1✔
267	k: v
268	for k, v in zip(
269	(
270	"database.host",
271	"database.user",
272	"database.password",
273	"external.aws_access_key_id",
274	"external.aws_secret_access_key",
275	),
276	map(
277	os.getenv,
278	(
279	"DJ_HOST",
280	"DJ_USER",
281	"DJ_PASS",
282	"DJ_AWS_ACCESS_KEY_ID",
283	"DJ_AWS_SECRET_ACCESS_KEY",
284	),
285	),
286	)
287	if v is not None
288	}
289	config.update(mapping)	1✔
290
291	logger.setLevel(log_levels[config["loglevel"]])	1✔

datajoint / datajoint-python / #12396

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous