#2326

pending completion

Build # #2326

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/87f22d4a3f23df45949f827735c7bc5d88eb1534">87f22d4a3<a href="https://github.com/datajoint/datajoint-python/commit/87f22d4a3f23df45949f827735c7bc5d88eb1534">">Merge </a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/92989d0205950d6dcd62a69e8d054e5ee934bc16">92989d020</a>">92989d020</a><a href="https://github.com/datajoint/datajoint-python/commit/87f22d4a3f23df45949f827735c7bc5d88eb1534"> into 1e0e4b2f7">1e0e4b2f7</a>

Run Details

1082 of 1082 new or added lines in 21 files covered. (100.0%)

2635 of 2890 relevant lines covered (91.18%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.81

/datajoint/settings.py

"""
Settings for DataJoint.
"""
from contextlib import contextmanager
import json
import os
import pprint
from collections import OrderedDict
import logging
import collections
from enum import Enum
from .errors import DataJointError

LOCALCONFIG = 'dj_local_conf.json'
GLOBALCONFIG = '.datajoint_config.json'

DEFAULT_SUBFOLDING = (2, 2)  # subfolding for external storage in filesystem.  2, 2 means that file abcdef is stored as /ab/cd/abcdef

validators = collections.defaultdict(lambda: lambda value: True)
validators['database.port'] = lambda a: isinstance(a, int)

Role = Enum('Role', 'manual lookup imported computed job')
role_to_prefix = {
    Role.manual: '',
    Role.lookup: '#',
    Role.imported: '_',
    Role.computed: '__',
    Role.job: '~'
}
prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix))

default = OrderedDict({
    'database.host': 'localhost',
    'database.password': None,
    'database.user': None,
    'database.port': 3306,
    'database.reconnect': True,
    'connection.init_function': None,
    'connection.charset': '',   # pymysql uses '' as default
    'loglevel': 'INFO',
    'safemode': True,
    'fetch_format': 'array',
    'display.limit': 12,
    'display.width': 14,
    'display.show_tuple_count': True,
    'database.use_tls': None
})

logger = logging.getLogger(__name__)
log_levels = {
    'INFO': logging.INFO,
    'WARNING': logging.WARNING,
    'CRITICAL': logging.CRITICAL,
    'DEBUG': logging.DEBUG,
    'ERROR': logging.ERROR,
    None:  logging.NOTSET
}


class Config(collections.MutableMapping):

    instance = None

    def __init__(self, *args, **kwargs):
            if not Config.instance:
                Config.instance = Config.__Config(*args, **kwargs)
            else:
                Config.instance._conf.update(dict(*args, **kwargs))

    def __getattr__(self, name):
        return getattr(self.instance, name)

    def __getitem__(self, item):
        return self.instance.__getitem__(item)

    def __setitem__(self, item, value):
        self.instance.__setitem__(item, value)

    def __str__(self):
        return pprint.pformat(self.instance._conf, indent=4)

    def __repr__(self):
        return self.__str__()

    def __delitem__(self, key):
        del self.instance._conf[key]

    def __iter__(self):
        return iter(self.instance._conf)

    def __len__(self):
        return len(self.instance._conf)

    def save(self, filename, verbose=False):
        """
        Saves the settings in JSON format to the given file path.
        :param filename: filename of the local JSON settings file.
        :param verbose: report having saved the settings file
        """
        with open(filename, 'w') as fid:
            json.dump(self._conf, fid, indent=4)
        if verbose:
            print('Saved settings in ' + filename)

    def load(self, filename):
        """
        Updates the setting from config file in JSON format.
        :param filename: filename of the local JSON settings file. If None, the local config file is used.
        """
        if filename is None:
            filename = LOCALCONFIG
        with open(filename, 'r') as fid:
            self._conf.update(json.load(fid))

    def save_local(self, verbose=False):
        """
        saves the settings in the local config file
        """
        self.save(LOCALCONFIG, verbose)

    def save_global(self, verbose=False):
        """
        saves the settings in the global config file
        """
        self.save(os.path.expanduser(os.path.join('~', GLOBALCONFIG)), verbose)

    def get_store_spec(self, store):
        """
        find configuration of external stores for blobs and attachments
        """
        try:
            spec = self['stores'][store]
        except KeyError:
            raise DataJointError('Storage {store} is requested but not configured'.format(store=store)) from None

        spec['subfolding'] = spec.get('subfolding', DEFAULT_SUBFOLDING)
        spec_keys = {  # REQUIRED in uppercase and allowed in lowercase
            'file': ('PROTOCOL', 'LOCATION', 'subfolding', 'stage'),
            's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION', 'subfolding', 'stage')}

        try:
            spec_keys = spec_keys[spec.get('protocol', '').lower()]
        except KeyError:
            raise DataJointError(
                'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(store=store)) from None

        # check that all required keys are present in spec
        try:
            raise DataJointError('dj.config["stores"]["{store}" is missing "{k}"'.format(
                store=store, k=next(k.lower() for k in spec_keys if k.isupper() and k.lower() not in spec)))
        except StopIteration:
            pass

        # check that only allowed keys are present in spec
        try:
            raise DataJointError('Invalid key "{k}" in dj.config["stores"]["{store}"]'.format(
                store=store, k=next(k for k in spec if k.upper() not in spec_keys and k.lower() not in spec_keys)))
        except StopIteration:
            pass  # no invalid keys

        return spec

    @contextmanager
    def __call__(self, **kwargs):
        """
        The config object can also be used in a with statement to change the state of the configuration
        temporarily. kwargs to the context manager are the keys into config, where '.' is replaced by a
        double underscore '__'. The context manager yields the changed config object.

        Example:
        >>> import datajoint as dj
        >>> with dj.config(safemode=False, database__host="localhost") as cfg:
        >>>     # do dangerous stuff here
        """

        try:
            backup = self.instance
            self.instance = Config.__Config(self.instance._conf)
            new = {k.replace('__', '.'): v for k, v in kwargs.items()}
            self.instance._conf.update(new)
            yield self
        except:
            self.instance = backup
            raise
        else:
            self.instance = backup

    class __Config:
        """
        Stores datajoint settings. Behaves like a dictionary, but applies validator functions
        when certain keys are set.

        The default parameters are stored in datajoint.settings.default . If a local config file
        exists, the settings specified in this file override the default settings.
        """

        def __init__(self, *args, **kwargs):
            self._conf = dict(default)
            self._conf.update(dict(*args, **kwargs))  # use the free update to set keys

        def __getitem__(self, key):
            return self._conf[key]

        def __setitem__(self, key, value):
            logger.log(logging.INFO, u"Setting {0:s} to {1:s}".format(str(key), str(value)))
            if validators[key](value):
                self._conf[key] = value
            else:
                raise DataJointError(u'Validator for {0:s} did not pass'.format(key))


# Load configuration from file
config = Config()
config_files = (os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join('~', GLOBALCONFIG)))
try:
    config_file = next(n for n in config_files if os.path.exists(n))
except StopIteration:
    pass
else:
    config.load(config_file)

# override login credentials with environment variables
mapping = {k: v for k, v in zip(
    ('database.host', 'database.user', 'database.password',
     'external.aws_access_key_id', 'external.aws_secret_access_key',),
    map(os.getenv, ('DJ_HOST', 'DJ_USER', 'DJ_PASS',
                    'DJ_AWS_ACCESS_KEY_ID', 'DJ_AWS_SECRET_ACCESS_KEY',)))
           if v is not None}
config.update(mapping)

logger.setLevel(log_levels[config['loglevel']])

1	"""
2	Settings for DataJoint.
3	"""
4	from contextlib import contextmanager	1✔
5	import json	1✔
6	import os	1✔
7	import pprint	1✔
8	from collections import OrderedDict	1✔
9	import logging	1✔
10	import collections	1✔
11	from enum import Enum	1✔
12	from .errors import DataJointError	1✔
13
14	LOCALCONFIG = 'dj_local_conf.json'	1✔
15	GLOBALCONFIG = '.datajoint_config.json'	1✔
16
17	DEFAULT_SUBFOLDING = (2, 2) # subfolding for external storage in filesystem. 2, 2 means that file abcdef is stored as /ab/cd/abcdef	1✔
18
19	validators = collections.defaultdict(lambda: lambda value: True)	1✔
20	validators['database.port'] = lambda a: isinstance(a, int)	1✔
21
22	Role = Enum('Role', 'manual lookup imported computed job')	1✔
23	role_to_prefix = {	1✔
24	Role.manual: '',
25	Role.lookup: '#',
26	Role.imported: '_',
27	Role.computed: '__',
28	Role.job: '~'
29	}
30	prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix))	1✔
31
32	default = OrderedDict({	1✔
33	'database.host': 'localhost',
34	'database.password': None,
35	'database.user': None,
36	'database.port': 3306,
37	'database.reconnect': True,
38	'connection.init_function': None,
39	'connection.charset': '', # pymysql uses '' as default
40	'loglevel': 'INFO',
41	'safemode': True,
42	'fetch_format': 'array',
43	'display.limit': 12,
44	'display.width': 14,
45	'display.show_tuple_count': True,
46	'database.use_tls': None
47	})
48
49	logger = logging.getLogger(__name__)	1✔
50	log_levels = {	1✔
51	'INFO': logging.INFO,
52	'WARNING': logging.WARNING,
53	'CRITICAL': logging.CRITICAL,
54	'DEBUG': logging.DEBUG,
55	'ERROR': logging.ERROR,
56	None: logging.NOTSET
57	}
58
59
60	class Config(collections.MutableMapping):	1✔
61
62	instance = None	1✔
63
64	def __init__(self, args, *kwargs):	1✔
65	if not Config.instance:	1✔
66	Config.instance = Config.__Config(args, *kwargs)	1✔
67	else:
68	Config.instance._conf.update(dict(args, *kwargs))	1✔
69
70	def __getattr__(self, name):	1✔
71	return getattr(self.instance, name)	1✔
72
73	def __getitem__(self, item):	1✔
74	return self.instance.__getitem__(item)	1✔
75
76	def __setitem__(self, item, value):	1✔
77	self.instance.__setitem__(item, value)	1✔
78
79	def __str__(self):	1✔
80	return pprint.pformat(self.instance._conf, indent=4)	1✔
81
82	def __repr__(self):	1✔
83	return self.__str__()	1✔
84
85	def __delitem__(self, key):	1✔
86	del self.instance._conf[key]	1✔
87
88	def __iter__(self):	1✔
89	return iter(self.instance._conf)	1✔
90
91	def __len__(self):	1✔
92	return len(self.instance._conf)	1✔
93
94	def save(self, filename, verbose=False):	1✔
95	"""
96	Saves the settings in JSON format to the given file path.
97	:param filename: filename of the local JSON settings file.
98	:param verbose: report having saved the settings file
99	"""
100	with open(filename, 'w') as fid:	1✔
101	json.dump(self._conf, fid, indent=4)	1✔
102	if verbose:	1✔
103	print('Saved settings in ' + filename)	×
104
105	def load(self, filename):	1✔
106	"""
107	Updates the setting from config file in JSON format.
108	:param filename: filename of the local JSON settings file. If None, the local config file is used.
109	"""
110	if filename is None:	1✔
111	filename = LOCALCONFIG	×
112	with open(filename, 'r') as fid:	1✔
113	self._conf.update(json.load(fid))	1✔
114
115	def save_local(self, verbose=False):	1✔
116	"""
117	saves the settings in the local config file
118	"""
119	self.save(LOCALCONFIG, verbose)	1✔
120
121	def save_global(self, verbose=False):	1✔
122	"""
123	saves the settings in the global config file
124	"""
125	self.save(os.path.expanduser(os.path.join('~', GLOBALCONFIG)), verbose)	×
126
127	def get_store_spec(self, store):	1✔
128	"""
129	find configuration of external stores for blobs and attachments
130	"""
131	try:	1✔
132	spec = self['stores'][store]	1✔
133	except KeyError:	×
134	raise DataJointError('Storage {store} is requested but not configured'.format(store=store)) from None	×
135
136	spec['subfolding'] = spec.get('subfolding', DEFAULT_SUBFOLDING)	1✔
137	spec_keys = { # REQUIRED in uppercase and allowed in lowercase	1✔
138	'file': ('PROTOCOL', 'LOCATION', 'subfolding', 'stage'),
139	's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION', 'subfolding', 'stage')}
140
141	try:	1✔
142	spec_keys = spec_keys[spec.get('protocol', '').lower()]	1✔
143	except KeyError:	×
144	raise DataJointError(	×
145	'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(store=store)) from None
146
147	# check that all required keys are present in spec
148	try:	1✔
149	raise DataJointError('dj.config["stores"]["{store}" is missing "{k}"'.format(	1✔
150	store=store, k=next(k.lower() for k in spec_keys if k.isupper() and k.lower() not in spec)))
151	except StopIteration:	1✔
152	pass	1✔
153
154	# check that only allowed keys are present in spec
155	try:	1✔
156	raise DataJointError('Invalid key "{k}" in dj.config["stores"]["{store}"]'.format(	1✔
157	store=store, k=next(k for k in spec if k.upper() not in spec_keys and k.lower() not in spec_keys)))
158	except StopIteration:	1✔
159	pass # no invalid keys	1✔
160
161	return spec	1✔
162
163	@contextmanager	1✔
164	def __call__(self, **kwargs):
165	"""
166	The config object can also be used in a with statement to change the state of the configuration
167	temporarily. kwargs to the context manager are the keys into config, where '.' is replaced by a
168	double underscore '__'. The context manager yields the changed config object.
169
170	Example:
171	>>> import datajoint as dj
172	>>> with dj.config(safemode=False, database__host="localhost") as cfg:
173	>>> # do dangerous stuff here
174	"""
175
176	try:	1✔
177	backup = self.instance	1✔
178	self.instance = Config.__Config(self.instance._conf)	1✔
179	new = {k.replace('__', '.'): v for k, v in kwargs.items()}	1✔
180	self.instance._conf.update(new)	1✔
181	yield self	1✔
182	except:	×
183	self.instance = backup	×
184	raise	×
185	else:
186	self.instance = backup	1✔
187
188	class __Config:	1✔
189	"""
190	Stores datajoint settings. Behaves like a dictionary, but applies validator functions
191	when certain keys are set.
192
193	The default parameters are stored in datajoint.settings.default . If a local config file
194	exists, the settings specified in this file override the default settings.
195	"""
196
197	def __init__(self, args, *kwargs):	1✔
198	self._conf = dict(default)	1✔
199	self._conf.update(dict(args, *kwargs)) # use the free update to set keys	1✔
200
201	def __getitem__(self, key):	1✔
202	return self._conf[key]	1✔
203
204	def __setitem__(self, key, value):	1✔
205	logger.log(logging.INFO, u"Setting {0:s} to {1:s}".format(str(key), str(value)))	1✔
206	if validators[key](value):	1✔
207	self._conf[key] = value	1✔
208	else:
209	raise DataJointError(u'Validator for {0:s} did not pass'.format(key))	1✔
210
211
212	# Load configuration from file
213	config = Config()	1✔
214	config_files = (os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join('~', GLOBALCONFIG)))	1✔
215	try:	1✔
216	config_file = next(n for n in config_files if os.path.exists(n))	1✔
217	except StopIteration:	1✔
218	pass	1✔
219	else:
220	config.load(config_file)	×
221
222	# override login credentials with environment variables
223	mapping = {k: v for k, v in zip(	1✔
224	('database.host', 'database.user', 'database.password',
225	'external.aws_access_key_id', 'external.aws_secret_access_key',),
226	map(os.getenv, ('DJ_HOST', 'DJ_USER', 'DJ_PASS',
227	'DJ_AWS_ACCESS_KEY_ID', 'DJ_AWS_SECRET_ACCESS_KEY',)))
228	if v is not None}
229	config.update(mapping)	1✔
230
231	logger.setLevel(log_levels[config['loglevel']])	1✔

datajoint / datajoint-python / #2326

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous