2309

pending completion

Build # 2309

Build Type

push

travis-ci

Committed by

computron

Commit Message

<a href="https://github.com/materialsproject/fireworks/commit/<a class=hub.com/materialsproject/fireworks/commit/6a2ce4ae8a640eef6e9be76ae4d315b50d3c547b">6a2ce4ae8<a href="https://github.com/materialsproject/fireworks/commit/6a2ce4ae8a640eef6e9be76ae4d315b50d3c547b">&quot;&gt;Revert &amp;quot;try fixing docs for RTD&quot;

This reverts commit </a><a class="double-link" href="https://github.com/materialsproject/fireworks/commit/<a class="double-link" href="https://github.com/materialsproject/fireworks/commit/19cfaa9eee4a87a07cc0fabec311e2c5a23a06b3">19cfaa9ee</a>">19cfaa9ee</a><a href="https://github.com/materialsproject/fireworks/commit/6a2ce4ae8a640eef6e9be76ae4d315b50d3c547b">.

Run Details

2809 of 4575 relevant lines covered (61.4%)

0.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.29

/fireworks/utilities/fw_serializers.py

# coding: utf-8

from __future__ import unicode_literals

"""
This module aids in serializing and deserializing objects.

To serialize a FW object, refer to the documentation for the FWSerializable class. To de-serialize
an object, refer to the documentation for the FWSerializable class and load_object() method.
    - you can de-serialize explicitly, e.g. FWObject.from_dict() to enforce a FWObject instance as the result
    - you can de-serialize implicitly, e.g. load_object() to search for the correct Class dynamically

The implicit method is especially useful if you don't know in advance which subclass of a base class
your serialization might point to. e.g. if you require some type of Quadrilateral, a serialization
from your collaborator might point to a Square, Rhombus, or Rectangle, and you might not know which
one in advance...

Some advantages:
    - Robust with regard to code refactorings even in implicit loading given certain reasonable
        guidelines on fw_name.
    - Simple to allow a superclass to define all the serializations for its subclasses, removing
        code repetition(in particular, note that from_dict is a class method rather than a static
        method, allowing use of self)
    - Decorators aid in some of the routine parts of the serialization, such as adding the _fw_name key
    - Both JSON and YAML file import/export are naturally and concisely supported within the framework.
    - Auto-detect and proper loading of JSON and YAML files
    - Proper JSON handling of datetime (both encoding and decoding) and UTF-8 strings
    - In some cases, objects can be serialized/deserialized extremely concisely, by use of only
        their fw_name (if no parameters are needed to describe the object)

"""

import traceback
import pkgutil
import inspect
import json  # note that ujson is faster, but at this time does not support "default" in dumps()
import importlib
import datetime
import abc
import sys

import yaml
# Use CLoader for faster performance where possible.
try:
    from yaml import CLoader as Loader, CSafeDumper as Dumper
except ImportError:
    from yaml import Loader as Loader, SafeDumper as Dumper
import six

from monty.json import MontyDecoder, MSONable

from fireworks.fw_config import FW_NAME_UPDATES, YAML_STYLE, USER_PACKAGES, DECODE_MONTY, ENCODE_MONTY

__author__ = 'Anubhav Jain'
__copyright__ = 'Copyright 2012, The Materials Project'
__version__ = '0.1'
__maintainer__ = 'Anubhav Jain'
__email__ = 'ajain@lbl.gov'
__date__ = 'Dec 13, 2012'

# TODO: consider *somehow* switching FireWorks to monty serialization. e.g., numpy serialization is better handled.

SAVED_FW_MODULES = {}
DATETIME_HANDLER = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) else None

if sys.version_info > (3, 0, 0):
    ENCODING_PARAMS = {"encoding": "utf-8"}
else:
    ENCODING_PARAMS = {}

try:
    import numpy as np
    NUMPY_INSTALLED = True
except:
    NUMPY_INSTALLED = False


def recursive_dict(obj, preserve_unicode=True):
    if obj is None:
        return None

    if ENCODE_MONTY and hasattr(obj, 'as_dict'):  # compatible with new monty JSONEncoder (MontyEncoder)
        return recursive_dict(obj.as_dict(), preserve_unicode)

    if hasattr(obj, 'to_dict'):
        return recursive_dict(obj.to_dict(), preserve_unicode)

    if isinstance(obj, dict):
        return {recursive_dict(k, preserve_unicode): recursive_dict(v, preserve_unicode)
                for k, v in obj.items()}

    if isinstance(obj, (list, tuple)):
        return [recursive_dict(v, preserve_unicode) for v in obj]

    if isinstance(obj, int) or isinstance(obj, float):
        return obj

    if isinstance(obj, datetime.datetime):
        return obj.isoformat()

    if preserve_unicode and isinstance(obj, six.text_type) and obj != obj.encode('ascii', 'ignore'):
        return obj

    if NUMPY_INSTALLED and isinstance(obj, np.ndarray):
        return [recursive_dict(v, preserve_unicode) for v in obj.tolist()]

    return str(obj)


# TODO: is reconstitute_dates really needed? Can this method just do everything?
def _recursive_load(obj):
    if obj is None:
        return None

    if hasattr(obj, '_fw_name') or isinstance(obj, MSONable):
        return obj

    if isinstance(obj, dict):
        if '_fw_name' in obj:
            return load_object(obj)

        if DECODE_MONTY and '@module' in obj and '@class' in obj:  # MontyDecoder compatibility
            return json.loads(json.dumps(obj), cls=MontyDecoder)

        return {k: _recursive_load(v) for k, v in obj.items()}

    if isinstance(obj, (list, tuple)):
        return [_recursive_load(v) for v in obj]

    if isinstance(obj, six.string_types):
        try:
            # convert String to datetime if really datetime
            return reconstitute_dates(obj)
        except:
            # convert unicode to ASCII if not really unicode
            if obj == obj.encode('ascii', 'ignore'):
                return str(obj)

    return obj


def recursive_serialize(func):
    """
    a decorator to add FW serializations keys
    see documentation of FWSerializable for more details
    """
    def _decorator(self, *args, **kwargs):
        m_dict = func(self, *args, **kwargs)
        m_dict = recursive_dict(m_dict)
        return m_dict

    return _decorator


def recursive_deserialize(func):
    """
    a decorator to add FW serializations keys
    see documentation of FWSerializable for more details
    """
    def _decorator(self, *args, **kwargs):
        new_args = [a for a in args]
        new_args[0] = {k: _recursive_load(v) for k, v in args[0].items()}
        m_dict = func(self, *new_args, **kwargs)
        return m_dict

    return _decorator


def serialize_fw(func):
    """
    a decorator to add FW serializations keys
    see documentation of FWSerializable for more details
    """
    def _decorator(self, *args, **kwargs):
        m_dict = func(self, *args, **kwargs)
        m_dict['_fw_name'] = self.fw_name
        return m_dict

    return _decorator


@six.add_metaclass(abc.ABCMeta)
class FWSerializable(object):
    """
    To create a serializable object within FireWorks, you should subclass this
    class and implement the to_dict() and from_dict() methods.

    If you want the load_object() implicit de-serialization to work, you must
    also:
        - Use the @serialize_fw decorator on to_dict()
        - Override the _fw_name parameter with a unique key.

    See documentation of load_object() for more details.

    The use of @classmethod for from_dict allows you to define a superclass
    that implements the to_dict() and from_dict() for all its subclasses.

    For an example of serialization, see the class QueueAdapterBase.
    """

    @property
    def fw_name(self):
        try:
            return self._fw_name
        except AttributeError:
            return get_default_serialization(self.__class__)

    @abc.abstractmethod
    def to_dict(self):
        raise NotImplementedError('FWSerializable object did not implement to_dict()!')

    def to_db_dict(self):
        return self.to_dict()

    def as_dict(self):
        # strictly for pseudo-compatibility with MSONable
        # Note that FWSerializable is not MSONable, it uses _fw_name instead of __class__ and
        # __module__
        return self.to_dict()

    @classmethod
    @abc.abstractmethod
    def from_dict(cls, m_dict):
        raise NotImplementedError('FWSerializable object did not implement from_dict()!')

    def __repr__(self):
        return json.dumps(self.to_dict(), default=DATETIME_HANDLER)

    def to_format(self, f_format='json', **kwargs):
        """
        returns a String representation in the given format

        Args:
            f_format (str): the format to output to (default json)
        """
        if f_format == 'json':
            return json.dumps(self.to_dict(), default=DATETIME_HANDLER, **kwargs)
        elif f_format == 'yaml':
            # start with the JSON format, and convert to YAML
            return yaml.dump(self.to_dict(), default_flow_style=YAML_STYLE,
                             allow_unicode=True, Dumper=Dumper)
        else:
            raise ValueError('Unsupported format {}'.format(f_format))

    @classmethod
    def from_format(cls, f_str, f_format='json'):
        """
        convert from a String representation to its Object.

        Args:
            f_str (str): the String representation
            f_format (str): serialization format of the String (default json)

        Returns:
            FWSerializable
        """
        if f_format == 'json':
            return cls.from_dict(reconstitute_dates(json.loads(f_str)))
        elif f_format == 'yaml':
            return cls.from_dict(reconstitute_dates(
                yaml.load(f_str, Loader=Loader)))
        else:
            raise ValueError('Unsupported format {}'.format(f_format))

    def to_file(self, filename, f_format=None, **kwargs):
        """
        Write a serialization of this object to a file.

        Args:
            filename(str): filename to write to
            f_format (str): serialization format, default checks the filename extension
        """
        if f_format is None:
            f_format = filename.split('.')[-1]
        with open(filename, 'w', **ENCODING_PARAMS) as f:
            f.write(self.to_format(f_format=f_format, **kwargs))

    @classmethod
    def from_file(cls, filename, f_format=None):
        """
        Load a serialization of this object from a file.

        Args:
            filename (str): filename to read
            f_format (str): serialization format, default checks the filename extension

        Returns:
            FWSerializable
        """
        if f_format is None:
            f_format = filename.split('.')[-1]
        with open(filename, 'r', **ENCODING_PARAMS) as f:
            return cls.from_format(f.read(), f_format=f_format)

    def __getstate__(self):
        return self.to_dict()

    def __setstate__(self, state):
        fw_obj = self.from_dict(state)
        for k, v in fw_obj.__dict__.items():
            self.__dict__[k] = v


# TODO: make this quicker the first time around
def load_object(obj_dict):
    """
    Creates an instantiation of a class based on a dictionary representation. We implicitly
    determine the Class through introspection along with information in the dictionary.

    We search for a class with the _fw_name property equal to obj_dict['_fw_name']
    If the @module key is set, that module is checked first for a matching class
    to improve speed of lookup.
    Afterwards, the modules in the USER_PACKAGES global parameter are checked.

    Refactoring class names, module names, etc. will not break object loading
    as long as:

    i) the _fw_name property is maintained the same AND
    ii) the refactored module is kept within USER_PACKAGES

    You can get around these limitations if you really want:
    i) If you want to change the fw_name of an object you can set the FW_NAME_UPDATES key
    ii) If you want to put a refactored module in a new place add an entry to USER_PACKAGES

    Args:
        obj_dict (dict): the dict representation of the class
    """

    # override the name in the obj_dict if there's an entry in FW_NAME_UPDATES
    fw_name = FW_NAME_UPDATES.get(obj_dict['_fw_name'], obj_dict['_fw_name'])
    obj_dict['_fw_name'] = fw_name

    # check for explicit serialization, e.g. {{fireworks.tasks.MyTask}} - based on pymatgen method
    if fw_name.startswith('{{') and fw_name.endswith('}}'):
        modname, classname = fw_name.strip('{} ').rsplit(".", 1)
        mod = __import__(modname, globals(), locals(), [classname], 0)
        if hasattr(mod, classname):
            cls_ = getattr(mod, classname)
            return cls_.from_dict(obj_dict)

    # first try to load from known location
    if fw_name in SAVED_FW_MODULES:
        m_module = importlib.import_module(SAVED_FW_MODULES[fw_name])
        m_object = _search_module_for_obj(m_module, obj_dict)
        if m_object is not None:
            return m_object

    # failing that, look for the object within all of USER_PACKAGES
    # this will be slow, but only needed the first time

    found_objects = [] # used to make sure we don't find multiple hits
    for package in USER_PACKAGES:
        root_module = importlib.import_module(package)
        for loader, mod_name, is_pkg in pkgutil.walk_packages(
                root_module.__path__, package + '.'):
            try:
                m_module = importlib.import_module(mod_name)
                m_object = _search_module_for_obj(m_module, obj_dict)
                if m_object is not None:
                    found_objects.append((m_object, mod_name))
            except ImportError as ex:
                import warnings
                warnings.warn(
                    "%s in %s cannot be loaded because of %s. Skipping.."
                    % (m_object, mod_name, str(ex)))
                traceback.print_exc(ex)

    if len(found_objects) == 1:
        SAVED_FW_MODULES[fw_name] = found_objects[0][1]
        return found_objects[0][0]
    elif len(found_objects) > 0:
        raise ValueError(
            'load_object() found multiple objects with cls._fw_name {} -- {}'
            .format(fw_name, found_objects))

    raise ValueError('load_object() could not find a class with cls._fw_name {}'.format(fw_name))


def load_object_from_file(filename, f_format=None):
    """
    Implicitly load an object from a file. just a friendly wrapper to
    load_object()

    Args:
        filename (str): the filename to load an object from
        f_format (str): the serialization format (default is auto-detect based on
            filename extension)
    """
    m_dict = {}
    if f_format is None:
        f_format = filename.split('.')[-1]

    with open(filename, 'r', **ENCODING_PARAMS) as f:
        if f_format == 'json':
            m_dict = reconstitute_dates(json.loads(f.read()))
        elif f_format == 'yaml':
            m_dict = reconstitute_dates(yaml.load(f, Loader=Loader))
        else:
            raise ValueError('Unknown file format {} cannot be loaded!'.format(f_format))

    return load_object(m_dict)


def _search_module_for_obj(m_module, obj_dict):
    """
    internal method that looks in a module for a class with a given _fw_name
    """
    obj_name = obj_dict['_fw_name']

    for name, obj in inspect.getmembers(m_module):
        # check if the member is a Class matching our description
        if inspect.isclass(obj) and obj.__module__ == m_module.__name__ and \
                getattr(obj, '_fw_name', get_default_serialization(obj)) == obj_name:
            return obj.from_dict(obj_dict)


def reconstitute_dates(obj_dict):
    if obj_dict is None:
        return None

    if isinstance(obj_dict, dict):
        return {k: reconstitute_dates(v) for k, v in obj_dict.items()}

    if isinstance(obj_dict, (list, tuple)):
        return [reconstitute_dates(v) for v in obj_dict]

    if isinstance(obj_dict, six.string_types):
        try:
            return datetime.datetime.strptime(obj_dict, "%Y-%m-%dT%H:%M:%S.%f")
        except:
            try:
                return datetime.datetime.strptime(obj_dict, "%Y-%m-%dT%H:%M:%S")
            except:
                pass
    return obj_dict


def get_default_serialization(cls):
    root_mod = cls.__module__.split('.')[0]
    if root_mod == '__main__':
        raise ValueError("Cannot get default serialization; try "
                         "instantiating your object from a different module "
                         "from which it is defined rather than defining your "
                         "object in the __main__ (running) module.")
    return root_mod + '::' + cls.__name__  # e.g. fireworks.ABC

1	# coding: utf-8
2
3	from __future__ import unicode_literals	1✔
4
5	"""
6	This module aids in serializing and deserializing objects.
7
8	To serialize a FW object, refer to the documentation for the FWSerializable class. To de-serialize
9	an object, refer to the documentation for the FWSerializable class and load_object() method.
10	- you can de-serialize explicitly, e.g. FWObject.from_dict() to enforce a FWObject instance as the result
11	- you can de-serialize implicitly, e.g. load_object() to search for the correct Class dynamically
12
13	The implicit method is especially useful if you don't know in advance which subclass of a base class
14	your serialization might point to. e.g. if you require some type of Quadrilateral, a serialization
15	from your collaborator might point to a Square, Rhombus, or Rectangle, and you might not know which
16	one in advance...
17
18	Some advantages:
19	- Robust with regard to code refactorings even in implicit loading given certain reasonable
20	guidelines on fw_name.
21	- Simple to allow a superclass to define all the serializations for its subclasses, removing
22	code repetition(in particular, note that from_dict is a class method rather than a static
23	method, allowing use of self)
24	- Decorators aid in some of the routine parts of the serialization, such as adding the _fw_name key
25	- Both JSON and YAML file import/export are naturally and concisely supported within the framework.
26	- Auto-detect and proper loading of JSON and YAML files
27	- Proper JSON handling of datetime (both encoding and decoding) and UTF-8 strings
28	- In some cases, objects can be serialized/deserialized extremely concisely, by use of only
29	their fw_name (if no parameters are needed to describe the object)
30
31	"""
32
33	import traceback	1✔
34	import pkgutil	1✔
35	import inspect	1✔
36	import json # note that ujson is faster, but at this time does not support "default" in dumps()	1✔
37	import importlib	1✔
38	import datetime	1✔
39	import abc	1✔
40	import sys	1✔
41
42	import yaml	1✔
43	# Use CLoader for faster performance where possible.
44	try:	1✔
45	from yaml import CLoader as Loader, CSafeDumper as Dumper	1✔
46	except ImportError:	×
47	from yaml import Loader as Loader, SafeDumper as Dumper	×
48	import six	1✔
49
50	from monty.json import MontyDecoder, MSONable	1✔
51
52	from fireworks.fw_config import FW_NAME_UPDATES, YAML_STYLE, USER_PACKAGES, DECODE_MONTY, ENCODE_MONTY	1✔
53
54	__author__ = 'Anubhav Jain'	1✔
55	__copyright__ = 'Copyright 2012, The Materials Project'	1✔
56	__version__ = '0.1'	1✔
57	__maintainer__ = 'Anubhav Jain'	1✔
58	__email__ = 'ajain@lbl.gov'	1✔
59	__date__ = 'Dec 13, 2012'	1✔
60
61	# TODO: consider somehow switching FireWorks to monty serialization. e.g., numpy serialization is better handled.
62
63	SAVED_FW_MODULES = {}	1✔
64	DATETIME_HANDLER = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) else None	1✔
65
66	if sys.version_info > (3, 0, 0):	1✔
67	ENCODING_PARAMS = {"encoding": "utf-8"}	×
68	else:
69	ENCODING_PARAMS = {}	1✔
70
71	try:	1✔
72	import numpy as np	1✔
73	NUMPY_INSTALLED = True	1✔
74	except:	×
75	NUMPY_INSTALLED = False	×
76
77
78	def recursive_dict(obj, preserve_unicode=True):	1✔
79	if obj is None:	1✔
80	return None	1✔
81
82	if ENCODE_MONTY and hasattr(obj, 'as_dict'): # compatible with new monty JSONEncoder (MontyEncoder)	1✔
83	return recursive_dict(obj.as_dict(), preserve_unicode)	1✔
84
85	if hasattr(obj, 'to_dict'):	1✔
86	return recursive_dict(obj.to_dict(), preserve_unicode)	×
87
88	if isinstance(obj, dict):	1✔
89	return {recursive_dict(k, preserve_unicode): recursive_dict(v, preserve_unicode)	1✔
90	for k, v in obj.items()}
91
92	if isinstance(obj, (list, tuple)):	1✔
93	return [recursive_dict(v, preserve_unicode) for v in obj]	1✔
94
95	if isinstance(obj, int) or isinstance(obj, float):	1✔
96	return obj	1✔
97
98	if isinstance(obj, datetime.datetime):	1✔
99	return obj.isoformat()	1✔
100
101	if preserve_unicode and isinstance(obj, six.text_type) and obj != obj.encode('ascii', 'ignore'):	1✔
102	return obj	×
103
104	if NUMPY_INSTALLED and isinstance(obj, np.ndarray):	1✔
105	return [recursive_dict(v, preserve_unicode) for v in obj.tolist()]	1✔
106
107	return str(obj)	1✔
108
109
110	# TODO: is reconstitute_dates really needed? Can this method just do everything?
111	def _recursive_load(obj):	1✔
112	if obj is None:	1✔
113	return None	1✔
114
115	if hasattr(obj, '_fw_name') or isinstance(obj, MSONable):	1✔
116	return obj	1✔
117
118	if isinstance(obj, dict):	1✔
119	if '_fw_name' in obj:	1✔
120	return load_object(obj)	1✔
121
122	if DECODE_MONTY and '@module' in obj and '@class' in obj: # MontyDecoder compatibility	1✔
123	return json.loads(json.dumps(obj), cls=MontyDecoder)	×
124
125	return {k: _recursive_load(v) for k, v in obj.items()}	1✔
126
127	if isinstance(obj, (list, tuple)):	1✔
128	return [_recursive_load(v) for v in obj]	1✔
129
130	if isinstance(obj, six.string_types):	1✔
131	try:	1✔
132	# convert String to datetime if really datetime
133	return reconstitute_dates(obj)	1✔
134	except:	×
135	# convert unicode to ASCII if not really unicode
136	if obj == obj.encode('ascii', 'ignore'):	×
137	return str(obj)	×
138
139	return obj	1✔
140
141
142	def recursive_serialize(func):	1✔
143	"""
144	a decorator to add FW serializations keys
145	see documentation of FWSerializable for more details
146	"""
147	def _decorator(self, args, *kwargs):	1✔
148	m_dict = func(self, args, *kwargs)	1✔
149	m_dict = recursive_dict(m_dict)	1✔
150	return m_dict	1✔
151
152	return _decorator	1✔
153
154
155	def recursive_deserialize(func):	1✔
156	"""
157	a decorator to add FW serializations keys
158	see documentation of FWSerializable for more details
159	"""
160	def _decorator(self, args, *kwargs):	1✔
161	new_args = [a for a in args]	1✔
162	new_args[0] = {k: _recursive_load(v) for k, v in args[0].items()}	1✔
163	m_dict = func(self, new_args, *kwargs)	1✔
164	return m_dict	1✔
165
166	return _decorator	1✔
167
168
169	def serialize_fw(func):	1✔
170	"""
171	a decorator to add FW serializations keys
172	see documentation of FWSerializable for more details
173	"""
174	def _decorator(self, args, *kwargs):	1✔
175	m_dict = func(self, args, *kwargs)	1✔
176	m_dict['_fw_name'] = self.fw_name	1✔
177	return m_dict	1✔
178
179	return _decorator	1✔
180
181
182	@six.add_metaclass(abc.ABCMeta)	1✔
183	class FWSerializable(object):	1✔
184	"""
185	To create a serializable object within FireWorks, you should subclass this
186	class and implement the to_dict() and from_dict() methods.
187
188	If you want the load_object() implicit de-serialization to work, you must
189	also:
190	- Use the @serialize_fw decorator on to_dict()
191	- Override the _fw_name parameter with a unique key.
192
193	See documentation of load_object() for more details.
194
195	The use of @classmethod for from_dict allows you to define a superclass
196	that implements the to_dict() and from_dict() for all its subclasses.
197
198	For an example of serialization, see the class QueueAdapterBase.
199	"""
200
201	@property	1✔
202	def fw_name(self):
203	try:	1✔
204	return self._fw_name	1✔
205	except AttributeError:	×
206	return get_default_serialization(self.__class__)	×
207
208	@abc.abstractmethod	1✔
209	def to_dict(self):
210	raise NotImplementedError('FWSerializable object did not implement to_dict()!')	×
211
212	def to_db_dict(self):	1✔
213	return self.to_dict()	×
214
215	def as_dict(self):	1✔
216	# strictly for pseudo-compatibility with MSONable
217	# Note that FWSerializable is not MSONable, it uses _fw_name instead of __class__ and
218	# __module__
219	return self.to_dict()	1✔
220
221	@classmethod	1✔
222	@abc.abstractmethod	1✔
223	def from_dict(cls, m_dict):
224	raise NotImplementedError('FWSerializable object did not implement from_dict()!')	×
225
226	def __repr__(self):	1✔
227	return json.dumps(self.to_dict(), default=DATETIME_HANDLER)	×
228
229	def to_format(self, f_format='json', **kwargs):	1✔
230	"""
231	returns a String representation in the given format
232
233	Args:
234	f_format (str): the format to output to (default json)
235	"""
236	if f_format == 'json':	1✔
237	return json.dumps(self.to_dict(), default=DATETIME_HANDLER, **kwargs)	1✔
238	elif f_format == 'yaml':	1✔
239	# start with the JSON format, and convert to YAML
240	return yaml.dump(self.to_dict(), default_flow_style=YAML_STYLE,	1✔
241	allow_unicode=True, Dumper=Dumper)
242	else:
243	raise ValueError('Unsupported format {}'.format(f_format))	×
244
245	@classmethod	1✔
246	def from_format(cls, f_str, f_format='json'):	1✔
247	"""
248	convert from a String representation to its Object.
249
250	Args:
251	f_str (str): the String representation
252	f_format (str): serialization format of the String (default json)
253
254	Returns:
255	FWSerializable
256	"""
257	if f_format == 'json':	1✔
258	return cls.from_dict(reconstitute_dates(json.loads(f_str)))	1✔
259	elif f_format == 'yaml':	1✔
260	return cls.from_dict(reconstitute_dates(	1✔
261	yaml.load(f_str, Loader=Loader)))
262	else:
263	raise ValueError('Unsupported format {}'.format(f_format))	×
264
265	def to_file(self, filename, f_format=None, **kwargs):	1✔
266	"""
267	Write a serialization of this object to a file.
268
269	Args:
270	filename(str): filename to write to
271	f_format (str): serialization format, default checks the filename extension
272	"""
273	if f_format is None:	1✔
274	f_format = filename.split('.')[-1]	1✔
275	with open(filename, 'w', **ENCODING_PARAMS) as f:	1✔
276	f.write(self.to_format(f_format=f_format, **kwargs))	1✔
277
278	@classmethod	1✔
279	def from_file(cls, filename, f_format=None):	1✔
280	"""
281	Load a serialization of this object from a file.
282
283	Args:
284	filename (str): filename to read
285	f_format (str): serialization format, default checks the filename extension
286
287	Returns:
288	FWSerializable
289	"""
290	if f_format is None:	1✔
291	f_format = filename.split('.')[-1]	1✔
292	with open(filename, 'r', **ENCODING_PARAMS) as f:	1✔
293	return cls.from_format(f.read(), f_format=f_format)	1✔
294
295	def __getstate__(self):	1✔
296	return self.to_dict()	×
297
298	def __setstate__(self, state):	1✔
299	fw_obj = self.from_dict(state)	×
300	for k, v in fw_obj.__dict__.items():	×
301	self.__dict__[k] = v	×
302
303
304	# TODO: make this quicker the first time around
305	def load_object(obj_dict):	1✔
306	"""
307	Creates an instantiation of a class based on a dictionary representation. We implicitly
308	determine the Class through introspection along with information in the dictionary.
309
310	We search for a class with the _fw_name property equal to obj_dict['_fw_name']
311	If the @module key is set, that module is checked first for a matching class
312	to improve speed of lookup.
313	Afterwards, the modules in the USER_PACKAGES global parameter are checked.
314
315	Refactoring class names, module names, etc. will not break object loading
316	as long as:
317
318	i) the _fw_name property is maintained the same AND
319	ii) the refactored module is kept within USER_PACKAGES
320
321	You can get around these limitations if you really want:
322	i) If you want to change the fw_name of an object you can set the FW_NAME_UPDATES key
323	ii) If you want to put a refactored module in a new place add an entry to USER_PACKAGES
324
325	Args:
326	obj_dict (dict): the dict representation of the class
327	"""
328
329	# override the name in the obj_dict if there's an entry in FW_NAME_UPDATES
330	fw_name = FW_NAME_UPDATES.get(obj_dict['_fw_name'], obj_dict['_fw_name'])	1✔
331	obj_dict['_fw_name'] = fw_name	1✔
332
333	# check for explicit serialization, e.g. {{fireworks.tasks.MyTask}} - based on pymatgen method
334	if fw_name.startswith('{{') and fw_name.endswith('}}'):	1✔
335	modname, classname = fw_name.strip('{} ').rsplit(".", 1)	1✔
336	mod = __import__(modname, globals(), locals(), [classname], 0)	1✔
337	if hasattr(mod, classname):	1✔
338	cls_ = getattr(mod, classname)	1✔
339	return cls_.from_dict(obj_dict)	1✔
340
341	# first try to load from known location
342	if fw_name in SAVED_FW_MODULES:	1✔
343	m_module = importlib.import_module(SAVED_FW_MODULES[fw_name])	1✔
344	m_object = _search_module_for_obj(m_module, obj_dict)	1✔
345	if m_object is not None:	1✔
346	return m_object	1✔
347
348	# failing that, look for the object within all of USER_PACKAGES
349	# this will be slow, but only needed the first time
350
351	found_objects = [] # used to make sure we don't find multiple hits	1✔
352	for package in USER_PACKAGES:	1✔
353	root_module = importlib.import_module(package)	1✔
354	for loader, mod_name, is_pkg in pkgutil.walk_packages(	1✔
355	root_module.__path__, package + '.'):
356	try:	1✔
357	m_module = importlib.import_module(mod_name)	1✔
358	m_object = _search_module_for_obj(m_module, obj_dict)	1✔
359	if m_object is not None:	1✔
360	found_objects.append((m_object, mod_name))	1✔
361	except ImportError as ex:	×
362	import warnings	×
363	warnings.warn(	×
364	"%s in %s cannot be loaded because of %s. Skipping.."
365	% (m_object, mod_name, str(ex)))
366	traceback.print_exc(ex)	×
367
368	if len(found_objects) == 1:	1✔
369	SAVED_FW_MODULES[fw_name] = found_objects[0][1]	1✔
370	return found_objects[0][0]	1✔
371	elif len(found_objects) > 0:	×
372	raise ValueError(	×
373	'load_object() found multiple objects with cls._fw_name {} -- {}'
374	.format(fw_name, found_objects))
375
376	raise ValueError('load_object() could not find a class with cls._fw_name {}'.format(fw_name))	×
377
378
379	def load_object_from_file(filename, f_format=None):	1✔
380	"""
381	Implicitly load an object from a file. just a friendly wrapper to
382	load_object()
383
384	Args:
385	filename (str): the filename to load an object from
386	f_format (str): the serialization format (default is auto-detect based on
387	filename extension)
388	"""
389	m_dict = {}	1✔
390	if f_format is None:	1✔
391	f_format = filename.split('.')[-1]	1✔
392
393	with open(filename, 'r', **ENCODING_PARAMS) as f:	1✔
394	if f_format == 'json':	1✔
395	m_dict = reconstitute_dates(json.loads(f.read()))	×
396	elif f_format == 'yaml':	1✔
397	m_dict = reconstitute_dates(yaml.load(f, Loader=Loader))	1✔
398	else:
399	raise ValueError('Unknown file format {} cannot be loaded!'.format(f_format))	×
400
401	return load_object(m_dict)	1✔
402
403
404	def _search_module_for_obj(m_module, obj_dict):	1✔
405	"""
406	internal method that looks in a module for a class with a given _fw_name
407	"""
408	obj_name = obj_dict['_fw_name']	1✔
409
410	for name, obj in inspect.getmembers(m_module):	1✔
411	# check if the member is a Class matching our description
412	if inspect.isclass(obj) and obj.__module__ == m_module.__name__ and \	1✔
413	getattr(obj, '_fw_name', get_default_serialization(obj)) == obj_name:
414	return obj.from_dict(obj_dict)	1✔
415
416
417	def reconstitute_dates(obj_dict):	1✔
418	if obj_dict is None:	1✔
419	return None	1✔
420
421	if isinstance(obj_dict, dict):	1✔
422	return {k: reconstitute_dates(v) for k, v in obj_dict.items()}	1✔
423
424	if isinstance(obj_dict, (list, tuple)):	1✔
425	return [reconstitute_dates(v) for v in obj_dict]	1✔
426
427	if isinstance(obj_dict, six.string_types):	1✔
428	try:	1✔
429	return datetime.datetime.strptime(obj_dict, "%Y-%m-%dT%H:%M:%S.%f")	1✔
430	except:	1✔
431	try:	1✔
432	return datetime.datetime.strptime(obj_dict, "%Y-%m-%dT%H:%M:%S")	1✔
433	except:	1✔
434	pass	1✔
435	return obj_dict	1✔
436
437
438	def get_default_serialization(cls):	1✔
439	root_mod = cls.__module__.split('.')[0]	1✔
440	if root_mod == '__main__':	1✔
441	raise ValueError("Cannot get default serialization; try "	×
442	"instantiating your object from a different module "
443	"from which it is defined rather than defining your "
444	"object in the __main__ (running) module.")
445	return root_mod + '::' + cls.__name__ # e.g. fireworks.ABC	1✔

materialsproject / fireworks / 2309

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous