2216974467

Build Type

github

Committed by GitHub

Commit Message

Merge 1c7dd3504 into 45b93a41d

Pull Request Pull Request #99: Orm separation

Run Details

1193 of 1827 branches covered (65.3%)

Branch coverage included in aggregate %.

236 of 348 new or added lines in 28 files covered. (67.82%)

4 existing lines in 3 files now uncovered.

2751 of 4629 relevant lines covered (59.43%)

3.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.8

/simpleml/persistables/base_persistable.py

"""
Base class for all database tracked records, called "Persistables"
"""

__author__ = "Elisha Yadgaran"


import logging
import uuid
from abc import abstractmethod
from collections import defaultdict
from typing import Any, Dict, List, Optional, Type, Union

from simpleml.persistables.hashing import CustomHasherMixin
from simpleml.registries import (
    LOAD_METHOD_REGISTRY,
    ORM_REGISTRY,
    SAVE_METHOD_REGISTRY,
    SIMPLEML_REGISTRY,
    PersistableRegistry,
)
from simpleml.utils.errors import SimpleMLError
from simpleml.utils.library_versions import INSTALLED_LIBRARIES

LOGGER = logging.getLogger(__name__)


class Persistable(CustomHasherMixin, metaclass=PersistableRegistry):
    """
    Base class for all SimpleML persistable objects.

    Uses private class attributes for internal artifact registry
    Does not need to be persisted because it gets populated on import
    (and can therefore be changed between versions)
    cls._ARTIFACT_{artifact_name} = {'save': save_attribute, 'restore': restore_attribute}

    -------
    Schema
    -------
    id: Random UUID(4). Used over auto incrementing id to minimize collision probability
        with distributed trainings and authors (especially if using central server
        to combine results across different instantiations of SimpleML)

    hash_id: Use hash of object to uniquely identify the contents at train time
    registered_name: class name of object defined when importing
        Can be used for the drag and drop GUI - also for prescribing training config
    author: creator
    project: Project objects are associated with. Useful if multiple persistables
        relate to the same project and want to be grouped (but have different names)
        also good for implementing row based security across teams
    name: friendly name - primary way of tracking evolution of "same" object over time
    version: autoincrementing id of "friendly name"
    version_description: description that explains what is new or different about this version

    # Persistence of fitted states
    has_external_files = boolean field to signify presence of saved files not in (main) db
    filepaths = JSON object with external file details
        The nested notation is because any persistable can implement multiple save
        options (with arbitrary priority) and arbitrary inputs. Simple serialization
        could have only a single string location whereas complex artifacts might have
        a list or map of filepaths

        Structure:
        {
            artifact_name: {
                'save_pattern': filepath_data
            },
            "example": {
                "disk_pickled": path to file, relative to base simpleml folder (default ~/.simpleml),
                "database": {"schema": schema, "table": table_name}, # (for files extractable with `select * from`)
                ...
            }
        }


    metadata: Generic JSON store for random attributes
    """

    object_type = "PERSISTABLE"

    def __init__(
        self,
        id: uuid.UUID = None,
        hash_: str = None,
        name: Optional[str] = "default",
        has_external_files: bool = False,
        author: Optional[str] = "default",
        project: Optional[str] = "default",
        version: Optional[int] = None,
        version_description: Optional[str] = "",
        save_patterns: Optional[Dict[str, List[str]]] = None,
        filepaths: Optional[Dict] = None,
        metadata_: Optional[Dict] = None,
        **kwargs,
    ):
        # Initialize values expected to exist at time of instantiation
        self.registered_name: str = self.__class__.__name__
        self.id: uuid.UUID = id or uuid.uuid4()
        self.author = author
        self.project = project
        self.name = name
        self.has_external_files = has_external_files
        self.version_description = version_description

        # expected null values
        self.hash_ = hash_
        self.version = version
        self.filepaths = filepaths or {}

        # Special place for SimpleML internal params
        # Think of as the config to initialize objects
        self.metadata_: Dict[str, Any] = (
            metadata_ or {}
        )  # Place for any arbitrary metadata

        if "config" not in self.metadata_:
            self.metadata_[
                "config"
            ] = (
                {}
            )  # Place for parameters that uniquely configure an instance on initialization
        if "state" not in self.metadata_:
            self.metadata_[
                "state"
            ] = (
                {}
            )  # Place for transitory values that may be set post initialization (and want to be persisted)

        save_patterns = save_patterns or self.state.get("save_patterns")
        if has_external_files and save_patterns is None:
            raise SimpleMLError(
                "Persistable has external artifacts, but has not specified any save patterns.\nTry reinitializing persistable with `Persistable(save_patterns={artifact_name: [save_patterns]})`"
            )

        # Store save pattern in state metadata as an operational setting, otherwise
        # it could affect the hash and result in a different object per save location
        self.state["save_patterns"] = save_patterns

    def __post_init__(self):
        self._configure_unmapped_attributes()

    def __post_restore__(self):
        self._configure_unmapped_attributes()

    def _configure_unmapped_attributes(self):
        """
        Unified entry for unmapped attributes. need to be restored when loading
        classes
        """
        self.unloaded_artifacts: List[str]
        # Track the list of artifacts
        # New persistables without a specified filepath dictionary have type
        # sqlalchemy.sql.schema.Column - calling list(Column.keys()) would fail
        if not isinstance(self.filepaths, dict):
            LOGGER.warning("Load appears to being called on an unsaved Persistable")
            self.unloaded_artifacts = []
        else:
            self.unloaded_artifacts = list(self.filepaths.keys())

    """
    metadata accessor properties
    """

    @property
    def config(self) -> Dict[str, Any]:
        return self.metadata_["config"]

    @property
    def state(self) -> Dict[str, Any]:
        return self.metadata_["state"]

    @property
    def library_versions(self) -> Dict[str, str]:
        return self.metadata_.get("library_versions", {})

    """
    abstract definitions
    """

    @abstractmethod
    def _hash(self):
        """
        Each subclass should implement a hashing routine to uniquely AND consistently
        identify the object contents. Consistency is important to ensure ability
        to assert identity across code definitions
        """

    """
    persistence hooks
    """

    def _get_latest_version(self) -> int:
        """
        Versions should be autoincrementing for each object (constrained over
        friendly name). Executes a database lookup and increments..
        """
        return self.orm_cls.get_latest_version(name=self.name)

    def save(self) -> None:
        """
        Each subclass needs to instantiate a save routine to persist to the
        database and any other required filestore

        sqlalchemy_mixins supports active record style TableModel.save()
        so can still call super(Persistable, self).save()
        """
        if self.has_external_files:  # todo: insert hook to not save on updates
            self.save_external_files()

        # Hash contents upon save
        if self.hash_ is None:
            self.hash_ = self._hash()

        # Get the latest version for this "friendly name"
        if self.version is None:
            self.version = self._get_latest_version()

        # Store library versions in case of future loads into unsupported environments
        if "library_versions" not in self.metadata_:
            self.metadata_["library_versions"] = INSTALLED_LIBRARIES

        self.orm_cls.save_record(**self.to_dict())

    @property
    def orm_cls(self):
        cls = ORM_REGISTRY.get(self.object_type)
        if cls is None:
            raise SimpleMLError(f"No registered ORM class for {self.object_type}")
        return cls

    def save_external_files(self) -> None:
        """
        Main routine to save registered external artifacts. Each save pattern
        is defined using the standard api for the save params defined here. If
        a pattern requires more imports, it needs to be added here

        Uses a standardized nomenclature to reuse params regardless of save pattern
        {
            'persistable_id': the database id of the persistable. typically used as the root name of the saved object. implementations will pre/suffix,
            'persistable_type': the persistable type (DATASET/PIPELINE..),
            'overwrite': boolean. shortcut in case save pattern redefines a serialization routine
        }
        """
        save_params: Dict[str, Union[str, bool]]
        save_params = {
            "persistable_id": str(self.id),
            "persistable_type": self.object_type,
            "overwrite": False,
        }
        # Iterate through each artifact and save
        for artifact_name, save_patterns in self.state.get("save_patterns", {}).items():
            # Artifact has to be registered in self.ARTIFACTS
            obj = self.get_artifact(artifact_name)
            # Iterate through list of save methods
            for save_pattern in save_patterns:
                self.save_external_file(
                    artifact_name=artifact_name,
                    filepath=f"{save_params['persistable_type']}-{save_params['persistable_id']}-{artifact_name}",
                    save_pattern=save_pattern,
                    obj=obj,
                    **save_params,
                )

    def save_external_file(
        self,
        artifact_name: str,
        save_pattern: str,
        cls: Optional[Type] = None,
        **save_params,
    ) -> None:
        """
        Abstracted pattern to save an artifact via one of the registered
        patterns and update the filepaths location
        """
        if cls is None:
            # Look up in registry
            save_cls = SAVE_METHOD_REGISTRY.get(save_pattern)
        else:
            LOGGER.info("Custom save class passed, skipping registry lookup")
            save_cls = cls

        if save_cls is None:
            raise SimpleMLError(f"No registered save pattern for {save_pattern}")

        filepath_data = save_cls.save(artifact_name=artifact_name, **save_params)

        # Update filepaths
        if self.filepaths is None:
            self.filepaths = {}
        if self.filepaths.get(artifact_name, None) is None:
            self.filepaths[artifact_name] = {}
        self.filepaths[artifact_name][save_pattern] = filepath_data

    def get_artifact(self, artifact_name: str) -> Any:
        """
        Accessor method to lookup the artifact in the registry and return
        the corresponding data value
        """
        registered_attribute = f"_ARTIFACT_{artifact_name}"
        if not hasattr(self, registered_attribute):
            raise SimpleMLError(
                "Cannot retrieve artifacts before registering. Make sure to decorate class with @ExternalArtifactDecorators.register_artifact"
            )
        save_attribute = getattr(self, registered_attribute)["save"]
        return getattr(self, save_attribute)

    @classmethod
    def from_dict(cls, **kwargs) -> "Persistable":
        """
        Parameterize a persistable from a dict. Used in deserialization from ORM
        objects
        """
        # skip init calls
        obj = cls.__new__(cls)
        for attr, value in kwargs.items():
            setattr(obj, attr, value)
        # execute __post_restore__ hook
        obj.__post_restore__()
        return obj

    def to_dict(self):
        return vars(self)

    def load_external_files(self, artifact_name: Optional[str] = None) -> None:
        """
        Main routine to restore registered external artifacts. Will iterate
        through save patterns and break after the first successful restore
        (allows robustness in the event of unavailable resources)
        """

        def _load(artifact_name: str, save_patterns: Dict[str, Any]):
            # Iterate through dict of save methods and file data
            for save_pattern in save_patterns:
                try:
                    obj = self.load_external_file(artifact_name, save_pattern)
                    self.restore_artifact(artifact_name, obj)
                    break
                except Exception as e:
                    LOGGER.error(
                        f"Failed to restore {artifact_name} via {save_pattern} ({e}). Trying next save pattern..."
                    )
            else:
                raise SimpleMLError(
                    f"Unable to restore {artifact_name} via any registered pattern"
                )

        # Iterate through each artifact and restore
        # Dont use self.unloaded_artifacts list to force a full reload
        if artifact_name is None:
            for artifact_name, save_patterns in self.filepaths.items():
                _load(artifact_name, save_patterns)
        else:
            _load(artifact_name, self.filepaths.get(artifact_name, {}))

    def load_external_file(
        self, artifact_name: str, save_pattern: str, cls: Optional[Type] = None
    ) -> Any:
        """
        Define pattern for loading external files
        returns the object for assignment
        Inverted operation from saving. Registered functions should take in
        the same data (in the same form) of what is saved in the filepath
        """
        if cls is None:
            # Look up in registry
            load_cls = LOAD_METHOD_REGISTRY.get(save_pattern)
        else:
            LOGGER.info("Custom load class passed, skipping registry lookup")
            load_cls = cls

        if load_cls is None:
            raise SimpleMLError(f"No registered load class for {save_pattern}")

        # Do some validation in case attempting to load unsaved artifact
        artifact = self.filepaths.get(artifact_name, None)
        if artifact is None:
            raise SimpleMLError(f"No artifact saved for {artifact_name}")
        if save_pattern not in artifact:
            raise SimpleMLError(
                f"No artifact saved using save pattern {save_pattern} for {artifact_name}"
            )

        filepath_data = artifact[save_pattern]
        if not isinstance(filepath_data, dict):
            # legacy wrap for old filepath formats
            filepath_data = {"legacy": filepath_data}
        return load_cls.load(**filepath_data)

    def restore_artifact(self, artifact_name: str, obj: Any) -> None:
        """
        Setter method to lookup the restore attribute and set to the passed object
        """
        registered_attribute = f"_ARTIFACT_{artifact_name}"
        if not hasattr(self, registered_attribute):
            raise SimpleMLError(
                "Cannot restore artifacts before registering. Make sure to decorate class with @ExternalArtifactDecorators.register_artifact"
            )
        restore_attribute = getattr(self, registered_attribute)["restore"]
        setattr(self, restore_attribute, obj)

        # Make note that the artifact was loaded
        if hasattr(self, "unloaded_artifacts"):
            try:
                self.unloaded_artifacts.remove(artifact_name)
            except ValueError:
                pass

    def load_if_unloaded(self, artifact_name: str) -> None:
        """
        Convenience method to load an artifact if not already loaded.
        Easy dropin in property methods
        ```
        @property
        def artifact(self):
            self.load_if_unloaded(artifact_name)
            if not hasattr(self, artifact_attribute):
                self.create_artifact()
            return self.artifact_attribute
        ```
        """
        if artifact_name in self.unloaded_artifacts:
            self.load_external_files(artifact_name=artifact_name)

1	"""
2	Base class for all database tracked records, called "Persistables"
3	"""
4
5	__author__ = "Elisha Yadgaran"
6
7
8	import logging
9	import uuid
10	from abc import abstractmethod
11	from collections import defaultdict
12	from typing import Any, Dict, List, Optional, Type, Union
13
14	from simpleml.persistables.hashing import CustomHasherMixin
15	from simpleml.registries import (
16	LOAD_METHOD_REGISTRY,
17	ORM_REGISTRY,
18	SAVE_METHOD_REGISTRY,
19	SIMPLEML_REGISTRY,
20	PersistableRegistry,
21	)
22	from simpleml.utils.errors import SimpleMLError
23	from simpleml.utils.library_versions import INSTALLED_LIBRARIES
24
25	LOGGER = logging.getLogger(__name__)	6✔
26
27
28	class Persistable(CustomHasherMixin, metaclass=PersistableRegistry):	6✔
29	"""
30	Base class for all SimpleML persistable objects.
31
32	Uses private class attributes for internal artifact registry
33	Does not need to be persisted because it gets populated on import
34	(and can therefore be changed between versions)
35	cls._ARTIFACT_{artifact_name} = {'save': save_attribute, 'restore': restore_attribute}
36
37	-------
38	Schema
39	-------
40	id: Random UUID(4). Used over auto incrementing id to minimize collision probability
41	with distributed trainings and authors (especially if using central server
42	to combine results across different instantiations of SimpleML)
43
44	hash_id: Use hash of object to uniquely identify the contents at train time
45	registered_name: class name of object defined when importing
46	Can be used for the drag and drop GUI - also for prescribing training config
47	author: creator
48	project: Project objects are associated with. Useful if multiple persistables
49	relate to the same project and want to be grouped (but have different names)
50	also good for implementing row based security across teams
51	name: friendly name - primary way of tracking evolution of "same" object over time
52	version: autoincrementing id of "friendly name"
53	version_description: description that explains what is new or different about this version
54
55	# Persistence of fitted states
56	has_external_files = boolean field to signify presence of saved files not in (main) db
57	filepaths = JSON object with external file details
58	The nested notation is because any persistable can implement multiple save
59	options (with arbitrary priority) and arbitrary inputs. Simple serialization
60	could have only a single string location whereas complex artifacts might have
61	a list or map of filepaths
62
63	Structure:
64	{
65	artifact_name: {
66	'save_pattern': filepath_data
67	},
68	"example": {
69	"disk_pickled": path to file, relative to base simpleml folder (default ~/.simpleml),
70	"database": {"schema": schema, "table": table_name}, # (for files extractable with `select * from`)
71	...
72	}
73	}
74
75
76	metadata: Generic JSON store for random attributes
77	"""
78
79	object_type = "PERSISTABLE"	6✔
80
81	def __init__(	6✔
82	self,
83	id: uuid.UUID = None,
84	hash_: str = None,
85	name: Optional[str] = "default",
86	has_external_files: bool = False,
87	author: Optional[str] = "default",
88	project: Optional[str] = "default",
89	version: Optional[int] = None,
90	version_description: Optional[str] = "",
91	save_patterns: Optional[Dict[str, List[str]]] = None,
92	filepaths: Optional[Dict] = None,
93	metadata_: Optional[Dict] = None,
94	**kwargs,
95	):
96	# Initialize values expected to exist at time of instantiation
97	self.registered_name: str = self.__class__.__name__	9✔
98	self.id: uuid.UUID = id or uuid.uuid4()	9✔
99	self.author = author	9✔
100	self.project = project	9✔
101	self.name = name	9✔
102	self.has_external_files = has_external_files	9✔
103	self.version_description = version_description	9✔
104
105	# expected null values
106	self.hash_ = hash_	9✔
107	self.version = version	9✔
108	self.filepaths = filepaths or {}	9✔
109
110	# Special place for SimpleML internal params
111	# Think of as the config to initialize objects
112	self.metadata_: Dict[str, Any] = (	9✔
113	metadata_ or {}
114	) # Place for any arbitrary metadata
115
116	if "config" not in self.metadata_:	9!
117	self.metadata_[	9✔
118	"config"
119	] = (
120	{}
121	) # Place for parameters that uniquely configure an instance on initialization
122	if "state" not in self.metadata_:	9!
123	self.metadata_[	9✔
124	"state"
125	] = (
126	{}
127	) # Place for transitory values that may be set post initialization (and want to be persisted)
128
129	save_patterns = save_patterns or self.state.get("save_patterns")	9✔
130	if has_external_files and save_patterns is None:	9!
131	raise SimpleMLError(	×
132	"Persistable has external artifacts, but has not specified any save patterns.\nTry reinitializing persistable with `Persistable(save_patterns={artifact_name: [save_patterns]})`"
133	)
134
135	# Store save pattern in state metadata as an operational setting, otherwise
136	# it could affect the hash and result in a different object per save location
137	self.state["save_patterns"] = save_patterns	9✔
138
139	def __post_init__(self):	6✔
140	self._configure_unmapped_attributes()	9✔
141
142	def __post_restore__(self):	6✔
143	self._configure_unmapped_attributes()	3✔
144
145	def _configure_unmapped_attributes(self):	6✔
146	"""
147	Unified entry for unmapped attributes. need to be restored when loading
148	classes
149	"""
150	self.unloaded_artifacts: List[str]	9✔
151	# Track the list of artifacts
152	# New persistables without a specified filepath dictionary have type
153	# sqlalchemy.sql.schema.Column - calling list(Column.keys()) would fail
154	if not isinstance(self.filepaths, dict):	9!
NEW 155	LOGGER.warning("Load appears to being called on an unsaved Persistable")	×
NEW 156	self.unloaded_artifacts = []	×
157	else:
158	self.unloaded_artifacts = list(self.filepaths.keys())	9✔
159
160	"""
161	metadata accessor properties
162	"""
163
164	@property	6✔
165	def config(self) -> Dict[str, Any]:	6✔
166	return self.metadata_["config"]	9✔
167
168	@property	6✔
169	def state(self) -> Dict[str, Any]:	6✔
170	return self.metadata_["state"]	9✔
171
172	@property	6✔
173	def library_versions(self) -> Dict[str, str]:	6✔
174	return self.metadata_.get("library_versions", {})	3✔
175
176	"""
177	abstract definitions
178	"""
179
180	@abstractmethod	6✔
181	def _hash(self):	4✔
182	"""
183	Each subclass should implement a hashing routine to uniquely AND consistently
184	identify the object contents. Consistency is important to ensure ability
185	to assert identity across code definitions
186	"""
187
188	"""
189	persistence hooks
190	"""
191
192	def _get_latest_version(self) -> int:	6✔
193	"""
194	Versions should be autoincrementing for each object (constrained over
195	friendly name). Executes a database lookup and increments..
196	"""
197	return self.orm_cls.get_latest_version(name=self.name)	3✔
198
199	def save(self) -> None:	6✔
200	"""
201	Each subclass needs to instantiate a save routine to persist to the
202	database and any other required filestore
203
204	sqlalchemy_mixins supports active record style TableModel.save()
205	so can still call super(Persistable, self).save()
206	"""
207	if self.has_external_files: # todo: insert hook to not save on updates	3!
208	self.save_external_files()	3✔
209
210	# Hash contents upon save
211	if self.hash_ is None:	3!
212	self.hash_ = self._hash()	3✔
213
214	# Get the latest version for this "friendly name"
215	if self.version is None:	3!
216	self.version = self._get_latest_version()	3✔
217
218	# Store library versions in case of future loads into unsupported environments
219	if "library_versions" not in self.metadata_:	3!
220	self.metadata_["library_versions"] = INSTALLED_LIBRARIES	3✔
221
222	self.orm_cls.save_record(**self.to_dict())	3✔
223
224	@property	6✔
225	def orm_cls(self):	4✔
226	cls = ORM_REGISTRY.get(self.object_type)	3✔
227	if cls is None:	3!
NEW 228	raise SimpleMLError(f"No registered ORM class for {self.object_type}")	×
229	return cls	3✔
230
231	def save_external_files(self) -> None:	6✔
232	"""
233	Main routine to save registered external artifacts. Each save pattern
234	is defined using the standard api for the save params defined here. If
235	a pattern requires more imports, it needs to be added here
236
237	Uses a standardized nomenclature to reuse params regardless of save pattern
238	{
239	'persistable_id': the database id of the persistable. typically used as the root name of the saved object. implementations will pre/suffix,
240	'persistable_type': the persistable type (DATASET/PIPELINE..),
241	'overwrite': boolean. shortcut in case save pattern redefines a serialization routine
242	}
243	"""
244	save_params: Dict[str, Union[str, bool]]
245	save_params = {	6✔
246	"persistable_id": str(self.id),
247	"persistable_type": self.object_type,
248	"overwrite": False,
249	}
250	# Iterate through each artifact and save
251	for artifact_name, save_patterns in self.state.get("save_patterns", {}).items():	6✔
252	# Artifact has to be registered in self.ARTIFACTS
253	obj = self.get_artifact(artifact_name)	6✔
254	# Iterate through list of save methods
255	for save_pattern in save_patterns:	6✔
256	self.save_external_file(	6✔
257	artifact_name=artifact_name,
258	filepath=f"{save_params['persistable_type']}-{save_params['persistable_id']}-{artifact_name}",
259	save_pattern=save_pattern,
260	obj=obj,
261	**save_params,
262	)
263
264	def save_external_file(	6✔
265	self,
266	artifact_name: str,
267	save_pattern: str,
268	cls: Optional[Type] = None,
269	**save_params,
270	) -> None:
271	"""
272	Abstracted pattern to save an artifact via one of the registered
273	patterns and update the filepaths location
274	"""
275	if cls is None:	6!
276	# Look up in registry
277	save_cls = SAVE_METHOD_REGISTRY.get(save_pattern)	6✔
278	else:
279	LOGGER.info("Custom save class passed, skipping registry lookup")	×
280	save_cls = cls	×
281
282	if save_cls is None:	6!
283	raise SimpleMLError(f"No registered save pattern for {save_pattern}")	×
284
285	filepath_data = save_cls.save(artifact_name=artifact_name, **save_params)	6✔
286
287	# Update filepaths
288	if self.filepaths is None:	6!
UNCOV 289	self.filepaths = {}	×
290	if self.filepaths.get(artifact_name, None) is None:	6!
291	self.filepaths[artifact_name] = {}	6✔
292	self.filepaths[artifact_name][save_pattern] = filepath_data	6✔
293
294	def get_artifact(self, artifact_name: str) -> Any:	6✔
295	"""
296	Accessor method to lookup the artifact in the registry and return
297	the corresponding data value
298	"""
299	registered_attribute = f"_ARTIFACT_{artifact_name}"	6✔
300	if not hasattr(self, registered_attribute):	6!
301	raise SimpleMLError(	×
302	"Cannot retrieve artifacts before registering. Make sure to decorate class with @ExternalArtifactDecorators.register_artifact"
303	)
304	save_attribute = getattr(self, registered_attribute)["save"]	6✔
305	return getattr(self, save_attribute)	6✔
306
307	@classmethod	6✔
308	def from_dict(cls, **kwargs) -> "Persistable":	6✔
309	"""
310	Parameterize a persistable from a dict. Used in deserialization from ORM
311	objects
312	"""
313	# skip init calls
314	obj = cls.__new__(cls)	3✔
315	for attr, value in kwargs.items():	3✔
316	setattr(obj, attr, value)	3✔
317	# execute __post_restore__ hook
318	obj.__post_restore__()	3✔
319	return obj	3✔
320
321	def to_dict(self):	6✔
322	return vars(self)	3✔
323
324	def load_external_files(self, artifact_name: Optional[str] = None) -> None:	6✔
325	"""
326	Main routine to restore registered external artifacts. Will iterate
327	through save patterns and break after the first successful restore
328	(allows robustness in the event of unavailable resources)
329	"""
330
331	def _load(artifact_name: str, save_patterns: Dict[str, Any]):	6✔
332	# Iterate through dict of save methods and file data
333	for save_pattern in save_patterns:	6!
334	try:	6✔
335	obj = self.load_external_file(artifact_name, save_pattern)	6✔
336	self.restore_artifact(artifact_name, obj)	6✔
337	break	6✔
338	except Exception as e:	×
339	LOGGER.error(	×
340	f"Failed to restore {artifact_name} via {save_pattern} ({e}). Trying next save pattern..."
341	)
342	else:
343	raise SimpleMLError(	×
344	f"Unable to restore {artifact_name} via any registered pattern"
345	)
346
347	# Iterate through each artifact and restore
348	# Dont use self.unloaded_artifacts list to force a full reload
349	if artifact_name is None:	6!
350	for artifact_name, save_patterns in self.filepaths.items():	6✔
351	_load(artifact_name, save_patterns)	6✔
352	else:
353	_load(artifact_name, self.filepaths.get(artifact_name, {}))	×
354
355	def load_external_file(	6✔
356	self, artifact_name: str, save_pattern: str, cls: Optional[Type] = None
357	) -> Any:
358	"""
359	Define pattern for loading external files
360	returns the object for assignment
361	Inverted operation from saving. Registered functions should take in
362	the same data (in the same form) of what is saved in the filepath
363	"""
364	if cls is None:	6!
365	# Look up in registry
366	load_cls = LOAD_METHOD_REGISTRY.get(save_pattern)	6✔
367	else:
368	LOGGER.info("Custom load class passed, skipping registry lookup")	×
369	load_cls = cls	×
370
371	if load_cls is None:	6!
372	raise SimpleMLError(f"No registered load class for {save_pattern}")	×
373
374	# Do some validation in case attempting to load unsaved artifact
375	artifact = self.filepaths.get(artifact_name, None)	6✔
376	if artifact is None:	6!
377	raise SimpleMLError(f"No artifact saved for {artifact_name}")	×
378	if save_pattern not in artifact:	6!
379	raise SimpleMLError(	×
380	f"No artifact saved using save pattern {save_pattern} for {artifact_name}"
381	)
382
383	filepath_data = artifact[save_pattern]	6✔
384	if not isinstance(filepath_data, dict):	6!
385	# legacy wrap for old filepath formats
386	filepath_data = {"legacy": filepath_data}	×
387	return load_cls.load(**filepath_data)	6✔
388
389	def restore_artifact(self, artifact_name: str, obj: Any) -> None:	6✔
390	"""
391	Setter method to lookup the restore attribute and set to the passed object
392	"""
393	registered_attribute = f"_ARTIFACT_{artifact_name}"	6✔
394	if not hasattr(self, registered_attribute):	6!
395	raise SimpleMLError(	×
396	"Cannot restore artifacts before registering. Make sure to decorate class with @ExternalArtifactDecorators.register_artifact"
397	)
398	restore_attribute = getattr(self, registered_attribute)["restore"]	6✔
399	setattr(self, restore_attribute, obj)	6✔
400
401	# Make note that the artifact was loaded
402	if hasattr(self, "unloaded_artifacts"):	6!
403	try:	6✔
404	self.unloaded_artifacts.remove(artifact_name)	6✔
405	except ValueError:	3✔
406	pass	3✔
407
408	def load_if_unloaded(self, artifact_name: str) -> None:	6✔
409	"""
410	Convenience method to load an artifact if not already loaded.
411	Easy dropin in property methods
412	```
413	@property
414	def artifact(self):
415	self.load_if_unloaded(artifact_name)
416	if not hasattr(self, artifact_attribute):
417	self.create_artifact()
418	return self.artifact_attribute
419	```
420	"""
421	if artifact_name in self.unloaded_artifacts:	9!
422	self.load_external_files(artifact_name=artifact_name)	×

eyadgaran / SimpleML / 2216974467

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous