2026984255

Build # 2026984255

Build Type

github

Committed by GitHub

Commit Message

Merge 492508f48 into 6122a52f0

Pull Request Pull Request #99: Orm separation

Run Details

1173 of 1810 branches covered (64.81%)

Branch coverage included in aggregate %.

242 of 356 new or added lines in 29 files covered. (67.98%)

16 existing lines in 7 files now uncovered.

2743 of 4614 relevant lines covered (59.45%)

3.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.79

/simpleml/metrics/base_metric.py

__author__ = 'Elisha Yadgaran'


import logging
import uuid
import weakref
from typing import Any, Optional, Union

from simpleml.datasets.base_dataset import Dataset
from simpleml.models.base_model import Model
from simpleml.persistables.base_persistable import Persistable
from simpleml.registries import MetricRegistry
from simpleml.utils.errors import MetricError

LOGGER = logging.getLogger(__name__)


class Metric(Persistable, metaclass=MetricRegistry):
    '''
    Base class for all Metric objects
    '''
    object_type: str = 'METRIC'

    def __init__(self,
                 dataset_id: Optional[Union[str, uuid.uuid4]] = None,
                 model_id: Optional[Union[str, uuid.uuid4]] = None,
                 **kwargs):
        super().__init__(**kwargs)

        # initialize null references
        self.dataset_id = dataset_id
        self.model_id = model_id

    def add_dataset(self, dataset: Dataset) -> None:
        '''
        Setter method for dataset used
        '''
        if dataset is None:
            return
        self.dataset_id = dataset.id
        self.dataset = dataset

    @property
    def dataset(self):
        '''
        Use a weakref to bind linked dataset so it doesnt bloat usage
        returns dataset if still available or tries to fetch otherwise
        '''
        # still referenced weakref
        if hasattr(self, '_dataset') and self._dataset() is not None:
            return self._dataset()

        # null return if no associated dataset (governed by dataset_id)
        elif self.dataset_id is None:
            return None

        # else regenerate weakref
        LOGGER.info('No referenced object available. Refreshing weakref')
        dataset = self._load_dataset()
        self._dataset = weakref.ref(dataset)
        return dataset

    @dataset.setter
    def dataset(self, dataset: Dataset) -> None:
        '''
        Need to be careful not to set as the orm object
        Cannot load if wrong type because of recursive behavior (will
        propagate down the whole dependency chain)
        '''
        self._dataset = weakref.ref(dataset)

    def _load_dataset(self):
        '''
        Helper to fetch the dataset
        '''
        return self.orm_cls.load_dataset(self.dataset_id)

    def add_model(self, model: Model) -> None:
        '''
        Setter method for model used
        '''
        if model is None:
            return
        self.model_id = model.id
        self.model = model

    @property
    def model(self):
        '''
        Use a weakref to bind linked model so it doesnt bloat usage
        returns model if still available or tries to fetch otherwise
        '''
        # still referenced weakref
        if hasattr(self, '_model') and self._model() is not None:
            return self._model()

        # null return if no associated model (governed by model_id)
        elif self.model_id is None:
            return None

        # else regenerate weakref
        LOGGER.info('No referenced object available. Refreshing weakref')
        model = self._load_model()
        self._model = weakref.ref(model)
        return model

    @model.setter
    def model(self, model: Model) -> None:
        '''
        Need to be careful not to set as the orm object
        Cannot load if wrong type because of recursive behavior (will
        propagate down the whole dependency chain)
        '''
        self._model = weakref.ref(model)

    def _load_model(self):
        '''
        Helper to fetch the model
        '''
        return self.orm_cls.load_model(self.model_id)

    def _hash(self) -> str:
        '''
        Hash is the combination of the:
            1) Model
            2) Dataset (optional)
            3) Metric
            4) Config
        '''
        model_hash = self.model.hash_ or self.model._hash()
        if self.dataset is not None:
            dataset_hash = self.dataset.hash_ or self.dataset._hash()
        else:
            dataset_hash = None
        metric = self.__class__.__name__
        config = self.config

        return self.custom_hasher((model_hash, dataset_hash, metric, config))

    def _get_latest_version(self) -> int:
        '''
        Versions should be autoincrementing for each object (constrained over
        friendly name and model). Executes a database lookup and increments..
        '''
        return self.orm_cls.get_latest_version(name=self.name, model_id=self.model.id)

    def _get_pipeline_split(self, column: str, split: str, **kwargs) -> Any:
        '''
        For special case where dataset is the same as the model's dataset, the
        dataset splits can refer to the pipeline imposed splits, not the inherent
        dataset's splits. Use the pipeline split then
        ex: RandomSplitPipeline on NoSplitDataset evaluating "in_sample" performance
        '''
        return getattr(self.model.pipeline.get_dataset_split(split=split, **kwargs), column)

    def _get_dataset_split(self, **kwargs) -> Any:
        '''
        Default accessor for dataset data. REFERS TO RAW DATASETS
        not the pipelines superimposed. That means that datasets that do not
        define explicit splits will have no notion of downstream splits
        (e.g. RandomSplitPipeline)
        '''
        return self.dataset.get(**kwargs)

    def save(self, **kwargs) -> None:
        '''
        Extend parent function with a few additional save routines
        '''
        if self.model is None:
            raise MetricError('Must set model before saving')

        if self.values is None:
            raise MetricError('Must score metric before saving')

        super().save(**kwargs)

    def score(self, **kwargs):
        '''
        Abstract method for each metric to define

        Should set self.values
        '''
        raise NotImplementedError

1	__author__ = 'Elisha Yadgaran'
2
3
4	import logging
5	import uuid
6	import weakref
7	from typing import Any, Optional, Union
8
9	from simpleml.datasets.base_dataset import Dataset
10	from simpleml.models.base_model import Model
11	from simpleml.persistables.base_persistable import Persistable
12	from simpleml.registries import MetricRegistry
13	from simpleml.utils.errors import MetricError
14
15	LOGGER = logging.getLogger(__name__)	6✔
16
17
18	class Metric(Persistable, metaclass=MetricRegistry):	6✔
19	'''
20	Base class for all Metric objects
21	'''
22	object_type: str = 'METRIC'	6✔
23
24	def __init__(self,	6✔
25	dataset_id: Optional[Union[str, uuid.uuid4]] = None,
26	model_id: Optional[Union[str, uuid.uuid4]] = None,
27	**kwargs):
28	super().__init__(**kwargs)	3✔
29
30	# initialize null references
31	self.dataset_id = dataset_id	3✔
32	self.model_id = model_id	3✔
33
34	def add_dataset(self, dataset: Dataset) -> None:	6✔
35	'''
36	Setter method for dataset used
37	'''
38	if dataset is None:	3!
NEW 39	return	×
40	self.dataset_id = dataset.id	3✔
41	self.dataset = dataset	3✔
42
43	@property	6✔
44	def dataset(self):	4✔
45	'''
46	Use a weakref to bind linked dataset so it doesnt bloat usage
47	returns dataset if still available or tries to fetch otherwise
48	'''
49	# still referenced weakref
50	if hasattr(self, '_dataset') and self._dataset() is not None:	3✔
51	return self._dataset()	3✔
52
53	# null return if no associated dataset (governed by dataset_id)
54	elif self.dataset_id is None:	3!
55	return None	3✔
56
57	# else regenerate weakref
NEW 58	LOGGER.info('No referenced object available. Refreshing weakref')	×
NEW 59	dataset = self._load_dataset()	×
NEW 60	self._dataset = weakref.ref(dataset)	×
NEW 61	return dataset	×
62
63	@dataset.setter	6✔
64	def dataset(self, dataset: Dataset) -> None:	6✔
65	'''
66	Need to be careful not to set as the orm object
67	Cannot load if wrong type because of recursive behavior (will
68	propagate down the whole dependency chain)
69	'''
70	self._dataset = weakref.ref(dataset)	3✔
71
72	def _load_dataset(self):	6✔
73	'''
74	Helper to fetch the dataset
75	'''
NEW 76	return self.orm_cls.load_dataset(self.dataset_id)	×
77
78	def add_model(self, model: Model) -> None:	6✔
79	'''
80	Setter method for model used
81	'''
82	if model is None:	3!
NEW 83	return	×
84	self.model_id = model.id	3✔
85	self.model = model	3✔
86
87	@property	6✔
88	def model(self):	4✔
89	'''
90	Use a weakref to bind linked model so it doesnt bloat usage
91	returns model if still available or tries to fetch otherwise
92	'''
93	# still referenced weakref
94	if hasattr(self, '_model') and self._model() is not None:	3!
95	return self._model()	3✔
96
97	# null return if no associated model (governed by model_id)
NEW 98	elif self.model_id is None:	×
NEW 99	return None	×
100
101	# else regenerate weakref
NEW 102	LOGGER.info('No referenced object available. Refreshing weakref')	×
NEW 103	model = self._load_model()	×
NEW 104	self._model = weakref.ref(model)	×
NEW 105	return model	×
106
107	@model.setter	6✔
108	def model(self, model: Model) -> None:	6✔
109	'''
110	Need to be careful not to set as the orm object
111	Cannot load if wrong type because of recursive behavior (will
112	propagate down the whole dependency chain)
113	'''
114	self._model = weakref.ref(model)	3✔
115
116	def _load_model(self):	6✔
117	'''
118	Helper to fetch the model
119	'''
NEW 120	return self.orm_cls.load_model(self.model_id)	×
121
122	def _hash(self) -> str:	6✔
123	'''
124	Hash is the combination of the:
125	1) Model
126	2) Dataset (optional)
127	3) Metric
128	4) Config
129	'''
130	model_hash = self.model.hash_ or self.model._hash()	3✔
131	if self.dataset is not None:	3!
132	dataset_hash = self.dataset.hash_ or self.dataset._hash()	3✔
133	else:
134	dataset_hash = None	×
135	metric = self.__class__.__name__	3✔
136	config = self.config	3✔
137
138	return self.custom_hasher((model_hash, dataset_hash, metric, config))	3✔
139
140	def _get_latest_version(self) -> int:	6✔
141	'''
142	Versions should be autoincrementing for each object (constrained over
143	friendly name and model). Executes a database lookup and increments..
144	'''
NEW 145	return self.orm_cls.get_latest_version(name=self.name, model_id=self.model.id)	×
146
147	def _get_pipeline_split(self, column: str, split: str, **kwargs) -> Any:	6✔
148	'''
149	For special case where dataset is the same as the model's dataset, the
150	dataset splits can refer to the pipeline imposed splits, not the inherent
151	dataset's splits. Use the pipeline split then
152	ex: RandomSplitPipeline on NoSplitDataset evaluating "in_sample" performance
153	'''
154	return getattr(self.model.pipeline.get_dataset_split(split=split, **kwargs), column)	3✔
155
156	def _get_dataset_split(self, **kwargs) -> Any:	6✔
157	'''
158	Default accessor for dataset data. REFERS TO RAW DATASETS
159	not the pipelines superimposed. That means that datasets that do not
160	define explicit splits will have no notion of downstream splits
161	(e.g. RandomSplitPipeline)
162	'''
163	return self.dataset.get(**kwargs)	3✔
164
165	def save(self, **kwargs) -> None:	6✔
166	'''
167	Extend parent function with a few additional save routines
168	'''
169	if self.model is None:	×
170	raise MetricError('Must set model before saving')	×
171
172	if self.values is None:	×
173	raise MetricError('Must score metric before saving')	×
174
NEW 175	super().save(**kwargs)	×
176
177	def score(self, **kwargs):	6✔
178	'''
179	Abstract method for each metric to define
180
181	Should set self.values
182	'''
183	raise NotImplementedError

eyadgaran / SimpleML / 2026984255

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous