1189008285

Build Type

github

Committed by GitHub

Commit Message

Merge 94dcf9d31 into d1ad2140e

Pull Request Pull Request #79: Pipeline data mutability

Run Details

214 of 775 branches covered (27.61%)

Branch coverage included in aggregate %.

67 of 117 new or added lines in 7 files covered. (57.26%)

8 existing lines in 3 files now uncovered.

461 of 3745 relevant lines covered (12.31%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

49.25

/simpleml/pipelines/validation_split_mixins.py

'''
Module for different pipeline split methods for cross validation

    1) No Split -- Just use all the data
    2) Explicit Split -- dataset class defines the split
    3) Percentage -- random split support for train, validation, test
    4) Chronological -- time based split support for train, validation, test
    5) KFold
'''

__author__ = 'Elisha Yadgaran'

import pandas as pd

from abc import ABCMeta, abstractmethod
from sklearn.model_selection import train_test_split
from future.utils import with_metaclass
from typing import Dict, List, Optional, Union

from simpleml.constants import TRAIN_SPLIT, VALIDATION_SPLIT, TEST_SPLIT
from simpleml.datasets.dataset_splits import Split, SplitContainer
from .projected_splits import IdentityProjectedDatasetSplit, IndexBasedProjectedDatasetSplit


class SplitMixin(with_metaclass(ABCMeta, object)):
    @abstractmethod
    def split_dataset(self):
        '''
        Set the split criteria

        Must set self._dataset_splits
        '''

    def containerize_split(self, split_dict: Dict[str, Split]) -> SplitContainer:
        return SplitContainer(**split_dict)

    def get_split_names(self) -> List[str]:
        if not hasattr(self, '_dataset_splits') or self._dataset_splits is None:
            self.split_dataset()
        return list(self._dataset_splits.keys())


class NoSplitMixin(SplitMixin):
    def split_dataset(self) -> None:
        '''
        Non-split mixin class. Returns full dataset for any split name
        '''
        default_split = IdentityProjectedDatasetSplit(dataset=self.dataset, split=None)
        self._dataset_splits = self.containerize_split({
            'default_factory': lambda: default_split
        })


class ExplicitSplitMixin(SplitMixin):
    def split_dataset(self) -> None:
        '''
        Method to split the dataframe into different sets. Assumes dataset
        explicitly delineates between different splits

        Passes forward dataset split names so uniquely named splits will propagate
        and can be referenced the same way
        '''
        self._dataset_splits = self.containerize_split({
            split_name: IdentityProjectedDatasetSplit(dataset=self.dataset, split=split_name)
            for split_name in self.dataset.get_split_names()
        })


class RandomSplitMixin(SplitMixin):
    '''
    Class to randomly split dataset into different sets

    **Redefines splits so custom named splits in dataset cannot be referenced
    by the same names. Only TRAIN/TEST/VALIDATION**
    '''

    def __init__(self,
                 train_size: Union[float, int],
                 test_size: Optional[Union[float, int]] = None,
                 validation_size: Union[float, int] = 0.0,
                 random_state: int = 123,
                 shuffle: bool = True, **kwargs):
        '''
        Set splitting params:
        By default validation is 0.0 because it is only used for hyperparameter
        tuning
        '''
        super(RandomSplitMixin, self).__init__(**kwargs)

        if train_size is None:
            train_size = 1.0 - validation_size

        if test_size is None:
            test_size = 1.0 - train_size - validation_size

        # Pipeline Params
        self.config.update({
            'train_size': train_size,
            'validation_size': validation_size,
            'test_size': test_size,
            'random_state': random_state,
            'shuffle': shuffle
        })

    @staticmethod
    def get_index(data) -> List[int]:
        '''
        Helper to extract the index from a dataset. Generates a range index
        if none exists
        '''
        if isinstance(data, (pd.DataFrame, pd.Series)):
            return data.index

        else:
            # no named index, use a linear range
            return range(len(data))

    def split_dataset(self) -> None:
        '''
        Overwrite method to split by percentage
        '''
        train_size = self.config.get('train_size')
        validation_size = self.config.get('validation_size')
        test_size = self.config.get('test_size')
        random_state = self.config.get('random_state')
        shuffle = self.config.get('shuffle')

        # Sklearn's train test split can only accomodate one split per iteration
        # find the indices that match to each split
        # use the X split section
        index = self.get_index(self.dataset.X)

        if test_size == 0:  # No split necessary
            test_indices = []
            remaining_indices = index
        else:
            remaining_indices, test_indices = train_test_split(
                index, test_size=test_size, random_state=random_state, shuffle=shuffle)

        calibrated_validation_size = float(validation_size) / (validation_size + train_size)
        if calibrated_validation_size == 0:  # No split necessary
            train_indices = remaining_indices
            validation_indices = []
        else:
            train_indices, validation_indices = train_test_split(
                remaining_indices, test_size=calibrated_validation_size, random_state=random_state, shuffle=shuffle)

        self._dataset_splits = self.containerize_split({
            TRAIN_SPLIT: IndexBasedProjectedDatasetSplit(dataset=self.dataset, split=None, indices=train_indices),
            VALIDATION_SPLIT: IndexBasedProjectedDatasetSplit(dataset=self.dataset, split=None, indices=validation_indices),
            TEST_SPLIT: IndexBasedProjectedDatasetSplit(dataset=self.dataset, split=None, indices=test_indices)
        })


class ChronologicalSplitMixin(SplitMixin):
    def __init__(self, **kwargs):
        super(ChronologicalSplitMixin, self).__init__(**kwargs)


class KFoldSplitMixin(SplitMixin):
    '''
    TBD on how to implement this. KFold requires K models and unique datasets
    so may be easier to wrap a parallelized implementation that internally
    creates K new Pipeline and Model objects
    '''
    pass

1	'''
2	Module for different pipeline split methods for cross validation
3
4	1) No Split -- Just use all the data
5	2) Explicit Split -- dataset class defines the split
6	3) Percentage -- random split support for train, validation, test
7	4) Chronological -- time based split support for train, validation, test
8	5) KFold
9	'''
10
11	__author__ = 'Elisha Yadgaran'	×
12
13	import pandas as pd
14
15	from abc import ABCMeta, abstractmethod
16	from sklearn.model_selection import train_test_split
17	from future.utils import with_metaclass
18	from typing import Dict, List, Optional, Union
19
20	from simpleml.constants import TRAIN_SPLIT, VALIDATION_SPLIT, TEST_SPLIT
21	from simpleml.datasets.dataset_splits import Split, SplitContainer
22	from .projected_splits import IdentityProjectedDatasetSplit, IndexBasedProjectedDatasetSplit
23
24
25	class SplitMixin(with_metaclass(ABCMeta, object)):	×
26	@abstractmethod	×
27	def split_dataset(self):
28	'''
29	Set the split criteria
30
31	Must set self._dataset_splits
32	'''
33
34	def containerize_split(self, split_dict: Dict[str, Split]) -> SplitContainer:	×
35	return SplitContainer(**split_dict)	3✔
36
37	def get_split_names(self) -> List[str]:	×
38	if not hasattr(self, '_dataset_splits') or self._dataset_splits is None:	×
39	self.split_dataset()	×
40	return list(self._dataset_splits.keys())	×
41
42
43	class NoSplitMixin(SplitMixin):	×
44	def split_dataset(self) -> None:	×
45	'''
46	Non-split mixin class. Returns full dataset for any split name
47	'''
48	default_split = IdentityProjectedDatasetSplit(dataset=self.dataset, split=None)	3✔
49	self._dataset_splits = self.containerize_split({	3✔
50	'default_factory': lambda: default_split
51	})
52
53
54	class ExplicitSplitMixin(SplitMixin):	×
55	def split_dataset(self) -> None:	×
56	'''
57	Method to split the dataframe into different sets. Assumes dataset
58	explicitly delineates between different splits
59
60	Passes forward dataset split names so uniquely named splits will propagate
61	and can be referenced the same way
62	'''
63	self._dataset_splits = self.containerize_split({	3✔
64	split_name: IdentityProjectedDatasetSplit(dataset=self.dataset, split=split_name)
65	for split_name in self.dataset.get_split_names()
66	})
67
68
69	class RandomSplitMixin(SplitMixin):	×
70	'''
71	Class to randomly split dataset into different sets
72
73	**Redefines splits so custom named splits in dataset cannot be referenced
74	by the same names. Only TRAIN/TEST/VALIDATION**
75	'''
76
77	def __init__(self,	×
78	train_size: Union[float, int],
79	test_size: Optional[Union[float, int]] = None,
80	validation_size: Union[float, int] = 0.0,
81	random_state: int = 123,
82	shuffle: bool = True, **kwargs):
83	'''
84	Set splitting params:
85	By default validation is 0.0 because it is only used for hyperparameter
86	tuning
87	'''
88	super(RandomSplitMixin, self).__init__(**kwargs)	3✔
89
90	if train_size is None:	3!
91	train_size = 1.0 - validation_size	×
92
93	if test_size is None:	3!
94	test_size = 1.0 - train_size - validation_size	3✔
95
96	# Pipeline Params
97	self.config.update({	3✔
98	'train_size': train_size,
99	'validation_size': validation_size,
100	'test_size': test_size,
101	'random_state': random_state,
102	'shuffle': shuffle
103	})
104
NEW 105	@staticmethod	×
NEW 106	def get_index(data) -> List[int]:	×
107	'''
108	Helper to extract the index from a dataset. Generates a range index
109	if none exists
110	'''
111	if isinstance(data, (pd.DataFrame, pd.Series)):	3!
112	return data.index	3✔
113
114	else:
115	# no named index, use a linear range
NEW 116	return range(len(data))	×
117
UNCOV 118	def split_dataset(self) -> None:	×
119	'''
120	Overwrite method to split by percentage
121	'''
122	train_size = self.config.get('train_size')	3✔
123	validation_size = self.config.get('validation_size')	3✔
124	test_size = self.config.get('test_size')	3✔
125	random_state = self.config.get('random_state')	3✔
126	shuffle = self.config.get('shuffle')	3✔
127
128	# Sklearn's train test split can only accomodate one split per iteration
129	# find the indices that match to each split
130	# use the X split section
131	index = self.get_index(self.dataset.X)	3✔
132
133	if test_size == 0: # No split necessary	3!
NEW 134	test_indices = []	×
NEW 135	remaining_indices = index	×
136	else:
137	remaining_indices, test_indices = train_test_split(	3✔
138	index, test_size=test_size, random_state=random_state, shuffle=shuffle)
139
140	calibrated_validation_size = float(validation_size) / (validation_size + train_size)	3✔
141	if calibrated_validation_size == 0: # No split necessary	3!
142	train_indices = remaining_indices	3✔
143	validation_indices = []	3✔
144	else:
NEW 145	train_indices, validation_indices = train_test_split(	×
146	remaining_indices, test_size=calibrated_validation_size, random_state=random_state, shuffle=shuffle)
147
148	self._dataset_splits = self.containerize_split({	3✔
149	TRAIN_SPLIT: IndexBasedProjectedDatasetSplit(dataset=self.dataset, split=None, indices=train_indices),
150	VALIDATION_SPLIT: IndexBasedProjectedDatasetSplit(dataset=self.dataset, split=None, indices=validation_indices),
151	TEST_SPLIT: IndexBasedProjectedDatasetSplit(dataset=self.dataset, split=None, indices=test_indices)
152	})
153
154
155	class ChronologicalSplitMixin(SplitMixin):	×
156	def __init__(self, **kwargs):	×
157	super(ChronologicalSplitMixin, self).__init__(**kwargs)	×
158
159
160	class KFoldSplitMixin(SplitMixin):	×
161	'''
162	TBD on how to implement this. KFold requires K models and unique datasets
163	so may be easier to wrap a parallelized implementation that internally
164	creates K new Pipeline and Model objects
165	'''
166	pass	×

eyadgaran / SimpleML / 1189008285

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous