1324963996

Build Type

github

Committed by GitHub

Commit Message

Merge 9209236bd into 4e6b1e66a

Pull Request Pull Request #79: Pipeline data mutability

Run Details

225 of 803 branches covered (28.02%)

Branch coverage included in aggregate %.

67 of 192 new or added lines in 8 files covered. (34.9%)

12 existing lines in 4 files now uncovered.

476 of 3830 relevant lines covered (12.43%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

51.28

/simpleml/pipelines/projected_splits.py

'''
Module for dataset projection into pipelines. Defines transfer objects
returned from pipelines
'''

__author__ = 'Elisha Yadgaran'


import pandas as pd
import numpy as np

from typing import Optional
from abc import ABCMeta, abstractmethod

from simpleml.datasets.base_dataset import Dataset
from simpleml.datasets.dataset_splits import Split


class ProjectedDatasetSplit(metaclass=ABCMeta):
    '''
    Transfer object to pass dataset splits through pipelines

    Contains a reference to the dataset and internal logic to
    project the split (references the dataset on each call to
    avoid mutability issues)

    Wraps the normal Split object but delegates behavior so can be used
    interchangeably
    '''

    def __init__(self,
                 dataset: Dataset,
                 split: Optional[str]):
        self.dataset = dataset
        self.split = split

    @property
    def dataset_split(self) -> Split:
        '''
        Passthrough method to retrieve the raw split
        '''
        return self.dataset.get_split(split=self.split)

    @abstractmethod
    def apply_projection(self, dataset_split: Split) -> Split:
        '''
        Main method to apply projection logic on the dataset split
        Returns a new Split with the data subset
        '''

    @property
    def projected_split(self) -> Split:
        '''
        Wrapper property to retrieve the dataset split and manipulate into a
        projected split. Returns a split object already parsed
        '''
        return self.apply_projection(self.dataset_split)

    def __getattr__(self, attr):
        '''
        Passthrough to treat a projected split like a normal split
        '''
        return getattr(self.projected_split, attr)

    def __getitem__(self, item):
        return getattr(self, item)


class IdentityProjectedDatasetSplit(ProjectedDatasetSplit):
    '''
    Straight passthrough variety of projection (ie projected split == dataset split)
    '''

    def apply_projection(self, dataset_split: Split) -> Split:
        '''
        Identity return
        '''
        return dataset_split.squeeze()


class IndexBasedProjectedDatasetSplit(ProjectedDatasetSplit):
    '''
    Index based subset. Compatible with dataset splits that support indexing
    '''

    def __init__(self, indices, **kwargs):
        super().__init__(**kwargs)
        self.indices = indices

    @classmethod
    def indexing_method(cls, df, *args, **kwargs):
        '''
        Infer indexing method to use based on type
        '''
        if isinstance(df, (pd.DataFrame, pd.Series)):
            return cls.pandas_indexing(df, *args, **kwargs)

        elif isinstance(df, np.ndarray):
            return cls.numpy_indexing(df, *args, **kwargs)

        else:
            raise NotImplementedError('Only pandas and numpy datasets are supported right now. Add additional indexing methods to support other dtypes')

    @staticmethod
    def pandas_indexing(df, indices):
        return df.loc[indices]

    @staticmethod
    def numpy_indexing(df, indices):
        return df[indices]

    def apply_projection(self, dataset_split: Split) -> Split:
        '''
        Index subset return
        '''
        return Split(
            **{k: self.indexing_method(v, self.indices) for k, v in dataset_split.items()}
        ).squeeze()

1	'''
2	Module for dataset projection into pipelines. Defines transfer objects
3	returned from pipelines
4	'''
5
NEW 6	__author__ = 'Elisha Yadgaran'	×
7
8
9	import pandas as pd
10	import numpy as np
11
12	from typing import Optional
13	from abc import ABCMeta, abstractmethod
14
15	from simpleml.datasets.base_dataset import Dataset
16	from simpleml.datasets.dataset_splits import Split
17
18
NEW 19	class ProjectedDatasetSplit(metaclass=ABCMeta):	×
20	'''
21	Transfer object to pass dataset splits through pipelines
22
23	Contains a reference to the dataset and internal logic to
24	project the split (references the dataset on each call to
25	avoid mutability issues)
26
27	Wraps the normal Split object but delegates behavior so can be used
28	interchangeably
29	'''
30
NEW 31	def __init__(self,	×
32	dataset: Dataset,
33	split: Optional[str]):
34	self.dataset = dataset	6✔
35	self.split = split	6✔
36
NEW 37	@property	×
NEW 38	def dataset_split(self) -> Split:	×
39	'''
40	Passthrough method to retrieve the raw split
41	'''
42	return self.dataset.get_split(split=self.split)	6✔
43
NEW 44	@abstractmethod	×
NEW 45	def apply_projection(self, dataset_split: Split) -> Split:	×
46	'''
47	Main method to apply projection logic on the dataset split
48	Returns a new Split with the data subset
49	'''
50
NEW 51	@property	×
NEW 52	def projected_split(self) -> Split:	×
53	'''
54	Wrapper property to retrieve the dataset split and manipulate into a
55	projected split. Returns a split object already parsed
56	'''
57	return self.apply_projection(self.dataset_split)	6✔
58
NEW 59	def __getattr__(self, attr):	×
60	'''
61	Passthrough to treat a projected split like a normal split
62	'''
63	return getattr(self.projected_split, attr)	6✔
64
NEW 65	def __getitem__(self, item):	×
66	return getattr(self, item)	3✔
67
68
NEW 69	class IdentityProjectedDatasetSplit(ProjectedDatasetSplit):	×
70	'''
71	Straight passthrough variety of projection (ie projected split == dataset split)
72	'''
73
NEW 74	def apply_projection(self, dataset_split: Split) -> Split:	×
75	'''
76	Identity return
77	'''
78	return dataset_split.squeeze()	6✔
79
80
NEW 81	class IndexBasedProjectedDatasetSplit(ProjectedDatasetSplit):	×
82	'''
83	Index based subset. Compatible with dataset splits that support indexing
84	'''
85
NEW 86	def __init__(self, indices, **kwargs):	×
87	super().__init__(**kwargs)	6✔
88	self.indices = indices	6✔
89
NEW 90	@classmethod	×
91	def indexing_method(cls, df, args, *kwargs):
92	'''
93	Infer indexing method to use based on type
94	'''
95	if isinstance(df, (pd.DataFrame, pd.Series)):	6✔
96	return cls.pandas_indexing(df, args, *kwargs)	6✔
97
98	elif isinstance(df, np.ndarray):	3✔
99	return cls.numpy_indexing(df, args, *kwargs)	3✔
100
101	else:
102	raise NotImplementedError('Only pandas and numpy datasets are supported right now. Add additional indexing methods to support other dtypes')
103
NEW 104	@staticmethod	×
105	def pandas_indexing(df, indices):
106	return df.loc[indices]	6✔
107
NEW 108	@staticmethod	×
109	def numpy_indexing(df, indices):
110	return df[indices]	3✔
111
NEW 112	def apply_projection(self, dataset_split: Split) -> Split:	×
113	'''
114	Index subset return
115	'''
116	return Split(	6✔
117	**{k: self.indexing_method(v, self.indices) for k, v in dataset_split.items()}
118	).squeeze()

eyadgaran / SimpleML / 1324963996

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous