EpistasisLab / pennai / 825

pending completion

Build # 825

Build Type

Pull #273

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/EpistasisLab/pennai/commit/<a class=hub.com/EpistasisLab/pennai/commit/<a class="double-link" href="https://git"><a class=hub.com/EpistasisLab/pennai/commit/<a class="double-link" href="https://git"><a class=hub.com/EpistasisLab/pennai/commit/<a class="double-link" href="https://git"><a class=hub.com/EpistasisLab/pennai/commit/<a class="double-link" href="https://git"><a class=hub.com/EpistasisLab/pennai/commit/bb313a850d9d00709cf38b298ea706ba8d24a423">bb313a850">&lt;a href=&quot;https://github.com/EpistasisLab/pennai/commit/</a><a class="double-link" href="https://github.com/EpistasisLab/pennai/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/EpistasisLab/pennai/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/EpistasisLab/pennai/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/EpistasisLab/pennai/commit/bb313a850d9d00709cf38b298ea706ba8d24a423">bb313a850</a><a href="https://github.com/EpistasisLab/pennai/commit/bb313a850d9d00709cf38b298ea706ba8d24a423">&lt;a href=&quot;https://github.com/EpistasisLab/pennai/commit/bb313a850d9d00709cf38b298ea706ba8d24a423&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/EpistasisLab/pennai/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/EpistasisLab/pennai/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/EpistasisLab/pennai/commit/bb313a850d9d00709cf38b298ea706ba8d24a423&amp;quot;&amp;gt;bb313a850&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/EpistasisLab/pennai/commit/bb313a850d9d00709cf38b298ea706ba8d24a423&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/EpistasisLab/pennai/commit/bb313a850d9d00709cf38b298ea706ba&amp;lt;/a&amp;gt;8d24a42</a>3&quot;&gt;&amp;quot;&amp;gt;Merge &amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/EpistasisLab/pennai/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/EpistasisLab/pennai/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/EpistasisLab/pennai/commit/16e25c1c0695da60a63a7b550... (continued)

Pull Request Pull Request #273: Sklearn-API of PennAI AI engine

Run Details

65 of 65 new or added lines in 9 files covered. (100.0%)

3682 of 4233 relevant lines covered (86.98%)

0.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.68

/ai/recommender/surprise_recommenders.py

"""~This file is part of the PennAI library~

Copyright (C) 2017 Epistasis Lab, University of Pennsylvania

PennAI is maintained by:
    - Heather Williams (hwilli@upenn.edu)
    - Weixuan Fu (weixuanf@upenn.edu)
    - William La Cava (lacava@upenn.edu)
    - Michael Stauffer (stauffer@upenn.edu)
    - and many other generous open source contributors

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.

(Autogenerated header, do not modify)

"""
# Recommender system for Penn AI.
import pandas as pd
from pandas.util import hash_pandas_object
import hashlib
import os
import gzip
import pickle
import copy
# import json
# import urllib.request, urllib.parse
from .base import BaseRecommender
#from ..metalearning import get_metafeatures
# from sklearn.preprocessing import RobustScaler
# from sklearn.pipeline import Pipeline
import numpy as np
from collections import defaultdict, OrderedDict
import pdb
from surprise import (Reader, Dataset, CoClustering, SlopeOne, KNNWithMeans,
                      KNNBasic, mySVD)
# import pyximport
# pyximport.install()
# from .svdedit import mySVD
from collections import defaultdict
import itertools as it
import logging

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
formatter = logging.Formatter('%(module)s: %(levelname)s: %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)


class SurpriseRecommender(BaseRecommender):
    """Class to support generic recommenders from the Surprise library.
    Not intended to be used as a standalone class.

    Parameters
    ----------
    ml_type: str, 'classifier' or 'regressor'
        Recommending classifiers or regressors. Used to determine ML options.

    metric: str (default: accuracy for classifiers, mse for regressors)
        The metric by which to assess performance on the datasets.
    """
    def __init__(self,
            ml_type='classifier',
            metric=None,
            ml_p=None,
            random_state=None,
            knowledgebase_results=None,
            knowledgebase_metafeatures=None,
            load_serialized_rec="if_exists",
            serialized_rec_directory=None,
            serialized_rec_filename=None):

        """ set default recommender specific parameters; might be overwritten by loading serialized recommender"""
        if self.__class__.__name__ == 'SurpriseRecommender':
            raise RuntimeError('Do not instantiate the SurpriseRecommender class '
            'directly; use one of the method-specific classes instead.')
        self.set_algo()
        self.random_state = random_state
        if hasattr(self.algo, 'random_state'):
            self.algo.random_state = self.random_state
        # store results
        self.results_df = pd.DataFrame()
        self.first_fit = True

        # reader for translating btw PennAI results and Suprise training set
        self.reader = Reader()

        self.ml_type = ml_type

        if metric is None:
            logger.debug('metric is None, setting...')
            self.metric='bal_accuracy' if self.ml_type=='classifier' else 'mse'
        else:
            self.metric = metric
        assert(self.metric is not None)

        logger.debug('self.algo_name: '+self.algo_name)
        logger.debug('ml_type: '+self.ml_type)
        logger.debug('metric: '+self.metric)

        self.min_epochs = 10
        self.max_epochs = 100

        """Initialize recommendation system."""
        super().__init__(
            ml_type,
            metric,
            ml_p,
            serialized_rec_directory=serialized_rec_directory,
            serialized_rec_filename=serialized_rec_filename,
            load_serialized_rec=load_serialized_rec,
            knowledgebase_results=knowledgebase_results,
            random_state=random_state)


    @property
    def algo_name(self):
        if type(self.algo).__name__ is None:
            return type(self).__name__
        return type(self.algo).__name__

    def _reconstruct_training_data(self, results_data, results_mf=None,
            source='pennai'):
        """Used for loading pickled recomenders to set results_df
        without training.
        :param results_data: DataFrame with columns corresponding to:
                'dataset'
                'algorithm'
                'parameters'
                self.metric
        :param results_mf: metafeatures for the datasets in results_data
        :param source: str, optional (default: 'pennai')
            if 'pennai', will update tally of trained dataset models
        """
        # update trained dataset models and hash table
        super().update(results_data, results_mf, source)
        # updates self.results_df and self.trainset
        self._update_training_data(results_data, shuffle=True)
        # check whether the set train data matches the pickled recommender's
        # training data.
        rowHashes = hash_pandas_object(self.results_df).values
        newHash = hashlib.sha256(rowHashes).hexdigest()
        if hasattr(self, 'results_df_hash'):
            if newHash == self.results_df_hash:
                logger.info('results_df hashes match')
            else:
                error_msg = 'the results_df hash from the pickle is different'
                logger.error(error_msg)
                raise ValueError(error_msg)
            del self.results_df_hash

    def load(self, filename=None, knowledgebase = None):
        """Load a saved recommender state."""
        if knowledgebase is None:
            logger.warning('A knowledgebase needs to be provided to load '
            'Surprise Recommenders from file. Not loading.')
            return
        loaded = super().load(filename=filename)
        if loaded:
            logger.info('setting training data...')
            self._reconstruct_training_data(knowledgebase,
                    source='knowledgebase')

    def update(self, results_data, results_mf=None, source='pennai'):
        """Update ML / Parameter recommendations based on overall performance in
        results_data.

        :param results_data: DataFrame with columns corresponding to:
                'dataset'
                'algorithm'
                'parameters'
                self.metric
        :param results_mf: metafeatures for the datasets in results_data
        """

        # update trained dataset models and hash table
        super().update(results_data, results_mf, source)

        # update internal model
        self._update_model(results_data)

    def _update_training_data(self, results_data, shuffle=False):
        """Appends results_data to self.results_df. Sets the trainset for
        the surprise recommender.

        :param results_data: DataFrame with columns corresponding to:
                'dataset'
                'algorithm'
                'parameters'
                self.metric
        :param shuffle: boolean, optional (default: False)
            If true, results_data is shuffled before it is added to
            self.results_df or self.trainset.
        """

        if shuffle:
            # shuffle the results data
            logger.debug('shuffling results_data')
            results_data = results_data.sample(frac=1,
                    random_state=self.random_state)

        results_data.loc[:, 'algorithm-parameters'] =  (
                results_data['algorithm'].values + '|' +
                results_data['parameter_hash'].values)
        results_data.rename(columns={self.metric:'score'},inplace=True)
        logger.info('append and drop dupes')
        self.results_df = self.results_df.append(
                results_data[['algorithm-parameters','_id','score']]
                                                ).drop_duplicates()
        logger.info('load_from_df')
        data = Dataset.load_from_df(self.results_df[['_id',
                                                     'algorithm-parameters',
                                                     'score']],
                                    self.reader, rating_scale=(0,1))
        # build training set from the data
        self.trainset = data.build_full_trainset()
        logger.debug('self.trainset # of ML-P combos: ' +
                str(self.trainset.n_items))
        logger.debug('self.trainset # of datasets: '
                + str(self.trainset.n_users))

    def _update_model(self,results_data):
        """Stores new results and updates algo."""
        logger.debug('updating '+self.algo_name+' model')

        self._update_training_data(results_data, self.first_fit)
        self.first_fit=False

        logger.debug('fitting self.algo...')
        # set the number of training iterations proportionally to the amount of
        # results_data
        self.algo.fit(self.trainset)
        logger.debug('done.')
        logger.debug('model '+self.algo_name+' updated')

    def recommend(self, dataset_id, n_recs=1, dataset_mf = None):
        """Return a model and parameter values expected to do best on dataset.

        Parameters
        ----------
        dataset_id: string
            ID of the dataset for which the recommender is generating
            recommendations.
        n_recs: int (default: 1), optional
            Return a list of length n_recs in order of estimators and
            parameters expected to do
            best.
        """
        # dataset hash table
        super().recommend(dataset_id, n_recs, dataset_mf)
        # dataset_hash = self.dataset_id_to_hash[dataset_id]
        try:
            predictions = []
            filtered =0
            for alg_params in self.mlp_combos:
                if (dataset_id+'|'+alg_params not in
                    self.trained_dataset_models):
                    predictions.append(self.algo.predict(dataset_id, alg_params,
                                                         clip=False))
                else:
                    filtered +=1
            logger.debug('filtered '+ str(filtered) + ' recommendations')
            logger.debug('getting top n predictions')
            ml_rec, phash_rec, score_rec = self._get_top_n(predictions, n_recs)
            logger.debug('returning ml recs')

        except Exception as e:
            logger.error( 'error running self.best_model_prediction for'+
                    str(dataset_id))
            raise e
        # update the recommender's memory with the new algorithm-parameter combos
        # that it recommended
        self._update_trained_dataset_models_from_rec(dataset_id,
                                                    ml_rec, phash_rec)

        p_rec = [self.hash_2_param[ph] for ph in phash_rec]
        return ml_rec, p_rec, score_rec

    def _get_top_n(self,predictions, n=10):
        '''Return the top-N recommendation for each user from a set of predictions.

        Args:
            predictions(list of Prediction objects): The list of predictions, as
                returned by the test method of an algorithm.
            n(int): The number of recommendation to output for each user. Default
                is 10.

        Returns:
            ml recs, parameter recs, and their scores in three lists
        '''

        # grabs the ml ids and their estimated scores for this dataset
        top_n = []
        ml_dist = {}
        for uid, iid, true_r, est, _ in predictions:
            top_n.append((iid, est))
            ml = iid.split('|')[0]
            if ml in ml_dist.keys():
                ml_dist[ml] += 1.0
            else:
                ml_dist[ml] = 1.0
        n_ml = len(ml_dist.keys())

        ######
        # Shuffle top_n just to remove tied algorithm bias when sorting
        # Make uniform random choices from the Algorithms, then uniform random
        # choices from their parameters to shuffle top_n
        # the probability for each ML method is 1/total_methods/(# instances of that
        # method)
        inv_ml_dist = {k:1/n_ml/v for k,v in ml_dist.items()}
        top_n_dist = np.array([inv_ml_dist[tn[0].split('|')[0]]
            for tn in top_n])
        top_n_idx = np.arange(len(top_n))
        top_n_idx_s = np.random.choice(top_n_idx, len(top_n), replace=False,
                                       p=top_n_dist)
        top_n = [top_n[i] for i in top_n_idx_s]
        #####

        # sort top_n
        top_n = sorted(top_n, key=lambda x: x[1], reverse=True)
        top_n = top_n[:n]

        logger.debug('filtered top_n:'+str(top_n))
        ml_rec = [n[0].split('|')[0] for n in top_n]
        p_rec = [n[0].split('|')[1] for n in top_n]
        score_rec = [n[1] for n in top_n]
        return ml_rec, p_rec, score_rec

class CoClusteringRecommender(SurpriseRecommender):
    """Generates recommendations via CoClustering, see
    https://surprise.readthedocs.io/en/stable/co_clustering.html
    """
    def set_algo(self):
        self.algo = CoClustering(n_cltr_u = 10)

    # def __init__(self, ml_type='classifier', metric=None, ml_p=None,
    #         algo=None):
    #     super().__init__(ml_type, metric, ml_p, algo)
    #     # set n clusters for ML equal to # of ML methods
    #     self.
    def _update_model(self,results_data):
        """Stores new results and updates algo."""
        self.algo.n_cltr_i = self.ml_p.algorithm.nunique()
        super()._update_model(results_data)

class KNNWithMeansRecommender(SurpriseRecommender):
    """Generates recommendations via KNNWithMeans, see
    https://surprise.readthedocs.io/en/stable/knn_inspired.html
    """
    def set_algo(self):
        self.algo = KNNWithMeans()

class KNNDatasetRecommender(SurpriseRecommender):
    """Generates recommendations via KNN with clusters defined over datasets, see
    https://surprise.readthedocs.io/en/stable/knn_inspired.html
    """
    def set_algo(self):
        self.algo = KNNBasic(sim_options={'user_based':True})

    @property
    def algo_name(self):
        return 'KNN-Dataset'

class KNNMLRecommender(SurpriseRecommender):
    """Generates recommendations via KNN with clusters defined over algorithms, see
    https://surprise.readthedocs.io/en/stable/knn_inspired.html
    """
    def set_algo(self):
        self.algo = KNNBasic(sim_options={'user_based':False})
    @property
    def algo_name(self):
        return 'KNN-ML'

class SlopeOneRecommender(SurpriseRecommender):
    """Generates recommendations via SlopeOne, see
    https://surprise.readthedocs.io/en/stable/slope_one.html
    """
    def set_algo(self):
        self.algo = SlopeOne()


class SVDRecommender(SurpriseRecommender):
    """SVD recommender.
    see https://surprise.readthedocs.io/en/stable/matrix_factorization.html
    Recommends machine learning algorithms and parameters using the SVD algorithm.
        - stores ML + P and every dataset.
        - learns a matrix factorization on the non-missing data.
        - given a dataset, estimates the rankings of all ML+P and returns the top
        n_recs.

    Note that we use a custom online version of SVD found here:
    https://github.com/lacava/surprise
    """

    def set_algo(self, surprise_kwargs={}):
        alg_kwargs = {'n_factors':20,
                'biased':True,
                'init_mean':0,
                'init_std_dev':.2,
                'lr_all':.01,
                'reg_all':.02,
                'verbose':False}
        alg_kwargs.update(surprise_kwargs)
        self.algo = mySVD(**alg_kwargs)

    # def __init__(self, ml_type='classifier', metric=None, ml_p=None,
    #         filename=None, knowledgebase=None, random_state=None,
    #         surprise_kwargs={}):
    #     super().__init__(ml_type=ml_type, metric=metric, ml_p=ml_p,
    #             filename=filename, knowledgebase=knowledgebase,
    #             random_state=random_state)

    def _update_model(self,results_data):
        """Stores new results and updates SVD."""
        logger.info('updating SVD model')
        # shuffle the results data the first time
        if self.first_fit:
            logger.debug('shuffling results_data')
            results_data = results_data.sample(frac=1,
                    random_state=self.random_state)
            self.first_fit=False

        self._update_training_data(results_data)
        # set the number of training iterations proportionally to the amount of
        # results_data
        logger.info('algo random_state: '+str(self.algo.random_state))
        self.algo.n_epochs = min(len(results_data),self.max_epochs)
        self.algo.n_epochs = max(self.algo.n_epochs,self.min_epochs)

        logger.debug('fitting self.algo...')
        self.algo.partial_fit(self.trainset)
        logger.debug('done.')
        logger.debug('model SVD updated')

1	"""~This file is part of the PennAI library~
2
3	Copyright (C) 2017 Epistasis Lab, University of Pennsylvania
4
5	PennAI is maintained by:
6	- Heather Williams (hwilli@upenn.edu)
7	- Weixuan Fu (weixuanf@upenn.edu)
8	- William La Cava (lacava@upenn.edu)
9	- Michael Stauffer (stauffer@upenn.edu)
10	- and many other generous open source contributors
11
12	This program is free software: you can redistribute it and/or modify
13	it under the terms of the GNU General Public License as published by
14	the Free Software Foundation, either version 3 of the License, or
15	(at your option) any later version.
16
17	This program is distributed in the hope that it will be useful,
18	but WITHOUT ANY WARRANTY; without even the implied warranty of
19	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20	GNU General Public License for more details.
21
22	You should have received a copy of the GNU General Public License
23	along with this program. If not, see <https://www.gnu.org/licenses/>.
24
25	(Autogenerated header, do not modify)
26
27	"""
28	# Recommender system for Penn AI.
29	import pandas as pd	1✔
30	from pandas.util import hash_pandas_object	1✔
31	import hashlib	1✔
32	import os	1✔
33	import gzip	1✔
34	import pickle	1✔
35	import copy	1✔
36	# import json
37	# import urllib.request, urllib.parse
38	from .base import BaseRecommender	1✔
39	#from ..metalearning import get_metafeatures
40	# from sklearn.preprocessing import RobustScaler
41	# from sklearn.pipeline import Pipeline
42	import numpy as np	1✔
43	from collections import defaultdict, OrderedDict	1✔
44	import pdb	1✔
45	from surprise import (Reader, Dataset, CoClustering, SlopeOne, KNNWithMeans,	1✔
46	KNNBasic, mySVD)
47	# import pyximport
48	# pyximport.install()
49	# from .svdedit import mySVD
50	from collections import defaultdict	1✔
51	import itertools as it	1✔
52	import logging	1✔
53
54	logger = logging.getLogger(__name__)	1✔
55	logger.setLevel(logging.DEBUG)	1✔
56	ch = logging.StreamHandler()	1✔
57	formatter = logging.Formatter('%(module)s: %(levelname)s: %(message)s')	1✔
58	ch.setFormatter(formatter)	1✔
59	logger.addHandler(ch)	1✔
60
61
62	class SurpriseRecommender(BaseRecommender):	1✔
63	"""Class to support generic recommenders from the Surprise library.
64	Not intended to be used as a standalone class.
65
66	Parameters
67	----------
68	ml_type: str, 'classifier' or 'regressor'
69	Recommending classifiers or regressors. Used to determine ML options.
70
71	metric: str (default: accuracy for classifiers, mse for regressors)
72	The metric by which to assess performance on the datasets.
73	"""
74	def __init__(self,	1✔
75	ml_type='classifier',
76	metric=None,
77	ml_p=None,
78	random_state=None,
79	knowledgebase_results=None,
80	knowledgebase_metafeatures=None,
81	load_serialized_rec="if_exists",
82	serialized_rec_directory=None,
83	serialized_rec_filename=None):
84
85	""" set default recommender specific parameters; might be overwritten by loading serialized recommender"""
86	if self.__class__.__name__ == 'SurpriseRecommender':	1✔
87	raise RuntimeError('Do not instantiate the SurpriseRecommender class '	×
88	'directly; use one of the method-specific classes instead.')
89	self.set_algo()	1✔
90	self.random_state = random_state	1✔
91	if hasattr(self.algo, 'random_state'):	1✔
92	self.algo.random_state = self.random_state	1✔
93	# store results
94	self.results_df = pd.DataFrame()	1✔
95	self.first_fit = True	1✔
96
97	# reader for translating btw PennAI results and Suprise training set
98	self.reader = Reader()	1✔
99
100	self.ml_type = ml_type	1✔
101
102	if metric is None:	1✔
103	logger.debug('metric is None, setting...')	1✔
104	self.metric='bal_accuracy' if self.ml_type=='classifier' else 'mse'	1✔
105	else:
106	self.metric = metric	×
107	assert(self.metric is not None)	1✔
108
109	logger.debug('self.algo_name: '+self.algo_name)	1✔
110	logger.debug('ml_type: '+self.ml_type)	1✔
111	logger.debug('metric: '+self.metric)	1✔
112
113	self.min_epochs = 10	1✔
114	self.max_epochs = 100	1✔
115
116	"""Initialize recommendation system."""
117	super().__init__(	1✔
118	ml_type,
119	metric,
120	ml_p,
121	serialized_rec_directory=serialized_rec_directory,
122	serialized_rec_filename=serialized_rec_filename,
123	load_serialized_rec=load_serialized_rec,
124	knowledgebase_results=knowledgebase_results,
125	random_state=random_state)
126
127
128	@property	1✔
129	def algo_name(self):
130	if type(self.algo).__name__ is None:	1✔
131	return type(self).__name__	×
132	return type(self.algo).__name__	1✔
133
134	def _reconstruct_training_data(self, results_data, results_mf=None,	1✔
135	source='pennai'):
136	"""Used for loading pickled recomenders to set results_df
137	without training.
138	:param results_data: DataFrame with columns corresponding to:
139	'dataset'
140	'algorithm'
141	'parameters'
142	self.metric
143	:param results_mf: metafeatures for the datasets in results_data
144	:param source: str, optional (default: 'pennai')
145	if 'pennai', will update tally of trained dataset models
146	"""
147	# update trained dataset models and hash table
148	super().update(results_data, results_mf, source)	1✔
149	# updates self.results_df and self.trainset
150	self._update_training_data(results_data, shuffle=True)	1✔
151	# check whether the set train data matches the pickled recommender's
152	# training data.
153	rowHashes = hash_pandas_object(self.results_df).values	1✔
154	newHash = hashlib.sha256(rowHashes).hexdigest()	1✔
155	if hasattr(self, 'results_df_hash'):	1✔
156	if newHash == self.results_df_hash:	1✔
157	logger.info('results_df hashes match')	1✔
158	else:
159	error_msg = 'the results_df hash from the pickle is different'	×
160	logger.error(error_msg)	×
161	raise ValueError(error_msg)	×
162	del self.results_df_hash	1✔
163
164	def load(self, filename=None, knowledgebase = None):	1✔
165	"""Load a saved recommender state."""
166	if knowledgebase is None:	1✔
167	logger.warning('A knowledgebase needs to be provided to load '	×
168	'Surprise Recommenders from file. Not loading.')
169	return	×
170	loaded = super().load(filename=filename)	1✔
171	if loaded:	1✔
172	logger.info('setting training data...')	1✔
173	self._reconstruct_training_data(knowledgebase,	1✔
174	source='knowledgebase')
175
176	def update(self, results_data, results_mf=None, source='pennai'):	1✔
177	"""Update ML / Parameter recommendations based on overall performance in
178	results_data.
179
180	:param results_data: DataFrame with columns corresponding to:
181	'dataset'
182	'algorithm'
183	'parameters'
184	self.metric
185	:param results_mf: metafeatures for the datasets in results_data
186	"""
187
188	# update trained dataset models and hash table
189	super().update(results_data, results_mf, source)	1✔
190
191	# update internal model
192	self._update_model(results_data)	1✔
193
194	def _update_training_data(self, results_data, shuffle=False):	1✔
195	"""Appends results_data to self.results_df. Sets the trainset for
196	the surprise recommender.
197
198	:param results_data: DataFrame with columns corresponding to:
199	'dataset'
200	'algorithm'
201	'parameters'
202	self.metric
203	:param shuffle: boolean, optional (default: False)
204	If true, results_data is shuffled before it is added to
205	self.results_df or self.trainset.
206	"""
207
208	if shuffle:	1✔
209	# shuffle the results data
210	logger.debug('shuffling results_data')	1✔
211	results_data = results_data.sample(frac=1,	1✔
212	random_state=self.random_state)
213
214	results_data.loc[:, 'algorithm-parameters'] = (	1✔
215	results_data['algorithm'].values + '\|' +
216	results_data['parameter_hash'].values)
217	results_data.rename(columns={self.metric:'score'},inplace=True)	1✔
218	logger.info('append and drop dupes')	1✔
219	self.results_df = self.results_df.append(	1✔
220	results_data[['algorithm-parameters','_id','score']]
221	).drop_duplicates()
222	logger.info('load_from_df')	1✔
223	data = Dataset.load_from_df(self.results_df[['_id',	1✔
224	'algorithm-parameters',
225	'score']],
226	self.reader, rating_scale=(0,1))
227	# build training set from the data
228	self.trainset = data.build_full_trainset()	1✔
229	logger.debug('self.trainset # of ML-P combos: ' +	1✔
230	str(self.trainset.n_items))
231	logger.debug('self.trainset # of datasets: '	1✔
232	+ str(self.trainset.n_users))
233
234	def _update_model(self,results_data):	1✔
235	"""Stores new results and updates algo."""
236	logger.debug('updating '+self.algo_name+' model')	1✔
237
238	self._update_training_data(results_data, self.first_fit)	1✔
239	self.first_fit=False	1✔
240
241	logger.debug('fitting self.algo...')	1✔
242	# set the number of training iterations proportionally to the amount of
243	# results_data
244	self.algo.fit(self.trainset)	1✔
245	logger.debug('done.')	1✔
246	logger.debug('model '+self.algo_name+' updated')	1✔
247
248	def recommend(self, dataset_id, n_recs=1, dataset_mf = None):	1✔
249	"""Return a model and parameter values expected to do best on dataset.
250
251	Parameters
252	----------
253	dataset_id: string
254	ID of the dataset for which the recommender is generating
255	recommendations.
256	n_recs: int (default: 1), optional
257	Return a list of length n_recs in order of estimators and
258	parameters expected to do
259	best.
260	"""
261	# dataset hash table
262	super().recommend(dataset_id, n_recs, dataset_mf)	1✔
263	# dataset_hash = self.dataset_id_to_hash[dataset_id]
264	try:	1✔
265	predictions = []	1✔
266	filtered =0	1✔
267	for alg_params in self.mlp_combos:	1✔
268	if (dataset_id+'\|'+alg_params not in	1✔
269	self.trained_dataset_models):
270	predictions.append(self.algo.predict(dataset_id, alg_params,	1✔
271	clip=False))
272	else:
273	filtered +=1	1✔
274	logger.debug('filtered '+ str(filtered) + ' recommendations')	1✔
275	logger.debug('getting top n predictions')	1✔
276	ml_rec, phash_rec, score_rec = self._get_top_n(predictions, n_recs)	1✔
277	logger.debug('returning ml recs')	1✔
278
279	except Exception as e:	×
280	logger.error( 'error running self.best_model_prediction for'+	×
281	str(dataset_id))
282	raise e	×
283	# update the recommender's memory with the new algorithm-parameter combos
284	# that it recommended
285	self._update_trained_dataset_models_from_rec(dataset_id,	1✔
286	ml_rec, phash_rec)
287
288	p_rec = [self.hash_2_param[ph] for ph in phash_rec]	1✔
289	return ml_rec, p_rec, score_rec	1✔
290
291	def _get_top_n(self,predictions, n=10):	1✔
292	'''Return the top-N recommendation for each user from a set of predictions.
293
294	Args:
295	predictions(list of Prediction objects): The list of predictions, as
296	returned by the test method of an algorithm.
297	n(int): The number of recommendation to output for each user. Default
298	is 10.
299
300	Returns:
301	ml recs, parameter recs, and their scores in three lists
302	'''
303
304	# grabs the ml ids and their estimated scores for this dataset
305	top_n = []	1✔
306	ml_dist = {}	1✔
307	for uid, iid, true_r, est, _ in predictions:	1✔
308	top_n.append((iid, est))	1✔
309	ml = iid.split('\|')[0]	1✔
310	if ml in ml_dist.keys():	1✔
311	ml_dist[ml] += 1.0	1✔
312	else:
313	ml_dist[ml] = 1.0	1✔
314	n_ml = len(ml_dist.keys())	1✔
315
316	######
317	# Shuffle top_n just to remove tied algorithm bias when sorting
318	# Make uniform random choices from the Algorithms, then uniform random
319	# choices from their parameters to shuffle top_n
320	# the probability for each ML method is 1/total_methods/(# instances of that
321	# method)
322	inv_ml_dist = {k:1/n_ml/v for k,v in ml_dist.items()}	1✔
323	top_n_dist = np.array([inv_ml_dist[tn[0].split('\|')[0]]	1✔
324	for tn in top_n])
325	top_n_idx = np.arange(len(top_n))	1✔
326	top_n_idx_s = np.random.choice(top_n_idx, len(top_n), replace=False,	1✔
327	p=top_n_dist)
328	top_n = [top_n[i] for i in top_n_idx_s]	1✔
329	#####
330
331	# sort top_n
332	top_n = sorted(top_n, key=lambda x: x[1], reverse=True)	1✔
333	top_n = top_n[:n]	1✔
334
335	logger.debug('filtered top_n:'+str(top_n))	1✔
336	ml_rec = [n[0].split('\|')[0] for n in top_n]	1✔
337	p_rec = [n[0].split('\|')[1] for n in top_n]	1✔
338	score_rec = [n[1] for n in top_n]	1✔
339	return ml_rec, p_rec, score_rec	1✔
340
341	class CoClusteringRecommender(SurpriseRecommender):	1✔
342	"""Generates recommendations via CoClustering, see
343	https://surprise.readthedocs.io/en/stable/co_clustering.html
344	"""
345	def set_algo(self):	1✔
346	self.algo = CoClustering(n_cltr_u = 10)	1✔
347
348	# def __init__(self, ml_type='classifier', metric=None, ml_p=None,
349	# algo=None):
350	# super().__init__(ml_type, metric, ml_p, algo)
351	# # set n clusters for ML equal to # of ML methods
352	# self.
353	def _update_model(self,results_data):	1✔
354	"""Stores new results and updates algo."""
355	self.algo.n_cltr_i = self.ml_p.algorithm.nunique()	1✔
356	super()._update_model(results_data)	1✔
357
358	class KNNWithMeansRecommender(SurpriseRecommender):	1✔
359	"""Generates recommendations via KNNWithMeans, see
360	https://surprise.readthedocs.io/en/stable/knn_inspired.html
361	"""
362	def set_algo(self):	1✔
363	self.algo = KNNWithMeans()	1✔
364
365	class KNNDatasetRecommender(SurpriseRecommender):	1✔
366	"""Generates recommendations via KNN with clusters defined over datasets, see
367	https://surprise.readthedocs.io/en/stable/knn_inspired.html
368	"""
369	def set_algo(self):	1✔
370	self.algo = KNNBasic(sim_options={'user_based':True})	1✔
371
372	@property	1✔
373	def algo_name(self):
374	return 'KNN-Dataset'	1✔
375
376	class KNNMLRecommender(SurpriseRecommender):	1✔
377	"""Generates recommendations via KNN with clusters defined over algorithms, see
378	https://surprise.readthedocs.io/en/stable/knn_inspired.html
379	"""
380	def set_algo(self):	1✔
381	self.algo = KNNBasic(sim_options={'user_based':False})	1✔
382	@property	1✔
383	def algo_name(self):
384	return 'KNN-ML'	1✔
385
386	class SlopeOneRecommender(SurpriseRecommender):	1✔
387	"""Generates recommendations via SlopeOne, see
388	https://surprise.readthedocs.io/en/stable/slope_one.html
389	"""
390	def set_algo(self):	1✔
391	self.algo = SlopeOne()	1✔
392
393
394	class SVDRecommender(SurpriseRecommender):	1✔
395	"""SVD recommender.
396	see https://surprise.readthedocs.io/en/stable/matrix_factorization.html
397	Recommends machine learning algorithms and parameters using the SVD algorithm.
398	- stores ML + P and every dataset.
399	- learns a matrix factorization on the non-missing data.
400	- given a dataset, estimates the rankings of all ML+P and returns the top
401	n_recs.
402
403	Note that we use a custom online version of SVD found here:
404	https://github.com/lacava/surprise
405	"""
406
407	def set_algo(self, surprise_kwargs={}):	1✔
408	alg_kwargs = {'n_factors':20,	1✔
409	'biased':True,
410	'init_mean':0,
411	'init_std_dev':.2,
412	'lr_all':.01,
413	'reg_all':.02,
414	'verbose':False}
415	alg_kwargs.update(surprise_kwargs)	1✔
416	self.algo = mySVD(**alg_kwargs)	1✔
417
418	# def __init__(self, ml_type='classifier', metric=None, ml_p=None,
419	# filename=None, knowledgebase=None, random_state=None,
420	# surprise_kwargs={}):
421	# super().__init__(ml_type=ml_type, metric=metric, ml_p=ml_p,
422	# filename=filename, knowledgebase=knowledgebase,
423	# random_state=random_state)
424
425	def _update_model(self,results_data):	1✔
426	"""Stores new results and updates SVD."""
427	logger.info('updating SVD model')	1✔
428	# shuffle the results data the first time
429	if self.first_fit:	1✔
430	logger.debug('shuffling results_data')	1✔
431	results_data = results_data.sample(frac=1,	1✔
432	random_state=self.random_state)
433	self.first_fit=False	1✔
434
435	self._update_training_data(results_data)	1✔
436	# set the number of training iterations proportionally to the amount of
437	# results_data
438	logger.info('algo random_state: '+str(self.algo.random_state))	1✔
439	self.algo.n_epochs = min(len(results_data),self.max_epochs)	1✔
440	self.algo.n_epochs = max(self.algo.n_epochs,self.min_epochs)	1✔
441
442	logger.debug('fitting self.algo...')	1✔
443	self.algo.partial_fit(self.trainset)	1✔
444	logger.debug('done.')	1✔
445	logger.debug('model SVD updated')	1✔

EpistasisLab / pennai / 825

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous