ishikota / kyoka / 130

Committed 24 Dec 2016 - 0:25 coverage: 98.031% (+0.03%) from 98.0%

Build # 130

Build Type

Pull #33

travis-ci

Committed by

web-flow

Commit Message

Bump up version to 0.2.1

Pull Request Pull Request #33: release v0.2.1

Run Details

77 of 80 new or added lines in 9 files covered. (96.25%)

1 existing line in 1 file now uncovered.

747 of 762 relevant lines covered (98.03%)

0.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.75

/kyoka/value_function.py

import os

from kyoka.utils import build_not_implemented_msg, pickle_data, unpickle_data


class BaseActionValueFunction(object):
    """Base class of tabular and approximation action value function.

    The responsibility of action value function is to impelement two methods
    - predict_value
    - backup
    """

    def predict_value(self, state, action):
        """Predict the value of passed state-action pair.
        Returns:
            value: predicted value of passed state-action pair
        """
        err_msg = build_not_implemented_msg(self, "predict_value")
        raise NotImplementedError(err_msg)

    def backup(self, state, action, backup_target, alpha):
        """Update the value of passed state-action pair
        Args:
            state : state of state-action pair to update the value
            action : action of state-action pair to update the value
            backup_target : update the value by using this target which created from RL algorithm
            alpha : learning parameter passed from RL algorithm
        """
        err_msg = build_not_implemented_msg(self, "backup")
        raise NotImplementedError(err_msg)

    def setup(self):
      pass

    def save(self, save_dir_path):
      pass

    def load(self, load_dir_path):
      pass


class BaseTabularActionValueFunction(BaseActionValueFunction):
    """Base class of tabular value function used in RL algorithms

    property:
        table : the table to store the values. This property is initialized
                in setup method by using "generate_initial_table" method.
    """

    BASE_SAVE_FILE_NAME = "table_action_value_function_data.pickle"

    def generate_initial_table(self):
        """Initialize table to store the values of state-action pairs.
        Returns:
            table: this table is passed to "fetch_value_from_table" and
                   "insert_value_into_table" methods.
        """
        err_msg = build_not_implemented_msg(self, "generate_initial_table")
        raise NotImplementedError(err_msg)

    def fetch_value_from_table(self, table, state, action):
        """Define how to fetch the value of state-action pair from table.
        Args:
            table : current table object which initialzed by "generate_initial_table" method
            state : state of state-action pair to fetch the value from table
            action : action of state-action pair to fetch the value from table
        Returns:
            value : the value of state-action pair fetched from the table
        """
        err_msg = build_not_implemented_msg(self, "fetch_value_from_table")
        raise NotImplementedError(err_msg)

    def insert_value_into_table(self, table, state, action, new_value):
        """how to insert the new_item into table indexed by state-action pair

        This method directly update passed table by inserting new_value.
        (so thie method causes side-effect through table object)

        Args:
            table : table to insert the value (initialized by "generate_initial_table" method)
            state : state of state-action pair to index where to insert the new_value
            action: action of state-action pair to index where to insert the new_value
            new_value : new_value to insert into the table
        Returns:
            nothing : because directly update passed table object
        """
        err_msg = build_not_implemented_msg(self, "insert_value_into_table")
        raise NotImplementedError(err_msg)

    def define_save_file_prefix(self):
        """
        If you return "boo" then "self.save("some_dir")" will create
        "some_dir/boo_table_action_value_function_data.pickle"
        """
        return ""

    def setup(self):
        self.table = self.generate_initial_table()

    def predict_value(self, state, action):
        return self.fetch_value_from_table(self.table, state, action)

    def save(self, save_dir_path):
        pickle_data(self._gen_table_data_file_path(save_dir_path), self.table)

    def load(self, load_dir_path):
        file_path = self._gen_table_data_file_path(load_dir_path)
        if not os.path.exists(file_path):
            raise IOError('The saved data of "TableActionValueFunction"' +
                          ' is not found on [ %s ]' % load_dir_path)
        self.table = unpickle_data(file_path)

    def _gen_table_data_file_path(self, dir_path):
        return os.path.join(dir_path, self._gen_table_data_file_name())

    def _gen_table_data_file_name(self):
        prefix = self.define_save_file_prefix()
        if len(prefix) != 0: prefix += "_"
        return prefix + self.BASE_SAVE_FILE_NAME


class BaseApproxActionValueFunction(BaseActionValueFunction):
    """Base class of approximation value function

    Child class needs to implement following 3 methods for
    "predict_value" and "backup" which is necessary for action value function.

    - construct_features : transform state to feature representation
    - approx_predict_value : predict value by using feature representation
    - approx_backup : backup target valie by using feature representation
    """

    def predict_value(self, state, action):
        return self.approx_predict_value(self.construct_features(state, action))

    def backup(self, state, action, backup_target, alpha):
        self.approx_backup(self.construct_features(state, action), backup_target, alpha)

    def construct_features(self, state, action):
        """Transform state to compact feature representation
        Args:
            state: state of state-action pair to transform
            action: action of state-action pair to transform
        Returns:
            features: features which represents passed state-action pair
        """
        err_msg = build_not_implemented_msg(self, "construct_features")
        raise NotImplementedError(err_msg)

    def approx_predict_value(self, features):
        """Predict value by using feature representation of state-action pair
        Args:
            features: transformed by "construct_features" method
        Returns:
            value : predict the value of state-action pair by using features
        """
        err_msg = build_not_implemented_msg(self, "approx_predict_value")
        raise NotImplementedError(err_msg)

    def approx_backup(self, features, backup_target, alpha):
        """Update value by using feature representation of state-action pair
        Args:
            features: transformed by "construct_features" method
            backup_target : update the value by using this target which created from RL algorithm
            alpha : learning parameter passed from RL algorithm
        """
        err_msg = build_not_implemented_msg(self, "approx_backup")
        raise NotImplementedError(err_msg)


1	import os	1×
2
3	from kyoka.utils import build_not_implemented_msg, pickle_data, unpickle_data	1×
4
5
6	class BaseActionValueFunction(object):	1×
7	"""Base class of tabular and approximation action value function.
8
9	The responsibility of action value function is to impelement two methods
10	- predict_value
11	- backup
12	"""
13
14	def predict_value(self, state, action):	1×
15	"""Predict the value of passed state-action pair.
16	Returns:
17	value: predicted value of passed state-action pair
18	"""
19	err_msg = build_not_implemented_msg(self, "predict_value")	!
20	raise NotImplementedError(err_msg)	!
21
22	def backup(self, state, action, backup_target, alpha):	1×
23	"""Update the value of passed state-action pair
24	Args:
25	state : state of state-action pair to update the value
26	action : action of state-action pair to update the value
27	backup_target : update the value by using this target which created from RL algorithm
28	alpha : learning parameter passed from RL algorithm
29	"""
30	err_msg = build_not_implemented_msg(self, "backup")	1×
31	raise NotImplementedError(err_msg)	1×
32
33	def setup(self):	1×
34	pass	!
35
36	def save(self, save_dir_path):	1×
37	pass	!
38
39	def load(self, load_dir_path):	1×
40	pass	!
41
42
43	class BaseTabularActionValueFunction(BaseActionValueFunction):	1×
44	"""Base class of tabular value function used in RL algorithms
45
46	property:
47	table : the table to store the values. This property is initialized
48	in setup method by using "generate_initial_table" method.
49	"""
50
51	BASE_SAVE_FILE_NAME = "table_action_value_function_data.pickle"	1×
52
53	def generate_initial_table(self):	1×
54	"""Initialize table to store the values of state-action pairs.
55	Returns:
56	table: this table is passed to "fetch_value_from_table" and
57	"insert_value_into_table" methods.
58	"""
59	err_msg = build_not_implemented_msg(self, "generate_initial_table")	1×
60	raise NotImplementedError(err_msg)	1×
61
62	def fetch_value_from_table(self, table, state, action):	1×
63	"""Define how to fetch the value of state-action pair from table.
64	Args:
65	table : current table object which initialzed by "generate_initial_table" method
66	state : state of state-action pair to fetch the value from table
67	action : action of state-action pair to fetch the value from table
68	Returns:
69	value : the value of state-action pair fetched from the table
70	"""
71	err_msg = build_not_implemented_msg(self, "fetch_value_from_table")	1×
72	raise NotImplementedError(err_msg)	1×
73
74	def insert_value_into_table(self, table, state, action, new_value):	1×
75	"""how to insert the new_item into table indexed by state-action pair
76
77	This method directly update passed table by inserting new_value.
78	(so thie method causes side-effect through table object)
79
80	Args:
81	table : table to insert the value (initialized by "generate_initial_table" method)
82	state : state of state-action pair to index where to insert the new_value
83	action: action of state-action pair to index where to insert the new_value
84	new_value : new_value to insert into the table
85	Returns:
86	nothing : because directly update passed table object
87	"""
88	err_msg = build_not_implemented_msg(self, "insert_value_into_table")	!
89	raise NotImplementedError(err_msg)	!
90
91	def define_save_file_prefix(self):	1×
92	"""
93	If you return "boo" then "self.save("some_dir")" will create
94	"some_dir/boo_table_action_value_function_data.pickle"
95	"""
UNCOV 96	return ""	!
97
98	def setup(self):	1×
99	self.table = self.generate_initial_table()	1×
100
101	def predict_value(self, state, action):	1×
102	return self.fetch_value_from_table(self.table, state, action)	1×
103
104	def save(self, save_dir_path):	1×
105	pickle_data(self._gen_table_data_file_path(save_dir_path), self.table)	1×
106
107	def load(self, load_dir_path):	1×
108	file_path = self._gen_table_data_file_path(load_dir_path)	1×
109	if not os.path.exists(file_path):	1×
NEW 110	raise IOError('The saved data of "TableActionValueFunction"' +	!
111	' is not found on [ %s ]' % load_dir_path)
112	self.table = unpickle_data(file_path)	1×
113
114	def _gen_table_data_file_path(self, dir_path):	1×
115	return os.path.join(dir_path, self._gen_table_data_file_name())	1×
116
117	def _gen_table_data_file_name(self):	1×
118	prefix = self.define_save_file_prefix()	1×
119	if len(prefix) != 0: prefix += "_"	1×
120	return prefix + self.BASE_SAVE_FILE_NAME	1×
121
122
123	class BaseApproxActionValueFunction(BaseActionValueFunction):	1×
124	"""Base class of approximation value function
125
126	Child class needs to implement following 3 methods for
127	"predict_value" and "backup" which is necessary for action value function.
128
129	- construct_features : transform state to feature representation
130	- approx_predict_value : predict value by using feature representation
131	- approx_backup : backup target valie by using feature representation
132	"""
133
134	def predict_value(self, state, action):	1×
135	return self.approx_predict_value(self.construct_features(state, action))	1×
136
137	def backup(self, state, action, backup_target, alpha):	1×
138	self.approx_backup(self.construct_features(state, action), backup_target, alpha)	1×
139
140	def construct_features(self, state, action):	1×
141	"""Transform state to compact feature representation
142	Args:
143	state: state of state-action pair to transform
144	action: action of state-action pair to transform
145	Returns:
146	features: features which represents passed state-action pair
147	"""
148	err_msg = build_not_implemented_msg(self, "construct_features")	1×
149	raise NotImplementedError(err_msg)	1×
150
151	def approx_predict_value(self, features):	1×
152	"""Predict value by using feature representation of state-action pair
153	Args:
154	features: transformed by "construct_features" method
155	Returns:
156	value : predict the value of state-action pair by using features
157	"""
158	err_msg = build_not_implemented_msg(self, "approx_predict_value")	1×
159	raise NotImplementedError(err_msg)	1×
160
161	def approx_backup(self, features, backup_target, alpha):	1×
162	"""Update value by using feature representation of state-action pair
163	Args:
164	features: transformed by "construct_features" method
165	backup_target : update the value by using this target which created from RL algorithm
166	alpha : learning parameter passed from RL algorithm
167	"""
168	err_msg = build_not_implemented_msg(self, "approx_backup")	1×
169	raise NotImplementedError(err_msg)	1×
170

ishikota / kyoka / 130

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous