datamllab / rlcard / 240

Build # 240

Build Type

push

travis-ci

Committed by Daochen

Commit Message

<a href="https://github.com/datamllab/rlcard/commit/<a class=hub.com/datamllab/rlcard/commit/<a class="double-link" href="https://git"><a class=hub.com/datamllab/rlcard/commit/<a class="double-link" href="https://git"><a class=hub.com/datamllab/rlcard/commit/<a class="double-link" href="https://git"><a class=hub.com/datamllab/rlcard/commit/<a class="double-link" href="https://git"><a class=hub.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024">02a537bd2<a href="https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024">&lt;a href=&quot;https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024">&lt;a href=&quot;https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024&amp;amp;quot;&amp;amp;gt;&amp;amp;amp;quot;&amp;amp;amp;gt;Squashed commit of the following:

commit &amp;amp;lt;/a&amp;amp;gt;&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://github.com/datamllab/rlcard/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://github.com/datamllab/rlcard/commit/32abdb3705e677688190394f7766e7fcbdb7556a&amp;amp;quot;&amp;amp;gt;32abdb370&amp;amp;lt;/a&amp;amp;gt;&amp;amp;quot;&amp;amp;gt;32abdb370&amp;amp;lt;/a&amp;amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024&amp;quot;&amp;gt;
Merge: 18b65b1 d462bf3
Author: songyih &amp;amp;lt;songyih1996@gmail.com&amp;amp;gt;
Date:   Tue Oct 8 18:52:36 2019 -0700

    Merge branch &amp;amp;#39;dev&amp;amp;#39; of https://github.com/datamllab/rlcard into dev

commit &amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/datamllab/rlcard/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/datamllab/rlcard/commit/d462bf35ba46d982f803bcefdeaf1476c8d96c75&amp;quot;&gt;d462bf35b&lt;/a&gt;&quot;&gt;d462bf35b&lt;/a&gt;&lt;a href=&quot;https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024&quot;&gt;
Author: Kwei-Herng Lai &amp;lt;khlai037@datalab3.engr.tamu.edu&amp;gt;
Date:   Tue Oct 8 18:51:59 2019 -0700

    mahjong (fix table)

commit &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/datamllab/rlcard/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/datamllab/rlcard/commit/18b65b18334df5f8442aceff9123374505ddfcb1&quot;&gt;18b65b183&lt;/a&gt;&quot;&gt;18b65b183&lt;/a&gt;&lt;a href=&quot;https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024">
Author: songyih &lt;songyih1996@gmail.com&gt;
Date:   Tue Oct 8 18:50:50 2019 -0700

    setup.py for pip hosting

commit </a><a class="double-link" href="https://github.com/datamllab/rlcard/commit/<a class="double-link" href="https://github.com/datamllab/rlcard/commit/f54c6e3812267646dcbbee1fe427319a107c6153">f54c6e381</a>">f54c6e381</a><a href="https://github.com/datamllab/rlcard/commit/02a537bd2753a1e5a6ca32c28a2eb3b827fb0024">
Author: Kwei-Herng Lai &lt;khlai037@datalab3.engr.tamu.edu&gt;
Date:   Tue Oct 8 18:50:38 2019 -0700

    mahjong (table)

commit </a><a class="double-link" href="https://github.com/datamllab/rlcard/commit/bfd6e149a">bfd6e149a
Author: songyih <songyih1996@gmail.com>
Date:   Tue Oct 8 18:49:38 2019 -0700

    setup.py for pip hosting

commit 
97.37

/rlcard/agents/cfr_agent.py

import numpy as np
import collections

from rlcard.utils.utils import *

class CFRAgent():
    ''' Implement CFR algorithm
    '''

    def __init__(self, env):
        ''' Initilize Agent
        
        Args:
            env (Env): Env class
        '''
        self.env = env
        
        # A policy is a dict state_str -> action probabilities
        self.policy = collections.defaultdict(list)
        self.average_policy =collections.defaultdict(np.array)

        # Regret is a dict state_str -> action regrets
        self.regrets = collections.defaultdict(np.array)

        self.iteration = 0

    def train(self):
        ''' Do one iteration of CFR
        '''
        self.iteration += 1
        # Firstly, tranvers tree to compute counterfactual regret for each player
        # The regrets are recorded in traversal
        for player_id in range(self.env.player_num):
            self.env.init_game()
            probs = np.ones(self.env.player_num)
            self.traverse_tree(probs, player_id)

        # Update policy
        self.update_policy()

    def traverse_tree(self, probs, player_id):
        ''' Traverse the game tree, update the regrets

        Args:
            probs: The reach probability of the current node
            player_id: The player to update the value

        Returns:
            state_utilities (list): The expected utilities for all the players
        '''
        if self.env.is_over():
            return self.env.get_payoffs()

        current_player = self.env.get_player_id()

        action_utilities = {}
        state_utility = np.zeros(self.env.player_num)
        obs, legal_actions = self.get_state(current_player)
        action_probs = self.action_probs(obs, legal_actions, self.policy)

        for action in legal_actions:
            action_prob = action_probs[action]
            new_probs = probs.copy()
            new_probs[current_player] *= action_prob

            # Keep traversing the child state
            self.env.step(action)
            utility = self.traverse_tree(new_probs, player_id)
            self.env.step_back()

            state_utility += action_prob * utility
            action_utilities[action] = utility

        if not current_player == player_id:
            return state_utility

        # If it is current player, we record the policy and compute regret
        player_prob = probs[current_player]
        counterfactual_prob = (np.prod(probs[:current_player]) *
                                np.prod(probs[current_player + 1:]))
        player_state_utility = state_utility[current_player]

        if obs not in self.regrets:
            self.regrets[obs] = np.zeros(self.env.action_num)
        if obs not in self.average_policy:
            self.average_policy[obs] = np.zeros(self.env.action_num)
        for action in legal_actions:
            action_prob = action_probs[action]
            regret = counterfactual_prob * (action_utilities[action][current_player]
                    - player_state_utility)
            self.regrets[obs][action] += regret
            self.average_policy[obs][action] += self.iteration * player_prob * action_prob
        return state_utility

    def update_policy(self):
        ''' Update policy based on the current regrets
        '''
        for obs in self.regrets:
            self.policy[obs] = self.regret_matching(obs)

    def regret_matching(self, obs):
        ''' Apply regret matching
        
        Args:
            obs (string): The state_str
        '''
        regret = self.regrets[obs]
        positive_regret_sum = sum([r for r in regret if r > 0])

        action_probs = np.zeros(self.env.action_num)
        if positive_regret_sum > 0:
            for action in range(self.env.action_num):
                action_probs[action] = max(0.0, regret[action] / positive_regret_sum)
        else:
            for action in range(self.env.action_num):
                action_probs[action] = 1.0 / self.env.action_num

        return action_probs

    def action_probs(self, obs, legal_actions, policy):
        ''' Obtain the action probabilities of the current state

        Args:
            obs (str): state_str
            legal_actions (list): List of leagel actions
            player_id (int): The current player
            policy (dict): The used policy

        Returns:
            (tuple) that contains:
                action_probs(numpy.array): The action probabilities
                legal_actions (list): Indices of legal actions
        '''
        if obs not in policy:
            action_probs = np.array([1.0/self.env.action_num for _ in range(self.env.action_num)])
            self.policy[obs] = action_probs
        else:
            action_probs = policy[obs]
        action_probs = remove_illegal(action_probs, legal_actions)
        return action_probs

    def eval_step(self, state):
        ''' Given a state, predict action based on average policy

        Args:
            state (numpy.array): State representation

        Returns:
            action (int): Predicted action
        '''
        probs = self.action_probs(state['obs'].tostring(), state['legal_actions'], self.average_policy)
        action = np.random.choice(len(probs), p=probs)
        return action

    def get_state(self, player_id):
        ''' Get state_str of the player

        Args:
            player_id (int): The player id

        Returns:
            (tuple) that contains:
                state (str): The state str
                legal_actions (list): Indices of legal actions
        '''
        state = self.env.get_state(player_id)
        return state['obs'].tostring(), state['legal_actions']

1	import numpy as np	1✔
2	import collections	1✔
3
4	from rlcard.utils.utils import *	1✔
5
6	class CFRAgent():	1✔
7	''' Implement CFR algorithm
8	'''
9
10	def __init__(self, env):	1✔
11	''' Initilize Agent
12
13	Args:
14	env (Env): Env class
15	'''
16	self.env = env	1✔
17
18	# A policy is a dict state_str -> action probabilities
19	self.policy = collections.defaultdict(list)	1✔
20	self.average_policy =collections.defaultdict(np.array)	1✔
21
22	# Regret is a dict state_str -> action regrets
23	self.regrets = collections.defaultdict(np.array)	1✔
24
25	self.iteration = 0	1✔
26
27	def train(self):	1✔
28	''' Do one iteration of CFR
29	'''
30	self.iteration += 1	1✔
31	# Firstly, tranvers tree to compute counterfactual regret for each player
32	# The regrets are recorded in traversal
33	for player_id in range(self.env.player_num):	1✔
34	self.env.init_game()	1✔
35	probs = np.ones(self.env.player_num)	1✔
36	self.traverse_tree(probs, player_id)	1✔
37
38	# Update policy
39	self.update_policy()	1✔
40
41	def traverse_tree(self, probs, player_id):	1✔
42	''' Traverse the game tree, update the regrets
43
44	Args:
45	probs: The reach probability of the current node
46	player_id: The player to update the value
47
48	Returns:
49	state_utilities (list): The expected utilities for all the players
50	'''
51	if self.env.is_over():	1✔
52	return self.env.get_payoffs()	1✔
53
54	current_player = self.env.get_player_id()	1✔
55
56	action_utilities = {}	1✔
57	state_utility = np.zeros(self.env.player_num)	1✔
58	obs, legal_actions = self.get_state(current_player)	1✔
59	action_probs = self.action_probs(obs, legal_actions, self.policy)	1✔
60
61	for action in legal_actions:	1✔
62	action_prob = action_probs[action]	1✔
63	new_probs = probs.copy()	1✔
64	new_probs[current_player] *= action_prob	1✔
65
66	# Keep traversing the child state
67	self.env.step(action)	1✔
68	utility = self.traverse_tree(new_probs, player_id)	1✔
69	self.env.step_back()	1✔
70
71	state_utility += action_prob * utility	1✔
72	action_utilities[action] = utility	1✔
73
74	if not current_player == player_id:	1✔
75	return state_utility	1✔
76
77	# If it is current player, we record the policy and compute regret
78	player_prob = probs[current_player]	1✔
79	counterfactual_prob = (np.prod(probs[:current_player]) *	1✔
80	np.prod(probs[current_player + 1:]))
81	player_state_utility = state_utility[current_player]	1✔
82
83	if obs not in self.regrets:	1✔
84	self.regrets[obs] = np.zeros(self.env.action_num)	1✔
85	if obs not in self.average_policy:	1✔
86	self.average_policy[obs] = np.zeros(self.env.action_num)	1✔
87	for action in legal_actions:	1✔
88	action_prob = action_probs[action]	1✔
89	regret = counterfactual_prob * (action_utilities[action][current_player]	1✔
90	- player_state_utility)
91	self.regrets[obs][action] += regret	1✔
92	self.average_policy[obs][action] += self.iteration * player_prob * action_prob	1✔
93	return state_utility	1✔
94
95	def update_policy(self):	1✔
96	''' Update policy based on the current regrets
97	'''
98	for obs in self.regrets:	1✔
99	self.policy[obs] = self.regret_matching(obs)	1✔
100
101	def regret_matching(self, obs):	1✔
102	''' Apply regret matching
103
104	Args:
105	obs (string): The state_str
106	'''
107	regret = self.regrets[obs]	1✔
108	positive_regret_sum = sum([r for r in regret if r > 0])	1✔
109
110	action_probs = np.zeros(self.env.action_num)	1✔
111	if positive_regret_sum > 0:	1✔
112	for action in range(self.env.action_num):	1✔
113	action_probs[action] = max(0.0, regret[action] / positive_regret_sum)	1✔
114	else:
115	for action in range(self.env.action_num):	×
116	action_probs[action] = 1.0 / self.env.action_num	×
117
118	return action_probs	1✔
119
120	def action_probs(self, obs, legal_actions, policy):	1✔
121	''' Obtain the action probabilities of the current state
122
123	Args:
124	obs (str): state_str
125	legal_actions (list): List of leagel actions
126	player_id (int): The current player
127	policy (dict): The used policy
128
129	Returns:
130	(tuple) that contains:
131	action_probs(numpy.array): The action probabilities
132	legal_actions (list): Indices of legal actions
133	'''
134	if obs not in policy:	1✔
135	action_probs = np.array([1.0/self.env.action_num for _ in range(self.env.action_num)])	1✔
136	self.policy[obs] = action_probs	1✔
137	else:
138	action_probs = policy[obs]	1✔
139	action_probs = remove_illegal(action_probs, legal_actions)	1✔
140	return action_probs	1✔
141
142	def eval_step(self, state):	1✔
143	''' Given a state, predict action based on average policy
144
145	Args:
146	state (numpy.array): State representation
147
148	Returns:
149	action (int): Predicted action
150	'''
151	probs = self.action_probs(state['obs'].tostring(), state['legal_actions'], self.average_policy)	1✔
152	action = np.random.choice(len(probs), p=probs)	1✔
153	return action	1✔
154
155	def get_state(self, player_id):	1✔
156	''' Get state_str of the player
157
158	Args:
159	player_id (int): The player id
160
161	Returns:
162	(tuple) that contains:
163	state (str): The state str
164	legal_actions (list): Indices of legal actions
165	'''
166	state = self.env.get_state(player_id)	1✔
167	return state['obs'].tostring(), state['legal_actions']	1✔