这里通过zoe走迷宫例子再次学习Q-learning。与强化学习系列(1)中思想一致，其区别主要是通过两个类，迷宫环境Maze和zoe大脑QLearningTable来规范化程序，同时在运行函数步骤来清晰化Q学习的过程。

Part 1. 迷宫环境class Maze

import numpy as np 
import time 
import sys# 窗口界面库
if sys.version_info.major == 2:import Tkinter as tk 
else:import tkinter as tkUNIT = 40 # 像素
MAZE_H = 4 # 网格高度
MAZE_W = 4 # 网格宽度class Maze(tk.Tk, object): # 新类继承父类tk.Tkdef __init__(self):super(Maze, self).__init__() # super类继承方法,初始化self.action_space = ['u', 'd', 'l', 'r']self.n_actions = len(self.action_space)self.title('maze')self.geometry('{0}x{1}'.format(MAZE_H * UNIT, MAZE_H * UNIT)) # format格式化函数 geometry分辨率函数self._build_maze()# 迷宫布景def _build_maze(self):self.canvas = tk.Canvas(self, bg='white', # tk画图组件height = MAZE_H * UNIT,width = MAZE_W * UNIT)for c in range(0, MAZE_W * UNIT, UNIT): # 网格x0, y0, x1, y1 = c, 0, c, MAZE_H * UNITself.canvas.create_line(x0, y0, x1, y1)for r in range(0, MAZE_H * UNIT, UNIT):x0, y0, x1, y1 = 0, r, MAZE_W * UNIT, rself.canvas.create_line(x0, y0, x1, y1)# 原点origin = np.array([20, 20])# hell 地狱hell1_center = origin + np.array([UNIT * 2, UNIT])self.hell1 = self.canvas.create_rectangle(hell1_center[0] - 15, hell1_center[1] - 15, # [5, 5]hell1_center[0] + 15, hell1_center[1] + 15, # [35, 35]fill='black')hell2_center = origin + np.array([UNIT, UNIT * 2])self.hell2 = self.canvas.create_rectangle(hell2_center[0] - 15, hell2_center[1] - 15,hell2_center[0] + 15, hell2_center[1] + 15,fill='black')# oval 宝藏oval_center = origin + UNIT * 2self.oval = self.canvas.create_oval(oval_center[0] - 15, oval_center[1] - 15,oval_center[0] + 15, oval_center[1] + 15,fill='yellow')# rect zoeself.rect = self.canvas.create_rectangle(origin[0] - 15, origin[1] - 15,origin[0] + 15, origin[1] + 15,fill='red')self.canvas.pack() # 打包#return(self.canvas)# zoe复位def reset(self):self.update()time.sleep(0.5)self.canvas.delete(self.rect)origin = np.array([20, 20])self.rect = self.canvas.create_rectangle(origin[0] - 15, origin[1] - 15,origin[0] + 15, origin[1] + 15,fill='red')return self.canvas.coords(self.rect)# 生成下一状态和奖惩机制策略def step(self, action):s = self.canvas.coords(self.rect)base_action = np.array([0, 0])if action == 0: # upif s[1] > UNIT:base_action[1] -= UNITelif action == 1: # downif s[1] < (MAZE_H - 1) * UNIT:base_action[1] += UNITelif action == 2: # rightif s[0] < (MAZE_W - 1) * UNIT:base_action[0] += UNITelif action == 3: # leftif s[0] > UNIT:base_action[0] -= UNITself.canvas.move(self.rect, base_action[0], base_action[1])s_ = self.canvas.coords(self.rect)# 奖惩判断if s_ == self.canvas.coords(self.oval):reward = 1done = Trues_ = 'terminal'elif s_ in [self.canvas.coords(self.hell1), self.canvas.coords(self.hell2)]:reward = -1done = Trues_ = 'terminal'else:reward = 0done = Falsereturn s_, reward, donedef render(self):time.sleep(0.1)self.update()# --------------------------------------------------------
# 主函数：程序入口
"""
if __name__ == "__main__":env = Maze()env.after(100, update)env.mainloop()"""

Part 2. zoe大脑class QLearningTable

import numpy as np 
import pandas as pd class QLearningTable:def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):self.actions = actionsself.lr = learning_rateself.gamma = reward_decayself.epsilon = e_greedyself.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64)# 根据当前观测状态 s 选择动作 adef choose_action(self, observation):self.check_state_exist(observation)if np.random.uniform() < self.epsilon:state_action = self.q_table.loc[observation, :]action = np.random.choice(state_action[state_action == np.max(state_action)].index)else:action = np.random.choice(self.actions)return action# 得到（s, a, r, s_）序列后进行状态动作值更新def learn(self, s, a, r, s_):self.check_state_exist(s_)q_predict = self.q_table.loc[s, a]if s_ != 'terminal':q_target = r + self.gamma * self.q_table.loc[s_, :].max()else:q_target = rself.q_table.loc[s, a] += self.lr * (q_target - q_predict)def check_state_exist(self, state):if state not in self.q_table.index:self.q_table = self.q_table.append(pd.Series([0]*len(self.actions),index=self.q_table.columns,name=state))

Part 3. 运行函数

from maze_env import Maze 
from RL_brain import QLearningTabledef update():for episode in range(100):observation = env.reset()# print(str(observation))step_counter = 0while True:step_counter += 1env.render()action = RL.choose_action(str(observation))# print(str(observation))observation_, reward, done = env.step(action)RL.learn(str(observation), action, reward, str(observation_))observation = observation_if done:breakprint('Episode %s: total_steps = %s' % (episode, step_counter))print('game over')env.destroy()# --------------------------------------------------
# 程序入口，main函数
if __name__ == "__main__":env = Maze()RL = QLearningTable(actions=list(range(env.n_actions)))env.after(100, update)env.mainloop()

Part 4. 演示效果