Repository: morningsky/NTU-ReinforcementLearning-Notes Branch: master Commit: d4a9dbf584ae Files: 21 Total size: 152.5 KB Directory structure: gitextract_v2chlczr/ ├── README.md ├── code/ │ ├── actor_critic_advantage.py │ ├── ddpg_update.py │ ├── deep_deterministic_policy_gradient.py │ ├── policy_gradient.py │ ├── proximal_policy_optimization.py │ └── tensrolayer-implemented/ │ ├── a3c.py │ ├── ac.py │ ├── ddpg.py │ ├── dqn.py │ ├── dqn_variants.py │ ├── pg.py │ ├── ppo.py │ ├── qlearning.py │ └── tutorial_wrappers.py └── notes/ ├── 1 Introduction.md ├── 2 Policy Gradient.md ├── 3 Q - Learning.md ├── 4 Actor Critic.md ├── 5 Sparse Reward.md └── 6 Imitation Learning.md ================================================ FILE CONTENTS ================================================ ================================================ FILE: README.md ================================================ # 李宏毅深度强化学习 笔记 ### 课程主页：[NTU-MLDS18](http://speech.ee.ntu.edu.tw/~tlkagk/courses_MLDS18.html) ### 视频： - [youtube](https://www.youtube.com/playlist?list=PLJV_el3uVTsODxQFgzMzPLa16h6B8kWM_) - [B站](https://www.bilibili.com/video/av24724071/?spm_id_from=333.788.videocard.4) ![1](http://oss.hackslog.cn/imgs/075034.png) 这门课的学习路线如上，强化学习是作为单独一个模块介绍。李宏毅老师讲这门课不是从MDP开始讲起，而是从如何获得最大化奖励出发，直接引出Policy Gradient（以及PPO），再讲Q-learning（原始Q-learning，DQN，各种DQN的升级），然后是A2C（以及A3C, DDPG），紧接着介绍了一些Reward Shaping的方法（主要是Curiosity，Curriculum Learning ，Hierarchical Learning），最后介绍Imitation Learning (Inverse RL)。比较全面的展现了深度强化学习的核心内容，也比较直观。跟伯克利学派的课类似，与UCL上来就讲MDP，解各种value iteration的思路有较大区别。 文档中的notes以对slides的批注为主，方便在阅读slides时理解，code以纯tensorflow实现，主要参考[莫凡RL教学](https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/)，修正部分代码以保持前后一致性，已经加入便于理解的注释。 ### 参考资料： [作业代码参考](https://github.com/JasonYao81000/MLDS2018SPRING/tree/master/hw4) [纯numpy实现非Deep的RL算法](https://github.com/ddbourgin/numpy-ml/tree/master/numpy_ml/rl_models) [OpenAI tutorial](https://github.com/openai/spinningup/tree/master/docs) [莫凡RL教学](https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/) - code中的tensorlayer实现来自于[Tensorlayer-RL](https://github.com/tensorlayer/tensorlayer/tree/master/examples/reinforcement_learning),比起原生tensorflow更加简洁 ================================================ FILE: code/actor_critic_advantage.py ================================================ import tensorflow as tf import numpy as np import gym ''' 比较PG算法 PG loss = log_prob * v估计 （来自贝尔曼公式） A2C loss = log_prob * TD-error（来自critic网络 表达当前动作的价值比平均动作的价值好多少） DDPG ： critic不仅能影响actor actor也能影响critic 相当于critic不仅告诉actor的行为好不好，还告诉他应该怎么改进才能更好(传一个梯度 dq/da) PPO: 对PG的更新加了限制，提高训练稳定性 相比于A2C 只是actor网络更加复杂 ''' class Actor(object): #本质还是policy gradient 不过A2C是单步更新 def __init__(self, sess, #两个网络需要共用一个session 所以外部初始化 n_actions, n_features, lr=0.01, ): #self.ep_obs, self.ep_as, self.ep_rs =[],[],[] #由于是单步更新 所以不需要存储每个episode的数据 self.sess = sess self.s = tf.placeholder(tf.float32, [1, n_features], "state") self.a = tf.placeholder(tf.int32, None, "act") # self.td_error = tf.placeholder(tf.float32, None, "td_error") # TD_error更新的幅度 td 的理解应该是 Q(s, a) - V(s), 某个动作价值减去平均动作价值 with tf.variable_scope('Actor'): #将原来的name_scope换成variable_scope ，可以在一个scope里面共享变量 l1 = tf.layers.dense( inputs=self.s, units=20, # number of hidden units activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(0., .1), # weights bias_initializer=tf.constant_initializer(0.1), # biases name='l1' ) self.acts_prob = tf.layers.dense( inputs=l1, units=n_actions, # output units activation=tf.nn.softmax, # get action probabilities kernel_initializer=tf.random_normal_initializer(0., .1), # weights bias_initializer=tf.constant_initializer(0.1), # biases name='acts_prob' ) #with tf.name_scope('loss'): # 最大化 总体 reward (log_p * R) 就是在最小化 -(log_p * R), 而 tf 的功能里只有最小化 loss #neg_log_prob = tf.reduce_sum(-tf.log(self.all_act_prob)*tf.one_hot(self.tf_acts, self.n_actions), axis=1) #加- 变为梯度下降 #loss = tf.reduce_mean(neg_log_prob * self.tf_vt) with tf.variable_scope('loss'): log_prob = tf.log(self.acts_prob[0,self.a]) #[[0.1,0.2,0.3]] -> 0.1, if a=0 self.loss = log_prob * self.td_error # advantage (TD_error) guided loss with tf.name_scope('train'): self.train_op = tf.train.AdamOptimizer(lr).minimize(-self.loss) def choose_action(self, s): #选择行为 s = s[np.newaxis, :] probs = self.sess.run(self.acts_prob, {self.s: s}) # get probabilities for all actions action = np.random.choice(np.arange(probs.shape[1]), p=probs.ravel()) return action # return a int def learn(self, s, a, td): s = s[np.newaxis, :] feed_dict = {self.s: s, self.a: a, self.td_error: td} _, loss = self.sess.run([self.train_op, self.loss], feed_dict) return loss class Critic(object): def __init__(self, sess, n_features, lr=0.01, gamma=0.9): self.sess = sess self.s = tf.placeholder(tf.float32, [1, n_features], "state") self.v_ = tf.placeholder(tf.float32, [1, 1], "v_next") self.r = tf.placeholder(tf.float32, None, 'r') with tf.variable_scope('Critic'): l1 = tf.layers.dense( inputs=self.s, units=20, # number of hidden units activation=tf.nn.relu, # None # have to be linear to make sure the convergence of actor. # But linear approximator seems hardly learns the correct Q. kernel_initializer=tf.random_normal_initializer(0., .1), # weights bias_initializer=tf.constant_initializer(0.1), # biases name='l1' ) self.v = tf.layers.dense( inputs=l1, units=1, # output units activation=None, kernel_initializer=tf.random_normal_initializer(0., .1), # weights bias_initializer=tf.constant_initializer(0.1), # biases name='V' ) with tf.variable_scope('squared_TD_error'): self.td_error = self.r + gamma * self.v_ - self.v self.loss = tf.square(self.td_error) # TD_error = (r+gamma*V_next) - V_eval with tf.variable_scope('train'): self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss) def learn(self, s, r, s_): s, s_ = s[np.newaxis, :], s_[np.newaxis, :] v_ = self.sess.run(self.v, {self.s: s_}) td_error, _ = self.sess.run([self.td_error, self.train_op], {self.s: s, self.v_: v_, self.r: r}) return td_error np.random.seed(2) tf.set_random_seed(2) # reproducible # Superparameters OUTPUT_GRAPH = False MAX_EPISODE = 100#3000 DISPLAY_REWARD_THRESHOLD = 200 # renders environment if total episode reward is greater then this threshold MAX_EP_STEPS = 1000 # maximum time step in one episode RENDER = False # rendering wastes time GAMMA = 0.9 # reward discount in TD error LR_A = 0.01 # learning rate for actor LR_C = 0.05 # learning rate for critic env = gym.make('CartPole-v0') env.seed(1) # reproducible env = env.unwrapped N_F = env.observation_space.shape[0] N_A = env.action_space.n from gym import Space sess = tf.Session() #两个网络共用一个session actor = Actor(sess, n_features=N_F, n_actions=N_A, lr=LR_A) critic = Critic(sess, n_features=N_F, lr=LR_C) # we need a good teacher, so the teacher should learn faster than the actor sess.run(tf.global_variables_initializer()) if OUTPUT_GRAPH: tf.summary.FileWriter("logs/", sess.graph) for i_episode in range(MAX_EPISODE): state = env.reset() t = 0 r_list = [] while True: if RENDER: env.render() action = actor.choose_action(state) state_, reward, done, info = env.step(action) if done: reward=-20 #最后一步的奖励 一个trick r_list.append(reward) td_error = critic.learn(state, reward, state_) actor.learn(state, action, td_error) state = state_ if done or t>= MAX_EP_STEPS: ep_rs_sum = sum(r_list) if 'running_reward' not in globals(): running_reward = ep_rs_sum else: running_reward = running_reward * 0.95 + ep_rs_sum * 0.05 if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = False # rendering print("episode:", i_episode, " reward:", int(running_reward)) break ================================================ FILE: code/ddpg_update.py ================================================ import tensorflow as tf import numpy as np import gym import time ##################### hyper parameters #################### MAX_EPISODES = 200 MAX_EP_STEPS = 200 LR_A = 0.01 # learning rate for actor LR_C = 0.02 # learning rate for critic GAMMA = 0.9 # reward discount TAU = 0.01 # soft replacement MEMORY_CAPACITY = 10000 BATCH_SIZE = 32 RENDER = False ENV_NAME = 'Pendulum-v0' #pendulum 动作与状态都是连续空间 #动作空间：只有一维力矩 长度为1 虽然是连续值，但是有bound【-2，2】 #状态空间：一维速度，长度为3 ############################### DDPG #################################### #离线训练 单步更新 按batch更新 引入replay buffer机制 class DDPG(object): def __init__(self, a_dim, s_dim, a_bound,): #初始化2个网络图 注意无论是critic还是actor网络都有target-network机制 target-network不训练 self.memory = np.zeros((MEMORY_CAPACITY, s_dim * 2 + a_dim +1), dtype=np.float32) #借鉴replay buff机制 s*2 : s, s_ self.pointer = 0 self.sess = tf.Session() self.a_dim, self.s_dim, self.a_bound = a_dim, s_dim, a_bound self.S = tf.placeholder(tf.float32, [None, s_dim], 's') #前面的None用来给batch size占位 self.S_ = tf.placeholder(tf.float32, [None, s_dim], 's_') self.R = tf.placeholder(tf.float32, [None,1], 'r') with tf.variable_scope('Actor'): self.a = self._build_a(self.S, scope='eval', trainable=True) #要训练的pi网络，也负责收集数据 # input s, output a a_ = self._build_a(self.S, scope='target', trainable=False) #target网络不训练，只负责输出动作给critic # input s_, output a, get a_ for critic with tf.variable_scope('Critic'): q = self._build_c(self.S, self.a, scope='eval', trainable=True) #要训练的Q， 与target输出的q算mse（td-error） 注意这个a来自于memory q_ = self._build_c(self.S_, a_, scope='target', trainable=False) #这个网络不训练, 用于给出 Actor 更新参数时的 Gradient ascent 强度 即dq/da 注意这个a来自于actor要更新参数时候的a # networks parameters self.ae_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/eval') self.at_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/target') self.ce_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval') self.ct_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target') #taget 网络更新 即从eval网络中复制参数 self.soft_replace = [tf.assign(t, (1-TAU)*t + TAU *e) for t, e in zip(self.at_params+self.ct_params,self.ae_params+self.ce_params)] #训练critic网络（eval） q_target = self.R + GAMMA * q_ #贝尔曼公式（里面的q_来自于Q-target网络输入(s_，a_)的输出） 得出q的”真实值“ 与预测值求mse td_error = tf.losses.mean_squared_error(labels=q_target, predictions=q) #预测值q 来自于q-eval网络输入当前时刻的(s,a)的输出 self.ctrain = tf.train.AdamOptimizer(LR_C).minimize(td_error, var_list = self.ce_params) #要train的是q-eval网络的参数 最小化mse #训练actor网络（eval） a_loss = -tf.reduce_mean(q) #maximize q self.atrain = tf.train.AdamOptimizer(LR_A).minimize(a_loss, var_list = self.ae_params) # self.sess.run(tf.global_variables_initializer()) def choose_action(self, s): s = s[np.newaxis, :] return self.sess.run(self.a, feed_dict={self.S: s})[0] # single action def learn(self): #每次学习都是先更新target网络参数 self.sess.run(self.soft_replace) indices = np.random.choice(MEMORY_CAPACITY, size=BATCH_SIZE) bt = self.memory[indices, : ] #从memory中取一个batch的数据来训练 bs = bt[:, :self.s_dim] #a batch of state ba = bt[:, self.s_dim: self.s_dim + self.a_dim] #a batch of action br = bt[:, -self.s_dim - 1: -self.s_dim] #a batch of reward bs_ = bt[:, -self.s_dim:] #一次训练一个batch 这一个batch的训练过程中target网络相当于固定不动 self.sess.run(self.atrain, {self.S: bs}) self.sess.run(self.ctrain, {self.S: bs, self.a: ba, self.R: br, self.S_: bs_}) def store_transition(self, s, a, r, s_): #离线训练算法标准操作 transition = np.hstack((s, a, [r], s_)) index = self.pointer % MEMORY_CAPACITY # replace the old memory with new memory self.memory[index, :] = transition self.pointer += 1 def _build_a(self, s, scope, trainable): #actor网络结构 直接输出动作确定a with tf.variable_scope(scope): net = tf.layers.dense(s, 30, activation=tf.nn.relu, name='l1', trainable=trainable) a = tf.layers.dense(net, self.a_dim, activation=tf.nn.tanh, name='a', trainable=trainable) #a经过了tanh 数值缩放到了【-1，1】 return tf.multiply(a, self.a_bound, name='scaled_a') #输出的每个a值都乘边界[max,] 可以保证输出范围在【-max，max】 如果最小 最大值不是相反数 得用clip正则化 def _build_c(self, s, a, scope, trainable): #critic网络结构 输出Q(s,a) with tf.variable_scope(scope): n_l1 = 30 w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], trainable=trainable) w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], trainable=trainable) b1 = tf.get_variable('b1', [1, n_l1], trainable=trainable) net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1) return tf.layers.dense(net, 1, trainable=trainable) # Q(s,a) env = gym.make(ENV_NAME) env = env.unwrapped env.seed(1) s_dim = env.observation_space.shape[0] a_dim = env.action_space.shape[0] a_bound = env.action_space.high ddpg = DDPG(a_dim, s_dim, a_bound) var = 3 # control exploration t1 = time.time() for i in range(MAX_EPISODES): s = env.reset() ep_reward = 0 for j in range(MAX_EP_STEPS): #没有明确停止条件的游戏都需要这么一个 if RENDER: env.render() a = ddpg.choose_action(s) a = np.clip(np.random.normal(a, var),-2,2) #增加exploration noise 以actor输出的a为均值，var为方差进行选择a 同时保证a的值在【-2，2】 s_, r, done, info = env.step(a) ddpg.store_transition(s, a, r/10, s_) if ddpg.pointer > MEMORY_CAPACITY: #存储的数据满了开始训练各个网络 var *= 0.9995 #降低动作选择的随机性 ddpg.learn() #超过10000才开始训练，每次从经验库中抽取一个batch，每走一步都会执行一次训练 单步更新 s = s_ ep_reward += r if j == MAX_EP_STEPS-1: print('Episode:', i, ' Reward: %i' % int(ep_reward), 'Explore: %.2f' % var, ) # if ep_reward > -300:RENDER = True break print('Running time: ', time.time() - t1) ================================================ FILE: code/deep_deterministic_policy_gradient.py ================================================ import tensorflow as tf import numpy as np import gym import time np.random.seed(1) tf.set_random_seed(1) MAX_EPISODES = 200 MAX_EP_STEPS = 200 LR_A = 0.001 # learning rate for actor LR_C = 0.001 # learning rate for critic GAMMA = 0.9 # reward discount REPLACEMENT = [ dict(name='soft', tau=0.01), dict(name='hard', rep_iter_a=600, rep_iter_c=500) ][0] # you can try different target replacement strategies MEMORY_CAPACITY = 10000 BATCH_SIZE = 32 RENDER = False OUTPUT_GRAPH = True ENV_NAME = 'Pendulum-v0' class Actor(object): def __init__(self, sess, action_dim, action_bound, learning_rate, replacement): self.sess = sess self.a_dim = action_dim self.action_bound = action_bound self.lr = learning_rate self.replacement = replacement self.t_replace_counter = 0 with tf.variable_scope('Actor'): # 这个网络用于及时更新参数 self.a = self._build_net(S, scope='eval_net', trainable=True) #由target网络给出确定的action #对比ppo 网络的输出是一个概率分布 #pi, pi_params = self._build_anet('pi', trainable=True) #self.sample_op = tf.squeeze(pi.sample(1), axis=0) #按概率分布pi选择一个action # 这个网络不及时更新参数, 用于预测 Critic 的 Q_target 中的 action self.a_ = self._build_net(S_, scope='target_net', trainable=False) self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/eval_net') self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/target_net') if self.replacement['name'] == 'hard': self.t_replace_counter = 0 self.hard_replace = [tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)] else: self.soft_replace = [tf.assign(t, (1 - self.replacement['tau']) * t + self.replacement['tau'] * e) for t, e in zip(self.t_params, self.e_params)] def _build_net(self, s, scope, trainable): with tf.variable_scope(scope): init_w = tf.random_normal_initializer(0., 0.3) init_b = tf.constant_initializer(0.1) net = tf.layers.dense(s, 30, activation=tf.nn.relu, kernel_initializer=init_w, bias_initializer=init_b, name='l1', trainable=trainable) with tf.variable_scope('a'): actions = tf.layers.dense(net, self.a_dim, activation=tf.nn.tanh, kernel_initializer=init_w, bias_initializer=init_b, name='a', trainable=trainable) scaled_a = tf.multiply(actions, self.action_bound, name='scaled_a') # Scale output to -action_bound to action_bound return scaled_a def learn(self, s): # batch update self.sess.run(self.train_op, feed_dict={S: s}) if self.replacement['name'] == 'soft': self.sess.run(self.soft_replace) else: if self.t_replace_counter % self.replacement['rep_iter_a'] == 0: self.sess.run(self.hard_replace) self.t_replace_counter += 1 def choose_action(self, s): s = s[np.newaxis, :] # single state return self.sess.run(self.a, feed_dict={S: s})[0] # single action #对比ppo a = self.sess.run(self.sample_op, {self.tfs:s})[0] ## 将 critic 产出的 dQ/da 加入到 Actor 的 Graph 中去 def add_grad_to_graph(self, a_grads): with tf.variable_scope('policy_grads'): # ys = policy; # xs = policy's parameters; # a_grads = the gradients of the policy to get more Q # tf.gradients will calculate dys/dxs with a initial gradients for ys, so this is dq/da * da/dparams self.policy_grads = tf.gradients(ys=self.a, xs=self.e_params, grad_ys=a_grads) ##grad_ys 这是从 Critic 来的 dQ/da with tf.variable_scope('A_train'): opt = tf.train.AdamOptimizer(-self.lr) # (- learning rate) for ascent policy self.train_op = opt.apply_gradients(zip(self.policy_grads, self.e_params)) class Critic(object): def __init__(self, sess, state_dim, action_dim, learning_rate, gamma, replacement, a, a_): self.sess = sess self.s_dim = state_dim self.a_dim = action_dim self.lr = learning_rate self.gamma = gamma self.replacement = replacement with tf.variable_scope('Critic'): # Input (s, a), output q self.a = tf.stop_gradient(a) # stop critic update flows to actor self.q = self._build_net(S, self.a, 'eval_net', trainable=True) # Input (s_, a_), output q_ for q_target self.q_ = self._build_net(S_, a_, 'target_net', trainable=False) # target_q is based on a_ from Actor's target_net self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval_net') self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target_net') with tf.variable_scope('target_q'): self.target_q = R + self.gamma * self.q_ ## self.q_ 根据 Actor 的 target_net 来 with tf.variable_scope('TD_error'): self.loss = tf.reduce_mean(tf.squared_difference(self.target_q, self.q)) # self.q 又基于 Actor 的 target_net with tf.variable_scope('C_train'): self.train_op = tf.train.AdamOptimizer(self.lr).minimize(self.loss) with tf.variable_scope('a_grad'): self.a_grads = tf.gradients(self.q, a)[0] # tensor of gradients of each sample (None, a_dim) if self.replacement['name'] == 'hard': self.t_replace_counter = 0 self.hard_replacement = [tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)] else: self.soft_replacement = [tf.assign(t, (1 - self.replacement['tau']) * t + self.replacement['tau'] * e) for t, e in zip(self.t_params, self.e_params)] def _build_net(self, s, a, scope, trainable): with tf.variable_scope(scope): init_w = tf.random_normal_initializer(0., 0.1) init_b = tf.constant_initializer(0.1) with tf.variable_scope('l1'): n_l1 = 30 w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], initializer=init_w, trainable=trainable) w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], initializer=init_w, trainable=trainable) b1 = tf.get_variable('b1', [1, n_l1], initializer=init_b, trainable=trainable) net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1) with tf.variable_scope('q'): q = tf.layers.dense(net, 1, kernel_initializer=init_w, bias_initializer=init_b, trainable=trainable) # Q(s,a) return q def learn(self, s, a, r, s_): self.sess.run(self.train_op, feed_dict={S: s, self.a: a, R: r, S_: s_}) if self.replacement['name'] == 'soft': self.sess.run(self.soft_replacement) else: if self.t_replace_counter % self.replacement['rep_iter_c'] == 0: self.sess.run(self.hard_replacement) self.t_replace_counter += 1 class Memory(object): def __init__(self, capacity, dims): self.capacity = capacity self.data = np.zeros((capacity, dims)) self.pointer = 0 def store_transition(self, s, a, r, s_): transition = np.hstack((s, a, [r], s_)) index = self.pointer % self.capacity # replace the old memory with new memory self.data[index, :] = transition self.pointer += 1 def sample(self, n): assert self.pointer >= self.capacity, 'Memory has not been fulfilled' indices = np.random.choice(self.capacity, size=n) return self.data[indices, :] env = gym.make(ENV_NAME) env = env.unwrapped env.seed(1) state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] action_bound = env.action_space.high # all placeholder for tf with tf.name_scope('S'): S = tf.placeholder(tf.float32, shape=[None, state_dim], name='s') with tf.name_scope('R'): R = tf.placeholder(tf.float32, [None, 1], name='r') with tf.name_scope('S_'): S_ = tf.placeholder(tf.float32, shape=[None, state_dim], name='s_') sess = tf.Session() # Create actor and critic. # They are actually connected to each other, details can be seen in tensorboard or in this picture: actor = Actor(sess, action_dim, action_bound, LR_A, REPLACEMENT) critic = Critic(sess, state_dim, action_dim, LR_C, GAMMA, REPLACEMENT, actor.a, actor.a_) # 将 actor 同它的 eval_net/target_net 产生的 a/a_ 传给 Critic actor.add_grad_to_graph(critic.a_grads) # 将 critic 产出的 dQ/da 加入到 Actor 的 Graph 中去 sess.run(tf.global_variables_initializer()) M = Memory(MEMORY_CAPACITY, dims=2 * state_dim + action_dim + 1) if OUTPUT_GRAPH: tf.summary.FileWriter("logs/", sess.graph) var = 3 # control exploration t1 = time.time() for i in range(MAX_EPISODES): s = env.reset() ep_reward = 0 for j in range(MAX_EP_STEPS): if RENDER: env.render() # Add exploration noise a = actor.choose_action(s) a = np.clip(np.random.normal(a, var), -2, 2) # add randomness to action selection for exploration s_, r, done, info = env.step(a) M.store_transition(s, a, r / 10, s_) if M.pointer > MEMORY_CAPACITY: var *= .9995 # decay the action randomness b_M = M.sample(BATCH_SIZE) b_s = b_M[:, :state_dim] b_a = b_M[:, state_dim: state_dim + action_dim] b_r = b_M[:, -state_dim - 1: -state_dim] b_s_ = b_M[:, -state_dim:] critic.learn(b_s, b_a, b_r, b_s_) actor.learn(b_s) s = s_ ep_reward += r if j == MAX_EP_STEPS-1: print('Episode:', i, ' Reward: %i' % int(ep_reward), 'Explore: %.2f' % var, ) if ep_reward > -300: RENDER = True break print('Running time: ', time.time()-t1) ================================================ FILE: code/policy_gradient.py ================================================ import tensorflow as tf import numpy as np import gym class PolicyGradient: def __init__(self, n_actions, n_features, learning_rate=0.01, reward_decay=0.95, output_graph=False ): self.n_actions = n_actions self.n_features = n_features self.lr = learning_rate self.gamma = reward_decay self.ep_obs, self.ep_as, self.ep_rs =[],[],[] #states,actions,rewards self.__build_net() self.sess = tf.Session() if output_graph: # $ tensorboard --logdir=logs # http://0.0.0.0:6006/ # tf.train.SummaryWriter soon be deprecated, use following tf.summary.FileWriter("logs/", self.sess.graph) self.sess.run(tf.global_variables_initializer()) def __build_net(self): #PG网络 with tf.name_scope('inputs'): self.tf_obs = tf.placeholder(tf.float32, [None, self.n_features],name="observations") self.tf_acts = tf.placeholder(tf.int32, [None,], name="actions_num") self.tf_vt = tf.placeholder(tf.float32, [None,], name="actions_value") #V(s,a) layer = tf.layers.dense( inputs = self.tf_obs, units = 10, activation = tf.nn.tanh, kernel_initializer=tf.random_normal_initializer(mean=0, stddev=0.3), bias_initializer=tf.constant_initializer(0.1), name = 'fc1' ) all_act = tf.layers.dense( inputs=layer, units=self.n_actions, # 输出个数 activation=None, # 之后再加 Softmax kernel_initializer=tf.random_normal_initializer(mean=0, stddev=0.3), bias_initializer=tf.constant_initializer(0.1), name='fc2' ) self.all_act_prob = tf.nn.softmax(all_act, name='act_prob') with tf.name_scope('loss'): # 最大化 总体 reward (log_p * R) 就是在最小化 -(log_p * R), 而 tf 的功能里只有最小化 loss neg_log_prob = tf.reduce_sum(-tf.log(self.all_act_prob)*tf.one_hot(self.tf_acts, self.n_actions), axis=1) #加- 变为梯度下降 loss = tf.reduce_mean(neg_log_prob * self.tf_vt) with tf.name_scope('train'): self.train_op = tf.train.AdamOptimizer(self.lr).minimize(loss) def choose_action(self, observation): #选择行为 prob_weights = self.sess.run(self.all_act_prob, feed_dict = {self.tf_obs: observation[np.newaxis, :]}) #[0,1,2]->[[0,1,2]] 所有action的概率 矩阵形式 action = np.random.choice(range(prob_weights.shape[1]), p=prob_weights.ravel()) # 根据概率来选 action range(prob_weights.shape[1]用0，1，2，表示动作 return action def store_transition(self, s, a, r):#存储一个回合的经验 self.ep_obs.append(s) self.ep_as.append(a) self.ep_rs.append(r) def learn(self): discounted_ep_rs_norm = self._discount_and_norm_rewards() # 衰减, 并标准化这回合的 reward self.sess.run(self.train_op, feed_dict={ self.tf_obs: np.vstack(self.ep_obs), # shape=[None, n_obs] self.tf_acts: np.array(self.ep_as), # shape=[None, ] self.tf_vt: discounted_ep_rs_norm, # shape=[None, ] }) self.ep_obs, self.ep_as, self.ep_rs = [],[],[] #清空回合数据 return discounted_ep_rs_norm # 返回这一回合的 state-action value def _discount_and_norm_rewards(self): #用bellman公式计算出vt(s,a) #discount discounted_ep_rs = np.zeros_like(self.ep_rs) running_add = 0 for t in reversed(range(0, len(self.ep_rs))): #倒数遍历这个episode中的reward running_add = running_add * self.gamma + self.ep_rs[t] discounted_ep_rs[t] = running_add # r1,r2,r3 -> r1+r2*gamma+r3*gamma^2, r2+r3*gamma, r3 #normalize discounted_ep_rs -= np.mean(discounted_ep_rs) discounted_ep_rs /= np.std(discounted_ep_rs) return discounted_ep_rs #将算法应用起来吧！ RENDER = False # 在屏幕上显示模拟窗口会拖慢运行速度, 我们等计算机学得差不多了再显示模拟 DISPLAY_REWARD_THRESHOLD = 1000 # 当 回合总 reward 大于 400 时显示模拟窗口 #env = gym.make('CartPole-v0') # CartPole 2个动作 向左 向右 env = gym.make('MountainCar-v0') #3个动作 左侧加速、不加速、右侧加速 env = env.unwrapped # 取消限制 env.seed(1) # 普通的 Policy gradient 方法, 使得回合的 variance 比较大, 所以我们选了一个好点的随机种子 print(env.action_space) # 显示可用 action print(env.observation_space) # 显示可用 state 的 observation print(env.observation_space.high) # 显示 observation 最高值 print(env.observation_space.low) # 显示 observation 最低值 # 定义 RL = PolicyGradient( n_actions=env.action_space.n, n_features=env.observation_space.shape[0], learning_rate=0.02, reward_decay=0.99, # gamma # output_graph=True, # 输出 tensorboard 文件 ) for i_episode in range(100): observation = env.reset() while True: if RENDER: env.render() action = RL.choose_action(observation) observation_, reward, done, info = env.step(action) RL.store_transition(observation, action, reward) if done: ep_rs_sum = sum(RL.ep_rs) if 'running_reward' not in globals(): running_reward = ep_rs_sum else: running_reward = running_reward *0.99 + ep_rs_sum *0.01 #不是简单的求和展示当下rewad 比较科学 print("episode:", i_episode, "reward:", int(running_reward)) vt = RL.learn() #学习 输出vt break observation = observation_ ================================================ FILE: code/proximal_policy_optimization.py ================================================ #pendulum #动作空间：只有一维力矩 长度为1 #状态空间：一维速度，长度为3 ''' Critic网络直接给出V(s) Actor网络由2部分组成 oldpi pi PPO升级于A2C（critic按batch更新，离线训练，有2个pi），升级于PG（加入critic网络，利用advantage引导pg优化） ''' import tensorflow as tf import numpy as np import matplotlib.pyplot as plt import gym EP_MAX = 1000 EP_LEN = 200 GAMMA = 0.9 A_LR = 0.0001 C_LR = 0.0002 BATCH = 32 A_UPDATE_STEPS = 10 C_UPDATE_STEPS = 10 S_DIM, A_DIM = 3, 1 #pendulum游戏 METHOD = [ dict(name='kl_pen', kl_target=0.01, lam=0.5), # KL penalty dict(name='clip', epsilon=0.2), # Clipped surrogate objective, find this is better ][1] # choose the method for optimization class PPO(object): def __init__(self): self.sess = tf.Session() self.tfs = tf.placeholder(tf.float32, [None, S_DIM], 'state') #[N_each_batch,DIM] #搭建AC网络 critic with tf.variable_scope('critic'): layer1 = tf.layers.dense(self.tfs, 100, tf.nn.relu) self.v = tf.layers.dense(layer1, 1) self.tfdc_r = tf.placeholder(tf.float32, [None, 1], 'discounted_r') self.advantage = self.tfdc_r - self.v # discounted reward - Critic 出来的 state value self.closs = tf.reduce_mean(tf.square(self.advantage)) # mse loss of critic self.ctrain_op = tf.train.AdamOptimizer(C_LR).minimize(self.closs) pi, pi_params = self._build_anet('pi', trainable=True) oldpi, oldpi_params = self._build_anet('oldpi', trainable=False) #每个pi的本质是一个概率分布 with tf.variable_scope('sample_action'): self.sample_op = tf.squeeze(pi.sample(1), axis=0) #按概率分布pi选择一个action with tf.variable_scope('update_oldpi'): self.update_oldpi_op = [oldp.assign(p) for p, oldp in zip(pi_params, oldpi_params)] #将pi的参数复制给oldpi self.tfa = tf.placeholder(tf.float32, [None, A_DIM], 'action') self.tfadv = tf.placeholder(tf.float32, [None, 1], 'advantage') with tf.variable_scope('loss'): with tf.variable_scope('surrogate'): ratio = pi.prob(self.tfa) / oldpi.prob(self.tfa) #(New Policy/Old Policy) 的比例 surr = ratio * self.tfadv #surrogate objective if METHOD['name'] == 'kl_pen': # 如果用 KL penatily self.tflam = tf.placeholder(tf.float32, None, 'lambda') kl = tf.distributions.kl_divergence(oldpi, pi) self.kl_mean = tf.reduce_mean(kl) self.aloss = tf.reduce_mean(surr - self.tflam * kl) #actor 最终的loss function else: # 如果用 clipping 的方式 self.aloss = tf.reduce_mean(tf.minimum(surr, tf.clip_by_value(ratio, 1-METHOD['epsilon'], 1+METHOD['epsilon'])*self.tfadv)) with tf.variable_scope('atrain'): self.atrain_op = tf.train.AdamOptimizer(A_LR).minimize(-self.aloss) self.sess.run(tf.global_variables_initializer()) def update(self, s, a, r): #update ppo # 先要将 oldpi 里的参数更新 pi 中的 self.sess.run(self.update_oldpi_op) adv = self.sess.run(self.advantage, {self.tfs:s, self.tfdc_r:r}) # adv = (adv - adv.mean())/(adv.std()+1e-6) # sometimes helpful # update actor # 更新 Actor 时, kl penalty 和 clipping 方式是不同的 if METHOD['name'] == 'kl_pen': for _ in range(A_UPDATE_STEPS): #actor 一次训练更新10次 _, kl = self.sess.run( [self.atrain_op, self.kl_mean], {self.tfs: s, self.tfa: a, self.tfadv: adv, self.tflam: METHOD['lam']}) if kl > 4*METHOD['kl_target']: # this in in google's paper break if kl < METHOD['kl_target'] / 1.5: # adaptive lambda, this is in OpenAI's paper METHOD['lam'] /= 2 elif kl > METHOD['kl_target'] * 1.5: METHOD['lam'] *= 2 METHOD['lam'] = np.clip(METHOD['lam'], 1e-4, 10) # sometimes explode, this clipping is my solution else: # clipping method, find this is better (OpenAI's paper) [self.sess.run(self.atrain_op, {self.tfs: s, self.tfa: a, self.tfadv: adv}) for _ in range(A_UPDATE_STEPS)] #actor 一次训练更新10次 # 更新 Critic 的时候, 他们是一样的 critic一次训练更新10次 [self.sess.run(self.ctrain_op, {self.tfs: s, self.tfdc_r: r}) for _ in range(C_UPDATE_STEPS)] def choose_action(self, s): s = s[np.newaxis, :] a = self.sess.run(self.sample_op, {self.tfs:s})[0] return np.clip(a,-2,2) #动作不要超出【-2，2】的范围 因为是按概率分布取动作 所以加上这一步很有必要！ def get_v(self, s): #V(s)状态值 由critic网络给出 if s.ndim < 2: s = s[np.newaxis, :] return self.sess.run(self.v, {self.tfs:s})[0,0] def _build_anet(self, name, trainable): #critic网络输出动作的概率分布 包含参数均值u与方差sigma with tf.variable_scope(name): l1 = tf.layers.dense(self.tfs, 100, tf.nn.relu, trainable=trainable) mu = 2 * tf.layers.dense(l1, A_DIM, tf.nn.tanh, trainable=trainable) sigma = tf.layers.dense(l1, A_DIM, tf.nn.softplus, trainable=trainable) norm_dist = tf.distributions.Normal(loc=mu, scale=sigma) params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name) return norm_dist, params env = gym.make('Pendulum-v0').unwrapped ppo = PPO() all_ep_r = [] for ep in range(EP_MAX): s = env.reset() buffer_s, buffer_a, buffer_r = [],[],[] ep_r = 0 for t in range(EP_LEN): env.render() a = ppo.choose_action(s) s_, r, done, info = env.step(a) buffer_s.append(s) buffer_a.append(a) buffer_r.append((r+8)/8) # normalize reward, 发现有帮助 s = s_ ep_r += r #一个episode的reward之和 # 如果 buffer 收集一个 batch 了或者 episode 完了 #则更新ppo if (t+1) % BATCH == 0 or t == EP_LEN -1: #计算折扣奖励 v_s_ = ppo.get_v(s_) discounted_r = [] for r in buffer_r[::-1]: v_s_ = r + GAMMA * v_s_ discounted_r.append(v_s_) discounted_r.reverse() bs, ba, br = np.vstack(buffer_s), np.vstack(buffer_a), np.array(discounted_r)[:, np.newaxis] #存入一个Batch #清空buffer buffer_s, buffer_a, buffer_r = [],[],[] ppo.update(bs, ba, br) #训练PPO if ep == 0: all_ep_r.append(ep_r) else: all_ep_r.append(all_ep_r[-1]*0.9 + ep_r*0.1) print('Ep: %i' % ep,"|Ep_r: %i" % ep_r,("|Lam: %.4f" % METHOD['lam']) if METHOD['name'] == 'kl_pen' else '',) #plt.plot(np.arange(len(all_ep_r)), all_ep_r) #plt.xlabel('Episode') #plt.ylabel('Moving averaged episode reward') #plt.show() ================================================ FILE: code/tensrolayer-implemented/a3c.py ================================================ """ Asynchronous Advantage Actor Critic (A3C) with Continuous Action Space. Actor Critic History ---------------------- A3C > DDPG (for continuous action space) > AC Advantage ---------- Train faster and more stable than AC. Disadvantage ------------- Have bias. Reference ---------- Original Paper: https://arxiv.org/pdf/1602.01783.pdf MorvanZhou's tutorial: https://morvanzhou.github.io/tutorials/ MorvanZhou's code: https://github.com/MorvanZhou/Reinforcement-learning-with-tensorflow/blob/master/experiments/Solve_BipedalWalker/A3C.py Environment ----------- BipedalWalker-v2 : https://gym.openai.com/envs/BipedalWalker-v2 Reward is given for moving forward, total 300+ points up to the far end. If the robot falls, it gets -100. Applying motor torque costs a small amount of points, more optimal agent will get better score. State consists of hull angle speed, angular velocity, horizontal speed, vertical speed, position of joints and joints angular speed, legs contact with ground, and 10 lidar rangefinder measurements. There's no coordinates in the state vector. Prerequisites -------------- tensorflow 2.0.0a0 tensorflow-probability 0.6.0 tensorlayer 2.0.0 && pip install box2d box2d-kengz --user To run ------ python tutorial_A3C.py --train/test """ import argparse import multiprocessing import threading import time import numpy as np import gym import tensorflow as tf import tensorflow_probability as tfp import tensorlayer as tl from tensorlayer.layers import DenseLayer, InputLayer tfd = tfp.distributions tl.logging.set_verbosity(tl.logging.DEBUG) np.random.seed(2) tf.random.set_seed(2) # reproducible # add arguments in command --train/test parser = argparse.ArgumentParser(description='Train or test neural net motor controller.') parser.add_argument('--train', dest='train', action='store_true', default=False) parser.add_argument('--test', dest='test', action='store_true', default=True) args = parser.parse_args() ##################### hyper parameters #################### GAME = 'BipedalWalker-v2' # BipedalWalkerHardcore-v2 BipedalWalker-v2 LunarLanderContinuous-v2 LOG_DIR = './log' # the log file N_WORKERS = multiprocessing.cpu_count() # number of workers accroding to number of cores in cpu # N_WORKERS = 2 # manually set number of workers MAX_GLOBAL_EP = 8 # number of training episodes GLOBAL_NET_SCOPE = 'Global_Net' UPDATE_GLOBAL_ITER = 10 # update global policy after several episodes GAMMA = 0.99 # reward discount factor ENTROPY_BETA = 0.005 # factor for entropy boosted exploration LR_A = 0.00005 # learning rate for actor LR_C = 0.0001 # learning rate for critic GLOBAL_RUNNING_R = [] GLOBAL_EP = 0 # will increase during training, stop training when it >= MAX_GLOBAL_EP ################### Asynchronous Advantage Actor Critic (A3C) #################################### class ACNet(object): def __init__(self, scope, globalAC=None): self.scope = scope self.save_path = './model' w_init = tf.keras.initializers.glorot_normal(seed=None) # initializer, glorot=xavier def get_actor(input_shape): # policy network with tf.name_scope(self.scope): ni = tl.layers.Input(input_shape, name='in') nn = tl.layers.Dense(n_units=500, act=tf.nn.relu6, W_init=w_init, name='la')(ni) nn = tl.layers.Dense(n_units=300, act=tf.nn.relu6, W_init=w_init, name='la2')(nn) mu = tl.layers.Dense(n_units=N_A, act=tf.nn.tanh, W_init=w_init, name='mu')(nn) sigma = tl.layers.Dense(n_units=N_A, act=tf.nn.softplus, W_init=w_init, name='sigma')(nn) return tl.models.Model(inputs=ni, outputs=[mu, sigma], name=scope + '/Actor') self.actor = get_actor([None, N_S]) self.actor.train() # train mode for Dropout, BatchNorm def get_critic(input_shape): # we use Value-function here, but not Q-function. with tf.name_scope(self.scope): ni = tl.layers.Input(input_shape, name='in') nn = tl.layers.Dense(n_units=500, act=tf.nn.relu6, W_init=w_init, name='lc')(ni) nn = tl.layers.Dense(n_units=300, act=tf.nn.relu6, W_init=w_init, name='lc2')(nn) v = tl.layers.Dense(n_units=1, W_init=w_init, name='v')(nn) return tl.models.Model(inputs=ni, outputs=v, name=scope + '/Critic') self.critic = get_critic([None, N_S]) self.critic.train() # train mode for Dropout, BatchNorm @tf.function # convert numpy functions to tf.Operations in the TFgraph, return tensor def update_global( self, buffer_s, buffer_a, buffer_v_target, globalAC ): # refer to the global Actor-Crtic network for updating it with samples ''' update the global critic ''' with tf.GradientTape() as tape: self.v = self.critic(buffer_s) self.v_target = buffer_v_target td = tf.subtract(self.v_target, self.v, name='TD_error') self.c_loss = tf.reduce_mean(tf.square(td)) self.c_grads = tape.gradient(self.c_loss, self.critic.trainable_weights) OPT_C.apply_gradients(zip(self.c_grads, globalAC.critic.trainable_weights)) # local grads applies to global net # del tape # Drop the reference to the tape ''' update the global actor ''' with tf.GradientTape() as tape: self.mu, self.sigma = self.actor(buffer_s) self.test = self.sigma[0] self.mu, self.sigma = self.mu * A_BOUND[1], self.sigma + 1e-5 normal_dist = tfd.Normal(self.mu, self.sigma) # no tf.contrib for tf2.0 self.a_his = buffer_a # float32 log_prob = normal_dist.log_prob(self.a_his) exp_v = log_prob * td # td is from the critic part, no gradients for it entropy = normal_dist.entropy() # encourage exploration self.exp_v = ENTROPY_BETA * entropy + exp_v self.a_loss = tf.reduce_mean(-self.exp_v) self.a_grads = tape.gradient(self.a_loss, self.actor.trainable_weights) OPT_A.apply_gradients(zip(self.a_grads, globalAC.actor.trainable_weights)) # local grads applies to global net return self.test # for test purpose @tf.function def pull_global(self, globalAC): # run by a local, pull weights from the global nets for l_p, g_p in zip(self.actor.trainable_weights, globalAC.actor.trainable_weights): l_p.assign(g_p) for l_p, g_p in zip(self.critic.trainable_weights, globalAC.critic.trainable_weights): l_p.assign(g_p) def choose_action(self, s): # run by a local s = s[np.newaxis, :] self.mu, self.sigma = self.actor(s) with tf.name_scope('wrap_a_out'): self.mu, self.sigma = self.mu * A_BOUND[1], self.sigma + 1e-5 normal_dist = tfd.Normal(self.mu, self.sigma) # for continuous action space self.A = tf.clip_by_value(tf.squeeze(normal_dist.sample(1), axis=0), *A_BOUND) return self.A.numpy()[0] def save_ckpt(self): # save trained weights tl.files.save_npz(self.actor.trainable_weights, name='model_actor.npz') tl.files.save_npz(self.critic.trainable_weights, name='model_critic.npz') def load_ckpt(self): # load trained weights tl.files.load_and_assign_npz(name='model_actor.npz', network=self.actor) tl.files.load_and_assign_npz(name='model_critic.npz', network=self.critic) class Worker(object): def __init__(self, name, globalAC): self.env = gym.make(GAME) self.name = name self.AC = ACNet(name, globalAC) # def work(self): def work(self, globalAC): global GLOBAL_RUNNING_R, GLOBAL_EP total_step = 1 buffer_s, buffer_a, buffer_r = [], [], [] while not COORD.should_stop() and GLOBAL_EP < MAX_GLOBAL_EP: s = self.env.reset() ep_r = 0 while True: # visualize Worker_0 during training if self.name == 'Worker_0' and total_step % 30 == 0: self.env.render() s = s.astype('float32') # double to float a = self.AC.choose_action(s) s_, r, done, _info = self.env.step(a) s_ = s_.astype('float32') # double to float # set robot falls reward to -2 instead of -100 if r == -100: r = -2 ep_r += r buffer_s.append(s) buffer_a.append(a) buffer_r.append(r) if total_step % UPDATE_GLOBAL_ITER == 0 or done: # update global and assign to local net if done: v_s_ = 0 # terminal else: v_s_ = self.AC.critic(s_[np.newaxis, :])[0, 0] # reduce dim from 2 to 0 buffer_v_target = [] for r in buffer_r[::-1]: # reverse buffer r v_s_ = r + GAMMA * v_s_ buffer_v_target.append(v_s_) buffer_v_target.reverse() buffer_s, buffer_a, buffer_v_target = ( np.vstack(buffer_s), np.vstack(buffer_a), np.vstack(buffer_v_target) ) # update gradients on global network self.AC.update_global(buffer_s, buffer_a, buffer_v_target.astype('float32'), globalAC) buffer_s, buffer_a, buffer_r = [], [], [] # update local network from global network self.AC.pull_global(globalAC) s = s_ total_step += 1 if done: if len(GLOBAL_RUNNING_R) == 0: # record running episode reward GLOBAL_RUNNING_R.append(ep_r) else: # moving average GLOBAL_RUNNING_R.append(0.95 * GLOBAL_RUNNING_R[-1] + 0.05 * ep_r) # print( # self.name, # "Episode: ", # GLOBAL_EP, # # "| pos: %i" % self.env.unwrapped.hull.position[0], # number of move # '| reward: %.1f' % ep_r, # "| running_reward: %.1f" % GLOBAL_RUNNING_R[-1], # # '| sigma:', test, # debug # # 'WIN ' * 5 if self.env.unwrapped.hull.position[0] >= 88 else '', # ) print('{}, Episode: {}/{} | Episode Reward: {:.4f} | Running Time: {:.4f}'\ .format(self.name, GLOBAL_EP, MAX_GLOBAL_EP, ep_r, time.time()-t0 )) GLOBAL_EP += 1 break if __name__ == "__main__": env = gym.make(GAME) N_S = env.observation_space.shape[0] N_A = env.action_space.shape[0] A_BOUND = [env.action_space.low, env.action_space.high] A_BOUND[0] = A_BOUND[0].reshape(1, N_A) A_BOUND[1] = A_BOUND[1].reshape(1, N_A) # print(A_BOUND) if args.train: # ============================= TRAINING =============================== t0 = time.time() with tf.device("/cpu:0"): OPT_A = tf.optimizers.RMSprop(LR_A, name='RMSPropA') OPT_C = tf.optimizers.RMSprop(LR_C, name='RMSPropC') GLOBAL_AC = ACNet(GLOBAL_NET_SCOPE) # we only need its params workers = [] # Create worker for i in range(N_WORKERS): i_name = 'Worker_%i' % i # worker name workers.append(Worker(i_name, GLOBAL_AC)) COORD = tf.train.Coordinator() # start TF threading worker_threads = [] for worker in workers: # t = threading.Thread(target=worker.work) job = lambda: worker.work(GLOBAL_AC) t = threading.Thread(target=job) t.start() worker_threads.append(t) COORD.join(worker_threads) import matplotlib.pyplot as plt plt.plot(GLOBAL_RUNNING_R) plt.xlabel('episode') plt.ylabel('global running reward') plt.savefig('a3c.png') plt.show() GLOBAL_AC.save_ckpt() if args.test: # ============================= EVALUATION ============================= # env = gym.make(GAME) # GLOBAL_AC = ACNet(GLOBAL_NET_SCOPE) GLOBAL_AC.load_ckpt() while True: s = env.reset() rall = 0 while True: env.render() s = s.astype('float32') # double to float a = GLOBAL_AC.choose_action(s) s, r, d, _ = env.step(a) rall += r if d: print("reward", rall) break ================================================ FILE: code/tensrolayer-implemented/ac.py ================================================ """ Actor-Critic ------------- It uses TD-error as the Advantage. Actor Critic History ---------------------- A3C > DDPG > AC Advantage ---------- AC converge faster than Policy Gradient. Disadvantage (IMPORTANT) ------------------------ The Policy is oscillated (difficult to converge), DDPG can solve this problem using advantage of DQN. Reference ---------- paper: https://papers.nips.cc/paper/1786-actor-critic-algorithms.pdf View more on MorvanZhou's tutorial page: https://morvanzhou.github.io/tutorials/ Environment ------------ CartPole-v0: https://gym.openai.com/envs/CartPole-v0 A pole is attached by an un-actuated joint to a cart, which moves along a frictionless track. The system is controlled by applying a force of +1 or -1 to the cart. The pendulum starts upright, and the goal is to prevent it from falling over. A reward of +1 is provided for every timestep that the pole remains upright. The episode ends when the pole is more than 15 degrees from vertical, or the cart moves more than 2.4 units from the center. Prerequisites -------------- tensorflow >=2.0.0a0 tensorlayer >=2.0.0 To run ------ python tutorial_AC.py --train/test """ import argparse import time import numpy as np import gym import tensorflow as tf import tensorlayer as tl tl.logging.set_verbosity(tl.logging.DEBUG) np.random.seed(2) tf.random.set_seed(2) # reproducible # add arguments in command --train/test parser = argparse.ArgumentParser(description='Train or test neural net motor controller.') parser.add_argument('--train', dest='train', action='store_true', default=False) parser.add_argument('--test', dest='test', action='store_true', default=True) args = parser.parse_args() ##################### hyper parameters #################### OUTPUT_GRAPH = False MAX_EPISODE = 3000 # number of overall episodes for training DISPLAY_REWARD_THRESHOLD = 100 # renders environment if running reward is greater then this threshold MAX_EP_STEPS = 1000 # maximum time step in one episode RENDER = False # rendering wastes time LAMBDA = 0.9 # reward discount in TD error LR_A = 0.001 # learning rate for actor LR_C = 0.01 # learning rate for critic ############################### Actor-Critic #################################### class Actor(object): def __init__(self, n_features, n_actions, lr=0.001): def get_model(inputs_shape): ni = tl.layers.Input(inputs_shape, name='state') nn = tl.layers.Dense( n_units=30, act=tf.nn.relu6, W_init=tf.random_uniform_initializer(0, 0.01), name='hidden' )(ni) nn = tl.layers.Dense( n_units=10, act=tf.nn.relu6, W_init=tf.random_uniform_initializer(0, 0.01), name='hidden2' )(nn) nn = tl.layers.Dense(n_units=n_actions, name='actions')(nn) return tl.models.Model(inputs=ni, outputs=nn, name="Actor") self.model = get_model([None, n_features]) self.model.train() self.optimizer = tf.optimizers.Adam(lr) def learn(self, s, a, td): with tf.GradientTape() as tape: _logits = self.model(np.array([s])) ## cross-entropy loss weighted by td-error (advantage), # the cross-entropy mearsures the difference of two probability distributions: the predicted logits and sampled action distribution, # then weighted by the td-error: small difference of real and predict actions for large td-error (advantage); and vice versa. _exp_v = tl.rein.cross_entropy_reward_loss(logits=_logits, actions=[a], rewards=td[0]) grad = tape.gradient(_exp_v, self.model.trainable_weights) self.optimizer.apply_gradients(zip(grad, self.model.trainable_weights)) return _exp_v def choose_action(self, s): _logits = self.model(np.array([s])) _probs = tf.nn.softmax(_logits).numpy() return tl.rein.choice_action_by_probs(_probs.ravel()) # sample according to probability distribution def choose_action_greedy(self, s): _logits = self.model(np.array([s])) # logits: probability distribution of actions _probs = tf.nn.softmax(_logits).numpy() return np.argmax(_probs.ravel()) def save_ckpt(self): # save trained weights tl.files.save_npz(self.model.trainable_weights, name='model_actor.npz') def load_ckpt(self): # load trained weights tl.files.load_and_assign_npz(name='model_actor.npz', network=self.model) class Critic(object): def __init__(self, n_features, lr=0.01): def get_model(inputs_shape): ni = tl.layers.Input(inputs_shape, name='state') nn = tl.layers.Dense( n_units=30, act=tf.nn.relu6, W_init=tf.random_uniform_initializer(0, 0.01), name='hidden' )(ni) nn = tl.layers.Dense( n_units=5, act=tf.nn.relu, W_init=tf.random_uniform_initializer(0, 0.01), name='hidden2' )(nn) nn = tl.layers.Dense(n_units=1, act=None, name='value')(nn) return tl.models.Model(inputs=ni, outputs=nn, name="Critic") self.model = get_model([1, n_features]) self.model.train() self.optimizer = tf.optimizers.Adam(lr) def learn(self, s, r, s_): v_ = self.model(np.array([s_])) with tf.GradientTape() as tape: v = self.model(np.array([s])) ## TD_error = r + lambd * V(newS) - V(S) td_error = r + LAMBDA * v_ - v loss = tf.square(td_error) grad = tape.gradient(loss, self.model.trainable_weights) self.optimizer.apply_gradients(zip(grad, self.model.trainable_weights)) return td_error def save_ckpt(self): # save trained weights tl.files.save_npz(self.model.trainable_weights, name='model_critic.npz') def load_ckpt(self): # load trained weights tl.files.load_and_assign_npz(name='model_critic.npz', network=self.model) if __name__ == '__main__': ''' choose environment 1. Openai gym: env = gym.make() 2. DeepMind Control Suite: env = dm_control2gym.make() ''' env = gym.make('CartPole-v0') # dm_control2gym.create_render_mode('example mode', show=True, return_pixel=False, height=240, width=320, camera_id=-1, overlays=(), # depth=False, scene_option=None) # env = dm_control2gym.make(domain_name="cartpole", task_name="balance") env.seed(2) # reproducible # env = env.unwrapped N_F = env.observation_space.shape[0] # N_A = env.action_space.shape[0] N_A = env.action_space.n print("observation dimension: %d" % N_F) # 4 print("observation high: %s" % env.observation_space.high) # [ 2.4 , inf , 0.41887902 , inf] print("observation low : %s" % env.observation_space.low) # [-2.4 , -inf , -0.41887902 , -inf] print("num of actions: %d" % N_A) # 2 : left or right actor = Actor(n_features=N_F, n_actions=N_A, lr=LR_A) # we need a good teacher, so the teacher should learn faster than the actor critic = Critic(n_features=N_F, lr=LR_C) if args.train: t0 = time.time() for i_episode in range(MAX_EPISODE): # episode_time = time.time() s = env.reset().astype(np.float32) t = 0 # number of step in this episode all_r = [] # rewards of all steps while True: if RENDER: env.render() a = actor.choose_action(s) s_new, r, done, info = env.step(a) s_new = s_new.astype(np.float32) if done: r = -20 # these may helpful in some tasks # if abs(s_new[0]) >= env.observation_space.high[0]: # # cart moves more than 2.4 units from the center # r = -20 # reward for the distance between cart to the center # r -= abs(s_new[0]) * .1 all_r.append(r) td_error = critic.learn( s, r, s_new ) # learn Value-function : gradient = grad[r + lambda * V(s_new) - V(s)] try: actor.learn(s, a, td_error) # learn Policy : true_gradient = grad[logPi(s, a) * td_error] except KeyboardInterrupt: # if Ctrl+C at running actor.learn(), then save model, or exit if not at actor.learn() actor.save_ckpt() critic.save_ckpt() # logging s = s_new t += 1 if done or t >= MAX_EP_STEPS: ep_rs_sum = sum(all_r) if 'running_reward' not in globals(): running_reward = ep_rs_sum else: running_reward = running_reward * 0.95 + ep_rs_sum * 0.05 # start rending if running_reward greater than a threshold # if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True # print("Episode: %d reward: %f running_reward %f took: %.5f" % \ # (i_episode, ep_rs_sum, running_reward, time.time() - episode_time)) print('Episode: {}/{} | Episode Reward: {:.4f} | Running Time: {:.4f}'\ .format(i_episode, MAX_EPISODE, ep_rs_sum, time.time()-t0 )) # Early Stopping for quick check if t >= MAX_EP_STEPS: print("Early Stopping") s = env.reset().astype(np.float32) rall = 0 while True: env.render() # a = actor.choose_action(s) a = actor.choose_action_greedy(s) # Hao Dong: it is important for this task s_new, r, done, info = env.step(a) s_new = np.concatenate((s_new[0:N_F], s[N_F:]), axis=0).astype(np.float32) rall += r s = s_new if done: print("reward", rall) s = env.reset().astype(np.float32) rall = 0 break actor.save_ckpt() critic.save_ckpt() if args.test: actor.load_ckpt() critic.load_ckpt() t0 = time.time() for i_episode in range(MAX_EPISODE): episode_time = time.time() s = env.reset().astype(np.float32) t = 0 # number of step in this episode all_r = [] # rewards of all steps while True: if RENDER: env.render() a = actor.choose_action(s) s_new, r, done, info = env.step(a) s_new = s_new.astype(np.float32) if done: r = -20 # these may helpful in some tasks # if abs(s_new[0]) >= env.observation_space.high[0]: # # cart moves more than 2.4 units from the center # r = -20 # reward for the distance between cart to the center # r -= abs(s_new[0]) * .1 all_r.append(r) s = s_new t += 1 if done or t >= MAX_EP_STEPS: ep_rs_sum = sum(all_r) if 'running_reward' not in globals(): running_reward = ep_rs_sum else: running_reward = running_reward * 0.95 + ep_rs_sum * 0.05 # start rending if running_reward greater than a threshold # if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True # print("Episode: %d reward: %f running_reward %f took: %.5f" % \ # (i_episode, ep_rs_sum, running_reward, time.time() - episode_time)) print('Episode: {}/{} | Episode Reward: {:.4f} | Running Time: {:.4f}'\ .format(i_episode, MAX_EPISODE, ep_rs_sum, time.time()-t0 )) # Early Stopping for quick check if t >= MAX_EP_STEPS: print("Early Stopping") s = env.reset().astype(np.float32) rall = 0 while True: env.render() # a = actor.choose_action(s) a = actor.choose_action_greedy(s) # Hao Dong: it is important for this task s_new, r, done, info = env.step(a) s_new = np.concatenate((s_new[0:N_F], s[N_F:]), axis=0).astype(np.float32) rall += r s = s_new if done: print("reward", rall) s = env.reset().astype(np.float32) rall = 0 break ================================================ FILE: code/tensrolayer-implemented/ddpg.py ================================================ """ Deep Deterministic Policy Gradient (DDPG) ----------------------------------------- An algorithm concurrently learns a Q-function and a policy. It uses off-policy data and the Bellman equation to learn the Q-function, and uses the Q-function to learn the policy. Reference --------- Deterministic Policy Gradient Algorithms, Silver et al. 2014 Continuous Control With Deep Reinforcement Learning, Lillicrap et al. 2016 MorvanZhou's tutorial page: https://morvanzhou.github.io/tutorials/ Environment ----------- Openai Gym Pendulum-v0, continual action space Prerequisites ------------- tensorflow >=2.0.0a0 tensorflow-probability 0.6.0 tensorlayer >=2.0.0 To run ------ python tutorial_DDPG.py --train/test """ import argparse import os import time import matplotlib.pyplot as plt import numpy as np import gym import tensorflow as tf import tensorlayer as tl parser = argparse.ArgumentParser(description='Train or test neural net motor controller.') parser.add_argument('--train', dest='train', action='store_true', default=True) parser.add_argument('--test', dest='train', action='store_false') args = parser.parse_args() ##################### hyper parameters #################### ENV_NAME = 'Pendulum-v0' # environment name RANDOMSEED = 1 # random seed LR_A = 0.001 # learning rate for actor LR_C = 0.002 # learning rate for critic GAMMA = 0.9 # reward discount TAU = 0.01 # soft replacement MEMORY_CAPACITY = 10000 # size of replay buffer BATCH_SIZE = 32 # update batchsize MAX_EPISODES = 200 # total number of episodes for training MAX_EP_STEPS = 200 # total number of steps for each episode TEST_PER_EPISODES = 10 # test the model per episodes VAR = 3 # control exploration ############################### DDPG #################################### class DDPG(object): """ DDPG class """ def __init__(self, a_dim, s_dim, a_bound): self.memory = np.zeros((MEMORY_CAPACITY, s_dim * 2 + a_dim + 1), dtype=np.float32) self.pointer = 0 self.a_dim, self.s_dim, self.a_bound = a_dim, s_dim, a_bound W_init = tf.random_normal_initializer(mean=0, stddev=0.3) b_init = tf.constant_initializer(0.1) def get_actor(input_state_shape, name=''): """ Build actor network :param input_state_shape: state :param name: name :return: act """ inputs = tl.layers.Input(input_state_shape, name='A_input') x = tl.layers.Dense(n_units=30, act=tf.nn.relu, W_init=W_init, b_init=b_init, name='A_l1')(inputs) x = tl.layers.Dense(n_units=a_dim, act=tf.nn.tanh, W_init=W_init, b_init=b_init, name='A_a')(x) x = tl.layers.Lambda(lambda x: np.array(a_bound) * x)(x) return tl.models.Model(inputs=inputs, outputs=x, name='Actor' + name) def get_critic(input_state_shape, input_action_shape, name=''): """ Build critic network :param input_state_shape: state :param input_action_shape: act :param name: name :return: Q value Q(s,a) """ s = tl.layers.Input(input_state_shape, name='C_s_input') a = tl.layers.Input(input_action_shape, name='C_a_input') x = tl.layers.Concat(1)([s, a]) x = tl.layers.Dense(n_units=60, act=tf.nn.relu, W_init=W_init, b_init=b_init, name='C_l1')(x) x = tl.layers.Dense(n_units=1, W_init=W_init, b_init=b_init, name='C_out')(x) return tl.models.Model(inputs=[s, a], outputs=x, name='Critic' + name) self.actor = get_actor([None, s_dim]) self.critic = get_critic([None, s_dim], [None, a_dim]) self.actor.train() self.critic.train() def copy_para(from_model, to_model): """ Copy parameters for soft updating :param from_model: latest model :param to_model: target model :return: None """ for i, j in zip(from_model.trainable_weights, to_model.trainable_weights): j.assign(i) self.actor_target = get_actor([None, s_dim], name='_target') copy_para(self.actor, self.actor_target) self.actor_target.eval() self.critic_target = get_critic([None, s_dim], [None, a_dim], name='_target') copy_para(self.critic, self.critic_target) self.critic_target.eval() self.R = tl.layers.Input([None, 1], tf.float32, 'r') self.ema = tf.train.ExponentialMovingAverage(decay=1 - TAU) # soft replacement self.actor_opt = tf.optimizers.Adam(LR_A) self.critic_opt = tf.optimizers.Adam(LR_C) def ema_update(self): """ Soft updating by exponential smoothing :return: None """ paras = self.actor.trainable_weights + self.critic.trainable_weights self.ema.apply(paras) for i, j in zip(self.actor_target.trainable_weights + self.critic_target.trainable_weights, paras): i.assign(self.ema.average(j)) def choose_action(self, s): """ Choose action :param s: state :return: act """ return self.actor(np.array([s], dtype=np.float32))[0] def learn(self): """ Update parameters :return: None """ indices = np.random.choice(MEMORY_CAPACITY, size=BATCH_SIZE) bt = self.memory[indices, :] bs = bt[:, :self.s_dim] ba = bt[:, self.s_dim:self.s_dim + self.a_dim] br = bt[:, -self.s_dim - 1:-self.s_dim] bs_ = bt[:, -self.s_dim:] with tf.GradientTape() as tape: a_ = self.actor_target(bs_) q_ = self.critic_target([bs_, a_]) y = br + GAMMA * q_ q = self.critic([bs, ba]) td_error = tf.losses.mean_squared_error(y, q) c_grads = tape.gradient(td_error, self.critic.trainable_weights) self.critic_opt.apply_gradients(zip(c_grads, self.critic.trainable_weights)) with tf.GradientTape() as tape: a = self.actor(bs) q = self.critic([bs, a]) a_loss = - tf.reduce_mean(q) # maximize the q a_grads = tape.gradient(a_loss, self.actor.trainable_weights) self.actor_opt.apply_gradients(zip(a_grads, self.actor.trainable_weights)) self.ema_update() def store_transition(self, s, a, r, s_): """ Store data in data buffer :param s: state :param a: act :param r: reward :param s_: next state :return: None """ s = s.astype(np.float32) s_ = s_.astype(np.float32) transition = np.hstack((s, a, [r], s_)) index = self.pointer % MEMORY_CAPACITY # replace the old memory with new memory self.memory[index, :] = transition self.pointer += 1 def save_ckpt(self): """ save trained weights :return: None """ if not os.path.exists('model'): os.makedirs('model') tl.files.save_weights_to_hdf5('model/ddpg_actor.hdf5', self.actor) tl.files.save_weights_to_hdf5('model/ddpg_actor_target.hdf5', self.actor_target) tl.files.save_weights_to_hdf5('model/ddpg_critic.hdf5', self.critic) tl.files.save_weights_to_hdf5('model/ddpg_critic_target.hdf5', self.critic_target) def load_ckpt(self): """ load trained weights :return: None """ tl.files.load_hdf5_to_weights_in_order('model/ddpg_actor.hdf5', self.actor) tl.files.load_hdf5_to_weights_in_order('model/ddpg_actor_target.hdf5', self.actor_target) tl.files.load_hdf5_to_weights_in_order('model/ddpg_critic.hdf5', self.critic) tl.files.load_hdf5_to_weights_in_order('model/ddpg_critic_target.hdf5', self.critic_target) if __name__ == '__main__': env = gym.make(ENV_NAME) env = env.unwrapped # reproducible env.seed(RANDOMSEED) np.random.seed(RANDOMSEED) tf.random.set_seed(RANDOMSEED) s_dim = env.observation_space.shape[0] a_dim = env.action_space.shape[0] a_bound = env.action_space.high ddpg = DDPG(a_dim, s_dim, a_bound) if args.train: # train reward_buffer = [] t0 = time.time() for i in range(MAX_EPISODES): t1 = time.time() s = env.reset() ep_reward = 0 for j in range(MAX_EP_STEPS): # Add exploration noise a = ddpg.choose_action(s) a = np.clip(np.random.normal(a, VAR), -2, 2) # add randomness to action selection for exploration s_, r, done, info = env.step(a) ddpg.store_transition(s, a, r / 10, s_) if ddpg.pointer > MEMORY_CAPACITY: ddpg.learn() s = s_ ep_reward += r if j == MAX_EP_STEPS - 1: print( '\rEpisode: {}/{} | Episode Reward: {:.4f} | Running Time: {:.4f}'.format( i, MAX_EPISODES, ep_reward, time.time() - t1 ), end='' ) plt.show() # test if i and not i % TEST_PER_EPISODES: t1 = time.time() s = env.reset() ep_reward = 0 for j in range(MAX_EP_STEPS): a = ddpg.choose_action(s) # without exploration noise s_, r, done, info = env.step(a) s = s_ ep_reward += r if j == MAX_EP_STEPS - 1: print( '\rEpisode: {}/{} | Episode Reward: {:.4f} | Running Time: {:.4f}'.format( i, MAX_EPISODES, ep_reward, time.time() - t1 ) ) reward_buffer.append(ep_reward) if reward_buffer: plt.ion() plt.cla() plt.title('DDPG') plt.plot(np.array(range(len(reward_buffer))) * TEST_PER_EPISODES, reward_buffer) # plot the episode vt plt.xlabel('episode steps') plt.ylabel('normalized state-action value') plt.ylim(-2000, 0) plt.show() plt.pause(0.1) plt.ioff() plt.show() print('\nRunning time: ', time.time() - t0) ddpg.save_ckpt() # test ddpg.load_ckpt() while True: s = env.reset() for i in range(MAX_EP_STEPS): env.render() s, r, done, info = env.step(ddpg.choose_action(s)) if done: break ================================================ FILE: code/tensrolayer-implemented/dqn.py ================================================ """ Deep Q-Network Q(a, s) ----------------------- TD Learning, Off-Policy, e-Greedy Exploration (GLIE). Q(S, A) <- Q(S, A) + alpha * (R + lambda * Q(newS, newA) - Q(S, A)) delta_w = R + lambda * Q(newS, newA) See David Silver RL Tutorial Lecture 5 - Q-Learning for more details. Reference ---------- original paper: https://storage.googleapis.com/deepmind-media/dqn/DQNNaturePaper.pdf EN: https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-0-q-learning-with-tables-and-neural-networks-d195264329d0#.5m3361vlw CN: https://zhuanlan.zhihu.com/p/25710327 Note: Policy Network has been proved to be better than Q-Learning, see tutorial_atari_pong.py Environment ----------- # The FrozenLake v0 environment https://gym.openai.com/envs/FrozenLake-v0 The agent controls the movement of a character in a grid world. Some tiles of the grid are walkable, and others lead to the agent falling into the water. Additionally, the movement direction of the agent is uncertain and only partially depends on the chosen direction. The agent is rewarded for finding a walkable path to a goal tile. SFFF (S: starting point, safe) FHFH (F: frozen surface, safe) FFFH (H: hole, fall to your doom) HFFG (G: goal, where the frisbee is located) The episode ends when you reach the goal or fall in a hole. You receive a reward of 1 if you reach the goal, and zero otherwise. Prerequisites -------------- tensorflow>=2.0.0a0 tensorlayer>=2.0.0 To run ------- python tutorial_DQN.py --train/test """ import argparse import time import numpy as np import gym import tensorflow as tf import tensorlayer as tl # add arguments in command --train/test parser = argparse.ArgumentParser(description='Train or test neural net motor controller.') parser.add_argument('--train', dest='train', action='store_true', default=False) parser.add_argument('--test', dest='test', action='store_true', default=True) args = parser.parse_args() tl.logging.set_verbosity(tl.logging.DEBUG) ##################### hyper parameters #################### lambd = .99 # decay factor e = 0.1 # e-Greedy Exploration, the larger the more random num_episodes = 10000 render = False # display the game environment running_reward = None ##################### DQN ########################## def to_one_hot(i, n_classes=None): a = np.zeros(n_classes, 'uint8') a[i] = 1 return a ## Define Q-network q(a,s) that ouput the rewards of 4 actions by given state, i.e. Action-Value Function. # encoding for state: 4x4 grid can be represented by one-hot vector with 16 integers. def get_model(inputs_shape): ni = tl.layers.Input(inputs_shape, name='observation') nn = tl.layers.Dense(4, act=None, W_init=tf.random_uniform_initializer(0, 0.01), b_init=None, name='q_a_s')(ni) return tl.models.Model(inputs=ni, outputs=nn, name="Q-Network") def save_ckpt(model): # save trained weights tl.files.save_npz(model.trainable_weights, name='dqn_model.npz') def load_ckpt(model): # load trained weights tl.files.load_and_assign_npz(name='dqn_model.npz', network=model) if __name__ == '__main__': qnetwork = get_model([None, 16]) qnetwork.train() train_weights = qnetwork.trainable_weights optimizer = tf.optimizers.SGD(learning_rate=0.1) env = gym.make('FrozenLake-v0') if args.train: t0 = time.time() for i in range(num_episodes): ## Reset environment and get first new observation # episode_time = time.time() s = env.reset() # observation is state, integer 0 ~ 15 rAll = 0 for j in range(99): # step index, maximum step is 99 if render: env.render() ## Choose an action by greedily (with e chance of random action) from the Q-network allQ = qnetwork(np.asarray([to_one_hot(s, 16)], dtype=np.float32)).numpy() a = np.argmax(allQ, 1) ## e-Greedy Exploration !!! sample random action if np.random.rand(1) < e: a[0] = env.action_space.sample() ## Get new state and reward from environment s1, r, d, _ = env.step(a[0]) ## Obtain the Q' values by feeding the new state through our network Q1 = qnetwork(np.asarray([to_one_hot(s1, 16)], dtype=np.float32)).numpy() ## Obtain maxQ' and set our target value for chosen action. maxQ1 = np.max(Q1) # in Q-Learning, policy is greedy, so we use "max" to select the next action. targetQ = allQ targetQ[0, a[0]] = r + lambd * maxQ1 ## Train network using target and predicted Q values # it is not real target Q value, it is just an estimation, # but check the Q-Learning update formula: # Q'(s,a) <- Q(s,a) + alpha(r + lambd * maxQ(s',a') - Q(s, a)) # minimizing |r + lambd * maxQ(s',a') - Q(s, a)|^2 equals to force Q'(s,a) ≈ Q(s,a) with tf.GradientTape() as tape: _qvalues = qnetwork(np.asarray([to_one_hot(s, 16)], dtype=np.float32)) _loss = tl.cost.mean_squared_error(targetQ, _qvalues, is_mean=False) grad = tape.gradient(_loss, train_weights) optimizer.apply_gradients(zip(grad, train_weights)) rAll += r s = s1 ## Reduce chance of random action if an episode is done. if d ==True: e = 1. / ((i / 50) + 10) # reduce e, GLIE: Greey in the limit with infinite Exploration break ## Note that, the rewards here with random action running_reward = rAll if running_reward is None else running_reward * 0.99 + rAll * 0.01 # print("Episode [%d/%d] sum reward: %f running reward: %f took: %.5fs " % \ # (i, num_episodes, rAll, running_reward, time.time() - episode_time)) print('Episode: {}/{} | Episode Reward: {:.4f} | Running Average Reward: {:.4f} | Running Time: {:.4f}'\ .format(i, num_episodes, rAll, running_reward, time.time()-t0 )) save_ckpt(qnetwork) # save model if args.test: t0 = time.time() load_ckpt(qnetwork) # load model for i in range(num_episodes): ## Reset environment and get first new observation episode_time = time.time() s = env.reset() # observation is state, integer 0 ~ 15 rAll = 0 for j in range(99): # step index, maximum step is 99 if render: env.render() ## Choose an action by greedily (with e chance of random action) from the Q-network allQ = qnetwork(np.asarray([to_one_hot(s, 16)], dtype=np.float32)).numpy() a = np.argmax(allQ, 1) # no epsilon, only greedy for testing ## Get new state and reward from environment s1, r, d, _ = env.step(a[0]) rAll += r s = s1 ## Reduce chance of random action if an episode is done. if d ==True: e = 1. / ((i / 50) + 10) # reduce e, GLIE: Greey in the limit with infinite Exploration break ## Note that, the rewards here with random action running_reward = rAll if running_reward is None else running_reward * 0.99 + rAll * 0.01 # print("Episode [%d/%d] sum reward: %f running reward: %f took: %.5fs " % \ # (i, num_episodes, rAll, running_reward, time.time() - episode_time)) print('Episode: {}/{} | Episode Reward: {:.4f} | Running Average Reward: {:.4f} | Running Time: {:.4f}'\ .format(i, num_episodes, rAll, running_reward, time.time()-t0 )) ================================================ FILE: code/tensrolayer-implemented/dqn_variants.py ================================================ """ DQN and its variants ------------------------ We implement Double DQN, Dueling DQN and Noisy DQN here. The max operator in standard DQN uses the same values both to select and to evaluate an action by Q(s_t, a_t) = R_{t+1} + \gamma * max_{a}Q_{tar}(s_{t+1}, a). Double DQN propose to use following evaluation to address overestimation problem of max operator: Q(s_t, a_t) = R_{t+1} + \gamma * Q_{tar}(s_{t+1}, max_{a}Q(s_{t+1}, a)). Dueling DQN uses dueling architecture where the value of state and the advantage of each action is estimated separately. Noisy DQN propose to explore by adding parameter noises. Reference: ------------------------ 1. Double DQN Van Hasselt H, Guez A, Silver D. Deep reinforcement learning with double q-learning[C]//Thirtieth AAAI Conference on Artificial Intelligence. 2016. 2. Dueling DQN Wang Z, Schaul T, Hessel M, et al. Dueling network architectures for deep reinforcement learning[J]. arXiv preprint arXiv:1511.06581, 2015. 3. Noisy DQN Plappert M, Houthooft R, Dhariwal P, et al. Parameter space noise for exploration[J]. arXiv preprint arXiv:1706.01905, 2017. Environment: ------------------------ Cartpole and Pong in OpenAI Gym Requirements: ------------------------ tensorflow>=2.0.0a0 tensorlayer>=2.0.0 To run: ------------------------ python tutorial_DQN_variantes.py --mode=train python tutorial_DQN_variantes.py --mode=test --save_path=dqn_variants/8000.npz """ import argparse import os import random import time import numpy as np import tensorflow as tf import tensorlayer as tl from tutorial_wrappers import build_env parser = argparse.ArgumentParser() parser.add_argument('--mode', help='train or test', default='train') parser.add_argument( '--save_path', default='dqn_variants', help='folder to save if mode == train else model path,' 'qnet will be saved once target net update' ) parser.add_argument('--seed', help='random seed', type=int, default=0) parser.add_argument('--env_id', default='CartPole-v0', help='CartPole-v0 or PongNoFrameskip-v4') args = parser.parse_args() if args.mode == 'train': os.makedirs(args.save_path, exist_ok=True) random.seed(args.seed) np.random.seed(args.seed) tf.random.set_seed(args.seed) # reproducible env_id = args.env_id env = build_env(env_id, seed=args.seed) # #################### hyper parameters #################### if env_id == 'CartPole-v0': qnet_type = 'MLP' number_timesteps = 10000 # total number of time steps to train on explore_timesteps = 100 # epsilon-greedy schedule, final exploit prob is 0.99 epsilon = lambda i_iter: 1 - 0.99 * min(1, i_iter / explore_timesteps) lr = 5e-3 # learning rate buffer_size = 1000 # replay buffer size target_q_update_freq = 50 # how frequency target q net update ob_scale = 1.0 # scale observations else: # reward will increase obviously after 1e5 time steps qnet_type = 'CNN' number_timesteps = int(1e6) # total number of time steps to train on explore_timesteps = 1e5 # epsilon-greedy schedule, final exploit prob is 0.99 epsilon = lambda i_iter: 1 - 0.99 * min(1, i_iter / explore_timesteps) lr = 1e-4 # learning rate buffer_size = 10000 # replay buffer size target_q_update_freq = 200 # how frequency target q net update ob_scale = 1.0 / 255 # scale observations in_dim = env.observation_space.shape out_dim = env.action_space.n reward_gamma = 0.99 # reward discount batch_size = 32 # batch size for sampling from replay buffer warm_start = buffer_size / 10 # sample times befor learning noise_update_freq = 50 # how frequency param noise net update # ############################## DQN #################################### class MLP(tl.models.Model): def __init__(self, name): super(MLP, self).__init__(name=name) self.h1 = tl.layers.Dense(64, tf.nn.tanh, in_channels=in_dim[0]) self.qvalue = tl.layers.Dense(out_dim, in_channels=64, name='q', W_init=tf.initializers.GlorotUniform()) self.svalue = tl.layers.Dense(1, in_channels=64, name='s', W_init=tf.initializers.GlorotUniform()) self.noise_scale = 0 def forward(self, ni): feature = self.h1(ni) # apply noise to all linear layer if self.noise_scale != 0: noises = [] for layer in [self.qvalue, self.svalue]: for var in layer.trainable_weights: noise = tf.random.normal(tf.shape(var), 0, self.noise_scale) noises.append(noise) var.assign_add(noise) qvalue = self.qvalue(feature) svalue = self.svalue(feature) if self.noise_scale != 0: idx = 0 for layer in [self.qvalue, self.svalue]: for var in layer.trainable_weights: var.assign_sub(noises[idx]) idx += 1 # dueling network out = svalue + qvalue - tf.reduce_mean(qvalue, 1, keepdims=True) return out class CNN(tl.models.Model): def __init__(self, name): super(CNN, self).__init__(name=name) h, w, in_channels = in_dim dense_in_channels = 64 * ((h - 28) // 8) * ((w - 28) // 8) self.conv1 = tl.layers.Conv2d( 32, (8, 8), (4, 4), tf.nn.relu, 'VALID', in_channels=in_channels, name='conv2d_1', W_init=tf.initializers.GlorotUniform() ) self.conv2 = tl.layers.Conv2d( 64, (4, 4), (2, 2), tf.nn.relu, 'VALID', in_channels=32, name='conv2d_2', W_init=tf.initializers.GlorotUniform() ) self.conv3 = tl.layers.Conv2d( 64, (3, 3), (1, 1), tf.nn.relu, 'VALID', in_channels=64, name='conv2d_3', W_init=tf.initializers.GlorotUniform() ) self.flatten = tl.layers.Flatten(name='flatten') self.preq = tl.layers.Dense( 256, tf.nn.relu, in_channels=dense_in_channels, name='pre_q', W_init=tf.initializers.GlorotUniform() ) self.qvalue = tl.layers.Dense(out_dim, in_channels=256, name='q', W_init=tf.initializers.GlorotUniform()) self.pres = tl.layers.Dense( 256, tf.nn.relu, in_channels=dense_in_channels, name='pre_s', W_init=tf.initializers.GlorotUniform() ) self.svalue = tl.layers.Dense(1, in_channels=256, name='state', W_init=tf.initializers.GlorotUniform()) self.noise_scale = 0 def forward(self, ni): feature = self.flatten(self.conv3(self.conv2(self.conv1(ni)))) # apply noise to all linear layer if self.noise_scale != 0: noises = [] for layer in [self.preq, self.qvalue, self.pres, self.svalue]: for var in layer.trainable_weights: noise = tf.random.normal(tf.shape(var), 0, self.noise_scale) noises.append(noise) var.assign_add(noise) qvalue = self.qvalue(self.preq(feature)) svalue = self.svalue(self.pres(feature)) if self.noise_scale != 0: idx = 0 for layer in [self.preq, self.qvalue, self.pres, self.svalue]: for var in layer.trainable_weights: var.assign_sub(noises[idx]) idx += 1 # dueling network return svalue + qvalue - tf.reduce_mean(qvalue, 1, keepdims=True) class ReplayBuffer(object): def __init__(self, size): self._storage = [] self._maxsize = size self._next_idx = 0 def __len__(self): return len(self._storage) def add(self, *args): if self._next_idx >= len(self._storage): self._storage.append(args) else: self._storage[self._next_idx] = args self._next_idx = (self._next_idx + 1) % self._maxsize def _encode_sample(self, idxes): b_o, b_a, b_r, b_o_, b_d = [], [], [], [], [] for i in idxes: o, a, r, o_, d = self._storage[i] b_o.append(o) b_a.append(a) b_r.append(r) b_o_.append(o_) b_d.append(d) return ( np.stack(b_o).astype('float32') * ob_scale, np.stack(b_a).astype('int32'), np.stack(b_r).astype('float32'), np.stack(b_o_).astype('float32') * ob_scale, np.stack(b_d).astype('float32'), ) def sample(self, batch_size): indexes = range(len(self._storage)) idxes = [random.choice(indexes) for _ in range(batch_size)] return self._encode_sample(idxes) def huber_loss(x): """Loss function for value""" return tf.where(tf.abs(x) < 1, tf.square(x) * 0.5, tf.abs(x) - 0.5) def sync(net, net_tar): """Copy q network to target q network""" for var, var_tar in zip(net.trainable_weights, net_tar.trainable_weights): var_tar.assign(var) def log_softmax(x, dim): temp = x - np.max(x, dim, keepdims=True) return temp - np.log(np.exp(temp).sum(dim, keepdims=True)) def softmax(x, dim): temp = np.exp(x - np.max(x, dim, keepdims=True)) return temp / temp.sum(dim, keepdims=True) if __name__ == '__main__': if args.mode == 'train': qnet = MLP('q') if qnet_type == 'MLP' else CNN('q') qnet.train() trainabel_weights = qnet.trainable_weights targetqnet = MLP('targetq') if qnet_type == 'MLP' else CNN('targetq') targetqnet.infer() sync(qnet, targetqnet) optimizer = tf.optimizers.Adam(learning_rate=lr) buffer = ReplayBuffer(buffer_size) o = env.reset() nepisode = 0 t = time.time() noise_scale = 1e-2 for i in range(1, number_timesteps + 1): eps = epsilon(i) # select action if random.random() < eps: a = int(random.random() * out_dim) else: # noise schedule is based on KL divergence between perturbed and # non-perturbed policy, see https://arxiv.org/pdf/1706.01905.pdf obv = np.expand_dims(o, 0).astype('float32') * ob_scale if i < explore_timesteps: qnet.noise_scale = noise_scale q_ptb = qnet(obv).numpy() qnet.noise_scale = 0 if i % noise_update_freq == 0: q = qnet(obv).numpy() kl_ptb = (log_softmax(q, 1) - log_softmax(q_ptb, 1)) kl_ptb = np.sum(kl_ptb * softmax(q, 1), 1).mean() kl_explore = -np.log(1 - eps + eps / out_dim) if kl_ptb < kl_explore: noise_scale *= 1.01 else: noise_scale /= 1.01 a = q_ptb.argmax(1)[0] else: a = qnet(obv).numpy().argmax(1)[0] # execute action and feed to replay buffer # note that `_` tail in var name means next o_, r, done, info = env.step(a) buffer.add(o, a, r, o_, done) if i >= warm_start: # sync q net and target q net if i % target_q_update_freq == 0: sync(qnet, targetqnet) path = os.path.join(args.save_path, '{}.npz'.format(i)) tl.files.save_npz(qnet.trainable_weights, name=path) # sample from replay buffer b_o, b_a, b_r, b_o_, b_d = buffer.sample(batch_size) # double q estimation b_a_ = tf.one_hot(tf.argmax(qnet(b_o_), 1), out_dim) b_q_ = (1 - b_d) * tf.reduce_sum(targetqnet(b_o_) * b_a_, 1) # calculate loss with tf.GradientTape() as q_tape: b_q = tf.reduce_sum(qnet(b_o) * tf.one_hot(b_a, out_dim), 1) loss = tf.reduce_mean(huber_loss(b_q - (b_r + reward_gamma * b_q_))) # backward gradients q_grad = q_tape.gradient(loss, trainabel_weights) optimizer.apply_gradients(zip(q_grad, trainabel_weights)) if done: o = env.reset() else: o = o_ # episode in info is real (unwrapped) message if info.get('episode'): nepisode += 1 reward, length = info['episode']['r'], info['episode']['l'] fps = int(length / (time.time() - t)) print( 'Time steps so far: {}, episode so far: {}, ' 'episode reward: {:.4f}, episode length: {}, FPS: {}'.format(i, nepisode, reward, length, fps) ) t = time.time() else: qnet = MLP('q') if qnet_type == 'MLP' else CNN('q') tl.files.load_and_assign_npz(name=args.save_path, network=qnet) qnet.eval() nepisode = 0 o = env.reset() for i in range(1, number_timesteps + 1): obv = np.expand_dims(o, 0).astype('float32') * ob_scale a = qnet(obv).numpy().argmax(1)[0] # execute action # note that `_` tail in var name means next o_, r, done, info = env.step(a) if done: o = env.reset() else: o = o_ # episode in info is real (unwrapped) message if info.get('episode'): nepisode += 1 reward, length = info['episode']['r'], info['episode']['l'] print( 'Time steps so far: {}, episode so far: {}, ' 'episode reward: {:.4f}, episode length: {}'.format(i, nepisode, reward, length) ) ================================================ FILE: code/tensrolayer-implemented/pg.py ================================================ """ Vanilla Policy Gradient(VPG or REINFORCE) ----------------------------------------- The policy gradient algorithm works by updating policy parameters via stochastic gradient ascent on policy performance. It's an on-policy algorithm can be used for environments with either discrete or continuous action spaces. Here is an example on discrete action space game CartPole-v0. To apply it on continuous action space, you need to change the last softmax layer and the choose_action function. Reference --------- Cookbook: Barto A G, Sutton R S. Reinforcement Learning: An Introduction[J]. 1998. MorvanZhou's tutorial page: https://morvanzhou.github.io/tutorials/ Environment ----------- Openai Gym CartPole-v0, discrete action space Prerequisites -------------- tensorflow >=2.0.0a0 tensorflow-probability 0.6.0 tensorlayer >=2.0.0 To run ------ python tutorial_PG.py --train/test """ import argparse import os import time import matplotlib.pyplot as plt import numpy as np import gym import tensorflow as tf import tensorlayer as tl parser = argparse.ArgumentParser(description='Train or test neural net motor controller.') parser.add_argument('--train', dest='train', action='store_true', default=True) parser.add_argument('--test', dest='train', action='store_false') args = parser.parse_args() ##################### hyper parameters #################### ENV_NAME = 'CartPole-v0' # environment name RANDOMSEED = 1 # random seed DISPLAY_REWARD_THRESHOLD = 400 # renders environment if total episode reward is greater then this threshold RENDER = False # rendering wastes time num_episodes = 300 ############################### PG #################################### class PolicyGradient: """ PG class """ def __init__(self, n_features, n_actions, learning_rate=0.01, reward_decay=0.95): self.n_actions = n_actions self.n_features = n_features self.lr = learning_rate self.gamma = reward_decay self.ep_obs, self.ep_as, self.ep_rs = [], [], [] def get_model(inputs_shape): """ Build a neural network model. :param inputs_shape: state_shape :return: act """ with tf.name_scope('inputs'): self.tf_obs = tl.layers.Input(inputs_shape, tf.float32, name="observations") self.tf_acts = tl.layers.Input([ None, ], tf.int32, name="actions_num") self.tf_vt = tl.layers.Input([ None, ], tf.float32, name="actions_value") # fc1 layer = tl.layers.Dense( n_units=30, act=tf.nn.tanh, W_init=tf.random_normal_initializer(mean=0, stddev=0.3), b_init=tf.constant_initializer(0.1), name='fc1' )(self.tf_obs) # fc2 all_act = tl.layers.Dense( n_units=self.n_actions, act=None, W_init=tf.random_normal_initializer(mean=0, stddev=0.3), b_init=tf.constant_initializer(0.1), name='all_act' )(layer) return tl.models.Model(inputs=self.tf_obs, outputs=all_act, name='PG model') self.model = get_model([None, n_features]) self.model.train() self.optimizer = tf.optimizers.Adam(self.lr) def choose_action(self, s): """ choose action with probabilities. :param s: state :return: act """ _logits = self.model(np.array([s], np.float32)) _probs = tf.nn.softmax(_logits).numpy() return tl.rein.choice_action_by_probs(_probs.ravel()) def choose_action_greedy(self, s): """ choose action with greedy policy :param s: state :return: act """ _probs = tf.nn.softmax(self.model(np.array([s], np.float32))).numpy() return np.argmax(_probs.ravel()) def store_transition(self, s, a, r): """ store data in memory buffer :param s: state :param a: act :param r: reward :return: """ self.ep_obs.append(np.array([s], np.float32)) self.ep_as.append(a) self.ep_rs.append(r) def learn(self): """ update policy parameters via stochastic gradient ascent :return: None """ # discount and normalize episode reward discounted_ep_rs_norm = self._discount_and_norm_rewards() with tf.GradientTape() as tape: _logits = self.model(np.vstack(self.ep_obs)) # to maximize total reward (log_p * R) is to minimize -(log_p * R), and the tf only have minimize(loss) neg_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=_logits, labels=np.array(self.ep_as)) # this is negative log of chosen action # or in this way: # neg_log_prob = tf.reduce_sum(-tf.log(self.all_act_prob)*tf.one_hot(self.tf_acts, self.n_actions), axis=1) loss = tf.reduce_mean(neg_log_prob * discounted_ep_rs_norm) # reward guided loss grad = tape.gradient(loss, self.model.trainable_weights) self.optimizer.apply_gradients(zip(grad, self.model.trainable_weights)) self.ep_obs, self.ep_as, self.ep_rs = [], [], [] # empty episode data return discounted_ep_rs_norm def _discount_and_norm_rewards(self): """ compute discount_and_norm_rewards :return: discount_and_norm_rewards """ # discount episode rewards discounted_ep_rs = np.zeros_like(self.ep_rs) running_add = 0 for t in reversed(range(0, len(self.ep_rs))): running_add = running_add * self.gamma + self.ep_rs[t] discounted_ep_rs[t] = running_add # normalize episode rewards discounted_ep_rs -= np.mean(discounted_ep_rs) discounted_ep_rs /= np.std(discounted_ep_rs) return discounted_ep_rs def save_ckpt(self): """ save trained weights :return: None """ if not os.path.exists('model'): os.makedirs('model') tl.files.save_weights_to_hdf5('model/pg_policy.hdf5', self.model) def load_ckpt(self): """ load trained weights :return: None """ tl.files.load_hdf5_to_weights_in_order('model/pg_policy.hdf5', self.model) if __name__ == '__main__': # reproducible np.random.seed(RANDOMSEED) tf.random.set_seed(RANDOMSEED) tl.logging.set_verbosity(tl.logging.DEBUG) env = gym.make(ENV_NAME) env.seed(RANDOMSEED) # reproducible, general Policy gradient has high variance env = env.unwrapped print(env.action_space) print(env.observation_space) print(env.observation_space.high) print(env.observation_space.low) RL = PolicyGradient( n_actions=env.action_space.n, n_features=env.observation_space.shape[0], learning_rate=0.02, reward_decay=0.99, # output_graph=True, ) if args.train: reward_buffer = [] for i_episode in range(num_episodes): episode_time = time.time() observation = env.reset() while True: if RENDER: env.render() action = RL.choose_action(observation) observation_, reward, done, info = env.step(action) RL.store_transition(observation, action, reward) if done: ep_rs_sum = sum(RL.ep_rs) if 'running_reward' not in globals(): running_reward = ep_rs_sum else: running_reward = running_reward * 0.99 + ep_rs_sum * 0.01 #打开渲染 可视化游戏界面 #if running_reward > DISPLAY_REWARD_THRESHOLD: # RENDER = True # rendering # print("episode:", i_episode, " reward:", int(running_reward)) print( "Episode [%d/%d] \tsum reward: %d \trunning reward: %f \ttook: %.5fs " % (i_episode, num_episodes, ep_rs_sum, running_reward, time.time() - episode_time) ) reward_buffer.append(running_reward) vt = RL.learn() break ''' plt.ion() plt.cla() plt.title('PG') plt.plot(reward_buffer, ) # plot the episode vt plt.xlabel('episode steps') plt.ylabel('normalized state-action value') plt.show() plt.pause(0.1) ''' observation = observation_ RL.save_ckpt() #plt.ioff() #plt.show() # test RL.load_ckpt() observation = env.reset() while True: env.render() action = RL.choose_action(observation) observation, reward, done, info = env.step(action) if done: observation = env.reset() ================================================ FILE: code/tensrolayer-implemented/ppo.py ================================================ """ Proximal Policy Optimization (PPO) ---------------------------- A simple version of Proximal Policy Optimization (PPO) using single thread. PPO is a family of first-order methods that use a few other tricks to keep new policies close to old. PPO methods are significantly simpler to implement, and empirically seem to perform at least as well as TRPO. Reference --------- Proximal Policy Optimization Algorithms, Schulman et al. 2017 High Dimensional Continuous Control Using Generalized Advantage Estimation, Schulman et al. 2016 Emergence of Locomotion Behaviours in Rich Environments, Heess et al. 2017 MorvanZhou's tutorial page: https://morvanzhou.github.io/tutorials Environment ----------- Openai Gym Pendulum-v0, continual action space Prerequisites -------------- tensorflow >=2.0.0a0 tensorflow-probability 0.6.0 tensorlayer >=2.0.0 To run ------ python tutorial_PPO.py --train/test """ import argparse import os import time import matplotlib.pyplot as plt import numpy as np import gym import tensorflow as tf import tensorflow_probability as tfp import tensorlayer as tl parser = argparse.ArgumentParser(description='Train or test neural net motor controller.') parser.add_argument('--train', dest='train', action='store_true', default=True) parser.add_argument('--test', dest='train', action='store_false') args = parser.parse_args() ##################### hyper parameters #################### ENV_NAME = 'Pendulum-v0' # environment name RANDOMSEED = 1 # random seed EP_MAX = 1000 # total number of episodes for training EP_LEN = 200 # total number of steps for each episode GAMMA = 0.9 # reward discount A_LR = 0.0001 # learning rate for actor C_LR = 0.0002 # learning rate for critic BATCH = 32 # update batchsize A_UPDATE_STEPS = 10 # actor update steps C_UPDATE_STEPS = 10 # critic update steps S_DIM, A_DIM = 3, 1 # state dimension, action dimension EPS = 1e-8 # epsilon METHOD = [ dict(name='kl_pen', kl_target=0.01, lam=0.5), # KL penalty dict(name='clip', epsilon=0.2), # Clipped surrogate objective, find this is better ][1] # choose the method for optimization ############################### PPO #################################### class PPO(object): ''' PPO class ''' def __init__(self): # critic 输出v值 tfs = tl.layers.Input([None, S_DIM], tf.float32, 'state') l1 = tl.layers.Dense(100, tf.nn.relu)(tfs) v = tl.layers.Dense(1)(l1) self.critic = tl.models.Model(tfs, v) self.critic.train() # actor 2个pi网络，固定一个，训练另一个 self.actor = self._build_anet('pi', trainable=True) self.actor_old = self._build_anet('oldpi', trainable=False) self.actor_opt = tf.optimizers.Adam(A_LR) self.critic_opt = tf.optimizers.Adam(C_LR) def a_train(self, tfs, tfa, tfadv): ''' Update policy network :param tfs: state :param tfa: act :param tfadv: advantage :return: ''' tfs = np.array(tfs, np.float32) tfa = np.array(tfa, np.float32) tfadv = np.array(tfadv, np.float32) #优势函数 -值函数的变形 with tf.GradientTape() as tape: mu, sigma = self.actor(tfs) pi = tfp.distributions.Normal(mu, sigma) #初始化策略 正态分布 mu_old, sigma_old = self.actor_old(tfs) oldpi = tfp.distributions.Normal(mu_old, sigma_old) # ratio = tf.exp(pi.log_prob(self.tfa) - oldpi.log_prob(self.tfa)) ratio = pi.prob(tfa) / (oldpi.prob(tfa) + EPS) surr = ratio * tfadv if METHOD['name'] == 'kl_pen': #使用KL作为惩罚项的loss tflam = METHOD['lam'] kl = tfp.distributions.kl_divergence(oldpi, pi) kl_mean = tf.reduce_mean(kl) aloss = -(tf.reduce_mean(surr - tflam * kl)) else: # clipping method, find this is better 使用clip作为loss aloss = -tf.reduce_mean( tf.minimum(surr, tf.clip_by_value(ratio, 1. - METHOD['epsilon'], 1. + METHOD['epsilon']) * tfadv) ) a_gard = tape.gradient(aloss, self.actor.trainable_weights) self.actor_opt.apply_gradients(zip(a_gard, self.actor.trainable_weights)) if METHOD['name'] == 'kl_pen': return kl_mean def update_old_pi(self): ''' Update old policy parameter :return: None ''' for p, oldp in zip(self.actor.trainable_weights, self.actor_old.trainable_weights): oldp.assign(p) #将新的pi的参数直接赋值给旧的pi def c_train(self, tfdc_r, s): #训练critic网络，mse优化 ''' Update actor network :param tfdc_r: cumulative reward :param s: state :return: None ''' tfdc_r = np.array(tfdc_r, dtype=np.float32) with tf.GradientTape() as tape: v = self.critic(s) advantage = tfdc_r - v closs = tf.reduce_mean(tf.square(advantage)) # print('tfdc_r value', tfdc_r) grad = tape.gradient(closs, self.critic.trainable_weights) self.critic_opt.apply_gradients(zip(grad, self.critic.trainable_weights)) def cal_adv(self, tfs, tfdc_r): ''' Calculate advantage :param tfs: state :param tfdc_r: cumulative reward :return: advantage ''' tfdc_r = np.array(tfdc_r, dtype=np.float32) advantage = tfdc_r - self.critic(tfs) return advantage.numpy() def update(self, s, a, r): ''' Update parameter with the constraint of KL divergent :param s: state :param a: act :param r: reward :return: None ''' s, a, r = s.astype(np.float32), a.astype(np.float32), r.astype(np.float32) self.update_old_pi() adv = self.cal_adv(s, r) # adv = (adv - adv.mean())/(adv.std()+1e-6) # sometimes helpful # update actor if METHOD['name'] == 'kl_pen': for _ in range(A_UPDATE_STEPS): kl = self.a_train(s, a, adv) if kl > 4 * METHOD['kl_target']: # this in in google's paper break if kl < METHOD['kl_target'] / 1.5: # adaptive lambda, this is in OpenAI's paper METHOD['lam'] /= 2 elif kl > METHOD['kl_target'] * 1.5: METHOD['lam'] *= 2 METHOD['lam'] = np.clip( METHOD['lam'], 1e-4, 10 ) # sometimes explode, this clipping is MorvanZhou's solution else: # clipping method, find this is better (OpenAI's paper) for _ in range(A_UPDATE_STEPS): self.a_train(s, a, adv) # update critic for _ in range(C_UPDATE_STEPS): self.c_train(r, s) #Critic的训练一次更新十步 def _build_anet(self, name, trainable): ''' Build policy network :param name: name :param trainable: trainable flag :return: policy network ''' tfs = tl.layers.Input([None, S_DIM], tf.float32, name + '_state') l1 = tl.layers.Dense(100, tf.nn.relu, name=name + '_l1')(tfs) a = tl.layers.Dense(A_DIM, tf.nn.tanh, name=name + '_a')(l1) mu = tl.layers.Lambda(lambda x: x * 2, name=name + '_lambda')(a) sigma = tl.layers.Dense(A_DIM, tf.nn.softplus, name=name + '_sigma')(l1) model = tl.models.Model(tfs, [mu, sigma], name) if trainable: model.train() else: model.eval() return model def choose_action(self, s): ''' Choose action :param s: state :return: clipped act ''' s = s[np.newaxis, :].astype(np.float32) mu, sigma = self.actor(s) pi = tfp.distributions.Normal(mu, sigma) a = tf.squeeze(pi.sample(1), axis=0)[0] # choosing action return np.clip(a, -2, 2) def get_v(self, s): ''' Compute value :param s: state :return: value ''' s = s.astype(np.float32) if s.ndim < 2: s = s[np.newaxis, :] return self.critic(s)[0, 0] def save_ckpt(self): """ save trained weights :return: None """ if not os.path.exists('model'): os.makedirs('model') tl.files.save_weights_to_hdf5('model/ppo_actor.hdf5', self.actor) tl.files.save_weights_to_hdf5('model/ppo_actor_old.hdf5', self.actor_old) tl.files.save_weights_to_hdf5('model/ppo_critic.hdf5', self.critic) def load_ckpt(self): """ load trained weights :return: None """ tl.files.load_hdf5_to_weights_in_order('model/ppo_actor.hdf5', self.actor) tl.files.load_hdf5_to_weights_in_order('model/ppo_actor_old.hdf5', self.actor_old) tl.files.load_hdf5_to_weights_in_order('model/ppo_critic.hdf5', self.critic) if __name__ == '__main__': env = gym.make(ENV_NAME).unwrapped # reproducible env.seed(RANDOMSEED) np.random.seed(RANDOMSEED) tf.random.set_seed(RANDOMSEED) ppo = PPO() if args.train: all_ep_r = [] for ep in range(EP_MAX): s = env.reset() buffer_s, buffer_a, buffer_r = [], [], [] ep_r = 0 t0 = time.time() for t in range(EP_LEN): # in one episode # env.render() a = ppo.choose_action(s) s_, r, done, _ = env.step(a) buffer_s.append(s) buffer_a.append(a) buffer_r.append((r + 8) / 8) # normalize reward, find to be useful s = s_ ep_r += r # update ppo if (t + 1) % BATCH == 0 or t == EP_LEN - 1: #采集了一个Batch的数据或者走到最后一步了才进行更新 v_s_ = ppo.get_v(s_) discounted_r = [] for r in buffer_r[::-1]: v_s_ = r + GAMMA * v_s_ discounted_r.append(v_s_) discounted_r.reverse() bs, ba, br = np.vstack(buffer_s), np.vstack(buffer_a), np.array(discounted_r)[:, np.newaxis] buffer_s, buffer_a, buffer_r = [], [], [] ppo.update(bs, ba, br) if ep == 0: all_ep_r.append(ep_r) else: all_ep_r.append(all_ep_r[-1] * 0.9 + ep_r * 0.1) print( 'Episode: {}/{} | Episode Reward: {:.4f} | Running Time: {:.4f}'.format( ep, EP_MAX, ep_r, time.time() - t0 ) ) plt.ion() plt.cla() plt.title('PPO') plt.plot(np.arange(len(all_ep_r)), all_ep_r) plt.ylim(-2000, 0) plt.xlabel('Episode') plt.ylabel('Moving averaged episode reward') plt.show() plt.pause(0.1) ppo.save_ckpt() plt.ioff() plt.show() # test ppo.load_ckpt() while True: s = env.reset() for i in range(EP_LEN): env.render() s, r, done, _ = env.step(ppo.choose_action(s)) if done: break ================================================ FILE: code/tensrolayer-implemented/qlearning.py ================================================ """Q-Table learning algorithm. Non deep learning - TD Learning, Off-Policy, e-Greedy Exploration Q(S, A) <- Q(S, A) + alpha * (R + lambda * Q(newS, newA) - Q(S, A)) See David Silver RL Tutorial Lecture 5 - Q-Learning for more details. For Q-Network, see tutorial_frozenlake_q_network.py EN: https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-0-q-learning-with-tables-and-neural-networks-d195264329d0#.5m3361vlw CN: https://zhuanlan.zhihu.com/p/25710327 tensorflow==2.0.0a0 tensorlayer==2.0.0 """ import time import numpy as np import gym ## Load the environment env = gym.make('FrozenLake-v0') render = False # display the game environment running_reward = None ##================= Implement Q-Table learning algorithm =====================## ## Initialize table with all zeros Q = np.zeros([env.observation_space.n, env.action_space.n]) ## Set learning parameters lr = .85 # alpha, if use value function approximation, we can ignore it lambd = .99 # decay factor num_episodes = 10000 rList = [] # rewards for each episode for i in range(num_episodes): ## Reset environment and get first new observation episode_time = time.time() s = env.reset() rAll = 0 ## The Q-Table learning algorithm for j in range(99): if render: env.render() ## Choose an action by greedily (with noise) picking from Q table a = np.argmax(Q[s, :] + np.random.randn(1, env.action_space.n) * (1. / (i + 1))) ## Get new state and reward from environment s1, r, d, _ = env.step(a) ## Update Q-Table with new knowledge Q[s, a] = Q[s, a] + lr * (r + lambd * np.max(Q[s1, :]) - Q[s, a]) rAll += r s = s1 if d ==True: break rList.append(rAll) running_reward = r if running_reward is None else running_reward * 0.99 + r * 0.01 print("Episode [%d/%d] sum reward: %f running reward: %f took: %.5fs " % \ (i, num_episodes, rAll, running_reward, time.time() - episode_time)) print("Final Q-Table Values:/n %s" % Q) ================================================ FILE: code/tensrolayer-implemented/tutorial_wrappers.py ================================================ """Env wrappers Note that this file is adapted from `https://pypi.org/project/gym-vec-env` and `https://github.com/openai/baselines/blob/master/baselines/common/*wrappers.py` """ from collections import deque from functools import partial from multiprocessing import Pipe, Process, cpu_count from sys import platform import numpy as np import cv2 import gym from gym import spaces __all__ = ( 'build_env', # build env 'TimeLimit', # Time limit wrapper 'NoopResetEnv', # Run random number of no-ops on reset 'FireResetEnv', # Reset wrapper for envs with fire action 'EpisodicLifeEnv', # end-of-life == end-of-episode wrapper 'MaxAndSkipEnv', # skip frame wrapper 'ClipRewardEnv', # clip reward wrapper 'WarpFrame', # warp observation wrapper 'FrameStack', # stack frame wrapper 'LazyFrames', # lazy store wrapper 'RewardScaler', # reward scale 'SubprocVecEnv', # vectorized env wrapper 'VecFrameStack', # stack frames in vectorized env 'Monitor', # Episode reward and length monitor ) cv2.ocl.setUseOpenCL(False) # env_id -> env_type id2type = dict() for _env in gym.envs.registry.all(): id2type[_env.id] = _env.entry_point.split(':')[0].rsplit('.', 1)[1] def build_env(env_id, vectorized=False, seed=0, reward_scale=1.0, nenv=0): """Build env based on options""" env_type = id2type[env_id] nenv = nenv or cpu_count() // (1 + (platform == 'darwin')) stack = env_type == 'atari' if not vectorized: env = _make_env(env_id, env_type, seed, reward_scale, stack) else: env = _make_vec_env(env_id, env_type, nenv, seed, reward_scale, stack) return env def _make_env(env_id, env_type, seed, reward_scale, frame_stack=True): """Make single env""" if env_type == 'atari': env = gym.make(env_id) assert 'NoFrameskip' in env.spec.id env = NoopResetEnv(env, noop_max=30) env = MaxAndSkipEnv(env, skip=4) env = Monitor(env) # deepmind wrap env = EpisodicLifeEnv(env) if 'FIRE' in env.unwrapped.get_action_meanings(): env = FireResetEnv(env) env = WarpFrame(env) env = ClipRewardEnv(env) if frame_stack: env = FrameStack(env, 4) elif env_type == 'classic_control': env = Monitor(gym.make(env_id)) else: raise NotImplementedError if reward_scale != 1: env = RewardScaler(env, reward_scale) env.seed(seed) return env def _make_vec_env(env_id, env_type, nenv, seed, reward_scale, frame_stack=True): """Make vectorized env""" env = SubprocVecEnv([partial(_make_env, env_id, env_type, seed + i, reward_scale, False) for i in range(nenv)]) if frame_stack: env = VecFrameStack(env, 4) return env class TimeLimit(gym.Wrapper): def __init__(self, env, max_episode_steps=None): super(TimeLimit, self).__init__(env) self._max_episode_steps = max_episode_steps self._elapsed_steps = 0 def step(self, ac): observation, reward, done, info = self.env.step(ac) self._elapsed_steps += 1 if self._elapsed_steps >= self._max_episode_steps: done = True info['TimeLimit.truncated'] = True return observation, reward, done, info def reset(self, **kwargs): self._elapsed_steps = 0 return self.env.reset(**kwargs) class NoopResetEnv(gym.Wrapper): def __init__(self, env, noop_max=30): """Sample initial states by taking random number of no-ops on reset. No-op is assumed to be action 0. """ super(NoopResetEnv, self).__init__(env) self.noop_max = noop_max self.override_num_noops = None self.noop_action = 0 assert env.unwrapped.get_action_meanings()[0] == 'NOOP' def reset(self, **kwargs): """ Do no-op action for a number of steps in [1, noop_max].""" self.env.reset(**kwargs) if self.override_num_noops is not None: noops = self.override_num_noops else: noops = self.unwrapped.np_random.randint(1, self.noop_max + 1) assert noops > 0 obs = None for _ in range(noops): obs, _, done, _ = self.env.step(self.noop_action) if done: obs = self.env.reset(**kwargs) return obs def step(self, ac): return self.env.step(ac) class FireResetEnv(gym.Wrapper): def __init__(self, env): """Take action on reset for environments that are fixed until firing.""" super(FireResetEnv, self).__init__(env) assert env.unwrapped.get_action_meanings()[1] == 'FIRE' assert len(env.unwrapped.get_action_meanings()) >= 3 def reset(self, **kwargs): self.env.reset(**kwargs) obs, _, done, _ = self.env.step(1) if done: self.env.reset(**kwargs) obs, _, done, _ = self.env.step(2) if done: self.env.reset(**kwargs) return obs def step(self, ac): return self.env.step(ac) class EpisodicLifeEnv(gym.Wrapper): def __init__(self, env): """Make end-of-life == end-of-episode, but only reset on true game over. Done by DeepMind for the DQN and co. since it helps value estimation. """ super(EpisodicLifeEnv, self).__init__(env) self.lives = 0 self.was_real_done = True def step(self, action): obs, reward, done, info = self.env.step(action) self.was_real_done = done # check current lives, make loss of life terminal, # then update lives to handle bonus lives lives = self.env.unwrapped.ale.lives() if 0 < lives < self.lives: # for Qbert sometimes we stay in lives == 0 condition for a few # frames so it's important to keep lives > 0, so that we only reset # once the environment advertises done. done = True self.lives = lives return obs, reward, done, info def reset(self, **kwargs): """Reset only when lives are exhausted. This way all states are still reachable even though lives are episodic, and the learner need not know about any of this behind-the-scenes. """ if self.was_real_done: obs = self.env.reset(**kwargs) else: # no-op step to advance from terminal/lost life state obs, _, _, _ = self.env.step(0) self.lives = self.env.unwrapped.ale.lives() return obs class MaxAndSkipEnv(gym.Wrapper): def __init__(self, env, skip=4): """Return only every `skip`-th frame""" super(MaxAndSkipEnv, self).__init__(env) # most recent raw observations (for max pooling across time steps) shape = (2, ) + env.observation_space.shape self._obs_buffer = np.zeros(shape, dtype=np.uint8) self._skip = skip def step(self, action): """Repeat action, sum reward, and max over last observations.""" total_reward = 0.0 done = info = None for i in range(self._skip): obs, reward, done, info = self.env.step(action) if i == self._skip - 2: self._obs_buffer[0] = obs if i == self._skip - 1: self._obs_buffer[1] = obs total_reward += reward if done: break # Note that the observation on the done=True frame doesn't matter max_frame = self._obs_buffer.max(axis=0) return max_frame, total_reward, done, info def reset(self, **kwargs): return self.env.reset(**kwargs) class ClipRewardEnv(gym.RewardWrapper): def __init__(self, env): super(ClipRewardEnv, self).__init__(env) def reward(self, reward): """Bin reward to {+1, 0, -1} by its sign.""" return np.sign(reward) class WarpFrame(gym.ObservationWrapper): def __init__(self, env, width=84, height=84, grayscale=True): """Warp frames to 84x84 as done in the Nature paper and later work.""" super(WarpFrame, self).__init__(env) self.width = width self.height = height self.grayscale = grayscale shape = (self.height, self.width, 1 if self.grayscale else 3) self.observation_space = spaces.Box(low=0, high=255, shape=shape, dtype=np.uint8) def observation(self, frame): if self.grayscale: frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) size = (self.width, self.height) frame = cv2.resize(frame, size, interpolation=cv2.INTER_AREA) if self.grayscale: frame = np.expand_dims(frame, -1) return frame class FrameStack(gym.Wrapper): def __init__(self, env, k): """Stack k last frames. Returns lazy array, which is much more memory efficient. See Also `LazyFrames` """ super(FrameStack, self).__init__(env) self.k = k self.frames = deque([], maxlen=k) shp = env.observation_space.shape shape = shp[:-1] + (shp[-1] * k, ) self.observation_space = spaces.Box(low=0, high=255, shape=shape, dtype=env.observation_space.dtype) def reset(self): ob = self.env.reset() for _ in range(self.k): self.frames.append(ob) return np.asarray(self._get_ob()) def step(self, action): ob, reward, done, info = self.env.step(action) self.frames.append(ob) return np.asarray(self._get_ob()), reward, done, info def _get_ob(self): assert len(self.frames) == self.k return LazyFrames(list(self.frames)) class LazyFrames(object): def __init__(self, frames): """This object ensures that common frames between the observations are only stored once. It exists purely to optimize memory usage which can be huge for DQN's 1M frames replay buffers. This object should only be converted to numpy array before being passed to the model. You'd not believe how complex the previous solution was. """ self._frames = frames self._out = None def _force(self): if self._out is None: self._out = np.concatenate(self._frames, axis=-1) self._frames = None return self._out def __array__(self, dtype=None): out = self._force() if dtype is not None: out = out.astype(dtype) return out def __len__(self): return len(self._force()) def __getitem__(self, i): return self._force()[i] class RewardScaler(gym.RewardWrapper): """Bring rewards to a reasonable scale for PPO. This is incredibly important and effects performance drastically. """ def __init__(self, env, scale=0.01): super(RewardScaler, self).__init__(env) self.scale = scale def reward(self, reward): return reward * self.scale class VecFrameStack(object): def __init__(self, env, k): self.env = env self.k = k self.action_space = env.action_space self.frames = deque([], maxlen=k) shp = env.observation_space.shape shape = shp[:-1] + (shp[-1] * k, ) self.observation_space = spaces.Box(low=0, high=255, shape=shape, dtype=env.observation_space.dtype) def reset(self): ob = self.env.reset() for _ in range(self.k): self.frames.append(ob) return np.asarray(self._get_ob()) def step(self, action): ob, reward, done, info = self.env.step(action) self.frames.append(ob) return np.asarray(self._get_ob()), reward, done, info def _get_ob(self): assert len(self.frames) == self.k return LazyFrames(list(self.frames)) def _worker(remote, parent_remote, env_fn_wrapper): parent_remote.close() env = env_fn_wrapper.x() while True: cmd, data = remote.recv() if cmd == 'step': ob, reward, done, info = env.step(data) if done: ob = env.reset() remote.send((ob, reward, done, info)) elif cmd == 'reset': ob = env.reset() remote.send(ob) elif cmd == 'reset_task': ob = env._reset_task() remote.send(ob) elif cmd == 'close': remote.close() break elif cmd == 'get_spaces': remote.send((env.observation_space, env.action_space)) else: raise NotImplementedError class CloudpickleWrapper(object): """ Uses cloudpickle to serialize contents """ def __init__(self, x): self.x = x def __getstate__(self): import cloudpickle return cloudpickle.dumps(self.x) def __setstate__(self, ob): import pickle self.x = pickle.loads(ob) class SubprocVecEnv(object): def __init__(self, env_fns): """ envs: list of gym environments to run in subprocesses """ self.num_envs = len(env_fns) self.waiting = False self.closed = False nenvs = len(env_fns) self.nenvs = nenvs self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)]) zipped_args = zip(self.work_remotes, self.remotes, env_fns) self.ps = [ Process(target=_worker, args=(work_remote, remote, CloudpickleWrapper(env_fn))) for (work_remote, remote, env_fn) in zipped_args ] for p in self.ps: # if the main process crashes, we should not cause things to hang p.daemon = True p.start() for remote in self.work_remotes: remote.close() self.remotes[0].send(('get_spaces', None)) observation_space, action_space = self.remotes[0].recv() self.observation_space = observation_space self.action_space = action_space def _step_async(self, actions): """ Tell all the environments to start taking a step with the given actions. Call step_wait() to get the results of the step. You should not call this if a step_async run is already pending. """ for remote, action in zip(self.remotes, actions): remote.send(('step', action)) self.waiting = True def _step_wait(self): """ Wait for the step taken with step_async(). Returns (obs, rews, dones, infos): - obs: an array of observations, or a tuple of arrays of observations. - rews: an array of rewards - dones: an array of "episode done" booleans - infos: a sequence of info objects """ results = [remote.recv() for remote in self.remotes] self.waiting = False obs, rews, dones, infos = zip(*results) return np.stack(obs), np.stack(rews), np.stack(dones), infos def reset(self): """ Reset all the environments and return an array of observations, or a tuple of observation arrays. If step_async is still doing work, that work will be cancelled and step_wait() should not be called until step_async() is invoked again. """ for remote in self.remotes: remote.send(('reset', None)) return np.stack([remote.recv() for remote in self.remotes]) def _reset_task(self): for remote in self.remotes: remote.send(('reset_task', None)) return np.stack([remote.recv() for remote in self.remotes]) def close(self): if self.closed: return if self.waiting: for remote in self.remotes: remote.recv() for remote in self.remotes: remote.send(('close', None)) for p in self.ps: p.join() self.closed = True def __len__(self): return self.nenvs def step(self, actions): self._step_async(actions) return self._step_wait() class Monitor(gym.Wrapper): def __init__(self, env): super(Monitor, self).__init__(env) self._monitor_rewards = None def reset(self, **kwargs): self._monitor_rewards = [] return self.env.reset(**kwargs) def step(self, action): o_, r, done, info = self.env.step(action) self._monitor_rewards.append(r) if done: info['episode'] = {'r': sum(self._monitor_rewards), 'l': len(self._monitor_rewards)} return o_, r, done, info class NormalizedActions(gym.ActionWrapper): def _action(self, action): low = self.action_space.low high = self.action_space.high action = low + (action + 1.0) * 0.5 * (high - low) action = np.clip(action, low, high) return action def _reverse_action(self, action): low = self.action_space.low high = self.action_space.high action = 2 * (action - low) / (high - low) - 1 action = np.clip(action, low, high) return action def unit_test(): env_id = 'CartPole-v0' unwrapped_env = gym.make(env_id) wrapped_env = build_env(env_id, False) o = wrapped_env.reset() print('Reset {} observation shape {}'.format(env_id, o.shape)) done = False while not done: a = unwrapped_env.action_space.sample() o_, r, done, info = wrapped_env.step(a) print('Take action {} get reward {} info {}'.format(a, r, info)) env_id = 'PongNoFrameskip-v4' nenv = 2 unwrapped_env = gym.make(env_id) wrapped_env = build_env(env_id, True, nenv=nenv) o = wrapped_env.reset() print('Reset {} observation shape {}'.format(env_id, o.shape)) for _ in range(1000): a = [unwrapped_env.action_space.sample() for _ in range(nenv)] a = np.asarray(a, 'int64') o_, r, done, info = wrapped_env.step(a) print('Take action {} get reward {} info {}'.format(a, r, info)) if __name__ == '__main__': unit_test() ================================================ FILE: notes/1 Introduction.md ================================================ # 李宏毅深度强化学习 笔记 课程主页：[NTU-MLDS18](http://speech.ee.ntu.edu.tw/~tlkagk/courses_MLDS18.html) 视频：[youtube](https://www.youtube.com/playlist?list=PLJV_el3uVTsODxQFgzMzPLa16h6B8kWM_) [B站](https://www.bilibili.com/video/av24724071/?spm_id_from=333.788.videocard.4) 参考资料： [作业代码参考](https://github.com/JasonYao81000/MLDS2018SPRING/tree/master/hw4) [纯numpy实现非Deep的RL算法](https://github.com/ddbourgin/numpy-ml/tree/master/numpy_ml/rl_models) [OpenAI tutorial](https://github.com/openai/spinningup/tree/master/docs) # 1. Introduction ![1](http://oss.hackslog.cn/imgs/075034.png) 这门课的学习路线如上，强化学习是作为单独一个模块介绍。李宏毅老师讲这门课不是从MDP开始讲起，而是从如何获得最大化奖励出发，直接引出Policy Gradient（以及PPO），再讲Q-learning（原始Q-learning，DQN，各种DQN的升级），然后是A2C（以及A3C, DDPG），紧接着介绍了一些Reward Shaping的方法（主要是Curiosity，Curriculum Learning ，Hierarchical RL），最后介绍Imitation Learning (Inverse RL)。比较全面的展现了深度强化学习的核心内容，也比较直观。 ![image-20191029211249836](http://oss.hackslog.cn/imgs/075024.png) 首先强化学习是一种解决序列决策问题的方法，他是通过与环境交互进行学习。首先会有一个Env，给agent一个state，agent根据得到的state执行一个action，这个action会改变Env，使自己跳转到下一个state，同时Env会反馈给agent一个reward，agent学习的目标就是通过采取action，使得reward的期望最大化。 ![image-20191029211454593](http://oss.hackslog.cn/imgs/075050.png) 在alpha go的例子中，state（又称observation）为所看到的棋盘，action就是落子，reward通过围棋的规则给出，如果最终赢了，得1，输了，得-1。 下面从2个例子中看强化学习与有监督学习的区别。RL不需要给定标签，但需要有reward。 ![image-20191029211749897](http://oss.hackslog.cn/imgs/075057.png) 实际上alphgo是从提前收集的数据上进行有监督学习，效果不错后，再去做强化学习，提高水平。 ![image-20191029211848429](http://oss.hackslog.cn/imgs/075104.png) 人没有告诉机器人具体哪里说错了，机器需要根据最终的评价自己总结，一般需要对话好多次。所以通常训练对话模型会训练2个agent互相对话 ![image-20191029212015623](http://oss.hackslog.cn/imgs/075108.png) ![image-20191029212144625](http://oss.hackslog.cn/imgs/075112.png) 一个难点是怎么判断对话的效果，一般会设置一些预先定义的规则。 ![image-20191029212313069](http://oss.hackslog.cn/imgs/075117.png) 强化学习还有很多成功的应用，凡是序列决策问题，大多数可以用RL解决。 ================================================ FILE: notes/2 Policy Gradient.md ================================================ # 2. Policy Gradient ## 2.1 Origin Policy Gradient ![](http://oss.hackslog.cn/imgs/075626.jpg) 在alpha go场景中，actor决定下哪个位置，env就是你的对手，reward是围棋的规则。强化学习三大基本组件里面，env和reward是事先给定的，我们唯一能做的就是通过调整actor，使得到的累积reward最大化。 ![](http://oss.hackslog.cn/imgs/075657.jpg) 一般把actor的策略定义成Policy，数学符号为$\pi$，参数是$\theta$，本质是一个NN（神经网络）。 那么针对Atari游戏：输入游戏的画面，Policy $\pi$输出各个动作的概率，agent根据这个概率分布采取行动。通过调整$\theta$, 我们就可以调整策略的输出。 ![](http://oss.hackslog.cn/imgs/075712.jpg)_page-0007) 每次采取一个行动会有一个reward ![](http://oss.hackslog.cn/imgs/075809.jpg) 玩一场游戏叫做一个episode，actor存在的目的就是最大化所能得到的return，这个return指的是每一个时间步得到的reward之和。注意我们期望最大化的是return，不是一个时刻的reward。 如果max的目标是当下时刻的reward，那么在Atari游戏中如果agent在某个s下执行开火，得到了较大的reward，那么可能agent就会一直选择开火。并不代表，最终能够取得游戏的胜利。 那么，怎么得到这个actor呢？ ![](http://oss.hackslog.cn/imgs/075903.jpg) 先定义玩一次游戏，即一个episode的游戏记录为trajectory $\tau$，内容如图所示，是s-a组成的序列对。 假设actor的参数$\theta$已经给定，则可以得到每个$\tau$出现的概率。这个概率取决于两部分，$p\left(s_{t+1} | s_{t}, a_{t}\right)$部分由env的机制决定，actor没法控制，我们能控制的是$p_{\theta}\left(a_{t} | s_{t}\right)$ 由$\pi$的参数$\theta$决定。 ![](http://oss.hackslog.cn/imgs/075918.jpg) 定义$R(\tau)$ 为一个episode的总的reward，即每个时间步下的即时reward相加，我习惯表述为return。 定义$\bar{R}_{\theta}$ 为$R(\tau)$的期望，等价于将每一个轨迹$\tau$出现的概率乘与其return，再求和。 由于$R(\tau)$是一个随机变量，因为actor本身在给定同样的state下会采取什么行为具有随机性，env在给定行为下输出什么state，也是随机的，所以只能算$R(\tau)$的期望。 我们的目标就变成了最大化Expected Reward，那么如何最大化？ ![](http://oss.hackslog.cn/imgs/075954.jpg) 优化算法是梯度更新，首先我们先计算出$\bar{R}_{\theta}$ 对$\theta$的梯度。 从公式中可以看出$R(\tau)$可以是不可微的，因为与参数无关，不需要求导。 第一个改写（红色部分）：将加权求和写成期望的形式。 第二个近似：实际上没有办法把所有可能的轨迹（游戏记录）都求出来，所以一般是采样N个轨迹 第三个改写：将$p_{\theta}\left(\tau^{n}\right)$的表达式展开(前2页slide)，去掉跟$\theta$无关的项（不需要求导），则可达到最终的简化结果。具体如下：首先用actor采集一个游戏记录 ![image-20191029215615001](http://oss.hackslog.cn/imgs/2019-11-06-080254.png) ![image-20191029220147651](http://oss.hackslog.cn/imgs/2019-11-06-080301.png) 最终得到的公式相当的直觉，在s下采取了a导致最终结果赢了，那么return就是正的，也就是会增加相应的s-a出现的概率P。 上面的公式推导中可能会有疑问，为什么要引入log？再乘一个概率除一个概率？原因非常的直觉，如下：如果动作b本来出现的次数就多，那么在加权平均所有的episode后，参数会偏好执行动作b，而实际上动作b得到的return比a低，所以除掉自身出现的概率，以降低其对训练的影响。 ![image-20191029220546039](http://oss.hackslog.cn/imgs/2019-11-06-080313.png) 那么，到底是怎么更新参数的呢？ ![](http://oss.hackslog.cn/imgs/080148.jpg) 首先会拿agent跟环境互动，收集大量游戏记录，然后把每一个游戏记录拿到右边，计算一个参数theta的更新值，更新参数后，再拿新的actor去收集游戏记录，不断循环。 注意：一般采样的数据只会用一次，用完就丢弃 ![](http://oss.hackslog.cn/imgs/2019-11-06-080339.jpg) 具体实现：可当成一个分类任务，只是分类的结果不是识别object，是给出actor要执行的动作。 如何构建训练集？ 采样得到的a，作为ground truth。然后去最小化loss function。 一般的分类问题loss function是交叉熵，在强化学习里面，只需要在前面乘一个weight，即交叉熵乘一个return。 实现的过程中还有一些tips可以提高效果： ![](http://oss.hackslog.cn/imgs/2019-11-06-080618.jpg) 如果 reward都是正的，那么理想的情况下：reward从大到小 b>a>c, 出现次数 b>a>c, 经过训练以后，reward值高的a，c会提高出现的概率，b会降低。但如果a没有采样到，则a出现的概率最终可能会下降，尽管a的reward高。 解决方法：增加一个baseline，用r-b作为新的reward，让其有正有负。最简单的做法是b取所有轨迹的平均回报。 一般r-b叫做优势函数Advantage Functions。我们不需要描述一个行动的绝对好坏，而只需要知道它相对于平均水平的优势。 ![](http://oss.hackslog.cn/imgs/2019-11-06-080716.jpg) 在这个公式里面，对于一个轨迹，每一个s-a的pair都会乘同一个weight，显然不公平，因为一局游戏里面往往有好的pair，有对结果不好的pair。所以我们希望给每一个pair乘不同的weight。整场游戏结果是好的，不代表每一个pair都是好的。如果sample次数够多，则不存在这个问题。 解决思路：在执行当下action之前的事情跟其没有关系，无论得到多少功劳都跟它没有关系，只考虑在当下执行pair之后的reward，这才是它真正的贡献。把原来的总的return，换成未来的return。 如图：对于第一组数据，在($s_a$,$a_1$)时候总的return是+3，那么如果对每一个pair都乘3，则($s_b$,$a_2$)会认为是有效的，但如果使用改进的思路，将其乘之后的return，即-2，则能有效抑制该pair对结果的贡献。 再改进：加一个折扣系数，如果时间拖得越长，对于越之后的reward，当下action的功劳越小。 ![](http://oss.hackslog.cn/imgs/2019-11-06-080806.jpg) 我们将R-b 记为 A，意义就是评价当前s执行动作a，相比于采取其他的a，它有多好。之后我们会用一个critic网络来估计这个评价值。 ## 2.2 PPO PPO算法是PG算法的变形,目的是把在线的学习变成离线的学习。 核心的idea是对每一条经验（又称轨迹，即一个episode的游戏记录）不止使用一次。 简单理解：在线学习就是一边玩一边学，离线学习就是先看着别人玩进行学习，之后再自己玩 ![](http://oss.hackslog.cn/imgs/2019-11-06-080842.jpg) ![](http://oss.hackslog.cn/imgs/2019-11-06-080853.jpg) Motivation：每次用$\pi_\theta$去采样数据之后，$\pi_\theta$都会更新，接下来又要采样新的数据。以至于PG算法大部分时间都在采样数据。那么能不能将这些数据保存下来，由另一个$\pi_{\theta'}$去更新参数？那么策略$\pi_\theta$采样的数据就能被$\pi_{\theta'}$多次利用。引入统计学中的经典方法： 重要性采样：如果想求一个函数的期望，但无法积分，则可以通过采样求平均的方法来近似，但是如果p分布不知道（无法采样），我们知道q分布，则如上图通过一个重要性权重，用q分布来替代p分布进行采样。这个重要性权重的作用就是修正两个分布的差异。 ![](http://oss.hackslog.cn/imgs/2019-11-06-080922.jpg) 存在的问题：如果p跟q的差异比较大，则方差会很大 ![](http://oss.hackslog.cn/imgs/2019-11-06-080951.jpg) 如果sample的次数不够多，比如按原分布p进行采样，最终f的期望值是负数（大部分概率都在左侧，左侧f是负值），如果按q分布进行sample，只sample到右边，则f就一直是正的，严重偏离原来的分布。当然采样次数够多的时候，q也sample到了左边，则p/q这个负weight非常大，会平衡掉右边的正值，会导致最终计算出的期望值仍然是负值。但实际上采样的次数总是有限的，出现这种问题的概率也很大。 先忽略这个问题，加入重要性采样之后，训练变成了离线的 ![](http://oss.hackslog.cn/imgs/2019-11-06-081110.jpg) 离线训练的实现：用另一个policy2与环境做互动，采集数据，然后在这个数据上训练policy1。尽管2个采集的数据分布不一样，但加入一个重要性的weights，可以修正其差异。等policy1训练的差不多以后，policy2再去采集数据，不断循环。 ![](http://oss.hackslog.cn/imgs/2019-11-06-081122.jpg) 由于我们得到的$A^{\theta}\left(s_{t}, a_{t}\right)$（执行当下action后得到reward-baseline）是由policy2采集的数据观察得到的，所以 $A^{\theta}\left(s_{t}, a_{t}\right)$的参数得修正为$\theta'$ 根据$\nabla f(x)=f(x) \nabla \log f(x)$反推目标函数$J$，注意要优化的参数是$\theta$ ，$\theta’$只负责采集数据。 利用 $\theta’$采集的数据来训练$\theta$，会不会有问题？（虽然有修正，但毕竟还是不同） 答案是我们需要保证他们的差异尽可能的小，那么在刚刚的公式里再加入一些限制保证其差异足够小，则诞生了 PPO算法。 ![](http://oss.hackslog.cn/imgs/2019-11-06-081210.jpg) 引入函数KL，KL衡量两个分布的距离。注意：不是参数上的距离，是2个$\pi$给同样的state之后基于各自参数输出的action space的距离 加入KL的公式直觉的理解：如果我们学习出来的$\theta$跟$\theta'$越像，则KL越小，J越大。我们的学习目标还是跟原先的PG算法一样，用梯度上升训练，最大化J。这个操作有点像正则化，用来解决重要性采样存在的问题。 TRPO是PPO是前身，把KL这个限制条件放在优化的目标函数外面。对于梯度上升的优化过程，这种限制比较难处理，使优化变得复杂，一般不用。 ![](http://oss.hackslog.cn/imgs/2019-11-06-081228.jpg) 实现过程：初始化policy参数，在每一个迭代里面，用$theta^k$采集很多数据，同时计算出奖励A值，接着train这个数据，更新$\theta$优化J。由于是离线训练，可以多次更新后，再去采集新的数据。 有一个trick是KL的权重beta也可以调整，使其更加的adaptive。 ![](http://oss.hackslog.cn/imgs/2019-11-06-081245.jpg) ![](http://oss.hackslog.cn/imgs/2019-11-06-081315.jpg) 实际上KL也是比较难计算的，所以有了PPO2算法，不计算KL，通过clip达到同样效果。 clip(a, b, c): if a b If a>c => c If b a 看图：绿色：min里面的第一项，蓝色：min里面的第二项，红色 min的输出 这个公式的直觉理解：希望$\theta$与$\theta^k$在优化之后不要差距太大。如果A>0，即这个state-action是好的，所以需要增加这个pair出现的几率，所以在max J的过程中会增大$\frac{p_{\theta}\left(a_{t} | s_{t}\right)}{p_{\theta^{k}}\left(a_{t} | s_{t}\right)}$, 但最大不要超过1+eplison，如果A<0，不断减小，小到1-eplison，始终不会相差太大。 ![](http://oss.hackslog.cn/imgs/2019-11-06-081323.jpg) PG算法效果非常不稳定，自从有了PPO，PG的算法可以在很多任务上work。 ================================================ FILE: notes/3 Q - Learning.md ================================================ # 3. Q - Learning ## 3.1 Q-learning 在之前的policy-based算法里，我们的目标是learn 一个actor，value-based的强化学习算法目标是learn一个critic。 定义一个Critic，也就是状态值函数$V^{\pi}(s)$，它的值是：当使用策略$\pi$进行游戏时，在观察到一个state s之后，环境输出的累积的reward值的期望。注意取决于两个值，一个是state s，一个是actor$\pi$。 ![](http://oss.hackslog.cn/imgs/2019-11-06-081510.jpg) ![0004](http://oss.hackslog.cn/imgs/2019-11-06-081524.jpg) 如果是不同的actor，在同样的state下，critic给出的值也是不同的。那么怎么估计出这个函数V呢？ 主要有MC和TD的方法，实际上还有DP的方法，但是用DP求解需要整个环境都是已知的。而在强化学习的大部分任务里，都是model-free的，需要agent自己去探索环境。 ![0005](http://oss.hackslog.cn/imgs/2019-11-06-081539.jpg) MC：直接让agent与环境互动，统计计算出在$S_a$之后直到一个episode结束的累积reward作为$G_a$。 训练的目标就是让$V^{\pi}(s)$的输出尽可能的接近$G_a$。 ![0006](http://oss.hackslog.cn/imgs/2019-11-06-081554.jpg) MC每次必须把游戏玩到结束，TD不需要把游戏玩到底，只需要玩了一次游戏，有一个状态的变化。 那么训练的目标就是让$V^{\pi}(s_t)$ 和$V^{\pi}(s_t+1)$的差接近$r_t$ ![0007](http://oss.hackslog.cn/imgs/2019-11-06-081607.jpg) MC方差大，因为$r$是一个随机变量，MC方法中的$G$是$r$之和，而TD方法只有$r$是随机变量，r的方差比G小。但TD方法的$V^{\pi}$有可能估计的不准。 ![0008](http://oss.hackslog.cn/imgs/2019-11-06-081621.jpg) 用MC和TD估计的结果不一样 ![0009](http://oss.hackslog.cn/imgs/2019-11-06-081633.jpg) 定义另一种Critic，状态-动作值函数$Q^{\pi}(s,a)$，有的地方叫做Q-function，输入是一个pair $(s,a)$，意思是用$\pi$玩游戏时，在s状态下强制执行动作a（策略$\pi$在s下不一定会执行a），所得到的累积reward。 有两种写法，输入pair，输出Q，此时的Q是一个标量。 另一种是输入s，输出所有可能的action的Q值，此时Q是一个向量。 那么Critic到底怎么用呢？ ![](http://oss.hackslog.cn/imgs/2019-11-06-081645.jpg) Q-learning的过程： 初始化一个actor $\pi$去收集数据，然后learn一个基于$ \pi$的Q-function，接着寻找一个新的比原来的$\pi$要好actor , 找到后更新$\pi$，再去寻找新的Q-function，不断循环，得到更好的policy。 可见Q-learning的核心思想是先找到最优的Q-function，再通过这个Q-function得出最优策略。而Policy-based的算法是直接去学习策略。这是本质区别。 那么，怎么样才算比原来的好？ ![0012](http://oss.hackslog.cn/imgs/2019-11-06-081701.jpg) 定义好的策略：对所有可能的s而言，$V_\pi(s)$一定小于$V_\pi'(s)$，则$V_\pi'(s)$就是更好的策略。 $\pi'(s)$的本质：假设已经学习到了一个actor $\pi$的Q-function，给一个state，把所有可能的action都代入Q，执行那个可以让Q最大的action。 注意：实际上，给定一个s，$ \pi$不一定会执行a，现在的做法是强制执行a，计算执行之后玩下去得到的reward进行比较。 在实现的时候$\pi'$没有额外的参数，依赖于Q。并且当动作是连续值的时候，无法进行argmax。 那么， 为什么actor $\pi’$能被找到？ ![0013](http://oss.hackslog.cn/imgs/2019-11-06-081720.jpg) 上面是为了证明：只要你估计出了一个actor的Q-function，则一定可以找到一个更好的actor。 核心思想：在一个episode中某一步把$\pi$换成了$ \pi'$比完全follow $ \pi$，得到的奖励期望值会更大。 注意$r_{t+1}$指的是在执行当下$a_t$得到的奖励，有的文献也会写成$r_t$ 训练的时候有一些Tips可以提高效率： ![0014](http://oss.hackslog.cn/imgs/2019-11-06-081740.jpg) Tips 1 引入target网络 训练的时候，每次需要两个Q function（两个的输入不同）一起更新，不稳定。 一般会固定一个Q作为Target，产生回归任务的label，在训练N次之后，再更新target的参数。回归任务的目标，让$Q^\pi(s_t,a_t)$与$\mathrm{Q}^{\pi}\left(s_{t+1}, \pi\left(s_{t+1}\right)\right))+r$越来越接近，即降低mse。最终希望训练得到的$Q^\pi$能直接估计出这个$(s_t,a_t)$未来的一个累积奖励。 注意：target网络的参数不需要训练，直接每隔N次复制Q的参数。训练的目标只有一个 Q。 ![0015](http://oss.hackslog.cn/imgs/2019-11-06-081754.jpg) Tips2 改进探索机制 PG算法，每次都会sample新的action，随机性比较大，大概率会尽可能的覆盖所有的动作。而之前的Q-learning，策略的本质是绝对贪婪策略，那么如果有的action没有被sample到，则可能之后再也不会选择这样的action。这种探索的机制（收集数据的方法）不好，所以改进贪心算法，让actor每次会$\varepsilon$的概率执行随机动作。 ![0016](http://oss.hackslog.cn/imgs/2019-11-06-081820.jpg) ![0017](http://oss.hackslog.cn/imgs/2019-11-06-081833.jpg) Tips 3 引入记忆池机制 将采集到的一些数据收集起来，放入replay buffer。好处： 1.可重复使用过去的policy采集的数据，降低agent与环境互动的次数，加快训练效率 。 2.replay buffer里面包含了不同actor采集的数据，这样每次随机抽取一个batch进行训练的时候，每个batch内的数据会有较大的差异（数据更加diverse），有助于训练。 那么，当我们训练的目标是$ \pi$的Q-function，训练数据混杂了$\pi’$,$\pi’'$,$\pi’''$采集的数据 有没有问题呢？没有，不是因为这些$ \pi$很像，主要原因是我们采样的不是一个轨迹，只是采样了一笔experience($s_t,a_t,r_t,s_{t+1}$)。这个理论上证明是没有问题的，很难解释... ![0018](http://oss.hackslog.cn/imgs/2019-11-06-081844.jpg) 采用了3个Tips的Q-learning训练过程如图： 注意图中省略了一个循环，即存储了很多笔experience之后才会进行sample。相比于原始的Q-learning，每次sample是从replay buff里面随机抽一个batch，然后计算用绝对贪心策略得到Q-target的值作为label，接着在回归任务中更新Q的参数。每训练多步后，更新Q-target的参数。 ## 3.2 Tips of Q-learning ![](http://oss.hackslog.cn/imgs/2019-11-06-081854.jpg) DQN估计出的值一般都高于实际的值，double DQN估计出的值与实际值比较接近。 ![0021](http://oss.hackslog.cn/imgs/2019-11-06-081906.jpg) Q是一个估计值，被高估的越多，越容易被选择。 ![0022](http://oss.hackslog.cn/imgs/2019-11-06-081948.jpg) Double的思想有点像行政跟立法分权。 用要训练的Q-network去选择动作，用固定不动的target-network去做估计，相比于DQN,只需要改一行代码！ ![0023](http://oss.hackslog.cn/imgs/2019-11-06-081958.jpg) 改了network架构，其他没动。每个网络结构的输出是一个标量+一个向量 ![0024](http://oss.hackslog.cn/imgs/2019-11-06-082009.jpg) 比如下一时刻，我们需要把3->4, 0->-1,那么Dueling结构里会倾向于不修改A，只调整V来达到目的，这样只需要把V中 0->1, 如果Q中的第三行-2没有被sample到，也进行了更新，提高效率，减少训练次数。 ![0025](http://oss.hackslog.cn/imgs/2019-11-06-082019.jpg) 实际实现的时候，通过添加了限制条件，也就是把A normalize，使得其和为0，这样只会更新V。 这种结构让DQN也能处理连续的动作空间。 ![0028](http://oss.hackslog.cn/imgs/2019-11-06-082253.jpg) 加入权重的replay buffer motivation：TD error大的数据应该更可能被采样到 注意论文原文实现的细节里，也修改了参数更新的方法 ![0029](http://oss.hackslog.cn/imgs/2019-11-06-082300.jpg) 原来收集一条experience是执行一个step，现在变成执行N个step。相比TD的好处：之前只sample一个$(s_t,a_t)$pair，现在sample多个才估测Q值，估计的误差会更小。坏处，与MC一样，reward的项数比较多，相加的方差更大。 调N就是一个trade-off的过程。 ![0030](http://oss.hackslog.cn/imgs/2019-11-06-082309.jpg) 在Q-function的参数空间上+noise 比较有意思的是，OpenAI DeepMind几乎在同一个时间发布了Noisy Net思想的论文。 ![0031](http://oss.hackslog.cn/imgs/2019-11-06-082316.jpg) 在同一个episode里面，在动作空间上加噪声，会导致相同state下执行的action不一样。而在参数空间加噪声，则在相同或者相似的state下，会采取同一个action。 注意加噪声只是为了在不同的episode的里面，train Q的时候不会针对特定的一个state永远只执行一个特定的action。 ![0033](http://oss.hackslog.cn/imgs/2019-11-06-082325.jpg) 带分布的Q-function Motivation：原来计算Q-function的值是通过累积reward的期望，也就是均值，但实际上累积的reward可能在不同的分布下会得到相同的Q值。 注意：每个Q-function的本质都是一个概率分布。 ![0034](http://oss.hackslog.cn/imgs/2019-11-06-082332.jpg) 让$Q^ \pi$直接输出每一个Q-function的分布，但实际上选择action的时候还是会根据mean值大的选。不过拥有了这个分布，可以计算方差，这样如果有的任务需要在追求回报最大的同时降低风险，则可以利用这个分布。 ![0036](http://oss.hackslog.cn/imgs/2019-11-06-082339.jpg) ![0037](http://oss.hackslog.cn/imgs/2019-11-06-082345.jpg) Rainbow：集成了7种升级技术的DQN 上图是一个一个改进拿掉之后的效果，看紫色似乎double 没啥用，实际上是因为有Q-function的分布存在，一般不会过高估计Q值，所以double 意义不大。 直觉的理解：使用分布DQN，即时Q值被高估很多，由于最终只会映射到对应的分布区间，所以最终的输出值也不会过大。 ## 3.3 Q-learning in continuous actions 在出现PPO之前, PG的算法非常不稳定。DQN 比较稳定，也容易train，因为DQN是只要估计出Q-function，就能得到好的policy，而估计Q-function就是一个回归问题，回归问题比较容易判断learn的效果，看mse。问题是Q-learning不太容易处理连续动作空间。比如汽车的速度，是一个连续变量。 ![0039](http://oss.hackslog.cn/imgs/2019-11-06-082354.jpg) 当动作值是连续时，怎么解argmax： 1. 通过映射，强行离散化 2. 使用梯度上升解这个公式，这相当于每次train完Q后，在选择action的时候又要train一次网络，比较耗时间。 ![0040](http://oss.hackslog.cn/imgs/2019-11-06-082404.jpg) 3. 设计特定的网络，使得输出还是一个标量。 ![0042](http://oss.hackslog.cn/imgs/2019-11-06-082433.jpg) 最有效的解决方法是，针对连续动作空间，不要使用Q-learning。使用AC算法！ ================================================ FILE: notes/4 Actor Critic.md ================================================ # 4. Actor Critic ## 4.1 Advantage Actor-Critic (A2C) ![](http://oss.hackslog.cn/imgs/2019-11-06-082605.jpg) 由于每次在执行PG算法之前，一般只能采样少量的数据，导致对于同一个$(s_t,a_t)$，得到的$G$的值方差很大，不稳定。那么能不能直接估计出期望值，来替代采样的结果？ ![AC-4](http://oss.hackslog.cn/imgs/2019-11-06-082530.jpg) 回顾下Q-learning中的定义，我们发现： ![AC-5](http://oss.hackslog.cn/imgs/2019-11-06-082602.jpg) PG算法中G的期望的定义恰好也是Q-learning算法中$Q^\pi(s,a)$的定义： 假设现在的policy是$ \pi$的情况下，在某一个s，采取某一个a以后得到的累积reward的期望值。 因此在这里将Q-learning引入到预估reward中，也即policy gradient和q-learning的结合,叫做Actor-Critic。 把原来的reward和baseline分别替换，PG算法中的减法就变成了$Q^{\pi_{\theta}}\left(s_{t}^{n}, a_{t}^{n}\right)-V^{\pi_{\theta}}\left(s_{t}^{n}\right)$。似乎我们需要训练2个网络？ ![AC-6](http://oss.hackslog.cn/imgs/2019-11-06-082629.jpg) 实际上Q与V可以互相转化，我们只需要训练V。转化公式中为什么要加期望？在s下执行a得到的$ r_t$和$s_{t+1}$是随机的。 实际将Q变成V的操作中，我们会去掉期望，使得只需要训练（估计）状态值函数$V^\pi$，这样会导致一点偏差，但比同时估计两个function导致的偏差要好。（A3C原始paper通过实验验证了这一点）。 ![AC-7](http://oss.hackslog.cn/imgs/2019-11-06-082641.jpg) A2C的训练流程：收集数据，估计出状态值函数$V^\pi(s)$，套用公式更新策略$\pi$，再利用新的$\pi$与环境互动收集新的数据，不断循环。 ![AC-8](http://oss.hackslog.cn/imgs/2019-11-06-082652.jpg) 训练过程中的2个Tips： 1. Actor与Critic的前几层一般会共用参数，因为输入都是state 2. 正则化：让采用不同action的概率尽量平均，希望有更大的entropy，这样能够探索更多情况。 ## 4.2 Asynchronous Advantage Actor-Critic (A3C) ![AC-9](http://oss.hackslog.cn/imgs/2019-11-06-082709.jpg) A3C算法的motivation：开分身学习~ ![AC-10](http://oss.hackslog.cn/imgs/2019-11-06-082718.jpg) 训练过程：每个agent复制一份全局参数，然后各自采样数据，计算梯度，更新这份全局参数，然后将结果传回，复制一份新的参数。 注意： 1. 初始条件会尽量的保证多样性(Diverse)，让每个agent探索的情况更加不一样。 2. 所有的actor都是平行跑的，每个worker把各自的参数传回去然后复制一份新的全局参数。此时可能这份全局参数已经发生了改变，没有关系。 ## 4.3 Pathwise Derivative Policy Gradient (PDPG) 在之前Actor-Critic框架里，Critic的作用是评估agent所执行的action好不好？那么Critic能不能不止给出评价，还给出指导意见呢？即告诉actor要怎样做才能更好？于是有了DPG算法： ![AC-12](http://oss.hackslog.cn/imgs/2019-11-06-082731.jpg) ![AC-13](http://oss.hackslog.cn/imgs/2019-11-06-082746.jpg) 在上面介绍A2C算法的motivation，主要是从改进PG算法引入。那么从Q-learning的角度来看，PDPG相当于learn一个actor，来解决argmax这个优化问题，以处理连续动作空间，直接根据输入的状态输出动作。 ![AC-14](http://oss.hackslog.cn/imgs/2019-11-06-082759.jpg) Actor+Critic连成一个大的网络，训练过程中也会采取TD-target的技巧，固定住Critic $\pi'$，使用梯度上升优化Actor ![AC-15](http://oss.hackslog.cn/imgs/2019-11-06-082809.jpg) 训练过程：Actor会学到策略$\pi$，使基于策略$\pi$，输入s可以获得能够最大化Q的action，天然地能够处理continuous的情况。当actor生成的$Q^\pi$效果比较好时，重新采样生成新的Q。有点像GAN中的判别器与生成器。 注意：从算法的流程可知，Actor 网络和 Critic 网络是分开训练的，但是两者的输入输出存在联系，Actor 网络输出的 action 是 Critic 网络的输入，同时 Critic 网络的输出会被用到 Actor 网路进行反向传播。 由于Critic模块是基于Q-learning算法，所以Q learning的技巧，探索机制，回忆缓冲都可以用上。 ![AC-16](http://oss.hackslog.cn/imgs/2019-11-06-082820.jpg) ![AC-17](http://oss.hackslog.cn/imgs/2019-11-06-082830.jpg) 与Q-learning相比的改进： - 不通过Q-function输出动作，直接用learn一个actor网络输出动作（Policy-based的算法的通用特性）。 - 对于连续变量，不好解argmax的优化问题，转化成了直接选择$\pi-target$ 输出的动作，再基于Q-target得出y。 - 引入$\pi-target$，也使得actor网络不会频繁更新，会通过采样一批数据训练好后再更新，提高训练效率。 ![AC-18](http://oss.hackslog.cn/imgs/2019-11-06-082845.jpg) 总结下：最基础的 Policy Gradient 是回合更新的，通过引入 Critic 后变成了单步更新，而这种结合了 policy 和 value 的方法也叫 Actor-Critic，Critic 有多种可选的方法。A3C在A2C的基础上引入了多个 agent 对网络进行异步更新。对于输出动作为连续值的情形，原始的输出动作概率分布的PG算法不能解决，同时Q-learning算法也不能处理这类问题，因此提出了 DPG 。 ================================================ FILE: notes/5 Sparse Reward.md ================================================ # 5. Sparse Reward 大多数RL的任务中，是没法得到reward，reward=0，导致reward空间非常的sparse。 比如我们需要赢得一局游戏才能知道胜负得到reward，那么玩这句游戏的很长一段时间内，我们得不到reward。比如如果机器人要将东西放入杯子才能得到一个reward，尝试了很多动作很有可能都是0。 但是人可以在非常sprse的环境下进行学习，所以这一章节提出的很多算法与人的一些学习机制比较类似。 ## 5.1 Reward Shaping 手动设计新的reward，让agent做的更好。但有些比较复杂的任务，需要domain knowledge去设计新的reward。 ![](http://oss.hackslog.cn/imgs/2019-11-06-082912.jpg) ![](http://oss.hackslog.cn/imgs/2019-11-06-082928.jpg) ## 5.2 Curiosity 好奇心机制非常的直觉，也非常的强大。有个案例：[Happy Bird](https://github.com/pathak22/noreward-rl) ![](http://oss.hackslog.cn/imgs/2019-11-06-082959.jpg) 好奇心也是reward shaping的一种，引入一个新的reward ：ICM，同时优化2个reward。如何设计一个ICM模块，使agent拥有好奇心？ ![](http://oss.hackslog.cn/imgs/2019-11-06-083015.jpg) 单独训练一个状态估计的模型，如果在某个state下采取某个action得到的下一个state难以预测，则鼓励agent进行尝试这个action。 不过有的state很难预测，但不重要。比如说某个游戏里面背景是树叶飘动，很难预测，接下来agent一直不动看着树叶飘动，没有意义。 ![](http://oss.hackslog.cn/imgs/2019-11-06-083031.jpg) 再设计一个moudle，判断环境中state的重要性：learn一个feature ext的网络，去掉环境中与action关系不大的state。 原理：输入两个处理过的state，预测action，使逼近真实的action。这样使得处理之后的state都是跟agent要采取的action相关的。 ## 5.3 Curriculum Learning 课程学习：为learning做规划，通常由易到难。 ![](http://oss.hackslog.cn/imgs/2019-11-06-092656.jpg) 设计不同难度的课程，一开始直接把板子放入柱子，则agent只要把板子压下去就能获得reward，接着把板子的初始位置提高一些，agent有可能把板子抽出则无法获得reward，接着更general的情况，把板子放倒柱子外面，再让agent去学习。 生成课程的方法通常如下：从目标反推，越靠近目标的state越简单，不断生成难度更高的state。 ![](http://oss.hackslog.cn/imgs/2019-11-06-092702.jpg) ![](http://oss.hackslog.cn/imgs/2019-11-06-092658.jpg) ## 5.4 Hierarchical RL 分层学习：把大的任务拆解成小任务 ![](http://oss.hackslog.cn/imgs/2019-11-06-092745.jpg) 上层的agent给下层的agent提供一个愿景，如果下层的达不到目标，会获得惩罚。如果下层的agent得到的错误的目标，那么它会假设最初的目标也是错的。 ================================================ FILE: notes/6 Imitation Learning.md ================================================ # 6. Imitation Learning 模仿学习，又叫学徒学习，反向强化学习 之前介绍的强化学习都有一个reward function，但生活中大多数任务无法定义reward，或者难以定义。但是这些任务中如果收集很厉害的范例（专家经验）比较简单，则可以用模仿学习解决。 ![](http://oss.hackslog.cn/imgs/2019-11-06-092747.jpg) ## 6.1 Behavior Cloning 本质是有监督学习 ![0004](http://oss.hackslog.cn/imgs/2019-11-06-092759.jpg) ![0005](http://oss.hackslog.cn/imgs/2019-11-06-092817.jpg) 存在问题：training data里面没有撞墙的case，则agent遇到这种情况不知如何决策 ![0006](http://oss.hackslog.cn/imgs/2019-11-06-092821.jpg) 一个直觉的解决方法是数据增强：每次通过牺牲一个专家，学会了一种新的case，策略$\pi$得到了增强。 ![](http://oss.hackslog.cn/imgs/2019-11-06-092831.jpg) 行为克隆还存在一个关键问题：agent不知道哪些行为对结局重要，哪些不重要。由于是采样学习，有可能只记住了多余的无用的行为。 ![0009](http://oss.hackslog.cn/imgs/2019-11-06-092847.jpg) 同时也由于RL的训练数据不是独立同分布，当下的action会影响之后的state，所以不能直接套用监督学习的框架。 为了解决这些问题，就有了反向强化学习，现在一般说模仿学习指的就是反向强化学习。 ## 6.2 Inverse RL ![0011](http://oss.hackslog.cn/imgs/2019-11-06-092859.jpg) 之前的强化学习是reard和env通过RL 学到一个最优的actor。 反向强化学习是，假设有一批expert的数据，通过env和IRL推导expert因为什么样子的reward function才会采取这样的行为。 好处：也许expert的行为复杂但reward function很简单。拿到这个reward function后我们就可以训练出好的agent。 ![0012](http://oss.hackslog.cn/imgs/2019-11-06-092907.jpg) IRL的框架：先射箭 再画靶。 具体过程： Expert先跟环境互动，玩N场游戏，存储记录，我们的actor $ \pi$也去互动，生成N场游戏记录。接下来定义一个reward function $R$，保证expert的$R$比我们的actor的$R$大就行。再根据定义的的$R$用RL的方法去学习一个新的actor ，这个过程也会采集新的游戏记录，等训练好这个actor，也就是当这个actor可以基于$R$获得高分的时候，重新定义一个新的reward function$R'$，让expert的$R'$大于agent，不断循环。 ![0013](http://oss.hackslog.cn/imgs/2019-11-06-092917.jpg) IRL与GAN的框架是一样的，学习 一个 reward function相当于学习一个判别器，这个判别器给expert高分，给我们的actor低分。 一个有趣的事实是给不同的expert，我们的agent最终也会学会不同的策略风格。如下蓝色是expert的行为，红色是学习到的actor的行为。 ![](http://oss.hackslog.cn/imgs/2019-11-06-092932.jpg) 针对训练robot的任务： IRL有个好处是不需要定义规则让robot执行动作，人给robot示范一下动作即可。但robot学习时候的视野跟它执行该动作时候的视野不一致，怎么把它在第三人称视野学到的策略泛化到第一人称视野呢？ ![](http://oss.hackslog.cn/imgs/2019-11-06-092943.jpg) ![0019](http://oss.hackslog.cn/imgs/2019-11-06-092953.jpg) 解决思路跟好奇心机制类似，抽出视野中不重要的因素，让第一人称和第三人称视野中的state都是有用的，与action强相关的。