企业官网建设流程全解析

1.估值网络简介
在强化学习中，除了上节提到的策略网络（Policy Based）直接选择Action的方法，还有一种学习Action对应的期望值（Expected Utility）的方法，称为Q-Learning，和Plolicy Based方法一样， Q-Learning不依赖环境模型。在有限马尔科夫决策过程中（Markov Decision Process）中，Q-Learning被证明最终可以找到最优的策略。简单来说，将旧的Q-Learning函数，向着学习目标（当前获得的Reward加上下一步可获得的最大期望价值）按一个较小的学习速率学习，得到新的Q-Learning函数，这个就是Q-Learning的具体的思想，学习率决定了覆盖之前掌握信息的比例，通常设为一个比较小的值，如果设定的值比较大，那么覆盖之前的信息比较多，那么会造成整个网络的动荡。

我们用来学习Q-Learning的模型可以是神经网络，这样得到的模型即是估值网络。如果其中的神经网络比较深，那就是DQN。在DQN的使用中会有很多的Trick。第一个是在DQN中引入卷积层，第二个是Experience Replay，第三个Trick就是可以再使用一个DQN网络来辅助训练，第四个Trick，如果再分拆出target DQN的方法上更进一步，那就是Double DQN，第五个Trick是使用dual DQN。

2.GridWorld的任务代码实现

#coding:utf-8#这里也是导入常用的依赖库#为了直接能够在终端中运行代码，我还是把魔法命定注释掉了，具体的魔法命令的解释可以看上一个实战importnumpyasnpimportrandomimportitertoolsimportscipy.miscimportmatplotlib.pyplotaspltimporttensorflowastfimportos# %matplotlib inline#先是创建环境内物体对象的classclassgameOb():def__init__(self,coordinates,size,intensity,channel,reward,name):self.x=coordinates[0]self.y=coordinates[1]self.size=size self.intensity=intensity self.channel=channel self.reward=reward self.name=name#创建GridWorld环境的classclassgameEnv():def__init__(self,size):self.sizeX=size self.sizeY=size self.actions=4self.objects=[]a=self.reset()plt.imshow(a,interpolation="nearest")#hero是用户控制的对象，4个goal的reward为1， 2个fire的reward为-1defreset(self):self.objects=[]hero=gameOb(self.newPosition(),1,1,2,None,'hero')self.objects.append(hero)goal=gameOb(self.newPosition(),1,1,1,1,'goal')self.objects.append(goal)hole=gameOb(self.newPosition(),1,1,0,-1,'fire')self.objects.append(hole)goal2=gameOb(self.newPosition(),1,1,1,1,'goal')self.objects.append(goal2)hole2=gameOb(self.newPosition(),1,1,0,-1,'fire')self.objects.append(hole2)goal3=gameOb(self.newPosition(),1,1,1,1,'goal')self.objects.append(goal3)goal4=gameOb(self.newPosition(),1,1,1,1,'goal')self.objects.append(goal4)state=self.renderEnv()self.state=statereturnstate#实现英雄角色移动的方向0，1， 2，3，分别代表下，上， 左，右defmoveChar(self,direction):hero=self.objects[0]heroX=hero.x heroY=hero.yifdirection==0andhero.y>=1:hero.y-=1ifdirection==1andhero.y<=self.sizeY-2:hero.y+=1ifdirection==2andhero.x>=1:hero.x-=1ifdirection==3andhero.x<=self.sizeX-2:hero.x+=1self.objects[0]=hero#定义新的位置defnewPosition(self):iterables=[range(self.sizeX),range(self.sizeY)]points=[]fortinitertools.product(*iterables):points.append(t)currentPositions=[]forobjectAinself.objects:if(objectA.x,objectA.y)notincurrentPositions:currentPositions.append((objectA.x,objectA.y))forposincurrentPositions:points.remove(pos)location=np.random.choice(range(len(points)),replace=False)returnpoints[location]#定义checkGoal函数，用来检查hero是否触碰了goal或者firedefcheckGoal(self):others=[]forobjinself.objects:ifobj.name=='hero':hero=objelse:others.append(obj)forotherinothers:ifhero.x==other.xandhero.y==other.y:self.objects.remove(other)ifother.reward==1:self.objects.append(gameOb(self.newPosition(),1,1,1,1,'goal'))else:self.objects.append(gameOb(self.newPosition(),1,1,0,-1,'fire'))returnother.reward,Falsereturn0.0,False#渲染图像尺寸defrenderEnv(self):a=np.ones([self.sizeY+2,self.sizeX+2,3])a[1:-1,1:-1,:]=0hero=Noneforiteminself.objects:a[item.y+1:item.y+item.size+1,item.x+1:item.x+item.size+1,item.channel]=item.intensity b=scipy.misc.imresize(a[:,:,0],[84,84,1],interp='nearest')c=scipy.misc.imresize(a[:,:,1],[84,84,1],interp='nearest')d=scipy.misc.imresize(a[:,:,2],[84,84,1],interp='nearest')a=np.stack([b,c,d],axis=2)returna#定义执行的Action的方法defstep(self,action):self.moveChar(action)reward,done=self.checkGoal()state=self.renderEnv()returnstate,reward,done#设置尺寸为5env=gameEnv(size=5)#定义DQN（Deep Q-Network）网络classQnetwork():def__init__(self,h_size):self.scalarInput=tf.placeholder(shape=[None,21168],dtype=tf.float32)self.imageIn=tf.reshape(self.scalarInput,shape=[-1,84,84,3])self.conv1=tf.contrib.layers.convolution2d(inputs=self.imageIn,num_outputs=32,kernel_size=[8,8],stride=[4,4],padding='VALID',biases_initializer=None)self.conv2=tf.contrib.layers.convolution2d(inputs=self.conv1,num_outputs=64,kernel_size=[4,4],stride=[2,2],padding='VALID',biases_initializer=None)self.conv3=tf.contrib.layers.convolution2d(inputs=self.conv2,num_outputs=64,kernel_size=[3,3],stride=[1,1],padding='VALID',biases_initializer=None)self.conv4=tf.contrib.layers.convolution2d(inputs=self.conv3,num_outputs=512,kernel_size=[7,7],stride=[1,1],padding='VALID',biases_initializer=None)self.streamAC,self.streamVC=tf.split(self.conv4,2,3)self.streamA=tf.contrib.layers.flatten(self.streamAC)self.streamV=tf.contrib.layers.flatten(self.streamVC)self.AW=tf.Variable(tf.random_normal([h_size//2,env.actions]))self.VW=tf.Variable(tf.random_normal([h_size//2,1]))self.Adavantage=tf.matmul(self.streamA,self.AW)self.Value=tf.matmul(self.streamV,self.VW)self.Qout=self.Value+tf.subtract(self.Adavantage,tf.reduce_mean(self.Adavantage,reduction_indices=1,keep_dims=True))self.predict=tf.argmax(self.Qout,1)self.targetQ=tf.placeholder(shape=[None],dtype=tf.float32)self.actions=tf.placeholder(shape=[None],dtype=tf.int32)self.actions_onehot=tf.one_hot(self.actions,env.actions,dtype=tf.float32)self.Q=tf.reduce_sum(tf.multiply(self.Qout,self.actions_onehot),reduction_indices=1)self.td_error=tf.square(self.targetQ-self.Q)self.loss=tf.reduce_mean(self.td_error)self.trainer=tf.train.AdamOptimizer(learning_rate=0.0001)self.UpdateModel=self.trainer.minimize(self.loss)#实现Experience Replay策略classexperience_buffer():def__init__(self,buffer_size=50000):self.buffer=[]self.buffer_size=buffer_sizedefadd(self,experience):iflen(self.buffer)+len(experience)>=self.buffer_size:self.buffer[0:(len(experience)+len(self.buffer))-self.buffer_size]=[]self.buffer.extend(experience)defsample(self,size):returnnp.reshape(np.array(random.sample(self.buffer,size)),[size,5])#把当前state扁平为1维向量的函数defprocessState(states):returnnp.reshape(states,[21168])#更新模型参数defupdateTargetGraph(tfVars,tau):total_vars=len(tfVars)op_holder=[]foridx,varinenumerate(tfVars[0:total_vars//2]):op_holder.append(tfVars[idx+total_vars//2].assign((var.value()*tau)+((1-tau)*tfVars[idx+total_vars//2].value())))returnop_holderdefupdateTarget(op_holder,sess):foropinop_holder:sess.run(op)#设置一些训练参数batch_size=32update_freq=4y=.99startE=1endE=0.1anneling_steps=10000.num_episodes=10000pre_train_steps=10000max_epLength=50load_model=Falsepath="./dqn"h_size=512tau=0.001#初始化mainQN=Qnetwork(h_size)targetQN=Qnetwork(h_size)init=tf.global_variables_initializer()trainables=tf.trainable_variables()targetOps=updateTargetGraph(trainables,tau)myBuffer=experience_buffer()e=startE stepDrop=(startE-endE)/anneling_steps rList=[]total_steps=0saver=tf.train.Saver()ifnotos.path.exists(path):os.makedirs(path)#创建默认的sessionwithtf.Session()assess:ifload_model==True:print('Load Model...')ckpt=tf.train.get_checkpoint_state(path)saver.restore(sess,ckpt.model_checkpoint_path)sess.run(init)updateTarget(targetOps,sess)foriinrange(num_episodes+1):episodeBuffer=experience_buffer()s=env.reset()s=processState(s)d=FalserAll=0j=0whilej<max_epLength:j+=1ifnp.random.rand(1)<eortotal_steps<pre_train_steps:a=np.random.randint(0,4)else:a=sess.run(mainQN.predict,feed_dict={mainQN.scalarInput:[s]})[0]s1,r,d=env.step(a)s1=processState(s1)total_steps+=1episodeBuffer.add(np.reshape(np.array([s,a,r,s1,d]),[1,5]))iftotal_steps>pre_train_steps:ife>endE:e-=stepDropiftotal_steps%(update_freq)==0:trainBatch=myBuffer.sample(batch_size)A=sess.run(mainQN.predict,feed_dict={mainQN.scalarInput:np.vstack(trainBatch[:,3])})Q=sess.run(targetQN.Qout,feed_dict={targetQN.scalarInput:np.vstack(trainBatch[:,3])})doubleQ=Q[range(batch_size),A]targetQ=trainBatch[:,2]+y*doubleQ _=sess.run(mainQN.UpdateModel,feed_dict={mainQN.scalarInput:np.vstack(trainBatch[:,0]),mainQN.targetQ:targetQ,mainQN.actions:trainBatch[:,1]})updateTarget(targetOps,sess)rAll+=r s=s1ifd==True:breakmyBuffer.add(episodeBuffer.buffer)rList.append(rAll)ifi>0andi%25==0:print('episode',i,', average reward of last 25 episode',np.mean(rList[-25:]))ifi>0andi%1000==0:saver.save(sess,path+'/model-'+str(i)+'.cptk')print("Saved Model")saver.save(sess,path+'/model-'+str(i)+'.cptk')rMat=np.resize(np.array(rList),[len(rList)//100,100])rMean=np.average(rMat,1)plt.plot(rMean)

这个还是要训练好久，不过还是蛮好玩的，如果可以用强化学习训练一个监督机器人，这样LZ就不会有拖延症啦O(∩_∩)O