【路径规划】基于深度强化学习DQN实现路径规划问题附matlab代码.zip

%忘记面孔的Batou %I thought what I'd do was I'd pretend I was one of those deaf-mutes, or should I? clear all; %定义全局参数，方便函数调用 global Gwidth Gheight N_act targetxy gamma; %网格迷宫参数 Gwidth=60; Gheight=35; N_act=4; %风速初始化 Windyworld.windx=zeros(Gheight,Gwidth); Windyworld.windy=zeros(Gheight,Gwidth); Windyworld.windy(:,6:9)=Windyworld.windy(:,6:9)+1; %%Windyworld.windy(:,7:8)=Windyworld.windy(:,7:8)+1; targetxy=[10,14]; %!!注意：第一个坐标为y坐标，第二个坐标为x坐标 startxy=[1,4]; step=1; %时间设置：观察期，训练期 T_obs=500; T_train=10000; T_episode=200; %DQN神经网络初始化 QNet_eval=fitnet([40,40]); Iniset=zeros(4,400); %前三行为输入，最后一行为目标输出 for i=1:40 Iniset(1,i)=unidrnd(Gheight); Iniset(2,i)=unidrnd(Gwidth); Iniset(3,i)=unidrnd(N_act); if Iniset(3,i)==1 Iniset(4,i)=0.2; else if Iniset(3,i)==2 Iniset(4,i)=-0.2; end %} end end clear i; QNet_eval=train(QNet_eval,Iniset(1:3,:),Iniset(4,:)); QNet_target=QNet_eval; %将神经网络训练算法设置为自适应动量梯度下降法 QNet_eval.trainFcn='traingdx'; %关闭训练图窗nntraintool QNet_eval.trainParam.showWindow=0; %Replaymemory初始化： S_memo=8000; Rmemo=zeros(5,S_memo); %1:2：rolexy,3：act,4:5：nextxy Memopointer=1; %Replay memory的写入指针 %学习参数初始化 gamma=0.99; nBatch=400; T_gap=400; %20,25,30 T_renew=3*T_gap; %记录列表初始化 Successmark=zeros(1,T_obs+T_train); AveSuccess=zeros(1,T_obs+T_train); TotalSuccess=zeros(1,T_obs+T_train); %动态绘图初始化 Plotset=zeros(2,1); p = plot(Plotset(1,:),Plotset(2,:),... 'EraseMode','background','MarkerSize',5); axis([0 T_obs+T_train 0 1]); %观察期：此期间不更新Qnet,仅记录replay memory for Ts=1:T_obs rolexy=startxy; %result(Ts).trace=zeros(40,3); %result(Ts).trace(1,:)=([1,rolexy(1),rolexy(2)]); for Tm=1:T_episode %根据tcegreedy策略执行动作 [act,Q_now]=tcegreedy(Ts,rolexy,QNet_eval); nextxy=movement(act,rolexy,Windyworld); %Replaymemory记录+指针更新 Rmemo(:,Memopointer)=[rolexy';act;nextxy']; Memopointer=PointerMove(Memopointer,S_memo); %更新位置 step=step+1; rolexy=nextxy; %result(Ts).trace(Tm+1,:)=([Tm+1,rolexy(1),rolexy(2)]); %判断是否跳出本episode if rolexy(1)==targetxy(1)&&rolexy(2)==targetxy(2) Successmark(Ts)=1; break; else if rolexy(1)<1||rolexy(1)>Gheight||rolexy(2)<1||rolexy(2)>Gwidth break; end end end %数据记录 TotalSuccess(Ts)=sum(Successmark(1:Ts)); AveSuccess(Ts)=TotalSuccess(Ts)/Ts; %动态绘图 % if mod(Ts,10)==0 % TempP=[Ts;AveSuccess(Ts)]; % Plotset=[Plotset,TempP]; % set(p,'XData',Plotset(1,:),'YData',Plotset(2,:)); % drawnow % axis([0 T_obs+T_train 0 1]); % end end %探索期 %探索期开始更新神经网络参数 Tnode1=1+T_obs; Tnode2=T_obs+T_train; %网络训练参数更新 for Ts=Tnode1:Tnode2 rolexy=startxy; %result(Ts).trace=zeros(40,3); %result(Ts).trace(1,:)=([1,rolexy(1),rolexy(2)]); for Tm=1:T_episode %根据tcegreedy策略执行动作 [act,Q_now]=tcegreedy(Ts,rolexy,QNet_eval); nextxy=movement(act,rolexy,Windyworld); %为提高效率，Q_target统一在训练神经网络时更新 %Replaymemory记录+指针更新 Rmemo(:,Memopointer)=[rolexy';act;nextxy']; Memopointer=PointerMove(Memopointer,S_memo); %更新位置 step=step+1; rolexy=nextxy; %result(Ts).trace(Tm+1,:)=([Tm+1,rolexy(1),rolexy(2)]); %判断是否跳出本episode if rolexy(1)==targetxy(1)&&rolexy(2)==targetxy(2) Successmark(Ts)=1; break; else if rolexy(1)<1||rolexy(1)>Gheight||rolexy(2)<1||rolexy(2)>Gwidth break; end end %按照T-renew间隔更新估计Q_target的目标神经网络QNet_target if mod(step,T_renew)==0 QNet_target=QNet_eval; end %按照T_gap的间隔训练估计Q_eval的评估神经网络QNet_eval if mod(step,T_gap)==0 %1. 利用Rmemo生成训练数据级 Trainset=zeros(6,nBatch); %前五行与replaymemory一致，后一行为利用QNet_target计算得到的Q_target i=1; while i<=nBatch num1=unidrnd(S_memo); %随机抽取ReplayMemory中的数据 if Rmemo(1,num1)>0 Trainset(1:5,i)=Rmemo(:,num1); i=i+1; end end %2. 计算Q_target Trainset(6,:)=CalculationQtarget(Trainset(1:5,:),QNet_target); %3. 训练QNet_eval QNet_eval=train(QNet_eval,Trainset(1:3,:),Trainset(6,:)); end end %数据记录 TotalSuccess(Ts)=sum(Successmark(1:Ts)); AveSuccess(Ts)=TotalSuccess(Ts)/Ts; % %动态绘图 % if mod(Ts,10)==0 % TempP=[Ts;AveSuccess(Ts)]; % Plotset=[Plotset,TempP]; % set(p,'XData',Plotset(1,:),'YData',Plotset(2,:)); % drawnow % axis([0 T_obs+T_train 0 1]); % end end