基于深度强化学习DQN的无人艇避障控制（matlab）

clc clear all; warning off; %数值仿真需要 opts = odeset('RelTol',1e-3,'AbsTol',1e-4); xyz0=[0 0 0]; uvr0=[0 0 0]; obs1_x_y=[1 0.5]; obs2_x_y=[0.5 1]; obs3_x_y=[2.5 0.5]; obs4_x_y=[1.5 1.5]; obs5_x_y=[2.8 0.8]; run('Fuzzy_set') run('AgentInitializer'); %训练周期设置：观察期，训练期 N_obs=200; N_train=1800; N_total=N_obs+N_train; T_episode=40; %每个周期的总时间 %数据记录初始化，状态记录在Episode开始前进行 Successmark=zeros(1,N_obs+N_train); AveSuccess=zeros(1,N_obs+N_train); TotalSuccess=zeros(1,N_obs+N_train); ATRpointer=1; %episode及其余设置 T_step=0.1; n_step=1; %动态绘图初始化 Plotset=zeros(2,1); p = plot(Plotset(1,:),Plotset(2,:),... 'EraseMode','background','MarkerSize',5); for Ns=1:N_total USVstate=[xyz0 uvr0 0 0 obs1_x_y obs2_x_y obs3_x_y obs4_x_y obs5_x_y]'; %USVstate为18*1矩阵，xyz,uvr,tao_uu,tao_rr,obs1_x_y obs2_x_y obs3_x_y obs4_x_y obs5_x_y T1=0; %初始化历史记录。记录下列内容：1.控制时长；2.每个episode的400个状态与控制力矩 TrackPointer=1; TrackRecord(Ns).Track=zeros(21,T_episode/T_step); while T1<=T_episode %根据tcegreedy策略选择动作 [act,Qnow]=tcegreedy(Ns,USVstate,QNet_eval); if act==1 Tao_uu=USVstate(7)+2.5; Tao_rr=5; end if act==2 Tao_uu=USVstate(7)+2.5; Tao_rr=-5; end if act==3 Tao_uu=USVstate(7)+2.5; Tao_rr=0; end if act==4 Tao_uu=USVstate(7)-2.5; Tao_rr=5; end if act==5 Tao_uu=USVstate(7)-2.5; Tao_rr=-5; end if act==6 Tao_uu=USVstate(7)-2.5; Tao_rr=0; end if act==7 Tao_uu=USVstate(7); Tao_rr=5; end if act==8 Tao_uu=USVstate(7); Tao_rr=-5; end if act==9 Tao_uu=USVstate(7); Tao_rr=0; end %使用Ode45执行动作 OdeInput=[USVstate(1:6); Tao_uu; Tao_rr; USVstate(9:18)]; [t,y]=ode45(@USV_Eqs,[0,T_step],OdeInput,opts); Nsize=size(y); Nsize=Nsize(1); Newstate=y(Nsize,1:18); Newstate=Newstate'; %Replaymemory记录+指针更新 Rmemo(:,Memopointer)=[USVstate;act;Newstate]; Memopointer=PointerMove(Memopointer,S_memo); %轨迹数据记录更新 TrackRecord(Ns).Track(:,TrackPointer)=[T1;USVstate;Tao_uu;Tao_rr]; TrackPointer=TrackPointer+1; %更新状态 T1=T1+T_step; n_step=n_step+1; USVstate=Newstate; %按照T-renew间隔更新估计Q_target的目标神经网络QNet_target if (mod(n_step,N_renew)==0)&&(Ns>=N_obs) QNet_target=QNet_eval; end %按照T_gap的间隔训练估计Q_eval的评估神经网络QNet_eval if (mod(n_step,N_gap)==0)&&(Ns>=N_obs) %1. 利用Rmemo生成训练数据级 Trainset=zeros(38,nBatch); %前37行与replaymemory一致，后一行为利用QNet_target计算得到的Q_target; i=1; while i<=nBatch num1=unidrnd(S_memo); if Rmemo(19,num1)>0 Trainset(1:37,i)=Rmemo(:,num1); i=i+1; end end %2. 计算Q_target Trainset(38,:)=CalculationQtarget(Trainset(20:37,:),QNet_target,fis); for pp=1:nBatch dec_obs(1,pp)=sig_t_Cal(Trainset(1:18,pp)); end %3. 训练QNet_eval QNet_eval=train(QNet_eval,[Trainset(1:8,:);dec_obs(1,:)],Trainset(38,:)); end %判断是否跳出本episode，并记录控制时长 if abs(USVstate(1)-2)<0.05&&abs(USVstate(2)-1.5)<0.05 Successmark(Ns)=1; break; else if Newstate(1)>=3.5 || Newstate(1)<=-0.5 || Newstate(2)>=3.5 || Newstate(2)<=-0.5 break; elseif T1>=T_episode break; end end end %动态绘图 TotalSuccess(Ns)=sum(Successmark(1:Ns)); AveSuccess(Ns)=TotalSuccess(Ns)/Ns; if mod(Ns,10)==0 TempP=[Ns;AveSuccess(Ns)]; Plotset=[Plotset,TempP]; set(p,'XData',Plotset(1,:),'YData',Plotset(2,:)); drawnow axis([0 N_obs+N_train 0 1]); end end % figure % plot(y(:,1),y(:,2)) % hold on % plot(y(:,9),y(:,10)) % plot(y(:,11),y(:,12)) % plot(y(:,13),y(:,14)) % plot(y(:,15),y(:,16)) % plot(y(:,17),y(:,18)) % plot(2,1.5,'*') %%目标点位置 % axis([-0.5 3.5 -0.5 3.5]);