深度强化学习应用无人机附python代码.rar

#-- coding:UTF-8 -- """ Reinforcement learning UAV-enabled MEC example. Basical setting: AoI: 100m*100m; Height_levels, maximal height, minimula height: 20, 100m, 200m; This script is the environment part of the UAV-enabled MEC. The RL is in RL_brain.py. View more on my information see paper: "Deep Reinforcement Learning based 3D-Trajectory Design and Task Offloading in UAV-enabled MEC System" by Haibo Mei, Kun Yang, Qiang Liu; """ import numpy as np import random as rd import time import math as mt import sys import copy from scipy.interpolate import interp1d from mpl_toolkits.mplot3d import Axes3D import matplotlib if sys.version_info.major == 2: import Tkinter as tk else: import tkinter as tk import matplotlib.pyplot as plt from TSP import tsp tsp=tsp() UNIT = 1 # pixels IOT_H = 100 # grid height IOT_W = 100 # grid width Max_Hight = 100 # maximum level of height Min_Hight = 50 # minimum level of height #weight variables for the reward function beta = 20 #gradent of the horizontal and vertical locations of the UAV D_k = 200 F_k = 2000 # 2M(2000), 2000 cycles # Initialize the wireless environement and some other verables. N_0 = mt.pow(10, ((-169 / 3) / 10)) # Noise power spectrum density is -169dBm/Hz; a = 9.61 # referenced from paper [Efficient 3-D Placement of an Aerial Base Station in Next Generation Cellular Networks, 2016, ICC] b = 0.16 # and paper [Optimal LAP Altitude for Maximum Coverage, IEEE WIRELESS COMMUNICATIONS LETTERS, VOL. 3, NO. 6, DECEMBER 2014] eta_los = 1 # Loss corresponding to the LoS connections defined in (2) of the paper; eta_nlos = 20 # Loss corresponding to the NLoS connections defined in (2)of the paper; A = eta_los - eta_nlos # A varable defined in (2) of the paper; C = 20 * np.log10( 4 * np.pi * 9 / 3) + eta_nlos # C varable defined in (2)of the paper, where carrier frequncy is 900Mhz=900*10^6, and light speed is c=3*10^8; then one has f/c=9/3; B = 2000 # overall Bandwith is 2Gb; Power = 5 * mt.pow(10, 5) # maximum uplink transimission power of one GT is 5mW; class UAV_MEC(object): def __init__(self): super(UAV_MEC, self).__init__() self.N_slot = 400 # number of time slots in one episode self.x_s = 10 self.y_s = 10 self.h_s = 2 self.GTs = 6 self.l_o_v = 50*self.h_s # initial vertical location self.l_f_v = 50*self.h_s # final vertical location self.l_o_h = [0, 0] # initial horizontal location self.l_f_h = [0, 0] # final horizontal location self.eps = 60 #number of episode self.UAV_trajectory_tsp = np.zeros((self.N_slot, 3), dtype=float) #cycles / s, #cycles / s; self.f_u = 100 self.f_g = 5 self.D_max = np.sqrt(mt.pow(self.l_o_h[0]*self.x_s - self.l_f_h[0]*self.x_s, 2) + mt.pow(self.l_o_h[1]*self.y_s - self.l_f_h[1]*self.y_s,2)) # the distance from initial point to final point # north, south, east, west, hover self.action_space_uav_horizontal = ['n', 's', 'e','w','h'] # ascend, descend, slf self.action_space_uav_vertical = ['a', 'd', 's'] # offloading, local exection self.action_space_task_offloading = np.zeros((self.GTs, 2), dtype=int) #overall_action_space self.n_actions = len(self.action_space_uav_horizontal)*len(self.action_space_uav_vertical)*mt.pow(2,self.GTs) self.n_features = 3 #horizontal:x, y, vertical trajectory of the UAV #generate action table; self.actions = np.zeros((int(self.n_actions),1+2+self.GTs), dtype=int) index = 0 for h in range(len(self.action_space_uav_horizontal)): for v in range(len(self.action_space_uav_vertical)): LL= self.brgd(self.GTs) #list all the possible combination of 0-1 offloading options among the GTs for l in range (len(LL)): o_string = LL[l] of = [] for ch in range (len(o_string)): if o_string[ch] == '0': of.append(0) else: of.append(1) self.actions[index,:]=[index, h, v]+ of[:] index = index + 1 self._build_uav_mec() def _build_uav_mec(self): # initilize the GT coordinates and tasks # model of GTs' location and task self.location = np.zeros((5, 2), dtype=float) self.location[0, :] = [rd.randint(0, int(IOT_H / 3)), rd.randint(0, int(IOT_W / 3))] self.location[1, :] = [rd.randint(int(IOT_H/3), int(2*IOT_H/3)), rd.randint(int(IOT_W/3), int(2*IOT_W/3))] self.location[2, :] = [rd.randint(int(2*IOT_H/3), int(3*IOT_H/3)), rd.randint(int(2*IOT_W/3), int(3*IOT_W/3))] self.location[0, :] = [rd.randint(0, int(2*IOT_H/3)), rd.randint(0, int(2*IOT_W/3))] self.location[1, :] = [rd.randint(0, int(2*IOT_H/3)), rd.randint(0, int(2*IOT_W/3))] self.location[2, :] = [rd.randint(0, int(2*IOT_H/3)), rd.randint(0, int(2*IOT_W/3))] self.w_k = np.zeros((self.GTs, 2), dtype=float) self.u_k = np.zeros((self.GTs, 2), dtype=float) for count in range(3): #3*2=6; for cou in range(2): g = count * 2 + cou # horizontal coordinate of the GT self.w_k[g, 0] = self.location[count, 0] + rd.randint(20, 40) self.w_k[g, 1] = self.location[count, 1] + rd.randint(20, 40) self.w_k[g, 0] = self.w_k[g, 0] * self.x_s + self.x_s * rd.random() self.w_k[g, 1] = self.w_k[g, 1] * self.y_s + self.y_s * rd.random() # D_k of the GT task self.u_k[g, 0] = D_k / 2 + (D_k / 2) * rd.random() # F_k of the GT task self.u_k[g, 1] = F_k / 2 + (F_k / 2) * rd.random() #initial UAV trajectory using TSP w_k_tmp = np.zeros((self.GTs+1, 2), dtype=float) for g in range(self.GTs): w_k_tmp[g,0]=self.w_k[g, 0] w_k_tmp[g,1]=self.w_k[g, 1] w_k_tmp[self.GTs, 0] = 0 w_k_tmp[self.GTs, 1] = 0 [tsp_result,sum_tsp_path]=tsp.solve(w_k_tmp) Delta_distance = sum_tsp_path /self.N_slot UAV_trajectory_tsp_tmp = np.zeros((self.N_slot, 3), dtype=float) UAV_trajectory_tsp_tmp[0,0] = w_k_tmp[tsp_result[0], 0] UAV_trajectory_tsp_tmp[0,1] = w_k_tmp[tsp_result[0], 1] UAV_trajectory_tsp_tmp[0,2] = 200 count = 0 text_dist=0 for n in range(0, self.GTs): current_note = tsp_result[n] next_note = tsp_result[n + 1] dis = np.sqrt((w_k_tmp[current_note, 0] - w_k_tmp[next_note, 0])**2 + (w_k_tmp[current_note, 1] - w_k_tmp[next_note, 1])**2) text_dist = text_dist + dis line_count =int(np.floor_divide(dis,Delta_distance)) if (np.remainder(dis,Delta_distance)>0): line_count+=1 x_dis = w_k_tmp[next_note, 0] - w_k_tmp[current_note, 0] y_dis = w_k_tmp[next_note, 1] - w_k_tmp[current_note, 1] delta_x_dis = x_dis / line_count delta_y_dis = y_dis / line_count for index in range(1,line_count): UAV_trajectory_tsp_tmp[count + 1,0]= UAV_trajectory_tsp_tmp[count,0] + delta_x_dis UAV_trajectory_tsp_tmp[count + 1,1]= UAV_trajectory_tsp_tmp[count,1] + delta_y_dis if (UAV_trajectory_tsp_tmp[count + 1,0]<0): UAV_trajectory_tsp_tmp[count + 1,0]=0 if (UAV_trajectory_tsp_tmp[count + 1,1]<0): UAV_trajectory_tsp_tmp[count + 1,1]=0 UAV_trajectory_tsp_tmp[count + 1, 2] =200 #if (count<=(self.N_slot/2)): #UAV_trajectory_tsp_tmp[count + 1,2] = UAV_trajectory_tsp_tmp[count,2]-(200.00/np