CR_Noma_DRL.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CR-Noma-DRL.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HYHVnb5sXTNW"
      },
      "source": [
        "#Import library for Environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ssjOn5OU_sQ"
      },
      "source": [
        "import scipy\r\n",
        "from scipy import special\r\n",
        "import numpy as np\r\n",
        "from scipy.special import lambertw\r\n",
        "import math\r\n",
        "\r\n",
        "dtype = np.float32"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RyeRzQIUXKR_"
      },
      "source": [
        "#Import library for DDPG Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MD38wTjaXJMT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3bff9c56-95ac-4dc8-de33-2aa945d2ab66"
      },
      "source": [
        "import tensorflow.compat.v1 as tf\r\n",
        "tf.disable_v2_behavior()\r\n",
        "import numpy as np\r\n",
        "# from enviroment import Env_cellular as env\r\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "non-resource variables are not supported in the long term\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VAeTJxIMVq4y"
      },
      "source": [
        "# Create environment for cellular BS"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pPkCcWnHVuqb"
      },
      "source": [
        "class Env_cellular():\r\n",
        "  def __init__(self,    MAX_EP_STEPS, s_dim, location_vector, location_GF,K, Pn,fading_n, fading_0):\r\n",
        "    self.emax = 0.1  # battery capacity\r\n",
        "    #self.P0 = P0\r\n",
        "    self.K = K\r\n",
        "    self.T =1\r\n",
        "    self.eta =0.7\r\n",
        "    self.Pn = Pn # grant-based user's power\r\n",
        "    self.Pmax = 0.1\r\n",
        "\r\n",
        "\r\n",
        "    BW = 10**6 # 10MHz\r\n",
        "    sigma2_dbm = -170 + 10 * np.log10(BW) #  Thermal noise in dBm\r\n",
        "    #sigma2_dbm = -94\r\n",
        "    self.noise = 10 ** ((sigma2_dbm - 30) / 10)\r\n",
        "    self.Pn = self.Pn\r\n",
        "\r\n",
        "    #self.P0n = 10 #grant free user's power\r\n",
        "    self.s_dim = s_dim\r\n",
        "\r\n",
        "    self.MAX_EP_STEPS = MAX_EP_STEPS\r\n",
        "\r\n",
        "\r\n",
        "    distance_GF = np.sqrt(np.sum((location_vector-location_GF)**2, axis=1))\r\n",
        "    distance_GB = np.sqrt(np.sum((location_vector)**2, axis=1))\r\n",
        "\r\n",
        "    distance = np.matrix.transpose(np.array([distance_GF, distance_GB]))\r\n",
        "    distance = np.maximum(distance, np.ones((self.K,2)))\r\n",
        "    PL_alpha = 3\r\n",
        "    PL = 1/distance**PL_alpha/(10**3.17)\r\n",
        "    self.hn = np.multiply( PL, fading_n)\r\n",
        "    distance_GF0 = np.sqrt(np.sum( location_GF ** 2, axis=1))\r\n",
        "    distance0 = np.maximum(distance_GF0, 1)\r\n",
        "    PL0 = 1/(distance0 ** PL_alpha)/(10**3.17)\r\n",
        "    self.h0 = fading_0*PL0\r\n",
        "    #print(f\"{self.hn} and {self.h0}\")\r\n",
        "\r\n",
        "    self.channel_sequence = np.zeros(( self.MAX_EP_STEPS,2))\r\n",
        "    for i in range(self.MAX_EP_STEPS):\r\n",
        "      id_index = i % self.K\r\n",
        "      self.channel_sequence[i,:] = self.hn[id_index,:]\r\n",
        "\r\n",
        "  def step(self, action, state, j):\r\n",
        "    hn = state[0,0]/self.noise\r\n",
        "    hn0 = state[0,1]\r\n",
        "    h0 = self.h0/self.noise #state[0,2]/self.noise\r\n",
        "    En = state[0,-1]\r\n",
        "\r\n",
        "\r\n",
        "    #print(uchannel)\r\n",
        "\r\n",
        "    En_bar = action*min(self.emax-En, self.T*self.eta*self.Pn*hn0) - (1-action)*min(En,self.T*self.Pmax)\r\n",
        "\r\n",
        "    #print(f\"energy is {self.T*self.eta*self.Pn*hn0}\")\r\n",
        "    #print(f\"en_bar is {En_bar}, and action is {action}\")\r\n",
        "    mu1 = self.eta*self.Pn*hn0*h0/(1+self.Pn*hn)\r\n",
        "    mu2 = En_bar*h0/self.T/(1+self.Pn*hn)\r\n",
        "    wx0 = np.real(lambertw(math.exp(-1)*(mu1-1), k=0))\r\n",
        "    #print(f\"{np.exp(wx0+1) } \")\r\n",
        "    alphaxx = (mu1-mu2)/(math.exp(wx0+1) - 1 + mu1)\r\n",
        "    alpha01 = 1 - (En+En_bar)/(self.T*self.eta*self.Pn*hn0)\r\n",
        "    alpha02 = (self.T*self.eta*self.Pn*hn0-En_bar)\\\r\n",
        "              /(self.T*self.eta*self.Pn*hn0 + self.T*self.Pmax)\r\n",
        "    alphax2 = max(alpha01, alpha02)\r\n",
        "    alphan = min(1, max(alphaxx,alphax2))\r\n",
        "    #print(alphan)\r\n",
        "    if En_bar == self.T * self.eta * self.Pn * hn0:\r\n",
        "      P0n=0\r\n",
        "      reward = 0  # remark in the paper\r\n",
        "    elif alphan ==0:#<= 0.00000001:\r\n",
        "      P0n=0\r\n",
        "      reward = 0\r\n",
        "    else:\r\n",
        "      P0n = (1-alphan)*self.eta*self.Pn*hn0/alphan - En_bar/alphan/self.T\r\n",
        "      #print(f\"power is {P0n}, en is {En}, enbar is {En_bar}, alpha is {alphan}\")\r\n",
        "      #print(f\"power {P0n} divided by noise {self.noise} as {P0n/self.noise}\")\r\n",
        "      reward = alphan*np.log(1 + P0n*h0/(1 +self.Pn*hn))\r\n",
        "\r\n",
        "    #print(f\"power is {P0n}, en is {En}, enbar is {En_bar}, alpha is {alphan}\")\r\n",
        "    #print(f\"signal power is {P0n*h0} and inter is {self.Pn*hn}\")\r\n",
        "    #print(f\"noise is {self.noise}\")\r\n",
        "    #print(f\"signal strentch={P0n} times {h0},   interference is {self.Pn * hn}\")\r\n",
        "    #print(f\"Pn is {self.Pn} and hn is {hn}\")\r\n",
        "    #print(f\"rate is {np.log(1 + P0n * h0 / (1 + self.Pn * hn))}\")\r\n",
        "    #print(f\"signal strentch={P0n} times {h0}, noise is {self.noise}, interference is {self.Pn*hn}\")\r\n",
        "    #print(f\"{j} iteration - reward is {reward}\")\r\n",
        "    #print(f\"get {(1-alphan)*self.T * self.eta * self.Pn * hn0} and consumed {alphan*self.T*P0n}\")\r\n",
        "    #print(f\"get {(1-alphan)}* {self.T} * {self.eta} * {self.Pn} * {hn0} \")\r\n",
        "\r\n",
        "    #print(f\"the first part of power{(1 - alphan)}*{self.eta * self.Pn * hn0 / alphan}\")\r\n",
        "    #print(f\"the second part of power{- En_bar/alphan/self.T}\")\r\n",
        "\r\n",
        "    if math.isnan(reward):\r\n",
        "      print(f\"start---------------------------------------\")\r\n",
        "      print(f\"action is{action} \")\r\n",
        "      print(f\"finallly alphan is {alphan}\")\r\n",
        "      print(f\"end-----------------------------------\")\r\n",
        "      reward = 0\r\n",
        "    #if reward <0:\r\n",
        "    #    print(f\"alpha is {alphan} and power is {P0n}, reward is {reward}, \"\r\n",
        "    #          f\"bound is {1 - (En_bar)/(self.T*self.eta*self.Pn*hn0)}, enbar is {En_bar}, exist battery is {En}\")\r\n",
        "\r\n",
        "\r\n",
        "    batter_new = min(self.emax, En + En_bar)\r\n",
        "\r\n",
        "    #print(f\"{j} - Energy:  {En} minus {(alphan)*self.T*P0n} plus {(1-alphan)*self.T*self.eta*self.Pn*hn0}, remain {batter_new}, enbar is {En_bar} \")\r\n",
        "    #print(f\"Parameters are - alpha {alphan}, power is {P0n},   GB-GF channel is {hn0},    reward is {reward}\")\r\n",
        "\r\n",
        "    state_next = self.channel_sequence[(j+1) % self.K, :].tolist()\r\n",
        "    #state_next.append(self.h0)\r\n",
        "    state_next.append(batter_new)\r\n",
        "    state_next = np.reshape(state_next, (1, self.s_dim))\r\n",
        "\r\n",
        "    done=False\r\n",
        "\r\n",
        "    return reward, state_next, done\r\n",
        "\r\n",
        "  def step_greedy(self,   state, j):\r\n",
        "    hn = state[0,0]/self.noise\r\n",
        "    hn0 = state[0,1]\r\n",
        "    h0 = self.h0/self.noise\r\n",
        "    En = state[0,-1]\r\n",
        "    #print(uchannel)\r\n",
        "\r\n",
        "    alphan = min(1, En/self.T/self.Pmax)\r\n",
        "    if alphan==0:\r\n",
        "      reward = 0\r\n",
        "    else:\r\n",
        "      reward = alphan*np.log(1 + self.Pmax*h0/(1 +self.Pn*hn))\r\n",
        "      #print(f\"random alpha is {alphan}- signal strentch={self.Pmax} times {h0}, noise is {self.noise}, interference is {self.Pn*hn}\")\r\n",
        "      #print(f\"{j}-iteration - energy is {En}-random reward is {reward}\")\r\n",
        "\r\n",
        "\r\n",
        "    if math.isnan(reward):\r\n",
        "      print(f\"alpha is {alphan}  \")\r\n",
        "      print(f\"{En}\")\r\n",
        "    batter_new = min(self.emax, En -alphan*self.T*self.Pmax +(1-alphan)*self.T*self.eta*self.Pn*hn0)\r\n",
        "\r\n",
        "    #print(f\"{j} random - Energy:  {En} minus {(alphan)*self.T*self.Pmax} plus {(1-alphan)*self.T*self.eta*self.Pn*hn0}, remain {batter_new} \")\r\n",
        "    #print(f\"Parameters are - alpha {alphan}, power is {self.Pmax},   GB-GF channel is {hn0},    reward is {reward}\")\r\n",
        "\r\n",
        "    state_next = self.channel_sequence[(j+1) % self.K, :].tolist()\r\n",
        "    #state_next.append(self.h0)\r\n",
        "    state_next.append(batter_new)\r\n",
        "    state_next = np.reshape(state_next, (1, self.s_dim))\r\n",
        "\r\n",
        "    done=False\r\n",
        "\r\n",
        "    return reward, state_next, done\r\n",
        "\r\n",
        "  def step_random(self,   state, j):\r\n",
        "    hn = state[0,0]/self.noise\r\n",
        "    hn0 = state[0,1]\r\n",
        "    h0 = self.h0/self.noise\r\n",
        "    En = state[0,-1]\r\n",
        "    #print(uchannel)\r\n",
        "\r\n",
        "    alphan = np.random.uniform(0, min(1, En/self.T/self.Pmax))\r\n",
        "    if alphan==0:\r\n",
        "      reward = 0\r\n",
        "    else:\r\n",
        "      reward = alphan*np.log(1 + self.Pmax*h0/(1 +self.Pn*hn))\r\n",
        "      #print(f\"random alpha is {alphan}- signal strentch={self.Pmax} times {h0}, noise is {self.noise}, interference is {self.Pn*hn}\")\r\n",
        "      #print(f\"{j}-iteration - energy is {En}-random reward is {reward}\")\r\n",
        "\r\n",
        "\r\n",
        "    if math.isnan(reward):\r\n",
        "      print(f\"alpha is {alphan}  \")\r\n",
        "      print(f\"{En}\")\r\n",
        "    batter_new = min(self.emax, En -alphan*self.T*self.Pmax +(1-alphan)*self.T*self.eta*self.Pn*hn0)\r\n",
        "\r\n",
        "    #print(f\"{j} random - Energy:  {En} minus {(alphan)*self.T*self.Pmax} plus {(1-alphan)*self.T*self.eta*self.Pn*hn0}, remain {batter_new} \")\r\n",
        "    #print(f\"Parameters are - alpha {alphan}, power is {self.Pmax},   GB-GF channel is {hn0},    reward is {reward}\")\r\n",
        "\r\n",
        "    state_next = self.channel_sequence[(j+1) % self.K, :].tolist()\r\n",
        "    #state_next.append(self.h0)\r\n",
        "    state_next.append(batter_new)\r\n",
        "    state_next = np.reshape(state_next, (1, self.s_dim))\r\n",
        "\r\n",
        "    done=False\r\n",
        "\r\n",
        "    return reward, state_next, done\r\n",
        "\r\n",
        "  def reset(self):\r\n",
        "    batter_ini = self.emax\r\n",
        "    return batter_ini"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EI6W9xJRWror"
      },
      "source": [
        "# Define DDPG Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SUWxv3p4WumZ"
      },
      "source": [
        "INIT = 0\r\n",
        "\r\n",
        "#####################  hyper parameters  ####################\r\n",
        "MAX_EP_STEPS = 100\r\n",
        "LR_A = 0.002    # learning rate for actor\r\n",
        "LR_C = 0.004    # learning rate for critic\r\n",
        "GAMMA = 0.9     # reward discount\r\n",
        "TAU = 0.01      # soft replacement\r\n",
        "MEMORY_CAPACITY = 10000\r\n",
        "BATCH_SIZE = 32\r\n",
        "\r\n",
        "#####################  INIT  ####################\r\n",
        "INIT = 0\r\n",
        "#### INIT:\r\n",
        "# RANDOM LOCATION WITH FADING K: 0\r\n",
        "# TWO USER CASE                : 1\r\n",
        "# SPECIAL CASE                 : 2\r\n",
        "if INIT == 0:         # RANDOM LOCATION WITH FADING K #\r\n",
        "  ct=500              # \r\n",
        "  MAX_EPISODES =10    #\r\n",
        "\r\n",
        "elif INIT == 1:    # TWO USER CASE #\r\n",
        "  Pn = 1              #\r\n",
        "  K=2                 #\r\n",
        "  MAX_EPISODES = 400  #\r\n",
        "  MAX_EP_STEPS = 100  #\r\n",
        "else:                 # SPECIAL CASE #\r\n",
        "  Pn = 1\r\n",
        "  K=10 # the number of grant based users\r\n",
        "  MAX_EPISODES = 400\r\n",
        "\r\n",
        "###############################  DDPG  ####################################\r\n",
        "class DDPG(object):\r\n",
        "  def __init__(self, a_dim, s_dim, a_bound,):\r\n",
        "    self.memory = np.zeros((MEMORY_CAPACITY, s_dim * 2 + a_dim + 1), dtype=np.float32)\r\n",
        "    self.pointer = 0          # memory of Experience Replay used pointer\r\n",
        "    self.sess = tf.Session()\r\n",
        "\r\n",
        "\r\n",
        "    self.a_dim, self.s_dim, self.a_bound = a_dim, s_dim, a_bound,\r\n",
        "    # Get input, hyper-parameters for model\r\n",
        "    self.S = tf.placeholder(tf.float32, [None, s_dim], 's')\r\n",
        "    self.S_ = tf.placeholder(tf.float32, [None, s_dim], 's_')\r\n",
        "    self.R = tf.placeholder(tf.float32, [None, 1], 'r')\r\n",
        "\r\n",
        "    self.a = self._build_a(self.S,)\r\n",
        "    q = self._build_c(self.S, self.a, )\r\n",
        "    a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Actor')\r\n",
        "    c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Critic')\r\n",
        "    ema = tf.train.ExponentialMovingAverage(decay=1 - TAU)          # soft replacement\r\n",
        "    ###################################################################\r\n",
        "\r\n",
        "    def ema_getter(getter, name, *args, **kwargs):\r\n",
        "      return ema.average(getter(name, *args, **kwargs))\r\n",
        "\r\n",
        "    target_update = [ema.apply(a_params), ema.apply(c_params)]      # soft update operation\r\n",
        "    a_ = self._build_a(self.S_, reuse=True, custom_getter=ema_getter)   # replaced target parameters\r\n",
        "    q_ = self._build_c(self.S_, a_, reuse=True, custom_getter=ema_getter)\r\n",
        "\r\n",
        "    a_loss = - tf.reduce_mean(q)  # maximize the q\r\n",
        "    self.atrain = tf.train.AdamOptimizer(LR_A).minimize(a_loss, var_list=a_params)\r\n",
        "\r\n",
        "    with tf.control_dependencies(target_update):    # soft replacement happened at here\r\n",
        "      q_target = self.R + GAMMA * q_\r\n",
        "      td_error = tf.losses.mean_squared_error(labels=q_target, predictions=q)\r\n",
        "      self.ctrain = tf.train.AdamOptimizer(LR_C).minimize(td_error, var_list=c_params)\r\n",
        "\r\n",
        "    self.sess.run(tf.global_variables_initializer())\r\n",
        "\r\n",
        "  def choose_action(self, s):\r\n",
        "    return self.sess.run(self.a, {self.S: s })[0]\r\n",
        "\r\n",
        "  def learn(self):\r\n",
        "    # Choose randomly BATCH_SIZE addresses which will be used to takes value from Experience Replay memory\r\n",
        "    indices = np.random.choice(min(MEMORY_CAPACITY,self.pointer), size=BATCH_SIZE)  \r\n",
        "    bt  = self.memory[indices, :]\r\n",
        "    bs  = bt[:, :self.s_dim]\r\n",
        "    ba  = bt[:, self.s_dim: self.s_dim + self.a_dim]\r\n",
        "    br  = bt[:, -self.s_dim - 1: -self.s_dim]\r\n",
        "    bs_ = bt[:, -self.s_dim:]\r\n",
        "\r\n",
        "    self.sess.run(self.atrain, {self.S: bs})\r\n",
        "    self.sess.run(self.ctrain, {self.S: bs, self.a: ba, self.R: br, self.S_: bs_})\r\n",
        "\r\n",
        "  def store_transition(self, s, a, r, s_):\r\n",
        "    r = np.reshape(r,(1,1))\r\n",
        "    a = np.reshape(a,(1,1))\r\n",
        "    #print(f\"state is {s}, action is {a}, reward is {r}, next state is {s_}\")\r\n",
        "    transition = np.hstack((s, a, r, s_))\r\n",
        "    index = self.pointer % MEMORY_CAPACITY  # replace the old memory with new memory\r\n",
        "    self.memory[index, :] = transition\r\n",
        "    self.pointer += 1\r\n",
        "\r\n",
        "  def _build_a(self, s, reuse=None, custom_getter=None):\r\n",
        "    trainable = True if reuse is None else False\r\n",
        "    with tf.variable_scope('Actor', reuse=reuse, custom_getter=custom_getter):\r\n",
        "      net = tf.layers.dense(s, 64, activation=tf.nn.relu, name='l1', trainable=trainable)\r\n",
        "      a2 = tf.layers.dense(net, 64, activation=tf.nn.tanh, name='l2', trainable=trainable)\r\n",
        "      #a3 = tf.layers.dense(a2, 30, activation=tf.nn.tanh, name='l3', trainable=trainable)\r\n",
        "\r\n",
        "      a = tf.layers.dense(a2, self.a_dim, activation=tf.nn.tanh, name='a', trainable=trainable)\r\n",
        "      return tf.multiply(a, self.a_bound, name='scaled_a')\r\n",
        "\r\n",
        "  def _build_c(self, s, a, reuse=None, custom_getter=None):\r\n",
        "    trainable = True if reuse is None else False\r\n",
        "    with tf.variable_scope('Critic', reuse=reuse, custom_getter=custom_getter):\r\n",
        "      n_l1 = 64\r\n",
        "      w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], trainable=trainable)\r\n",
        "      w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], trainable=trainable)\r\n",
        "      b1 = tf.get_variable('b1', [1, n_l1], trainable=trainable)\r\n",
        "      net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)\r\n",
        "      net2 = tf.layers.dense(net, 64, activation=tf.nn.relu, name='lx2', trainable=trainable)\r\n",
        "      #net3 = tf.layers.dense(net2, 30, activation=tf.nn.relu, name='lx3', trainable=trainable)\r\n",
        "\r\n",
        "      #not sure about this part\r\n",
        "      return tf.layers.dense(net2, 1, trainable=trainable)  # Q(s,a)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y9w_rMclaQf0"
      },
      "source": [
        "# DDPG Training: \r\n",
        "Action: alpha (percentage of duration T)\r\n",
        "\r\n",
        "Observation/State: "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8yyz6kiSWkCF"
      },
      "source": [
        "#**Random location with fading K: TRAINING**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JqDlAzTfXcYi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "64e5691b-c843-4e8a-a644-66047b6df993"
      },
      "source": [
        "###############################  training  ####################################\r\n",
        "s_dim = 3# dimsion of states\r\n",
        "a_dim = 1# dimension of action\r\n",
        "a_bound = 1 #bound of action\r\n",
        "state_am = 10000\r\n",
        "\r\n",
        "\r\n",
        "#GB_power_dBm = np.array([0,  10,  20,   30])\r\n",
        "K_range = np.array([2,4,6,8, 10])\r\n",
        "rate_DDPG = []\r\n",
        "rate_greedy = []\r\n",
        "rate_random = []\r\n",
        "for k in range(len(K_range)):\r\n",
        "    K = K_range[k]\r\n",
        "    ratect_DDPG = 0\r\n",
        "    ratect_greedy = 0\r\n",
        "    ratect_random = 0\r\n",
        "    for mct in range(ct):\r\n",
        "        var = 1  # control exploration\r\n",
        "        ddpg = DDPG(a_dim, s_dim, a_bound)\r\n",
        "\r\n",
        "        locationspace = np.linspace(1, 1000, num=K)\r\n",
        "        location_vector = np.zeros((K, 2))\r\n",
        "        location_vector[:, 1] = locationspace\r\n",
        "        location_GF = np.array([[1,1]])# np.ones((1, 2))\r\n",
        "\r\n",
        "        ##### fading for GB user\r\n",
        "        hnx1 = np.random.randn(K, 2)\r\n",
        "        hnx2 = np.random.randn(K, 2)\r\n",
        "        fading_n = hnx1 ** 2 + hnx2 ** 2\r\n",
        "        #### fading for GF user\r\n",
        "        h0x1 = np.random.randn(1, 1)\r\n",
        "        h0x2 = np.random.randn(1, 1)\r\n",
        "        fading_0 =  h0x1[0,0] ** 2 + h0x2[0,0] ** 2\r\n",
        "\r\n",
        "        Pn = 10**((30-30)/10)\r\n",
        "        myenv = Env_cellular( MAX_EP_STEPS, s_dim, location_vector,location_GF,K,Pn, fading_n,ffading_0 )\r\n",
        "\r\n",
        "        ratek_DDPG = 0\r\n",
        "        ratek_greedy = 0\r\n",
        "        ratek_random = 0\r\n",
        "        for i in range(MAX_EPISODES):\r\n",
        "            batter_ini = myenv.reset()\r\n",
        "            s = myenv.channel_sequence[i % myenv.K, :].tolist()  # the current GB user, 2 element [GB-GF, GB-BS]\r\n",
        "            # s.append(myenv.h0)\r\n",
        "            s.append(batter_ini)\r\n",
        "            s = np.reshape(s, (1, s_dim))\r\n",
        "            s = s * state_am  # amplify the state\r\n",
        "            s_greedy = s\r\n",
        "            s_random = s\r\n",
        "            reward_dpgg_vector =[]\r\n",
        "            reward_greedy_vector = []\r\n",
        "            reward_random_vector = []\r\n",
        "            #print(s[0,0:2])\r\n",
        "            for j in range(MAX_EP_STEPS):\r\n",
        "\r\n",
        "                # Add exploration noise\r\n",
        "                a = ddpg.choose_action(s)\r\n",
        "                a = np.clip(np.random.normal(a, var), 0, 1)    # add randomness to action selection for exploration\r\n",
        "                #print(myenv.location)\r\n",
        "                r, s_, done = myenv.step(a, s / state_am, j)\r\n",
        "                s_ = s_ * state_am\r\n",
        "                #print(f\"{j} state is {s}, action is {a}, reward is {r}, next state is {s_}\")\r\n",
        "                ddpg.store_transition(s, a, r, s_)\r\n",
        "                if var>0.05:\r\n",
        "                    var *= .9998  # decay the action randomness\r\n",
        "                ddpg.learn()\r\n",
        "                s = s_\r\n",
        "                reward_dpgg_vector.append(r)\r\n",
        "\r\n",
        "                ##### greedy\r\n",
        "                r_greedy, s_next_greedy, done = myenv.step_greedy(s_greedy/state_am, j)\r\n",
        "                s_greedy = s_next_greedy*state_am\r\n",
        "                reward_greedy_vector.append(r_greedy)\r\n",
        "\r\n",
        "                ##### random\r\n",
        "                r_random, s_next_random, done = myenv.step_random(s_random/state_am, j)\r\n",
        "                s_random = s_next_random*state_am\r\n",
        "                reward_random_vector.append(r_random)\r\n",
        "\r\n",
        "                if j == MAX_EP_STEPS-1:\r\n",
        "                    #print(f\"Iteration., {k}--{mct},  Episode:, {i},  Reward: {sum(reward_dpgg_vector)/MAX_EP_STEPS},\")\r\n",
        "                    #      f\" Reward Greedy:   {sum(reward_greedy_vector)/MAX_EP_STEPS},\"\r\n",
        "                    #      f\" Reward random:   { sum(reward_random_vector)/MAX_EP_STEPS}, Explore: {var} \")\r\n",
        "                    break\r\n",
        "            ratek_DDPG = sum(reward_dpgg_vector)/MAX_EP_STEPS\r\n",
        "            ratek_greedy = sum(reward_greedy_vector)/MAX_EP_STEPS\r\n",
        "            ratek_random = sum(reward_random_vector)/MAX_EP_STEPS\r\n",
        "\r\n",
        "        tf.keras.backend.clear_session()\r\n",
        "        ratect_DDPG +=ratek_DDPG\r\n",
        "        ratect_greedy +=ratek_greedy\r\n",
        "        ratect_random += ratek_random\r\n",
        "        print(ratek_DDPG)\r\n",
        "    print(f\"Iteration., {k}--{mct},  Episode:, {i},  Reward: {ratect_DDPG/ct },\")\r\n",
        "    rate_DDPG.append(ratect_DDPG/ct )\r\n",
        "    rate_greedy.append(ratect_greedy /ct )\r\n",
        "    rate_random.append(ratect_random  /ct)\r\n",
        "\r\n",
        "print(f\"rate_greedy is {rate_greedy} and rate_random is {rate_random}\")\r\n",
        "#GB_power_label = ['0',  '10',  '20', '30']\r\n",
        "GB_power_label = ['0', '5', '10', '15', '20', '25', '30']\r\n",
        "K_rangex =  ['2', '4', '6', '8', '10']\r\n",
        "plt.plot(K_rangex,rate_DDPG, \"^-\", label='DDPG: rewards')\r\n",
        "plt.plot(K_rangex,rate_greedy, \"+:\", label='Greedy: rewards')\r\n",
        "plt.plot(K_rangex,rate_random, \"o--\", label='Random: rewards')\r\n",
        "plt.xlabel(\"The Number of Primary Users (K)\")\r\n",
        "plt.ylabel(\"  Average  Data Rate (NPCU)\")\r\n",
        "plt.legend()\r\n",
        "plt.show()\r\n",
        "\r\n",
        "\r\n",
        "''' Save final results'''\r\n",
        "#np.savez_compressed('data/dataK', rate_DDPG=rate_DDPG, rate_greedy=rate_greedy, rate_random=rate_random)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/legacy_tf_layers/core.py:171: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
            "  warnings.warn('`tf.layers.dense` is deprecated and '\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py:1719: UserWarning: `layer.apply` is deprecated and will be removed in a future version. Please use `layer.__call__` method instead.\n",
            "  warnings.warn('`layer.apply` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-54e43c4da5c6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mmct\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mct\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mvar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m  \u001b[0;31m# control exploration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0mddpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDDPG\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma_bound\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mlocationspace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-28-4dc50b1d5261>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, a_dim, s_dim, a_bound)\u001b[0m\n\u001b[1;32m     43\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplaceholder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_a\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m     \u001b[0mq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_c\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0ma_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAINABLE_VARIABLES\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Actor'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-28-4dc50b1d5261>\u001b[0m in \u001b[0;36m_build_a\u001b[0;34m(self, s, reuse, custom_getter)\u001b[0m\n\u001b[1;32m     92\u001b[0m     \u001b[0mtrainable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreuse\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Actor'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_getter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_getter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m       \u001b[0mnet\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'l1'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m       \u001b[0ma2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'l2'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m       \u001b[0;31m#a3 = tf.layers.dense(a2, 30, activation=tf.nn.tanh, name='l3', trainable=trainable)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/legacy_tf_layers/core.py\u001b[0m in \u001b[0;36mdense\u001b[0;34m(inputs, units, activation, use_bias, kernel_initializer, bias_initializer, kernel_regularizer, bias_regularizer, activity_regularizer, kernel_constraint, bias_constraint, trainable, name, reuse)\u001b[0m\n\u001b[1;32m    186\u001b[0m                 \u001b[0m_scope\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m                 _reuse=reuse)\n\u001b[0;32m--> 188\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1720\u001b[0m                   \u001b[0;34m'will be removed in a future version. '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1721\u001b[0m                   'Please use `layer.__call__` method instead.')\n\u001b[0;32m-> 1722\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1723\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1724\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mdoc_controls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdo_not_doc_inheritable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/legacy_tf_layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    549\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    550\u001b[0m       \u001b[0;31m# Actually call layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 551\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    552\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    553\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    764\u001b[0m           \u001b[0;31m# Build layer if applicable (if the `build` method has been\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m           \u001b[0;31m# overridden).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 766\u001b[0;31m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_build\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    767\u001b[0m           \u001b[0mcast_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_inputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py\u001b[0m in \u001b[0;36m_maybe_build\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2104\u001b[0m         \u001b[0;31m# operations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2105\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaybe_init_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2106\u001b[0;31m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2107\u001b[0m       \u001b[0;31m# We must set also ensure that the layer is marked as built, and the build\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2108\u001b[0m       \u001b[0;31m# shape is stored since user defined build functions may not be calling\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, input_shape)\u001b[0m\n\u001b[1;32m   1190\u001b[0m         \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernel_constraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1192\u001b[0;31m         trainable=True)\n\u001b[0m\u001b[1;32m   1193\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_bias\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1194\u001b[0m       self.bias = self.add_weight(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/legacy_tf_layers/base.py\u001b[0m in \u001b[0;36madd_weight\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, trainable, constraint, use_resource, synchronization, aggregation, partitioner, **kwargs)\u001b[0m\n\u001b[1;32m    462\u001b[0m             \u001b[0maggregation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maggregation\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    463\u001b[0m             \u001b[0mgetter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 464\u001b[0;31m             **kwargs)\n\u001b[0m\u001b[1;32m    465\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    466\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py\u001b[0m in \u001b[0;36madd_weight\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, trainable, constraint, partitioner, use_resource, synchronization, aggregation, **kwargs)\u001b[0m\n\u001b[1;32m    455\u001b[0m         \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m         \u001b[0maggregation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maggregation\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m         caching_device=caching_device)\n\u001b[0m\u001b[1;32m    458\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mregularizer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m       \u001b[0;31m# TODO(fchollet): in the future, this should be handled at the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py\u001b[0m in \u001b[0;36m_add_variable_with_custom_getter\u001b[0;34m(self, name, shape, dtype, initializer, getter, overwrite, **kwargs_for_getter)\u001b[0m\n\u001b[1;32m    808\u001b[0m         \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m         \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 810\u001b[0;31m         **kwargs_for_getter)\n\u001b[0m\u001b[1;32m    811\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    812\u001b[0m     \u001b[0;31m# If we set an initializer and the variable processed it, tracking will not\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m   1591\u001b[0m       \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconstraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1592\u001b[0m       \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1593\u001b[0;31m       aggregation=aggregation)\n\u001b[0m\u001b[1;32m   1594\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1595\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m   1334\u001b[0m           \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconstraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1335\u001b[0m           \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1336\u001b[0;31m           aggregation=aggregation)\n\u001b[0m\u001b[1;32m   1337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1338\u001b[0m   def _get_partitioned_variable(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m    589\u001b[0m           \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconstraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    590\u001b[0m           \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 591\u001b[0;31m           aggregation=aggregation)\n\u001b[0m\u001b[1;32m    592\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    593\u001b[0m   def _get_partitioned_variable(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m    541\u001b[0m           \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconstraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    542\u001b[0m           \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 543\u001b[0;31m           aggregation=aggregation)\n\u001b[0m\u001b[1;32m    544\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m     synchronization, aggregation, trainable = (\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape, use_resource, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;34m\"tensorflow/python\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    892\u001b[0m         raise ValueError(\"%s Originally defined at:\\n\\n%s\" %\n\u001b[0;32m--> 893\u001b[0;31m                          (err_msg, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m    894\u001b[0m       \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    895\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Variable Actor/l1/kernel already exists, disallowed. Did you mean to set reuse=True or reuse=tf.AUTO_REUSE in VarScope? Originally defined at:\n\n  File \"<ipython-input-5-4086c11b8cac>\", line 99, in _build_a\n    net = tf.layers.dense(s, 64, activation=tf.nn.relu, name='l1', trainable=trainable)  # fully connected\n  File \"<ipython-input-5-4086c11b8cac>\", line 46, in __init__\n    self.a = self._build_a(self.S,)\n  File \"<ipython-input-6-44ba89d6b2af>\", line 22, in <module>\n    ddpg = DDPG(a_dim, s_dim, a_bound) # define a class of ddpg\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ocaOLkKzBCG7"
      },
      "source": [
        "#**Two user case**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBZJFbAyBMs6"
      },
      "source": [
        "\r\n",
        "s_dim = 3# dimsion of states\r\n",
        "a_dim = 1# dimension of action\r\n",
        "a_bound = 1 #bound of action\r\n",
        "state_am = 1000\r\n",
        "\r\n",
        "\r\n",
        "location_vector = np.array([[0, 1],[0,1000]]) #locations of GB users\r\n",
        "\r\n",
        "\r\n",
        "location_GF = np.array([[1,1]])# np.ones((1, 2))\r\n",
        "\r\n",
        "##### fading for GB user\r\n",
        "hnx1 = np.random.randn(K, 2)\r\n",
        "hnx2 = np.random.randn(K, 2)\r\n",
        "fading_n = 1#hnx1 ** 2 + hnx2 ** 2\r\n",
        "#### fading for GF user\r\n",
        "h0x1 = np.random.randn(1, 1)\r\n",
        "h0x2 = np.random.randn(1, 1)\r\n",
        "fading_0 = 1#h0x1[0,0] ** 2 + h0x2[0,0] ** 2\r\n",
        "\r\n",
        "\r\n",
        "myenv = env( MAX_EP_STEPS, s_dim, location_vector,location_GF,K,Pn, fading_n, fading_0)\r\n",
        "#myenv = env(P0, MAX_EP_STEPS, s_dim, location_vector,location_GF,K)\r\n",
        "\r\n",
        "#myenv = env(P0,MAX_EP_STEPS,s_dim)\r\n",
        "\r\n",
        "ddpg = DDPG(a_dim, s_dim, a_bound)\r\n",
        "\r\n",
        "var = 1  # control exploration\r\n",
        "t1 = time.time()\r\n",
        "ep_rewardall = []\r\n",
        "ep_rewardall_greedy = []\r\n",
        "ep_rewardall_random = []\r\n",
        "for i in range(MAX_EPISODES):\r\n",
        "  batter_ini = myenv.reset()\r\n",
        "  s = myenv.channel_sequence[i%myenv.K,:].tolist() #the current GB user, 2 element [GB-GF, GB-BS]\r\n",
        "  #s.append(myenv.h0)\r\n",
        "  s.append(batter_ini)\r\n",
        "  s = np.reshape(s,(1,s_dim))\r\n",
        "  s = s*state_am #amplify the state\r\n",
        "  s_greedy = s\r\n",
        "  s_random = s\r\n",
        "  #print(s[0,0:2])\r\n",
        "  ep_reward = 0\r\n",
        "  ep_reward_random = 0\r\n",
        "  ep_reward_greedy = 0\r\n",
        "  s_traj = []\r\n",
        "  s_traj_random = []\r\n",
        "  s_traj_greedy = []\r\n",
        "  for j in range(MAX_EP_STEPS):\r\n",
        "\r\n",
        "    # Add exploration noise\r\n",
        "    a = ddpg.choose_action(s)\r\n",
        "    #print(s)\r\n",
        "    a = np.clip(np.random.normal(a, var), 0, 1)    # add randomness to action selection for exploration\r\n",
        "    #print(myenv.location)\r\n",
        "    r, s_, done = myenv.step(a,s/state_am,j)\r\n",
        "    s_ = s_*state_am\r\n",
        "    s_traj.append(s_)\r\n",
        "    ddpg.store_transition(s, a, r, s_)\r\n",
        "\r\n",
        "    if var > 0.1:\r\n",
        "      var *= .9998  # decay the action randomness\r\n",
        "    ddpg.learn()\r\n",
        "\r\n",
        "    s = s_\r\n",
        "    ep_reward += r\r\n",
        "\r\n",
        "    ##### greedy\r\n",
        "    r_greedy, s_next_greedy, done = myenv.step_greedy(s_greedy/state_am, j)\r\n",
        "    s_traj_greedy.append(s_next_greedy)\r\n",
        "    s_greedy = s_next_greedy*state_am\r\n",
        "    ep_reward_greedy += r_greedy\r\n",
        "\r\n",
        "    ##### random\r\n",
        "    r_random, s_next_random, done = myenv.step_random(s_random/state_am, j)\r\n",
        "    s_traj_random.append(s_next_random)\r\n",
        "    s_random = s_next_random*state_am\r\n",
        "    ep_reward_random += r_random\r\n",
        "\r\n",
        "\r\n",
        "    if j == MAX_EP_STEPS-1:\r\n",
        "      #print(f\"Episode: {i}, reward is {ep_reward}, and Explore is {var}\")\r\n",
        "      print('Episode:', i, ' Reward: %i' % int(ep_reward),' Reward Greedy: %i' % int(ep_reward_greedy),' Reward random: %i' % int(ep_reward_random), 'Explore: %.2f' % var )\r\n",
        "      #print(myenv.location)\r\n",
        "      # if ep_reward > -300:RENDER = True\r\n",
        "      break\r\n",
        "  ep_reward = np.reshape(ep_reward/MAX_EP_STEPS, (1,))\r\n",
        "  ep_rewardall.append(ep_reward)\r\n",
        "\r\n",
        "  ep_reward_greedy = np.reshape(ep_reward_greedy/MAX_EP_STEPS, (1,))\r\n",
        "  ep_rewardall_greedy.append(ep_reward_greedy)\r\n",
        "\r\n",
        "  ep_reward_random = np.reshape(ep_reward_random/MAX_EP_STEPS, (1,))\r\n",
        "  ep_rewardall_random.append(ep_reward_random)\r\n",
        "\r\n",
        "#print(s_)\r\n",
        "print('Running time: ', time.time() - t1)\r\n",
        "\r\n",
        "print(f\"{myenv.hn} and {myenv.h0}\")\r\n",
        "plt.plot(ep_rewardall, \"^-\", label='DDPG: rewards')\r\n",
        "plt.plot(ep_rewardall_greedy, \"+:\", label='Greedy: rewards')\r\n",
        "plt.plot(ep_rewardall_random, \"o--\", label='Random: rewards')\r\n",
        "plt.xlabel(\"Episode\")\r\n",
        "plt.ylabel(\" Epsiodic Reward -  Data Rate (NPCU)\")\r\n",
        "plt.legend()\r\n",
        "plt.show()\r\n",
        "\r\n",
        "''' Save final results'''\r\n",
        "#np.savez_compressed('data/data_two_user', ep_rewardall=ep_rewardall, ep_rewardall_greedy=ep_rewardall_greedy, ep_rewardall_random=ep_rewardall_random)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lwczTnrTBVGz"
      },
      "source": [
        "#**Special case**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FzmqX3ePBZVj"
      },
      "source": [
        "s_dim = 3# dimsion of states\r\n",
        "a_dim = 1# dimension of action\r\n",
        "a_bound = 1 #bound of action\r\n",
        "state_am = 10000\r\n",
        "\r\n",
        "\r\n",
        "locationspace = np.linspace(1,1000, num=K)\r\n",
        "location_vector = np.zeros((K, 2))\r\n",
        "location_vector[:,1] = locationspace\r\n",
        "\r\n",
        "\r\n",
        "location_GF = np.array([[1,1]])# np.ones((1, 2))\r\n",
        "\r\n",
        "##### fading for GB user\r\n",
        "hnx1 = np.random.randn(K, 2)\r\n",
        "hnx2 = np.random.randn(K, 2)\r\n",
        "fading_n = hnx1 ** 2 + hnx2 ** 2\r\n",
        "#### fading for GF user\r\n",
        "h0x1 = np.random.randn(1, 1)\r\n",
        "h0x2 = np.random.randn(1, 1)\r\n",
        "fading_0 = h0x1[0,0] ** 2 + h0x2[0,0] ** 2\r\n",
        "\r\n",
        "\r\n",
        "myenv = env(  MAX_EP_STEPS, s_dim, location_vector,location_GF,K,Pn, fading_n, fading_0)\r\n",
        "#myenv = env(P0, MAX_EP_STEPS, s_dim, location_vector,location_GF,K)\r\n",
        "\r\n",
        "#myenv = env(P0,MAX_EP_STEPS,s_dim)\r\n",
        "\r\n",
        "ddpg = DDPG(a_dim, s_dim, a_bound)\r\n",
        "\r\n",
        "var = 1  # control exploration\r\n",
        "t1 = time.time()\r\n",
        "ep_rewardall = []\r\n",
        "ep_rewardall_greedy = []\r\n",
        "ep_rewardall_random = []\r\n",
        "for i in range(MAX_EPISODES):\r\n",
        "  batter_ini = myenv.reset()\r\n",
        "  s = myenv.channel_sequence[i%myenv.K,:].tolist()\r\n",
        "  #s.append(myenv.h0)\r\n",
        "  s.append(batter_ini)\r\n",
        "  s = np.reshape(s,(1,s_dim))\r\n",
        "  s = s*state_am #amplify the state\r\n",
        "  s_greedy = s\r\n",
        "  s_random = s\r\n",
        "  #print(s[0,0:2])\r\n",
        "  ep_reward = 0\r\n",
        "  ep_reward_random = 0\r\n",
        "  ep_reward_greedy = 0\r\n",
        "  for j in range(MAX_EP_STEPS):\r\n",
        "\r\n",
        "    # Add exploration noise\r\n",
        "    a = ddpg.choose_action(s)\r\n",
        "    a = np.clip(np.random.normal(a, var), 0, 1)    # add randomness to action selection for exploration\r\n",
        "    r, s_, done = myenv.step(a,s/state_am,j)\r\n",
        "    s_ = s_ * state_am\r\n",
        "    ddpg.store_transition(s, a, r, s_)\r\n",
        "    if var >0.1:\r\n",
        "      var *= .9998    # decay the action randomness\r\n",
        "\r\n",
        "    ddpg.learn()\r\n",
        "    s = s_\r\n",
        "    ep_reward += r\r\n",
        "\r\n",
        "    ##### greedy\r\n",
        "    r_greedy, s_next_greedy, done = myenv.step_greedy(s_greedy/state_am, j)\r\n",
        "    s_greedy = s_next_greedy*state_am\r\n",
        "    ep_reward_greedy += r_greedy\r\n",
        "\r\n",
        "    ##### random\r\n",
        "    r_random, s_next_random, done = myenv.step_random(s_random/state_am, j)\r\n",
        "    s_random = s_next_random*state_am\r\n",
        "    ep_reward_random += r_random\r\n",
        "\r\n",
        "\r\n",
        "    if j == MAX_EP_STEPS-1:\r\n",
        "      #print(f\"Episode: {i}, reward is {ep_reward}, and Explore is {var}\")\r\n",
        "      print('Episode:', i, ' Reward: %i' % int(ep_reward),' Reward Greedy: %i' % int(ep_reward_greedy),' Reward random: %i' % int(ep_reward_random), 'Explore: %.2f' % var )\r\n",
        "      #print(myenv.location)\r\n",
        "      # if ep_reward > -300:RENDER = True\r\n",
        "      break\r\n",
        "  ep_reward = np.reshape(ep_reward/MAX_EP_STEPS, (1,))\r\n",
        "  ep_rewardall.append(ep_reward)\r\n",
        "\r\n",
        "  ep_reward_greedy = np.reshape(ep_reward_greedy/MAX_EP_STEPS, (1,))\r\n",
        "  ep_rewardall_greedy.append(ep_reward_greedy)\r\n",
        "\r\n",
        "  ep_reward_random = np.reshape(ep_reward_random/MAX_EP_STEPS, (1,))\r\n",
        "  ep_rewardall_random.append(ep_reward_random)\r\n",
        "\r\n",
        "#print(s_)\r\n",
        "print('Running time: ', time.time() - t1)\r\n",
        "\r\n",
        "print(f\"{ep_reward}  \")\r\n",
        "print(ep_rewardall)\r\n",
        "plt.plot(ep_rewardall, \"^-\", label='DDPG: rewards')\r\n",
        "plt.plot(ep_rewardall_greedy, \"+:\", label='Greedy: rewards')\r\n",
        "plt.plot(ep_rewardall_random, \"o--\", label='Random: rewards')\r\n",
        "plt.xlabel(\"Episode\")\r\n",
        "plt.ylabel(\" Epsiodic Reward - Average Data Rate (NPCU)\")\r\n",
        "plt.legend( loc=3,  ncol=2)\r\n",
        "plt.show()\r\n",
        "\r\n",
        "''' Save final results'''\r\n",
        "#np.savez_compressed('data_snapshot', ep_rewardall=ep_rewardall, ep_rewardall_greedy=ep_rewardall_greedy, ep_rewardall_random=ep_rewardall_random)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T1nVKaOIRFeb"
      },
      "source": [
        "# Code Test"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hFgiIgQsRIot"
      },
      "source": [
        "# Memory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gcWXLO-PRHhr",
        "outputId": "a96a2707-38be-49aa-b458-be5704bac4a1"
      },
      "source": [
        "MEMORY_CAPACITY = 10001\r\n",
        "BATCH_SIZE = 32\r\n",
        "pointer = 0\r\n",
        "s_dim = 3\r\n",
        "a_dim = 1\r\n",
        "\r\n",
        "def store_transition(s, a, r, s_, pointer):\r\n",
        "  rr = np.reshape(r,(1,1))\r\n",
        "  aa = np.reshape(a,(1,1))\r\n",
        "  ss = np.reshape(s,(1,1))\r\n",
        "  ss_ = np.reshape(s_,(1,1))\r\n",
        "\r\n",
        "  #print(f\"state is {s}, action is {a}, reward is {r}, next state is {s_}\")\r\n",
        "  transition = np.hstack((ss, aa, rr, ss_))\r\n",
        "  index = pointer % MEMORY_CAPACITY  # replace the old memory with new memory\r\n",
        "  memory[index, :] = transition\r\n",
        "  pointer += 1\r\n",
        "  return pointer\r\n",
        "\r\n",
        "memory = np.zeros((MEMORY_CAPACITY, 2 + 1 + 1), dtype=np.float32)\r\n",
        "pointer = 0          # memory of Experience Replay used pointer\r\n",
        "\r\n",
        "for k in range(10000):\r\n",
        "  pointer = store_transition(k, k+1, k+2, k+3, pointer)\r\n",
        "print (\"memory is: \\n{}\\n\".format(memory))\r\n",
        "\r\n",
        "indices = np.random.choice(min(MEMORY_CAPACITY,pointer), size=BATCH_SIZE)\r\n",
        "print (\"all the indices are: \\n{}\\n\".format(indices))\r\n",
        "bt  = memory[indices, :]\r\n",
        "print (\"batch is: \\n{}\\n\".format(bt))\r\n",
        "\r\n",
        "# structure of observation is pre-defined\r\n",
        "bs  = bt[:,       : s_dim]\r\n",
        "ba  = bt[:,  s_dim: s_dim + a_dim]\r\n",
        "br  = bt[:, -s_dim - 1: -s_dim]\r\n",
        "bs_ = bt[:, -s_dim:]\r\n",
        "\r\n",
        "print(\"bs is: \\n{}\\n\".format(bs))\r\n",
        "print(\"ba is: \\n{}\\n\".format(ba))\r\n",
        "print(\"br is: \\n{}\\n\".format(br))\r\n",
        "print(\"bs_ is: \\n{}\\n\".format(bs_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "memory is: \n",
            "[[0.0000e+00 1.0000e+00 2.0000e+00 3.0000e+00]\n",
            " [1.0000e+00 2.0000e+00 3.0000e+00 4.0000e+00]\n",
            " [2.0000e+00 3.0000e+00 4.0000e+00 5.0000e+00]\n",
            " ...\n",
            " [9.9980e+03 9.9990e+03 1.0000e+04 1.0001e+04]\n",
            " [9.9990e+03 1.0000e+04 1.0001e+04 1.0002e+04]\n",
            " [0.0000e+00 0.0000e+00 0.0000e+00 0.0000e+00]]\n",
            "\n",
            "all the indices are: \n",
            "[1591 1924 8180 6783 1046 5470 8989 6245 4031 2453 1022 4325 8867  895\n",
            " 9807 2125 6194 9272 3539 5693 3206 5506 7375 9782 6410 6145 8088 5658\n",
            " 5101 6989 9920 3782]\n",
            "\n",
            "batch is: \n",
            "[[1591. 1592. 1593. 1594.]\n",
            " [1924. 1925. 1926. 1927.]\n",
            " [8180. 8181. 8182. 8183.]\n",
            " [6783. 6784. 6785. 6786.]\n",
            " [1046. 1047. 1048. 1049.]\n",
            " [5470. 5471. 5472. 5473.]\n",
            " [8989. 8990. 8991. 8992.]\n",
            " [6245. 6246. 6247. 6248.]\n",
            " [4031. 4032. 4033. 4034.]\n",
            " [2453. 2454. 2455. 2456.]\n",
            " [1022. 1023. 1024. 1025.]\n",
            " [4325. 4326. 4327. 4328.]\n",
            " [8867. 8868. 8869. 8870.]\n",
            " [ 895.  896.  897.  898.]\n",
            " [9807. 9808. 9809. 9810.]\n",
            " [2125. 2126. 2127. 2128.]\n",
            " [6194. 6195. 6196. 6197.]\n",
            " [9272. 9273. 9274. 9275.]\n",
            " [3539. 3540. 3541. 3542.]\n",
            " [5693. 5694. 5695. 5696.]\n",
            " [3206. 3207. 3208. 3209.]\n",
            " [5506. 5507. 5508. 5509.]\n",
            " [7375. 7376. 7377. 7378.]\n",
            " [9782. 9783. 9784. 9785.]\n",
            " [6410. 6411. 6412. 6413.]\n",
            " [6145. 6146. 6147. 6148.]\n",
            " [8088. 8089. 8090. 8091.]\n",
            " [5658. 5659. 5660. 5661.]\n",
            " [5101. 5102. 5103. 5104.]\n",
            " [6989. 6990. 6991. 6992.]\n",
            " [9920. 9921. 9922. 9923.]\n",
            " [3782. 3783. 3784. 3785.]]\n",
            "\n",
            "bs is: \n",
            "[[1591. 1592. 1593.]\n",
            " [1924. 1925. 1926.]\n",
            " [8180. 8181. 8182.]\n",
            " [6783. 6784. 6785.]\n",
            " [1046. 1047. 1048.]\n",
            " [5470. 5471. 5472.]\n",
            " [8989. 8990. 8991.]\n",
            " [6245. 6246. 6247.]\n",
            " [4031. 4032. 4033.]\n",
            " [2453. 2454. 2455.]\n",
            " [1022. 1023. 1024.]\n",
            " [4325. 4326. 4327.]\n",
            " [8867. 8868. 8869.]\n",
            " [ 895.  896.  897.]\n",
            " [9807. 9808. 9809.]\n",
            " [2125. 2126. 2127.]\n",
            " [6194. 6195. 6196.]\n",
            " [9272. 9273. 9274.]\n",
            " [3539. 3540. 3541.]\n",
            " [5693. 5694. 5695.]\n",
            " [3206. 3207. 3208.]\n",
            " [5506. 5507. 5508.]\n",
            " [7375. 7376. 7377.]\n",
            " [9782. 9783. 9784.]\n",
            " [6410. 6411. 6412.]\n",
            " [6145. 6146. 6147.]\n",
            " [8088. 8089. 8090.]\n",
            " [5658. 5659. 5660.]\n",
            " [5101. 5102. 5103.]\n",
            " [6989. 6990. 6991.]\n",
            " [9920. 9921. 9922.]\n",
            " [3782. 3783. 3784.]]\n",
            "\n",
            "ba is: \n",
            "[[1594.]\n",
            " [1927.]\n",
            " [8183.]\n",
            " [6786.]\n",
            " [1049.]\n",
            " [5473.]\n",
            " [8992.]\n",
            " [6248.]\n",
            " [4034.]\n",
            " [2456.]\n",
            " [1025.]\n",
            " [4328.]\n",
            " [8870.]\n",
            " [ 898.]\n",
            " [9810.]\n",
            " [2128.]\n",
            " [6197.]\n",
            " [9275.]\n",
            " [3542.]\n",
            " [5696.]\n",
            " [3209.]\n",
            " [5509.]\n",
            " [7378.]\n",
            " [9785.]\n",
            " [6413.]\n",
            " [6148.]\n",
            " [8091.]\n",
            " [5661.]\n",
            " [5104.]\n",
            " [6992.]\n",
            " [9923.]\n",
            " [3785.]]\n",
            "\n",
            "br is: \n",
            "[[1591.]\n",
            " [1924.]\n",
            " [8180.]\n",
            " [6783.]\n",
            " [1046.]\n",
            " [5470.]\n",
            " [8989.]\n",
            " [6245.]\n",
            " [4031.]\n",
            " [2453.]\n",
            " [1022.]\n",
            " [4325.]\n",
            " [8867.]\n",
            " [ 895.]\n",
            " [9807.]\n",
            " [2125.]\n",
            " [6194.]\n",
            " [9272.]\n",
            " [3539.]\n",
            " [5693.]\n",
            " [3206.]\n",
            " [5506.]\n",
            " [7375.]\n",
            " [9782.]\n",
            " [6410.]\n",
            " [6145.]\n",
            " [8088.]\n",
            " [5658.]\n",
            " [5101.]\n",
            " [6989.]\n",
            " [9920.]\n",
            " [3782.]]\n",
            "\n",
            "bs_ is: \n",
            "[[1592. 1593. 1594.]\n",
            " [1925. 1926. 1927.]\n",
            " [8181. 8182. 8183.]\n",
            " [6784. 6785. 6786.]\n",
            " [1047. 1048. 1049.]\n",
            " [5471. 5472. 5473.]\n",
            " [8990. 8991. 8992.]\n",
            " [6246. 6247. 6248.]\n",
            " [4032. 4033. 4034.]\n",
            " [2454. 2455. 2456.]\n",
            " [1023. 1024. 1025.]\n",
            " [4326. 4327. 4328.]\n",
            " [8868. 8869. 8870.]\n",
            " [ 896.  897.  898.]\n",
            " [9808. 9809. 9810.]\n",
            " [2126. 2127. 2128.]\n",
            " [6195. 6196. 6197.]\n",
            " [9273. 9274. 9275.]\n",
            " [3540. 3541. 3542.]\n",
            " [5694. 5695. 5696.]\n",
            " [3207. 3208. 3209.]\n",
            " [5507. 5508. 5509.]\n",
            " [7376. 7377. 7378.]\n",
            " [9783. 9784. 9785.]\n",
            " [6411. 6412. 6413.]\n",
            " [6146. 6147. 6148.]\n",
            " [8089. 8090. 8091.]\n",
            " [5659. 5660. 5661.]\n",
            " [5102. 5103. 5104.]\n",
            " [6990. 6991. 6992.]\n",
            " [9921. 9922. 9923.]\n",
            " [3783. 3784. 3785.]]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}