CR_Noma_DRL_Torch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CR-Noma-DRL-Torch.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "5mJ-hxOHWufD"
      },
      "source": [
        "import numpy as np\r\n",
        "from collections import deque\r\n",
        "import random\r\n",
        "\r\n",
        "import torch\r\n",
        "import torch.nn as nn\r\n",
        "import torch.nn.functional as F \r\n",
        "import torch.autograd\r\n",
        "from torch.autograd import Variable\r\n",
        "\r\n",
        "# from enviroment import Env_cellular as env\r\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UKJ__sygYqbJ"
      },
      "source": [
        "import scipy\r\n",
        "from scipy import special\r\n",
        "import numpy as np\r\n",
        "from scipy.special import lambertw\r\n",
        "import math\r\n",
        "\r\n",
        "dtype = np.float32"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3bBQKXNtxVQe"
      },
      "source": [
        "##################################################################################################################\r\n",
        "# Ornstein-Ulhenbeck Process\r\n",
        "# Put some noise into policy to make new exploration outside the right policy (exploitation)\r\n",
        "# Promote exploration -> Gaussian noise is added to the action determined by the policy\r\n",
        "class OUNoise(object):\r\n",
        "  def __init__(self, action_space, mu=0.0, theta=0.15, max_sigma=0.3, min_sigma=0.3, decay_period=100000):\r\n",
        "    self.mu           = mu                      # mu: policy function \r\n",
        "    self.theta        = theta                   # theta: parameters \r\n",
        "    self.sigma        = max_sigma               # initial value of sigma is at maximum value of sigma\r\n",
        "    self.max_sigma    = max_sigma               # \r\n",
        "    self.min_sigma    = min_sigma               # \r\n",
        "    self.decay_period = decay_period            # decay_period:\r\n",
        "    self.action_dim   = action_space.shape[0]   # action_dim: dimension of the action space\r\n",
        "    self.low          = action_space.low        # low: \r\n",
        "    self.high         = action_space.high       # high:\r\n",
        "    self.reset()                                # reset:\r\n",
        "\r\n",
        "  # Reset the state matrix back to matrix of MU <policy>\r\n",
        "  def reset(self):\r\n",
        "    self.state = np.ones(self.action_dim) * self.mu\r\n",
        "\r\n",
        "  # \r\n",
        "  def evolve_state(self):\r\n",
        "    # buffer the state to x\r\n",
        "    x  = self.state\r\n",
        "    dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim)\r\n",
        "\r\n",
        "    # state update\r\n",
        "    self.state = x + dx\r\n",
        "    return self.state\r\n",
        "\r\n",
        "  def get_action(self, action, t=0):\r\n",
        "    ou_state = self.evolve_state() # new state <Ornstein-Uhlenbeck state>\r\n",
        "    self.sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(1.0, t / self.decay_period)\r\n",
        "    return np.clip(action + ou_state, self.low, self.high) \r\n",
        "        \r\n",
        "##################################################################################################################\r\n",
        "# Work similar to what replay buffer 's theory do:\r\n",
        "\r\n",
        "class Memory:\r\n",
        "  def __init__(self, max_size):\r\n",
        "    self.max_size = max_size # max_size of deque is number of experience that allows to be saved in deque\r\n",
        "    self.buffer = deque(maxlen=max_size) # deque||double-end queue has the feature of adding and removing elements from either end || similar to numpy array but easier to work with\r\n",
        "  \r\n",
        "  # push new Experience into buffer // push old Experience out of buffer\r\n",
        "  def push(self, state, action, reward, next_state, done): \r\n",
        "    experience = (state, action, np.array([reward]), next_state, done) # why reward is in array \r\n",
        "    self.buffer.append(experience) # append new experience//observation into deque. If the data reach the buffer 's capacity limit => delete oldest data\r\n",
        "\r\n",
        "  def sample(self, batch_size):\r\n",
        "    state_batch = []\r\n",
        "    action_batch = []\r\n",
        "    reward_batch = []\r\n",
        "    next_state_batch = []\r\n",
        "    done_batch = []\r\n",
        "\r\n",
        "    batch = random.sample(self.buffer, batch_size)  # pick random batch (a packet of random data with size = batch_size // from buffer)\r\n",
        "\r\n",
        "    for experience in batch:  # loops over all group of [state, action, reward, next_state, done] in batch\r\n",
        "      state, action, reward, next_state, done = experience\r\n",
        "      # append all [state, action, reward, next_state, done] into batch // split into separated features\r\n",
        "      state_batch.append(state)\r\n",
        "      action_batch.append(action)\r\n",
        "      reward_batch.append(reward)\r\n",
        "      next_state_batch.append(next_state)\r\n",
        "      done_batch.append(done)\r\n",
        "  \r\n",
        "    return state_batch, action_batch, reward_batch, next_state_batch, done_batch\r\n",
        "\r\n",
        "  def __len__(self):\r\n",
        "    return len(self.buffer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w37AmwV82-3-"
      },
      "source": [
        "class Critic(nn.Module):\r\n",
        "  def __init__(self, input_size, hidden_size1, hidden_size2, output_size):\r\n",
        "    super(Critic, self).__init__()\r\n",
        "    self.linear1 = nn.Linear(input_size, hidden_size1)     # dense 64 output (hidden_size1)\r\n",
        "    self.linear2 = nn.Linear(hidden_size1, hidden_size2)   # dense 30 ouput  (hidden_size2)\r\n",
        "    self.linear3 = nn.Linear(hidden_size2, output_size)    # dense 1 output  (output_size)\r\n",
        "\r\n",
        "  def forward(self, state, action):\r\n",
        "    \"\"\"\r\n",
        "    Params state and actions are torch tensors\r\n",
        "    \"\"\"\r\n",
        "    x = torch.cat([state, action], 1)\r\n",
        "    x = F.relu(self.linear1(x))\r\n",
        "    x = F.relu(self.linear2(x))\r\n",
        "    x = self.linear3(x)          # difference between actor and critic\r\n",
        "\r\n",
        "    return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MxEtJ7T63CWN"
      },
      "source": [
        "class Actor(nn.Module):\r\n",
        "  def __init__(self, input_size, hidden_size1 = 64, hidden_size2 = 64, output_size):\r\n",
        "    super(Actor, self).__init__() #  lets you avoid referring to the base class explicitly\r\n",
        "    # (size-of-each-input-sample, size-of-each-output-sample, bias =0 )\r\n",
        "    self.linear1 = nn.Linear(input_size, hidden_size)    # dense 64 output \r\n",
        "    # Just do linear transformation, not classification\r\n",
        "    self.linear2 = nn.Linear(hidden_size, hidden_size)   # dense 64 output\r\n",
        "    self.linear3 = nn.Linear(hidden_size, output_size)   # dense action_dimension output\r\n",
        "      \r\n",
        "  def forward(self, state):\r\n",
        "    \"\"\"\r\n",
        "    Param state is a torch tensor\r\n",
        "    \"\"\"\r\n",
        "    x = F.relu(self.linear1(state))\r\n",
        "    x = torch.tanh(self.linear2(x))\r\n",
        "    x = torch.tanh(self.linear3(x))\r\n",
        "\r\n",
        "    return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9OJWkrDr3HZE"
      },
      "source": [
        "INIT = 0\r\n",
        "\r\n",
        "#####################  hyper parameters  ####################\r\n",
        "MAX_EP_STEPS = 100\r\n",
        "LR_A = 0.002    # learning rate for actor\r\n",
        "LR_C = 0.004    # learning rate for critic\r\n",
        "GAMMA = 0.9     # reward discount\r\n",
        "TAU = 0.01      # soft replacement\r\n",
        "MEMORY_CAPACITY = 10000\r\n",
        "BATCH_SIZE = 32\r\n",
        "\r\n",
        "#####################  INIT  ####################\r\n",
        "INIT = 0\r\n",
        "#### INIT:\r\n",
        "# RANDOM LOCATION WITH FADING K: 0\r\n",
        "# TWO USER CASE                : 1\r\n",
        "# SPECIAL CASE                 : 2\r\n",
        "if INIT == 0:         # RANDOM LOCATION WITH FADING K #\r\n",
        "  ct=500              # \r\n",
        "  MAX_EPISODES =10    #\r\n",
        "\r\n",
        "elif INIT == 1:    # TWO USER CASE #\r\n",
        "  Pn = 1              #\r\n",
        "  K=2                 #\r\n",
        "  MAX_EPISODES = 400  #\r\n",
        "  MAX_EP_STEPS = 100  #\r\n",
        "else:                 # SPECIAL CASE #\r\n",
        "  Pn = 1\r\n",
        "  K=10 # the number of grant based users\r\n",
        "  MAX_EPISODES = 400\r\n",
        "\r\n",
        "# env:\r\n",
        "# hidden_size\r\n",
        "# actor_learning_rate:   || critic_learning_rate\r\n",
        "# gamma: discount factor\r\n",
        "# tau: target network update parameters\r\n",
        "# max_memory_size: \r\n",
        "class DDPG:\r\n",
        "  def __init__(self, a_dim, s_dim, a_bound, hidden_size1=64, hidden_size2=30, \r\n",
        "               actor_learning_rate=LR_A, critic_learning_rate=LR_C, \r\n",
        "               gamma=GAMMA, tau=TAU, max_memory_size=MEMORY_CAPACITY):\r\n",
        "    # Params\r\n",
        "    self.num_states = s_dim\r\n",
        "    self.num_actions = a_dim\r\n",
        "    self.gamma = gamma\r\n",
        "    self.tau = tau\r\n",
        "\r\n",
        "    # Network layer definition\r\n",
        "    a_hidden_layer_1_size = hidden_size1  \r\n",
        "    a_hidden_layer_2_size = hidden_size1\r\n",
        "    c_hidden_layer_1_size = hidden_size1\r\n",
        "    c_hidden_layer_2_size = hidden_size2\r\n",
        "\r\n",
        "    # Networks\r\n",
        "    self.actor = Actor(self.num_states, a_hidden_layer_1_size,                             # input_size, hidden_size1\r\n",
        "                       a_hidden_layer_2_size, self.num_actions)                            # hidden_size2, output_size\r\n",
        "    self.actor_target = Actor(self.num_states, a_hidden_layer_1_size,                      # input_size, hidden_size1\r\n",
        "                              a_hidden_layer_2_size, self.num_actions)                     # hidden_size2, output_size\r\n",
        "\r\n",
        "    self.critic = Critic(self.num_states + self.num_actions, c_hidden_layer_1_size,        # input_size, hidden_size1\r\n",
        "                         c_hidden_layer_2_size, self.num_actions)                          # hidden_size2, output_size\r\n",
        "    self.critic_target = Critic(self.num_states + self.num_actions, c_hidden_layer_1_size, # input_size, hidden_size1\r\n",
        "                         c_hidden_layer_2_size, self.num_actions)                          # hidden_size2, output_size\r\n",
        "\r\n",
        "    # copy all parameters from actor network -> actor target network\r\n",
        "    for target_param, param in zip(self.actor_target.parameters(), self.actor.parameters()):\r\n",
        "      target_param.data.copy_(param.data)\r\n",
        "\r\n",
        "    # copy all parameters from critic network -> critic target network\r\n",
        "    for target_param, param in zip(self.critic_target.parameters(), self.critic.parameters()):\r\n",
        "      target_param.data.copy_(param.data)\r\n",
        "    \r\n",
        "    # Training\r\n",
        "    self.memory = Memory(max_memory_size)        \r\n",
        "    self.critic_criterion  = nn.MSELoss()\r\n",
        "    self.actor_optimizer  = optim.Adam(self.actor.parameters(), lr=actor_learning_rate)\r\n",
        "    self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_learning_rate)\r\n",
        "\r\n",
        "  def get_action(self, state):\r\n",
        "    state = Variable(torch.from_numpy(state).float().unsqueeze(0))\r\n",
        "    action = self.actor.forward(state)\r\n",
        "    action = action.detach().numpy()[0,0]\r\n",
        "    return action\r\n",
        "  \r\n",
        "  def update(self, batch_size):\r\n",
        "    states, actions, rewards, next_states, _ = self.memory.sample(batch_size)\r\n",
        "    states = torch.FloatTensor(states)\r\n",
        "    actions = torch.FloatTensor(actions)\r\n",
        "    rewards = torch.FloatTensor(rewards)\r\n",
        "    next_states = torch.FloatTensor(next_states)\r\n",
        "\r\n",
        "    # Critic loss    \r\n",
        "    # Critic depends on tuple[Q_target, mu_target, mu_main] \r\n",
        "    # mu_target = mu_main + delay + [noise from updating part of old mu]    \r\n",
        "    # Q_target  = Q_main  + delay + [noise from updating part of old Q]\r\n",
        "    Qvals = self.critic.forward(states, actions)\r\n",
        "    next_actions = self.actor_target.forward(next_states)\r\n",
        "    next_Q = self.critic_target.forward(next_states, next_actions.detach())\r\n",
        "    Qprime = rewards + self.gamma * next_Q\r\n",
        "    critic_loss = self.critic_criterion(Qvals, Qprime)\r\n",
        "    \r\n",
        "    \r\n",
        "    # Actor loss\r\n",
        "    # Simply the average sum of Q(s_i, mu(s_i))\r\n",
        "    policy_loss = -self.critic.forward(states, self.actor.forward(states)).mean()\r\n",
        "    \r\n",
        "    # update networks\r\n",
        "    self.actor_optimizer.zero_grad()\r\n",
        "    policy_loss.backward()\r\n",
        "    self.actor_optimizer.step()\r\n",
        "\r\n",
        "    self.critic_optimizer.zero_grad()\r\n",
        "    critic_loss.backward() \r\n",
        "    self.critic_optimizer.step()\r\n",
        "\r\n",
        "    # update target networks \r\n",
        "    for target_param, param in zip(self.actor_target.parameters(), self.actor.parameters()):\r\n",
        "      target_param.data.copy_(param.data * self.tau + target_param.data * (1.0 - self.tau))\r\n",
        "    \r\n",
        "    for target_param, param in zip(self.critic_target.parameters(), self.critic.parameters()):\r\n",
        "      target_param.data.copy_(param.data * self.tau + target_param.data * (1.0 - self.tau))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MIga8WbDLLVE"
      },
      "source": [
        "###############################  training  ####################################\r\n",
        "s_dim = 3# dimsion of states\r\n",
        "a_dim = 1# dimension of action\r\n",
        "a_bound = 1 #bound of action\r\n",
        "state_am = 10000\r\n",
        "\r\n",
        "\r\n",
        "#GB_power_dBm = np.array([0,  10,  20,   30])\r\n",
        "K_range = np.array([2,4,6,8, 10])\r\n",
        "rate_DDPG = []\r\n",
        "rate_greedy = []\r\n",
        "rate_random = []\r\n",
        "for k in range(len(K_range)):\r\n",
        "  K = K_range[k]\r\n",
        "  ratect_DDPG = 0\r\n",
        "  ratect_greedy = 0\r\n",
        "  ratect_random = 0\r\n",
        "  for mct in range(ct):\r\n",
        "    var = 1  # control exploration\r\n",
        "    agent = DDPG(a_dim, s_dim, a_bound,                                  # necessary parameters\r\n",
        "                hidden_size1=64, hidden_size2=30, \r\n",
        "                actor_learning_rate=LR_A, critic_learning_rate=LR_C, \r\n",
        "                gamma=GAMMA, tau=TAU, max_memory_size=MEMORY_CAPACITY)\r\n",
        "\r\n",
        "    locationspace = np.linspace(1, 1000, num=K)\r\n",
        "    location_vector = np.zeros((K, 2))\r\n",
        "    location_vector[:, 1] = locationspace\r\n",
        "    location_GF = np.array([[1,1]])# np.ones((1, 2))\r\n",
        "\r\n",
        "    ##### fading for GB user\r\n",
        "    hnx1 = np.random.randn(K, 2)\r\n",
        "    hnx2 = np.random.randn(K, 2)\r\n",
        "    fading_n = hnx1 ** 2 + hnx2 ** 2\r\n",
        "    #### fading for GF user\r\n",
        "    h0x1 = np.random.randn(1, 1)\r\n",
        "    h0x2 = np.random.randn(1, 1)\r\n",
        "    fading_0 =  h0x1[0,0] ** 2 + h0x2[0,0] ** 2\r\n",
        "\r\n",
        "    Pn = 10**((30-30)/10)\r\n",
        "    myenv = Env_cellular( MAX_EP_STEPS, s_dim, location_vector, location_GF, K,Pn, fading_n, fading_0)\r\n",
        "    noise = OUNoise(env.action_space)\r\n",
        "\r\n",
        "    ratek_DDPG = 0\r\n",
        "    ratek_greedy = 0\r\n",
        "    ratek_random = 0\r\n",
        "    for i in range(MAX_EPISODES):\r\n",
        "      batter_ini = myenv.reset()\r\n",
        "      state = myenv.channel_sequence[i % myenv.K, :].tolist()  # the current GB user, 2 element [GB-GF, GB-BS]\r\n",
        "      # s.append(myenv.h0)\r\n",
        "      state.append(batter_ini)\r\n",
        "      state = np.reshape(s, (1, s_dim))\r\n",
        "      state = state * state_am  # amplify the state\r\n",
        "      state_greedy = state\r\n",
        "      state_random = state\r\n",
        "      reward_dpgg_vector =[]\r\n",
        "      reward_greedy_vector = []\r\n",
        "      reward_random_vector = []\r\n",
        "      #print(s[0,0:2])\r\n",
        "      for step in range(MAX_EP_STEPS):\r\n",
        "\r\n",
        "        # Add exploration noise\r\n",
        "        a = agent.get_action(state)\r\n",
        "        action = noise.get_action(action, t=step, )\r\n",
        "        #print(myenv.location)\r\n",
        "        reward, next_state, done = myenv.step(a, state / state_am, step)\r\n",
        "        next_state = next_state * state_am\r\n",
        "        #print(f\"{step} state is {s}, action is {a}, reward is {r}, next state is {s_}\")\r\n",
        "        agent.memory.push(state, action, reward, new_state, done)\r\n",
        "        if var>0.05:\r\n",
        "          var *= .9998  # decay the action randomness\r\n",
        "        if len(agent.memory) > BATCH_SIZE:\r\n",
        "          agent.update(BATCH_SIZE)  \r\n",
        "        state = next_state\r\n",
        "        reward_dpgg_vector.append(r)\r\n",
        "\r\n",
        "        ##### greedy\r\n",
        "        reward_greedy, state_next_greedy, done = myenv.step_greedy(state_greedy/state_am, step)\r\n",
        "        state_greedy = state_next_greedy*state_am\r\n",
        "        reward_greedy_vector.append(reward_greedy)\r\n",
        "\r\n",
        "        ##### random\r\n",
        "        reward_random, state_next_random, done = myenv.step_random(state_random/state_am, step)\r\n",
        "        state_random = state_next_random*state_am\r\n",
        "        reward_random_vector.append(reward_random)\r\n",
        "\r\n",
        "        if step == MAX_EP_STEPS-1:\r\n",
        "          #print(f\"Iteration., {k}--{mct},  Episode:, {i},  Reward: {sum(reward_dpgg_vector)/MAX_EP_STEPS},\")\r\n",
        "          #      f\" Reward Greedy:   {sum(reward_greedy_vector)/MAX_EP_STEPS},\"\r\n",
        "          #      f\" Reward random:   { sum(reward_random_vector)/MAX_EP_STEPS}, Explore: {var} \")\r\n",
        "          break\r\n",
        "        ratek_DDPG = sum(reward_dpgg_vector)/MAX_EP_STEPS\r\n",
        "        ratek_greedy = sum(reward_greedy_vector)/MAX_EP_STEPS\r\n",
        "        ratek_random = sum(reward_random_vector)/MAX_EP_STEPS\r\n",
        "\r\n",
        "    tf.keras.backend.clear_session()\r\n",
        "    ratect_DDPG +=ratek_DDPG\r\n",
        "    ratect_greedy +=ratek_greedy\r\n",
        "    ratect_random += ratek_random\r\n",
        "    print(ratek_DDPG)\r\n",
        "  print(f\"Iteration., {k}--{mct},  Episode:, {i},  Reward: {ratect_DDPG/ct },\")\r\n",
        "  rate_DDPG.append(ratect_DDPG/ct )\r\n",
        "  rate_greedy.append(ratect_greedy /ct )\r\n",
        "  rate_random.append(ratect_random  /ct)\r\n",
        "\r\n",
        "print(f\"rate_greedy is {rate_greedy} and rate_random is {rate_random}\")\r\n",
        "#GB_power_label = ['0',  '10',  '20', '30']\r\n",
        "GB_power_label = ['0', '5', '10', '15', '20', '25', '30']\r\n",
        "K_rangex =  ['2', '4', '6', '8', '10']\r\n",
        "plt.plot(K_rangex,rate_DDPG, \"^-\", label='DDPG: rewards')\r\n",
        "plt.plot(K_rangex,rate_greedy, \"+:\", label='Greedy: rewards')\r\n",
        "plt.plot(K_rangex,rate_random, \"o--\", label='Random: rewards')\r\n",
        "plt.xlabel(\"The Number of Primary Users (K)\")\r\n",
        "plt.ylabel(\"  Average  Data Rate (NPCU)\")\r\n",
        "plt.legend()\r\n",
        "plt.show()\r\n",
        "\r\n",
        "\r\n",
        "''' Save final results'''\r\n",
        "#np.savez_compressed('data/dataK', rate_DDPG=rate_DDPG, rate_greedy=rate_greedy, rate_random=rate_random)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w_DAPnn_ZpQR"
      },
      "source": [
        "# Code Test "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9tuewWm2FvQA",
        "outputId": "396b5aa5-a949-4b85-d5d5-6c872ab911b9"
      },
      "source": [
        "MEMORY_CAPACITY = 10001\r\n",
        "BATCH_SIZE = 32\r\n",
        "pointer = 0\r\n",
        "s_dim = 3\r\n",
        "a_dim = 1\r\n",
        "\r\n",
        "def store_transition(s, a, r, s_, pointer):\r\n",
        "  rr = np.reshape(r,(1,1))\r\n",
        "  aa = np.reshape(a,(1,1))\r\n",
        "  ss = np.reshape(s,(1,1))\r\n",
        "  ss_ = np.reshape(s_,(1,1))\r\n",
        "\r\n",
        "  #print(f\"state is {s}, action is {a}, reward is {r}, next state is {s_}\")\r\n",
        "  transition = np.hstack((ss, aa, rr, ss_))\r\n",
        "  index = pointer % MEMORY_CAPACITY  # replace the old memory with new memory\r\n",
        "  memory[index, :] = transition\r\n",
        "  pointer += 1\r\n",
        "  return pointer\r\n",
        "\r\n",
        "memory = np.zeros((MEMORY_CAPACITY, 2 + 1 + 1), dtype=np.float32)\r\n",
        "pointer = 0          # memory of Experience Replay used pointer\r\n",
        "\r\n",
        "for k in range(10000):\r\n",
        "  pointer = store_transition(k, k+1, k+2, k+3, pointer)\r\n",
        "print (\"memory is: \\n{}\\n\".format(memory))\r\n",
        "\r\n",
        "indices = np.random.choice(min(MEMORY_CAPACITY,pointer), size=BATCH_SIZE)\r\n",
        "print (\"all the indices are: \\n{}\\n\".format(indices))\r\n",
        "bt  = memory[indices, :]\r\n",
        "print (\"batch is: \\n{}\\n\".format(bt))\r\n",
        "\r\n",
        "# structure of observation is pre-defined\r\n",
        "bs  = bt[:,       : s_dim]\r\n",
        "ba  = bt[:,  s_dim: s_dim + a_dim]\r\n",
        "br  = bt[:, -s_dim - 1: -s_dim]\r\n",
        "bs_ = bt[:, -s_dim:]\r\n",
        "\r\n",
        "print(\"bs is: \\n{}\\n\".format(bs))\r\n",
        "print(\"ba is: \\n{}\\n\".format(ba))\r\n",
        "print(\"br is: \\n{}\\n\".format(br))\r\n",
        "print(\"bs_ is: \\n{}\\n\".format(bs_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "memory is: \n",
            "[[0.0000e+00 1.0000e+00 2.0000e+00 3.0000e+00]\n",
            " [1.0000e+00 2.0000e+00 3.0000e+00 4.0000e+00]\n",
            " [2.0000e+00 3.0000e+00 4.0000e+00 5.0000e+00]\n",
            " ...\n",
            " [9.9980e+03 9.9990e+03 1.0000e+04 1.0001e+04]\n",
            " [9.9990e+03 1.0000e+04 1.0001e+04 1.0002e+04]\n",
            " [0.0000e+00 0.0000e+00 0.0000e+00 0.0000e+00]]\n",
            "\n",
            "all the indices are: \n",
            "[7568 7011 2873 8428 9115 3907 5605 4596  900 2083 5473 5751 8237 5282\n",
            " 9476 9935 5968 4306 1952 2803 2394 3701 4238 5539 1323 4488 8601 9796\n",
            " 2011 8414 8638 5534]\n",
            "\n",
            "batch is: \n",
            "[[7568. 7569. 7570. 7571.]\n",
            " [7011. 7012. 7013. 7014.]\n",
            " [2873. 2874. 2875. 2876.]\n",
            " [8428. 8429. 8430. 8431.]\n",
            " [9115. 9116. 9117. 9118.]\n",
            " [3907. 3908. 3909. 3910.]\n",
            " [5605. 5606. 5607. 5608.]\n",
            " [4596. 4597. 4598. 4599.]\n",
            " [ 900.  901.  902.  903.]\n",
            " [2083. 2084. 2085. 2086.]\n",
            " [5473. 5474. 5475. 5476.]\n",
            " [5751. 5752. 5753. 5754.]\n",
            " [8237. 8238. 8239. 8240.]\n",
            " [5282. 5283. 5284. 5285.]\n",
            " [9476. 9477. 9478. 9479.]\n",
            " [9935. 9936. 9937. 9938.]\n",
            " [5968. 5969. 5970. 5971.]\n",
            " [4306. 4307. 4308. 4309.]\n",
            " [1952. 1953. 1954. 1955.]\n",
            " [2803. 2804. 2805. 2806.]\n",
            " [2394. 2395. 2396. 2397.]\n",
            " [3701. 3702. 3703. 3704.]\n",
            " [4238. 4239. 4240. 4241.]\n",
            " [5539. 5540. 5541. 5542.]\n",
            " [1323. 1324. 1325. 1326.]\n",
            " [4488. 4489. 4490. 4491.]\n",
            " [8601. 8602. 8603. 8604.]\n",
            " [9796. 9797. 9798. 9799.]\n",
            " [2011. 2012. 2013. 2014.]\n",
            " [8414. 8415. 8416. 8417.]\n",
            " [8638. 8639. 8640. 8641.]\n",
            " [5534. 5535. 5536. 5537.]]\n",
            "\n",
            "bs is: \n",
            "[[7568.]\n",
            " [7011.]\n",
            " [2873.]\n",
            " [8428.]\n",
            " [9115.]\n",
            " [3907.]\n",
            " [5605.]\n",
            " [4596.]\n",
            " [ 900.]\n",
            " [2083.]\n",
            " [5473.]\n",
            " [5751.]\n",
            " [8237.]\n",
            " [5282.]\n",
            " [9476.]\n",
            " [9935.]\n",
            " [5968.]\n",
            " [4306.]\n",
            " [1952.]\n",
            " [2803.]\n",
            " [2394.]\n",
            " [3701.]\n",
            " [4238.]\n",
            " [5539.]\n",
            " [1323.]\n",
            " [4488.]\n",
            " [8601.]\n",
            " [9796.]\n",
            " [2011.]\n",
            " [8414.]\n",
            " [8638.]\n",
            " [5534.]]\n",
            "\n",
            "ba is: \n",
            "[[7569.]\n",
            " [7012.]\n",
            " [2874.]\n",
            " [8429.]\n",
            " [9116.]\n",
            " [3908.]\n",
            " [5606.]\n",
            " [4597.]\n",
            " [ 901.]\n",
            " [2084.]\n",
            " [5474.]\n",
            " [5752.]\n",
            " [8238.]\n",
            " [5283.]\n",
            " [9477.]\n",
            " [9936.]\n",
            " [5969.]\n",
            " [4307.]\n",
            " [1953.]\n",
            " [2804.]\n",
            " [2395.]\n",
            " [3702.]\n",
            " [4239.]\n",
            " [5540.]\n",
            " [1324.]\n",
            " [4489.]\n",
            " [8602.]\n",
            " [9797.]\n",
            " [2012.]\n",
            " [8415.]\n",
            " [8639.]\n",
            " [5535.]]\n",
            "\n",
            "br is: \n",
            "[[7570.]\n",
            " [7013.]\n",
            " [2875.]\n",
            " [8430.]\n",
            " [9117.]\n",
            " [3909.]\n",
            " [5607.]\n",
            " [4598.]\n",
            " [ 902.]\n",
            " [2085.]\n",
            " [5475.]\n",
            " [5753.]\n",
            " [8239.]\n",
            " [5284.]\n",
            " [9478.]\n",
            " [9937.]\n",
            " [5970.]\n",
            " [4308.]\n",
            " [1954.]\n",
            " [2805.]\n",
            " [2396.]\n",
            " [3703.]\n",
            " [4240.]\n",
            " [5541.]\n",
            " [1325.]\n",
            " [4490.]\n",
            " [8603.]\n",
            " [9798.]\n",
            " [2013.]\n",
            " [8416.]\n",
            " [8640.]\n",
            " [5536.]]\n",
            "\n",
            "bs_ is: \n",
            "[[7571.]\n",
            " [7014.]\n",
            " [2876.]\n",
            " [8431.]\n",
            " [9118.]\n",
            " [3910.]\n",
            " [5608.]\n",
            " [4599.]\n",
            " [ 903.]\n",
            " [2086.]\n",
            " [5476.]\n",
            " [5754.]\n",
            " [8240.]\n",
            " [5285.]\n",
            " [9479.]\n",
            " [9938.]\n",
            " [5971.]\n",
            " [4309.]\n",
            " [1955.]\n",
            " [2806.]\n",
            " [2397.]\n",
            " [3704.]\n",
            " [4241.]\n",
            " [5542.]\n",
            " [1326.]\n",
            " [4491.]\n",
            " [8604.]\n",
            " [9799.]\n",
            " [2014.]\n",
            " [8417.]\n",
            " [8641.]\n",
            " [5537.]]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B9Bcjq-3VwW7",
        "outputId": "abd42d2e-2dcf-42f1-a260-14b227e076ad"
      },
      "source": [
        "MAX_EP_STEPS = 100\r\n",
        "K = 5\r\n",
        "\r\n",
        "locationspace = np.linspace(1, 1000, num=K) \r\n",
        "# define a matrix of client-locations (each client includes 2 parts)\r\n",
        "location_vector = np.zeros((K, 2))\r\n",
        "# 2nd column definition\r\n",
        "location_vector[:, 1] = locationspace\r\n",
        "location_GF = np.array([[1,1]])# np.ones((1, 2))\r\n",
        "\r\n",
        "print(\"locationspace:\\n{}\\n\".format(locationspace))\r\n",
        "print(\"location_vector:\\n{}\\n\".format(location_vector))\r\n",
        "print(\"location_GF:\\n{}\\n\".format(location_GF))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "locationspace:\n",
            "[   1.    250.75  500.5   750.25 1000.  ]\n",
            "\n",
            "location_vector:\n",
            "[[   0.      1.  ]\n",
            " [   0.    250.75]\n",
            " [   0.    500.5 ]\n",
            " [   0.    750.25]\n",
            " [   0.   1000.  ]]\n",
            "\n",
            "location_GF:\n",
            "[[1 1]]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z1HvDaLXdIU_",
        "outputId": "89fc856f-8544-494a-e12d-ad0b20aac4ec"
      },
      "source": [
        "##### fading for GB (grant-based) user\r\n",
        "hnx1 = np.random.randn(K, 2)      # channel gain\r\n",
        "hnx2 = np.random.randn(K, 2)      # channel gain\r\n",
        "fading_n = hnx1 ** 2 + hnx2 ** 2\r\n",
        "#### fading for GF (grant-free) user\r\n",
        "h0x1 = np.random.randn(1, 1)      # channel gain \r\n",
        "h0x2 = np.random.randn(1, 1)      # channel gain\r\n",
        "fading_0 =  h0x1[0,0] ** 2 + h0x2[0,0] ** 2\r\n",
        "\r\n",
        "Pn = 10**((30-30)/10)\r\n",
        "\r\n",
        "print(\"hnx1: \\n{}\\n\".format(hnx1))\r\n",
        "print(\"hnx2: \\n{}\\n\".format(hnx2))\r\n",
        "print(\"fading_n: \\n{}\\n\".format(fading_n))\r\n",
        "print(\"h0x1: \\n{}\\n\".format(h0x1))\r\n",
        "print(\"h0x2: \\n{}\\n\".format(h0x2))\r\n",
        "print(\"fading_0: \\n{}\\n\".format(fading_0))\r\n",
        "print(\"Pn: \\n{}\\n\".format(Pn))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "hnx1: \n",
            "[[-0.7025477  -0.76816182]\n",
            " [ 0.51816404  1.27695805]\n",
            " [ 1.36956635  0.2337245 ]\n",
            " [ 0.0316882  -0.81842374]\n",
            " [-1.1495627  -2.03515967]]\n",
            "\n",
            "hnx2: \n",
            "[[-1.14285002  0.89788275]\n",
            " [-0.13430783 -0.06191845]\n",
            " [-0.38909785  1.52578619]\n",
            " [-1.52485374  1.78663988]\n",
            " [-1.97539743 -1.30185783]]\n",
            "\n",
            "fading_n: \n",
            "[[1.79967945 1.39626602]\n",
            " [0.28653256 1.63445576]\n",
            " [2.02710913 2.38265063]\n",
            " [2.32618307 3.86189949]\n",
            " [5.2236894  5.8367087 ]]\n",
            "\n",
            "h0x1: \n",
            "[[0.00397162]]\n",
            "\n",
            "h0x2: \n",
            "[[0.05871606]]\n",
            "\n",
            "fading_0: \n",
            "0.003463349416688293\n",
            "\n",
            "Pn: \n",
            "1.0\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sutAvGZMu275",
        "outputId": "6aa0837e-e6fe-4f09-afa9-8a85dd55aec5"
      },
      "source": [
        "s_dim = 3 # dimsion of states\r\n",
        "a_dim = 1 # dimension of action\r\n",
        "\r\n",
        "################################################################################\r\n",
        "# Noise calculator\r\n",
        "BW = 10**6 # 10MHz\r\n",
        "# -170 dBm/Hz: AWGN \r\n",
        "# -\r\n",
        "sigma2_dbm = -170 + 10 * np.log10(BW) #  Thermal noise in dBm \r\n",
        "#sigma2_dbm = -94\r\n",
        "noise = 10 ** ((sigma2_dbm - 30) / 10)\r\n",
        "# self.Pn = self.Pn\r\n",
        "################################################################################ \r\n",
        "\r\n",
        "#self.P0n = 10 #grant free user's power #transmit power of U0 at tn\r\n",
        "\r\n",
        "#defines whatever you want =)))\r\n",
        "\r\n",
        "# location_vector / location_GF: coordination of base station and secondary user\r\n",
        "\r\n",
        "# separate distance between transmitter-receiver\r\n",
        "# GF is reference coordinator\r\n",
        "distance_GF = np.sqrt(np.sum((location_vector-location_GF)**2, axis=1))\r\n",
        "\r\n",
        "# \r\n",
        "distance_GB = np.sqrt(np.sum((location_vector)**2, axis=1))\r\n",
        "\r\n",
        "\r\n",
        "# path loss\r\n",
        "# merge distance_GF and distance_GB into 1 matrix ( )\r\n",
        "distance = np.matrix.transpose(np.array([distance_GF, distance_GB]))   # Kx2 matrix\r\n",
        "distance = np.maximum(distance, np.ones((K,2))) # Less than one -> equal to one (normalize small value)\r\n",
        "PL_alpha = 3\r\n",
        "PL = 1/distance**PL_alpha/(10**3.17)\r\n",
        "\r\n",
        "distance_GF0 = np.sqrt(np.sum( location_GF ** 2, axis=1))\r\n",
        "distance0 = np.maximum(distance_GF0, 1)\r\n",
        "\r\n",
        "\r\n",
        "PL0 = 1/(distance0 ** PL_alpha)/(10**3.17)\r\n",
        "\r\n",
        "# channel = fading * pathloss ? \r\n",
        "hn = np.multiply( PL, fading_n)   \r\n",
        "h0 = fading_0*PL0\r\n",
        "#print(f\"{hn} and {h0}\")\r\n",
        "\r\n",
        "\r\n",
        "channel_sequence = np.zeros((MAX_EP_STEPS,2))\r\n",
        "for i in range(MAX_EP_STEPS):\r\n",
        "    id_index = i % K\r\n",
        "    # print(\"id_index: {} \\n\".format(id_index))\r\n",
        "    channel_sequence[i,:] = hn[id_index,:]\r\n",
        "\r\n",
        "print(\"distance_GF\\n{}\\n\".format(distance_GF))\r\n",
        "print(\"distance_GF0\\n{}\\n\".format(distance_GF0))\r\n",
        "print(\"distance_GB\\n{}\\n\".format(distance_GB))\r\n",
        "print(\"distance\\n{}\\n\".format(distance))\r\n",
        "print(\"distance0\\n{}\\n\".format(distance0))\r\n",
        "print(\"PL0\\n{}\\n\".format(PL0))                         \r\n",
        "print(\"h0\\n{}\\n\".format(h0))                           \r\n",
        "                                                       \r\n",
        "print(\"channel_sequence\\n{}\\n\".format(channel_sequence))\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "distance_GF\n",
            "[  1.         249.75200199 499.501001   749.25066733 999.0005005 ]\n",
            "\n",
            "distance_GF0\n",
            "[1.41421356]\n",
            "\n",
            "distance_GB\n",
            "[   1.    250.75  500.5   750.25 1000.  ]\n",
            "\n",
            "distance\n",
            "[[   1.            1.        ]\n",
            " [ 249.75200199  250.75      ]\n",
            " [ 499.501001    500.5       ]\n",
            " [ 749.25066733  750.25      ]\n",
            " [ 999.0005005  1000.        ]]\n",
            "\n",
            "distance0\n",
            "[1.41421356]\n",
            "\n",
            "PL0\n",
            "[0.00023903]\n",
            "\n",
            "h0\n",
            "[8.27849358e-07]\n",
            "\n",
            "channel_sequence\n",
            "[[1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]\n",
            " [1.21673264e-03 9.43991683e-04]\n",
            " [1.24350361e-11 7.00890777e-11]\n",
            " [1.09968434e-11 1.28483726e-11]\n",
            " [3.73906036e-12 6.18276805e-12]\n",
            " [3.54225831e-12 3.94609938e-12]]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}