L.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2020,
     "status": "ok",
     "timestamp": 1572770690617,
     "user": {
      "displayName": "Hale Damirchi",
      "photoUrl": "https://lh5.googleusercontent.com/-w2toi7wPDL8/AAAAAAAAAAI/AAAAAAAAAUw/U8_SysYvTY4/s64/photo.jpg",
      "userId": "00567801216308932433"
     },
     "user_tz": -210
    },
    "id": "1fpLNXcNlRLL",
    "outputId": "d118d3ff-a18a-49aa-d113-e5bce1fc2b6d",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "imported\n"
     ]
    }
   ],
   "source": [
    "# import libraries.\n",
    "import torch\n",
    "import numpy as np\n",
    "import datetime\n",
    "# tf.enable_eager_execution()\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data.sampler import Sampler\n",
    "from torch.autograd import Variable\n",
    "from torch.optim.lr_scheduler import MultiplicativeLR, StepLR, LambdaLR\n",
    "import h5py\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import torch.optim as optim\n",
    "from natsort import natsorted\n",
    "print('imported')\n",
    "# #######################\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Caution! change foldername before execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1042,
     "status": "ok",
     "timestamp": 1572770696598,
     "user": {
      "displayName": "Hale Damirchi",
      "photoUrl": "https://lh5.googleusercontent.com/-w2toi7wPDL8/AAAAAAAAAAI/AAAAAAAAAUw/U8_SysYvTY4/s64/photo.jpg",
      "userId": "00567801216308932433"
     },
     "user_tz": -210
    },
    "id": "K0kyLLwfqmZJ",
    "outputId": "a6079220-b1dc-4601-f0e7-9f96a79746de"
   },
   "outputs": [],
   "source": [
    "# Data_path = 'D:/studies/university/thesis/speech_separation_codes/du16/donesomestuff'\n",
    "# Data_path = 'D:/hale'\n",
    "# Data_path = '/content/drive/My Drive/thesis/datasets'\n",
    "Data_path = os.getcwd()[0:-5]\n",
    "ckpt_folder = '565'\n",
    "\n",
    "checkpoint_path = os.path.normpath(os.path.join(Data_path,'results','checkpoints',ckpt_folder))\n",
    "if not os.path.exists(checkpoint_path):\n",
    "    os.makedirs(checkpoint_path)\n",
    "files = glob.glob(os.path.normpath(os.path.join(checkpoint_path,'*')))\n",
    "for f in files:\n",
    "    os.remove(f)\n",
    "    \n",
    "loss_path = os.path.normpath(os.path.join(Data_path,'results','losses',ckpt_folder))\n",
    "if not os.path.exists(loss_path):\n",
    "    os.makedirs(loss_path)\n",
    "files = glob.glob(os.path.normpath(os.path.join(loss_path,'*')))\n",
    "for f in files:\n",
    "    os.remove(f)\n",
    "    \n",
    "time_path = os.path.normpath(os.path.join(Data_path,'results','times',ckpt_folder))\n",
    "if not os.path.exists(time_path):\n",
    "    os.makedirs(time_path)\n",
    "files = glob.glob(os.path.normpath(os.path.join(time_path,'*')))\n",
    "for f in files:\n",
    "    os.remove(f)\n",
    "\n",
    "    \n",
    "timestep = 16\n",
    "batch_size = 32\n",
    "n_features = 257\n",
    "epochs_num =100\n",
    "lr = 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset_loader():\n",
    "\n",
    "    def __init__(self, path, batch_size, timestep, X_, Y_):\n",
    "        self.write_path = path\n",
    "        self.batch_size = batch_size\n",
    "        self.len_data = 0\n",
    "        self.X_ = X_\n",
    "        self.Y_ = Y_\n",
    "        self.timestep = timestep\n",
    "        with h5py.File(os.path.normpath(os.path.join(self.write_path))+'.hdf5', \"r\") as f:\n",
    "            obj = f['X']\n",
    "            self.len_data = obj.shape[0]\n",
    "            self.n_features = obj.shape[1]\n",
    "        self.index_arr = np.arange(self.len_data)\n",
    "        self.index_arr = self.index_arr[0:(len(self.index_arr)//self.timestep)*self.timestep]\n",
    "        self.index_arr = self.index_arr.reshape((len(self.index_arr)//self.timestep,self.timestep))\n",
    "        np.random.shuffle(self.index_arr)\n",
    "\n",
    "    def check_if_left(self):\n",
    "        if len(self.index_arr)>=self.batch_size:\n",
    "            return True\n",
    "        return False\n",
    "\n",
    "    def get_batch(self):\n",
    "        self.X_out=self.X_[self.index_arr[0:self.batch_size].reshape(1,self.batch_size*self.timestep)]\n",
    "        self.Y_out=self.Y_[self.index_arr[0:self.batch_size].reshape(1,self.batch_size*self.timestep)]\n",
    "        self.index_arr = self.index_arr[self.batch_size:]\n",
    "#         print(self.X_out.reshape(self.batch_size,self.timestep,self.n_features))\n",
    "#         print(self.X_out.reshape(self.batch_size,self.timestep,self.n_features))\n",
    "        return self.X_out.reshape(self.batch_size,self.timestep,self.n_features),self.Y_out.reshape(self.batch_size,self.timestep,self.n_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.normpath(os.path.join(Data_path,'h5_files_e','30hdata_2','features','data_30h_loge'))\n",
    "path_val = os.path.normpath(os.path.join(Data_path,'h5_files_e','devdata_3','features','devdata_30h_loge'))\n",
    "files = 'data_30h_loge'\n",
    "files_val = 'devdata_30h_loge'\n",
    "with h5py.File(path+'.hdf5', \"r\") as f:\n",
    "    X = f['X'][:]\n",
    "with h5py.File(path+'.hdf5', \"r\") as f:\n",
    "    Y = f['Y'][:]\n",
    "with h5py.File(path_val+'.hdf5', \"r\") as f:\n",
    "    X_val = f['X'][:]\n",
    "with h5py.File(path_val+'.hdf5', \"r\") as f:\n",
    "    Y_val = f['Y'][:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_batch = Dataset_loader( path, batch_size, timestep, X, Y)\n",
    "load_batch.check_if_left()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (lstm): LSTM(257, 257, num_layers=2, batch_first=True, bidirectional=True)\n",
       "  (fc): Linear(in_features=514, out_features=257, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.lstm = nn.LSTM(input_size=n_features,hidden_size=257, num_layers=2, batch_first=True, bidirectional=True)\n",
    "        self.fc = nn.Linear(514, 257)\n",
    "    def forward(self, x):\n",
    "        x, _  = self.lstm(x)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "    \n",
    "net = Net()\n",
    "net.cuda()\n",
    "net.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class custom_losses(torch.nn.Module):\n",
    "    def __init__(self, y_true, y_pred, epoch):\n",
    "        super(custom_losses,self).__init__()\n",
    "        self.sample_rate = 16000\n",
    "        self.nfilt = 40\n",
    "        self.NFFT = 512\n",
    "        self.low_freq_mel = 0\n",
    "        self.y_true = y_true\n",
    "        self.y_pred = y_pred\n",
    "        self.base = 10\n",
    "        self.epoch = epoch\n",
    "        self.batch_size = y_true.shape[0]\n",
    "        self.n_features = y_true.shape[-1]\n",
    "        self.high_freq_mel = 2595 * np.log10(1 + (self.sample_rate / 2) / 700)  # Convert Hz to Mel\n",
    "        self.mel_points = np.linspace(self.low_freq_mel, self.high_freq_mel, self.nfilt + 2)  # Equally spaced in Mel scale\n",
    "        self.hz_points = (700 * (10**(self.mel_points / 2595) - 1))  # Convert Mel to Hz\n",
    "        self.binf = np.floor((self.NFFT + 1) * self.hz_points / self.sample_rate)\n",
    "#         print(binf)\n",
    "        self.fbank = np.zeros((self.nfilt, self.n_features),dtype=np.float32)\n",
    "    def forward(self):\n",
    "        self.true = torch.sqrt(torch.exp(self.y_true))\n",
    "        self.pred = torch.sqrt(torch.exp(self.y_pred))\n",
    "        for m in range(1, 41):\n",
    "            f_m_minus = np.int(self.binf[m - 1])  # left\n",
    "            f_m = np.int(self.binf[m])           # center\n",
    "            f_m_plus = np.int(self.binf[m + 1])    # right\n",
    "\n",
    "            for k in range(f_m_minus, f_m):\n",
    "                self.fbank[m - 1, k] = (k - self.binf[m - 1]) / (self.binf[m] - self.binf[m - 1])\n",
    "            for k in range(f_m, f_m_plus):\n",
    "                self.fbank[m - 1, k] = (self.binf[m + 1] - k) / (self.binf[m + 1] - self.binf[m])\n",
    "        self.fbank= self.fbank.reshape((1,self.fbank.shape[0],self.fbank.shape[1]))\n",
    "        self.fbank = np.repeat(self.fbank, self.batch_size, axis=0)\n",
    "        self.filtered_true = torch.einsum('mnp,mqp->mnq', self.true, torch.tensor(self.fbank,device=torch.device('cuda:0')))\n",
    "        self.filtered_pred = torch.einsum('mnp,mqp->mnq',self.pred, torch.tensor(self.fbank,device=torch.device('cuda:0')))\n",
    "#         print(torch.mean(torch.pow(self.filtered_true-self.filtered_pred,2)))\n",
    "#         print(torch.mean(torch.pow(self.y_true-self.y_pred,2)))\n",
    "        l = (torch.mean(torch.pow(self.filtered_true-self.filtered_pred,2))*1.4)+(1*torch.mean(torch.pow(self.y_true-self.y_pred,2)))\n",
    "        return l\n",
    "\n",
    "optimizer = optim.Adam(net.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda1 = lambda epoch: 0.9**(epoch-10) if epoch>10 else 1\n",
    "scheduler = LambdaLR(optimizer, lr_lambda=lambda1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch 1, minibatch  1000] loss: 52.441\n",
      "[epoch 1, minibatch  2000] loss: 24.978\n",
      "[epoch 1, minibatch  3000] loss: 21.178\n",
      "[epoch 1, minibatch  4000] loss: 18.993\n",
      "[epoch 1, minibatch  5000] loss: 17.390\n",
      "[epoch 1, minibatch  6000] loss: 16.084\n",
      "[epoch 1, minibatch  7000] loss: 15.057\n",
      "[epoch 1, minibatch  8000] loss: 14.292\n",
      "[epoch 1, minibatch  9000] loss: 13.613\n",
      "[epoch 1, minibatch 10000] loss: 13.230\n",
      "[epoch 1, minibatch 11000] loss: 12.737\n",
      "[epoch 1, minibatch 12000] loss: 12.523\n",
      "[epoch 1, minibatch 13000] loss: 12.244\n",
      "[epoch 1, minibatch 14000] loss: 11.839\n",
      "epoch 1, train loss: 17.908\n",
      "0.0001\n",
      "************ epoch 1, val loss: 19.962 ************\n",
      "[epoch 2, minibatch  1000] loss: 11.215\n",
      "[epoch 2, minibatch  2000] loss: 11.109\n",
      "[epoch 2, minibatch  3000] loss: 10.794\n",
      "[epoch 2, minibatch  4000] loss: 10.628\n",
      "[epoch 2, minibatch  5000] loss: 10.644\n",
      "[epoch 2, minibatch  6000] loss: 10.451\n",
      "[epoch 2, minibatch  7000] loss: 10.175\n",
      "[epoch 2, minibatch  8000] loss: 10.055\n",
      "[epoch 2, minibatch  9000] loss: 9.981\n",
      "[epoch 2, minibatch 10000] loss: 9.948\n",
      "[epoch 2, minibatch 11000] loss: 9.795\n",
      "[epoch 2, minibatch 12000] loss: 9.695\n",
      "[epoch 2, minibatch 13000] loss: 9.503\n",
      "[epoch 2, minibatch 14000] loss: 9.498\n",
      "epoch 2, train loss: 10.197\n",
      "0.0001\n",
      "************ epoch 2, val loss: 19.091 ************\n",
      "[epoch 3, minibatch  1000] loss: 9.067\n",
      "[epoch 3, minibatch  2000] loss: 9.093\n",
      "[epoch 3, minibatch  3000] loss: 8.854\n",
      "[epoch 3, minibatch  4000] loss: 9.000\n",
      "[epoch 3, minibatch  5000] loss: 8.779\n",
      "[epoch 3, minibatch  6000] loss: 8.823\n",
      "[epoch 3, minibatch  7000] loss: 8.679\n",
      "[epoch 3, minibatch  8000] loss: 8.712\n",
      "[epoch 3, minibatch  9000] loss: 8.509\n",
      "[epoch 3, minibatch 10000] loss: 8.341\n",
      "[epoch 3, minibatch 11000] loss: 8.445\n",
      "[epoch 3, minibatch 12000] loss: 8.454\n",
      "[epoch 3, minibatch 13000] loss: 8.297\n",
      "[epoch 3, minibatch 14000] loss: 8.297\n",
      "epoch 3, train loss: 8.645\n",
      "0.0001\n",
      "************ epoch 3, val loss: 17.926 ************\n",
      "[epoch 4, minibatch  1000] loss: 7.986\n",
      "[epoch 4, minibatch  2000] loss: 7.936\n",
      "[epoch 4, minibatch  3000] loss: 7.937\n",
      "[epoch 4, minibatch  4000] loss: 7.905\n",
      "[epoch 4, minibatch  5000] loss: 7.860\n",
      "[epoch 4, minibatch  6000] loss: 7.896\n",
      "[epoch 4, minibatch  7000] loss: 7.800\n",
      "[epoch 4, minibatch  8000] loss: 7.701\n",
      "[epoch 4, minibatch  9000] loss: 7.693\n",
      "[epoch 4, minibatch 10000] loss: 7.575\n",
      "[epoch 4, minibatch 11000] loss: 7.628\n",
      "[epoch 4, minibatch 12000] loss: 7.615\n",
      "[epoch 4, minibatch 13000] loss: 7.430\n",
      "[epoch 4, minibatch 14000] loss: 7.467\n",
      "epoch 4, train loss: 7.726\n",
      "0.0001\n",
      "************ epoch 4, val loss: 17.173 ************\n",
      "[epoch 5, minibatch  1000] loss: 7.297\n",
      "[epoch 5, minibatch  2000] loss: 7.211\n",
      "[epoch 5, minibatch  3000] loss: 7.177\n",
      "[epoch 5, minibatch  4000] loss: 7.188\n",
      "[epoch 5, minibatch  5000] loss: 7.082\n",
      "[epoch 5, minibatch  6000] loss: 7.169\n",
      "[epoch 5, minibatch  7000] loss: 7.124\n",
      "[epoch 5, minibatch  8000] loss: 7.104\n",
      "[epoch 5, minibatch  9000] loss: 7.062\n",
      "[epoch 5, minibatch 10000] loss: 7.017\n",
      "[epoch 5, minibatch 11000] loss: 7.040\n",
      "[epoch 5, minibatch 12000] loss: 7.045\n",
      "[epoch 5, minibatch 13000] loss: 6.966\n",
      "[epoch 5, minibatch 14000] loss: 6.999\n",
      "epoch 5, train loss: 7.095\n",
      "0.0001\n",
      "************ epoch 5, val loss: 16.264 ************\n",
      "[epoch 6, minibatch  1000] loss: 6.687\n",
      "[epoch 6, minibatch  2000] loss: 6.700\n",
      "[epoch 6, minibatch  3000] loss: 6.590\n",
      "[epoch 6, minibatch  4000] loss: 6.620\n",
      "[epoch 6, minibatch  5000] loss: 6.680\n",
      "[epoch 6, minibatch  6000] loss: 6.682\n",
      "[epoch 6, minibatch  7000] loss: 6.678\n",
      "[epoch 6, minibatch  8000] loss: 6.591\n",
      "[epoch 6, minibatch  9000] loss: 6.613\n",
      "[epoch 6, minibatch 10000] loss: 6.589\n",
      "[epoch 6, minibatch 11000] loss: 6.605\n",
      "[epoch 6, minibatch 12000] loss: 6.579\n",
      "[epoch 6, minibatch 13000] loss: 6.448\n",
      "[epoch 6, minibatch 14000] loss: 6.566\n",
      "epoch 6, train loss: 6.604\n",
      "0.0001\n",
      "************ epoch 6, val loss: 14.697 ************\n",
      "[epoch 7, minibatch  1000] loss: 6.291\n",
      "[epoch 7, minibatch  2000] loss: 6.236\n",
      "[epoch 7, minibatch  3000] loss: 6.243\n",
      "[epoch 7, minibatch  4000] loss: 6.310\n",
      "[epoch 7, minibatch  5000] loss: 6.217\n",
      "[epoch 7, minibatch  6000] loss: 6.278\n",
      "[epoch 7, minibatch  7000] loss: 6.248\n",
      "[epoch 7, minibatch  8000] loss: 6.253\n",
      "[epoch 7, minibatch  9000] loss: 6.196\n",
      "[epoch 7, minibatch 10000] loss: 6.181\n",
      "[epoch 7, minibatch 11000] loss: 6.187\n",
      "[epoch 7, minibatch 12000] loss: 6.120\n",
      "[epoch 7, minibatch 13000] loss: 6.182\n",
      "[epoch 7, minibatch 14000] loss: 6.148\n",
      "epoch 7, train loss: 6.215\n",
      "0.0001\n",
      "************ epoch 7, val loss: 14.926 ************\n",
      "[epoch 8, minibatch  1000] loss: 5.942\n",
      "[epoch 8, minibatch  2000] loss: 5.992\n",
      "[epoch 8, minibatch  3000] loss: 5.943\n",
      "[epoch 8, minibatch  4000] loss: 5.929\n",
      "[epoch 8, minibatch  5000] loss: 5.964\n",
      "[epoch 8, minibatch  6000] loss: 5.920\n",
      "[epoch 8, minibatch  7000] loss: 5.880\n",
      "[epoch 8, minibatch  8000] loss: 5.882\n",
      "[epoch 8, minibatch  9000] loss: 5.877\n",
      "[epoch 8, minibatch 10000] loss: 5.919\n",
      "[epoch 8, minibatch 11000] loss: 5.880\n",
      "[epoch 8, minibatch 12000] loss: 5.865\n",
      "[epoch 8, minibatch 13000] loss: 5.885\n",
      "[epoch 8, minibatch 14000] loss: 5.854\n",
      "epoch 8, train loss: 5.907\n",
      "0.0001\n",
      "************ epoch 8, val loss: 14.750 ************\n",
      "[epoch 9, minibatch  1000] loss: 5.613\n",
      "[epoch 9, minibatch  2000] loss: 5.644\n",
      "[epoch 9, minibatch  3000] loss: 5.716\n",
      "[epoch 9, minibatch  4000] loss: 5.647\n",
      "[epoch 9, minibatch  5000] loss: 5.641\n",
      "[epoch 9, minibatch  6000] loss: 5.630\n",
      "[epoch 9, minibatch  7000] loss: 5.656\n",
      "[epoch 9, minibatch  8000] loss: 5.639\n",
      "[epoch 9, minibatch  9000] loss: 5.649\n",
      "[epoch 9, minibatch 10000] loss: 5.564\n",
      "[epoch 9, minibatch 11000] loss: 5.614\n",
      "[epoch 9, minibatch 12000] loss: 5.659\n",
      "[epoch 9, minibatch 13000] loss: 5.620\n",
      "[epoch 9, minibatch 14000] loss: 5.588\n",
      "epoch 9, train loss: 5.633\n",
      "0.0001\n",
      "************ epoch 9, val loss: 14.286 ************\n",
      "[epoch 10, minibatch  1000] loss: 5.435\n",
      "[epoch 10, minibatch  2000] loss: 5.356\n",
      "[epoch 10, minibatch  3000] loss: 5.438\n",
      "[epoch 10, minibatch  4000] loss: 5.422\n",
      "[epoch 10, minibatch  5000] loss: 5.402\n",
      "[epoch 10, minibatch  6000] loss: 5.362\n",
      "[epoch 10, minibatch  7000] loss: 5.440\n",
      "[epoch 10, minibatch  8000] loss: 5.374\n",
      "[epoch 10, minibatch  9000] loss: 5.454\n",
      "[epoch 10, minibatch 10000] loss: 5.389\n",
      "[epoch 10, minibatch 11000] loss: 5.322\n",
      "[epoch 10, minibatch 12000] loss: 5.466\n",
      "[epoch 10, minibatch 13000] loss: 5.403\n",
      "[epoch 10, minibatch 14000] loss: 5.330\n",
      "epoch 10, train loss: 5.398\n",
      "0.0001\n",
      "************ epoch 10, val loss: 14.281 ************\n",
      "[epoch 11, minibatch  1000] loss: 5.205\n",
      "[epoch 11, minibatch  2000] loss: 5.152\n",
      "[epoch 11, minibatch  3000] loss: 5.198\n",
      "[epoch 11, minibatch  4000] loss: 5.195\n",
      "[epoch 11, minibatch  5000] loss: 5.170\n",
      "[epoch 11, minibatch  6000] loss: 5.236\n",
      "[epoch 11, minibatch  7000] loss: 5.254\n",
      "[epoch 11, minibatch  8000] loss: 5.187\n",
      "[epoch 11, minibatch  9000] loss: 5.189\n",
      "[epoch 11, minibatch 10000] loss: 5.234\n",
      "[epoch 11, minibatch 11000] loss: 5.265\n",
      "[epoch 11, minibatch 12000] loss: 5.130\n",
      "[epoch 11, minibatch 13000] loss: 5.207\n",
      "[epoch 11, minibatch 14000] loss: 5.168\n",
      "epoch 11, train loss: 5.201\n",
      "0.0001\n",
      "************ epoch 11, val loss: 13.125 ************\n",
      "[epoch 12, minibatch  1000] loss: 4.994\n",
      "[epoch 12, minibatch  2000] loss: 4.993\n",
      "[epoch 12, minibatch  3000] loss: 5.074\n",
      "[epoch 12, minibatch  4000] loss: 5.043\n",
      "[epoch 12, minibatch  5000] loss: 5.024\n",
      "[epoch 12, minibatch  6000] loss: 5.036\n",
      "[epoch 12, minibatch  7000] loss: 4.995\n",
      "[epoch 12, minibatch  8000] loss: 5.009\n",
      "[epoch 12, minibatch  9000] loss: 5.071\n",
      "[epoch 12, minibatch 10000] loss: 5.023\n",
      "[epoch 12, minibatch 11000] loss: 5.016\n",
      "[epoch 12, minibatch 12000] loss: 4.945\n",
      "[epoch 12, minibatch 13000] loss: 4.949\n",
      "[epoch 12, minibatch 14000] loss: 5.155\n",
      "epoch 12, train loss: 5.022\n",
      "0.0001\n",
      "************ epoch 12, val loss: 13.412 ************\n",
      "[epoch 13, minibatch  1000] loss: 4.854\n",
      "[epoch 13, minibatch  2000] loss: 4.803\n",
      "[epoch 13, minibatch  3000] loss: 4.893\n",
      "[epoch 13, minibatch  4000] loss: 4.881\n",
      "[epoch 13, minibatch  5000] loss: 4.862\n",
      "[epoch 13, minibatch  6000] loss: 4.853\n",
      "[epoch 13, minibatch  7000] loss: 4.871\n",
      "[epoch 13, minibatch  8000] loss: 4.887\n",
      "[epoch 13, minibatch  9000] loss: 4.842\n",
      "[epoch 13, minibatch 10000] loss: 4.898\n",
      "[epoch 13, minibatch 11000] loss: 4.842\n",
      "[epoch 13, minibatch 12000] loss: 4.797\n",
      "[epoch 13, minibatch 13000] loss: 4.858\n",
      "[epoch 13, minibatch 14000] loss: 4.892\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, train loss: 4.860\n",
      "0.0001\n",
      "************ epoch 13, val loss: 12.913 ************\n",
      "[epoch 14, minibatch  1000] loss: 4.674\n",
      "[epoch 14, minibatch  2000] loss: 4.741\n",
      "[epoch 14, minibatch  3000] loss: 4.686\n",
      "[epoch 14, minibatch  4000] loss: 4.688\n",
      "[epoch 14, minibatch  5000] loss: 4.826\n",
      "[epoch 14, minibatch  6000] loss: 4.690\n",
      "[epoch 14, minibatch  7000] loss: 4.731\n",
      "[epoch 14, minibatch  8000] loss: 4.730\n",
      "[epoch 14, minibatch  9000] loss: 4.765\n",
      "[epoch 14, minibatch 10000] loss: 4.686\n",
      "[epoch 14, minibatch 11000] loss: 4.728\n",
      "[epoch 14, minibatch 12000] loss: 4.752\n",
      "[epoch 14, minibatch 13000] loss: 4.733\n",
      "[epoch 14, minibatch 14000] loss: 4.705\n",
      "epoch 14, train loss: 4.729\n",
      "0.0001\n",
      "************ epoch 14, val loss: 13.568 ************\n",
      "[epoch 15, minibatch  1000] loss: 4.581\n",
      "[epoch 15, minibatch  2000] loss: 4.565\n",
      "[epoch 15, minibatch  3000] loss: 4.630\n",
      "[epoch 15, minibatch  4000] loss: 4.585\n",
      "[epoch 15, minibatch  5000] loss: 4.601\n",
      "[epoch 15, minibatch  6000] loss: 4.566\n",
      "[epoch 15, minibatch  7000] loss: 4.603\n",
      "[epoch 15, minibatch  8000] loss: 4.637\n",
      "[epoch 15, minibatch  9000] loss: 4.580\n",
      "[epoch 15, minibatch 10000] loss: 4.580\n",
      "[epoch 15, minibatch 11000] loss: 4.590\n",
      "[epoch 15, minibatch 12000] loss: 4.639\n",
      "[epoch 15, minibatch 13000] loss: 4.624\n",
      "[epoch 15, minibatch 14000] loss: 4.626\n",
      "epoch 15, train loss: 4.603\n",
      "0.0001\n",
      "************ epoch 15, val loss: 13.469 ************\n",
      "[epoch 16, minibatch  1000] loss: 4.451\n",
      "[epoch 16, minibatch  2000] loss: 4.509\n",
      "[epoch 16, minibatch  3000] loss: 4.467\n",
      "[epoch 16, minibatch  4000] loss: 4.527\n",
      "[epoch 16, minibatch  5000] loss: 4.454\n",
      "[epoch 16, minibatch  6000] loss: 4.449\n",
      "[epoch 16, minibatch  7000] loss: 4.484\n",
      "[epoch 16, minibatch  8000] loss: 4.528\n",
      "[epoch 16, minibatch  9000] loss: 4.485\n",
      "[epoch 16, minibatch 10000] loss: 4.481\n",
      "[epoch 16, minibatch 11000] loss: 4.483\n",
      "[epoch 16, minibatch 12000] loss: 4.506\n",
      "[epoch 16, minibatch 13000] loss: 4.514\n",
      "[epoch 16, minibatch 14000] loss: 4.477\n",
      "epoch 16, train loss: 4.490\n",
      "0.0001\n",
      "************ epoch 16, val loss: 12.906 ************\n",
      "[epoch 17, minibatch  1000] loss: 4.384\n",
      "[epoch 17, minibatch  2000] loss: 4.387\n",
      "[epoch 17, minibatch  3000] loss: 4.335\n",
      "[epoch 17, minibatch  4000] loss: 4.420\n",
      "[epoch 17, minibatch  5000] loss: 4.373\n",
      "[epoch 17, minibatch  6000] loss: 4.318\n",
      "[epoch 17, minibatch  7000] loss: 4.402\n",
      "[epoch 17, minibatch  8000] loss: 4.385\n",
      "[epoch 17, minibatch  9000] loss: 4.374\n",
      "[epoch 17, minibatch 10000] loss: 4.415\n",
      "[epoch 17, minibatch 11000] loss: 4.359\n",
      "[epoch 17, minibatch 12000] loss: 4.417\n",
      "[epoch 17, minibatch 13000] loss: 4.360\n",
      "[epoch 17, minibatch 14000] loss: 4.408\n",
      "epoch 17, train loss: 4.384\n",
      "0.0001\n",
      "************ epoch 17, val loss: 13.012 ************\n",
      "[epoch 18, minibatch  1000] loss: 4.238\n",
      "[epoch 18, minibatch  2000] loss: 4.221\n",
      "[epoch 18, minibatch  3000] loss: 4.268\n",
      "[epoch 18, minibatch  4000] loss: 4.274\n",
      "[epoch 18, minibatch  5000] loss: 4.272\n",
      "[epoch 18, minibatch  6000] loss: 4.336\n",
      "[epoch 18, minibatch  7000] loss: 4.271\n",
      "[epoch 18, minibatch  8000] loss: 4.311\n",
      "[epoch 18, minibatch  9000] loss: 4.307\n",
      "[epoch 18, minibatch 10000] loss: 4.276\n",
      "[epoch 18, minibatch 11000] loss: 4.289\n",
      "[epoch 18, minibatch 12000] loss: 4.336\n",
      "[epoch 18, minibatch 13000] loss: 4.315\n",
      "[epoch 18, minibatch 14000] loss: 4.293\n",
      "epoch 18, train loss: 4.287\n",
      "0.0001\n",
      "************ epoch 18, val loss: 12.939 ************\n",
      "[epoch 19, minibatch  1000] loss: 4.200\n",
      "[epoch 19, minibatch  2000] loss: 4.209\n",
      "[epoch 19, minibatch  3000] loss: 4.146\n",
      "[epoch 19, minibatch  4000] loss: 4.184\n",
      "[epoch 19, minibatch  5000] loss: 4.318\n",
      "[epoch 19, minibatch  6000] loss: 4.145\n",
      "[epoch 19, minibatch  7000] loss: 4.388\n",
      "[epoch 19, minibatch  8000] loss: 4.209\n",
      "[epoch 19, minibatch  9000] loss: 4.190\n",
      "[epoch 19, minibatch 10000] loss: 4.224\n",
      "[epoch 19, minibatch 11000] loss: 4.196\n",
      "[epoch 19, minibatch 12000] loss: 4.200\n",
      "[epoch 19, minibatch 13000] loss: 4.221\n",
      "[epoch 19, minibatch 14000] loss: 4.241\n",
      "epoch 19, train loss: 4.216\n",
      "0.0001\n",
      "************ epoch 19, val loss: 12.649 ************\n",
      "[epoch 20, minibatch  1000] loss: 4.072\n",
      "[epoch 20, minibatch  2000] loss: 4.040\n",
      "[epoch 20, minibatch  3000] loss: 4.122\n",
      "[epoch 20, minibatch  4000] loss: 4.124\n",
      "[epoch 20, minibatch  5000] loss: 4.128\n",
      "[epoch 20, minibatch  6000] loss: 4.131\n",
      "[epoch 20, minibatch  7000] loss: 4.126\n",
      "[epoch 20, minibatch  8000] loss: 4.110\n",
      "[epoch 20, minibatch  9000] loss: 4.174\n",
      "[epoch 20, minibatch 10000] loss: 4.193\n",
      "[epoch 20, minibatch 11000] loss: 4.108\n",
      "[epoch 20, minibatch 12000] loss: 4.084\n",
      "[epoch 20, minibatch 13000] loss: 4.143\n",
      "[epoch 20, minibatch 14000] loss: 4.144\n",
      "epoch 20, train loss: 4.121\n",
      "0.0001\n",
      "************ epoch 20, val loss: 12.508 ************\n",
      "[epoch 21, minibatch  1000] loss: 3.988\n",
      "[epoch 21, minibatch  2000] loss: 4.028\n",
      "[epoch 21, minibatch  3000] loss: 4.030\n",
      "[epoch 21, minibatch  4000] loss: 4.047\n",
      "[epoch 21, minibatch  5000] loss: 4.063\n",
      "[epoch 21, minibatch  6000] loss: 4.056\n",
      "[epoch 21, minibatch  7000] loss: 4.057\n",
      "[epoch 21, minibatch  8000] loss: 4.045\n",
      "[epoch 21, minibatch  9000] loss: 4.130\n",
      "[epoch 21, minibatch 10000] loss: 4.026\n",
      "[epoch 21, minibatch 11000] loss: 4.053\n",
      "[epoch 21, minibatch 12000] loss: 4.041\n",
      "[epoch 21, minibatch 13000] loss: 4.063\n",
      "[epoch 21, minibatch 14000] loss: 4.072\n",
      "epoch 21, train loss: 4.052\n",
      "0.0001\n",
      "************ epoch 21, val loss: 12.931 ************\n",
      "[epoch 22, minibatch  1000] loss: 3.917\n",
      "[epoch 22, minibatch  2000] loss: 3.964\n",
      "[epoch 22, minibatch  3000] loss: 3.932\n",
      "[epoch 22, minibatch  4000] loss: 3.960\n",
      "[epoch 22, minibatch  5000] loss: 4.017\n",
      "[epoch 22, minibatch  6000] loss: 4.007\n",
      "[epoch 22, minibatch  7000] loss: 3.993\n",
      "[epoch 22, minibatch  8000] loss: 3.947\n",
      "[epoch 22, minibatch  9000] loss: 3.998\n",
      "[epoch 22, minibatch 10000] loss: 4.053\n",
      "[epoch 22, minibatch 11000] loss: 3.959\n",
      "[epoch 22, minibatch 12000] loss: 3.976\n",
      "[epoch 22, minibatch 13000] loss: 3.993\n",
      "[epoch 22, minibatch 14000] loss: 3.999\n",
      "epoch 22, train loss: 3.979\n",
      "0.0001\n",
      "************ epoch 22, val loss: 12.889 ************\n",
      "[epoch 23, minibatch  1000] loss: 3.868\n",
      "[epoch 23, minibatch  2000] loss: 4.017\n",
      "[epoch 23, minibatch  3000] loss: 3.919\n",
      "[epoch 23, minibatch  4000] loss: 3.859\n",
      "[epoch 23, minibatch  5000] loss: 3.906\n",
      "[epoch 23, minibatch  6000] loss: 3.884\n",
      "[epoch 23, minibatch  7000] loss: 3.875\n",
      "[epoch 23, minibatch  8000] loss: 3.939\n",
      "[epoch 23, minibatch  9000] loss: 3.949\n",
      "[epoch 23, minibatch 10000] loss: 3.923\n",
      "[epoch 23, minibatch 11000] loss: 3.921\n",
      "[epoch 23, minibatch 12000] loss: 3.932\n",
      "[epoch 23, minibatch 13000] loss: 3.897\n",
      "[epoch 23, minibatch 14000] loss: 4.004\n",
      "epoch 23, train loss: 3.923\n",
      "0.0001\n",
      "************ epoch 23, val loss: 12.800 ************\n",
      "[epoch 24, minibatch  1000] loss: 3.793\n",
      "[epoch 24, minibatch  2000] loss: 3.841\n",
      "[epoch 24, minibatch  3000] loss: 3.842\n",
      "[epoch 24, minibatch  4000] loss: 3.823\n",
      "[epoch 24, minibatch  5000] loss: 3.861\n",
      "[epoch 24, minibatch  6000] loss: 3.828\n",
      "[epoch 24, minibatch  7000] loss: 3.873\n",
      "[epoch 24, minibatch  8000] loss: 3.887\n",
      "[epoch 24, minibatch  9000] loss: 3.858\n",
      "[epoch 24, minibatch 10000] loss: 3.844\n",
      "[epoch 24, minibatch 11000] loss: 3.877\n",
      "[epoch 24, minibatch 12000] loss: 3.904\n",
      "[epoch 24, minibatch 13000] loss: 3.854\n",
      "[epoch 24, minibatch 14000] loss: 3.869\n",
      "epoch 24, train loss: 3.855\n",
      "0.0001\n",
      "************ epoch 24, val loss: 12.268 ************\n",
      "[epoch 25, minibatch  1000] loss: 3.733\n",
      "[epoch 25, minibatch  2000] loss: 3.822\n",
      "[epoch 25, minibatch  3000] loss: 3.784\n",
      "[epoch 25, minibatch  4000] loss: 3.803\n",
      "[epoch 25, minibatch  5000] loss: 3.800\n",
      "[epoch 25, minibatch  6000] loss: 3.803\n",
      "[epoch 25, minibatch  7000] loss: 3.777\n",
      "[epoch 25, minibatch  8000] loss: 3.822\n",
      "[epoch 25, minibatch  9000] loss: 3.820\n",
      "[epoch 25, minibatch 10000] loss: 3.814\n",
      "[epoch 25, minibatch 11000] loss: 3.805\n",
      "[epoch 25, minibatch 12000] loss: 3.807\n",
      "[epoch 25, minibatch 13000] loss: 3.812\n",
      "[epoch 25, minibatch 14000] loss: 3.785\n",
      "epoch 25, train loss: 3.802\n",
      "0.0001\n",
      "************ epoch 25, val loss: 12.494 ************\n",
      "[epoch 26, minibatch  1000] loss: 3.713\n",
      "[epoch 26, minibatch  2000] loss: 3.705\n",
      "[epoch 26, minibatch  3000] loss: 3.754\n",
      "[epoch 26, minibatch  4000] loss: 3.749\n",
      "[epoch 26, minibatch  5000] loss: 3.754\n",
      "[epoch 26, minibatch  6000] loss: 3.766\n",
      "[epoch 26, minibatch  7000] loss: 3.720\n",
      "[epoch 26, minibatch  8000] loss: 3.748\n",
      "[epoch 26, minibatch  9000] loss: 3.772\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch 26, minibatch 10000] loss: 3.744\n",
      "[epoch 26, minibatch 11000] loss: 3.794\n",
      "[epoch 26, minibatch 12000] loss: 3.782\n",
      "[epoch 26, minibatch 13000] loss: 3.744\n",
      "[epoch 26, minibatch 14000] loss: 3.772\n",
      "epoch 26, train loss: 3.753\n",
      "0.0001\n",
      "************ epoch 26, val loss: 12.399 ************\n",
      "[epoch 27, minibatch  1000] loss: 3.640\n",
      "[epoch 27, minibatch  2000] loss: 3.659\n",
      "[epoch 27, minibatch  3000] loss: 3.675\n",
      "[epoch 27, minibatch  4000] loss: 3.669\n",
      "[epoch 27, minibatch  5000] loss: 3.684\n",
      "[epoch 27, minibatch  6000] loss: 3.723\n",
      "[epoch 27, minibatch  7000] loss: 3.701\n",
      "[epoch 27, minibatch  8000] loss: 3.709\n",
      "[epoch 27, minibatch  9000] loss: 3.764\n",
      "[epoch 27, minibatch 10000] loss: 3.727\n",
      "[epoch 27, minibatch 11000] loss: 3.749\n",
      "[epoch 27, minibatch 12000] loss: 3.699\n",
      "[epoch 27, minibatch 13000] loss: 3.699\n",
      "[epoch 27, minibatch 14000] loss: 3.718\n",
      "epoch 27, train loss: 3.703\n",
      "0.0001\n",
      "************ epoch 27, val loss: 12.430 ************\n",
      "[epoch 28, minibatch  1000] loss: 3.607\n",
      "[epoch 28, minibatch  2000] loss: 3.629\n",
      "[epoch 28, minibatch  3000] loss: 3.623\n",
      "[epoch 28, minibatch  4000] loss: 3.631\n",
      "[epoch 28, minibatch  5000] loss: 3.665\n",
      "[epoch 28, minibatch  6000] loss: 3.625\n",
      "[epoch 28, minibatch  7000] loss: 3.663\n",
      "[epoch 28, minibatch  8000] loss: 3.665\n",
      "[epoch 28, minibatch  9000] loss: 3.701\n",
      "[epoch 28, minibatch 10000] loss: 3.626\n",
      "[epoch 28, minibatch 11000] loss: 3.657\n",
      "[epoch 28, minibatch 12000] loss: 3.678\n",
      "[epoch 28, minibatch 13000] loss: 3.679\n",
      "[epoch 28, minibatch 14000] loss: 3.685\n",
      "epoch 28, train loss: 3.655\n",
      "0.0001\n",
      "************ epoch 28, val loss: 12.614 ************\n",
      "[epoch 29, minibatch  1000] loss: 3.580\n",
      "[epoch 29, minibatch  2000] loss: 3.582\n",
      "[epoch 29, minibatch  3000] loss: 3.605\n",
      "[epoch 29, minibatch  4000] loss: 3.745\n",
      "[epoch 29, minibatch  5000] loss: 3.597\n",
      "[epoch 29, minibatch  6000] loss: 3.575\n",
      "[epoch 29, minibatch  7000] loss: 3.621\n",
      "[epoch 29, minibatch  8000] loss: 3.626\n",
      "[epoch 29, minibatch  9000] loss: 3.617\n",
      "[epoch 29, minibatch 10000] loss: 3.651\n",
      "[epoch 29, minibatch 11000] loss: 3.622\n",
      "[epoch 29, minibatch 12000] loss: 3.635\n",
      "[epoch 29, minibatch 13000] loss: 3.607\n",
      "[epoch 29, minibatch 14000] loss: 3.648\n",
      "epoch 29, train loss: 3.623\n",
      "0.0001\n",
      "************ epoch 29, val loss: 12.115 ************\n",
      "[epoch 30, minibatch  1000] loss: 3.515\n",
      "[epoch 30, minibatch  2000] loss: 3.533\n",
      "[epoch 30, minibatch  3000] loss: 3.541\n",
      "[epoch 30, minibatch  4000] loss: 3.588\n",
      "[epoch 30, minibatch  5000] loss: 3.611\n",
      "[epoch 30, minibatch  6000] loss: 3.584\n",
      "[epoch 30, minibatch  7000] loss: 3.546\n",
      "[epoch 30, minibatch  8000] loss: 3.556\n",
      "[epoch 30, minibatch  9000] loss: 3.547\n",
      "[epoch 30, minibatch 10000] loss: 3.553\n",
      "[epoch 30, minibatch 11000] loss: 3.614\n",
      "[epoch 30, minibatch 12000] loss: 3.557\n",
      "[epoch 30, minibatch 13000] loss: 3.588\n",
      "[epoch 30, minibatch 14000] loss: 3.613\n",
      "epoch 30, train loss: 3.568\n",
      "0.0001\n",
      "************ epoch 30, val loss: 12.002 ************\n",
      "[epoch 31, minibatch  1000] loss: 3.464\n",
      "[epoch 31, minibatch  2000] loss: 3.482\n",
      "[epoch 31, minibatch  3000] loss: 3.513\n",
      "[epoch 31, minibatch  4000] loss: 3.516\n",
      "[epoch 31, minibatch  5000] loss: 3.590\n",
      "[epoch 31, minibatch  6000] loss: 3.538\n",
      "[epoch 31, minibatch  7000] loss: 3.506\n",
      "[epoch 31, minibatch  8000] loss: 3.510\n",
      "[epoch 31, minibatch  9000] loss: 3.588\n",
      "[epoch 31, minibatch 10000] loss: 3.542\n",
      "[epoch 31, minibatch 11000] loss: 3.570\n",
      "[epoch 31, minibatch 12000] loss: 3.548\n",
      "[epoch 31, minibatch 13000] loss: 3.531\n",
      "[epoch 31, minibatch 14000] loss: 3.544\n",
      "epoch 31, train loss: 3.533\n",
      "0.0001\n",
      "************ epoch 31, val loss: 12.299 ************\n",
      "[epoch 32, minibatch  1000] loss: 3.430\n",
      "[epoch 32, minibatch  2000] loss: 3.488\n",
      "[epoch 32, minibatch  3000] loss: 3.484\n",
      "[epoch 32, minibatch  4000] loss: 3.522\n",
      "[epoch 32, minibatch  5000] loss: 3.523\n",
      "[epoch 32, minibatch  6000] loss: 3.455\n",
      "[epoch 32, minibatch  7000] loss: 3.526\n",
      "[epoch 32, minibatch  8000] loss: 3.487\n",
      "[epoch 32, minibatch  9000] loss: 3.546\n",
      "[epoch 32, minibatch 10000] loss: 3.511\n",
      "[epoch 32, minibatch 11000] loss: 3.492\n",
      "[epoch 32, minibatch 12000] loss: 3.480\n",
      "[epoch 32, minibatch 13000] loss: 3.540\n",
      "[epoch 32, minibatch 14000] loss: 3.549\n",
      "epoch 32, train loss: 3.502\n",
      "0.0001\n",
      "************ epoch 32, val loss: 12.040 ************\n",
      "[epoch 33, minibatch  1000] loss: 3.424\n",
      "[epoch 33, minibatch  2000] loss: 3.447\n",
      "[epoch 33, minibatch  3000] loss: 3.434\n",
      "[epoch 33, minibatch  4000] loss: 3.464\n",
      "[epoch 33, minibatch  5000] loss: 3.464\n",
      "[epoch 33, minibatch  6000] loss: 3.437\n",
      "[epoch 33, minibatch  7000] loss: 3.508\n",
      "[epoch 33, minibatch  8000] loss: 3.497\n",
      "[epoch 33, minibatch  9000] loss: 3.464\n",
      "[epoch 33, minibatch 10000] loss: 3.480\n",
      "[epoch 33, minibatch 11000] loss: 3.478\n",
      "[epoch 33, minibatch 12000] loss: 3.474\n",
      "[epoch 33, minibatch 13000] loss: 3.495\n",
      "[epoch 33, minibatch 14000] loss: 3.446\n",
      "epoch 33, train loss: 3.467\n",
      "0.0001\n",
      "************ epoch 33, val loss: 12.196 ************\n",
      "[epoch 34, minibatch  1000] loss: 3.386\n",
      "[epoch 34, minibatch  2000] loss: 3.456\n",
      "[epoch 34, minibatch  3000] loss: 3.385\n",
      "[epoch 34, minibatch  4000] loss: 3.396\n",
      "[epoch 34, minibatch  5000] loss: 3.416\n",
      "[epoch 34, minibatch  6000] loss: 3.384\n",
      "[epoch 34, minibatch  7000] loss: 3.475\n",
      "[epoch 34, minibatch  8000] loss: 3.466\n",
      "[epoch 34, minibatch  9000] loss: 3.481\n",
      "[epoch 34, minibatch 10000] loss: 3.414\n",
      "[epoch 34, minibatch 11000] loss: 3.463\n",
      "[epoch 34, minibatch 12000] loss: 3.490\n",
      "[epoch 34, minibatch 13000] loss: 3.521\n",
      "[epoch 34, minibatch 14000] loss: 3.438\n",
      "epoch 34, train loss: 3.439\n",
      "0.0001\n",
      "************ epoch 34, val loss: 12.060 ************\n",
      "[epoch 35, minibatch  1000] loss: 3.358\n",
      "[epoch 35, minibatch  2000] loss: 3.397\n",
      "[epoch 35, minibatch  3000] loss: 3.362\n",
      "[epoch 35, minibatch  4000] loss: 3.384\n",
      "[epoch 35, minibatch  5000] loss: 3.387\n",
      "[epoch 35, minibatch  6000] loss: 3.441\n",
      "[epoch 35, minibatch  7000] loss: 3.391\n",
      "[epoch 35, minibatch  8000] loss: 3.401\n",
      "[epoch 35, minibatch  9000] loss: 3.385\n",
      "[epoch 35, minibatch 10000] loss: 3.403\n",
      "[epoch 35, minibatch 11000] loss: 3.402\n",
      "[epoch 35, minibatch 12000] loss: 3.476\n",
      "[epoch 35, minibatch 13000] loss: 3.445\n",
      "[epoch 35, minibatch 14000] loss: 3.465\n",
      "epoch 35, train loss: 3.405\n",
      "0.0001\n",
      "************ epoch 35, val loss: 11.902 ************\n",
      "[epoch 36, minibatch  1000] loss: 3.328\n",
      "[epoch 36, minibatch  2000] loss: 3.339\n",
      "[epoch 36, minibatch  3000] loss: 3.358\n",
      "[epoch 36, minibatch  4000] loss: 3.361\n",
      "[epoch 36, minibatch  5000] loss: 3.360\n",
      "[epoch 36, minibatch  6000] loss: 3.367\n",
      "[epoch 36, minibatch  7000] loss: 3.428\n",
      "[epoch 36, minibatch  8000] loss: 3.357\n",
      "[epoch 36, minibatch  9000] loss: 3.356\n",
      "[epoch 36, minibatch 10000] loss: 3.414\n",
      "[epoch 36, minibatch 11000] loss: 3.368\n",
      "[epoch 36, minibatch 12000] loss: 3.363\n",
      "[epoch 36, minibatch 13000] loss: 3.396\n",
      "[epoch 36, minibatch 14000] loss: 3.392\n",
      "epoch 36, train loss: 3.372\n",
      "0.0001\n",
      "************ epoch 36, val loss: 12.006 ************\n",
      "[epoch 37, minibatch  1000] loss: 3.328\n",
      "[epoch 37, minibatch  2000] loss: 3.301\n",
      "[epoch 37, minibatch  3000] loss: 3.368\n",
      "[epoch 37, minibatch  4000] loss: 3.325\n",
      "[epoch 37, minibatch  5000] loss: 3.353\n",
      "[epoch 37, minibatch  6000] loss: 3.312\n",
      "[epoch 37, minibatch  7000] loss: 3.349\n",
      "[epoch 37, minibatch  8000] loss: 3.307\n",
      "[epoch 37, minibatch  9000] loss: 3.383\n",
      "[epoch 37, minibatch 10000] loss: 3.328\n",
      "[epoch 37, minibatch 11000] loss: 3.368\n",
      "[epoch 37, minibatch 12000] loss: 3.360\n",
      "[epoch 37, minibatch 13000] loss: 3.362\n",
      "[epoch 37, minibatch 14000] loss: 3.398\n",
      "epoch 37, train loss: 3.347\n",
      "0.0001\n",
      "************ epoch 37, val loss: 11.607 ************\n",
      "[epoch 38, minibatch  1000] loss: 3.257\n",
      "[epoch 38, minibatch  2000] loss: 3.291\n",
      "[epoch 38, minibatch  3000] loss: 3.295\n",
      "[epoch 38, minibatch  4000] loss: 3.300\n",
      "[epoch 38, minibatch  5000] loss: 3.325\n",
      "[epoch 38, minibatch  6000] loss: 3.323\n",
      "[epoch 38, minibatch  7000] loss: 3.338\n",
      "[epoch 38, minibatch  8000] loss: 3.313\n",
      "[epoch 38, minibatch  9000] loss: 3.329\n",
      "[epoch 38, minibatch 10000] loss: 3.354\n",
      "[epoch 38, minibatch 11000] loss: 3.319\n",
      "[epoch 38, minibatch 12000] loss: 3.311\n",
      "[epoch 38, minibatch 13000] loss: 3.344\n",
      "[epoch 38, minibatch 14000] loss: 3.326\n",
      "epoch 38, train loss: 3.316\n",
      "0.0001\n",
      "************ epoch 38, val loss: 11.903 ************\n",
      "[epoch 39, minibatch  1000] loss: 3.254\n",
      "[epoch 39, minibatch  2000] loss: 3.337\n",
      "[epoch 39, minibatch  3000] loss: 3.342\n",
      "[epoch 39, minibatch  4000] loss: 3.295\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch 39, minibatch  5000] loss: 3.316\n",
      "[epoch 39, minibatch  6000] loss: 3.278\n",
      "[epoch 39, minibatch  7000] loss: 3.277\n",
      "[epoch 39, minibatch  8000] loss: 3.298\n",
      "[epoch 39, minibatch  9000] loss: 3.290\n",
      "[epoch 39, minibatch 10000] loss: 3.280\n",
      "[epoch 39, minibatch 11000] loss: 3.313\n",
      "[epoch 39, minibatch 12000] loss: 3.338\n",
      "[epoch 39, minibatch 13000] loss: 3.304\n",
      "[epoch 39, minibatch 14000] loss: 3.301\n",
      "epoch 39, train loss: 3.302\n",
      "0.0001\n",
      "************ epoch 39, val loss: 11.867 ************\n",
      "[epoch 40, minibatch  1000] loss: 3.220\n",
      "[epoch 40, minibatch  2000] loss: 3.242\n",
      "[epoch 40, minibatch  3000] loss: 3.267\n",
      "[epoch 40, minibatch  4000] loss: 3.241\n",
      "[epoch 40, minibatch  5000] loss: 3.293\n",
      "[epoch 40, minibatch  6000] loss: 3.249\n",
      "[epoch 40, minibatch  7000] loss: 3.255\n",
      "[epoch 40, minibatch  8000] loss: 3.293\n",
      "[epoch 40, minibatch  9000] loss: 3.257\n",
      "[epoch 40, minibatch 10000] loss: 3.280\n",
      "[epoch 40, minibatch 11000] loss: 3.247\n",
      "[epoch 40, minibatch 12000] loss: 3.294\n",
      "[epoch 40, minibatch 13000] loss: 3.292\n",
      "[epoch 40, minibatch 14000] loss: 3.285\n",
      "epoch 40, train loss: 3.266\n",
      "0.0001\n",
      "************ epoch 40, val loss: 11.844 ************\n",
      "[epoch 41, minibatch  1000] loss: 3.240\n",
      "[epoch 41, minibatch  2000] loss: 3.225\n",
      "[epoch 41, minibatch  3000] loss: 3.184\n",
      "[epoch 41, minibatch  4000] loss: 3.226\n",
      "[epoch 41, minibatch  5000] loss: 3.237\n",
      "[epoch 41, minibatch  6000] loss: 3.227\n",
      "[epoch 41, minibatch  7000] loss: 3.311\n",
      "[epoch 41, minibatch  8000] loss: 3.241\n",
      "[epoch 41, minibatch  9000] loss: 3.219\n",
      "[epoch 41, minibatch 10000] loss: 3.253\n",
      "[epoch 41, minibatch 11000] loss: 3.266\n",
      "[epoch 41, minibatch 12000] loss: 3.303\n",
      "[epoch 41, minibatch 13000] loss: 3.248\n",
      "[epoch 41, minibatch 14000] loss: 3.301\n",
      "epoch 41, train loss: 3.249\n",
      "0.0001\n",
      "************ epoch 41, val loss: 11.989 ************\n",
      "[epoch 42, minibatch  1000] loss: 3.172\n",
      "[epoch 42, minibatch  2000] loss: 3.281\n",
      "[epoch 42, minibatch  3000] loss: 3.196\n",
      "[epoch 42, minibatch  4000] loss: 3.177\n",
      "[epoch 42, minibatch  5000] loss: 3.243\n",
      "[epoch 42, minibatch  6000] loss: 3.222\n",
      "[epoch 42, minibatch  7000] loss: 3.220\n",
      "[epoch 42, minibatch  8000] loss: 3.226\n",
      "[epoch 42, minibatch  9000] loss: 3.246\n",
      "[epoch 42, minibatch 10000] loss: 3.275\n",
      "[epoch 42, minibatch 11000] loss: 3.257\n",
      "[epoch 42, minibatch 12000] loss: 3.206\n",
      "[epoch 42, minibatch 13000] loss: 3.197\n",
      "[epoch 42, minibatch 14000] loss: 3.378\n",
      "epoch 42, train loss: 3.236\n",
      "0.0001\n",
      "************ epoch 42, val loss: 12.008 ************\n",
      "[epoch 43, minibatch  1000] loss: 3.162\n",
      "[epoch 43, minibatch  2000] loss: 3.174\n",
      "[epoch 43, minibatch  3000] loss: 3.178\n",
      "[epoch 43, minibatch  4000] loss: 3.161\n",
      "[epoch 43, minibatch  5000] loss: 3.251\n",
      "[epoch 43, minibatch  6000] loss: 3.208\n",
      "[epoch 43, minibatch  7000] loss: 3.179\n",
      "[epoch 43, minibatch  8000] loss: 3.205\n",
      "[epoch 43, minibatch  9000] loss: 3.224\n",
      "[epoch 43, minibatch 10000] loss: 3.206\n",
      "[epoch 43, minibatch 11000] loss: 3.190\n",
      "[epoch 43, minibatch 12000] loss: 3.195\n",
      "[epoch 43, minibatch 13000] loss: 3.191\n",
      "[epoch 43, minibatch 14000] loss: 3.224\n",
      "epoch 43, train loss: 3.198\n",
      "0.0001\n",
      "************ epoch 43, val loss: 11.560 ************\n",
      "[epoch 44, minibatch  1000] loss: 3.140\n",
      "[epoch 44, minibatch  2000] loss: 3.165\n",
      "[epoch 44, minibatch  3000] loss: 3.134\n",
      "[epoch 44, minibatch  4000] loss: 3.159\n",
      "[epoch 44, minibatch  5000] loss: 3.156\n",
      "[epoch 44, minibatch  6000] loss: 3.150\n",
      "[epoch 44, minibatch  7000] loss: 3.194\n",
      "[epoch 44, minibatch  8000] loss: 3.166\n",
      "[epoch 44, minibatch  9000] loss: 3.176\n",
      "[epoch 44, minibatch 10000] loss: 3.214\n",
      "[epoch 44, minibatch 11000] loss: 3.173\n",
      "[epoch 44, minibatch 12000] loss: 3.205\n",
      "[epoch 44, minibatch 13000] loss: 3.211\n",
      "[epoch 44, minibatch 14000] loss: 3.187\n",
      "epoch 44, train loss: 3.176\n",
      "0.0001\n",
      "************ epoch 44, val loss: 11.973 ************\n",
      "[epoch 45, minibatch  1000] loss: 3.103\n",
      "[epoch 45, minibatch  2000] loss: 3.113\n",
      "[epoch 45, minibatch  3000] loss: 3.125\n",
      "[epoch 45, minibatch  4000] loss: 3.149\n",
      "[epoch 45, minibatch  5000] loss: 3.149\n",
      "[epoch 45, minibatch  6000] loss: 3.132\n",
      "[epoch 45, minibatch  7000] loss: 3.175\n",
      "[epoch 45, minibatch  8000] loss: 3.285\n",
      "[epoch 45, minibatch  9000] loss: 3.164\n",
      "[epoch 45, minibatch 10000] loss: 3.127\n",
      "[epoch 45, minibatch 11000] loss: 3.187\n",
      "[epoch 45, minibatch 12000] loss: 3.139\n",
      "[epoch 45, minibatch 13000] loss: 3.160\n",
      "[epoch 45, minibatch 14000] loss: 3.192\n",
      "epoch 45, train loss: 3.164\n",
      "0.0001\n",
      "************ epoch 45, val loss: 11.951 ************\n",
      "[epoch 46, minibatch  1000] loss: 3.167\n",
      "[epoch 46, minibatch  2000] loss: 3.119\n",
      "[epoch 46, minibatch  3000] loss: 3.114\n",
      "[epoch 46, minibatch  4000] loss: 3.117\n",
      "[epoch 46, minibatch  5000] loss: 3.142\n",
      "[epoch 46, minibatch  6000] loss: 3.140\n",
      "[epoch 46, minibatch  7000] loss: 3.133\n",
      "[epoch 46, minibatch  8000] loss: 3.159\n",
      "[epoch 46, minibatch  9000] loss: 3.130\n",
      "[epoch 46, minibatch 10000] loss: 3.153\n",
      "[epoch 46, minibatch 11000] loss: 3.146\n",
      "[epoch 46, minibatch 12000] loss: 3.150\n",
      "[epoch 46, minibatch 13000] loss: 3.151\n",
      "[epoch 46, minibatch 14000] loss: 3.226\n",
      "epoch 46, train loss: 3.147\n",
      "0.0001\n",
      "************ epoch 46, val loss: 11.665 ************\n",
      "[epoch 47, minibatch  1000] loss: 3.081\n",
      "[epoch 47, minibatch  2000] loss: 3.080\n",
      "[epoch 47, minibatch  3000] loss: 3.098\n",
      "[epoch 47, minibatch  4000] loss: 3.095\n",
      "[epoch 47, minibatch  5000] loss: 3.099\n",
      "[epoch 47, minibatch  6000] loss: 3.107\n",
      "[epoch 47, minibatch  7000] loss: 3.106\n",
      "[epoch 47, minibatch  8000] loss: 3.126\n",
      "[epoch 47, minibatch  9000] loss: 3.133\n",
      "[epoch 47, minibatch 10000] loss: 3.115\n",
      "[epoch 47, minibatch 11000] loss: 3.139\n",
      "[epoch 47, minibatch 12000] loss: 3.114\n",
      "[epoch 47, minibatch 13000] loss: 3.183\n",
      "[epoch 47, minibatch 14000] loss: 3.141\n",
      "epoch 47, train loss: 3.118\n",
      "0.0001\n",
      "************ epoch 47, val loss: 11.274 ************\n",
      "[epoch 48, minibatch  1000] loss: 3.079\n",
      "[epoch 48, minibatch  2000] loss: 3.063\n",
      "[epoch 48, minibatch  3000] loss: 3.087\n",
      "[epoch 48, minibatch  4000] loss: 3.076\n",
      "[epoch 48, minibatch  5000] loss: 3.083\n",
      "[epoch 48, minibatch  6000] loss: 3.096\n",
      "[epoch 48, minibatch  7000] loss: 3.132\n",
      "[epoch 48, minibatch  8000] loss: 3.093\n",
      "[epoch 48, minibatch  9000] loss: 3.102\n",
      "[epoch 48, minibatch 10000] loss: 3.099\n",
      "[epoch 48, minibatch 11000] loss: 3.127\n",
      "[epoch 48, minibatch 12000] loss: 3.111\n",
      "[epoch 48, minibatch 13000] loss: 3.108\n",
      "[epoch 48, minibatch 14000] loss: 3.125\n",
      "epoch 48, train loss: 3.104\n",
      "0.0001\n",
      "************ epoch 48, val loss: 11.604 ************\n",
      "[epoch 49, minibatch  1000] loss: 3.114\n",
      "[epoch 49, minibatch  2000] loss: 3.039\n",
      "[epoch 49, minibatch  3000] loss: 3.059\n",
      "[epoch 49, minibatch  4000] loss: 3.091\n",
      "[epoch 49, minibatch  5000] loss: 3.078\n",
      "[epoch 49, minibatch  6000] loss: 3.069\n",
      "[epoch 49, minibatch  7000] loss: 3.065\n",
      "[epoch 49, minibatch  8000] loss: 3.111\n",
      "[epoch 49, minibatch  9000] loss: 3.097\n",
      "[epoch 49, minibatch 10000] loss: 3.140\n",
      "[epoch 49, minibatch 11000] loss: 3.061\n",
      "[epoch 49, minibatch 12000] loss: 3.091\n",
      "[epoch 49, minibatch 13000] loss: 3.088\n",
      "[epoch 49, minibatch 14000] loss: 3.100\n",
      "epoch 49, train loss: 3.087\n",
      "0.0001\n",
      "************ epoch 49, val loss: 12.046 ************\n",
      "[epoch 50, minibatch  1000] loss: 3.046\n",
      "[epoch 50, minibatch  2000] loss: 3.065\n",
      "[epoch 50, minibatch  3000] loss: 3.057\n",
      "[epoch 50, minibatch  4000] loss: 3.056\n",
      "[epoch 50, minibatch  5000] loss: 3.031\n",
      "[epoch 50, minibatch  6000] loss: 3.074\n",
      "[epoch 50, minibatch  7000] loss: 3.063\n",
      "[epoch 50, minibatch  8000] loss: 3.095\n",
      "[epoch 50, minibatch  9000] loss: 3.092\n",
      "[epoch 50, minibatch 10000] loss: 3.178\n",
      "[epoch 50, minibatch 11000] loss: 3.057\n",
      "[epoch 50, minibatch 12000] loss: 3.058\n",
      "[epoch 50, minibatch 13000] loss: 3.074\n",
      "[epoch 50, minibatch 14000] loss: 3.079\n",
      "epoch 50, train loss: 3.076\n",
      "0.0001\n",
      "************ epoch 50, val loss: 11.826 ************\n",
      "[epoch 51, minibatch  1000] loss: 3.028\n",
      "[epoch 51, minibatch  2000] loss: 3.034\n",
      "[epoch 51, minibatch  3000] loss: 3.040\n",
      "[epoch 51, minibatch  4000] loss: 3.008\n",
      "[epoch 51, minibatch  5000] loss: 3.046\n",
      "[epoch 51, minibatch  6000] loss: 3.067\n",
      "[epoch 51, minibatch  7000] loss: 3.040\n",
      "[epoch 51, minibatch  8000] loss: 3.056\n",
      "[epoch 51, minibatch  9000] loss: 3.054\n",
      "[epoch 51, minibatch 10000] loss: 3.084\n",
      "[epoch 51, minibatch 11000] loss: 3.044\n",
      "[epoch 51, minibatch 12000] loss: 3.066\n",
      "[epoch 51, minibatch 13000] loss: 3.071\n",
      "[epoch 51, minibatch 14000] loss: 3.056\n",
      "epoch 51, train loss: 3.051\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0001\n",
      "************ epoch 51, val loss: 11.714 ************\n",
      "[epoch 52, minibatch  1000] loss: 3.004\n",
      "[epoch 52, minibatch  2000] loss: 2.993\n",
      "[epoch 52, minibatch  3000] loss: 3.052\n",
      "[epoch 52, minibatch  4000] loss: 3.080\n",
      "[epoch 52, minibatch  5000] loss: 3.025\n",
      "[epoch 52, minibatch  6000] loss: 3.038\n",
      "[epoch 52, minibatch  7000] loss: 3.003\n",
      "[epoch 52, minibatch  8000] loss: 3.020\n",
      "[epoch 52, minibatch  9000] loss: 3.025\n",
      "[epoch 52, minibatch 10000] loss: 3.067\n",
      "[epoch 52, minibatch 11000] loss: 3.060\n",
      "[epoch 52, minibatch 12000] loss: 3.024\n",
      "[epoch 52, minibatch 13000] loss: 3.113\n",
      "[epoch 52, minibatch 14000] loss: 3.054\n",
      "epoch 52, train loss: 3.042\n",
      "0.0001\n",
      "************ epoch 52, val loss: 11.602 ************\n",
      "[epoch 53, minibatch  1000] loss: 3.007\n",
      "[epoch 53, minibatch  2000] loss: 2.995\n",
      "[epoch 53, minibatch  3000] loss: 2.980\n",
      "[epoch 53, minibatch  4000] loss: 3.044\n",
      "[epoch 53, minibatch  5000] loss: 3.014\n",
      "[epoch 53, minibatch  6000] loss: 3.061\n",
      "[epoch 53, minibatch  7000] loss: 3.036\n",
      "[epoch 53, minibatch  8000] loss: 3.021\n",
      "[epoch 53, minibatch  9000] loss: 3.020\n",
      "[epoch 53, minibatch 10000] loss: 3.021\n",
      "[epoch 53, minibatch 11000] loss: 3.020\n",
      "[epoch 53, minibatch 12000] loss: 3.055\n",
      "[epoch 53, minibatch 13000] loss: 3.024\n",
      "[epoch 53, minibatch 14000] loss: 3.024\n",
      "epoch 53, train loss: 3.025\n",
      "0.0001\n",
      "************ epoch 53, val loss: 11.791 ************\n",
      "[epoch 54, minibatch  1000] loss: 3.012\n",
      "[epoch 54, minibatch  2000] loss: 2.988\n",
      "[epoch 54, minibatch  3000] loss: 2.976\n",
      "[epoch 54, minibatch  4000] loss: 2.979\n",
      "[epoch 54, minibatch  5000] loss: 2.979\n",
      "[epoch 54, minibatch  6000] loss: 3.004\n",
      "[epoch 54, minibatch  7000] loss: 3.007\n",
      "[epoch 54, minibatch  8000] loss: 2.985\n",
      "[epoch 54, minibatch  9000] loss: 3.019\n",
      "[epoch 54, minibatch 10000] loss: 3.037\n",
      "[epoch 54, minibatch 11000] loss: 3.034\n",
      "[epoch 54, minibatch 12000] loss: 3.028\n",
      "[epoch 54, minibatch 13000] loss: 3.034\n",
      "[epoch 54, minibatch 14000] loss: 3.022\n",
      "epoch 54, train loss: 3.008\n",
      "0.0001\n",
      "************ epoch 54, val loss: 11.648 ************\n",
      "[epoch 55, minibatch  1000] loss: 2.985\n",
      "[epoch 55, minibatch  2000] loss: 2.951\n",
      "[epoch 55, minibatch  3000] loss: 2.983\n",
      "[epoch 55, minibatch  4000] loss: 3.013\n",
      "[epoch 55, minibatch  5000] loss: 2.973\n",
      "[epoch 55, minibatch  6000] loss: 2.997\n",
      "[epoch 55, minibatch  7000] loss: 2.982\n",
      "[epoch 55, minibatch  8000] loss: 3.018\n",
      "[epoch 55, minibatch  9000] loss: 2.988\n",
      "[epoch 55, minibatch 10000] loss: 2.998\n",
      "[epoch 55, minibatch 11000] loss: 3.032\n",
      "[epoch 55, minibatch 12000] loss: 3.050\n",
      "[epoch 55, minibatch 13000] loss: 2.984\n",
      "[epoch 55, minibatch 14000] loss: 2.991\n",
      "epoch 55, train loss: 2.996\n",
      "0.0001\n",
      "************ epoch 55, val loss: 11.559 ************\n",
      "[epoch 56, minibatch  1000] loss: 2.914\n",
      "[epoch 56, minibatch  2000] loss: 2.967\n",
      "[epoch 56, minibatch  3000] loss: 2.961\n",
      "[epoch 56, minibatch  4000] loss: 2.967\n",
      "[epoch 56, minibatch  5000] loss: 2.971\n",
      "[epoch 56, minibatch  6000] loss: 2.985\n",
      "[epoch 56, minibatch  7000] loss: 3.036\n",
      "[epoch 56, minibatch  8000] loss: 2.980\n",
      "[epoch 56, minibatch  9000] loss: 2.968\n",
      "[epoch 56, minibatch 10000] loss: 2.971\n",
      "[epoch 56, minibatch 11000] loss: 2.978\n",
      "[epoch 56, minibatch 12000] loss: 3.009\n",
      "[epoch 56, minibatch 13000] loss: 3.041\n",
      "[epoch 56, minibatch 14000] loss: 2.979\n",
      "epoch 56, train loss: 2.984\n",
      "0.0001\n",
      "************ epoch 56, val loss: 11.366 ************\n",
      "[epoch 57, minibatch  1000] loss: 2.930\n",
      "[epoch 57, minibatch  2000] loss: 2.940\n",
      "[epoch 57, minibatch  3000] loss: 2.918\n",
      "[epoch 57, minibatch  4000] loss: 2.979\n",
      "[epoch 57, minibatch  5000] loss: 2.977\n",
      "[epoch 57, minibatch  6000] loss: 2.942\n",
      "[epoch 57, minibatch  7000] loss: 2.944\n",
      "[epoch 57, minibatch  8000] loss: 2.990\n",
      "[epoch 57, minibatch  9000] loss: 2.966\n",
      "[epoch 57, minibatch 10000] loss: 3.037\n",
      "[epoch 57, minibatch 11000] loss: 3.045\n",
      "[epoch 57, minibatch 12000] loss: 2.983\n",
      "[epoch 57, minibatch 13000] loss: 2.968\n",
      "[epoch 57, minibatch 14000] loss: 2.975\n",
      "epoch 57, train loss: 2.973\n",
      "0.0001\n",
      "************ epoch 57, val loss: 11.813 ************\n",
      "[epoch 58, minibatch  1000] loss: 2.904\n",
      "[epoch 58, minibatch  2000] loss: 2.959\n",
      "[epoch 58, minibatch  3000] loss: 2.927\n",
      "[epoch 58, minibatch  4000] loss: 2.942\n",
      "[epoch 58, minibatch  5000] loss: 2.930\n",
      "[epoch 58, minibatch  6000] loss: 2.952\n",
      "[epoch 58, minibatch  7000] loss: 2.979\n",
      "[epoch 58, minibatch  8000] loss: 2.966\n",
      "[epoch 58, minibatch  9000] loss: 2.952\n",
      "[epoch 58, minibatch 10000] loss: 2.961\n",
      "[epoch 58, minibatch 11000] loss: 2.971\n",
      "[epoch 58, minibatch 12000] loss: 2.963\n",
      "[epoch 58, minibatch 13000] loss: 2.983\n",
      "[epoch 58, minibatch 14000] loss: 3.010\n",
      "epoch 58, train loss: 2.957\n",
      "0.0001\n",
      "************ epoch 58, val loss: 11.580 ************\n",
      "[epoch 59, minibatch  1000] loss: 2.887\n",
      "[epoch 59, minibatch  2000] loss: 2.933\n",
      "[epoch 59, minibatch  3000] loss: 2.909\n",
      "[epoch 59, minibatch  4000] loss: 2.925\n",
      "[epoch 59, minibatch  5000] loss: 2.983\n",
      "[epoch 59, minibatch  6000] loss: 2.933\n",
      "[epoch 59, minibatch  7000] loss: 2.942\n",
      "[epoch 59, minibatch  8000] loss: 2.989\n",
      "[epoch 59, minibatch  9000] loss: 2.969\n",
      "[epoch 59, minibatch 10000] loss: 2.917\n",
      "[epoch 59, minibatch 11000] loss: 2.940\n",
      "[epoch 59, minibatch 12000] loss: 2.966\n",
      "[epoch 59, minibatch 13000] loss: 2.939\n",
      "[epoch 59, minibatch 14000] loss: 2.965\n",
      "epoch 59, train loss: 2.944\n",
      "0.0001\n",
      "************ epoch 59, val loss: 11.446 ************\n",
      "[epoch 60, minibatch  1000] loss: 2.874\n",
      "[epoch 60, minibatch  2000] loss: 2.907\n",
      "[epoch 60, minibatch  3000] loss: 2.921\n",
      "[epoch 60, minibatch  4000] loss: 2.901\n",
      "[epoch 60, minibatch  5000] loss: 2.954\n",
      "[epoch 60, minibatch  6000] loss: 2.950\n",
      "[epoch 60, minibatch  7000] loss: 2.924\n",
      "[epoch 60, minibatch  8000] loss: 2.941\n",
      "[epoch 60, minibatch  9000] loss: 2.954\n",
      "[epoch 60, minibatch 10000] loss: 2.941\n",
      "[epoch 60, minibatch 11000] loss: 2.971\n",
      "[epoch 60, minibatch 12000] loss: 2.921\n",
      "[epoch 60, minibatch 13000] loss: 2.924\n",
      "[epoch 60, minibatch 14000] loss: 2.993\n",
      "epoch 60, train loss: 2.937\n",
      "0.0001\n",
      "************ epoch 60, val loss: 11.665 ************\n",
      "[epoch 61, minibatch  1000] loss: 2.879\n",
      "[epoch 61, minibatch  2000] loss: 2.884\n",
      "[epoch 61, minibatch  3000] loss: 2.902\n",
      "[epoch 61, minibatch  4000] loss: 2.906\n",
      "[epoch 61, minibatch  5000] loss: 2.937\n",
      "[epoch 61, minibatch  6000] loss: 2.923\n",
      "[epoch 61, minibatch  7000] loss: 2.941\n",
      "[epoch 61, minibatch  8000] loss: 2.907\n",
      "[epoch 61, minibatch  9000] loss: 2.964\n",
      "[epoch 61, minibatch 10000] loss: 2.919\n",
      "[epoch 61, minibatch 11000] loss: 2.909\n",
      "[epoch 61, minibatch 12000] loss: 2.905\n",
      "[epoch 61, minibatch 13000] loss: 3.012\n",
      "[epoch 61, minibatch 14000] loss: 2.952\n",
      "epoch 61, train loss: 2.923\n",
      "0.0001\n",
      "************ epoch 61, val loss: 11.516 ************\n",
      "[epoch 62, minibatch  1000] loss: 2.879\n",
      "[epoch 62, minibatch  2000] loss: 2.887\n",
      "[epoch 62, minibatch  3000] loss: 2.876\n",
      "[epoch 62, minibatch  4000] loss: 2.899\n",
      "[epoch 62, minibatch  5000] loss: 2.866\n",
      "[epoch 62, minibatch  6000] loss: 2.901\n",
      "[epoch 62, minibatch  7000] loss: 2.916\n",
      "[epoch 62, minibatch  8000] loss: 2.900\n",
      "[epoch 62, minibatch  9000] loss: 2.960\n",
      "[epoch 62, minibatch 10000] loss: 2.919\n",
      "[epoch 62, minibatch 11000] loss: 2.916\n",
      "[epoch 62, minibatch 12000] loss: 2.919\n",
      "[epoch 62, minibatch 13000] loss: 2.938\n",
      "[epoch 62, minibatch 14000] loss: 2.935\n",
      "epoch 62, train loss: 2.909\n",
      "0.0001\n",
      "************ epoch 62, val loss: 11.748 ************\n",
      "[epoch 63, minibatch  1000] loss: 2.845\n",
      "[epoch 63, minibatch  2000] loss: 2.860\n",
      "[epoch 63, minibatch  3000] loss: 2.887\n",
      "[epoch 63, minibatch  4000] loss: 2.987\n",
      "[epoch 63, minibatch  5000] loss: 2.896\n",
      "[epoch 63, minibatch  6000] loss: 2.879\n",
      "[epoch 63, minibatch  7000] loss: 2.901\n",
      "[epoch 63, minibatch  8000] loss: 2.954\n",
      "[epoch 63, minibatch  9000] loss: 2.945\n",
      "[epoch 63, minibatch 10000] loss: 2.897\n",
      "[epoch 63, minibatch 11000] loss: 2.952\n",
      "[epoch 63, minibatch 12000] loss: 2.880\n",
      "[epoch 63, minibatch 13000] loss: 2.891\n",
      "[epoch 63, minibatch 14000] loss: 2.885\n",
      "epoch 63, train loss: 2.906\n",
      "0.0001\n",
      "************ epoch 63, val loss: 11.657 ************\n",
      "[epoch 64, minibatch  1000] loss: 2.855\n",
      "[epoch 64, minibatch  2000] loss: 2.838\n",
      "[epoch 64, minibatch  3000] loss: 2.848\n",
      "[epoch 64, minibatch  4000] loss: 2.909\n",
      "[epoch 64, minibatch  5000] loss: 2.929\n",
      "[epoch 64, minibatch  6000] loss: 2.971\n",
      "[epoch 64, minibatch  7000] loss: 2.922\n",
      "[epoch 64, minibatch  8000] loss: 2.857\n",
      "[epoch 64, minibatch  9000] loss: 2.871\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch 64, minibatch 10000] loss: 2.906\n",
      "[epoch 64, minibatch 11000] loss: 2.895\n",
      "[epoch 64, minibatch 12000] loss: 2.896\n",
      "[epoch 64, minibatch 13000] loss: 2.929\n",
      "[epoch 64, minibatch 14000] loss: 2.921\n",
      "epoch 64, train loss: 2.899\n",
      "0.0001\n",
      "************ epoch 64, val loss: 11.447 ************\n",
      "[epoch 65, minibatch  1000] loss: 2.817\n",
      "[epoch 65, minibatch  2000] loss: 2.864\n",
      "[epoch 65, minibatch  3000] loss: 2.907\n",
      "[epoch 65, minibatch  4000] loss: 2.856\n",
      "[epoch 65, minibatch  5000] loss: 2.880\n",
      "[epoch 65, minibatch  6000] loss: 2.854\n",
      "[epoch 65, minibatch  7000] loss: 2.878\n",
      "[epoch 65, minibatch  8000] loss: 2.886\n",
      "[epoch 65, minibatch  9000] loss: 2.857\n",
      "[epoch 65, minibatch 10000] loss: 2.905\n",
      "[epoch 65, minibatch 11000] loss: 2.957\n",
      "[epoch 65, minibatch 12000] loss: 2.960\n",
      "[epoch 65, minibatch 13000] loss: 2.862\n",
      "[epoch 65, minibatch 14000] loss: 2.879\n",
      "epoch 65, train loss: 2.882\n",
      "0.0001\n",
      "************ epoch 65, val loss: 11.100 ************\n",
      "[epoch 66, minibatch  1000] loss: 2.811\n",
      "[epoch 66, minibatch  2000] loss: 2.831\n",
      "[epoch 66, minibatch  3000] loss: 2.850\n",
      "[epoch 66, minibatch  4000] loss: 2.853\n",
      "[epoch 66, minibatch  5000] loss: 2.847\n",
      "[epoch 66, minibatch  6000] loss: 2.851\n",
      "[epoch 66, minibatch  7000] loss: 2.852\n",
      "[epoch 66, minibatch  8000] loss: 2.911\n",
      "[epoch 66, minibatch  9000] loss: 2.845\n",
      "[epoch 66, minibatch 10000] loss: 2.924\n",
      "[epoch 66, minibatch 11000] loss: 2.871\n",
      "[epoch 66, minibatch 12000] loss: 2.904\n",
      "[epoch 66, minibatch 13000] loss: 2.871\n",
      "[epoch 66, minibatch 14000] loss: 2.867\n",
      "epoch 66, train loss: 2.864\n",
      "0.0001\n",
      "************ epoch 66, val loss: 11.585 ************\n",
      "[epoch 67, minibatch  1000] loss: 2.818\n",
      "[epoch 67, minibatch  2000] loss: 2.840\n",
      "[epoch 67, minibatch  3000] loss: 2.830\n",
      "[epoch 67, minibatch  4000] loss: 2.845\n",
      "[epoch 67, minibatch  5000] loss: 2.861\n",
      "[epoch 67, minibatch  6000] loss: 2.826\n",
      "[epoch 67, minibatch  7000] loss: 2.840\n",
      "[epoch 67, minibatch  8000] loss: 2.865\n",
      "[epoch 67, minibatch  9000] loss: 2.857\n",
      "[epoch 67, minibatch 10000] loss: 2.857\n",
      "[epoch 67, minibatch 11000] loss: 2.873\n",
      "[epoch 67, minibatch 12000] loss: 2.864\n",
      "[epoch 67, minibatch 13000] loss: 2.909\n",
      "[epoch 67, minibatch 14000] loss: 2.879\n",
      "epoch 67, train loss: 2.856\n",
      "0.0001\n",
      "************ epoch 67, val loss: 11.416 ************\n",
      "[epoch 68, minibatch  1000] loss: 2.792\n",
      "[epoch 68, minibatch  2000] loss: 2.813\n",
      "[epoch 68, minibatch  3000] loss: 2.836\n",
      "[epoch 68, minibatch  4000] loss: 2.868\n",
      "[epoch 68, minibatch  5000] loss: 2.848\n",
      "[epoch 68, minibatch  6000] loss: 2.838\n",
      "[epoch 68, minibatch  7000] loss: 2.842\n",
      "[epoch 68, minibatch  8000] loss: 2.842\n",
      "[epoch 68, minibatch  9000] loss: 2.828\n",
      "[epoch 68, minibatch 10000] loss: 2.835\n",
      "[epoch 68, minibatch 11000] loss: 2.875\n",
      "[epoch 68, minibatch 12000] loss: 2.920\n",
      "[epoch 68, minibatch 13000] loss: 2.869\n",
      "[epoch 68, minibatch 14000] loss: 3.000\n",
      "epoch 68, train loss: 2.857\n",
      "0.0001\n",
      "************ epoch 68, val loss: 11.478 ************\n",
      "[epoch 69, minibatch  1000] loss: 2.771\n",
      "[epoch 69, minibatch  2000] loss: 2.790\n",
      "[epoch 69, minibatch  3000] loss: 2.833\n",
      "[epoch 69, minibatch  4000] loss: 2.905\n",
      "[epoch 69, minibatch  5000] loss: 2.808\n",
      "[epoch 69, minibatch  6000] loss: 2.845\n",
      "[epoch 69, minibatch  7000] loss: 3.008\n",
      "[epoch 69, minibatch  8000] loss: 2.877\n",
      "[epoch 69, minibatch  9000] loss: 2.814\n",
      "[epoch 69, minibatch 10000] loss: 2.823\n",
      "[epoch 69, minibatch 11000] loss: 2.840\n",
      "[epoch 69, minibatch 12000] loss: 2.841\n",
      "[epoch 69, minibatch 13000] loss: 2.858\n",
      "[epoch 69, minibatch 14000] loss: 2.831\n",
      "epoch 69, train loss: 2.846\n",
      "0.0001\n",
      "************ epoch 69, val loss: 11.691 ************\n",
      "[epoch 70, minibatch  1000] loss: 2.793\n",
      "[epoch 70, minibatch  2000] loss: 2.850\n",
      "[epoch 70, minibatch  3000] loss: 2.827\n",
      "[epoch 70, minibatch  4000] loss: 2.829\n",
      "[epoch 70, minibatch  5000] loss: 2.799\n",
      "[epoch 70, minibatch  6000] loss: 2.868\n",
      "[epoch 70, minibatch  7000] loss: 2.804\n",
      "[epoch 70, minibatch  8000] loss: 2.865\n",
      "[epoch 70, minibatch  9000] loss: 2.822\n",
      "[epoch 70, minibatch 10000] loss: 2.840\n",
      "[epoch 70, minibatch 11000] loss: 2.847\n",
      "[epoch 70, minibatch 12000] loss: 2.833\n",
      "[epoch 70, minibatch 13000] loss: 2.862\n",
      "[epoch 70, minibatch 14000] loss: 2.819\n",
      "epoch 70, train loss: 2.834\n",
      "0.0001\n",
      "************ epoch 70, val loss: 11.540 ************\n",
      "[epoch 71, minibatch  1000] loss: 2.784\n",
      "[epoch 71, minibatch  2000] loss: 2.795\n",
      "[epoch 71, minibatch  3000] loss: 2.797\n",
      "[epoch 71, minibatch  4000] loss: 2.828\n",
      "[epoch 71, minibatch  5000] loss: 2.823\n",
      "[epoch 71, minibatch  6000] loss: 2.797\n",
      "[epoch 71, minibatch  7000] loss: 2.861\n",
      "[epoch 71, minibatch  8000] loss: 2.833\n",
      "[epoch 71, minibatch  9000] loss: 2.810\n",
      "[epoch 71, minibatch 10000] loss: 2.848\n",
      "[epoch 71, minibatch 11000] loss: 2.829\n",
      "[epoch 71, minibatch 12000] loss: 2.825\n",
      "[epoch 71, minibatch 13000] loss: 2.826\n",
      "[epoch 71, minibatch 14000] loss: 2.819\n",
      "epoch 71, train loss: 2.820\n",
      "0.0001\n",
      "************ epoch 71, val loss: 11.466 ************\n",
      "[epoch 72, minibatch  1000] loss: 2.787\n",
      "[epoch 72, minibatch  2000] loss: 2.775\n",
      "[epoch 72, minibatch  3000] loss: 2.779\n",
      "[epoch 72, minibatch  4000] loss: 2.791\n",
      "[epoch 72, minibatch  5000] loss: 2.788\n",
      "[epoch 72, minibatch  6000] loss: 2.887\n",
      "[epoch 72, minibatch  7000] loss: 2.832\n",
      "[epoch 72, minibatch  8000] loss: 2.786\n",
      "[epoch 72, minibatch  9000] loss: 2.819\n",
      "[epoch 72, minibatch 10000] loss: 2.803\n",
      "[epoch 72, minibatch 11000] loss: 2.812\n",
      "[epoch 72, minibatch 12000] loss: 2.820\n",
      "[epoch 72, minibatch 13000] loss: 2.817\n",
      "[epoch 72, minibatch 14000] loss: 2.833\n",
      "epoch 72, train loss: 2.811\n",
      "0.0001\n",
      "************ epoch 72, val loss: 11.720 ************\n",
      "[epoch 73, minibatch  1000] loss: 2.781\n",
      "[epoch 73, minibatch  2000] loss: 2.774\n",
      "[epoch 73, minibatch  3000] loss: 2.768\n",
      "[epoch 73, minibatch  4000] loss: 2.788\n",
      "[epoch 73, minibatch  5000] loss: 2.894\n",
      "[epoch 73, minibatch  6000] loss: 2.799\n",
      "[epoch 73, minibatch  7000] loss: 2.847\n",
      "[epoch 73, minibatch  8000] loss: 2.789\n",
      "[epoch 73, minibatch  9000] loss: 2.785\n",
      "[epoch 73, minibatch 10000] loss: 2.847\n",
      "[epoch 73, minibatch 11000] loss: 2.780\n",
      "[epoch 73, minibatch 12000] loss: 2.812\n",
      "[epoch 73, minibatch 13000] loss: 2.827\n",
      "[epoch 73, minibatch 14000] loss: 2.815\n",
      "epoch 73, train loss: 2.812\n",
      "0.0001\n",
      "************ epoch 73, val loss: 11.562 ************\n",
      "[epoch 74, minibatch  1000] loss: 2.781\n",
      "[epoch 74, minibatch  2000] loss: 2.758\n",
      "[epoch 74, minibatch  3000] loss: 2.759\n",
      "[epoch 74, minibatch  4000] loss: 2.770\n",
      "[epoch 74, minibatch  5000] loss: 2.770\n",
      "[epoch 74, minibatch  6000] loss: 2.791\n",
      "[epoch 74, minibatch  7000] loss: 2.795\n",
      "[epoch 74, minibatch  8000] loss: 2.861\n",
      "[epoch 74, minibatch  9000] loss: 2.818\n",
      "[epoch 74, minibatch 10000] loss: 2.790\n",
      "[epoch 74, minibatch 11000] loss: 2.838\n",
      "[epoch 74, minibatch 12000] loss: 2.811\n",
      "[epoch 74, minibatch 13000] loss: 2.796\n",
      "[epoch 74, minibatch 14000] loss: 2.792\n",
      "epoch 74, train loss: 2.796\n",
      "0.0001\n",
      "************ epoch 74, val loss: 11.531 ************\n",
      "[epoch 75, minibatch  1000] loss: 2.765\n",
      "[epoch 75, minibatch  2000] loss: 2.921\n",
      "[epoch 75, minibatch  3000] loss: 2.756\n",
      "[epoch 75, minibatch  4000] loss: 2.763\n",
      "[epoch 75, minibatch  5000] loss: 2.750\n",
      "[epoch 75, minibatch  6000] loss: 2.776\n",
      "[epoch 75, minibatch  7000] loss: 2.783\n",
      "[epoch 75, minibatch  8000] loss: 2.787\n",
      "[epoch 75, minibatch  9000] loss: 2.845\n",
      "[epoch 75, minibatch 10000] loss: 2.826\n",
      "[epoch 75, minibatch 11000] loss: 2.875\n",
      "[epoch 75, minibatch 12000] loss: 2.765\n",
      "[epoch 75, minibatch 13000] loss: 2.775\n",
      "[epoch 75, minibatch 14000] loss: 2.769\n",
      "epoch 75, train loss: 2.798\n",
      "0.0001\n",
      "************ epoch 75, val loss: 12.150 ************\n",
      "[epoch 76, minibatch  1000] loss: 2.739\n",
      "[epoch 76, minibatch  2000] loss: 2.812\n",
      "[epoch 76, minibatch  3000] loss: 2.762\n",
      "[epoch 76, minibatch  4000] loss: 2.743\n",
      "[epoch 76, minibatch  5000] loss: 2.773\n",
      "[epoch 76, minibatch  6000] loss: 2.773\n",
      "[epoch 76, minibatch  7000] loss: 2.780\n",
      "[epoch 76, minibatch  8000] loss: 2.781\n",
      "[epoch 76, minibatch  9000] loss: 2.835\n",
      "[epoch 76, minibatch 10000] loss: 2.799\n",
      "[epoch 76, minibatch 11000] loss: 2.788\n",
      "[epoch 76, minibatch 12000] loss: 2.805\n",
      "[epoch 76, minibatch 13000] loss: 2.767\n",
      "[epoch 76, minibatch 14000] loss: 2.801\n",
      "epoch 76, train loss: 2.784\n",
      "0.0001\n",
      "************ epoch 76, val loss: 11.369 ************\n",
      "[epoch 77, minibatch  1000] loss: 2.745\n",
      "[epoch 77, minibatch  2000] loss: 2.747\n",
      "[epoch 77, minibatch  3000] loss: 2.734\n",
      "[epoch 77, minibatch  4000] loss: 2.814\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch 77, minibatch  5000] loss: 2.747\n",
      "[epoch 77, minibatch  6000] loss: 2.750\n",
      "[epoch 77, minibatch  7000] loss: 2.764\n",
      "[epoch 77, minibatch  8000] loss: 2.779\n",
      "[epoch 77, minibatch  9000] loss: 2.814\n",
      "[epoch 77, minibatch 10000] loss: 2.765\n",
      "[epoch 77, minibatch 11000] loss: 2.781\n",
      "[epoch 77, minibatch 12000] loss: 2.805\n",
      "[epoch 77, minibatch 13000] loss: 2.799\n",
      "[epoch 77, minibatch 14000] loss: 2.788\n",
      "epoch 77, train loss: 2.776\n",
      "0.0001\n",
      "************ epoch 77, val loss: 11.838 ************\n",
      "[epoch 78, minibatch  1000] loss: 2.761\n",
      "[epoch 78, minibatch  2000] loss: 2.749\n",
      "[epoch 78, minibatch  3000] loss: 2.724\n",
      "[epoch 78, minibatch  4000] loss: 2.750\n",
      "[epoch 78, minibatch  5000] loss: 2.751\n",
      "[epoch 78, minibatch  6000] loss: 2.784\n",
      "[epoch 78, minibatch  7000] loss: 2.737\n",
      "[epoch 78, minibatch  8000] loss: 2.738\n",
      "[epoch 78, minibatch  9000] loss: 2.770\n",
      "[epoch 78, minibatch 10000] loss: 2.791\n",
      "[epoch 78, minibatch 11000] loss: 2.756\n",
      "[epoch 78, minibatch 12000] loss: 2.771\n",
      "[epoch 78, minibatch 13000] loss: 2.817\n",
      "[epoch 78, minibatch 14000] loss: 2.794\n",
      "epoch 78, train loss: 2.763\n",
      "0.0001\n",
      "************ epoch 78, val loss: 11.497 ************\n",
      "[epoch 79, minibatch  1000] loss: 2.704\n",
      "[epoch 79, minibatch  2000] loss: 2.749\n",
      "[epoch 79, minibatch  3000] loss: 2.753\n",
      "[epoch 79, minibatch  4000] loss: 2.749\n",
      "[epoch 79, minibatch  5000] loss: 2.763\n",
      "[epoch 79, minibatch  6000] loss: 2.748\n",
      "[epoch 79, minibatch  7000] loss: 2.752\n",
      "[epoch 79, minibatch  8000] loss: 2.745\n",
      "[epoch 79, minibatch  9000] loss: 2.776\n",
      "[epoch 79, minibatch 10000] loss: 2.758\n",
      "[epoch 79, minibatch 11000] loss: 2.942\n",
      "[epoch 79, minibatch 12000] loss: 2.745\n",
      "[epoch 79, minibatch 13000] loss: 2.772\n",
      "[epoch 79, minibatch 14000] loss: 2.764\n",
      "epoch 79, train loss: 2.765\n",
      "0.0001\n",
      "************ epoch 79, val loss: 11.209 ************\n",
      "[epoch 80, minibatch  1000] loss: 2.701\n",
      "[epoch 80, minibatch  2000] loss: 2.720\n",
      "[epoch 80, minibatch  3000] loss: 2.712\n",
      "[epoch 80, minibatch  4000] loss: 2.747\n",
      "[epoch 80, minibatch  5000] loss: 2.738\n",
      "[epoch 80, minibatch  6000] loss: 2.754\n",
      "[epoch 80, minibatch  7000] loss: 2.741\n",
      "[epoch 80, minibatch  8000] loss: 2.730\n",
      "[epoch 80, minibatch  9000] loss: 2.753\n",
      "[epoch 80, minibatch 10000] loss: 2.849\n",
      "[epoch 80, minibatch 11000] loss: 2.748\n",
      "[epoch 80, minibatch 12000] loss: 2.746\n",
      "[epoch 80, minibatch 13000] loss: 2.748\n",
      "[epoch 80, minibatch 14000] loss: 2.793\n",
      "epoch 80, train loss: 2.750\n",
      "0.0001\n",
      "************ epoch 80, val loss: 11.538 ************\n",
      "[epoch 81, minibatch  1000] loss: 2.703\n",
      "[epoch 81, minibatch  2000] loss: 2.733\n",
      "[epoch 81, minibatch  3000] loss: 2.747\n",
      "[epoch 81, minibatch  4000] loss: 2.704\n",
      "[epoch 81, minibatch  5000] loss: 2.715\n",
      "[epoch 81, minibatch  6000] loss: 2.742\n",
      "[epoch 81, minibatch  7000] loss: 2.857\n",
      "[epoch 81, minibatch  8000] loss: 2.731\n",
      "[epoch 81, minibatch  9000] loss: 2.741\n",
      "[epoch 81, minibatch 10000] loss: 2.728\n",
      "[epoch 81, minibatch 11000] loss: 2.751\n",
      "[epoch 81, minibatch 12000] loss: 2.749\n",
      "[epoch 81, minibatch 13000] loss: 2.735\n",
      "[epoch 81, minibatch 14000] loss: 2.825\n",
      "epoch 81, train loss: 2.746\n",
      "0.0001\n",
      "************ epoch 81, val loss: 11.357 ************\n",
      "[epoch 82, minibatch  1000] loss: 2.707\n",
      "[epoch 82, minibatch  2000] loss: 2.717\n",
      "[epoch 82, minibatch  3000] loss: 2.680\n",
      "[epoch 82, minibatch  4000] loss: 2.776\n",
      "[epoch 82, minibatch  5000] loss: 2.771\n",
      "[epoch 82, minibatch  6000] loss: 2.734\n",
      "[epoch 82, minibatch  7000] loss: 2.725\n",
      "[epoch 82, minibatch  8000] loss: 2.783\n",
      "[epoch 82, minibatch  9000] loss: 2.706\n",
      "[epoch 82, minibatch 10000] loss: 2.777\n",
      "[epoch 82, minibatch 11000] loss: 2.765\n",
      "[epoch 82, minibatch 12000] loss: 2.769\n",
      "[epoch 82, minibatch 13000] loss: 3.022\n",
      "[epoch 82, minibatch 14000] loss: 2.734\n",
      "epoch 82, train loss: 2.759\n",
      "0.0001\n",
      "************ epoch 82, val loss: 11.161 ************\n",
      "[epoch 83, minibatch  1000] loss: 2.675\n",
      "[epoch 83, minibatch  2000] loss: 2.724\n",
      "[epoch 83, minibatch  3000] loss: 2.704\n",
      "[epoch 83, minibatch  4000] loss: 2.711\n",
      "[epoch 83, minibatch  5000] loss: 2.728\n",
      "[epoch 83, minibatch  6000] loss: 2.723\n",
      "[epoch 83, minibatch  7000] loss: 2.777\n",
      "[epoch 83, minibatch  8000] loss: 2.726\n",
      "[epoch 83, minibatch  9000] loss: 2.732\n",
      "[epoch 83, minibatch 10000] loss: 2.783\n",
      "[epoch 83, minibatch 11000] loss: 2.819\n",
      "[epoch 83, minibatch 12000] loss: 2.753\n",
      "[epoch 83, minibatch 13000] loss: 2.765\n",
      "[epoch 83, minibatch 14000] loss: 2.779\n",
      "epoch 83, train loss: 2.745\n",
      "0.0001\n",
      "************ epoch 83, val loss: 11.706 ************\n",
      "[epoch 84, minibatch  1000] loss: 2.811\n",
      "[epoch 84, minibatch  2000] loss: 2.677\n",
      "[epoch 84, minibatch  3000] loss: 2.691\n",
      "[epoch 84, minibatch  4000] loss: 2.718\n",
      "[epoch 84, minibatch  5000] loss: 2.701\n",
      "[epoch 84, minibatch  6000] loss: 2.726\n",
      "[epoch 84, minibatch  7000] loss: 2.913\n",
      "[epoch 84, minibatch  8000] loss: 2.725\n",
      "[epoch 84, minibatch  9000] loss: 2.694\n",
      "[epoch 84, minibatch 10000] loss: 2.708\n",
      "[epoch 84, minibatch 11000] loss: 2.742\n",
      "[epoch 84, minibatch 12000] loss: 2.786\n",
      "[epoch 84, minibatch 13000] loss: 2.735\n",
      "[epoch 84, minibatch 14000] loss: 2.729\n",
      "epoch 84, train loss: 2.740\n",
      "0.0001\n",
      "************ epoch 84, val loss: 11.309 ************\n",
      "[epoch 85, minibatch  1000] loss: 2.660\n",
      "[epoch 85, minibatch  2000] loss: 2.764\n",
      "[epoch 85, minibatch  3000] loss: 2.702\n",
      "[epoch 85, minibatch  4000] loss: 2.695\n",
      "[epoch 85, minibatch  5000] loss: 2.683\n",
      "[epoch 85, minibatch  6000] loss: 2.792\n",
      "[epoch 85, minibatch  7000] loss: 2.732\n",
      "[epoch 85, minibatch  8000] loss: 2.709\n",
      "[epoch 85, minibatch  9000] loss: 2.728\n",
      "[epoch 85, minibatch 10000] loss: 2.711\n",
      "[epoch 85, minibatch 11000] loss: 2.749\n",
      "[epoch 85, minibatch 12000] loss: 2.719\n",
      "[epoch 85, minibatch 13000] loss: 2.778\n",
      "[epoch 85, minibatch 14000] loss: 2.717\n",
      "epoch 85, train loss: 2.724\n",
      "0.0001\n",
      "************ epoch 85, val loss: 11.565 ************\n",
      "[epoch 86, minibatch  1000] loss: 2.691\n",
      "[epoch 86, minibatch  2000] loss: 2.671\n",
      "[epoch 86, minibatch  3000] loss: 2.671\n",
      "[epoch 86, minibatch  4000] loss: 2.676\n",
      "[epoch 86, minibatch  5000] loss: 2.724\n",
      "[epoch 86, minibatch  6000] loss: 2.703\n",
      "[epoch 86, minibatch  7000] loss: 2.728\n",
      "[epoch 86, minibatch  8000] loss: 2.707\n",
      "[epoch 86, minibatch  9000] loss: 2.697\n",
      "[epoch 86, minibatch 10000] loss: 2.690\n",
      "[epoch 86, minibatch 11000] loss: 2.895\n",
      "[epoch 86, minibatch 12000] loss: 2.775\n",
      "[epoch 86, minibatch 13000] loss: 2.685\n",
      "[epoch 86, minibatch 14000] loss: 2.742\n",
      "epoch 86, train loss: 2.720\n",
      "0.0001\n",
      "************ epoch 86, val loss: 11.335 ************\n",
      "[epoch 87, minibatch  1000] loss: 2.678\n",
      "[epoch 87, minibatch  2000] loss: 2.678\n",
      "[epoch 87, minibatch  3000] loss: 2.676\n",
      "[epoch 87, minibatch  4000] loss: 2.722\n",
      "[epoch 87, minibatch  5000] loss: 2.696\n",
      "[epoch 87, minibatch  6000] loss: 2.698\n",
      "[epoch 87, minibatch  7000] loss: 2.706\n",
      "[epoch 87, minibatch  8000] loss: 2.692\n",
      "[epoch 87, minibatch  9000] loss: 2.691\n",
      "[epoch 87, minibatch 10000] loss: 2.705\n",
      "[epoch 87, minibatch 11000] loss: 2.711\n",
      "[epoch 87, minibatch 12000] loss: 2.714\n",
      "[epoch 87, minibatch 13000] loss: 2.794\n",
      "[epoch 87, minibatch 14000] loss: 2.700\n",
      "epoch 87, train loss: 2.705\n",
      "0.0001\n",
      "************ epoch 87, val loss: 11.348 ************\n",
      "[epoch 88, minibatch  1000] loss: 2.655\n",
      "[epoch 88, minibatch  2000] loss: 2.660\n",
      "[epoch 88, minibatch  3000] loss: 2.709\n",
      "[epoch 88, minibatch  4000] loss: 2.666\n",
      "[epoch 88, minibatch  5000] loss: 2.705\n",
      "[epoch 88, minibatch  6000] loss: 2.693\n",
      "[epoch 88, minibatch  7000] loss: 2.718\n",
      "[epoch 88, minibatch  8000] loss: 2.708\n",
      "[epoch 88, minibatch  9000] loss: 2.727\n",
      "[epoch 88, minibatch 10000] loss: 2.687\n",
      "[epoch 88, minibatch 11000] loss: 2.721\n",
      "[epoch 88, minibatch 12000] loss: 2.713\n",
      "[epoch 88, minibatch 13000] loss: 2.685\n",
      "[epoch 88, minibatch 14000] loss: 2.788\n",
      "epoch 88, train loss: 2.704\n",
      "0.0001\n",
      "************ epoch 88, val loss: 11.381 ************\n",
      "[epoch 89, minibatch  1000] loss: 2.648\n",
      "[epoch 89, minibatch  2000] loss: 2.658\n",
      "[epoch 89, minibatch  3000] loss: 2.716\n",
      "[epoch 89, minibatch  4000] loss: 2.706\n",
      "[epoch 89, minibatch  5000] loss: 2.669\n",
      "[epoch 89, minibatch  6000] loss: 2.672\n",
      "[epoch 89, minibatch  7000] loss: 2.663\n",
      "[epoch 89, minibatch  8000] loss: 2.688\n",
      "[epoch 89, minibatch  9000] loss: 2.689\n",
      "[epoch 89, minibatch 10000] loss: 2.720\n",
      "[epoch 89, minibatch 11000] loss: 2.709\n",
      "[epoch 89, minibatch 12000] loss: 2.709\n",
      "[epoch 89, minibatch 13000] loss: 2.704\n",
      "[epoch 89, minibatch 14000] loss: 2.693\n",
      "epoch 89, train loss: 2.690\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0001\n",
      "************ epoch 89, val loss: 11.380 ************\n",
      "[epoch 90, minibatch  1000] loss: 2.653\n",
      "[epoch 90, minibatch  2000] loss: 2.645\n",
      "[epoch 90, minibatch  3000] loss: 2.692\n",
      "[epoch 90, minibatch  4000] loss: 2.698\n",
      "[epoch 90, minibatch  5000] loss: 2.655\n",
      "[epoch 90, minibatch  6000] loss: 2.691\n",
      "[epoch 90, minibatch  7000] loss: 2.673\n",
      "[epoch 90, minibatch  8000] loss: 2.706\n",
      "[epoch 90, minibatch  9000] loss: 2.669\n",
      "[epoch 90, minibatch 10000] loss: 2.803\n",
      "[epoch 90, minibatch 11000] loss: 2.690\n",
      "[epoch 90, minibatch 12000] loss: 2.688\n",
      "[epoch 90, minibatch 13000] loss: 2.677\n",
      "[epoch 90, minibatch 14000] loss: 2.782\n",
      "epoch 90, train loss: 2.695\n",
      "0.0001\n",
      "************ epoch 90, val loss: 11.358 ************\n",
      "[epoch 91, minibatch  1000] loss: 2.630\n",
      "[epoch 91, minibatch  2000] loss: 2.665\n",
      "[epoch 91, minibatch  3000] loss: 2.693\n",
      "[epoch 91, minibatch  4000] loss: 2.660\n",
      "[epoch 91, minibatch  5000] loss: 2.664\n",
      "[epoch 91, minibatch  6000] loss: 2.710\n",
      "[epoch 91, minibatch  7000] loss: 2.674\n",
      "[epoch 91, minibatch  8000] loss: 2.692\n",
      "[epoch 91, minibatch  9000] loss: 2.664\n",
      "[epoch 91, minibatch 10000] loss: 2.725\n",
      "[epoch 91, minibatch 11000] loss: 2.679\n",
      "[epoch 91, minibatch 12000] loss: 2.683\n",
      "[epoch 91, minibatch 13000] loss: 2.716\n",
      "[epoch 91, minibatch 14000] loss: 2.722\n",
      "epoch 91, train loss: 2.686\n",
      "0.0001\n",
      "************ epoch 91, val loss: 11.167 ************\n",
      "[epoch 92, minibatch  1000] loss: 2.635\n",
      "[epoch 92, minibatch  2000] loss: 2.631\n",
      "[epoch 92, minibatch  3000] loss: 2.731\n",
      "[epoch 92, minibatch  4000] loss: 2.659\n",
      "[epoch 92, minibatch  5000] loss: 2.662\n",
      "[epoch 92, minibatch  6000] loss: 2.661\n",
      "[epoch 92, minibatch  7000] loss: 2.682\n",
      "[epoch 92, minibatch  8000] loss: 2.661\n",
      "[epoch 92, minibatch  9000] loss: 2.676\n",
      "[epoch 92, minibatch 10000] loss: 2.725\n",
      "[epoch 92, minibatch 11000] loss: 2.671\n",
      "[epoch 92, minibatch 12000] loss: 2.673\n",
      "[epoch 92, minibatch 13000] loss: 2.687\n",
      "[epoch 92, minibatch 14000] loss: 2.702\n",
      "epoch 92, train loss: 2.675\n",
      "0.0001\n",
      "************ epoch 92, val loss: 11.316 ************\n",
      "[epoch 93, minibatch  1000] loss: 2.638\n",
      "[epoch 93, minibatch  2000] loss: 2.629\n",
      "[epoch 93, minibatch  3000] loss: 2.654\n",
      "[epoch 93, minibatch  4000] loss: 2.721\n",
      "[epoch 93, minibatch  5000] loss: 2.687\n",
      "[epoch 93, minibatch  6000] loss: 2.651\n",
      "[epoch 93, minibatch  7000] loss: 2.653\n",
      "[epoch 93, minibatch  8000] loss: 2.701\n",
      "[epoch 93, minibatch  9000] loss: 2.667\n",
      "[epoch 93, minibatch 10000] loss: 2.675\n",
      "[epoch 93, minibatch 11000] loss: 2.680\n",
      "[epoch 93, minibatch 12000] loss: 2.678\n",
      "[epoch 93, minibatch 13000] loss: 2.661\n",
      "[epoch 93, minibatch 14000] loss: 2.759\n",
      "epoch 93, train loss: 2.676\n",
      "0.0001\n",
      "************ epoch 93, val loss: 11.328 ************\n",
      "[epoch 94, minibatch  1000] loss: 2.613\n",
      "[epoch 94, minibatch  2000] loss: 2.668\n",
      "[epoch 94, minibatch  3000] loss: 2.636\n",
      "[epoch 94, minibatch  4000] loss: 2.735\n",
      "[epoch 94, minibatch  5000] loss: 2.648\n",
      "[epoch 94, minibatch  6000] loss: 2.639\n",
      "[epoch 94, minibatch  7000] loss: 2.668\n",
      "[epoch 94, minibatch  8000] loss: 2.719\n",
      "[epoch 94, minibatch  9000] loss: 2.694\n",
      "[epoch 94, minibatch 10000] loss: 2.667\n",
      "[epoch 94, minibatch 11000] loss: 2.661\n",
      "[epoch 94, minibatch 12000] loss: 2.664\n",
      "[epoch 94, minibatch 13000] loss: 2.656\n",
      "[epoch 94, minibatch 14000] loss: 2.774\n",
      "epoch 94, train loss: 2.676\n",
      "0.0001\n",
      "************ epoch 94, val loss: 11.366 ************\n",
      "[epoch 95, minibatch  1000] loss: 2.642\n",
      "[epoch 95, minibatch  2000] loss: 2.618\n",
      "[epoch 95, minibatch  3000] loss: 2.659\n",
      "[epoch 95, minibatch  4000] loss: 2.643\n",
      "[epoch 95, minibatch  5000] loss: 2.660\n",
      "[epoch 95, minibatch  6000] loss: 2.735\n",
      "[epoch 95, minibatch  7000] loss: 2.645\n",
      "[epoch 95, minibatch  8000] loss: 2.642\n",
      "[epoch 95, minibatch  9000] loss: 2.648\n",
      "[epoch 95, minibatch 10000] loss: 2.664\n",
      "[epoch 95, minibatch 11000] loss: 2.666\n",
      "[epoch 95, minibatch 12000] loss: 2.644\n",
      "[epoch 95, minibatch 13000] loss: 2.660\n",
      "[epoch 95, minibatch 14000] loss: 2.692\n",
      "epoch 95, train loss: 2.657\n",
      "0.0001\n",
      "************ epoch 95, val loss: 11.627 ************\n",
      "[epoch 96, minibatch  1000] loss: 2.629\n",
      "[epoch 96, minibatch  2000] loss: 2.640\n",
      "[epoch 96, minibatch  3000] loss: 2.613\n",
      "[epoch 96, minibatch  4000] loss: 2.622\n",
      "[epoch 96, minibatch  5000] loss: 2.718\n",
      "[epoch 96, minibatch  6000] loss: 2.653\n",
      "[epoch 96, minibatch  7000] loss: 2.631\n",
      "[epoch 96, minibatch  8000] loss: 2.699\n",
      "[epoch 96, minibatch  9000] loss: 2.640\n",
      "[epoch 96, minibatch 10000] loss: 2.655\n",
      "[epoch 96, minibatch 11000] loss: 2.696\n",
      "[epoch 96, minibatch 12000] loss: 2.667\n",
      "[epoch 96, minibatch 13000] loss: 2.643\n",
      "[epoch 96, minibatch 14000] loss: 2.662\n",
      "epoch 96, train loss: 2.654\n",
      "0.0001\n",
      "************ epoch 96, val loss: 11.241 ************\n",
      "[epoch 97, minibatch  1000] loss: 2.644\n",
      "[epoch 97, minibatch  2000] loss: 2.608\n",
      "[epoch 97, minibatch  3000] loss: 2.616\n",
      "[epoch 97, minibatch  4000] loss: 2.655\n",
      "[epoch 97, minibatch  5000] loss: 2.633\n",
      "[epoch 97, minibatch  6000] loss: 2.644\n",
      "[epoch 97, minibatch  7000] loss: 2.651\n",
      "[epoch 97, minibatch  8000] loss: 2.651\n",
      "[epoch 97, minibatch  9000] loss: 2.646\n",
      "[epoch 97, minibatch 10000] loss: 2.672\n",
      "[epoch 97, minibatch 11000] loss: 2.654\n",
      "[epoch 97, minibatch 12000] loss: 2.662\n",
      "[epoch 97, minibatch 13000] loss: 2.650\n",
      "[epoch 97, minibatch 14000] loss: 2.681\n",
      "epoch 97, train loss: 2.649\n",
      "0.0001\n",
      "************ epoch 97, val loss: 11.223 ************\n",
      "[epoch 98, minibatch  1000] loss: 2.609\n",
      "[epoch 98, minibatch  2000] loss: 2.624\n",
      "[epoch 98, minibatch  3000] loss: 2.630\n",
      "[epoch 98, minibatch  4000] loss: 2.642\n",
      "[epoch 98, minibatch  5000] loss: 2.637\n",
      "[epoch 98, minibatch  6000] loss: 2.620\n",
      "[epoch 98, minibatch  7000] loss: 2.661\n",
      "[epoch 98, minibatch  8000] loss: 2.940\n",
      "[epoch 98, minibatch  9000] loss: 2.651\n",
      "[epoch 98, minibatch 10000] loss: 2.630\n",
      "[epoch 98, minibatch 11000] loss: 2.614\n",
      "[epoch 98, minibatch 12000] loss: 2.639\n",
      "[epoch 98, minibatch 13000] loss: 2.648\n",
      "[epoch 98, minibatch 14000] loss: 2.648\n",
      "epoch 98, train loss: 2.659\n",
      "0.0001\n",
      "************ epoch 98, val loss: 11.238 ************\n",
      "[epoch 99, minibatch  1000] loss: 2.624\n",
      "[epoch 99, minibatch  2000] loss: 2.625\n",
      "[epoch 99, minibatch  3000] loss: 2.623\n",
      "[epoch 99, minibatch  4000] loss: 2.620\n",
      "[epoch 99, minibatch  5000] loss: 2.640\n",
      "[epoch 99, minibatch  6000] loss: 2.626\n",
      "[epoch 99, minibatch  7000] loss: 2.641\n",
      "[epoch 99, minibatch  8000] loss: 2.632\n",
      "[epoch 99, minibatch  9000] loss: 2.638\n",
      "[epoch 99, minibatch 10000] loss: 2.663\n",
      "[epoch 99, minibatch 11000] loss: 2.628\n",
      "[epoch 99, minibatch 12000] loss: 2.630\n",
      "[epoch 99, minibatch 13000] loss: 2.692\n",
      "[epoch 99, minibatch 14000] loss: 2.633\n",
      "epoch 99, train loss: 2.641\n",
      "0.0001\n",
      "************ epoch 99, val loss: 11.479 ************\n",
      "[epoch 100, minibatch  1000] loss: 2.588\n",
      "[epoch 100, minibatch  2000] loss: 2.619\n",
      "[epoch 100, minibatch  3000] loss: 2.638\n",
      "[epoch 100, minibatch  4000] loss: 2.607\n",
      "[epoch 100, minibatch  5000] loss: 2.652\n",
      "[epoch 100, minibatch  6000] loss: 2.611\n",
      "[epoch 100, minibatch  7000] loss: 2.599\n",
      "[epoch 100, minibatch  8000] loss: 2.675\n",
      "[epoch 100, minibatch  9000] loss: 2.611\n",
      "[epoch 100, minibatch 10000] loss: 2.705\n",
      "[epoch 100, minibatch 11000] loss: 2.619\n",
      "[epoch 100, minibatch 12000] loss: 2.616\n",
      "[epoch 100, minibatch 13000] loss: 2.641\n",
      "[epoch 100, minibatch 14000] loss: 2.660\n",
      "epoch 100, train loss: 2.632\n",
      "0.0001\n",
      "************ epoch 100, val loss: 11.207 ************\n",
      "Finished Training\n",
      "Finished saving\n"
     ]
    }
   ],
   "source": [
    "\n",
    "train_loss = np.zeros((epochs_num,1))\n",
    "val_loss = np.zeros((epochs_num,1))+np.inf\n",
    "start = time.time()\n",
    "for epoch in range(epochs_num): \n",
    "    running_loss = 0.0\n",
    "    loss_tot = 0.0\n",
    "    v_loss_tot = 0.0\n",
    "    i=0\n",
    "    load_batch = Dataset_loader(path, batch_size, timestep, X, Y)\n",
    "    load_batch_val = Dataset_loader(path_val, batch_size, timestep, X_val, Y_val)\n",
    "    net.train()\n",
    "    while load_batch.check_if_left():\n",
    "        \n",
    "        inputs, labels = load_batch.get_batch()\n",
    "        \n",
    "#         print(stop-start)\n",
    "        inputs=Variable(torch.tensor(inputs).float())\n",
    "        labels=Variable(torch.tensor(labels).float())\n",
    "        inputs = inputs.cuda()\n",
    "        labels = labels.cuda()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        criterion = custom_losses(outputs, labels, epoch)\n",
    "        loss = criterion()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += loss.item()\n",
    "        loss_tot += loss.item()\n",
    "        if (i+1) % 1000 == 0:   \n",
    "            print('[epoch %d, minibatch %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, running_loss / 1000))\n",
    "            running_loss = 0.0\n",
    "        i = i+1\n",
    "    train_loss[epoch] = loss_tot/i\n",
    "    print('epoch %d, train loss: %.3f' %(epoch+1, loss_tot/i))\n",
    "    net.eval()\n",
    "    i=0\n",
    "    while load_batch_val.check_if_left():\n",
    "        inputs_val, labels_val = load_batch_val.get_batch()\n",
    "        inputs_val=Variable(torch.tensor(inputs_val).float())\n",
    "        labels_val=Variable(torch.tensor(labels_val).float())\n",
    "        inputs_val = inputs_val.cuda()\n",
    "        labels_val = labels_val.cuda()\n",
    "        outputs_val = net(inputs_val)\n",
    "        criterion = custom_losses(outputs_val, labels_val, epoch)\n",
    "        v_loss = criterion()\n",
    "        v_loss_tot += v_loss.item()\n",
    "        i=i+1\n",
    "    print(optimizer.param_groups[0]['lr'])\n",
    "#     scheduler.step()\n",
    "    if (v_loss_tot/i)<np.min(val_loss):\n",
    "        torch.save(net.state_dict(), os.path.normpath(os.path.join(checkpoint_path,'ckpt_'+str(epoch+1)+'.pth')))\n",
    "    val_loss[epoch] = v_loss_tot/i\n",
    "    print('************ epoch %d, val loss: %.3f ************' %(epoch+1, v_loss_tot/i))\n",
    "print('Finished Training')\n",
    "stop = time.time()\n",
    "with open((os.path.normpath(os.path.join(time_path,str(ckpt_folder)+'.txt'))), 'w') as f:\n",
    "  f.write('%d' % int(stop-start))\n",
    "np.savetxt(os.path.normpath(os.path.join(loss_path,'train_loss.txt')),train_loss)\n",
    "np.savetxt(os.path.normpath(os.path.join(loss_path,'val_loss.txt')),val_loss)\n",
    "model_path = os.path.normpath(os.path.join(Data_path,'results','models',str(ckpt_folder)+'.pth'))\n",
    "torch.save(net.state_dict(), model_path)\n",
    "print('Finished saving')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss.data.cpu().numpy().reshape(1,1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "du16_2_rnn_mi.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}