NEC_Extreme.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import math\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "\n",
    "class E_L(nn.Module):\n",
    "    def __init__(self, input_dim=1, hidden_dim=256, layer_dim=8):\n",
    "        super(E_L, self).__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.layer_dim = layer_dim\n",
    "        self.input_dim = input_dim\n",
    "\n",
    "        self.lstm = nn.LSTM(self.input_dim, self.hidden_dim, self.layer_dim, dropout=0.4, bidirectional=False, batch_first=True)\n",
    "\n",
    "    def forward(self, x, s_h, s_c):\n",
    "        # Initialize hidden and cell state\n",
    "        h0 = s_h\n",
    "        c0 = s_c\n",
    "        out, (hn, cn) = self.lstm(x, (h0,c0))\n",
    "        return out, hn, cn\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "    \n",
    "class E_F(nn.Module):\n",
    "    def __init__(self, hidden_dim=64, output_dim=24):\n",
    "        super(E_F, self).__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.output_dim=output_dim\n",
    "        self.bn1 = nn.BatchNorm1d(num_features=self.hidden_dim)\n",
    "        self.L_out1 = nn.Linear(self.hidden_dim, 512) \n",
    "        self.bn2 = nn.BatchNorm1d(num_features=512)\n",
    "        self.L_out2 = nn.Linear(512, 256) \n",
    "        self.bn3 = nn.BatchNorm1d(num_features=256)\n",
    "        self.L_out3 = nn.Linear(256, self.output_dim) \n",
    "    def forward(self, x, id=-1):\n",
    "        linear_out = self.L_out1(self.bn1(x[:,id,:]))\n",
    "        linear_out = self.L_out2(self.bn2(linear_out))\n",
    "        linear_out = self.L_out3(self.bn3(linear_out))\n",
    "        return linear_out\n",
    "\n",
    "\n",
    "# prepare train and val set\n",
    "from torch.utils.data import TensorDataset\n",
    "from torch.utils.data import DataLoader\n",
    "class RnnDataset(TensorDataset):\n",
    "    def __init__(self, data, label):\n",
    "        self.data = data\n",
    "        self.label=label\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        return self.data[index],self.label[index] \n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def read_dataset(sensor_id, start_point, train_point):\n",
    "    # read sensor data to vector\n",
    "    if(str(sensor_id)[0]=='4'):\n",
    "        trainX = pd.read_csv('./data'+'/reservoir_stor_'+str(sensor_id)+'_sof24.tsv', sep='\\t')\n",
    "    elif(str(sensor_id)[0]=='6'):\n",
    "        trainX = pd.read_csv('./data'+'/raingauge_byhour_'+str(sensor_id)+'_sof.tsv', sep='\\t')\n",
    "    else:\n",
    "        print(\"Error: the support sensor type is 4***or 6****.\")\n",
    "        sys.exit()\n",
    "    start_num = trainX[trainX[\"TSC_TSTAMP_UTC\"]==start_point].index.values[0]\n",
    "    print(\"for sensor \", sensor_id,\"start_num is: \", start_num)\n",
    "    idx_num = 0\n",
    "    print(len(trainX))\n",
    "    print(trainX[:3])\n",
    "\n",
    "    val_skips = []\n",
    "    \n",
    "    val_set = pd.read_csv('./data/'+str(sensor_id)+'_validation_timestamps_24avg.tsv',sep='\\t')\n",
    "    val_points = val_set[\"Hold Out Start\"]\n",
    "    for val_i in val_points:\n",
    "        valskip = trainX[trainX[\"TSC_TSTAMP_UTC\"]==val_i].index.values[0] - start_num\n",
    "        val_skips.append(valskip)\n",
    "        \n",
    "    # id of test_start and test_end, save it in the last two values in val_skips\n",
    "    test_start = trainX[trainX[\"TSC_TSTAMP_UTC\"]=='2017-01-01 14:30:00'].index.values[0] - start_num \n",
    "    test_end = trainX[trainX[\"TSC_TSTAMP_UTC\"]=='2018-12-31 14:30:00'].index.values[0] - start_num\n",
    "    val_skips.append(test_start)\n",
    "    val_skips.append(test_end) \n",
    "        \n",
    "    print(\"val_skips is: \", val_skips)\n",
    "\n",
    "    #foot label of train_end\n",
    "    train_end = trainX[trainX[\"TSC_TSTAMP_UTC\"]==train_point].index.values[0] - start_num \n",
    "    print(\"train_end is : \", train_end)\n",
    "\n",
    "    #the whole dataset to be preprocessed\n",
    "    train_x = trainX[start_num:] \n",
    "    print(len(train_x))\n",
    "    sensor_data = np.array(train_x[\"TSC_VALUE_F\"])\n",
    "    \n",
    "    return sensor_data, train_end, val_skips\n",
    " \n",
    "def diff_norm_dataset(sensor_data0):\n",
    "    a = sensor_data0\n",
    "    b = a[:-1]\n",
    "    a = a[1:]-b\n",
    "    c = np.array([0]+a.tolist())\n",
    "    mean = c.mean()\n",
    "    print(\"mean is: \",mean)\n",
    "    std = c.std()\n",
    "    print(\"std is \",std)\n",
    "    c = (c-mean)/std\n",
    "\n",
    "    extream_num = 0\n",
    "    print(\"extream boundary is: \", std*alpha_extreams)\n",
    "    total_num = len(c)\n",
    "    for i in range(total_num):\n",
    "        if(abs(c[i])>alpha_extreams):\n",
    "            extream_num += 1\n",
    "    print(\" the total number is: \", total_num)\n",
    "    print(\" the number of extreams is: \", extream_num, \" the ratio is: \", extream_num/total_num)\n",
    "\n",
    "    return c, mean, std\n",
    "\n",
    "def std_norm_dataset(sensor_data0):\n",
    "\n",
    "    c = sensor_data0\n",
    "    mean = c.mean()\n",
    "    print(\"mean is: \", mean)\n",
    "    std = c.std()\n",
    "    print(\"std is \", std)\n",
    "    c = (c - mean) / std\n",
    "  \n",
    "    return c, mean, std\n",
    "\n",
    "\n",
    "import random\n",
    "def split_dataset(norm_data,local_data, train_end, input_dim, output_size, train_days, predict_days, train_volum):\n",
    "\n",
    "    en_seq_len = train_days\n",
    "    de_seq_len = predict_days\n",
    "\n",
    "    DATA = []\n",
    "    Label = []\n",
    "    print(\"norm data size:\",len(norm_data))\n",
    "\n",
    "    train_size = train_end + 1 - (train_days*1 + output_size * predict_days)\n",
    "    print(\"train size is: \",train_size)\n",
    "\n",
    "    val_skips_extend = []\n",
    "    l = len(val_skips)\n",
    "    for i in range(l-2):\n",
    "        for j in range(output_size*predict_days):\n",
    "            val_skips_extend.append(val_skips[i]+j)\n",
    "    \n",
    "    test_start = val_skips[l-2]\n",
    "    print(\"test set start ID is: \", test_start)\n",
    "    test_end = val_skips[l-1]\n",
    "    print(\"test set end ID is: \", test_end)\n",
    "    \n",
    "    \n",
    "    # step 1, randomly choose train data\n",
    "    if(is_over_sampling==1):\n",
    "        norm_num = train_volum * norm_percent\n",
    "        extreme_num = train_volum * (1 - norm_percent)\n",
    "        extreme_total = 0\n",
    "        norm_total = 0\n",
    "   \n",
    "        while(extreme_total<extreme_num or norm_total<norm_num):\n",
    "            i = random.randint(0, train_size)\n",
    "            if  (i+train_days not in val_skips_extend ) and ((i+train_days)<test_start or (i+train_days)>test_end):  \n",
    "                if(is_watersheds==1 or is_prob_feature==1):\n",
    "                    data0 = np.array(local_data[i:(i+train_days*1)])\n",
    "                else:\n",
    "                    data0 = np.array(norm_data[i:(i+train_days*1)]).reshape(train_days,-1)\n",
    "                label0 = np.array(norm_data[(i+train_days*1):(i+train_days*1+output_size*predict_days)]) \n",
    "                label1 = np.array(class_label[(i+train_days*1):(i+train_days*1+output_size*predict_days)])\n",
    "            \n",
    "            # step 2, over sampling extreme points\n",
    "            if(label1.sum()>=1 and extreme_total<extreme_num):\n",
    "                extreme_total += 1;\n",
    "                DATA.append(data0)\n",
    "                Label.append(label0)\n",
    "            if(label1.sum()==0 and norm_total<norm_num):\n",
    "                norm_total += 1;\n",
    "                DATA.append(data0)\n",
    "                Label.append(label0) \n",
    "    else:  \n",
    "        ii=0\n",
    "        while (ii+train_days) < train_volum:\n",
    "            i=random.randint(0, train_size-1)\n",
    "            if (i+train_days not in val_skips_extend ) and ((i+train_days)<test_start or (i+train_days)>test_end) :   \n",
    "                if(is_watersheds==1 or is_prob_feature==1):\n",
    "                    data0=np.array(local_data[i:(i+train_days*1)])\n",
    "                else:\n",
    "                    data0=np.array(norm_data[i:(i+train_days*1)]).reshape(train_days,-1)\n",
    "            label0=np.array(norm_data[(i+train_days*1):(i+train_days*1+output_size*predict_days)])\n",
    "            DATA.append(data0)\n",
    "            Label.append(label0)\n",
    "            ii=ii+1\n",
    "        \n",
    "    dataset1=RnnDataset(DATA,Label)\n",
    "    data_loader = DataLoader(dataset1, \n",
    "                         batch_size,\n",
    "                         shuffle=True,\n",
    "                         num_workers=2,\n",
    "                         pin_memory=True,\n",
    "                         collate_fn=lambda x: x)\n",
    "    return data_loader\n",
    "\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "def create_model(input_dim, hidden_dim, layer_dim, output_dim):\n",
    "    encoder = E_L(input_dim, hidden_dim, layer_dim)\n",
    "    outlinear = E_F(hidden_dim, predict_days) \n",
    "\n",
    "    encoder = encoder.to(device)\n",
    "    outlinear = outlinear.to(device)\n",
    "\n",
    "    criterion = nn.MSELoss(reduction='sum')\n",
    "    encoder_optimizer = torch.optim.SGD(encoder.parameters(),0.001)  \n",
    "    outlinear_optimizer = torch.optim.Adam(outlinear.parameters(),0.0005) \n",
    "    \n",
    "    return encoder, outlinear, encoder_optimizer, outlinear_optimizer, criterion\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_loop( ):\n",
    "    num_epochs = 100\n",
    "    early_stop = 0\n",
    "    old_val_loss = 1000\n",
    "    min_RMSE = 500000\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        print_loss_total = 0\n",
    "        encoder.train()\n",
    "        outlinear.train()\n",
    "        \n",
    "        for i, batch in enumerate(dataloader):\n",
    "            x_train=[TrainData for TrainData, _ in batch]\n",
    "            y_train=[TrainLabel for _, TrainLabel in batch]\n",
    "            x_train=torch.from_numpy(np.array(x_train, np.float32)).to(device)\n",
    "            y_train=torch.from_numpy(np.array(y_train,np.float32)).to(device)\n",
    "\n",
    "            # Clear gradients w.r.t. parameters\n",
    "            encoder_optimizer.zero_grad()\n",
    "            outlinear_optimizer.zero_grad()\n",
    "            loss = 0\n",
    "\n",
    "            # Forward pass\n",
    "            s_h = torch.zeros(layer_dim, x_train.size(0), hidden_dim).to(device)\n",
    "            s_c = torch.zeros(layer_dim, x_train.size(0), hidden_dim).to(device)\n",
    "        \n",
    "            encoder_out, encoder_h, encoder_c = encoder(x_train, s_h, s_c)\n",
    "            e_out = outlinear(encoder_out)\n",
    "\n",
    "            # selective backpropagation\n",
    "        \n",
    "            if(is_normal==1):\n",
    "                for ii in range(y_train.size(0)):\n",
    "                    for jj in range(len(e_out[ii])):\n",
    "                        if(abs(y_train[ii][jj])>alpha_extreams):\n",
    "                            e_out[ii][jj]=y_train[ii][jj]\n",
    "                        \n",
    "            if(is_extreme==1):\n",
    "                for ii in range(y_train.size(0)):\n",
    "                    for jj in range(len(e_out[ii])):\n",
    "                        if(abs(y_train[ii][jj])<=alpha_extreams):\n",
    "                            e_out[ii][jj]=y_train[ii][jj] \n",
    "                        \n",
    "            loss = criterion(e_out, y_train) \n",
    "\n",
    "            # Backward pass \n",
    "            loss.backward()\n",
    "            encoder_optimizer.step()\n",
    "            outlinear_optimizer.step()\n",
    "            print_loss_total += loss.item()\n",
    "\n",
    "        encoder.eval()\n",
    "        outlinear.eval()\n",
    "        val_loss, min_RMSE = generate_val_rmse(min_RMSE)\n",
    "        \n",
    "        print('-----------Epoch: {}. train_Loss>: {:.6f}. --------------------'.format(epoch, print_loss_total)) \n",
    "        print('-----------Epoch: {}. val_Loss>: {:.6f}. --------------------'.format(epoch, val_loss)) \n",
    "\n",
    "        #early stop\n",
    "        if(val_loss>old_val_loss):\n",
    "            early_stop+=1\n",
    "        else:\n",
    "            early_stop=0\n",
    "        if(early_stop>=4):\n",
    "            break\n",
    "        old_val_loss=val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inference_test(encoder, outlinear,  x_test):\n",
    "    y_predict = []\n",
    "    d_out = torch.tensor([]).to(device)\n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        x_test = torch.from_numpy(np.array(x_test, np.float32)).to(device)\n",
    "        # Forward pass \n",
    "        s_h = torch.zeros(layer_dim, x_test.size(0), hidden_dim).to(device)\n",
    "        s_c = torch.zeros(layer_dim, x_test.size(0), hidden_dim).to(device)\n",
    "        x_test_new = x_test.squeeze(2)\n",
    "        encoder_out, _, _ = encoder(x_test, s_h, s_c)\n",
    "        d_out = outlinear(encoder_out)\n",
    "            \n",
    "        y_predict.extend(d_out[0])\n",
    "        y_predict = [y_predict[i].item() for i in range(len(y_predict))]\n",
    "        y_predict = np.array(y_predict).reshape(1, -1) \n",
    "    \n",
    "    return y_predict\n",
    "\n",
    "\n",
    "def diff_denorm_dataset(predict_y0, y0, pre_gt, y_test):\n",
    "    a2 = predict_y0\n",
    "    a2 = [(ii*std+mean) for ii in a2] \n",
    "    y0 = y0.values\n",
    "    a3 = np.zeros(len(y0)) \n",
    "    tt = 0\n",
    "  \n",
    "    if(is_single==1):\n",
    "        a3[0] = a2[0] + pre_gt[0]\n",
    "        for ii in range(predict_days-1):\n",
    "            a3[ii+1] = a3[ii] + a2[ii+1]\n",
    "    \n",
    "    if(is_single==0) and (is_normal==1):\n",
    "        for ii in range(predict_days):\n",
    "            if abs(y_test[ii]) <= alpha_extreams:\n",
    "                a3[tt]=a2[tt] + pre_gt[ii]\n",
    "            else:\n",
    "                a3[tt] = y0[ii]\n",
    "            tt += 1\n",
    "    \n",
    "    if(is_single==0) and (is_extreme==1):\n",
    "        for ii in range(predict_days):\n",
    "            if abs(y_test[ii]) > alpha_extreams:\n",
    "                a3[tt] = a2[tt] + pre_gt[ii]\n",
    "            else:\n",
    "                a3[tt] = y0[ii]\n",
    "            tt += 1 \n",
    "    return a3, a2\n",
    "\n",
    "def std_denorm_dataset(predict_y0):\n",
    "\n",
    "    a2 = predict_y0\n",
    "    a2 = [(ii*std+mean) for ii in a2]\n",
    "  \n",
    "    return a2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(test_point):\n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "\n",
    "    test_predict = np.zeros(predict_days*output_dim)\n",
    "    \n",
    "    if(str(sensor_id)[0]=='4'):\n",
    "        trainX = pd.read_csv('./data'+'/reservoir_stor_'+str(sensor_id)+'_sof24.tsv', sep='\\t')\n",
    "    elif(str(sensor_id)[0]=='6'):\n",
    "        trainX = pd.read_csv('./data'+'/raingauge_byhour_'+str(sensor_id)+'_sof.tsv', sep='\\t')\n",
    "    else:\n",
    "        print(\"Error: the support sensor type is 4*** or 6****.\")\n",
    "        sys.exit()\n",
    "\n",
    "    point = trainX[trainX[\"TSC_TSTAMP_UTC\"]==test_point].index.values[0]\n",
    "    start_num = trainX[trainX[\"TSC_TSTAMP_UTC\"]==start_point].index.values[0]\n",
    "    test_point = point - start_num\n",
    "    pre_gt = trainX[point-1 : point+71][\"TSC_VALUE_F\"].values.tolist()\n",
    "    y = trainX[point:point+72][\"TSC_VALUE_F\"]\n",
    "\n",
    "    #inference\n",
    "    norm_data = sensor_data_norm\n",
    "    if(is_watersheds==1 or is_prob_feature==1):\n",
    "        x_test = np.array(sensor_data_norm_1[test_point-train_days*1:test_point], np.float32).reshape(train_days,-1)\n",
    "    else:\n",
    "        x_test = np.array(norm_data[test_point-train_days*1:test_point], np.float32).reshape(train_days,-1)\n",
    "    y_test = np.array(norm_data[test_point:test_point+72], np.float32).reshape(72,-1)\n",
    "    x_test = [x_test]\n",
    "    y_predict = inference_test(encoder=encoder, outlinear=outlinear, x_test=x_test)\n",
    "    y_predict = np.array(y_predict.tolist())[0]\n",
    "    y_predict = [y_predict[i].item() for i in range(len(y_predict))]\n",
    "\n",
    "    if(is_diff==0):\n",
    "        test_predict = std_denorm_dataset(y_predict)\n",
    "        diff_predict = []\n",
    "    else:\n",
    "        test_predict, diff_predict = diff_denorm_dataset(y_predict, y, pre_gt, y_test)\n",
    "\n",
    "    return test_predict, y, diff_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_single(test_point):\n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "    test_predict = np.zeros(predict_days*output_dim)\n",
    "\n",
    "    if(str(sensor_id)[0]=='4'):\n",
    "        trainX = pd.read_csv('./data'+'/reservoir_stor_'+str(sensor_id)+'_sof24.tsv', sep='\\t')\n",
    "    elif(str(sensor_id)[0]=='6'):\n",
    "        trainX = pd.read_csv('./data'+'/raingauge_byhour_'+str(sensor_id)+'_sof.tsv', sep='\\t')\n",
    "    else:\n",
    "        print(\"Error: the support sensor type is 4*** or 6****.\")\n",
    "        sys.exit()\n",
    "            \n",
    "    #foot label of test_data\n",
    "    point = trainX[trainX[\"TSC_TSTAMP_UTC\"]==test_point].index.values[0]\n",
    "    start_num = trainX[trainX[\"TSC_TSTAMP_UTC\"]==start_point].index.values[0]\n",
    "    test_point = point - start_num\n",
    "    pre_gt = trainX[point-1:point+71][\"TSC_VALUE_F\"].values.tolist()\n",
    "    y = trainX[point:point+72][\"TSC_VALUE_F\"]\n",
    "\n",
    "    #inference\n",
    "    norm_data = sensor_data_norm\n",
    "    if(is_watersheds==1 or is_prob_feature==1):\n",
    "        x_test = np.array(sensor_data_norm_1[test_point-train_days*1:test_point], np.float32).reshape(train_days, -1)\n",
    "\n",
    "    else:\n",
    "        x_test = np.array(norm_data[test_point-train_days*1:test_point], np.float32).reshape(train_days, -1)\n",
    "\n",
    "    y_test = np.array(norm_data[test_point:test_point+72], np.float32).reshape(72, -1)\n",
    "    x_test = [x_test]\n",
    "\n",
    "    y_predict = inference_test(encoder=encoder, outlinear=outlinear, x_test=x_test)\n",
    "    y_predict = np.array(y_predict.tolist())[0]\n",
    "    y_predict = [y_predict[i].item() for i in range(len(y_predict))]\n",
    "    \n",
    "    if(is_diff==0):\n",
    "        test_predict = std_denorm_dataset(y_predict)\n",
    "        diff_predict = []\n",
    "    else:\n",
    "        test_predict, diff_predict = diff_denorm_dataset(y_predict, y, pre_gt, y_test)\n",
    "\n",
    "    return test_predict, y, diff_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_val_rmse(min_RMSE):\n",
    "    \n",
    "    test_predict = np.zeros(predict_days*output_dim)\n",
    "    val_set = pd.read_csv('./data/'+str(sensor_id)+'_validation_timestamps_24avg.tsv',sep='\\t')\n",
    "    val_points = val_set[\"Hold Out Start\"]\n",
    "\n",
    "    total = 0\n",
    "    val_rmse_list = []\n",
    "    val_diff_predicts = []\n",
    "    val_std_predicts = []\n",
    "    for i in range(len(val_points)):\n",
    "        val_point = val_points[i]\n",
    "        test_predict, ground_truth, diff_predict = test_single(test_point=val_point)\n",
    "        test_predict = [test_predict[i].item() for i in range(len(test_predict))]\n",
    "        test_predict = [k if k>0 else 0 for k in test_predict]\n",
    "    \n",
    "        val_diff_predicts.append(diff_predict)\n",
    "        val_std_predicts.append(test_predict)\n",
    "    \n",
    "        val_MSE = np.square(np.subtract(ground_truth, test_predict)).mean() \n",
    "        val_RMSE = math.sqrt(val_MSE)\n",
    "        val_rmse_list.append(val_RMSE)\n",
    "        total += val_RMSE\n",
    "        print(val_point, \" val Root Mean Square Error: \", val_RMSE)\n",
    "    \n",
    "    temp0 = [ k+0.1 for k in range(72)]\n",
    "    pd__temp = pd.DataFrame([temp0])\n",
    "    if(is_diff==1):\n",
    "        for i in range(len(val_diff_predicts)):\n",
    "            pd__temp.loc[i] = val_diff_predicts[i]\n",
    "    else:\n",
    "        for i in range(len(val_std_predicts)):\n",
    "            pd__temp.loc[i] = val_std_predicts[i]\n",
    "\n",
    "        \n",
    "    if(is_diff==0):\n",
    "        norm = \"std\"\n",
    "    else:\n",
    "        norm = \"diff\"\n",
    "    if(is_single==1):\n",
    "        single = \"single\"\n",
    "    else:\n",
    "        if(is_normal==1):\n",
    "            single = \"normal\"\n",
    "        else:\n",
    "            single = \"extreme\"\n",
    "        \n",
    "    if(is_watersheds==1):\n",
    "        if(is_prob_feature==0):\n",
    "            watersheds = \"shed\"\n",
    "        else:\n",
    "            watersheds = \"Shed-ProbFeature\"\n",
    "    else:\n",
    "        if(is_prob_feature==0):\n",
    "            watersheds = \"solo\"\n",
    "        else:\n",
    "            watersheds = \"ProbFeature\"\n",
    "        \n",
    "    new_min_RMSE = min_RMSE\n",
    "    if (total<min_RMSE):\n",
    "        pd__temp.to_csv(basic_path+'_'+norm+'_'+single+'.tsv',sep='\\t')\n",
    "        new_min_RMSE = total\n",
    "        #save_model\n",
    "        encoder_name = basic_encoder_model_path + '_' + norm + '_' + watersheds + \".pt\"\n",
    "        outlinear_name = basic_decoder_model_path + '_' + norm + '_' + watersheds + \".pt\"      \n",
    "        torch.save(outlinear, outlinear_name) \n",
    "        torch.save(encoder, encoder_name)\n",
    "        \n",
    "    print(\"val total RMSE: \", total)\n",
    "    print(\"val min RMSE: \", new_min_RMSE)\n",
    "    \n",
    "    return total, new_min_RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_test(path, filename):\n",
    "    \n",
    "    test_predict = np.zeros(predict_days*output_dim)\n",
    "    #test_data\n",
    "    val_set = pd.read_csv('./data/'+str(sensor_id)+'_test_timestamps_24avg.tsv',sep='\\t')\n",
    "    val_points = val_set[\"Start\"]\n",
    "\n",
    "    val_diff_predicts = []\n",
    "    val_std_predicts = []\n",
    "    for i in range(len(val_points)):\n",
    "        val_point = val_points[i]\n",
    "        test_predict, ground_truth, diff_predict = test_single(test_point=val_point)\n",
    "        test_predict = [test_predict[i].item() for i in range(len(test_predict))]\n",
    "    \n",
    "        val_diff_predicts.append(diff_predict)\n",
    "        val_std_predicts.append(test_predict)\n",
    "    \n",
    "    temp0 = [ k+0.1 for k in range(72)]\n",
    "    pd__temp = pd.DataFrame([temp0])\n",
    "    if(is_diff==1):\n",
    "        for i in range(len(val_diff_predicts)):\n",
    "            pd__temp.loc[i] = val_diff_predicts[i]\n",
    "    else:\n",
    "        for i in range(len(val_std_predicts)):\n",
    "            pd__temp.loc[i] = val_std_predicts[i]\n",
    "    pd__temp.to_csv(path+filename+'.tsv', sep='\\t')\n",
    "    \n",
    "    print(\"test prediction is saved at:  \" + path + filename + '.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import sys\n",
    "import sklearn\n",
    "from sklearn.mixture import GaussianMixture\n",
    "\n",
    "sensor_ids = [4001]\n",
    "local_sets = [[0]] # if watershed mode is used, the rain sensors involved should be set here.\n",
    "\n",
    "for i in range(len(sensor_ids)):\n",
    "    # sensor's data\n",
    "    sensor_id = sensor_ids[i]\n",
    "    local_set = local_sets[i]\n",
    "\n",
    "    # dataset parameters\n",
    "    start_point = '1980-12-31 14:30:00'\n",
    "    train_end = 0 \n",
    "    train_point = '2019-12-31 14:30:00'\n",
    "    print(\"Begin!! Sensor id is :---------------------\", sensor_id)\n",
    "\n",
    "    #training hyperparameters    \n",
    "    input_dim = 1      # univari forecasting, set 1\n",
    "    output_dim = 1       # univari forecasting, set 1\n",
    "    train_days = 15*24   # history length, h=360\n",
    "    predict_days = 24*3  # forecasting length,f=72\n",
    "    alpha_extreams = 1.5 # normal/extreme boundary, \\epsilon=1.5\n",
    "    is_diff = 1          # first order difference preprocessing, set1\n",
    "    is_single = 0        # N, E, or C, so not single\n",
    "    is_normal = 0        \n",
    "    is_extreme = 1       # E model, set 1\n",
    "    is_watersheds = 0    # no exogenous variables for this sensor, set 0\n",
    "    is_prob_feature = 1  # use GMM Indicator, set 1\n",
    "    is_over_sampling = 1 # E model use oversampling, set 1\n",
    "    norm_percent = 0.0   # oversampling ratio, OS% = 1\n",
    "\n",
    "    # model hyperparameters\n",
    "    hidden_dim = 512   # E hidden\n",
    "    layer_dim = 4      # E layers\n",
    "    batch_size = 32     # E batch_size\n",
    "    train_volum = 50000  # E volume\n",
    "    print(\"hidden dim is: \", hidden_dim)\n",
    "    print(\"layer num is: \", layer_dim)\n",
    "    print(\"train_volum is: \", train_volum)\n",
    "\n",
    "    print(\" is_over_sampling: \", is_over_sampling, \" is_prob_feature: \", is_prob_feature, \" is_diff: \", is_diff,\" is_single: \", is_single, \" is_normal: \", is_normal, \" is_extreme: \", is_extreme, \" is_watersheds: \", is_watersheds)\n",
    "    if(is_watersheds==1):\n",
    "        input_dim = len(local_set) + 1\n",
    "        print(\"the input_dim is: \", input_dim)\n",
    "    \n",
    "    if(is_prob_feature==1):\n",
    "        input_dim += 1\n",
    "        print(\"the input_dim with GM is: \", input_dim)\n",
    "    \n",
    "    if(is_extreme+is_single+is_normal!=1):\n",
    "        print(\"Error: is_extreme, is_single,is_normal can and only can be chosen one.\")\n",
    "        sys.exit()\n",
    "    \n",
    "    val_skips = []\n",
    "\n",
    "    if(is_watersheds==1):\n",
    "        # set according to the rain sensor's available data scope\n",
    "        start_point = '1991-12-18 14:30:00'\n",
    "        train_point = '2017-12-18 14:30:00'\n",
    "        \n",
    "    sensor_data, train_end, val_skips = read_dataset(sensor_id=sensor_id, start_point=start_point, train_point=train_point)\n",
    "  \n",
    "    if(is_diff==0):\n",
    "        sensor_data_norm, mean, std = std_norm_dataset(sensor_data)\n",
    "        print(\"I am using std_norm.\")\n",
    "    else:\n",
    "        sensor_data_norm, mean, std = diff_norm_dataset(sensor_data)\n",
    "        print(\"I am using diff_std_norm.\")\n",
    "\n",
    "\n",
    "    print(\"If I am using over_sampling, the norm_percent is: \", norm_percent)\n",
    "    \n",
    "    if(is_diff==0):\n",
    "        norm = \"std\"\n",
    "    else:\n",
    "        norm = \"diff\"\n",
    "        \n",
    "    if(is_over_sampling==1):\n",
    "        OS = \"_OS\" + str(norm_percent)\n",
    "    else:\n",
    "        OS = \"_OS-null\"\n",
    "        \n",
    "    if(is_single==1):\n",
    "        single = \"single\"\n",
    "    else:\n",
    "        if(is_normal==1):\n",
    "            single = \"normal\"\n",
    "        else:\n",
    "            single = \"extreme\"\n",
    "            \n",
    "    if(is_watersheds==1):\n",
    "        if(is_prob_feature==0):\n",
    "            watersheds = \"shed\"\n",
    "        else:\n",
    "            watersheds = \"Shed-ProbFeature\"\n",
    "    else:\n",
    "        if(is_prob_feature==0):\n",
    "            watersheds = \"solo\"\n",
    "        else:\n",
    "            watersheds = \"ProbFeature\"\n",
    " \n",
    "    sensor_data_norm_1 = [[ff] for ff in sensor_data_norm] \n",
    "\n",
    "    if(is_watersheds==1):\n",
    "        sensor_data_norm = sensor_data_norm[:train_end]\n",
    "        sensor_data_norm_1 = [[ff] for ff in sensor_data_norm] \n",
    "        for k in range(len(local_set)):\n",
    "            sensor_data_local, _ , _ = read_dataset(sensor_id=local_set[k], start_point=start_point, train_point=train_point)\n",
    "            if(is_diff==1):\n",
    "                sensor_data_norm_local, mean_local, std_local = std_norm_dataset(sensor_data_local)\n",
    "            else:\n",
    "                sensor_data_norm_local, mean_local, std_local = diff_norm_dataset(sensor_data_local)\n",
    "            print(\"local sensor id is: \", local_set[k], \" mean is: \", mean_local, \" std is: \", std_local)\n",
    "            sensor_data_norm_local = sensor_data_norm_local[:train_end]\n",
    "            sensor_data_norm_local = [[ff] for ff in sensor_data_norm_local]\n",
    "            sensor_data_norm_1 = np.concatenate((sensor_data_norm_1, sensor_data_norm_local), 1)\n",
    "\n",
    "    if(is_prob_feature==1):\n",
    "        gm = GaussianMixture(n_components=3, )  # using GMM, M=3\n",
    "        sensor_data_norm_prob = sensor_data_norm.reshape(-1, 1)\n",
    "        gm.fit(sensor_data_norm_prob)\n",
    "        print(\"gm.means are: \", gm.means_)\n",
    "        print(\"gm.covariances are: \", gm.covariances_)\n",
    "        print(\"gm.weights are: \", gm.weights_)\n",
    "        weights = gm.weights_\n",
    "        data_prob = gm.predict_proba(sensor_data_norm_prob)\n",
    "        prob_in_distribution = data_prob[:, 0] * weights[0] + data_prob[:, 1] * weights[1] + data_prob[:, 2] * weights[2]\n",
    "        prob_like_outlier = 1- prob_in_distribution\n",
    "        prob_like_outlier = prob_like_outlier.reshape((len(sensor_data_norm), 1))\n",
    "        sensor_data_norm_1 = np.concatenate((sensor_data_norm_1,prob_like_outlier), 1)\n",
    "          \n",
    "    class_label = []\n",
    "    for i in range(len(sensor_data_norm)):\n",
    "        if abs(sensor_data_norm[i])>alpha_extreams:\n",
    "            class_label.append(1)\n",
    "        else:\n",
    "            class_label.append(0)\n",
    "    \n",
    "    if(is_single==0):\n",
    "        if(is_normal==1):\n",
    "            print(\"I am training only normal data.\")        \n",
    "        else: \n",
    "            if(is_extreme==1):\n",
    "                print(\"I am training only extremem data.\")\n",
    "    else:\n",
    "        print(\"I am training on the whole data set.\")\n",
    "\n",
    "\n",
    "    #paths settings\n",
    "    basic_path = './val/' + str(sensor_id) + '_' + watersheds       #path used to save validation set prediction\n",
    "    basic_encoder_model_path = \"./model/\" + str(sensor_id) + single + '_encoder'   # E_LSTM model path\n",
    "    basic_decoder_model_path = \"./model/\" + str(sensor_id) + single + '_outlinear' # E_FC model path\n",
    "    \n",
    "    #get train data_loader \n",
    "    dataloader = split_dataset(norm_data=sensor_data_norm, local_data=sensor_data_norm_1,train_end=train_end, input_dim=input_dim, output_size=output_dim, train_days=train_days, predict_days=predict_days,train_volum=train_volum)\n",
    "        \n",
    "    #create_model\n",
    "    encoder, outlinear, encoder_optimizer, outlinear_optimizer, criterion = create_model(input_dim=input_dim, hidden_dim=hidden_dim, layer_dim=layer_dim, output_dim=input_dim)\n",
    "\n",
    "    #train the model\n",
    "    train_loop()    #if only inferencing, just hide this line.\n",
    "    \n",
    "    print(\"Finish!! Sensor id is :---------------------\", sensor_id)    \n",
    "\n",
    "    #generate test prediction\n",
    "    encoder = torch.load(basic_encoder_model_path+'_'+norm+'_'+watersheds+\".pt\")\n",
    "    outlinear = torch.load(basic_decoder_model_path+'_'+norm+'_'+watersheds+\".pt\" )\n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "    generate_test(\"./test/\", str(sensor_id)+single) # the path of generated test prediction\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}