finalsubmission.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Submission.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z5i5ApQf52nU",
        "colab_type": "text"
      },
      "source": [
        "# Basic Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ao8TxEyY0yXn",
        "colab_type": "text"
      },
      "source": [
        "1. Connect Google Drive and setup dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vDOW8ALuAR3q",
        "colab_type": "code",
        "outputId": "3044e183-9f52-4a9d-f9d1-0061f3d60f8f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')\n",
        "! ln -s '/gdrive/My Drive/Kitten' /content/\n",
        "! pip install visdom"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /gdrive\n",
            "Collecting visdom\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/97/c4/5f5356fd57ae3c269e0e31601ea6487e0622fedc6756a591e4a5fd66cc7a/visdom-0.1.8.8.tar.gz (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 9.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.8 in /usr/local/lib/python3.6/dist-packages (from visdom) (1.16.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from visdom) (1.3.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from visdom) (2.21.0)\n",
            "Requirement already satisfied: tornado in /usr/local/lib/python3.6/dist-packages (from visdom) (4.5.3)\n",
            "Requirement already satisfied: pyzmq in /usr/local/lib/python3.6/dist-packages (from visdom) (17.0.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from visdom) (1.12.0)\n",
            "Collecting torchfile (from visdom)\n",
            "  Downloading https://files.pythonhosted.org/packages/91/af/5b305f86f2d218091af657ddb53f984ecbd9518ca9fe8ef4103a007252c9/torchfile-0.1.0.tar.gz\n",
            "Collecting websocket-client (from visdom)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/29/19/44753eab1fdb50770ac69605527e8859468f3c0fd7dc5a76dd9c4dbd7906/websocket_client-0.56.0-py2.py3-none-any.whl (200kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 48.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from visdom) (5.4.1)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->visdom) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->visdom) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->visdom) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->visdom) (2019.3.9)\n",
            "Building wheels for collected packages: visdom, torchfile\n",
            "  Building wheel for visdom (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Stored in directory: /root/.cache/pip/wheels/ee/87/ce/a5023722374ca73b57fc8d4284ba6f973c01219b3c385a07e0\n",
            "  Building wheel for torchfile (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Stored in directory: /root/.cache/pip/wheels/b1/c3/d6/9a1cc8f3a99a0fc1124cae20153f36af59a6e683daca0a0814\n",
            "Successfully built visdom torchfile\n",
            "Installing collected packages: torchfile, websocket-client, visdom\n",
            "Successfully installed torchfile-0.1.0 visdom-0.1.8.8 websocket-client-0.56.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PVNr3GEY7lQr",
        "colab_type": "code",
        "outputId": "46c203dc-0520-47aa-9203-739e74d05739",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# change to AlphaPose directory for importing modules\n",
        "% cd '/content/Kitten/AlphaPose'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/gdrive/My Drive/Kitten/AlphaPose\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wjIpNXRg1UKq",
        "colab_type": "text"
      },
      "source": [
        "2. Import from everywhere"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bfrVuFwI7d54",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# import modules\n",
        "\n",
        "import torch\n",
        "from torch import optim\n",
        "from torch.autograd import Variable\n",
        "import torch.nn.functional as F\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.utils.data\n",
        "import torch.utils.data as data\n",
        "\n",
        "import os\n",
        "import sys\n",
        "import cv2\n",
        "import json\n",
        "import time\n",
        "import numpy as np\n",
        "import math\n",
        "\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "from opt import opt\n",
        "from tqdm import tqdm\n",
        "\n",
        "from yolo.preprocess import prep_image\n",
        "from yolo.darknet import Darknet\n",
        "from yolo.util import dynamic_write_results\n",
        "\n",
        "from dataloader import Mscoco, crop_from_dets\n",
        "from SPPE.src.utils.eval import getPrediction\n",
        "from SPPE.src.utils.img import load_image, cropBox, im_to_torch\n",
        "from SPPE.src.main_fast_inference import InferenNet\n",
        "from pPose_nms import pose_nms, write_json\n",
        "from fn import vis_frame\n",
        "\n",
        "# from PIL import Image"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rJNEhaSk1cXg",
        "colab_type": "text"
      },
      "source": [
        "3. Paths"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cics8eHRhNyx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_dir = '/content/Kitten/data/merged'\n",
        "train_folder = os.path.join(base_dir, 'train')\n",
        "val_folder = os.path.join(base_dir, 'val')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q6ffTeGU1fXS",
        "colab_type": "text"
      },
      "source": [
        "4. Load labels with corresponding index numbers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjDtfxjljN4t",
        "colab_type": "code",
        "outputId": "82a1af5e-1025-48e2-cf04-4da32ccd242e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "classes = os.listdir(train_folder)\n",
        "classes = sorted(classes)\n",
        "key_dict = {}\n",
        "\n",
        "for k, v in enumerate(classes):\n",
        "    key_dict[k] = v\n",
        "\n",
        "print(key_dict) # ensure that the classes are extracted correctly"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{0: 'ChairPose', 1: 'ChildPose', 2: 'Dabbing', 3: 'HandGun', 4: 'HandShake', 5: 'HulkSmash', 6: 'KoreanHeart', 7: 'KungfuCrane', 8: 'KungfuSalute', 9: 'Salute', 10: 'WarriorPose'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wc0-EuSn3UbY",
        "colab_type": "text"
      },
      "source": [
        "7. Define some helper functions for pose extraction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VBuBceCBnKCw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "opt.vis_dim = 64"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "czfnniGR7hy6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_models():\n",
        "    # Load yolov3 model\n",
        "    det_model = Darknet(os.path.join(\"yolo/cfg/yolov3-spp.cfg\"))\n",
        "    det_model.load_weights(os.path.join('models/yolo/yolov3-spp.weights'))\n",
        "    det_model.net_info['height'] = opt.inp_dim\n",
        "    det_model.cuda()\n",
        "    det_model.eval()\n",
        "\n",
        "    # Load pose model\n",
        "    pose_dataset = Mscoco()\n",
        "    pose_model = InferenNet(4 * 1 + 1, pose_dataset)\n",
        "    pose_model.cuda()\n",
        "    pose_model.eval()\n",
        "    \n",
        "    return det_model, pose_model\n",
        "\n",
        "def letterbox_image(img, inp_dim=(opt.vis_dim, opt.vis_dim), bg = 128):\n",
        "    '''resize image with unchanged aspect ratio using padding'''\n",
        "    img_w, img_h = img.shape[1], img.shape[0]\n",
        "    w, h = inp_dim\n",
        "    new_w = int(img_w * min(w / img_w, h / img_h))\n",
        "    new_h = int(img_h * min(w / img_w, h / img_h))\n",
        "    resized_image = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_CUBIC)\n",
        "\n",
        "    canvas = np.full((inp_dim[1], inp_dim[0], 3), bg)\n",
        "\n",
        "    canvas[(h - new_h) // 2:(h - new_h) // 2 + new_h, (w - new_w) // 2:(w - new_w) // 2 + new_w, :] = resized_image\n",
        "\n",
        "    return canvas\n",
        "\n",
        "def predict_numpy_image(img_np, det_model, pose_model):\n",
        "    \"\"\"\n",
        "    This functions reads from a single numpy array of pixel values and returns \n",
        "    all the pose estimations found sucessfully, None otherwise\n",
        "    \"\"\"\n",
        "    \n",
        "    inp_dim = int(opt.inp_dim)\n",
        "    \n",
        "    im_dim = img_np.shape[1], img_np.shape[0]\n",
        "    im_dim = torch.FloatTensor(im_dim).repeat(1,2)\n",
        "    \n",
        "    img = letterbox_image(img_np, (inp_dim, inp_dim))\n",
        "    img = img.transpose((2, 0, 1)).copy()\n",
        "    img = torch.from_numpy(img).float().div(255.0).unsqueeze(0)\n",
        "\n",
        "    det_inp_dim = int(det_model.net_info['height'])\n",
        "\n",
        "    with torch.no_grad():\n",
        "        \n",
        "        im_dim = torch.FloatTensor(im_dim).repeat(1,2)    \n",
        "        # Human Detection using yolov3\n",
        "        img = img.cuda()\n",
        "        prediction = det_model(img, CUDA=True)\n",
        "\n",
        "\n",
        "        # NMS process\n",
        "        dets = dynamic_write_results(prediction, opt.confidence,\n",
        "                            opt.num_classes, nms=True, nms_conf=opt.nms_thesh)\n",
        "        try:\n",
        "            dets = dets.cpu()\n",
        "        except:\n",
        "            return None\n",
        "        \n",
        "        im_dim_list = torch.index_select(im_dim, 0, dets[:, 0].long())\n",
        "        scaling_factor = torch.min(det_inp_dim / im_dim, 1)[0].view(-1, 1)\n",
        "\n",
        "        # coordinate transfer\n",
        "        dets[:, [1, 3]] -= (det_inp_dim - scaling_factor * im_dim_list[:, 0].view(-1, 1)) / 2\n",
        "        dets[:, [2, 4]] -= (det_inp_dim - scaling_factor * im_dim_list[:, 1].view(-1, 1)) / 2\n",
        "\n",
        "\n",
        "        dets[:, 1:5] /= scaling_factor\n",
        "        for j in range(dets.shape[0]):\n",
        "            dets[j, [1, 3]] = torch.clamp(dets[j, [1, 3]], 0.0, im_dim_list[j, 0])\n",
        "            dets[j, [2, 4]] = torch.clamp(dets[j, [2, 4]], 0.0, im_dim_list[j, 1])\n",
        "        boxes = dets[:, 1:5]\n",
        "        scores = dets[:, 5:6]\n",
        "\n",
        "        boxes = boxes[dets[:,0]==0]\n",
        "\n",
        "        inps = torch.zeros(boxes.size(0), 3, opt.inputResH, opt.inputResW)\n",
        "        pt1 = torch.zeros(boxes.size(0), 2)\n",
        "        pt2 = torch.zeros(boxes.size(0), 2)\n",
        "        scores = scores[dets[:,0]==0]\n",
        "        \n",
        "        # pose estimation\n",
        "\n",
        "        inp = im_to_torch(img_np)\n",
        "        inps, pt1, pt2 = crop_from_dets(inp, boxes, inps, pt1, pt2)\n",
        "\n",
        "        inps = inps.cuda()\n",
        "        hm = pose_model(inps)\n",
        "        hm = hm.cpu()\n",
        "\n",
        "\n",
        "    if boxes is not None:\n",
        "        # filter detections\n",
        "        preds_hm, preds_img, preds_scores = getPrediction(hm, pt1, pt2, \n",
        "                                                          opt.inputResH, opt.inputResW,\n",
        "                                                          opt.outputResH, opt.outputResW)\n",
        "        result = pose_nms(boxes, scores, preds_img, preds_scores)\n",
        "\n",
        "        if len(result):\n",
        "            return result\n",
        "\n",
        "def process_results(result, img_dim):\n",
        "    \"\"\"\n",
        "    Due to multiple poses detected, we assume that the keypoints with the \n",
        "    largest bounding box is the one of interest\n",
        "    \"\"\"\n",
        "    img_h, img_w = img_dim\n",
        "    kps = []\n",
        "    scores = []\n",
        "    areas = []\n",
        "    coordinates = []\n",
        "    for human in result:\n",
        "        kp_preds = human['keypoints']\n",
        "        kp_scores = human['kp_score']\n",
        "        kp_preds = torch.cat((kp_preds, torch.unsqueeze((kp_preds[5,:]+kp_preds[6,:])/2,0)))\n",
        "        kp_scores = torch.cat((kp_scores, torch.unsqueeze((kp_scores[5,:]+kp_scores[6,:])/2,0)))\n",
        "        for n in range(kp_scores.shape[0]):\n",
        "            if kp_scores[n] <= 0.05:\n",
        "                kp_preds[n] = torch.zeros_like(kp_preds[n])\n",
        "                continue\n",
        "        kps.append(kp_preds)\n",
        "        scores.append(kp_scores)\n",
        "        \n",
        "        x_nonzero = kp_preds[:,0]\n",
        "        y_nonzero = kp_preds[:,1]\n",
        "        x_nonzero = x_nonzero[x_nonzero != 0]\n",
        "        y_nonzero = y_nonzero[y_nonzero != 0]\n",
        "        \n",
        "        min_x = int(torch.min(x_nonzero))\n",
        "        min_y = int(torch.min(y_nonzero))\n",
        "        max_x = int(torch.max(x_nonzero))\n",
        "        max_y = int(torch.max(y_nonzero))\n",
        "        \n",
        "        area = (max_x - min_x) * (max_y - min_y)\n",
        "        areas.append(float(area))\n",
        "        \n",
        "        coor = (min_y, max_y, min_x, max_x)\n",
        "        coordinates.append(coor)\n",
        "        \n",
        "        \n",
        "    if len(kps):\n",
        "        # return the set of keypoints covering largest area and corner coordinates\n",
        "        idx = np.argmax(areas)\n",
        "        return kps[idx], scores[idx], coordinates[idx]\n",
        "    \n",
        "def vis_kps(kp_preds, kp_scores, coor, orig_dim):\n",
        "    '''\n",
        "    kps: keypoints of a single person\n",
        "    coor: (min_y, max_y, min_x, max_x)\n",
        "    orig_dim: dimensions of the original image\n",
        "    return rendered letter-boxed image\n",
        "    '''\n",
        "    l_pair = [\n",
        "        (0, 1), (0, 2), (1, 3), (2, 4),  # Head\n",
        "        (5, 6), (5, 7), (7, 9), (6, 8), (8, 10),\n",
        "        (17, 11), (17, 12),  # Body\n",
        "        (11, 13), (12, 14), (13, 15), (14, 16)\n",
        "    ]\n",
        "\n",
        "    p_color = [(0, 255, 255), (0, 191, 255),(0, 255, 102),(0, 77, 255), (0, 255, 0), #Nose, LEye, REye, LEar, REar\n",
        "                (77,255,255), (77, 255, 204), (77,204,255), (191, 255, 77), (77,191,255), (191, 255, 77), #LShoulder, RShoulder, LElbow, RElbow, LWrist, RWrist\n",
        "                (204,77,255), (77,255,204), (191,77,255), (77,255,191), (127,77,255), (77,255,127), (0, 255, 255)] #LHip, RHip, LKnee, Rknee, LAnkle, RAnkle, Neck\n",
        "    line_color = [(0, 215, 255), (0, 255, 204), (0, 134, 255), (0, 255, 50), \n",
        "                (77,255,222), (77,196,255), (77,135,255), (191,255,77), (77,255,77), \n",
        "                (77,222,255), (255,156,127), \n",
        "                (0,127,255), (255,127,77), (0,77,255), (255,77,36)]\n",
        "    \n",
        "    height,width = orig_dim\n",
        "    min_y, max_y, min_x, max_x = coor\n",
        "    \n",
        "    bleed = 8\n",
        "    min_x = max(min_x - bleed, 0) // 2\n",
        "    min_y = max(min_y - bleed, 0) // 2\n",
        "    max_x = min(max_x + bleed, width) // 2\n",
        "    max_y = min(max_y + bleed, height) // 2\n",
        "    \n",
        "    # create black background\n",
        "    img = np.zeros((int(height/2), int(width/2), 3), dtype=np.uint8)\n",
        "    part_line = {}\n",
        "    # Draw keypoints\n",
        "    for n in range(kp_scores.shape[0]):\n",
        "        cor_x, cor_y = int(kp_preds[n, 0]), int(kp_preds[n, 1])\n",
        "        if kp_scores[n] <= 0.05:\n",
        "            continue\n",
        "        part_line[n] = (int(cor_x/2), int(cor_y/2))\n",
        "        bg = img.copy()\n",
        "        cv2.circle(bg, (int(cor_x/2), int(cor_y/2)), 2, p_color[n], -1)\n",
        "        # Now create a mask of logo and create its inverse mask also\n",
        "        transparency = max(0, min(1, kp_scores[n]))\n",
        "        img = cv2.addWeighted(bg, transparency, img, 1-transparency, 0)\n",
        "    # Draw limbs\n",
        "    for i, (start_p, end_p) in enumerate(l_pair):\n",
        "        if start_p in part_line and end_p in part_line:\n",
        "            start_xy = part_line[start_p]\n",
        "            end_xy = part_line[end_p]\n",
        "            bg = img.copy()\n",
        "\n",
        "            X = (start_xy[0], end_xy[0])\n",
        "            Y = (start_xy[1], end_xy[1])\n",
        "            mX = np.mean(X)\n",
        "            mY = np.mean(Y)\n",
        "            length = ((Y[0] - Y[1]) ** 2 + (X[0] - X[1]) ** 2) ** 0.5\n",
        "            angle = math.degrees(math.atan2(Y[0] - Y[1], X[0] - X[1]))\n",
        "            stickwidth = (kp_scores[start_p] + kp_scores[end_p]) + 1\n",
        "            polygon = cv2.ellipse2Poly((int(mX),int(mY)), \n",
        "                                       (int(length/2), stickwidth), \n",
        "                                       int(angle), 0, 360, 1)\n",
        "            cv2.fillConvexPoly(bg, polygon, line_color[i])\n",
        "            transparency = max(0, min(1, 0.5*(kp_scores[start_p] + \n",
        "                                              kp_scores[end_p])))\n",
        "            img = cv2.addWeighted(bg, transparency, img, 1-transparency, 0)\n",
        "                \n",
        "    cropped_img = img[min_y:max_y, min_x:max_x, :]\n",
        "    vis = letterbox_image(cropped_img, bg=0)\n",
        "    \n",
        "    return vis\n",
        "    \n",
        "def img_to_vis(img_list, del_list = None):\n",
        "    \"\"\"\n",
        "    Take in an array of numpy images as input and output a tensor with the \n",
        "    visualisation of each one\n",
        "    \"\"\"\n",
        "    dim = opt.vis_dim\n",
        "    \n",
        "    # load models\n",
        "    det_model, pose_model = load_models()\n",
        "    \n",
        "    vis_list = []\n",
        "    for i in range(len(img_list)):\n",
        "        orig_dim = img_list[i].shape[:2]\n",
        "        result = predict_numpy_image(img_list[i], det_model, pose_model)\n",
        "        if result is not None:\n",
        "            # when there is at least one pose detection\n",
        "            (keypoints, scores, coor) = process_results(result, orig_dim)\n",
        "            vis = vis_kps(keypoints, scores, coor, orig_dim)\n",
        "            vis_list.append(vis)\n",
        "            continue\n",
        "        else:\n",
        "            # either no humans are detected successfully\n",
        "            # or pose estimation failed for detected humans\n",
        "            vis_list.append(np.zeros((dim, dim, 3), dtype=np.uint8))\n",
        "            if del_list is not None:\n",
        "                del_list.append(i)\n",
        "                \n",
        "    return np.stack(vis_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u1zMB6ZE5ZEQ",
        "colab_type": "text"
      },
      "source": [
        "8. Convert images into keypoint visualisations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zh54oSVp9FDe",
        "colab_type": "code",
        "outputId": "5cd0e2a4-f1a7-4bf4-9451-8c18e22b1d45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        }
      },
      "source": [
        "train_del = []\n",
        "val_del = []\n",
        "if not load_vis:\n",
        "    x_train_vis = img_to_vis(x_train, train_del)\n",
        "    x_val_vis = img_to_vis(x_val, val_del)\n",
        "    # BGR to RGB\n",
        "    x_train_vis = x_train_vis[:,:,:,::-1].transpose((0, 3, 1, 2)).copy()\n",
        "    x_val_vis = x_val_vis[:,:,:,::-1].transpose((0, 3, 1, 2)).copy()\n",
        "    \n",
        "    # delete non-detections to avoid interference with training\n",
        "    x_train = [item for itr, item in enumerate(x_train) if itr not in train_del]\n",
        "    x_train_vis = np.delete(x_train_vis, train_del, 0)\n",
        "    y_train = np.delete(y_train, train_del, 0)\n",
        "    x_val = [item for itr, item in enumerate(x_val) if itr not in val_del]\n",
        "    x_val_vis = np.delete(x_val_vis, val_del, 0)\n",
        "    y_val = np.delete(y_val, val_del, 0)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-8f631f75b05d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtrain_del\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mval_del\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mload_vis\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mx_train_vis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimg_to_vis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_del\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mx_val_vis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimg_to_vis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_del\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'load_vis' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C3vhDpK63bAa",
        "colab_type": "text"
      },
      "source": [
        "# Augmentation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCPRQmR637-O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import imgaug as ia\n",
        "from imgaug import augmenters as iaa\n",
        "\n",
        "augmentation_seq = iaa.Sequential([\n",
        "    iaa.Multiply((0.9, 1.1)),\n",
        "    iaa.Fliplr(0.5),\n",
        "    iaa.Affine(rotate=(-20, 20), scale=(0.9, 1.1), translate_px=(-3, 3)),\n",
        "    iaa.PerspectiveTransform(scale=(0.0, 0.05), keep_size=True)\n",
        "])\n",
        "\n",
        "augmentations = 10\n",
        "x_train_vis_aug = []\n",
        "\n",
        "transposed = np.transpose(x_train_vis, (0, 2, 3, 1)).astype(np.uint8)\n",
        "\n",
        "for i in range(augmentations):\n",
        "    augmentation_batch = []\n",
        "    for image in transposed:\n",
        "        augmentation_batch.append(augmentation_seq.augment_image(image))\n",
        "    x_train_vis_aug.append(augmentation_batch)\n",
        "\n",
        "x_train_vis_aug = np.array(x_train_vis_aug)\n",
        "x_train_vis_aug = np.transpose(x_train_vis_aug, (0, 1, 4, 2, 3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t0z-vXZG5pMi",
        "colab_type": "text"
      },
      "source": [
        "# Pose Estimation to Pose Classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VyvDEyNs6CCR",
        "colab_type": "text"
      },
      "source": [
        "All are pytorch tensors\n",
        "\n",
        "Training keypoints:  `x_train_vis`\n",
        "\n",
        "Validation keypoints: `x_val_vis`\n",
        "\n",
        "Training ground truth: `y_train`\n",
        "\n",
        "Validation ground truth: `y_val`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0EbODN-JaZlH",
        "colab_type": "text"
      },
      "source": [
        "## Convert to tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15X47J5waYjM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train_vis = torch.tensor(x_train_vis, dtype=torch.float32)\n",
        "x_train_vis_aug = torch.tensor(x_train_vis_aug, dtype=torch.float32)\n",
        "x_val_vis = torch.tensor(x_val_vis, dtype=torch.float32)\n",
        "y_train = torch.tensor(y_train, dtype=torch.long)\n",
        "y_val = torch.tensor(y_val, dtype=torch.long)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vduq-4okapdD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train_vis /= 256.0\n",
        "x_train_vis_aug /= 256.0\n",
        "x_val_vis /= 256.0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cubjG-276xA6",
        "colab_type": "text"
      },
      "source": [
        "## *Define neural network here*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tW8L_-ZeIVgu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
        "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
        "        self.pool = nn.MaxPool2d(2, 2)\n",
        "        self.fc1 = nn.Linear(16 * 13 * 13, 256)\n",
        "        self.fc2 = nn.Linear(256, 256)\n",
        "        self.fc3 = nn.Linear(512, 128)\n",
        "        self.fc4 = nn.Linear(256, 11)\n",
        "        self.do  = nn.Dropout()\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.pool(F.leaky_relu(self.conv1(x)))\n",
        "        x = self.do(x)\n",
        "        x = self.pool(F.leaky_relu(self.conv2(x)))\n",
        "        x = self.do(x)\n",
        "        x = x.view(-1, self.num_flat_features(x))\n",
        "        x = self.fc1(x)\n",
        "        x = self.do(x)\n",
        "        x = F.leaky_relu(x)\n",
        "        x = self.fc2(x)\n",
        "        x = self.do(x)\n",
        "        x = F.leaky_relu(x)\n",
        "#         x = self.fc3(x)\n",
        "#         x = self.do(x)\n",
        "#         x = F.relu(x)\n",
        "        x = self.fc4(x)\n",
        "        return x\n",
        "    \n",
        "    def num_flat_features(self, x):\n",
        "        size = x.size()[1:]  # all dimensions except the batch dimension\n",
        "        num_features = 1\n",
        "        for s in size:\n",
        "            num_features *= s\n",
        "        return num_features\n",
        "    \n",
        "model = Model()\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adadelta(model.parameters(), lr=0.1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UJBMxzCe8Alf",
        "colab_type": "text"
      },
      "source": [
        "## Load model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_tijXLMoADae",
        "colab_type": "text"
      },
      "source": [
        "Method 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "382RRAx3y02O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "best_model_path = ''\n",
        "infer_model = torch.load(best_model_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tPLnbJLSAGX6",
        "colab_type": "text"
      },
      "source": [
        "Method 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QH0z8pxdAFrR",
        "colab_type": "code",
        "outputId": "0e685e48-5b46-4b3b-805c-7a7214955df1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "# infer_model = Model()\n",
        "# infer_model.load_state_dict(torch.load('/content/Kitten/test.pth'))\n",
        "# infer_model.eval()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Model(\n",
              "  (fc1): Linear(in_features=34, out_features=256, bias=True)\n",
              "  (fc2): Linear(in_features=256, out_features=256, bias=True)\n",
              "  (fc3): Linear(in_features=512, out_features=128, bias=True)\n",
              "  (fc4): Linear(in_features=256, out_features=11, bias=True)\n",
              "  (do): Dropout(p=0.5)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GAKc4UEN_ii_",
        "colab_type": "text"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8IiYdyoeuS_v",
        "colab_type": "code",
        "outputId": "53494ae9-8af3-4297-bbfa-b964dacf7aa2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "! pip install folium==0.2.1 imgaug==0.2.5 pillow==5.4.1\n",
        "! pip install --force-reinstall --no-warn-script-location -q https://ai-camp.s3-us-west-2.amazonaws.com/AiCampEval-1.7-py3-none-any.whl"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[31mERROR: Invalid requirement: 'pillow=5.4.1'\n",
            "= is not a valid operator. Did you mean == ?\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OUTTGO9muyOi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from AiCampEval import eval_submit\n",
        "\n",
        "def evaluate_images(list_of_np_arrays):\n",
        "    new_h = 480\n",
        "    for img in list_of_np_arrays:\n",
        "        width, height = img.shape[:2]\n",
        "        new_w = width * new_h // height\n",
        "        img = resized_image = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_CUBIC)\n",
        "    preprocessed_imgs = img_to_vis( list_of_np_arrays )\n",
        "    preprocessed_imgs = preprocessed_imgs[:,:,:,::-1].transpose((0, 3, 1, 2)).copy()\n",
        "    infer_model.eval()\n",
        "    preprocessed_imgs = torch.tensor(preprocessed_imgs, dtype=torch.float32)\n",
        "    preprocessed_imgs = preprocessed_imgs.cuda()\n",
        "    preprocessed_imgs.div(256.)\n",
        "    output = infer_model(preprocessed_imgs)\n",
        "    output = output.cpu()\n",
        "    pred = torch.argmax(output, 1).numpy()\n",
        "    # Convert the list of class_indices to a list of predictions (in str)\n",
        "    # predictions is a list of labels: ['KoreanHeart', 'KoreanHeart','ChairPose',.......]\n",
        "    predictions = [ key_dict[k] for k in pred ]\n",
        "    return predictions"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4L_rnCJu9Us",
        "colab_type": "code",
        "outputId": "68226627-cc96-4ab9-d619-c9b51be6ee12",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "TEAM_ID = \"Team Chi\"\n",
        "SUBMISSION_TYPE = \"testset_11classes_1_01010\"\n",
        "\n",
        "eval_submit(evaluate_images, SUBMISSION_TYPE, TEAM_ID)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Predicting batch 1/1...\n",
            "Loading pose model from ./models/sppe/duc_se.pth\n",
            "\n",
            "Total time taken for model evaluation: 40.847s\n",
            "\n",
            "Submitting predictions\n",
            "{'accuracy': 0.6014,\n",
            " 'submission_time': '2019-06-13 09:57:07.789606+08:00',\n",
            " 'submission_type': 'testset_11classes_1_01010',\n",
            " 'team_id': 'Team Chi'}\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}