Copy_of_Yolov3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Yolov3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/silver612/ML_pathfinder/blob/master/Copy_of_Yolov3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7mBKBn01Q02z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3f82ab19-c469-4e12-8ba2-a20cd08c237b"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lwA7QYiRnfu_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        },
        "outputId": "80b75759-7618-48d4-b3a2-2f10253e6f0c"
      },
      "source": [
        "# download weights\n",
        "!wget https://pjreddie.com/media/files/yolov3.weights"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-09-18 12:57:53--  https://pjreddie.com/media/files/yolov3.weights\n",
            "Resolving pjreddie.com (pjreddie.com)... 128.208.4.108\n",
            "Connecting to pjreddie.com (pjreddie.com)|128.208.4.108|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 248007048 (237M) [application/octet-stream]\n",
            "Saving to: ‘yolov3.weights’\n",
            "\n",
            "yolov3.weights      100%[===================>] 236.52M   710KB/s    in 9m 33s  \n",
            "\n",
            "2020-09-18 13:07:27 (423 KB/s) - ‘yolov3.weights’ saved [248007048/248007048]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yhB1AQhTT1hE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#download test video\n",
        "import requests \n",
        "file_url = \"https://assets.mixkit.co/videos/download/mixkit-quiet-tokyo-street-at-night-4451.mp4\"\n",
        "\t\n",
        "r = requests.get(file_url, stream = True) \n",
        "\n",
        "with open(\"/content/gdrive/My Drive/python.pdf\", \"wb\") as file: \n",
        "\tfor block in r.iter_content(chunk_size = 1024): \n",
        "\t\tif block: \n",
        "\t\t\tfile.write(block) \n"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZzT9Aei52B40",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import scipy.io\n",
        "import scipy.misc\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import PIL\n",
        "import struct\n",
        "import cv2\n",
        "from numpy import expand_dims\n",
        "import tensorflow as tf\n",
        "from skimage.transform import resize\n",
        "from keras import backend as K\n",
        "from keras.layers import Input, Lambda, Conv2D, BatchNormalization, LeakyReLU, ZeroPadding2D, UpSampling2D\n",
        "from keras.models import load_model, Model\n",
        "from keras.layers.merge import add, concatenate\n",
        "from keras.preprocessing.image import load_img\n",
        "from keras.preprocessing.image import img_to_array\n",
        "from matplotlib import pyplot\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.pyplot import imshow\n",
        "from matplotlib.patches import Rectangle\n",
        "%matplotlib inline"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ia18mAg7RAUq",
        "colab_type": "text"
      },
      "source": [
        "**Step 1:** `WeightReader` class is used to parse the \"yolov3.weights\" file and load the model weights into memory in a format that we can set into keras model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5q0Aj3GUbadT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class WeightReader:\n",
        "\tdef __init__(self, weight_file):\n",
        "\t\twith open(weight_file, 'rb') as w_f:\n",
        "\t\t\tmajor,\t= struct.unpack('i', w_f.read(4))\n",
        "\t\t\tminor,\t= struct.unpack('i', w_f.read(4))\n",
        "\t\t\trevision, = struct.unpack('i', w_f.read(4))\n",
        "\t\t\tif (major*10 + minor) >= 2 and major < 1000 and minor < 1000:\n",
        "\t\t\t\tw_f.read(8)\n",
        "\t\t\telse:\n",
        "\t\t\t\tw_f.read(4)\n",
        "\t\t\ttranspose = (major > 1000) or (minor > 1000)\n",
        "\t\t\tbinary = w_f.read()\n",
        "\t\tself.offset = 0\n",
        "\t\tself.all_weights = np.frombuffer(binary, dtype='float32')\n",
        " \n",
        "\tdef read_bytes(self, size):\n",
        "\t\tself.offset = self.offset + size\n",
        "\t\treturn self.all_weights[self.offset-size:self.offset]\n",
        " \n",
        "\tdef load_weights(self, model):\n",
        "\t\tfor i in range(106):\n",
        "\t\t\ttry:\n",
        "\t\t\t\tconv_layer = model.get_layer('conv_' + str(i))\n",
        "\t\t\t\tprint(\"loading weights of convolution #\" + str(i))\n",
        "\t\t\t\tif i not in [81, 93, 105]:\n",
        "\t\t\t\t\tnorm_layer = model.get_layer('bnorm_' + str(i))\n",
        "\t\t\t\t\tsize = np.prod(norm_layer.get_weights()[0].shape)\n",
        "\t\t\t\t\tbeta  = self.read_bytes(size) # bias\n",
        "\t\t\t\t\tgamma = self.read_bytes(size) # scale\n",
        "\t\t\t\t\tmean  = self.read_bytes(size) # mean\n",
        "\t\t\t\t\tvar   = self.read_bytes(size) # variance\n",
        "\t\t\t\t\tweights = norm_layer.set_weights([gamma, beta, mean, var])\n",
        "\t\t\t\tif len(conv_layer.get_weights()) > 1:\n",
        "\t\t\t\t\tbias   = self.read_bytes(np.prod(conv_layer.get_weights()[1].shape))\n",
        "\t\t\t\t\tkernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
        "\t\t\t\t\tkernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
        "\t\t\t\t\tkernel = kernel.transpose([2,3,1,0])\n",
        "\t\t\t\t\tconv_layer.set_weights([kernel, bias])\n",
        "\t\t\t\telse:\n",
        "\t\t\t\t\tkernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
        "\t\t\t\t\tkernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
        "\t\t\t\t\tkernel = kernel.transpose([2,3,1,0])\n",
        "\t\t\t\t\tconv_layer.set_weights([kernel])\n",
        "\t\t\texcept ValueError:\n",
        "\t\t\t\tprint(\"no convolution #\" + str(i))\n",
        " \n",
        "\tdef reset(self):\n",
        "\t\tself.offset = 0"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j63SVedVbb0a",
        "colab_type": "text"
      },
      "source": [
        "**Step 2:** \n",
        "- `_conv_block(input, convs, skip=True)` is a function to create convolutional layer\n",
        "- `make_yolov3_model()` is a function to create layers of convoluational and stack together as a whole yolo model\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdjWkAa4blVT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def _conv_block(inp, convs, skip=True):\n",
        "\tx = inp\n",
        "\tcount = 0\n",
        "\tfor conv in convs:\n",
        "\t\tif count == (len(convs) - 2) and skip:\n",
        "\t\t\tskip_connection = x\n",
        "\t\tcount += 1\n",
        "\t\tif conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top\n",
        "\t\tx = Conv2D(conv['filter'],\n",
        "\t\t\t\t   conv['kernel'],\n",
        "\t\t\t\t   strides=conv['stride'],\n",
        "\t\t\t\t   padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top\n",
        "\t\t\t\t   name='conv_' + str(conv['layer_idx']),\n",
        "\t\t\t\t   use_bias=False if conv['bnorm'] else True)(x)\n",
        "\t\tif conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x)\n",
        "\t\tif conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x)\n",
        "\treturn add([skip_connection, x]) if skip else x\n",
        "\n",
        "def make_yolov3_model():\n",
        "\tinput_image = Input(shape=(None, None, 3))\n",
        "\t# Layer  0 => 4\n",
        "\tx = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0},\n",
        "\t\t\t\t\t\t\t\t  {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1},\n",
        "\t\t\t\t\t\t\t\t  {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2},\n",
        "\t\t\t\t\t\t\t\t  {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}])\n",
        "\t# Layer  5 => 8\n",
        "\tx = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5},\n",
        "\t\t\t\t\t\t{'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6},\n",
        "\t\t\t\t\t\t{'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}])\n",
        "\t# Layer  9 => 11\n",
        "\tx = _conv_block(x, [{'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9},\n",
        "\t\t\t\t\t\t{'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}])\n",
        "\t# Layer 12 => 15\n",
        "\tx = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12},\n",
        "\t\t\t\t\t\t{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13},\n",
        "\t\t\t\t\t\t{'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}])\n",
        "\t# Layer 16 => 36\n",
        "\tfor i in range(7):\n",
        "\t\tx = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3},\n",
        "\t\t\t\t\t\t\t{'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}])\n",
        "\tskip_36 = x\n",
        "\t# Layer 37 => 40\n",
        "\tx = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37},\n",
        "\t\t\t\t\t\t{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38},\n",
        "\t\t\t\t\t\t{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}])\n",
        "\t# Layer 41 => 61\n",
        "\tfor i in range(7):\n",
        "\t\tx = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3},\n",
        "\t\t\t\t\t\t\t{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}])\n",
        "\tskip_61 = x\n",
        "\t# Layer 62 => 65\n",
        "\tx = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62},\n",
        "\t\t\t\t\t\t{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63},\n",
        "\t\t\t\t\t\t{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}])\n",
        "\t# Layer 66 => 74\n",
        "\tfor i in range(3):\n",
        "\t\tx = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3},\n",
        "\t\t\t\t\t\t\t{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}])\n",
        "\t# Layer 75 => 79\n",
        "\tx = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75},\n",
        "\t\t\t\t\t\t{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76},\n",
        "\t\t\t\t\t\t{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77},\n",
        "\t\t\t\t\t\t{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78},\n",
        "\t\t\t\t\t\t{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False)\n",
        "\t# Layer 80 => 82\n",
        "\tyolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 80},\n",
        "\t\t\t\t\t\t\t  {'filter':  255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False)\n",
        "\t# Layer 83 => 86\n",
        "\tx = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False)\n",
        "\tx = UpSampling2D(2)(x)\n",
        "\tx = concatenate([x, skip_61])\n",
        "\t# Layer 87 => 91\n",
        "\tx = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87},\n",
        "\t\t\t\t\t\t{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88},\n",
        "\t\t\t\t\t\t{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89},\n",
        "\t\t\t\t\t\t{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90},\n",
        "\t\t\t\t\t\t{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False)\n",
        "\t# Layer 92 => 94\n",
        "\tyolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 92},\n",
        "\t\t\t\t\t\t\t  {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False)\n",
        "\t# Layer 95 => 98\n",
        "\tx = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True,   'layer_idx': 96}], skip=False)\n",
        "\tx = UpSampling2D(2)(x)\n",
        "\tx = concatenate([x, skip_36])\n",
        "\t# Layer 99 => 106\n",
        "\tyolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 99},\n",
        "\t\t\t\t\t\t\t   {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 100},\n",
        "\t\t\t\t\t\t\t   {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 101},\n",
        "\t\t\t\t\t\t\t   {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 102},\n",
        "\t\t\t\t\t\t\t   {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 103},\n",
        "\t\t\t\t\t\t\t   {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 104},\n",
        "\t\t\t\t\t\t\t   {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False)\n",
        "\tmodel = Model(input_image, [yolo_82, yolo_94, yolo_106])\n",
        "\treturn model"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eIh-jDWU8bR",
        "colab_type": "text"
      },
      "source": [
        "**Step 3:** \n",
        "- define the model\n",
        "- load the weight\n",
        "- save the model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e-9FSEgMe-sM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "bf83aafc-bacd-496f-ebe1-2136813179cb"
      },
      "source": [
        "# define the yolo v3 model\n",
        "yolov3 = make_yolov3_model()\n",
        "\n",
        "# load the weights\n",
        "weight_reader = WeightReader('/content/yolov3.weights')\n",
        "\n",
        "# set the weights\n",
        "weight_reader.load_weights(yolov3)\n",
        "\n",
        "# save the model to file\n",
        "yolov3.save('model.h5')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading weights of convolution #0\n",
            "loading weights of convolution #1\n",
            "loading weights of convolution #2\n",
            "loading weights of convolution #3\n",
            "no convolution #4\n",
            "loading weights of convolution #5\n",
            "loading weights of convolution #6\n",
            "loading weights of convolution #7\n",
            "no convolution #8\n",
            "loading weights of convolution #9\n",
            "loading weights of convolution #10\n",
            "no convolution #11\n",
            "loading weights of convolution #12\n",
            "loading weights of convolution #13\n",
            "loading weights of convolution #14\n",
            "no convolution #15\n",
            "loading weights of convolution #16\n",
            "loading weights of convolution #17\n",
            "no convolution #18\n",
            "loading weights of convolution #19\n",
            "loading weights of convolution #20\n",
            "no convolution #21\n",
            "loading weights of convolution #22\n",
            "loading weights of convolution #23\n",
            "no convolution #24\n",
            "loading weights of convolution #25\n",
            "loading weights of convolution #26\n",
            "no convolution #27\n",
            "loading weights of convolution #28\n",
            "loading weights of convolution #29\n",
            "no convolution #30\n",
            "loading weights of convolution #31\n",
            "loading weights of convolution #32\n",
            "no convolution #33\n",
            "loading weights of convolution #34\n",
            "loading weights of convolution #35\n",
            "no convolution #36\n",
            "loading weights of convolution #37\n",
            "loading weights of convolution #38\n",
            "loading weights of convolution #39\n",
            "no convolution #40\n",
            "loading weights of convolution #41\n",
            "loading weights of convolution #42\n",
            "no convolution #43\n",
            "loading weights of convolution #44\n",
            "loading weights of convolution #45\n",
            "no convolution #46\n",
            "loading weights of convolution #47\n",
            "loading weights of convolution #48\n",
            "no convolution #49\n",
            "loading weights of convolution #50\n",
            "loading weights of convolution #51\n",
            "no convolution #52\n",
            "loading weights of convolution #53\n",
            "loading weights of convolution #54\n",
            "no convolution #55\n",
            "loading weights of convolution #56\n",
            "loading weights of convolution #57\n",
            "no convolution #58\n",
            "loading weights of convolution #59\n",
            "loading weights of convolution #60\n",
            "no convolution #61\n",
            "loading weights of convolution #62\n",
            "loading weights of convolution #63\n",
            "loading weights of convolution #64\n",
            "no convolution #65\n",
            "loading weights of convolution #66\n",
            "loading weights of convolution #67\n",
            "no convolution #68\n",
            "loading weights of convolution #69\n",
            "loading weights of convolution #70\n",
            "no convolution #71\n",
            "loading weights of convolution #72\n",
            "loading weights of convolution #73\n",
            "no convolution #74\n",
            "loading weights of convolution #75\n",
            "loading weights of convolution #76\n",
            "loading weights of convolution #77\n",
            "loading weights of convolution #78\n",
            "loading weights of convolution #79\n",
            "loading weights of convolution #80\n",
            "loading weights of convolution #81\n",
            "no convolution #82\n",
            "no convolution #83\n",
            "loading weights of convolution #84\n",
            "no convolution #85\n",
            "no convolution #86\n",
            "loading weights of convolution #87\n",
            "loading weights of convolution #88\n",
            "loading weights of convolution #89\n",
            "loading weights of convolution #90\n",
            "loading weights of convolution #91\n",
            "loading weights of convolution #92\n",
            "loading weights of convolution #93\n",
            "no convolution #94\n",
            "no convolution #95\n",
            "loading weights of convolution #96\n",
            "no convolution #97\n",
            "no convolution #98\n",
            "loading weights of convolution #99\n",
            "loading weights of convolution #100\n",
            "loading weights of convolution #101\n",
            "loading weights of convolution #102\n",
            "loading weights of convolution #103\n",
            "loading weights of convolution #104\n",
            "loading weights of convolution #105\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x779EbsufcLj",
        "colab_type": "text"
      },
      "source": [
        "**step 4:** Prediction\n",
        " by loading the image to model and make prediction\n",
        " \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IisXY-Whl3az",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_image_pixels(filename, shape):\n",
        "  # load image to get its shape\n",
        "  image = load_img(filename)\n",
        "  width, height = image.size\n",
        "\n",
        "  # load image with required size\n",
        "  image = load_img(filename, target_size=shape)\n",
        "  image = img_to_array(image)\n",
        "\n",
        "  # grayscale image normalization\n",
        "  image = image.astype('float32')\n",
        "  image /= 255.0\n",
        "\n",
        "  # add a dimension so that we have one sample\n",
        "  image = expand_dims(image, 0)\n",
        "  return image, width, height"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vAST0Xi0odmj",
        "colab_type": "text"
      },
      "source": [
        "**Step 4:** Decode the prediction output to rectangle coordinates\n",
        "- `BoundBox` class is used to return object bounding box coordinates, object name and threshold score\n",
        "- `decode_netout` function is used to decode the prediction output to rectangle coordinates"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dd4lNvV6G_S_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BoundBox:\n",
        "  def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None):\n",
        "    self.xmin = xmin\n",
        "    self.ymin = ymin\n",
        "    self.xmax = xmax\n",
        "    self.ymax = ymax\n",
        "    self.objness = objness\n",
        "    self.classes = classes\n",
        "    self.label = -1\n",
        "    self.score = -1\n",
        "\n",
        "  def get_label(self):\n",
        "    if self.label == -1:\n",
        "      self.label = np.argmax(self.classes)\n",
        "    \n",
        "    return self.label\n",
        "  \n",
        "  def get_score(self):\n",
        "    if self.score == -1:\n",
        "      self.score = self.classes[self.get_label()]\n",
        "    return self.get_score\n",
        "\n",
        "def _sigmoid(x):\n",
        "  return 1. /(1. + np.exp(-x))\n",
        "\n",
        "def decode_netout(netout, anchors, obj_thresh, net_h, net_w):\n",
        "\tgrid_h, grid_w = netout.shape[:2]\n",
        "\tnb_box = 3\n",
        "\tnetout = netout.reshape((grid_h, grid_w, nb_box, -1))\n",
        "\tnb_class = netout.shape[-1] - 5\n",
        "\tboxes = []\n",
        "\tnetout[..., :2]  = _sigmoid(netout[..., :2])\n",
        "\tnetout[..., 4:]  = _sigmoid(netout[..., 4:])\n",
        "\tnetout[..., 5:]  = netout[..., 4][..., np.newaxis] * netout[..., 5:]\n",
        "\tnetout[..., 5:] *= netout[..., 5:] > obj_thresh\n",
        "\t\n",
        "\tfor i in range(grid_h*grid_w):\n",
        "\t\trow = i / grid_w\n",
        "\t\tcol = i % grid_w\n",
        "\t\tfor b in range(nb_box):\n",
        "\t\t\t# 4th element is objectness score\n",
        "\t\t\tobjectness = netout[int(row)][int(col)][b][4]\n",
        "\t\t\tif(objectness.all() <= obj_thresh): continue\n",
        "\t\t\t# first 4 elements are x, y, w, and h\n",
        "\t\t\tx, y, w, h = netout[int(row)][int(col)][b][:4]\n",
        "\t\t\tx = (col + x) / grid_w # center position, unit: image width\n",
        "\t\t\ty = (row + y) / grid_h # center position, unit: image height\n",
        "\t\t\tw = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n",
        "\t\t\th = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height\n",
        "\t\t\t# last elements are class probabilities\n",
        "\t\t\tclasses = netout[int(row)][col][b][5:]\n",
        "\t\t\tbox = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)\n",
        "\t\t\tboxes.append(box)\n",
        "\treturn boxes"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xy27Iyevk-SR",
        "colab_type": "text"
      },
      "source": [
        "**Step 5:** strech the box to be fit to the image normal shape"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fZgL2rGkXZk2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n",
        "\tnew_w, new_h = net_w, net_h\n",
        "\tfor i in range(len(boxes)):\n",
        "\t\tx_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w\n",
        "\t\ty_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h\n",
        "\t\tboxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n",
        "\t\tboxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n",
        "\t\tboxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n",
        "\t\tboxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6FKaEI6KlKue",
        "colab_type": "text"
      },
      "source": [
        "**Step 6:** implementing IOU\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MYY2hRhkMYzf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def _interval_overlap(interval_a, interval_b):\n",
        "\tx1, x2 = interval_a\n",
        "\tx3, x4 = interval_b\n",
        "\tif x3 < x1:\n",
        "\t\tif x4 < x1:\n",
        "\t\t\treturn 0\n",
        "\t\telse:\n",
        "\t\t\treturn min(x2,x4) - x1\n",
        "\telse:\n",
        "\t\tif x2 < x3:\n",
        "\t\t\t return 0\n",
        "\t\telse:\n",
        "\t\t\treturn min(x2,x4) - x3\n",
        " \n",
        "def bbox_iou(box1, box2):\n",
        "\tintersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n",
        "\tintersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])\n",
        "\tintersect = intersect_w * intersect_h\n",
        "\tw1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n",
        "\tw2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n",
        "\tunion = w1*h1 + w2*h2 - intersect\n",
        "\treturn float(intersect) / union\n",
        " \n",
        "def do_nms(boxes, nms_thresh):\n",
        "\tif len(boxes) > 0:\n",
        "\t\tnb_class = len(boxes[0].classes)\n",
        "\telse:\n",
        "\t\treturn\n",
        "\tfor c in range(nb_class):\n",
        "\t\tsorted_indices = np.argsort([-box.classes[c] for box in boxes])\n",
        "\t\tfor i in range(len(sorted_indices)):\n",
        "\t\t\tindex_i = sorted_indices[i]\n",
        "\t\t\tif boxes[index_i].classes[c] == 0: continue\n",
        "\t\t\tfor j in range(i+1, len(sorted_indices)):\n",
        "\t\t\t\tindex_j = sorted_indices[j]\n",
        "\t\t\t\tif bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n",
        "\t\t\t\t\tboxes[index_j].classes[c] = 0\n",
        "\n",
        "# get all of the results above a threshold\n",
        "def get_boxes(boxes, labels, thresh):\n",
        "\tv_boxes, v_labels, v_scores = list(), list(), list()\n",
        "\t# enumerate all boxes\n",
        "\tfor box in boxes:\n",
        "\t\t# enumerate all possible labels\n",
        "\t\tfor i in range(len(labels)):\n",
        "\t\t\t# check if the threshold for this label is high enough\n",
        "\t\t\tif box.classes[i] > thresh:\n",
        "\t\t\t\tv_boxes.append(box)\n",
        "\t\t\t\tv_labels.append(labels[i])\n",
        "\t\t\t\tv_scores.append(box.classes[i]*100)\n",
        "\t\t\t\t# don't break, many labels may trigger for one box\n",
        "\treturn v_boxes, v_labels, v_scores\n",
        "\n",
        "# draw all results\n",
        "def draw_boxes(filename, v_boxes, v_labels, v_scores):\n",
        "  \n",
        "\t# load the image\n",
        "\tdata = pyplot.imread(filename)\n",
        "\t# plot the image\n",
        "\tpyplot.imshow(data)\n",
        "\t# get the context for drawing boxes\n",
        "\tax = pyplot.gca()\n",
        "\t# plot each box\n",
        "\tfor i in range(len(v_boxes)):\n",
        "\t\tbox = v_boxes[i]\n",
        "\t\t# get coordinates\n",
        "\t\ty1, x1, y2, x2 = box.ymin, box.xmin, box.ymax, box.xmax\n",
        "\t\t# calculate width and height of the box\n",
        "\t\twidth, height = x2 - x1, y2 - y1\n",
        "\n",
        "\t\tif v_labels[i] == 'person':\n",
        "\n",
        "\t\t\t\n",
        "\t\t\tx1 = x1 + float(width) / 4\n",
        "\t\t\theight = height / 7\n",
        "\t\t\twidth = width / 2 \n",
        "\t\t# create the shape\n",
        "\t\t\trect = Rectangle((x1, y1), width, height, fill=False, color='red', linewidth = '2')\n",
        "\t\t# draw the box\n",
        "\t\t\tax.add_patch(rect)\n",
        "\t\t# draw text and score in top left corner\n",
        "\t\t\tlabel = \"%s\" % ('face' + str(i+1))\n",
        "\t\t\tpyplot.text(x1, y1, label, color='red')\n",
        "\t# show the plot\n",
        "\tpyplot.show()"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AcJbSoiBlWwY",
        "colab_type": "text"
      },
      "source": [
        "**step 7:** declare several configuration\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LglOJs4IOYi7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# define the anchors\n",
        "anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]\n",
        "\n",
        "# define the probability threshold for detected objects\n",
        "class_threshold = 0.6\n",
        "\n",
        "# define the labels\n",
        "labels = [\"person\", \"bicycle\", \"car\", \"motorbike\", \"aeroplane\", \"bus\", \"train\", \"truck\",\n",
        "\t\"boat\", \"traffic light\", \"fire hydrant\", \"stop sign\", \"parking meter\", \"bench\",\n",
        "\t\"bird\", \"cat\", \"dog\", \"horse\", \"sheep\", \"cow\", \"elephant\", \"bear\", \"zebra\", \"giraffe\",\n",
        "\t\"backpack\", \"umbrella\", \"handbag\", \"tie\", \"suitcase\", \"frisbee\", \"skis\", \"snowboard\",\n",
        "\t\"sports ball\", \"kite\", \"baseball bat\", \"baseball glove\", \"skateboard\", \"surfboard\",\n",
        "\t\"tennis racket\", \"bottle\", \"wine glass\", \"cup\", \"fork\", \"knife\", \"spoon\", \"bowl\", \"banana\",\n",
        "\t\"apple\", \"sandwich\", \"orange\", \"broccoli\", \"carrot\", \"hot dog\", \"pizza\", \"donut\", \"cake\",\n",
        "\t\"chair\", \"sofa\", \"pottedplant\", \"bed\", \"diningtable\", \"toilet\", \"tvmonitor\", \"laptop\", \"mouse\",\n",
        "\t\"remote\", \"keyboard\", \"cell phone\", \"microwave\", \"oven\", \"toaster\", \"sink\", \"refrigerator\",\n",
        "\t\"book\", \"clock\", \"vase\", \"scissors\", \"teddy bear\", \"hair drier\", \"toothbrush\"]"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MDeJobbelfJe",
        "colab_type": "text"
      },
      "source": [
        "**Step 8:** Make Prediction\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kxDM1To3POkD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#from google.colab import files \n",
        "#upload = files.upload()\n",
        "\n",
        "#for fn in upload.keys():\n",
        "# photo_filename = '/content/' + fn\n",
        "\n",
        "  # define the expected input shape for the model\n",
        "input_w, input_h = 416, 416\n",
        "\n",
        "# for camera use, put 0 as argument\n",
        "# currently running on test video\n",
        "vidObj = cv2.VideoCapture('/content/gdrive/My Drive/python.pdf')\n",
        "count = 0\n",
        "success = 1\n",
        "while success:\n",
        "  success,image = vidObj.read()\n",
        "  photo_filename = str(count) + \".jpg\"\n",
        "  cv2.imwrite(photo_filename,image)\n",
        "  count += 1 \n",
        "  image, image_w, image_h = load_image_pixels(photo_filename, (input_w, input_h))\n",
        "\n",
        "  # make prediction\n",
        "  yhat = yolov3.predict(image)\n",
        "  # summarize the shape of the list of arrays\n",
        "  #print([a.shape for a in yhat])\n",
        "  \n",
        "  boxes = list() \n",
        "  for i in range(len(yhat)):\n",
        "    # decode the output of the network\n",
        "    boxes += decode_netout(yhat[i][0], anchors[i], class_threshold, input_h, input_w)\n",
        " \n",
        "  # correct the sizes of the bounding boxes for the shape of the image\n",
        "  correct_yolo_boxes(boxes, image_h, image_w, input_h, input_w)\n",
        "\n",
        "  # suppress non-maximal boxes\n",
        "  do_nms(boxes, 0.5)\n",
        "\n",
        "  # get the details of the detected objects\n",
        "  v_boxes, v_labels, v_scores = get_boxes(boxes, labels, class_threshold)\n",
        "\n",
        "  # summarize what we found\n",
        "  for i in range(len(v_boxes)):\n",
        "    if v_labels[i] == 'person':\n",
        "      print('face '+str(i+1))\n",
        "  print(photo_filename)\n",
        "  # draw what we found\n",
        "  draw_boxes(photo_filename, v_boxes, v_labels, v_scores)\n",
        "\n",
        "  try:\n",
        "    os.remove('/content/' + photo_filename)\n",
        "  except: pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l7PdGDohhBIv",
        "colab_type": "text"
      },
      "source": [
        "# References:\n",
        "\n",
        "\n",
        "1.   Answer by berak: https://stackoverflow.com/questions/25156547/delete-an-image-in-opencv-using-python\n",
        "2.  Sample YOLO program by christienatashiaarchie:https://github.com/christienatashiaarchie/YOLOv3-Object-Detection/blob/master/Yolov3.ipynb\n",
        "3. Test video 'Quiet Tokyo Street at Night':https://mixkit.co/free-stock-video/quiet-tokyo-street-at-night-4451/\n",
        "\n"
      ]
    }
  ]
}