0-yolo+opencv-customweights.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: opencv-contrib-python in c:\\programdata\\anaconda3\\envs\\pytorch\\lib\\site-packages (4.2.0.32)\n",
      "Requirement already satisfied: numpy>=1.11.3 in c:\\programdata\\anaconda3\\envs\\pytorch\\lib\\site-packages (from opencv-contrib-python) (1.16.5)\n"
     ]
    }
   ],
   "source": [
    "#!pip install opencv-python\n",
    "\"\"\"\n",
    "use this instead pip3 install opencv-contrib-python\n",
    "\"\"\"\n",
    "!pip install opencv-contrib-python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I have downloaded .names, .cfg and .weights file. \n",
    "First, I will build a yolov3 with opencv demo. Just to quil demonstrate how it works."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2 as cv\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "confidence_threshold = 0.10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NMS:\n",
    "\n",
    "Input: A list of Proposal boxes B, corresponding confidence scores S and overlap threshold N.\n",
    "\n",
    "Output: A list of filtered proposals D."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "nms_threshold = 0.40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp_height = 416\n",
    "inp_width = 416"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes_file = \"obj.names\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_ids = []\n",
    "confidences = []\n",
    "boxes = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The rstrip() method returns a copy of the string with trailing characters removed \n",
    "split('\\n')Split a string into a list where each word is a list item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(classes_file,'rt')as f:\n",
    "    classes = f.read().rstrip('\\n').split('n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config = \"yolov3-tiny-obj.cfg\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_weights = \"yolov3-tiny-obj_5000.weights\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# setting up our neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "neural_net = cv.dnn.readNetFromDarknet(model_config, model_weights)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ask network to use specific computation backend where it supports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#neural_net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV)\n",
    "neural_net.setPreferableBackend(cv.dnn.DNN_BACKEND_CUDA)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ask network to make computations on specific target device.\n",
    "I will change this later on my cuda gpu and see if performs better but for now I will assume it's for raspberry pi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#neural_net.setPreferableTarget(cv.dnn.DNN_BACKEND_OPENCV)\n",
    "neural_net.setPreferableTarget(cv.dnn.DNN_BACKEND_OPENCV)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "setting up opencv window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "win_name = \"TinyYolov3 + OpenCV + Rocket\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv.namedWindow(win_name, cv.WINDOW_NORMAL)\n",
    "cv.resizeWindow(win_name, 800,800)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "capture = cv.VideoCapture(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getOutputsNames(neural_net):\n",
    "    layer_names = neural_net.getLayerNames()\n",
    "    #print(layer_names)\n",
    "    #print(layer_names[i[0] - 1] for i in neural_net.getUnconnectedOutLayers)\n",
    "    return [layer_names[i[0] - 1] for i in neural_net.getUnconnectedOutLayers()] #\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drawPred(class_id, confidence, left, top, right, bottom):\n",
    "    cv.rectangle(frame, (left,top), (right,bottom), (255, 178, 50))\n",
    "    label = \"%2f\"%confidence\n",
    "    \n",
    "    #get confidence and label for class name\n",
    "    if classes:\n",
    "        assert(class_id < len(classes))\n",
    "        label = \"%s:%s\"%(classes[class_id] , label)\n",
    "        \n",
    "    cv.putText(frame, label, (left,top), cv.FONT_HERSHEY_PLAIN, 2, (0,255,255), 4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def postProcess(frame, outputs):\n",
    "    frame_height = frame.shape[0]\n",
    "    frame_width = frame.shape[1]\n",
    "    \n",
    "    for output in outputs:\n",
    "        for detection in output:\n",
    "            scores = detection[5:] #list has centerx,centery,width,height,confi,0,1,...80 etc\n",
    "            class_id = np.argmax(scores)\n",
    "            confidence = scores[class_id]\n",
    "            if confidence > confidence_threshold:\n",
    "                \n",
    "                center_x = int(detection[0]*frame_width)\n",
    "                center_y = int(detection[1]*frame_height)\n",
    "                \n",
    "                width = int(detection[2]*frame_width)\n",
    "                height = int(detection[3]*frame_height)\n",
    "                \n",
    "                left = int(center_x - width/2)\n",
    "                top = int(center_y - height/2)\n",
    "                \n",
    "                class_ids.append(class_id)\n",
    "                confidences.append(float(confidence))\n",
    "                \n",
    "                boxes.append([left, top, width, height])\n",
    "                \n",
    "            indices = cv.dnn.NMSBoxes(boxes, confidences,\n",
    "                                      confidence_threshold,\n",
    "                                      nms_threshold)\n",
    "            \n",
    "            for i in indices:\n",
    "                i = i[0]\n",
    "                box = boxes[i]\n",
    "                left = box[0]\n",
    "                top = box[1]\n",
    "                width = box[2]\n",
    "                height = box[3]\n",
    "                \n",
    "                #we want to draw prediction\n",
    "                class_ids_i = class_ids[i]\n",
    "                confidences_i = confidences[i]\n",
    "                drawPred(class_ids_i,confidences_i,left, top, left + width, top+ height)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Incase you are curious what cv.waitKey does - https://stackoverflow.com/questions/5217519/what-does-opencvs-cvwaitkey-function-do "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "error",
     "evalue": "OpenCV(4.2.0) C:\\projects\\opencv-python\\opencv\\modules\\dnn\\src\\dnn.cpp:1322: error: (-215:Assertion failed) preferableBackend != DNN_BACKEND_CUDA || IS_DNN_CUDA_TARGET(preferableTarget) in function 'cv::dnn::dnn4_v20191202::Net::Impl::setUpNet'\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31merror\u001b[0m                                     Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-21-6b9c0247652f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     16\u001b[0m     \u001b[0mcustom\u001b[0m \u001b[0mfunctions\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m     \"\"\"\n\u001b[1;32m---> 18\u001b[1;33m     \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mneural_net\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgetOutputsNames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mneural_net\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     19\u001b[0m     \u001b[1;31m#post processing\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m     \u001b[0mpostProcess\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31merror\u001b[0m: OpenCV(4.2.0) C:\\projects\\opencv-python\\opencv\\modules\\dnn\\src\\dnn.cpp:1322: error: (-215:Assertion failed) preferableBackend != DNN_BACKEND_CUDA || IS_DNN_CUDA_TARGET(preferableTarget) in function 'cv::dnn::dnn4_v20191202::Net::Impl::setUpNet'\n"
     ]
    }
   ],
   "source": [
    "while cv.waitKey(1) <0:\n",
    "    has_frame,frame = capture.read() #capturing\n",
    "    \"\"\"blob is binary large object that\n",
    "    we want to fee to our neural network.\n",
    "    We convert pixel values to 0..1 by divinding each pixel value with 255(color,remember?).\n",
    "    The the image is resized depending on what we want without cropping!\n",
    "    We don't want to do any mean subtraction so just pass [0,0,0,0] in mean parameter else it's 1 y default\"\"\"\n",
    "    blob_from_image = cv.dnn.blobFromImage(frame, 1/255,\n",
    "                                     (inp_width,inp_height),\n",
    "                                     [0,0,0,0], crop = False\n",
    "                                          )\n",
    "    neural_net.setInput(blob_from_image)\n",
    "    #outs = neural_net.forward (getOutputsNames(net))\n",
    "    #net.getUnconnectedOutLayersNames()\n",
    "    \"\"\"\n",
    "    custom functions\n",
    "    \"\"\"\n",
    "    outputs = neural_net.forward(getOutputsNames(neural_net))\n",
    "    #post processing\n",
    "    postProcess(frame, outputs)\n",
    "    \n",
    "    #showing image\n",
    "    cv.imshow(win_name, frame)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}