u-ml-face-parts-detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Detect eyes, nose, lips, and jaw with dlib, OpenCV, and Python\n",
    "\n",
    "In this lab we are going to learn how to extract face regions individually using dlib, OpenCV, and Python.\n",
    "\n",
    "<img src=\"images/ex-face-detection.png\" width=\"500\">\n",
    "\n",
    "Original source https://www.pyimagesearch.com/2017/04/10/detect-eyes-nose-lips-jaw-dlib-opencv-python/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.spatial import distance as dist\n",
    "from imutils.video import VideoStream\n",
    "from imutils import face_utils\n",
    "import argparse\n",
    "import imutils\n",
    "import time\n",
    "import dlib\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] loading facial landmark predictor...\n"
     ]
    }
   ],
   "source": [
    "args = {}\n",
    "args[\"shape_predictor\"] = \"downloads/models-cia/shape_predictor_68_face_landmarks.dat\"\n",
    "\n",
    "# define two constants, one for the eye aspect ratio to indicate\n",
    "# blink and then a second constant for the number of consecutive\n",
    "# frames the eye must be below the threshold for to set off the\n",
    "# alarm\n",
    "EYE_AR_THRESH = 0.3\n",
    "EYE_AR_CONSEC_FRAMES = 38\n",
    "\n",
    "MOUTH_AR_CONSEC_FRAMES = 28\n",
    "MOUTH_AR_THRESH = 1.0\n",
    "\n",
    "# initialize the frame counter as well as a boolean used to\n",
    "# indicate if the alarm is going off\n",
    "EYE_COUNTER = 0\n",
    "MOUTH_COUNTER = 0.3\n",
    "\n",
    "\n",
    "# initialize dlib's face detector (HOG-based) and then create\n",
    "# the facial landmark predictor\n",
    "print(\"[INFO] loading facial landmark predictor...\")\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "predictor = dlib.shape_predictor(args[\"shape_predictor\"])\n",
    "\n",
    "# grab the indexes of the facial landmarks for the left and\n",
    "# right eye, respectively\n",
    "(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS[\"left_eye\"]\n",
    "(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS[\"right_eye\"]\n",
    "\n",
    "(mouthStart, mouthEnd) = face_utils.FACIAL_LANDMARKS_IDXS[\"mouth\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def roi_aspect_ratio(roi):\n",
    "\t# compute the euclidean distances between the two sets of\n",
    "\t# vertical roi landmarks (x, y)-coordinates\n",
    "\tA = dist.euclidean(roi[1], roi[5])\n",
    "\tB = dist.euclidean(roi[2], roi[4])\n",
    "\n",
    "\t# compute the euclidean distance between the horizontal\n",
    "\t# eye landmark (x, y)-coordinates\n",
    "\tC = dist.euclidean(roi[0], roi[3])\n",
    "\n",
    "\t# compute the eye aspect ratio\n",
    "\tear = (A + B) / (2.0 * C)\n",
    "\n",
    "\t# return the eye aspect ratio\n",
    "\treturn ear\n",
    "\n",
    "\n",
    "\n",
    "def drowsiness_process(ratio):\n",
    "\tglobal EYE_COUNTER\n",
    "\t# check to see if the eye aspect ratio is below the blink\n",
    "\t# threshold, and if so, increment the blink frame counter\n",
    "\tif ratio < EYE_AR_THRESH:\n",
    "\t\tEYE_COUNTER += 1\n",
    "\n",
    "\t\t# if the eyes were closed for a sufficient number of\n",
    "\t\t# then sound the alarm\n",
    "\t\tif EYE_COUNTER >= EYE_AR_CONSEC_FRAMES:\n",
    "\t\t\t# draw an alarm on the frame\n",
    "\t\t\tcv2.putText(frame, \"eyes closed\", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "\n",
    "\t# otherwise, the eye aspect ratio is not below the blink\n",
    "\t# threshold, so reset the counter and alarm\n",
    "\telse:\n",
    "\t\tEYE_COUNTER = 0\n",
    "\n",
    "\n",
    "# loop over frames from the video stream\n",
    "def happiness_process(ratio):\n",
    "\tglobal MOUTH_COUNTER\n",
    "\t# check to see if the mouth aspect ratio is below the threshold, and if so, increment the mount frame counter\n",
    "\tif ratio > MOUTH_AR_THRESH:\n",
    "\t\tMOUTH_COUNTER += 1\n",
    "\n",
    "\t\t# if the eyes were closed for a sufficient number of\n",
    "\t\t# then sound the alarm\n",
    "\t\tif MOUTH_COUNTER >= MOUTH_AR_CONSEC_FRAMES:\n",
    "\t\t\t# draw an alarm on the frame\n",
    "\t\t\tcv2.putText(frame, \"Smilling!\", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)\n",
    "\n",
    "\t# otherwise, the eye aspect ratio is not below the threshold, so reset the counter and alarm\n",
    "\telse:\n",
    "\t\tMOUTH_COUNTER = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] starting video stream thread...\n"
     ]
    },
    {
     "ename": "error",
     "evalue": "OpenCV(4.2.0) /Users/travis/build/skvark/opencv-python/opencv/modules/imgproc/src/resize.cpp:4045: error: (-215:Assertion failed) !ssize.empty() in function 'resize'\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31merror\u001b[0m                                     Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-5265775243df>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     64\u001b[0m                         \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mboundingRect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m                         \u001b[0mroi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mframe\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mw\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m                         \u001b[0mroi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mroi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m250\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mINTER_CUBIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0;31m# show the frame\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/ComputerVision/lib/python3.7/site-packages/imutils/convenience.py\u001b[0m in \u001b[0;36mresize\u001b[0;34m(image, width, height, inter)\u001b[0m\n\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;31m# resize the image\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m     \u001b[0mresized\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minterpolation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;31m# return the resized image\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31merror\u001b[0m: OpenCV(4.2.0) /Users/travis/build/skvark/opencv-python/opencv/modules/imgproc/src/resize.cpp:4045: error: (-215:Assertion failed) !ssize.empty() in function 'resize'\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# start the video stream thread\n",
    "print(\"[INFO] starting video stream thread...\")\n",
    "vs = VideoStream(src=0).start()\n",
    "time.sleep(1.0)\n",
    "\n",
    "while True:\n",
    "\t# grab the frame from the threaded video file stream, resize\n",
    "\t# it, and convert it to grayscale\n",
    "\t# channels)\n",
    "\tframe = vs.read()\n",
    "\tframe = imutils.resize(frame, width=450)\n",
    "\tgray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "\t# detect faces in the grayscale frame\n",
    "\trects = detector(gray, 0)\n",
    "\n",
    "\t# loop over the face detections\n",
    "\tfor rect in rects:\n",
    "\t\t# determine the facial landmarks for the face region, then\n",
    "\t\t# convert the facial landmark (x, y)-coordinates to a NumPy\n",
    "\t\t# array\n",
    "\t\tshape = predictor(gray, rect)\n",
    "\t\tshape = face_utils.shape_to_np(shape)\n",
    "\n",
    "\t\t# extract the left and right eye coordinates, then use the\n",
    "\t\t# coordinates to compute the eye aspect ratio for both eyes\n",
    "\t\tleftEye = shape[lStart:lEnd]\n",
    "\t\trightEye = shape[rStart:rEnd]\n",
    "\t\tleftEAR = roi_aspect_ratio(leftEye)\n",
    "\t\trightEAR = roi_aspect_ratio(rightEye)\n",
    "\n",
    "\t\t# average the eye aspect ratio together for both eyes\n",
    "\t\tear = (leftEAR + rightEAR) / 2.0\n",
    "\n",
    "\t\t# compute the convex hull for the left and right eye, then\n",
    "\t\t# visualize each of the eyes\n",
    "\t\tleftEyeHull = cv2.convexHull(leftEye)\n",
    "\t\trightEyeHull = cv2.convexHull(rightEye)\n",
    "\t\tcv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)\n",
    "\t\tcv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)\n",
    "\n",
    "\t\t# Doing the same process for the mouth\n",
    "\t\tmouth = shape[mouthStart:mouthEnd]\n",
    "\t\tmouthEAR = roi_aspect_ratio(mouth)\n",
    "\t\tmouthHull = cv2.convexHull(mouth)\n",
    "\t\tcv2.drawContours(frame, [mouthHull], -1, (0, 255, 0), 1)\n",
    "\n",
    "\t\tdrowsiness_process(ear)\n",
    "\t\thappiness_process(mouthEAR)\n",
    "\n",
    "\t\t# draw the computed eye aspect ratio on the frame to help\n",
    "\t\t# with debugging and setting the correct eye aspect ratio\n",
    "\t\t# thresholds and frame counters\n",
    "\t\tcv2.putText(frame, \"EAR: {:.2f}\".format(mouthEAR), (300, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "\n",
    "\t\t# loop over all face parts\n",
    "\t\tfor (name, (i, j)) in face_utils.FACIAL_LANDMARKS_IDXS.items():\n",
    "\t\t\t# loop over the subset of facial landmarks, drawing the\n",
    "\t\t\t# specific face part\n",
    "\t\t\tfor (x, y) in shape[i:j]:\n",
    "\t\t\t\tcv2.circle(frame, (x, y), 1, (0, 0, 255), -1)\n",
    "\n",
    "\t\t\t# extract the ROI of the face region as a separate image\n",
    "\t\t\t(x, y, w, h) = cv2.boundingRect(np.array([shape[i:j]]))\n",
    "\t\t\troi = frame[y:y + h, x:x + w]\n",
    "\t\t\troi = imutils.resize(roi, width=250, inter=cv2.INTER_CUBIC)\n",
    "\n",
    "\t# show the frame\n",
    "\tcv2.imshow(\"Frame\", frame)\n",
    "\tkey = cv2.waitKey(1) & 0xFF\n",
    "\n",
    "\t# if the `q` key was pressed, break from the loop\n",
    "\tif key == ord(\"q\"):\n",
    "\t\tbreak\n",
    "\n",
    "# do a bit of cleanup\n",
    "cv2.destroyAllWindows()\n",
    "vs.stop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}