Backend.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a3593e3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version:  2.6.0\n",
      "/device:GPU:0\n",
      "setup is ok\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'NVIDIA GeForce GTX 1650'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import cv2\n",
    "import os\n",
    "import random\n",
    "import math\n",
    "import datetime as dt\n",
    "from collections import deque\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import statistics\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import joblib\n",
    "\n",
    "import numpy as np\n",
    "from numpy.lib.scimath import sqrt # used for hoF\n",
    "from numpy import arctan2 # used for hoF\n",
    "\n",
    "from scipy import pi, cos, sin # used for HoF\n",
    "from scipy.ndimage import uniform_filter # used for hoF\n",
    "\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.svm import SVR\n",
    "\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.models import Sequential, Model, load_model\n",
    "from tensorflow.keras.utils import to_categorical, plot_model\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint,CSVLogger\n",
    "from tensorflow.keras.losses import categorical_crossentropy\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow as tf\n",
    "print(\"Tensorflow version: \", tf.__version__)\n",
    "print(tf.test.gpu_device_name())\n",
    "\n",
    "#for i3d extraction\n",
    "from models.i3d.extract_i3d import ExtractI3D\n",
    "from utils.utils import build_cfg_path\n",
    "from omegaconf import OmegaConf\n",
    "import torch\n",
    "\n",
    "import requests as requests\n",
    "from ast import literal_eval as make_tuple\n",
    "import os\n",
    "import urllib.request\n",
    "from flask import Flask, flash, request, redirect, url_for, render_template, jsonify\n",
    "from flask_cors import CORS\n",
    "from werkzeug.utils import secure_filename\n",
    "from werkzeug.wrappers import Request, Response\n",
    "import json\n",
    "import base64\n",
    "\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "torch.cuda.get_device_name(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3835b463",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Global Variables\n",
    "dive_action_labels  = ['Entry', 'Flight', 'Takeoff']\n",
    "temp_segment_model = None\n",
    "autoscore_model = None\n",
    "ss_twist_classifier_model = None\n",
    "ss_twist_endpoint = 'http://localhost:5001/sstwist'\n",
    "hybrid_model = None\n",
    "minmaxScalar = None\n",
    "folderpath      = 'modelcheckpoints/'\n",
    "\n",
    "#i3d variables\n",
    "i3dextractor = None\n",
    "stack_size = 12\n",
    "step_size = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c6a7d93a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_temporal_segment_model():\n",
    "    print('loading temporal segment model')\n",
    "    global temp_segment_model\n",
    "    image_height, image_width = 64, 64\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', input_shape = (image_height, image_width, 3)))\n",
    "    model.add(Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "    model.add(GlobalAveragePooling2D())\n",
    "    model.add(Dense(256, activation = 'relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(3, activation = 'softmax'))\n",
    "    model.compile(loss=categorical_crossentropy, optimizer=Adam(), metrics=['accuracy'])\n",
    "    model.load_weights(folderpath+\"model_2D.h5\")\n",
    "    temp_segment_model = model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dc672abe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_autoscore_model():\n",
    "    print('loading autoscore model')\n",
    "    global autoscore_model\n",
    "    if (autoscore_model == None):\n",
    "        autoscore_model = load_model('modelcheckpoints/fullyconnected_ID128D96D1_0.1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22f1eb0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_ss_twist_classifier_model():\n",
    "    print('loading ss_twist_classifier model')\n",
    "    global ss_twist_classifier_model\n",
    "    image_height, image_width = 64, 64\n",
    "    optim = tf.keras.optimizers.Adam(0.0001)\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu', input_shape = (image_height, image_width, 3)))\n",
    "    model.add(Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "    model.add(Conv2D(filters = 64, kernel_size = (3, 3), activation = 'relu'))\n",
    "    model.add(GlobalAveragePooling2D())\n",
    "    model.add(Dense(256, activation = 'relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(2, activation = 'softmax'))\n",
    "    model.compile(loss=categorical_crossentropy, optimizer=optim, metrics=['accuracy'])\n",
    "    model.load_weights(folderpath+\"model_flightClassify_2D_3.h5\")\n",
    "    ss_twist_classifier_model = model\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9753b34e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_hybrid_model():\n",
    "    print('loading hybrid model')\n",
    "    global hybrid_model, minmaxScalar\n",
    "    hybrid_model = joblib.load(folderpath+'svr')\n",
    "    minmaxScalar = joblib.load(folderpath+'minMaxScaler')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "27ceb3f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ensureDirectoryClean(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)\n",
    "    else:\n",
    "        for f in os.listdir(dirpath):\n",
    "            os.remove(os.path.join(dirpath, f))\n",
    "\n",
    "def extractFolderAndFileNameFromAbsPath(absFilePath):\n",
    "    filename_sep = absFilePath.rindex('\\\\')\n",
    "    extension_sep = absFilePath.rindex(\".\")\n",
    "    folder = absFilePath[0: filename_sep]\n",
    "    shortfilename = absFilePath[filename_sep+1:extension_sep]\n",
    "    ext = absFilePath[extension_sep+1:len(absFilePath)]\n",
    "    return folder, shortfilename, ext\n",
    "\n",
    "def extractEventNoAndDiveNo(folderPath):\n",
    "    tokens = folderPath.split(\"\\\\\")\n",
    "    diveno = tokens[len(tokens)-1]\n",
    "    eventno = tokens[len(tokens)-2]\n",
    "    return eventno, diveno"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bb7121b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_temporal_segmentation(vidpath, imgOutputDir):\n",
    "    global temp_segment_model\n",
    "    window_size=3\n",
    "    predicted_label_list = []\n",
    "    predicted_labels_probabilities_deque = deque(maxlen = window_size)\n",
    " \n",
    "    video_reader = cv2.VideoCapture(vidpath)\n",
    " \n",
    "    original_video_width = int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "    original_video_height = int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "    fps = int(video_reader.get(cv2.CAP_PROP_FPS))\n",
    "    \n",
    "    folder, shortfilename, _ = extractFolderAndFileNameFromAbsPath(vidpath)\n",
    "    for dive_action_label in dive_action_labels:\n",
    "        subdir = imgOutputDir+\"\\\\\"+shortfilename+\"\\\\\"+ dive_action_label \n",
    "        ensureDirectoryClean(subdir)\n",
    "    \n",
    "    #video_writer = cv2.VideoWriter(imgOutputDir, cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), fps, (original_video_width, original_video_height))\n",
    "    count_seg=0\n",
    "    image_height, image_width = 64, 64\n",
    "    while True: \n",
    "        count_seg += 1 \n",
    "        status, frame = video_reader.read() \n",
    " \n",
    "        if not status:\n",
    "            break\n",
    "         \n",
    "        # predict frame type (Takeoff/Flight/Entry)\n",
    "        resized_frame = cv2.resize(frame, (image_height, image_width))\n",
    "        normalized_frame = resized_frame / 255 \n",
    "        predicted_labels_probabilities = temp_segment_model.predict(np.expand_dims(normalized_frame, axis = 0))[0] \n",
    "        predicted_labels_probabilities_deque.append(predicted_labels_probabilities)\n",
    " \n",
    "        if len(predicted_labels_probabilities_deque) == window_size:\n",
    "            predicted_labels_probabilities_np = np.array(predicted_labels_probabilities_deque)\n",
    "            #print('predicted_labels_probabilities_np : ',predicted_labels_probabilities_np)\n",
    "            predicted_labels_probabilities_averaged = predicted_labels_probabilities_np.mean(axis = 0)\n",
    "            #print('predicted_labels_probabilities_averaged',predicted_labels_probabilities_averaged)\n",
    "            predicted_label = np.argmax(predicted_labels_probabilities_averaged)\n",
    "            \n",
    "        else: # len(predicted_labels_probabilities_deque) < window_size\n",
    "            predicted_label = np.argmax(predicted_labels_probabilities) \n",
    "        \n",
    "        predicted_label_list.append(predicted_label)\n",
    "        predicted_class_name = dive_action_labels[predicted_label]\n",
    "        #print('predicted_class' , predicted_class_name)\n",
    "        \n",
    "        subdir = imgOutputDir+\"\\\\\"+shortfilename+\"\\\\\"+ predicted_class_name \n",
    "        imagename = subdir+\"\\\\\"+(\"frame%04d.jpg\" % count_seg)\n",
    "        #print('write to ', imagename)\n",
    "        cv2.imwrite(imagename, frame)\n",
    "\n",
    "    video_reader.release()\n",
    "    #video_writer.release()\n",
    "    return predicted_label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "13167e9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#resizeFrameDim [width, height]\n",
    "def createVideo(image_folder, video_folder, divephase, vidname, resizeFrame=False, resizeFrameDim=[64,64]):\n",
    "    images = []\n",
    "    #folders = [image_folder+\"\\\\Ntakeoff\", image_folder+\"\\\\Nflight\", image_folder+\"\\\\Nentry\"]\n",
    "    subfolder_images = sorted(os.listdir(image_folder))\n",
    "    for subfolder_image in subfolder_images:\n",
    "        if subfolder_image.endswith(\".jpg\"):\n",
    "            images.append(image_folder+\"\\\\\"+subfolder_image)\n",
    "    if (len(images)==0):\n",
    "        return\n",
    "    \n",
    "    frame = cv2.imread(images[0])\n",
    "\n",
    "    height, width, layers = frame.shape\n",
    "    if (resizeFrame == True):\n",
    "        height = resizeFrameDim[1]\n",
    "        width = resizeFrameDim[0]\n",
    "    vidFullName = video_folder+'\\\\'+vidname+\"_\"+divephase+\".mp4\"\n",
    "    print('writing video to ', vidFullName , ' framewidth ', width, ' frameheight ', height)\n",
    "    fps = 25\n",
    "    video = cv2.VideoWriter(vidFullName, cv2.VideoWriter_fourcc('M', 'P', '4', 'V'), fps, (width,height))\n",
    "\n",
    "    \n",
    "    for image in images:\n",
    "        frame = cv2.imread(image)\n",
    "        if (resizeFrame == True):\n",
    "            frame = cv2.resize(frame, (height, width))\n",
    "        video.write(frame)\n",
    "\n",
    "    cv2.destroyAllWindows()\n",
    "    video.release()\n",
    "    return vidFullName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e3209b60",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#cleanUp flag to delete temp folder created for normalizing number of images\n",
    "def createNormalizedVideos(vidpath, imgOutputDir, numImages, cleanUp=True):\n",
    "    video_list = []\n",
    "    folder, shortfilename, _ = extractFolderAndFileNameFromAbsPath(vidpath)\n",
    "    subdirNorm = imgOutputDir+\"\\\\\"+shortfilename+\"\\\\N_all\" \n",
    "    ensureDirectoryClean(subdirNorm)\n",
    "    count_norm = 0\n",
    "    for dive_action_label in ['Takeoff', 'Flight', 'Entry']: #Need to redefine the oder instead of using dive_action_label\n",
    "        subdir = imgOutputDir+\"\\\\\"+shortfilename+\"\\\\\"+ dive_action_label \n",
    "        index = {}\n",
    "        files = os.listdir(subdir)\n",
    "        numFiles = len(files)\n",
    "        if (numFiles == 0):\n",
    "            os.rmdir(subdirNorm)\n",
    "            continue\n",
    "        for file in files:\n",
    "            index[file] = 1\n",
    "        if (numFiles < numImages):\n",
    "            diff = numImages - numFiles\n",
    "            count_diff = 0;\n",
    "            while (count_diff < diff):\n",
    "                rand = random.choice(files)\n",
    "                index[rand] = index[rand]+1\n",
    "                count_diff += 1\n",
    "            \n",
    "        elif (numFiles > numImages):\n",
    "            diff = numFiles - numImages\n",
    "            count_diff = 0;\n",
    "            while (count_diff < diff):\n",
    "                rand = random.choice(files)\n",
    "                if (index[rand] > 0):\n",
    "                    index[rand] = index[rand]-1\n",
    "                    count_diff += 1\n",
    "        sortedkeys = list(index.keys())\n",
    "        sortedkeys.sort()\n",
    "        #count = 0\n",
    " \n",
    "        for key in sortedkeys:\n",
    "            frame = cv2.imread(subdir+\"\\\\\"+key)\n",
    "            for i in range(index[key]):\n",
    "                count_norm +=1\n",
    "                imagename = subdirNorm+\"\\\\\"+(\"frame%04d.jpg\" % count_norm)\n",
    "                cv2.imwrite(imagename, frame)\n",
    "    #write video to directory\n",
    "    imgInputDir = imgOutputDir+\"\\\\\"+shortfilename+ \"\\\\\"+ \"N_all\"\n",
    "    vidFullName = createVideo(imgInputDir, imgOutputDir+\"\\\\\"+shortfilename, \"N_all\", shortfilename,\n",
    "                             resizeFrame=True, resizeFrameDim=[64,64])\n",
    "    video_list.append(vidFullName)\n",
    "        \n",
    "    #cleanup\n",
    "    if(cleanUp == True):\n",
    "        for f in os.listdir(subdirNorm):\n",
    "            os.remove(subdirNorm+\"\\\\\"+f)\n",
    "        os.rmdir(subdirNorm) \n",
    "            \n",
    "    return video_list\n",
    "#createNormalizedVideos(\".\\\\uploads\\\\01_10_all.mp4\", '.\\\\images', 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "afab8780",
   "metadata": {},
   "outputs": [],
   "source": [
    "def videosAreComplete(vidpath, imgOutputDir):\n",
    "    hasTakeoff = False\n",
    "    hasFlight = False\n",
    "    hasEntry = False\n",
    "    folder, shortfilename, _ = extractFolderAndFileNameFromAbsPath(vidpath)\n",
    "    print (imgOutputDir, ' === ' , shortfilename)\n",
    "    print('tkoff files ', getNumFiles(imgOutputDir+\"\\\\\" + shortfilename+\"\\\\Takeoff\"))\n",
    "    print('Flight files ', getNumFiles(imgOutputDir+\"\\\\\" + shortfilename+\"\\\\Flight\"))\n",
    "    print('Entry files ', getNumFiles(imgOutputDir+\"\\\\\" + shortfilename+\"\\\\Entry\"))\n",
    "    \n",
    "    if (getNumFiles(imgOutputDir+\"\\\\\" + shortfilename+\"\\\\Takeoff\") > 0):\n",
    "        hasTakeoff = True\n",
    "    else:\n",
    "        print('Error: No Takeoff files!')\n",
    "        \n",
    "    if (getNumFiles(imgOutputDir+\"\\\\\" + shortfilename+\"\\\\Flight\") > 0):\n",
    "        hasFlight = True\n",
    "    else:\n",
    "        print('Error: No Flight files!')\n",
    "        \n",
    "    if (getNumFiles(imgOutputDir+\"\\\\\" + shortfilename+\"\\\\Entry\") > 0):\n",
    "        hasEntry = True\n",
    "    else:\n",
    "        print('Error: No Entry files!')\n",
    "    \n",
    "    return hasTakeoff and hasFlight and hasEntry\n",
    "\n",
    "def getNumFiles(folderpath):\n",
    "    return len([f for f in os.listdir(folderpath)  if f.endswith('.jpg') and os.path.isfile(os.path.join(folderpath, f))])\n",
    "\n",
    "\n",
    "#videosAreComplete = videosAreComplete(\".\\\\uploads\\\\01_10_all.mp4\", \".\\\\images\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9561a626",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractI3DFeatures(vidpaths):\n",
    "    global i3dextractor, stack_size, step_size\n",
    "    if (i3dextractor == None):\n",
    "        args = OmegaConf.load(build_cfg_path('i3d'))\n",
    "        args.stack_size = stack_size\n",
    "        args.step_size = step_size\n",
    "        args.extraction_fps = 25\n",
    "        args.device='cpu'#force to cpu to prevent OOM error\n",
    "        args.flow_type = 'raft' # 'pwc' is not supported on Google Colab (cupy version mismatch)\n",
    "        i3dextractor = ExtractI3D(args)\n",
    "    rgb_features = [] \n",
    "    for vidpath in vidpaths:\n",
    "        print(f'I3D Extracting for {vidpath}')\n",
    "        feature_dict = i3dextractor.extract(vidpath)\n",
    "        vidname = vidpath.replace('.mp4','')\n",
    "        for k, v in feature_dict.items():\n",
    "            if (k=='rgb'): # or k=='flow'):\n",
    "                rgb_features.append(v)\n",
    "    i3d_features = np.concatenate(rgb_features, axis=None)\n",
    "    numWindows = (96-stack_size)/step_size\n",
    "    return i3d_features.reshape(1, int(numWindows*1024))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7344dc01",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_autoscore(vidpath, imgOutputDir):\n",
    "    print('predict autoscore')\n",
    "    global autoscore_model\n",
    "    # check that there are 3 videos, else decline to proceed\n",
    "    if (videosAreComplete(vidpath, imgOutputDir)==False):\n",
    "        print('videos are incomplete! missing either entry, flight or takeoff phase')\n",
    "        return -1\n",
    "    numImages=32\n",
    "    norm_vidpaths = createNormalizedVideos(vidpath, imgOutputDir, numImages)\n",
    "\n",
    "    videos_features = extractI3DFeatures(norm_vidpaths)\n",
    "    score = autoscore_model.predict(videos_features)[0][0]\n",
    "    for video in norm_vidpaths:\n",
    "        os.remove(video)\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a851b32a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_ss_or_twist(imgFolder):\n",
    "    print('predict ss or twist')\n",
    "    global ss_twist_classifier_model\n",
    "    window_size = 1\n",
    "    images = sorted(os.listdir(imgFolder))\n",
    "    predicted_label_list = []\n",
    "    predicted_labels_probabilities_deque = deque(maxlen = window_size)\n",
    "    eventno, diveno = extractEventNoAndDiveNo(imgFolder)\n",
    "    numTwistFrames = 0\n",
    "    numSSFrames = 0\n",
    "    fps = 25\n",
    "    frame = cv2.imread(imgFolder+\"\\\\\"+images[0])\n",
    "    image_height = frame.shape[0]\n",
    "    image_width = frame.shape[1]\n",
    "    videoSS = cv2.VideoWriter(imgFolder+eventno+\"_\"+diveno+\"_ss.mp4\", cv2.VideoWriter_fourcc(*'mp4v'), fps, (image_width,image_height))\n",
    "    videoTW = cv2.VideoWriter(imgFolder+eventno+\"_\"+diveno+\"_tw.mp4\", cv2.VideoWriter_fourcc(*'mp4v'), fps, (image_width,image_height))\n",
    "    for i in range(len(images)):\n",
    "        frame = cv2.imread(imgFolder+\"\\\\\"+images[i])\n",
    "        \n",
    "        # Resize the Frame to fixed Dimensions\n",
    "        resized_frame = cv2.resize(frame, (64,64))\n",
    "        \n",
    "        # Normalize the resized frame by dividing it with 255 so that each pixel value then lies between 0 and 1\n",
    "        normalized_frame = resized_frame / 255\n",
    " \n",
    "        # Passing the Image Normalized Frame to the model and receiving Predicted Probabilities.\n",
    "        predicted_labels_probabilities = ss_twist_classifier_model.predict(np.expand_dims(normalized_frame, axis = 0))[0]\n",
    " \n",
    "        # Appending predicted label probabilities to the deque object\n",
    "        predicted_labels_probabilities_deque.append(predicted_labels_probabilities)\n",
    " \n",
    "        # Assuring that the Deque is completely filled before starting the averaging process\n",
    "        if len(predicted_labels_probabilities_deque) == window_size:\n",
    " \n",
    "            # Converting Predicted Labels Probabilities Deque into Numpy array\n",
    "            predicted_labels_probabilities_np = np.array(predicted_labels_probabilities_deque)\n",
    "            #print('predicted_labels_probabilities_np : ',predicted_labels_probabilities_np)\n",
    " \n",
    "            # Calculating Average of Predicted Labels Probabilities Column Wise \n",
    "            predicted_labels_probabilities_averaged = predicted_labels_probabilities_np.mean(axis = 0)\n",
    "            #print('predicted_labels_probabilities_averaged',predicted_labels_probabilities_averaged)\n",
    " \n",
    "            # Converting the predicted probabilities into labels by returning the index of the maximum value.\n",
    "            predicted_label = np.argmax(predicted_labels_probabilities_averaged)\n",
    " \n",
    "        else: # len(predicted_labels_probabilities_deque) < window_size\n",
    "            predicted_label = np.argmax(predicted_labels_probabilities) \n",
    " \n",
    "\n",
    "        # Accessing The Class Name using predicted label.\n",
    "        #predicted_class_name = dive_action_labels[predicted_label]\n",
    "        \n",
    "        if (predicted_label == 0): #SS\n",
    "            numSSFrames += 1\n",
    "            videoSS.write(frame)\n",
    "            predicted_label_list.append('SS')\n",
    "        elif (predicted_label == 1):\n",
    "            numTwistFrames += 1\n",
    "            videoTW.write(frame)\n",
    "            predicted_label_list.append('TW')\n",
    "    print('numSSFrames ', numSSFrames)\n",
    "    print('numTwistFrames ', numTwistFrames)\n",
    "    print(predicted_label_list)\n",
    "    cv2.destroyAllWindows()\n",
    "    videoSS.release()\n",
    "    videoTW.release()\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8686d677",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def getSecondValue (tupleStr):\n",
    "    my_tuple = make_tuple(tupleStr)\n",
    "    return my_tuple[1]\n",
    "    \n",
    "def predict_num_somersaults_twists(vidName):\n",
    "    x = requests.post(ss_twist_endpoint, data = {'videoname': vidName})\n",
    "    xjson = x.json()\n",
    "    numSS = float(getSecondValue(str(xjson['numSomersaults'])))\n",
    "    numTW = float(getSecondValue(str(xjson['numTwists'])))\n",
    "    return numSS, numTW\n",
    "\n",
    "#predict_num_somersaults_twists('testdive5.mp4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "98d07001",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate entry and splash\n",
    "\n",
    "def calculate_mean_color_value(frames):\n",
    "    frame_mean_values = []\n",
    "\n",
    "    for frame in frames:\n",
    "        frame_mean_value = np.mean(frame)\n",
    "        frame_mean_values.append(frame_mean_value)\n",
    "\n",
    "    # video_mean_color_value = np.mean(frame_mean_values)\n",
    "    return frame_mean_values\n",
    "\n",
    "\n",
    "def blue_color_proportion(frame, blur_kernel_size=(5, 5)):\n",
    "    blurred_frame = cv2.GaussianBlur(frame, blur_kernel_size, 0)\n",
    "    blue_channel = blurred_frame[:, :, 0]\n",
    "    total_pixels = blurred_frame.size // 3\n",
    "    blue_pixels = np.sum(blue_channel > 210)  # Threshold for blue color intensity (0-255)\n",
    "    proportion = blue_pixels / total_pixels\n",
    "    return proportion\n",
    "\n",
    "def get_blue_proportions(frames, blur_kernel_size=(5, 5)):\n",
    "    proportion_list = []\n",
    "    for i, frame in enumerate(frames):\n",
    "        proportion = blue_color_proportion(frame, blur_kernel_size)\n",
    "        proportion_list.append(proportion)\n",
    "    return proportion_list  \n",
    "\n",
    "def moving_average(data, window_size):\n",
    "    return [\n",
    "        sum(data[i:i + window_size]) / window_size\n",
    "        for i in range(len(data) - window_size + 1)\n",
    "    ]\n",
    "\n",
    "def find_splash_frame_index(data, window_size=5, threshold = 0.01):\n",
    "    smoothed_data = moving_average(data, window_size)\n",
    "    stdev = statistics.stdev(data)\n",
    "\n",
    "    # if no index find        \n",
    "    for i in range(len(smoothed_data) - 1):\n",
    "        if abs(smoothed_data[i+1] - smoothed_data[i]) < threshold - stdev/2:\n",
    "            return i + window_size // 2\n",
    "\n",
    "    return len(data)//5\n",
    "\n",
    "def get_entry_splash_frames(frames):\n",
    "    color_info = get_blue_proportions(frames)\n",
    "    splash_frame_index = find_splash_frame_index(color_info)\n",
    "    if splash_frame_index is not None:\n",
    "        entry_frames = frames[:splash_frame_index]\n",
    "        splash_frames = frames[splash_frame_index:]\n",
    "        return entry_frames, splash_frames\n",
    "    else:\n",
    "        print(\"No splash frame detected\")\n",
    "\n",
    "def crop_image(img,  crop_r_w = 0.5, crop_r_h = 0):\n",
    "    h, w ,_ = img.shape\n",
    "    # Define the coordinates of the top-left and bottom-right corners of the rectangle to crop\n",
    "    x1, y1 = int(w*crop_r_w/2), int(h*crop_r_h/2)  # top-left corner\n",
    "    x2, y2 = int(w*(1-crop_r_w/2)), int(h*(1-crop_r_h/2))  # bottom-right corner\n",
    "    return img.copy()[y1:y2, x1:x2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f50c55ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find entry key frame\n",
    "def remove_bg(img):\n",
    "    # Convert the image to the HSV color space\n",
    "    hsv_image = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n",
    "\n",
    "    # # Define the lower and upper bounds of the blue color range\n",
    "    lower_range = np.array([90, 50, 50])\n",
    "    upper_range = np.array([120, 255, 255])\n",
    "\n",
    "    # Threshold the image to extract the blue pixels\n",
    "    mask = cv2.inRange(hsv_image, lower_range, upper_range)\n",
    "    mask = cv2.bitwise_not(mask) \n",
    "    mask = cv2.GaussianBlur(mask, (5, 5), 0)\n",
    "\n",
    "    # Define the kernel for the morphological operations\n",
    "    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))\n",
    "\n",
    "    # Apply dilation to fill small holes and connect nearby contours\n",
    "    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)\n",
    "    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)\n",
    "\n",
    "    # Apply binary threshold to ensure only black and white pixels\n",
    "    _, mask = cv2.threshold(mask, 0, 255, cv2.THRESH_BINARY)\n",
    "    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)\n",
    "\n",
    "    # retunr binary mask \n",
    "    return mask\n",
    "\n",
    "def find_human_contour(img):\n",
    "    height, width= img.shape\n",
    "    # find all contour\n",
    "    ctrs = cv2.findContours(img, \n",
    "                          cv2.RETR_EXTERNAL,\n",
    "                          cv2.CHAIN_APPROX_SIMPLE)\n",
    "    ctrs = ctrs[0] \n",
    "\n",
    "    ctrs = sorted(ctrs, key=cv2.contourArea, reverse=True)[:5]\n",
    "    for c in ctrs:\n",
    "        (x, y, w, h)  = cv2.boundingRect(c)\n",
    "        aspr = h/float(w)\n",
    "        if x != 0 and x+w != width and y+h != height and aspr > 0.8:   \n",
    "            return c\n",
    "    # print('no contour found')\n",
    "    return None\n",
    "\n",
    "def get_consecutive_numbers(input_list):\n",
    "    consecutive_numbers = []\n",
    "    temp = []\n",
    "\n",
    "    for i, num in enumerate(input_list):\n",
    "        if i == 0: \n",
    "            temp.append(num)\n",
    "            continue\n",
    "\n",
    "        if num == input_list[i - 1] + 1:\n",
    "            temp.append(num)\n",
    "        else:\n",
    "            temp = [num]\n",
    "\n",
    "    return temp\n",
    "\n",
    "def get_consecutive_numbers_front(input_list):\n",
    "    consecutive_numbers = []\n",
    "    temp = []\n",
    "\n",
    "    for i, num in enumerate(input_list):\n",
    "        if i == 0: \n",
    "            temp.append(num)\n",
    "            continue\n",
    "\n",
    "        if num == input_list[i - 1] + 1:\n",
    "            temp.append(num)\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    return temp\n",
    "\n",
    "def find_decreasing_point_smoothed(data, window_size=2):\n",
    "    smoothed_data = moving_average(data, window_size)\n",
    "    stdev = statistics.stdev(data)\n",
    "    for i in range(1, len(smoothed_data) - 1):\n",
    "        if smoothed_data[i] < smoothed_data[i - 1] and smoothed_data[i] <= smoothed_data[i + 1]:\n",
    "        # if smoothed_data[i] - smoothed_data[i+1] > 0:\n",
    "            return i   # Return the index adjusted for the moving average window\n",
    "\n",
    "    return -1  # If the curve does not decrease, return -1\n",
    "\n",
    "def find_key_frame_index(ctr_areas):\n",
    "    # select the key frame\n",
    "    thr = sum(ctr_areas) / len(ctr_areas)\n",
    "    frame_indexs = []\n",
    "    for i in range(len(ctr_areas)):\n",
    "        if ctr_areas[i] > thr * 0.8:\n",
    "            frame_indexs.append(i)\n",
    "\n",
    "    frame_indexs = get_consecutive_numbers(frame_indexs)\n",
    "\n",
    "    if len(frame_indexs) > 0:\n",
    "   \n",
    "\n",
    "        frame_areas = [ctr_areas[i] for i in frame_indexs]   \n",
    "        thr = sum(frame_areas) / len(frame_areas) * 1\n",
    "        key_frame_indexs = []\n",
    "\n",
    "        for i in frame_indexs:\n",
    "            if thr * 1.5 > ctr_areas[i] >= thr:\n",
    "                key_frame_indexs.append(i)\n",
    "\n",
    "        if len(key_frame_indexs) > 0:\n",
    "          # key_frame_indexs = get_consecutive_numbers_front(frame_indexs)\n",
    "\n",
    "          return key_frame_indexs[-1]\n",
    "\n",
    "    # print(key_frame_indexs)\n",
    "    return len(ctr_areas)//2\n",
    "\n",
    "# return the key frame and the key frame ctr\n",
    "def find_key_frame(frames):\n",
    "    ctr_areas = []\n",
    "    ctr_list = []\n",
    "    ctr_images = []\n",
    "    for frame in frames:\n",
    "        mask = remove_bg(frame)\n",
    "        ctr = find_human_contour(mask)\n",
    "        ctr_list.append(ctr)\n",
    "    \n",
    "        if ctr is not None: \n",
    "            ctr_areas.append(cv2.contourArea(ctr))\n",
    "\n",
    "        else:\n",
    "            ctr_areas.append(0)    \n",
    "\n",
    "    # find key frame\n",
    "    key_frame_index = find_key_frame_index(ctr_areas)\n",
    "\n",
    "    return frames[key_frame_index], ctr_list[key_frame_index], ctr_areas[key_frame_index]\n",
    "\n",
    "def analyse_entry_action(frames):\n",
    "    ctr_areas = []\n",
    "    ctr_list = []\n",
    "    ctr_images = []\n",
    "    for frame in entry_frames:\n",
    "        # cv2_imshow(frame)\n",
    "        mask = remove_bg(frame)\n",
    "        ctr = find_human_contour(mask)\n",
    "        ctr_list.append(ctr)\n",
    "    \n",
    "        if ctr is not None: \n",
    "            ctr_areas.append(cv2.contourArea(ctr))\n",
    "\n",
    "        else:\n",
    "            ctr_areas.append(0)    \n",
    "\n",
    "    # find key frame\n",
    "    key_frame_index = find_key_frame(ctr_areas)\n",
    "    for i in range(len(entry_frames)):\n",
    "        img = entry_frames[i].copy()\n",
    "        ctr = ctr_list[i]\n",
    "        if ctr is not None: \n",
    "            if i != key_frame_index:\n",
    "                cv2.drawContours(img, [ctr], -1, (0, 255, 0), 2)\n",
    "            else:\n",
    "                cv2.drawContours(img, [ctr], -1, (0, 0, 255), 2)\n",
    "        ctr_images.append(img)\n",
    "\n",
    "  \n",
    "    return ctr_images, key_frame_index, ctr_list, ctr_areas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7156dd84",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis angle\n",
    "\n",
    "# Calculate the angle of the line segment against the horizontal line\n",
    "def calculate_angle(line1, line2 = None):\n",
    "\n",
    "    if line2: \n",
    "        slope1 = slope2 = 1\n",
    "        # calculate the slopes of the two lines\n",
    "        if line1[1][0] - line1[0][0]:\n",
    "            slope1 = (line1[1][1] - line1[0][1]) / (line1[1][0] - line1[0][0])  \n",
    "        if line2[1][0] - line2[0][0]:\n",
    "            slope2 = (line2[1][1] - line2[0][1]) / (line2[1][0] - line2[0][0])\n",
    "        # calculate the angle between the two lines  \n",
    "        angle = math.atan(abs((slope2 - slope1) / (1 + slope1 * slope2))) * 180 / math.pi\n",
    "        angle = abs(angle)\n",
    "\n",
    "    else:\n",
    "        h = line1[1][1] - line1[0][1] \n",
    "        w = line1[1][0] - line1[0][0]  \n",
    "        if w: \n",
    "            angle = math.atan(abs(h/w))* 180 / math.pi\n",
    "            angle = abs(angle)\n",
    "        else:\n",
    "            angle = 90.0\n",
    "  \n",
    "    return angle\n",
    "\n",
    "def compute_bend_angle(line1, line2):\n",
    "    # Calculate the vectors of the lines\n",
    "    vector1 = [line1[1][0] - line1[0][0], line1[1][1] - line1[0][1]]\n",
    "    vector2 = [line2[1][0] - line2[0][0], line2[1][1] - line2[0][1]]\n",
    "\n",
    "    # Calculate the dot product of the vectors\n",
    "    dot_product = vector1[0] * vector2[0] + vector1[1] * vector2[1]\n",
    "\n",
    "    # Calculate the magnitudes of the vectors\n",
    "    magnitude1 = math.sqrt(vector1[0] ** 2 + vector1[1] ** 2)\n",
    "    magnitude2 = math.sqrt(vector2[0] ** 2 + vector2[1] ** 2)\n",
    "\n",
    "    # Compute the cosine of the angle\n",
    "    cos_angle = dot_product / (magnitude1 * magnitude2)\n",
    "\n",
    "    # Clamp the cosine value to the valid range of -1 to 1\n",
    "    cos_angle = max(min(cos_angle, 1), -1)\n",
    "\n",
    "    # Calculate the angle in radians\n",
    "    angle_rad = math.acos(cos_angle)\n",
    "\n",
    "    # Convert the angle to degrees\n",
    "    angle_deg = math.degrees(angle_rad)\n",
    "\n",
    "    # Calculate the intersection angle, which is the smaller angle\n",
    "    intersection_angle = min(angle_deg, 360 - angle_deg)\n",
    "\n",
    "    return intersection_angle\n",
    "\n",
    "# plot the annotation\n",
    "def plot_angle_annotation(image, top_point, center_point, bottom_point):\n",
    "    human_img = image.copy()\n",
    "    pts = np.array([top_point, center_point, bottom_point ],np.int32)\n",
    "    pts = pts.reshape((-1, 1, 2))\n",
    "    cv2.polylines(human_img, [pts], False, (0, 0, 255), thickness=2)\n",
    "    cv2.circle(human_img, center_point, 3, (0, 0, 255), -1)\n",
    "    \n",
    "    \n",
    "    return human_img\n",
    "\n",
    "def compute_angle(img, ctr):\n",
    "    top_point = tuple(ctr[ctr[:,:,1].argmin()][0])\n",
    "    bottom_point = tuple(ctr[ctr[:,:,1].argmax()][0])\n",
    "    left_point = tuple(ctr[ctr[:,:,0].argmin()][0])\n",
    "    right_point = tuple(ctr[ctr[:,:,0].argmax()][0])\n",
    "\n",
    "    # find the center part\n",
    "    M = cv2.moments(ctr)\n",
    "    if M['m00'] != 0:\n",
    "        cx = int(M['m10']/M['m00'])\n",
    "        cy = int(M['m01']/M['m00'])      \n",
    "        center_point=(cx, cy)     \n",
    "    # analyse the angle\n",
    "    entry_angle = calculate_angle((center_point,bottom_point))\n",
    "    bend_angle = compute_bend_angle([top_point, center_point], [bottom_point, center_point])\n",
    "\n",
    "  \n",
    "    human_img = plot_angle_annotation(img, top_point, center_point, bottom_point)\n",
    "    \n",
    "    return bend_angle, entry_angle, human_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "03b729ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splash analysis\n",
    "def increase_contrast(frame, min_val=0, max_val=255):\n",
    "    gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "    min_pixel = np.min(gray_frame)\n",
    "    max_pixel = np.max(gray_frame)\n",
    "\n",
    "    contrast_frame = (gray_frame - min_pixel) * ((max_val - min_val) / (max_pixel - min_pixel)) + min_val\n",
    "    return contrast_frame.astype(np.uint8)\n",
    "\n",
    "\n",
    "def get_brightness_mask(image, threshold_factor=1.2):\n",
    "    # Calculate the average brightness\n",
    "    average_brightness = np.mean(image)\n",
    "\n",
    "    # Threshold the grayscale image to create a mask of areas with brightness above the average\n",
    "    _, brightness_mask = cv2.threshold(image, threshold_factor * average_brightness, 255, cv2.THRESH_BINARY)\n",
    "\n",
    "    return brightness_mask\n",
    "\n",
    "\n",
    "def splash_size_pixel(frames, flow_threshold=2, skip_frames=2):\n",
    "    prev_frame = increase_contrast(frames[0])\n",
    "    splash_sizes = []\n",
    "\n",
    "    for i in range(1, len(frames), skip_frames + 1):\n",
    "        curr_frame = increase_contrast(frames[i])\n",
    "\n",
    "        # Calculate the optical flow using Farneback method\n",
    "        flow = cv2.calcOpticalFlowFarneback(prev_frame, curr_frame, None, 0.5, 3, 15, 3, 5, 1.2, 0)\n",
    "\n",
    "        # Calculate the magnitude and angle of the 2D vectors\n",
    "        magnitude, _ = cv2.cartToPolar(flow[..., 0], flow[..., 1])\n",
    "\n",
    "        # Threshold the magnitude to capture significant motion\n",
    "        motion_mask = np.where(magnitude > flow_threshold, 1, 0)\n",
    "\n",
    "         # Threshold the magnitude to capture significant motion\n",
    "        motion_mask = np.where(magnitude > flow_threshold, 1, 0)\n",
    "\n",
    "        # Threshold the brightness to capture only the brighter parts\n",
    "        bright_mask = get_brightness_mask(curr_frame)\n",
    "\n",
    "        # Combine the motion and brightness masks to focus on the bright splash\n",
    "        splash_mask = motion_mask * bright_mask\n",
    "\n",
    "        splash_size = np.sum(splash_mask == 255)\n",
    "\n",
    "        splash_sizes.append(splash_size)\n",
    "\n",
    "        prev_frame = curr_frame\n",
    "\n",
    "    # Calculate the mean and max splash sizes\n",
    "    mean_splash_size = np.mean(splash_sizes)\n",
    "    max_splash_size = max(splash_sizes)\n",
    "\n",
    "    return mean_splash_size, max_splash_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f1f2de79",
   "metadata": {},
   "outputs": [],
   "source": [
    "CTR_MEAN = 1463.82755\n",
    "\n",
    "# range from 0 and 1\n",
    "def compute_ratio(size, area):\n",
    "    ratio = 0\n",
    "    if area>0:\n",
    "        ratio = size/area\n",
    "    ratio = size/CTR_MEAN\n",
    "    if ratio > 10:\n",
    "        return 10\n",
    "    return ratio\n",
    "    \n",
    "def analyze_entry(imgFolder):\n",
    "    print('analyze entry')\n",
    "    images = sorted(os.listdir(imgFolder))\n",
    "    frames = [crop_image(cv2.imread(imgFolder+\"\\\\\"+image)) for image in images]\n",
    "    \n",
    "    with open(folderpath+\"mean_splash_index_scaler.pkl\", \"rb\") as f:\n",
    "        mean_splash_index_scaler = pickle.load(f)\n",
    "\n",
    "    with open(folderpath+\"max_splash_index_scaler.pkl\", \"rb\") as f:\n",
    "        max_splash_index_scaler = pickle.load(f)\n",
    "\n",
    "    # separate the action and splash\n",
    "    entry_frames, splash_frames = get_entry_splash_frames(frames)\n",
    "\n",
    "    # find key frame\n",
    "    key_frame, ctr, ctr_area= find_key_frame(entry_frames)\n",
    "    bend_angle = 0 \n",
    "    entry_angle = 0\n",
    "    human_imag = key_frame\n",
    "\n",
    "    # if key frame not found, return angle = 0\n",
    "    if (key_frame is None or ctr is None):\n",
    "        print('no human found')\n",
    "    else:\n",
    "        bend_angle, entry_angle, human_img = compute_angle(key_frame, ctr)\n",
    "\n",
    "    mean_splash_size, max_splash_size = splash_size_pixel(splash_frames)\n",
    "\n",
    "    mean_splash_ratio = compute_ratio(mean_splash_size, ctr_area)\n",
    "    max_splash_ratio = compute_ratio(max_splash_size, ctr_area)\n",
    "\n",
    "    mean_splash_index = mean_splash_index_scaler.transform(np.array([mean_splash_ratio]).reshape(-1, 1))[0][0]\n",
    "    max_splash_index = max_splash_index_scaler.transform(np.array([max_splash_ratio]).reshape(-1, 1))[0][0]\n",
    "\n",
    "#     return bend_angle, entry_angle, ctr_area, \\\n",
    "#         mean_splash_size, max_splash_size, \\\n",
    "#         mean_splash_ratio, max_splash_ratio,\\\n",
    "#         mean_splash_index, max_splash_index\\\n",
    "        # human_img \n",
    "   \n",
    "    return ctr_area, bend_angle, entry_angle, mean_splash_index, max_splash_index, human_img\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7aae1e2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_final_score(autoscore, numSomersaults, numTwists, bend_angle, entry_angle, mean_splash_index, ctr_area):\n",
    "    global hybrid_model, minmaxScalar\n",
    "    if (numSomersaults == None):\n",
    "        numSomersaults = 2.5\n",
    "    if (numTwists == None):\n",
    "        numTwists = 1.0\n",
    "    if (autoscore == None):\n",
    "        autoscore = 0.6223464999999999\n",
    "    df = pd.DataFrame([[numTwists, bend_angle, entry_angle, numSomersaults, autoscore, mean_splash_index, ctr_area]], \n",
    "                      columns=['pred_twist','bend_angle','entry_angle','pred_somersault','updated_predscore','mean_splash_index','ctr_area'])\n",
    "    minmaxScalar.transform (df)\n",
    "    svr_pred = round(hybrid_model.predict(df)[0],4)\n",
    "    return svr_pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d589b0e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def processVideo(vidpath):\n",
    "    print('processing ', vidpath)\n",
    "    folder, shortfilename, _ = extractFolderAndFileNameFromAbsPath(vidpath)\n",
    "    predict_temporal_segmentation(vidpath, '.\\\\images')\n",
    "    autoscore = predict_autoscore(vidpath, '.\\\\images')\n",
    "    predict_ss_or_twist(\".\\\\images\\\\\"+shortfilename+\"\\\\Flight\")\n",
    "    numSomersaults, numTwists = predict_num_somersaults_twists(shortfilename+\".mp4\")\n",
    "    ctr_area, bend_angle, entry_angle, mean_splash_index, max_splash_index, human_img = analyze_entry(\".\\\\images\\\\\"+shortfilename+\"\\\\Entry\")\n",
    "    final_score = predict_final_score(autoscore, numSomersaults, numTwists, bend_angle, entry_angle, mean_splash_index, ctr_area)\n",
    "    print('autoscore: ', autoscore, \n",
    "          ', numSomersaults: ', numSomersaults, \n",
    "          ', numTwists: ', numTwists,\n",
    "          ', angleOfEntry: ', (bend_angle, entry_angle),\n",
    "          ', splashIndex: ', (mean_splash_index, max_splash_index),\n",
    "          ', final_score: ', final_score)\n",
    "    \n",
    "    return final_score, autoscore, numSomersaults, numTwists, bend_angle, entry_angle, mean_splash_index, max_splash_index, human_img\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2a3669c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    load_temporal_segment_model()\n",
    "    load_autoscore_model()\n",
    "    load_ss_twist_classifier_model()\n",
    "    load_hybrid_model()\n",
    "    #final_score, autoscore, numSomersaults, numTwists, angleOfEntry, splashIndex = processVideo(\".\\\\testdive5.mp4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5b9b6731",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading temporal segment model\n",
      "loading autoscore model\n",
      "loading ss_twist_classifier model\n",
      "loading hybrid model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\sklearn\\base.py:329: UserWarning: Trying to unpickle estimator SVR from version 1.1.1 when using version 1.1.3. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n",
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\sklearn\\base.py:329: UserWarning: Trying to unpickle estimator MinMaxScaler from version 1.1.1 when using version 1.1.3. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b48595aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.\n",
      " * Running on http://localhost:5000\n",
      "Press CTRL+C to quit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing  .\\uploads\\uploaded_video.mp4\n",
      "predict autoscore\n",
      ".\\images  ===  uploaded_video\n",
      "tkoff files  14\n",
      "Flight files  34\n",
      "Entry files  35\n",
      "writing video to  .\\images\\uploaded_video\\uploaded_video_N_all.mp4  framewidth  64  frameheight  64\n",
      "using device  cpu\n",
      "I3D Extracting for .\\images\\uploaded_video\\uploaded_video_N_all.mp4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\torch\\functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at C:\\cb\\pytorch_1000000000000\\work\\aten\\src\\ATen\\native\\TensorShape.cpp:3191.)\n",
      "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predict ss or twist\n",
      "numSSFrames  17\n",
      "numTwistFrames  17\n",
      "['SS', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS']\n",
      "analyze entry\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\sklearn\\base.py:329: UserWarning: Trying to unpickle estimator MinMaxScaler from version 1.2.2 when using version 1.1.3. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n",
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\sklearn\\base.py:443: UserWarning: X has feature names, but SVR was fitted without feature names\n",
      "  warnings.warn(\n",
      "127.0.0.1 - - [20/May/2023 23:17:32] \"POST /videoupload HTTP/1.1\" 200 -\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "autoscore:  0.7432131 , numSomersaults:  1.5 , numTwists:  0.5 , angleOfEntry:  (160.98338811957265, 81.11934084947976) , splashIndex:  (0.18010067136354657, 0.27838195999495086) , final_score:  52.9082\n",
      "processing  .\\uploads\\uploaded_video.mp4\n",
      "predict autoscore\n",
      ".\\images  ===  uploaded_video\n",
      "tkoff files  14\n",
      "Flight files  34\n",
      "Entry files  35\n",
      "writing video to  .\\images\\uploaded_video\\uploaded_video_N_all.mp4  framewidth  64  frameheight  64\n",
      "I3D Extracting for .\\images\\uploaded_video\\uploaded_video_N_all.mp4\n",
      "predict ss or twist\n",
      "numSSFrames  17\n",
      "numTwistFrames  17\n",
      "['SS', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'TW', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS']\n",
      "analyze entry\n",
      "autoscore:  0.7597108 , numSomersaults:  1.5 , numTwists:  0.5 , angleOfEntry:  (160.98338811957265, 81.11934084947976) , splashIndex:  (0.18010067136354657, 0.27838195999495086) , final_score:  52.9221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\sklearn\\base.py:329: UserWarning: Trying to unpickle estimator MinMaxScaler from version 1.2.2 when using version 1.1.3. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n",
      "C:\\Users\\Grace\\anaconda3\\envs\\gpu\\lib\\site-packages\\sklearn\\base.py:443: UserWarning: X has feature names, but SVR was fitted without feature names\n",
      "  warnings.warn(\n",
      "127.0.0.1 - - [20/May/2023 23:26:48] \"POST /videoupload HTTP/1.1\" 200 -\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing  .\\uploads\\uploaded_video.mp4\n",
      "predict autoscore\n",
      ".\\images  ===  uploaded_video\n",
      "tkoff files  48\n",
      "Flight files  39\n",
      "Entry files  30\n",
      "writing video to  .\\images\\uploaded_video\\uploaded_video_N_all.mp4  framewidth  64  frameheight  64\n",
      "I3D Extracting for .\\images\\uploaded_video\\uploaded_video_N_all.mp4\n",
      "predict ss or twist\n",
      "numSSFrames  39\n",
      "numTwistFrames  0\n",
      "['SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS', 'SS']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "127.0.0.1 - - [20/May/2023 23:37:53] \"POST /videoupload HTTP/1.1\" 500 -\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An exception occurred: Expecting value: line 1 column 1 (char 0)\n",
      "processing  .\\uploads\\uploaded_video.mp4\n",
      "predict autoscore\n",
      ".\\images  ===  uploaded_video\n",
      "tkoff files  48\n",
      "Flight files  39\n",
      "Entry files  30\n",
      "writing video to  .\\images\\uploaded_video\\uploaded_video_N_all.mp4  framewidth  64  frameheight  64\n",
      "I3D Extracting for .\\images\\uploaded_video\\uploaded_video_N_all.mp4\n"
     ]
    }
   ],
   "source": [
    "app = Flask(__name__)\n",
    "CORS(app)\n",
    "app.secret_key = \"secret key\"\n",
    "app.config[\"UPLOAD_FOLDER\"] = \".\\\\uploads\\\\\"\n",
    "\n",
    "if not os.path.exists(app.config[\"UPLOAD_FOLDER\"]):\n",
    "    os.makedirs(app.config[\"UPLOAD_FOLDER\"])\n",
    "\n",
    "@app.route('/videoupload', methods=['POST'])\n",
    "\n",
    "def upload_video():\n",
    "    try:\n",
    "        if \"video\" not in request.files:\n",
    "            print(\"No video in request.files\")\n",
    "            return jsonify({\"error\": \"No video file\"}), 400\n",
    "\n",
    "        video = request.files[\"video\"]\n",
    "        \n",
    "        if video and video.filename != \"\":\n",
    "            filename = secure_filename(video.filename)\n",
    "            video.save(os.path.join(app.config[\"UPLOAD_FOLDER\"], filename))\n",
    "            final_score, auto_score, numSomersaults, numTwists, bending_angle, entry_angle, mean_splash_index, max_splash_index,human_imag = processVideo(app.config['UPLOAD_FOLDER']+filename)\n",
    "            _, img_encoded = cv2.imencode('.jpg', human_imag)\n",
    "            img_str = base64.b64encode(img_encoded).decode('utf-8')\n",
    "\n",
    "\n",
    "            # process video file\n",
    "            result = {\n",
    "                \"final_score\" : str(final_score),\n",
    "                \"auto_score\" : str(auto_score),\n",
    "                \"numSomersaults\" : str(numSomersaults),\n",
    "                \"numTwists\" : str(numTwists),\n",
    "                \"EntryAngle\" : str(entry_angle),\n",
    "                \"BendingAngle\" : str(bending_angle),\n",
    "                \"MeanSplashIndex\" : str(mean_splash_index),\n",
    "                \"MaxSplashIndex\" : str(max_splash_index),\n",
    "                \"EntryImage\": img_str\n",
    "            }\n",
    "\n",
    "            return jsonify(result), 200\n",
    "        else:\n",
    "            print(\"File not allowed\")\n",
    "            return jsonify({\"error\": \"File not allowed\"}), 400\n",
    "        \n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"An exception occurred: \" + str(e))\n",
    "        return jsonify({\"error\": \"An internal server error occurred\"}), 500\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    from werkzeug.serving import run_simple\n",
    "    run_simple('localhost', 5000, app)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99aa5e7f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gpuEnv",
   "language": "python",
   "name": "gpuenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}