kaggle_inference_notebook_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "540c9f3c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:09.817393Z",
     "iopub.status.busy": "2023-04-29T15:22:09.816987Z",
     "iopub.status.idle": "2023-04-29T15:22:22.368941Z",
     "shell.execute_reply": "2023-04-29T15:22:22.367634Z"
    },
    "papermill": {
     "duration": 12.565065,
     "end_time": "2023-04-29T15:22:22.372253",
     "exception": false,
     "start_time": "2023-04-29T15:22:09.807188",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tflite-runtime\r\n",
      "  Downloading tflite_runtime-2.11.0-cp37-cp37m-manylinux2014_x86_64.whl (2.5 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m25.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.19.2 in /opt/conda/lib/python3.7/site-packages (from tflite-runtime) (1.21.6)\r\n",
      "Installing collected packages: tflite-runtime\r\n",
      "Successfully installed tflite-runtime-2.11.0\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "#!pip install onnx_tf\n",
    "!pip install tflite-runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "92edf29b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:22.389049Z",
     "iopub.status.busy": "2023-04-29T15:22:22.388594Z",
     "iopub.status.idle": "2023-04-29T15:22:35.195115Z",
     "shell.execute_reply": "2023-04-29T15:22:35.193147Z"
    },
    "papermill": {
     "duration": 12.818416,
     "end_time": "2023-04-29T15:22:35.198317",
     "exception": false,
     "start_time": "2023-04-29T15:22:22.379901",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "import os\n",
    "import gc\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torchinfo import summary\n",
    "import onnx\n",
    "import tflite_runtime.interpreter as tflite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "46c295b4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:35.215108Z",
     "iopub.status.busy": "2023-04-29T15:22:35.214262Z",
     "iopub.status.idle": "2023-04-29T15:22:35.219113Z",
     "shell.execute_reply": "2023-04-29T15:22:35.218122Z"
    },
    "papermill": {
     "duration": 0.015928,
     "end_time": "2023-04-29T15:22:35.221331",
     "exception": false,
     "start_time": "2023-04-29T15:22:35.205403",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#train = pd.read_csv('/kaggle/input/asl-signs/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "679b9d60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:35.237697Z",
     "iopub.status.busy": "2023-04-29T15:22:35.236852Z",
     "iopub.status.idle": "2023-04-29T15:22:35.243187Z",
     "shell.execute_reply": "2023-04-29T15:22:35.242003Z"
    },
    "papermill": {
     "duration": 0.01755,
     "end_time": "2023-04-29T15:22:35.245915",
     "exception": false,
     "start_time": "2023-04-29T15:22:35.228365",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from typing import Dict, Optional\n",
    " \n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch import Tensor\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b6116c5",
   "metadata": {
    "papermill": {
     "duration": 0.006808,
     "end_time": "2023-04-29T15:22:35.259970",
     "exception": false,
     "start_time": "2023-04-29T15:22:35.253162",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Torch model and weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5b0b44b6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:35.276412Z",
     "iopub.status.busy": "2023-04-29T15:22:35.275626Z",
     "iopub.status.idle": "2023-04-29T15:22:48.641003Z",
     "shell.execute_reply": "2023-04-29T15:22:48.639323Z"
    },
    "papermill": {
     "duration": 13.376894,
     "end_time": "2023-04-29T15:22:48.643901",
     "exception": false,
     "start_time": "2023-04-29T15:22:35.267007",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting efficientnet_pytorch\r\n",
      "  Downloading efficientnet_pytorch-0.7.1.tar.gz (21 kB)\r\n",
      "  Preparing metadata (setup.py) ... \u001b[?25l-\b \bdone\r\n",
      "\u001b[?25hRequirement already satisfied: torch in /opt/conda/lib/python3.7/site-packages (from efficientnet_pytorch) (1.13.0+cpu)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch->efficientnet_pytorch) (4.4.0)\r\n",
      "Building wheels for collected packages: efficientnet_pytorch\r\n",
      "  Building wheel for efficientnet_pytorch (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for efficientnet_pytorch: filename=efficientnet_pytorch-0.7.1-py3-none-any.whl size=16446 sha256=4c6f15c5b739a1de58f6ea67280c7ccb4e4165547ac2e7acd822cba42bc096b6\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/96/3f/5f/13976445f67f3b4e77b054e65f7f4c39016e92e8358fe088db\r\n",
      "Successfully built efficientnet_pytorch\r\n",
      "Installing collected packages: efficientnet_pytorch\r\n",
      "Successfully installed efficientnet_pytorch-0.7.1\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install efficientnet_pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "090e6bfb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:48.662030Z",
     "iopub.status.busy": "2023-04-29T15:22:48.661559Z",
     "iopub.status.idle": "2023-04-29T15:22:48.726427Z",
     "shell.execute_reply": "2023-04-29T15:22:48.725043Z"
    },
    "papermill": {
     "duration": 0.077148,
     "end_time": "2023-04-29T15:22:48.729126",
     "exception": false,
     "start_time": "2023-04-29T15:22:48.651978",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['utils/.idea/.gitignore',\n",
       " 'utils/.idea/utils.iml',\n",
       " 'utils/.idea/modules.xml',\n",
       " 'utils/.idea/inspectionProfiles/profiles_settings.xml',\n",
       " 'utils/.idea/inspectionProfiles/Project_Default.xml',\n",
       " 'utils/.idea/misc.xml',\n",
       " 'utils/keras_models/preprocess.py',\n",
       " 'utils/keras_models/efficientnet.py',\n",
       " 'utils/keras_models/transfer.py',\n",
       " 'utils/keras_models/common.py',\n",
       " 'utils/keras_models/rexnet.py',\n",
       " 'utils/keras_models/transformer.py',\n",
       " 'utils/keras_models/__init__.py',\n",
       " 'utils/augmentations.py',\n",
       " 'utils/__init__.py']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from distutils.dir_util import copy_tree\n",
    "copy_tree(\"/kaggle/input/keras-repo-v2/utils\", \"utils\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd1e7225",
   "metadata": {
    "papermill": {
     "duration": 0.007683,
     "end_time": "2023-04-29T15:22:48.744870",
     "exception": false,
     "start_time": "2023-04-29T15:22:48.737187",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Set params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1624f447",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:48.763077Z",
     "iopub.status.busy": "2023-04-29T15:22:48.762074Z",
     "iopub.status.idle": "2023-04-29T15:22:48.775817Z",
     "shell.execute_reply": "2023-04-29T15:22:48.774547Z"
    },
    "papermill": {
     "duration": 0.025539,
     "end_time": "2023-04-29T15:22:48.778324",
     "exception": false,
     "start_time": "2023-04-29T15:22:48.752785",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def transfer_efficientnet_artem(torch_model_path, model_name, size = (160, 80)):\n",
    "    stt = torch.load(torch_model_path, map_location = \"cpu\")[\"model\"]\n",
    "    stt = dict([(n.split('base_model.')[-1], p) for n, p in stt.items() if 'hyper' not in n])\n",
    "    model = EfficientNet.from_name('efficientnet-b0', num_classes=250, in_channels=3)\n",
    "    model.eval()\n",
    "    model.load_state_dict(stt)\n",
    "\n",
    "\n",
    "    input_layer = K.layers.Input((*size, 3), batch_size = 1, name = \"inputs\")\n",
    "    model_layer = KerasEfficientNet(model, name = \"model\")(input_layer)\n",
    "    output_layer = KerasIdentity(None, name = \"outputs\")(model_layer)\n",
    "    keras_model = K.models.Model(input_layer, output_layer, name = model_name)\n",
    "\n",
    "    newstt = {k[1:].replace(\"._\", \".\"): v for k, v in model.state_dict().items()}\n",
    "\n",
    "    original_names = [_.name for _ in keras_model.weights]\n",
    "    names = [replace(_.name)[:-2] for _ in keras_model.weights]\n",
    "    shapes = {replace(_.name)[:-2]: _.numpy().shape for _ in keras_model.weights}\n",
    "    for name, original_name in zip(names, original_names):\n",
    "        if modify(name, original_name, newstt).shape != shapes[name]:\n",
    "            print(name, \", \", modify(name, original_name, newstt).shape, \"-->\", shapes[name])\n",
    "\n",
    "    keras_model.set_weights([modify(name, original_name, newstt) for name, original_name in zip(names, original_names)])\n",
    "    return model, keras_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d22dcaee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:22:48.797898Z",
     "iopub.status.busy": "2023-04-29T15:22:48.797076Z",
     "iopub.status.idle": "2023-04-29T15:24:28.207409Z",
     "shell.execute_reply": "2023-04-29T15:24:28.205699Z"
    },
    "papermill": {
     "duration": 99.427499,
     "end_time": "2023-04-29T15:24:28.213706",
     "exception": false,
     "start_time": "2023-04-29T15:22:48.786207",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Amount of transformers: 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/kaggle/working/utils/keras_models/transfer.py:34: UserWarning: The use of `x.T` on tensors of dimension other than 2 to reverse their shape is deprecated and it will throw an error in a future release. Consider `x.mT` to transpose batches of matrices or `x.permute(*torch.arange(x.ndim - 1, -1, -1))` to reverse the dimensions of a tensor. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3277.)\n",
      "  w = w.permute(2, 3, 1, 0) if len(w.shape) == 4 else w.T\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "from utils.keras_models.preprocess import KerasPreprocessing, KerasTransformerPreprocess\n",
    "from utils.keras_models.transfer import *\n",
    "\n",
    "#####################################################################################\n",
    "INPUT_SIZE = (160, 80) #(160, 80)\n",
    "\n",
    "# WEIGHTs FOR CNN - TRANSFORMER\n",
    "CNN_TRANS_WGT = (0.6, 0.4)\n",
    "\n",
    "# CHOOSE CNN: 'rex' or 'eff'\n",
    "CNN = 'eff'\n",
    "# SET NUMBER OF TRANSFORMER MODELS: 3 or 4\n",
    "NUM_TRANS = 2\n",
    "print(f'Amount of transformers: {NUM_TRANS}')\n",
    "\n",
    "#####################################################################################\n",
    "\n",
    "################# CNN\n",
    "#effckpt = '/kaggle/input/islr-share/8963/partialaug_OPT_effb0x160x80_8f_f0_0_bs64_sm0.58_norm1_inter0.23_time13_freq80_repl0.11_lr0.002391_mixup0.42-4.16_aug0.18_rot0.25_zero0.00_shift0.13_scale0.25_inver0.28_ep179/checkpoints/img_v0_fold0_final_loop_best_main_0.8963_0.9503.pth'\n",
    "\n",
    "# FULL TRAIN CKPT\n",
    "#effckpt = '/kaggle/input/full-data-train/_FULL_TRAIN_1_8963params/checkpoints/img_v0_FULL_ep179.pth'\n",
    "\n",
    "# NEW effb0 f8\n",
    "effckpt = '/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_0_pwbad0.82_pwcom1.40_sm0.51_inter0.37_repl0.24_lr0.002705_mixup0.49_aug0.24_rot0.17_shift0.20_scale0.31_inver0.22_ep182/checkpoints/img_v0_fold0_final_loop_best_main_0.8986_0.9518.pth'\n",
    "#effckpt = '/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_0_pwbad0.81_pwcom0.86_sm0.54_inter0.32_repl0.16_lr0.002544_mixup0.46_aug0.26_rot0.11_shift0.21_scale0.28_inver0.32_ep179/checkpoints/img_v0_fold0_final_loop_best_main_0.8975_0.9531.pth'\n",
    "#effckpt = '/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_1_pwbad1.04_pwcom1.63_sm0.51_inter0.36_repl0.13_lr0.002719_mixup0.44_aug0.21_rot0.34_shift0.31_scale0.25_inver0.24_ep178/checkpoints/img_v0_fold0_final_loop_best_main_0.8966_0.9517.pth'\n",
    "\n",
    "\n",
    "artem_ckpt = '/kaggle/input/160x80-weights/twohand_OPT_HYP_SCEX_0.5_DROPLAST/checkpoints/img_v0_fold0_final_loop_best_main_0.8992_0.9513.pth'\n",
    "#eff b0\n",
    "\n",
    "torch_cnn, keras_cnn = transfer_efficientnet(effckpt, \"eff\", size=INPUT_SIZE)\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "preprocess_layer = KerasPreprocessing(size=INPUT_SIZE, name = \"preprocess\")(input_layer)\n",
    "cnn_out = keras_cnn(preprocess_layer)\n",
    "keras_cnn_model = K.models.Model(input_layer, cnn_out)\n",
    "tf.saved_model.save(keras_cnn_model, 'effb0_160')\n",
    "\n",
    "'''torch_cnn, keras_cnn = transfer_efficientnet_artem(artem_ckpt, \"eff\", size=INPUT_SIZE)\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "preprocess_layer = KerasPreprocessing(size=INPUT_SIZE, name = \"preprocess\")(input_layer)\n",
    "cnn_out = keras_cnn(preprocess_layer)\n",
    "keras_cnn_model = K.models.Model(input_layer, cnn_out)\n",
    "tf.saved_model.save(keras_cnn_model, 'effb0_160_artem')'''\n",
    "\n",
    "\n",
    "'''#16f ckpts\n",
    "effckpt16 = '/kaggle/input/effb0-f16-x160x80/n1_effb0_x160x80_16f_f0_2_pwbad0.92_pwcom0.81_sm0.54_inter0.22_repl0.11_lr0.002575_mixup0.65_aug0.16_rot0.24_shift0.16_scale0.23_inver0.30_ep172/checkpoints/img_v0_fold0_final_loop_best_main_0.8986_0.9522.pth'\n",
    "#effckpt16 = '/kaggle/input/effb0-f16-x160x80/n1_effb0_x160x80_16f_f0_2_pwbad1.24_pwcom0.92_sm0.54_inter0.23_repl0.09_lr0.002272_mixup0.68_aug0.17_rot0.26_shift0.17_scale0.23_inver0.27_ep185/checkpoints/img_v0_fold0_final_loop_best_main_0.8992_0.9522.pth'\n",
    "\n",
    "torch_cnn, keras_cnn = transfer_efficientnet(effckpt16, \"eff\", size=INPUT_SIZE)\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "preprocess_layer = KerasPreprocessing(size=INPUT_SIZE, name = \"preprocess\")(input_layer)\n",
    "cnn_out = keras_cnn(preprocess_layer)\n",
    "keras_cnn_model = K.models.Model(input_layer, cnn_out)\n",
    "tf.saved_model.save(keras_cnn_model, 'effb0_160x16f')'''\n",
    "\n",
    "\n",
    "\n",
    "'''rexckpt = \"/kaggle/input/rexnet-ckpts/timm_fold0_final_loop_best_main_0.8953_0.9499.pth\" # 160\n",
    "#rexckpt = \"/kaggle/input/rexnet-ckpts/timm_fold0_ep165_acc0.8950.pth\" # 128\n",
    "#rexnet\n",
    "torch_cnn, keras_cnn = transfer_rexnet(rexckpt, \"rex\", size=INPUT_SIZE)\n",
    "\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "preprocess_layer = KerasPreprocessing(size=(160, 80), name = \"preprocess\")(input_layer)\n",
    "cnn_out = keras_cnn(preprocess_layer)\n",
    "keras_cnn_model = K.models.Model(input_layer, cnn_out)\n",
    "tf.saved_model.save(keras_cnn_model, 'rex160')'''\n",
    "\n",
    "\n",
    "################# BERT\n",
    "bert_full, keras_full = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/bert_full_data.ckpt\", \"bert_full\")\n",
    "bert_configf0, keras_bertf0 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/bert_f0.ckpt\", \"bertf0\")\n",
    "bert_configf1, keras_bertf1 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/bert_f1.ckpt\", \"bertf1\")\n",
    "\n",
    "bert_configf0x16, keras_bertf0x16 = transfer_transformer(\"/kaggle/input/islr-share/bert_16f0.ckpt\", \"bertf0x16\")\n",
    "\n",
    "\n",
    "tf.saved_model.save(keras_full, 'bert_full')\n",
    "tf.saved_model.save(keras_bertf0, 'bert_f0')\n",
    "tf.saved_model.save(keras_bertf1, 'bert_f1')\n",
    "tf.saved_model.save(keras_bertf0x16, 'bert_f0x16')\n",
    "\n",
    "\n",
    "################# DEBERTA\n",
    "deberta_config, keras_deberta = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/deberta_full_data.ckpt\", \"deberta\")\n",
    "deberta_configf0, keras_debertaf0 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/deberta_f0.ckpt\", \"debertaf0\")\n",
    "deberta_configf1, keras_debertaf1 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/deberta_f1.ckpt\", \"debertaf1\")\n",
    "deberta_configf0x16, keras_debertaf0x16 = transfer_transformer(\"/kaggle/input/islr-share/deberta_16f0.ckpt\", \"debertaf0x16\")\n",
    "\n",
    "\n",
    "tf.saved_model.save(keras_deberta,'deberta_full')\n",
    "tf.saved_model.save(keras_debertaf0, 'deberta_f0')\n",
    "tf.saved_model.save(keras_debertaf1, 'deberta_f1')\n",
    "tf.saved_model.save(keras_debertaf0x16, 'deberta_f0x16')\n",
    "\n",
    "\n",
    "################# TRANS PREPROC\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "transformer_preprocess_layer = KerasTransformerPreprocess(bert_full, name=\"trans_preprocess\")(input_layer)\n",
    "keras_tr_prep = K.models.Model(input_layer, transformer_preprocess_layer)\n",
    "tf.saved_model.save(keras_tr_prep, 'keras_tr_prep')\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "615851a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:24:28.231772Z",
     "iopub.status.busy": "2023-04-29T15:24:28.231375Z",
     "iopub.status.idle": "2023-04-29T15:24:28.681918Z",
     "shell.execute_reply": "2023-04-29T15:24:28.680521Z"
    },
    "papermill": {
     "duration": 0.462994,
     "end_time": "2023-04-29T15:24:28.684688",
     "exception": false,
     "start_time": "2023-04-29T15:24:28.221694",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "base_dir = '/kaggle/input/asl-signs/'\n",
    "ROWS_PER_FRAME = 543  # number of landmarks per frame\n",
    "\n",
    "def load_relevant_data_subset(pq_path):\n",
    "    data_columns = ['x', 'y', 'z']\n",
    "    data = pd.read_parquet(pq_path, columns=data_columns)\n",
    "    n_frames = int(len(data) / ROWS_PER_FRAME)\n",
    "    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n",
    "    return data.astype(np.float32)\n",
    "\n",
    "train = pd.read_csv(base_dir + '/train.csv')\n",
    "yy = load_relevant_data_subset(base_dir + train.loc[45, 'path'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "419091b9",
   "metadata": {
    "papermill": {
     "duration": 0.007689,
     "end_time": "2023-04-29T15:24:28.700580",
     "exception": false,
     "start_time": "2023-04-29T15:24:28.692891",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# TFLite Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "45fc2f48",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:24:28.719286Z",
     "iopub.status.busy": "2023-04-29T15:24:28.718148Z",
     "iopub.status.idle": "2023-04-29T15:24:47.844815Z",
     "shell.execute_reply": "2023-04-29T15:24:47.843403Z"
    },
    "papermill": {
     "duration": 19.139033,
     "end_time": "2023-04-29T15:24:47.847564",
     "exception": false,
     "start_time": "2023-04-29T15:24:28.708531",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48\n",
      "4.784736394882202\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "\n",
    "class TFLiteModel(tf.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(TFLiteModel, self).__init__()\n",
    "        \n",
    "        self.cnn = tf.saved_model.load('effb0_160')\n",
    "        ####self.cnn = tf.saved_model.load('effb0_160_artem')\n",
    "        ####self.cnn = tf.saved_model.load('rex160')        \n",
    "        #self.cnn = tf.saved_model.load('effb0_160x16f')\n",
    "        \n",
    "        \n",
    "        self.tr_prep = tf.saved_model.load('keras_tr_prep')\n",
    "        \n",
    "        self.bertfull = tf.saved_model.load('bert_full')\n",
    "        #self.bertf0 = tf.saved_model.load('bert_f0')\n",
    "        #self.bertf1 = tf.saved_model.load('bert_f1')        \n",
    "        #self.bertf0x16 = tf.saved_model.load('bert_f0x16')\n",
    "        \n",
    "        self.debertafull = tf.saved_model.load('deberta_full')\n",
    "        #self.debertaf0 = tf.saved_model.load('deberta_f0')\n",
    "        #self.debertaf1 = tf.saved_model.load('deberta_f1')        \n",
    "        #self.debertaf0x16 = tf.saved_model.load('deberta_f0x16')\n",
    "\n",
    "        # artem hyper 0.9045723962743438 (0.58, 0.42000000000000004, 0.65, 0.35)\n",
    "        # 0.90042 (0.57, 0.43, 0.7, 0.3) our current best effb0 with 2 trans\n",
    "        \n",
    "        # (f16 8992) 0.9036409822184589 (0.65, 0.35, 0.8, 0.19999999999999996)\n",
    "        # (f16 8986) 0.9048264182895851 (0.63, 0.37, 0.6000000000000001, 0.3999999999999999)\n",
    "        \n",
    "        # 8986  0.904233700254022 (0.58, 0.42000000000000004, 0.8500000000000001, 0.1499999999999999)\n",
    "        # 8975  0.9022015241320914 (0.56, 0.43999999999999995, 0.8500000000000001, 0.1499999999999999)\n",
    "        # 8966  0.9022015241320914 (0.5700000000000001, 0.42999999999999994, 0.45, 0.55)\n",
    "        \n",
    "        self.wgt = (0.58, 0.42) \n",
    "        \n",
    "        self.bert_wgt = 0.85   \n",
    "        self.deberta_wgt = 0.15\n",
    "\n",
    "    @tf.function(input_signature=[tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')])\n",
    "    def __call__(self, inputs):\n",
    "        \n",
    "        cnn_out = self.cnn(inputs)\n",
    "        \n",
    "        tr_input = self.tr_prep(inputs)\n",
    "        \n",
    "        bertfull = self.bertfull(tr_input)\n",
    "        #bertf0 = self.bertf0(tr_input)\n",
    "        #bertf1 = self.bertf1(tr_input)\n",
    "        #bertf0x16 = self.bertf0x16(tr_input)\n",
    "        \n",
    "        debertafull = self.debertafull(tr_input)\n",
    "        #debertaf0 = self.debertaf0(tr_input)\n",
    "        #debertaf1 = self.debertaf1(tr_input)\n",
    "        #debertaf0x16 = self.debertaf0x16(tr_input)\n",
    "        \n",
    "        trans_out = tf.add_n([\n",
    "                            tf.multiply(bertfull, self.bert_wgt),\n",
    "                            #tf.multiply(bertf0, self.bert_wgt),\n",
    "                            #tf.multiply(bertf1, self.bert_wgt),\n",
    "                            #tf.multiply(bertf0x16, self.bert_wgt),\n",
    "            \n",
    "                            tf.multiply(debertafull, self.deberta_wgt),\n",
    "                            #tf.multiply(debertaf0, self.deberta_wgt),\n",
    "                            #tf.multiply(debertaf1, self.deberta_wgt),\n",
    "                            #tf.multiply(debertaf0x16, self.deberta_wgt),\n",
    "                            ])        \n",
    "        \n",
    "        out = tf.add_n([tf.multiply(cnn_out, self.wgt[0]), tf.multiply(trans_out, self.wgt[1])])\n",
    "        return {'outputs': out}      \n",
    "\n",
    "tflite_keras_model = TFLiteModel()\n",
    "st = time.time()\n",
    "demo_output = tflite_keras_model(yy)[\"outputs\"]\n",
    "print(np.argmax(demo_output))\n",
    "print(time.time() - st)\n",
    "#decoder(np.argmax(demo_output.numpy(), axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "61a6e129",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:24:47.866043Z",
     "iopub.status.busy": "2023-04-29T15:24:47.865578Z",
     "iopub.status.idle": "2023-04-29T15:25:45.885234Z",
     "shell.execute_reply": "2023-04-29T15:25:45.883533Z"
    },
    "papermill": {
     "duration": 58.03233,
     "end_time": "2023-04-29T15:25:45.888250",
     "exception": false,
     "start_time": "2023-04-29T15:24:47.855920",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  adding: kaggle/working/model_FINAL.tflite (deflated 8%)\r\n",
      "Done with 0.97 minutes\n"
     ]
    }
   ],
   "source": [
    "st = time.time()\n",
    "keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n",
    "\n",
    "use_FP16 = False\n",
    "if use_FP16: #(CNN == 'eff' and NUM_TRANS > 2) or NUM_TRANS > 3:\n",
    "    print(f'Using FP16')\n",
    "    keras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
    "    keras_model_converter.target_spec.supported_types = [tf.float16]\n",
    "\n",
    "tflite_model = keras_model_converter.convert()\n",
    "with open('/kaggle/working/model_FINAL.tflite', 'wb') as f:\n",
    "    f.write(tflite_model)\n",
    "!zip submission.zip /kaggle/working/model_FINAL.tflite\n",
    "\n",
    "print(f'Done with {(time.time() - st)/60:.2f} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8e56f125",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:25:45.908137Z",
     "iopub.status.busy": "2023-04-29T15:25:45.907640Z",
     "iopub.status.idle": "2023-04-29T15:25:46.072995Z",
     "shell.execute_reply": "2023-04-29T15:25:46.071729Z"
    },
    "papermill": {
     "duration": 0.179235,
     "end_time": "2023-04-29T15:25:46.075806",
     "exception": false,
     "start_time": "2023-04-29T15:25:45.896571",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.14821624755859375\n",
      "48\n"
     ]
    }
   ],
   "source": [
    "import tflite_runtime.interpreter as tflite\n",
    "\n",
    "interpreter = tflite.Interpreter('/kaggle/working/model_FINAL.tflite')\n",
    "found_signatures = list(interpreter.get_signature_list().keys())\n",
    "prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n",
    "\n",
    "st = time.time()\n",
    "output = prediction_fn(inputs=yy)\n",
    "print(time.time() - st)\n",
    "sign = np.argmax(output[\"outputs\"])\n",
    "\n",
    "print(sign)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f0260ec7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:25:46.095369Z",
     "iopub.status.busy": "2023-04-29T15:25:46.094892Z",
     "iopub.status.idle": "2023-04-29T15:25:46.169843Z",
     "shell.execute_reply": "2023-04-29T15:25:46.168611Z"
    },
    "papermill": {
     "duration": 0.087471,
     "end_time": "2023-04-29T15:25:46.172279",
     "exception": false,
     "start_time": "2023-04-29T15:25:46.084808",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(94477, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sign</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_landmark_files/26734/1000035562.parquet</td>\n",
       "      <td>26734</td>\n",
       "      <td>1000035562</td>\n",
       "      <td>blow</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_landmark_files/28656/1000106739.parquet</td>\n",
       "      <td>28656</td>\n",
       "      <td>1000106739</td>\n",
       "      <td>wait</td>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_landmark_files/16069/100015657.parquet</td>\n",
       "      <td>16069</td>\n",
       "      <td>100015657</td>\n",
       "      <td>cloud</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_landmark_files/25571/1000210073.parquet</td>\n",
       "      <td>25571</td>\n",
       "      <td>1000210073</td>\n",
       "      <td>bird</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_landmark_files/62590/1000240708.parquet</td>\n",
       "      <td>62590</td>\n",
       "      <td>1000240708</td>\n",
       "      <td>owie</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            path  participant_id  sequence_id  \\\n",
       "0  train_landmark_files/26734/1000035562.parquet           26734   1000035562   \n",
       "1  train_landmark_files/28656/1000106739.parquet           28656   1000106739   \n",
       "2   train_landmark_files/16069/100015657.parquet           16069    100015657   \n",
       "3  train_landmark_files/25571/1000210073.parquet           25571   1000210073   \n",
       "4  train_landmark_files/62590/1000240708.parquet           62590   1000240708   \n",
       "\n",
       "    sign  label  \n",
       "0   blow     25  \n",
       "1   wait    232  \n",
       "2  cloud     48  \n",
       "3   bird     23  \n",
       "4   owie    164  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import json\n",
    "def read_dict(file_path):\n",
    "    path = os.path.expanduser(file_path)\n",
    "    with open(path, \"r\") as f:\n",
    "        dic = json.load(f)\n",
    "    return dic\n",
    "\n",
    "label_index = read_dict(f\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\")\n",
    "index_label = dict([(label_index[key], key) for key in label_index])\n",
    "train[\"label\"] = train[\"sign\"].map(lambda sign: label_index[sign])\n",
    "print(train.shape)\n",
    "display(train.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ead47f83",
   "metadata": {
    "papermill": {
     "duration": 0.008367,
     "end_time": "2023-04-29T15:25:46.189359",
     "exception": false,
     "start_time": "2023-04-29T15:25:46.180992",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Timings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c4625493",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:25:46.209520Z",
     "iopub.status.busy": "2023-04-29T15:25:46.208805Z",
     "iopub.status.idle": "2023-04-29T15:25:56.469321Z",
     "shell.execute_reply": "2023-04-29T15:25:56.468004Z"
    },
    "papermill": {
     "duration": 10.275025,
     "end_time": "2023-04-29T15:25:56.473642",
     "exception": false,
     "start_time": "2023-04-29T15:25:46.198617",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 115/115 [00:10<00:00, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 1.00000\n",
      "Mean time: 0.0891357\n",
      "Mean time only infer: 0.0660898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "st = time.time()\n",
    "cnt = 0\n",
    "total = 115\n",
    "model_time = 0\n",
    "\n",
    "for i in tqdm(np.linspace(0, len(train)-1, total, dtype=int)):\n",
    "    sample = train.loc[i] # 45 len=225\n",
    "\n",
    "    yy = load_relevant_data_subset(base_dir + sample['path'])\n",
    "    md_st = time.time()\n",
    "    output = prediction_fn(inputs=yy)\n",
    "    model_time += time.time() - md_st\n",
    "    sign = np.argmax(output[\"outputs\"])\n",
    "    #print(sample['sign'], sample['label'], sign)\n",
    "    \n",
    "    if sample['label'] == sign: cnt +=1\n",
    "\n",
    "print(f'accuracy: {cnt / total:.5f}')  \n",
    "\n",
    "print(f'Mean time: {(time.time() - st)/total:.7f}')\n",
    "print(f'Mean time only infer: {model_time/total:.7f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c6d0320",
   "metadata": {
    "papermill": {
     "duration": 0.012091,
     "end_time": "2023-04-29T15:25:56.498374",
     "exception": false,
     "start_time": "2023-04-29T15:25:56.486283",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "efa9ba52",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:25:56.526678Z",
     "iopub.status.busy": "2023-04-29T15:25:56.526272Z",
     "iopub.status.idle": "2023-04-29T15:25:56.537507Z",
     "shell.execute_reply": "2023-04-29T15:25:56.536406Z"
    },
    "papermill": {
     "duration": 0.027868,
     "end_time": "2023-04-29T15:25:56.539749",
     "exception": false,
     "start_time": "2023-04-29T15:25:56.511881",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'import tflite_runtime.interpreter as tflite\\nimport time\\n\\nfrom utils.keras_models.preprocess import KerasPreprocessing, KerasTransformerPreprocess\\nfrom utils.keras_models.transfer import *\\n\\nINPUT_SIZE = (160, 80)\\n\\n#here_cnn = \\'/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_0_pwbad0.82_pwcom1.40_sm0.51_inter0.37_repl0.24_lr0.002705_mixup0.49_aug0.24_rot0.17_shift0.20_scale0.31_inver0.22_ep182/checkpoints/img_v0_fold0_final_loop_best_main_0.8986_0.9518.pth\\'\\n#here_cnn = \\'/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_0_pwbad0.81_pwcom0.86_sm0.54_inter0.32_repl0.16_lr0.002544_mixup0.46_aug0.26_rot0.11_shift0.21_scale0.28_inver0.32_ep179/checkpoints/img_v0_fold0_final_loop_best_main_0.8975_0.9531.pth\\'\\nhere_cnn = \\'/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_1_pwbad1.04_pwcom1.63_sm0.51_inter0.36_repl0.13_lr0.002719_mixup0.44_aug0.21_rot0.34_shift0.31_scale0.25_inver0.24_ep178/checkpoints/img_v0_fold0_final_loop_best_main_0.8966_0.9517.pth\\'\\ntorch_cnn, keras_cnn = transfer_efficientnet(here_cnn, \"eff\", size=INPUT_SIZE)\\ninput_layer = K.layers.Input((543, 3), name = \"inputs\")\\npreprocess_layer = KerasPreprocessing(size=INPUT_SIZE, name = \"preprocess\")(input_layer)\\ncnn_out = keras_cnn(preprocess_layer)\\nkeras_cnn_model = K.models.Model(input_layer, cnn_out)\\ntf.saved_model.save(keras_cnn_model, \\'effb0_160_test\\')\\n\\ncnn_md = tf.saved_model.load(\\'effb0_160_test\\')\\n\\ncnn_conver = tf.lite.TFLiteConverter.from_keras_model(cnn_md)\\n\\ntflite_model = cnn_conver.convert()\\nwith open(\\'/kaggle/working/cnn_conver.tflite\\', \\'wb\\') as f:\\n    f.write(tflite_model)\\n    \\ninterpreter = tflite.Interpreter(\\'/kaggle/working/cnn_conver.tflite\\')\\nfound_signatures = list(interpreter.get_signature_list().keys())\\nprediction_fn_cnn = interpreter.get_signature_runner(\"serving_default\")\\n\\n\\nbert_configf0, keras_bertf0 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/bert_f0.ckpt\", \"bertf0\")\\ndeberta_configf0, keras_debertaf0 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/deberta_f0.ckpt\", \"debertaf0\")\\n\\n#bert_configf0, keras_bertf0 = transfer_transformer(\"/kaggle/input/islr-share/bert_16f0.ckpt\", \"bertf0\")\\n#deberta_configf0, keras_debertaf0 = transfer_transformer(\"/kaggle/input/islr-share/deberta_16f0.ckpt\", \"debertaf0\")\\n\\n\\ninput_layer = K.layers.Input((543, 3), name = \"inputs\")\\ntransformer_preprocess_layer = KerasTransformerPreprocess(bert_configf0, name=\"trans_preprocess\")(input_layer)\\nbertf0_out = keras_bertf0(transformer_preprocess_layer)\\ndebertaf0_out = keras_debertaf0(transformer_preprocess_layer)\\n\\noutput_layer = KerasIdentity(None, name = \"outputs\")((bertf0_out + debertaf0_out) / 2)\\ncarno = K.models.Model(input_layer, output_layer)\\n\\ntrans_conver = tf.lite.TFLiteConverter.from_keras_model(carno)\\n\\ntflite_model = trans_conver.convert()\\nwith open(\\'/kaggle/working/trans_conver.tflite\\', \\'wb\\') as f:\\n    f.write(tflite_model)\\n    \\ninterpreter = tflite.Interpreter(\\'/kaggle/working/trans_conver.tflite\\')\\nfound_signatures = list(interpreter.get_signature_list().keys())\\nprediction_fn_trans = interpreter.get_signature_runner(\"serving_default\")\\n\\n########################################################\\ninput_layer = K.layers.Input((543, 3), name = \"inputs\")\\ntransformer_preprocess_layer = KerasTransformerPreprocess(bert_configf0, name=\"trans_preprocess\")(input_layer)\\nbertf0_out = keras_bertf0(transformer_preprocess_layer)\\noutput_layer = KerasIdentity(None, name = \"outputs\")(bertf0_out)\\nbertf0__ = K.models.Model(input_layer, output_layer)\\ntrans_conver11 = tf.lite.TFLiteConverter.from_keras_model(bertf0__)\\n\\ntflite_model = trans_conver11.convert()\\nwith open(\\'/kaggle/working/bertf0_test.tflite\\', \\'wb\\') as f:\\n    f.write(tflite_model)\\n    \\ninterpreter = tflite.Interpreter(\\'/kaggle/working/bertf0_test.tflite\\')\\nfound_signatures = list(interpreter.get_signature_list().keys())\\nprediction_fn_trans11 = interpreter.get_signature_runner(\"serving_default\")\\n\\ninput_layer = K.layers.Input((543, 3), name = \"inputs\")\\ntransformer_preprocess_layer = KerasTransformerPreprocess(bert_configf0, name=\"trans_preprocess\")(input_layer)\\ndebertaf0_out = keras_debertaf0(transformer_preprocess_layer)\\noutput_layer = KerasIdentity(None, name = \"outputs\")(debertaf0_out)\\ndebertf0__ = K.models.Model(input_layer, output_layer)\\ntrans_conver22 = tf.lite.TFLiteConverter.from_keras_model(debertf0__)\\n\\ntflite_model = trans_conver22.convert()\\nwith open(\\'/kaggle/working/debertf0_test.tflite\\', \\'wb\\') as f:\\n    f.write(tflite_model)\\n    \\ninterpreter = tflite.Interpreter(\\'/kaggle/working/debertf0_test.tflite\\')\\nfound_signatures = list(interpreter.get_signature_list().keys())\\nprediction_fn_trans22 = interpreter.get_signature_runner(\"serving_default\")\\n\\n\\nprint(\\'done\\')'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''import tflite_runtime.interpreter as tflite\n",
    "import time\n",
    "\n",
    "from utils.keras_models.preprocess import KerasPreprocessing, KerasTransformerPreprocess\n",
    "from utils.keras_models.transfer import *\n",
    "\n",
    "INPUT_SIZE = (160, 80)\n",
    "\n",
    "#here_cnn = '/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_0_pwbad0.82_pwcom1.40_sm0.51_inter0.37_repl0.24_lr0.002705_mixup0.49_aug0.24_rot0.17_shift0.20_scale0.31_inver0.22_ep182/checkpoints/img_v0_fold0_final_loop_best_main_0.8986_0.9518.pth'\n",
    "#here_cnn = '/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_0_pwbad0.81_pwcom0.86_sm0.54_inter0.32_repl0.16_lr0.002544_mixup0.46_aug0.26_rot0.11_shift0.21_scale0.28_inver0.32_ep179/checkpoints/img_v0_fold0_final_loop_best_main_0.8975_0.9531.pth'\n",
    "here_cnn = '/kaggle/input/160x80-weights/eff_HYP_SCEX_DROPLAST_0.5_5_Rx160x80_8f_f0_1_pwbad1.04_pwcom1.63_sm0.51_inter0.36_repl0.13_lr0.002719_mixup0.44_aug0.21_rot0.34_shift0.31_scale0.25_inver0.24_ep178/checkpoints/img_v0_fold0_final_loop_best_main_0.8966_0.9517.pth'\n",
    "torch_cnn, keras_cnn = transfer_efficientnet(here_cnn, \"eff\", size=INPUT_SIZE)\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "preprocess_layer = KerasPreprocessing(size=INPUT_SIZE, name = \"preprocess\")(input_layer)\n",
    "cnn_out = keras_cnn(preprocess_layer)\n",
    "keras_cnn_model = K.models.Model(input_layer, cnn_out)\n",
    "tf.saved_model.save(keras_cnn_model, 'effb0_160_test')\n",
    "\n",
    "cnn_md = tf.saved_model.load('effb0_160_test')\n",
    "\n",
    "cnn_conver = tf.lite.TFLiteConverter.from_keras_model(cnn_md)\n",
    "\n",
    "tflite_model = cnn_conver.convert()\n",
    "with open('/kaggle/working/cnn_conver.tflite', 'wb') as f:\n",
    "    f.write(tflite_model)\n",
    "    \n",
    "interpreter = tflite.Interpreter('/kaggle/working/cnn_conver.tflite')\n",
    "found_signatures = list(interpreter.get_signature_list().keys())\n",
    "prediction_fn_cnn = interpreter.get_signature_runner(\"serving_default\")\n",
    "\n",
    "\n",
    "bert_configf0, keras_bertf0 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/bert_f0.ckpt\", \"bertf0\")\n",
    "deberta_configf0, keras_debertaf0 = transfer_transformer(\"/kaggle/input/islr-share/transformers_weights/deberta_f0.ckpt\", \"debertaf0\")\n",
    "\n",
    "#bert_configf0, keras_bertf0 = transfer_transformer(\"/kaggle/input/islr-share/bert_16f0.ckpt\", \"bertf0\")\n",
    "#deberta_configf0, keras_debertaf0 = transfer_transformer(\"/kaggle/input/islr-share/deberta_16f0.ckpt\", \"debertaf0\")\n",
    "\n",
    "\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "transformer_preprocess_layer = KerasTransformerPreprocess(bert_configf0, name=\"trans_preprocess\")(input_layer)\n",
    "bertf0_out = keras_bertf0(transformer_preprocess_layer)\n",
    "debertaf0_out = keras_debertaf0(transformer_preprocess_layer)\n",
    "\n",
    "output_layer = KerasIdentity(None, name = \"outputs\")((bertf0_out + debertaf0_out) / 2)\n",
    "carno = K.models.Model(input_layer, output_layer)\n",
    "\n",
    "trans_conver = tf.lite.TFLiteConverter.from_keras_model(carno)\n",
    "\n",
    "tflite_model = trans_conver.convert()\n",
    "with open('/kaggle/working/trans_conver.tflite', 'wb') as f:\n",
    "    f.write(tflite_model)\n",
    "    \n",
    "interpreter = tflite.Interpreter('/kaggle/working/trans_conver.tflite')\n",
    "found_signatures = list(interpreter.get_signature_list().keys())\n",
    "prediction_fn_trans = interpreter.get_signature_runner(\"serving_default\")\n",
    "\n",
    "########################################################\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "transformer_preprocess_layer = KerasTransformerPreprocess(bert_configf0, name=\"trans_preprocess\")(input_layer)\n",
    "bertf0_out = keras_bertf0(transformer_preprocess_layer)\n",
    "output_layer = KerasIdentity(None, name = \"outputs\")(bertf0_out)\n",
    "bertf0__ = K.models.Model(input_layer, output_layer)\n",
    "trans_conver11 = tf.lite.TFLiteConverter.from_keras_model(bertf0__)\n",
    "\n",
    "tflite_model = trans_conver11.convert()\n",
    "with open('/kaggle/working/bertf0_test.tflite', 'wb') as f:\n",
    "    f.write(tflite_model)\n",
    "    \n",
    "interpreter = tflite.Interpreter('/kaggle/working/bertf0_test.tflite')\n",
    "found_signatures = list(interpreter.get_signature_list().keys())\n",
    "prediction_fn_trans11 = interpreter.get_signature_runner(\"serving_default\")\n",
    "\n",
    "input_layer = K.layers.Input((543, 3), name = \"inputs\")\n",
    "transformer_preprocess_layer = KerasTransformerPreprocess(bert_configf0, name=\"trans_preprocess\")(input_layer)\n",
    "debertaf0_out = keras_debertaf0(transformer_preprocess_layer)\n",
    "output_layer = KerasIdentity(None, name = \"outputs\")(debertaf0_out)\n",
    "debertf0__ = K.models.Model(input_layer, output_layer)\n",
    "trans_conver22 = tf.lite.TFLiteConverter.from_keras_model(debertf0__)\n",
    "\n",
    "tflite_model = trans_conver22.convert()\n",
    "with open('/kaggle/working/debertf0_test.tflite', 'wb') as f:\n",
    "    f.write(tflite_model)\n",
    "    \n",
    "interpreter = tflite.Interpreter('/kaggle/working/debertf0_test.tflite')\n",
    "found_signatures = list(interpreter.get_signature_list().keys())\n",
    "prediction_fn_trans22 = interpreter.get_signature_runner(\"serving_default\")\n",
    "\n",
    "\n",
    "print('done')'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7da55656",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:25:56.568633Z",
     "iopub.status.busy": "2023-04-29T15:25:56.567253Z",
     "iopub.status.idle": "2023-04-29T15:25:56.577345Z",
     "shell.execute_reply": "2023-04-29T15:25:56.576131Z"
    },
    "papermill": {
     "duration": 0.027041,
     "end_time": "2023-04-29T15:25:56.579622",
     "exception": false,
     "start_time": "2023-04-29T15:25:56.552581",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nimport json\\ndef read_dict(file_path):\\n    path = os.path.expanduser(file_path)\\n    with open(path, \"r\") as f:\\n        dic = json.load(f)\\n    return dic\\n\\nclass CFG:\\n    base_path = \\'/kaggle/input/\\'\\n    fold_group = False\\n    n_fold = 8 #8\\ntrain = pd.read_csv(\\'/kaggle/input/asl-signs/train.csv\\')\\nlabel_index = read_dict(f\"{CFG.base_path}/asl-signs/sign_to_prediction_index_map.json\")\\nindex_label = dict([(label_index[key], key) for key in label_index])\\ntrain[\"label\"] = train[\"sign\"].map(lambda sign: label_index[sign])\\nprint(train.shape)\\ndisplay(train.head())\\n\\nimport numpy as np\\nfrom sklearn.model_selection import StratifiedGroupKFold, StratifiedKFold\\n\\nsplit = StratifiedKFold(CFG.n_fold, random_state=42, shuffle=True) #rs = 42\\n\\nfor k, (_, test_idx) in enumerate(split.split(train, train.sign)):\\n    train.loc[test_idx, \\'fold\\'] = k\\n\\ntrain.fold = train.fold.astype(int)\\ndisplay(train.groupby(\\'fold\\').size())\\n     \\n    \\ninter_dallata = train[train.fold==0].reset_index(drop=True)\\n\\ncnn, tran1, tran2 = [], [], []\\nfor i in tqdm(range(len(inter_dallata))):\\n    sample = inter_dallata.loc[i] # 45 len=225\\n\\n    yy = load_relevant_data_subset(base_dir + sample[\\'path\\'])\\n    md_st = time.time()\\n    #output = prediction_fn(inputs=yy)[\\'outputs\\']\\n    output = prediction_fn_cnn(inputs=yy)[\\'eff\\'] #[\\'eff\\'] [\\'rex\\']\\n    output2 = prediction_fn_trans11(inputs=yy)[\\'outputs\\']\\n    output3 = prediction_fn_trans22(inputs=yy)[\\'outputs\\']\\n    cnn.append(output)\\n    tran1.append(output2)\\n    tran2.append(output3)\\n    \\ncnn2 = np.array(cnn)\\ntran1 = np.array(tran1)\\ntran2 = np.array(tran2)\\n\\nfrom sklearn.metrics import accuracy_score\\n\\nintr = inter_dallata.copy()\\nranges = np.arange(0, 1.01, 0.01)\\nranges2 = np.arange(0, 1.05, 0.05)\\nlabs = inter_dallata.label.values\\n\\nbest_score = 0\\nbest_coefs = (None, None)\\nfor ran in ranges:\\n    for ran2 in ranges2:\\n        k_cnn = ran\\n        k_tran = 1 - k_cnn\\n        \\n        k_in = ran2\\n        k_in2 = 1 - k_in\\n\\n        prds = cnn2 * k_cnn + k_tran * (tran1 * k_in + tran2 * k_in2)\\n        acc = accuracy_score(labs, prds.argmax(2))\\n\\n        if acc > best_score:\\n            print(f\\'cnn: {k_cnn} | tran {k_tran} | bert {k_in} and deberta {k_in2}\\')\\n            print(f\\'score: {acc:.5f}\\')\\n            print(f\\'Improve {best_score:.5f} --> {acc:.5f}\\')\\n            best_score = acc\\n            best_coefs = k_cnn, k_tran, k_in, k_in2\\n            print()\\n\\nprint(); print()\\nprint(best_score, best_coefs)'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "import json\n",
    "def read_dict(file_path):\n",
    "    path = os.path.expanduser(file_path)\n",
    "    with open(path, \"r\") as f:\n",
    "        dic = json.load(f)\n",
    "    return dic\n",
    "\n",
    "class CFG:\n",
    "    base_path = '/kaggle/input/'\n",
    "    fold_group = False\n",
    "    n_fold = 8 #8\n",
    "train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n",
    "label_index = read_dict(f\"{CFG.base_path}/asl-signs/sign_to_prediction_index_map.json\")\n",
    "index_label = dict([(label_index[key], key) for key in label_index])\n",
    "train[\"label\"] = train[\"sign\"].map(lambda sign: label_index[sign])\n",
    "print(train.shape)\n",
    "display(train.head())\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedGroupKFold, StratifiedKFold\n",
    "\n",
    "split = StratifiedKFold(CFG.n_fold, random_state=42, shuffle=True) #rs = 42\n",
    "\n",
    "for k, (_, test_idx) in enumerate(split.split(train, train.sign)):\n",
    "    train.loc[test_idx, 'fold'] = k\n",
    "\n",
    "train.fold = train.fold.astype(int)\n",
    "display(train.groupby('fold').size())\n",
    "     \n",
    "    \n",
    "inter_dallata = train[train.fold==0].reset_index(drop=True)\n",
    "\n",
    "cnn, tran1, tran2 = [], [], []\n",
    "for i in tqdm(range(len(inter_dallata))):\n",
    "    sample = inter_dallata.loc[i] # 45 len=225\n",
    "\n",
    "    yy = load_relevant_data_subset(base_dir + sample['path'])\n",
    "    md_st = time.time()\n",
    "    #output = prediction_fn(inputs=yy)['outputs']\n",
    "    output = prediction_fn_cnn(inputs=yy)['eff'] #['eff'] ['rex']\n",
    "    output2 = prediction_fn_trans11(inputs=yy)['outputs']\n",
    "    output3 = prediction_fn_trans22(inputs=yy)['outputs']\n",
    "    cnn.append(output)\n",
    "    tran1.append(output2)\n",
    "    tran2.append(output3)\n",
    "    \n",
    "cnn2 = np.array(cnn)\n",
    "tran1 = np.array(tran1)\n",
    "tran2 = np.array(tran2)\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "intr = inter_dallata.copy()\n",
    "ranges = np.arange(0, 1.01, 0.01)\n",
    "ranges2 = np.arange(0, 1.05, 0.05)\n",
    "labs = inter_dallata.label.values\n",
    "\n",
    "best_score = 0\n",
    "best_coefs = (None, None)\n",
    "for ran in ranges:\n",
    "    for ran2 in ranges2:\n",
    "        k_cnn = ran\n",
    "        k_tran = 1 - k_cnn\n",
    "        \n",
    "        k_in = ran2\n",
    "        k_in2 = 1 - k_in\n",
    "\n",
    "        prds = cnn2 * k_cnn + k_tran * (tran1 * k_in + tran2 * k_in2)\n",
    "        acc = accuracy_score(labs, prds.argmax(2))\n",
    "\n",
    "        if acc > best_score:\n",
    "            print(f'cnn: {k_cnn} | tran {k_tran} | bert {k_in} and deberta {k_in2}')\n",
    "            print(f'score: {acc:.5f}')\n",
    "            print(f'Improve {best_score:.5f} --> {acc:.5f}')\n",
    "            best_score = acc\n",
    "            best_coefs = k_cnn, k_tran, k_in, k_in2\n",
    "            print()\n",
    "\n",
    "print(); print()\n",
    "print(best_score, best_coefs)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "495afc7f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-29T15:25:56.607048Z",
     "iopub.status.busy": "2023-04-29T15:25:56.606575Z",
     "iopub.status.idle": "2023-04-29T15:25:56.611898Z",
     "shell.execute_reply": "2023-04-29T15:25:56.610700Z"
    },
    "papermill": {
     "duration": 0.022007,
     "end_time": "2023-04-29T15:25:56.614468",
     "exception": false,
     "start_time": "2023-04-29T15:25:56.592461",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 8986  0.904233700254022 (0.58, 0.42000000000000004, 0.8500000000000001, 0.1499999999999999)\n",
    "# 8975  0.9022015241320914 (0.56, 0.43999999999999995, 0.8500000000000001, 0.1499999999999999)\n",
    "# 8966  0.9022015241320914 (0.5700000000000001, 0.42999999999999994, 0.45, 0.55)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e143330",
   "metadata": {
    "papermill": {
     "duration": 0.01245,
     "end_time": "2023-04-29T15:25:56.679247",
     "exception": false,
     "start_time": "2023-04-29T15:25:56.666797",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 240.186694,
   "end_time": "2023-04-29T15:25:59.414109",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-04-29T15:21:59.227415",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}