notebooks/linear-probing-stl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d05ceca",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../\")\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "from torchvision import models, transforms\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "from relic.encoders import resnet18, resnet50\n",
    "from relic.aug import get_relic_aug_inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ea87b9e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ec7dccba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "IMAGE_SIZE = 96\n",
    "transform = get_relic_aug_inference(image_size=(IMAGE_SIZE, IMAGE_SIZE))\n",
    "\n",
    "train_ds = torchvision.datasets.STL10(\"../data\",\n",
    "                                  split='train',\n",
    "                                  transform=transform,\n",
    "                                  download=True)\n",
    "val_ds = torchvision.datasets.STL10(\"../data\",\n",
    "                                  split='test',\n",
    "                                  transform=transform,\n",
    "                                  download=True)\n",
    "\n",
    "train_loader = DataLoader(train_ds,\n",
    "                          batch_size=256,\n",
    "                          num_workers=4)\n",
    "val_loader = DataLoader(val_ds,\n",
    "                       batch_size=256,\n",
    "                       num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a900a33e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 8000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ds), len(val_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a402d632",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# ckpt = torch.load(\"../models/sota_stl/encoder.pth\")\n",
    "ckpt = torch.load(\"../models/sota_stl_2048/encoder.pth\")\n",
    "\n",
    "# model = resnet18()\n",
    "model = resnet50()\n",
    "model.load_state_dict(ckpt)\n",
    "\n",
    "model = model.eval().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cda08697",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "import numpy as np\n",
    "\n",
    "def get_embs_labels(dl):\n",
    "    idx = 0\n",
    "    embs, labels = [], []\n",
    "    for idx, (images, targets) in enumerate(tqdm(dl)):\n",
    "        with torch.no_grad():\n",
    "            images = images.to(device)\n",
    "            out = model(images)\n",
    "            features = out.cpu().detach().tolist()\n",
    "            embs.extend(features)\n",
    "            labels.extend(targets.cpu().detach().tolist())\n",
    "    return np.array(embs), np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e4788ba1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3993162ecdb413abcfc1d8beb5e3b09",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "983d5eab2e1f446190876a0e064c1a4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "embeddings, labels = get_embs_labels(train_loader)\n",
    "embeddings_val, labels_val = get_embs_labels(val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "61140b47-4bbf-4da8-8d6a-3299c8cc4891",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 2048)\n",
      "(5000,)\n",
      "(8000, 2048)\n",
      "(8000,)\n"
     ]
    }
   ],
   "source": [
    "print(embeddings.shape)\n",
    "print(labels.shape)\n",
    "print(embeddings_val.shape)\n",
    "print(labels_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2e79aa7-fcfa-4bf5-8ed7-3d3b818da188",
   "metadata": {},
   "source": [
    "### Resnet 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c8786aaf-51d0-4ece-92a0-337e7a1f28e2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 5000 5000\n",
      "test 8000 8000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.804\n",
      "Confusion matrix: \n",
      " [[710  32   3  14   1   3   6   1  18  12]\n",
      " [  8 641   0  43  11  33   7  56   1   0]\n",
      " [ 16   2 723   9   0   5   3   1   5  36]\n",
      " [  0  46   0 540  55  82  19  58   0   0]\n",
      " [  0  30   0  45 630  32  41  22   0   0]\n",
      " [  1  33   0 111  41 470  67  76   0   1]\n",
      " [  0   8   0  11  32  48 672  24   0   5]\n",
      " [  0  57   0  53  18  45  13 614   0   0]\n",
      " [ 26   1   3   6   1   1   0   0 744  18]\n",
      " [ 10   2  35   9   1   2   7   2  44 688]]\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.89      0.90       800\n",
      "           1       0.75      0.80      0.78       800\n",
      "           2       0.95      0.90      0.92       800\n",
      "           3       0.64      0.68      0.66       800\n",
      "           4       0.80      0.79      0.79       800\n",
      "           5       0.65      0.59      0.62       800\n",
      "           6       0.80      0.84      0.82       800\n",
      "           7       0.72      0.77      0.74       800\n",
      "           8       0.92      0.93      0.92       800\n",
      "           9       0.91      0.86      0.88       800\n",
      "\n",
      "    accuracy                           0.80      8000\n",
      "   macro avg       0.81      0.80      0.80      8000\n",
      "weighted avg       0.81      0.80      0.80      8000\n",
      "\n",
      "Classification report train: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       500\n",
      "           1       1.00      1.00      1.00       500\n",
      "           2       1.00      1.00      1.00       500\n",
      "           3       0.99      0.99      0.99       500\n",
      "           4       0.99      1.00      0.99       500\n",
      "           5       1.00      0.97      0.98       500\n",
      "           6       0.99      1.00      0.99       500\n",
      "           7       0.98      1.00      0.99       500\n",
      "           8       0.99      1.00      1.00       500\n",
      "           9       1.00      0.98      0.99       500\n",
      "\n",
      "    accuracy                           0.99      5000\n",
      "   macro avg       0.99      0.99      0.99      5000\n",
      "weighted avg       0.99      0.99      0.99      5000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    " \n",
    "X_train, X_test = embeddings, embeddings_val\n",
    "y_train, y_test = labels, labels_val\n",
    "\n",
    "print(\"train\", X_train.shape[0], len(y_train))\n",
    "print(\"test\", X_test.shape[0], len(y_test))\n",
    " \n",
    "clf = LogisticRegression(max_iter=100)\n",
    "clf = CalibratedClassifierCV(clf)\n",
    " \n",
    "clf.fit(X_train, y_train)\n",
    " \n",
    "y_pred = clf.predict(X_test)\n",
    " \n",
    "acc = accuracy_score(y_test, y_pred)\n",
    "conf_matrix = confusion_matrix(y_test, y_pred)\n",
    "class_report = classification_report(y_test, y_pred)\n",
    " \n",
    "print(\"Accuracy: \", acc)\n",
    "print(\"Confusion matrix: \\n\", conf_matrix)\n",
    "print(\"Classification report: \\n\", class_report)\n",
    " \n",
    "y_pred_train = clf.predict(X_train)\n",
    "class_report = classification_report(y_train, y_pred_train)\n",
    "print(\"Classification report train: \\n\", class_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cf02cbb-d000-4331-858c-c7e0544895aa",
   "metadata": {},
   "source": [
    "### Resnet 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ab945605-e86b-48e8-8561-bc62aa229b6a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 5000 5000\n",
      "test 8000 8000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.761\n",
      "Confusion matrix: \n",
      " [[700  30  14   5   0   4   5   0  31  11]\n",
      " [ 18 592   1  62  16  27   8  71   4   1]\n",
      " [ 17   3 726   3   0   4   3   3   6  35]\n",
      " [  0  72   0 488  61  87  18  71   1   2]\n",
      " [  6  28   1  43 594  37  58  30   3   0]\n",
      " [  1  37   3 125  43 410  93  87   1   0]\n",
      " [  4   7   4  14  39  49 655  22   4   2]\n",
      " [  3  61   0  69  16  66  28 557   0   0]\n",
      " [ 29   4   8   1   0   2   0   0 738  18]\n",
      " [ 21   2  63  13   0   1   8   2  62 628]]\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.88      0.88       800\n",
      "           1       0.71      0.74      0.72       800\n",
      "           2       0.89      0.91      0.90       800\n",
      "           3       0.59      0.61      0.60       800\n",
      "           4       0.77      0.74      0.76       800\n",
      "           5       0.60      0.51      0.55       800\n",
      "           6       0.75      0.82      0.78       800\n",
      "           7       0.66      0.70      0.68       800\n",
      "           8       0.87      0.92      0.89       800\n",
      "           9       0.90      0.79      0.84       800\n",
      "\n",
      "    accuracy                           0.76      8000\n",
      "   macro avg       0.76      0.76      0.76      8000\n",
      "weighted avg       0.76      0.76      0.76      8000\n",
      "\n",
      "Classification report train: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.97       500\n",
      "           1       0.88      0.92      0.90       500\n",
      "           2       0.96      0.98      0.97       500\n",
      "           3       0.82      0.83      0.82       500\n",
      "           4       0.90      0.93      0.91       500\n",
      "           5       0.87      0.71      0.78       500\n",
      "           6       0.88      0.96      0.92       500\n",
      "           7       0.85      0.88      0.86       500\n",
      "           8       0.95      0.99      0.97       500\n",
      "           9       0.99      0.88      0.93       500\n",
      "\n",
      "    accuracy                           0.91      5000\n",
      "   macro avg       0.91      0.91      0.90      5000\n",
      "weighted avg       0.91      0.91      0.90      5000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wavelet/projects/relic/.env/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    " \n",
    "X_train, X_test = embeddings, embeddings_val\n",
    "y_train, y_test = labels, labels_val\n",
    "\n",
    "print(\"train\", X_train.shape[0], len(y_train))\n",
    "print(\"test\", X_test.shape[0], len(y_test))\n",
    " \n",
    "clf = LogisticRegression(max_iter=100)\n",
    "clf = CalibratedClassifierCV(clf)\n",
    " \n",
    "clf.fit(X_train, y_train)\n",
    " \n",
    "y_pred = clf.predict(X_test)\n",
    " \n",
    "acc = accuracy_score(y_test, y_pred)\n",
    "conf_matrix = confusion_matrix(y_test, y_pred)\n",
    "class_report = classification_report(y_test, y_pred)\n",
    " \n",
    "print(\"Accuracy: \", acc)\n",
    "print(\"Confusion matrix: \\n\", conf_matrix)\n",
    "print(\"Classification report: \\n\", class_report)\n",
    " \n",
    "y_pred_train = clf.predict(X_train)\n",
    "class_report = classification_report(y_train, y_pred_train)\n",
    "print(\"Classification report train: \\n\", class_report)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}