Task5_POPE_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71e86b47",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "from tqdm.auto import tqdm\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from utils.func import read_data\n",
    "from utils.metric import evaluate, eval_pope"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e61cffa",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"LLaVA-7B\"\n",
    "prompt = \"oe\"\n",
    "\n",
    "train_data, x_train, y_train = read_data(model_name, \"POPE\", split=\"train\", \n",
    "                                prompt=prompt, token_idx=0, return_data=True)\n",
    "val_data, x_val, y_val = read_data(model_name, \"POPE\", split=\"val\", \n",
    "                                   prompt=prompt, token_idx=0, return_data=True)\n",
    "\n",
    "print(x_train.shape, y_train.shape, x_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1df19f3",
   "metadata": {},
   "source": [
    "### The original performance of LVLMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b288ae23",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(val_data)):\n",
    "    val_data[i]['pred'] = 1 if val_data[i]['response'].lower().startswith('yes') else 0\n",
    "    \n",
    "for category in [\"adversarial\", \"popular\", \"random\"]:\n",
    "    print(category)\n",
    "    \n",
    "    label_list = [ins['label'] for ins in val_data\n",
    "                  if ins['category'] == category]\n",
    "    pred_list = [ins['pred'] for ins in val_data\n",
    "                  if ins['category'] == category]\n",
    "    \n",
    "    eval_pope(label_list, pred_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ba76ff7",
   "metadata": {},
   "source": [
    "### Linear probing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b250191",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logits\n",
    "print(x_train.shape, x_val.shape)\n",
    "\n",
    "model = LogisticRegression()\n",
    "model.fit(x_train, y_train)\n",
    "\n",
    "y_pred = model.predict_proba(x_val)[:, 1]\n",
    "evaluate(y_val, y_pred, show=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b236953",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(val_data)):\n",
    "    val_data[i]['pred'] = 1 if y_pred[i] > 0.5 else 0\n",
    "    \n",
    "for category in [\"adversarial\", \"popular\", \"random\"]:\n",
    "    print(category)\n",
    "    label_list = [ins['label'] for ins in val_data\n",
    "                  if ins['category'] == category]\n",
    "    pred_list = [ins['pred'] for ins in val_data\n",
    "                  if ins['category'] == category]\n",
    "    eval_pope(label_list, pred_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}