model_concatenate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e03399b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-11 14:15:08.775589: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-06-11 14:15:09.647860: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvrtc.so.11.1: cannot open shared object file: No such file or directory\n",
      "2023-06-11 14:15:09.648676: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvrtc.so.11.1: cannot open shared object file: No such file or directory\n",
      "2023-06-11 14:15:09.648685: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random seed set as 42\n",
      "Random seed set as 42\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "import transformers\n",
    "from transformers import ViTModel, ViTConfig, ViTForImageClassification\n",
    "from tqdm.auto import tqdm\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import clear_output\n",
    "from copy import deepcopy\n",
    "import random\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR\n",
    "from vit_train import validate, to_rgb, pretrained_model, create_dataset\n",
    "from vit import MemoryCapableViT\n",
    "from torchvision.transforms import ToTensor, Lambda, Compose\n",
    "from torchvision.transforms.functional import to_pil_image, to_grayscale\n",
    "\n",
    "\n",
    "device = \"cuda:0\"\n",
    "seed = 42\n",
    "random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "# Directories for cache and datasets\n",
    "home_dir = \"/hdd/ege\"\n",
    "cache_dir = os.path.join(home_dir, \"ceng502\")\n",
    "datasets_dir = os.path.join(home_dir, \"datasets\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "926accdf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch32-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch32-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = pretrained_model(cache_dir = cache_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2a615c5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = model\n",
    "    \n",
    "model = MemoryCapableViT(deepcopy(base_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2aaf341c",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_parameters = model.add_head(memory_tokens=1, num_classes=100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3078498b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(\"models/CIFAR100_model.pt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fe317caf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch32-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch32-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model_2 = pretrained_model(cache_dir = cache_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fa103b74",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2 = MemoryCapableViT(deepcopy(model_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f001d919",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_parameters_2 = model_2.add_head(memory_tokens=1, num_classes=365)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "643574e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2.load_state_dict(torch.load(\"models/Places_model.pt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "97305121",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.concatenate(model_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "87714b6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# Load and preprocess the dataset\n",
    "transform = transforms.Compose([\n",
    "    transforms.RandomResizedCrop((224)),\n",
    "    transforms.ToTensor(),\n",
    "    #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "])\n",
    "\n",
    "\n",
    "# CIFAR100\n",
    "train_dataset = datasets.CIFAR100(root=datasets_dir, train=True, transform=transform, download=True)\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=64, shuffle=True)\n",
    "validation_dataset = datasets.CIFAR100(root=datasets_dir, train=False, transform=transform, download=True)\n",
    "validation_loader = DataLoader(dataset=validation_dataset, batch_size=64, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fdb93304",
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "                transforms.Resize((224, 224)),\n",
    "                transforms.ToTensor(),\n",
    "                #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "])\n",
    "\n",
    "try:\n",
    "    train_dataset_2 = datasets.Places365(root=datasets_dir,small=True, split=\"train-standard\", transform=transform, download=True)\n",
    "\n",
    "except:\n",
    "    train_dataset_2 = datasets.Places365(root=datasets_dir,small=True, split=\"train-standard\", transform=transform)\n",
    "train_loader_2 = DataLoader(dataset=train_dataset_2, batch_size=64, shuffle=True)\n",
    "\n",
    "try:\n",
    "    validation_dataset_2 = datasets.Places365(root=datasets_dir,small=True, split=\"val\", transform=transform, download=True)\n",
    "\n",
    "except:\n",
    "    validation_dataset_2 = datasets.Places365(root=datasets_dir,small=True, split=\"val\", transform=transform)\n",
    "\n",
    "validation_loader_2 = DataLoader(dataset=validation_dataset_2, batch_size=64, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c8e36c64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/157 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.6408"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate(model, validation_loader, output_head=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3d188dd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/571 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.5046575342465753"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate(model, validation_loader_2, output_head=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a149c6a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/571 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.0031232876712328768"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate(model_2, validation_loader_2, output_head = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "34b8dda4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(model, dataloader, output_head=None):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for data, targets in tqdm(dataloader, leave=False):\n",
    "            data = data.to(device)\n",
    "            targets = targets.to(device)\n",
    "\n",
    "            outputs = model(data)\n",
    "            if output_head is not None:\n",
    "                outputs = outputs[output_head]\n",
    "            _, predicted = torch.max(outputs.logits, 1)\n",
    "            total += targets.size(0)\n",
    "            correct += (predicted == targets).sum().item()\n",
    "    \n",
    "    return correct / total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a63e8994",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/571 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.0025205479452054796"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate(model, validation_loader_2, output_head=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c54148f7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}