play-gpt2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import GPT2LMHeadModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed072943a97649a4b396c6556b9ad68e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/548M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\hp\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\huggingface_hub\\file_download.py:148: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\hp\\.cache\\huggingface\\hub\\models--gpt2. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9a8f18d29a54857b4728cda0065c37e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.wte.weight torch.Size([50257, 768])\n",
      "transformer.wpe.weight torch.Size([1024, 768])\n",
      "transformer.h.0.ln_1.weight torch.Size([768])\n",
      "transformer.h.0.ln_1.bias torch.Size([768])\n",
      "transformer.h.0.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.0.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.0.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.0.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.0.ln_2.weight torch.Size([768])\n",
      "transformer.h.0.ln_2.bias torch.Size([768])\n",
      "transformer.h.0.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.0.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.0.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.0.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.1.ln_1.weight torch.Size([768])\n",
      "transformer.h.1.ln_1.bias torch.Size([768])\n",
      "transformer.h.1.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.1.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.1.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.1.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.1.ln_2.weight torch.Size([768])\n",
      "transformer.h.1.ln_2.bias torch.Size([768])\n",
      "transformer.h.1.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.1.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.1.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.1.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.2.ln_1.weight torch.Size([768])\n",
      "transformer.h.2.ln_1.bias torch.Size([768])\n",
      "transformer.h.2.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.2.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.2.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.2.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.2.ln_2.weight torch.Size([768])\n",
      "transformer.h.2.ln_2.bias torch.Size([768])\n",
      "transformer.h.2.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.2.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.2.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.2.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.3.ln_1.weight torch.Size([768])\n",
      "transformer.h.3.ln_1.bias torch.Size([768])\n",
      "transformer.h.3.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.3.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.3.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.3.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.3.ln_2.weight torch.Size([768])\n",
      "transformer.h.3.ln_2.bias torch.Size([768])\n",
      "transformer.h.3.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.3.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.3.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.3.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.4.ln_1.weight torch.Size([768])\n",
      "transformer.h.4.ln_1.bias torch.Size([768])\n",
      "transformer.h.4.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.4.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.4.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.4.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.4.ln_2.weight torch.Size([768])\n",
      "transformer.h.4.ln_2.bias torch.Size([768])\n",
      "transformer.h.4.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.4.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.4.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.4.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.5.ln_1.weight torch.Size([768])\n",
      "transformer.h.5.ln_1.bias torch.Size([768])\n",
      "transformer.h.5.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.5.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.5.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.5.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.5.ln_2.weight torch.Size([768])\n",
      "transformer.h.5.ln_2.bias torch.Size([768])\n",
      "transformer.h.5.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.5.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.5.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.5.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.6.ln_1.weight torch.Size([768])\n",
      "transformer.h.6.ln_1.bias torch.Size([768])\n",
      "transformer.h.6.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.6.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.6.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.6.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.6.ln_2.weight torch.Size([768])\n",
      "transformer.h.6.ln_2.bias torch.Size([768])\n",
      "transformer.h.6.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.6.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.6.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.6.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.7.ln_1.weight torch.Size([768])\n",
      "transformer.h.7.ln_1.bias torch.Size([768])\n",
      "transformer.h.7.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.7.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.7.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.7.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.7.ln_2.weight torch.Size([768])\n",
      "transformer.h.7.ln_2.bias torch.Size([768])\n",
      "transformer.h.7.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.7.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.7.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.7.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.8.ln_1.weight torch.Size([768])\n",
      "transformer.h.8.ln_1.bias torch.Size([768])\n",
      "transformer.h.8.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.8.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.8.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.8.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.8.ln_2.weight torch.Size([768])\n",
      "transformer.h.8.ln_2.bias torch.Size([768])\n",
      "transformer.h.8.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.8.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.8.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.8.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.9.ln_1.weight torch.Size([768])\n",
      "transformer.h.9.ln_1.bias torch.Size([768])\n",
      "transformer.h.9.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.9.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.9.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.9.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.9.ln_2.weight torch.Size([768])\n",
      "transformer.h.9.ln_2.bias torch.Size([768])\n",
      "transformer.h.9.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.9.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.9.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.9.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.10.ln_1.weight torch.Size([768])\n",
      "transformer.h.10.ln_1.bias torch.Size([768])\n",
      "transformer.h.10.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.10.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.10.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.10.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.10.ln_2.weight torch.Size([768])\n",
      "transformer.h.10.ln_2.bias torch.Size([768])\n",
      "transformer.h.10.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.10.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.10.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.10.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.h.11.ln_1.weight torch.Size([768])\n",
      "transformer.h.11.ln_1.bias torch.Size([768])\n",
      "transformer.h.11.attn.c_attn.weight torch.Size([768, 2304])\n",
      "transformer.h.11.attn.c_attn.bias torch.Size([2304])\n",
      "transformer.h.11.attn.c_proj.weight torch.Size([768, 768])\n",
      "transformer.h.11.attn.c_proj.bias torch.Size([768])\n",
      "transformer.h.11.ln_2.weight torch.Size([768])\n",
      "transformer.h.11.ln_2.bias torch.Size([768])\n",
      "transformer.h.11.mlp.c_fc.weight torch.Size([768, 3072])\n",
      "transformer.h.11.mlp.c_fc.bias torch.Size([3072])\n",
      "transformer.h.11.mlp.c_proj.weight torch.Size([3072, 768])\n",
      "transformer.h.11.mlp.c_proj.bias torch.Size([768])\n",
      "transformer.ln_f.weight torch.Size([768])\n",
      "transformer.ln_f.bias torch.Size([768])\n",
      "lm_head.weight torch.Size([50257, 768])\n"
     ]
    }
   ],
   "source": [
    "#Call the GPT model of 124M\n",
    "model_hf = GPT2LMHeadModel.from_pretrained(\"gpt2\")\n",
    "sd_hf = model_hf.state_dict()\n",
    "\n",
    "for k,v in sd_hf.items():\n",
    "    print(k,v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}