demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5d7ff7fc",
   "metadata": {},
   "source": [
    "## 正常"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bccca5a9",
   "metadata": {},
   "source": [
    "### 微调"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9ba733da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 111\n",
      "CUDA SETUP: Loading binary /home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda111.so...\n",
      "Loading tokenizer and model of THUDM/chatglm-6b\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /home/wyd/miniconda3/envs/flexgen did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64 did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
      "  warn(msg)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0082d1e636164fd2b7c20bd6bd9a6511",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing peft model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/peft/tuners/lora.py:175: UserWarning: fan_in_fan_out is set to True but the target module is not a Conv1D. Setting fan_in_fan_out to False.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 3670016 || all params: 6176956416 || trainable%: 0.05941463324063059\n"
     ]
    }
   ],
   "source": [
    "import hcgf\n",
    "gl = hcgf.GlmLora(\"THUDM/chatglm-6b\", device=\"cuda:0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0e75df24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data with max_seq_len: 32\n",
      "Switch to training mode...\n",
      "Begining tunning\n",
      "Total data batches: 19, Epoch update steps: 19, Total update steps: 38, Warmup update steps: 0, Validation steps: 6, Early stop steps: 57\n",
      "\n",
      "\n",
      "Epoch 0/2\n",
      "Step: 2, Loss: 13.5151, LearningRate: 0.000189 \n",
      "Step: 4, Loss: 11.9978, LearningRate: 0.000179 \n",
      "Step: 6, Loss: 11.0744, LearningRate: 0.000168 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 6/0,  Total Step: 6,  LearningRate: 0.000168, \n",
      "TrainLoss: 11.0744,  ValidLoss: 9.3150\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 8, Loss: 10.4971, LearningRate: 0.000158 \n",
      "Step: 10, Loss: 10.1384, LearningRate: 0.000147 \n",
      "Step: 12, Loss: 9.6661, LearningRate: 0.000137 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 12/0,  Total Step: 12,  LearningRate: 0.000137, \n",
      "TrainLoss: 9.6661,  ValidLoss: 6.0409\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 14, Loss: 9.3166, LearningRate: 0.000126 \n",
      "Step: 16, Loss: 8.8593, LearningRate: 0.000116 \n",
      "Step: 18, Loss: 8.5360, LearningRate: 0.000105 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 18/0,  Total Step: 18,  LearningRate: 0.000105, \n",
      "TrainLoss: 8.5360,  ValidLoss: 4.5296\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Epoch: 0 | time in 0.1 minutes, 6 seconds \n",
      "\tEpoch TrainLoss: 8.3046  \n",
      "\tEpoch ValidLoss: 4.3499  \n",
      "\n",
      "\n",
      "Epoch 1/2\n",
      "Step: 2, Loss: 4.9558, LearningRate: 0.000089 \n",
      "Step: 4, Loss: 3.9609, LearningRate: 0.000079 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 5/1,  Total Step: 24,  LearningRate: 0.000074, \n",
      "TrainLoss: 3.7336,  ValidLoss: 3.5598\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 6, Loss: 3.4965, LearningRate: 0.000068 \n",
      "Step: 8, Loss: 3.6708, LearningRate: 0.000058 \n",
      "Step: 10, Loss: 3.6976, LearningRate: 0.000047 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 11/1,  Total Step: 30,  LearningRate: 0.000042, \n",
      "TrainLoss: 3.5090,  ValidLoss: 2.9730\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 12, Loss: 3.5064, LearningRate: 0.000037 \n",
      "Step: 14, Loss: 3.4383, LearningRate: 0.000026 \n",
      "Step: 16, Loss: 3.3320, LearningRate: 0.000016 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 17/1,  Total Step: 36,  LearningRate: 0.000011, \n",
      "TrainLoss: 3.2836,  ValidLoss: 2.7118\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 18, Loss: 3.2084, LearningRate: 0.000005 \n",
      "Epoch: 1 | time in 0.1 minutes, 9 seconds \n",
      "\tEpoch TrainLoss: 3.1074  \n",
      "\tEpoch ValidLoss: 2.6803  \n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    \"lr\": 2e-4,\n",
    "    \"num_epochs\": 2, \n",
    "    \"warmup_steps\": 0, \n",
    "    \"accumulate_steps\": 1, \n",
    "    \"print_every\": 2, \n",
    "}\n",
    "gl.load_data(\"./tests/test_data/test_data.json\", max_seq_len=32).tune(**params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "925aa224",
   "metadata": {},
   "source": [
    "### 推理\n",
    "\n",
    "重启下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a08992e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 111\n",
      "CUDA SETUP: Loading binary /home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda111.so...\n",
      "Loading tokenizer and model of THUDM/chatglm-6b\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /home/wyd/miniconda3/envs/flexgen did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64 did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
      "  warn(msg)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0fa962e4049411eb5ff04917e923a73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing peft model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/peft/tuners/lora.py:175: UserWarning: fan_in_fan_out is set to True but the target module is not a Conv1D. Setting fan_in_fan_out to False.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 3670016 || all params: 6176956416 || trainable%: 0.05941463324063059\n"
     ]
    }
   ],
   "source": [
    "import hcgf\n",
    "gl = hcgf.GlmLora(\"THUDM/chatglm-6b\", device=\"cuda:1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5594ca0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Switch to inference mode...\n"
     ]
    }
   ],
   "source": [
    "gl.load_pretrained(\"output/ckpt/lora-ckpt-last-38.pt\").eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d11ddbfa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('我是一休小和尚。一休小和尚是日本著名的小和尚，因为经常光头示人，所以被称为一休小和尚。一休小和尚聪明伶俐，善于言谈，喜欢读书，喜欢思考。',\n",
       " [('你是谁？',\n",
       "   '我是一休小和尚。一休小和尚是日本著名的小和尚，因为经常光头示人，所以被称为一休小和尚。一休小和尚聪明伶俐，善于言谈，喜欢读书，喜欢思考。')])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gl.chat(\"你是谁？\", temperature=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4400f7ef",
   "metadata": {},
   "source": [
    "## 8bit"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd758105",
   "metadata": {},
   "source": [
    "### 微调"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60789e32",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /home/wyd/miniconda3/envs/flexgen did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64 did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
      "  warn(msg)\n",
      "Overriding torch_dtype=None with `torch_dtype=torch.float16` due to requirements of `bitsandbytes` to enable model loading in mixed int8. Either pass torch_dtype=torch.float16 or don't pass this argument at all to remove this warning.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 111\n",
      "CUDA SETUP: Loading binary /home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda111.so...\n",
      "Loading tokenizer and model of THUDM/chatglm-6b\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c95ef9d9c02947908968fd3a51409f69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing peft model\n",
      "trainable params: 3670016 || all params: 6176956416 || trainable%: 0.05941463324063059\n"
     ]
    }
   ],
   "source": [
    "import hcgf\n",
    "gl = hcgf.GlmLora(\"THUDM/chatglm-6b\", load_in_8bit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1dac8c40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data with max_seq_len: 512\n",
      "Switch to training mode...\n",
      "Begining tunning\n",
      "Total data batches: 19, Epoch update steps: 19, Total update steps: 38, Warmup update steps: 0, Validation steps: 6, Early stop steps: 57\n",
      "\n",
      "\n",
      "Epoch 0/2\n",
      "Step: 2, Loss: 0.8712, LearningRate: 0.000189 \n",
      "Step: 4, Loss: 1.1272, LearningRate: 0.000179 \n",
      "Step: 6, Loss: 1.2099, LearningRate: 0.000168 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 6/0,  Total Step: 6,  LearningRate: 0.000168, \n",
      "TrainLoss: 1.2099,  ValidLoss: 1.3976\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 8, Loss: 1.3223, LearningRate: 0.000158 \n",
      "Step: 10, Loss: 1.4480, LearningRate: 0.000147 \n",
      "Step: 12, Loss: 1.3649, LearningRate: 0.000137 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 12/0,  Total Step: 12,  LearningRate: 0.000137, \n",
      "TrainLoss: 1.3649,  ValidLoss: 0.8056\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 14, Loss: 1.2276, LearningRate: 0.000126 \n",
      "Step: 16, Loss: 1.2283, LearningRate: 0.000116 \n",
      "Step: 18, Loss: 1.1352, LearningRate: 0.000105 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 18/0,  Total Step: 18,  LearningRate: 0.000105, \n",
      "TrainLoss: 1.1352,  ValidLoss: 0.2342\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Epoch: 0 | time in 0.2 minutes, 11 seconds \n",
      "\tEpoch TrainLoss: 1.0865  \n",
      "\tEpoch ValidLoss: 0.1962  \n",
      "\n",
      "\n",
      "Epoch 1/2\n",
      "Step: 2, Loss: 0.2360, LearningRate: 0.000089 \n",
      "Step: 4, Loss: 0.2072, LearningRate: 0.000079 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 5/1,  Total Step: 24,  LearningRate: 0.000074, \n",
      "TrainLoss: 0.1936,  ValidLoss: 0.1323\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 6, Loss: 0.1829, LearningRate: 0.000068 \n",
      "Step: 8, Loss: 0.1649, LearningRate: 0.000058 \n",
      "Step: 10, Loss: 0.1557, LearningRate: 0.000047 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 11/1,  Total Step: 30,  LearningRate: 0.000042, \n",
      "TrainLoss: 0.1494,  ValidLoss: 0.0998\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 12, Loss: 0.1425, LearningRate: 0.000037 \n",
      "Step: 14, Loss: 0.1341, LearningRate: 0.000026 \n",
      "Step: 16, Loss: 0.1290, LearningRate: 0.000016 \n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "Step(Batch)/Epoch: 17/1,  Total Step: 36,  LearningRate: 0.000011, \n",
      "TrainLoss: 0.1261,  ValidLoss: 0.0860\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "Step: 18, Loss: 0.1242, LearningRate: 0.000005 \n",
      "Epoch: 1 | time in 0.4 minutes, 22 seconds \n",
      "\tEpoch TrainLoss: 0.1214  \n",
      "\tEpoch ValidLoss: 0.0852  \n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    \"lr\": 2e-4,\n",
    "    \"num_epochs\": 2, \n",
    "    \"warmup_steps\": 0, \n",
    "    \"accumulate_steps\": 1, \n",
    "    \"print_every\": 2, \n",
    "}\n",
    "gl.load_data(\"tests/test_data/test_data.json\").tune(**params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75c72d70",
   "metadata": {},
   "source": [
    "### 推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a083ccb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /home/wyd/miniconda3/envs/flexgen did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64 did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
      "  warn(msg)\n",
      "Overriding torch_dtype=None with `torch_dtype=torch.float16` due to requirements of `bitsandbytes` to enable model loading in mixed int8. Either pass torch_dtype=torch.float16 or don't pass this argument at all to remove this warning.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 111\n",
      "CUDA SETUP: Loading binary /home/wyd/miniconda3/envs/flexgen/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda111.so...\n",
      "Loading tokenizer and model of THUDM/chatglm-6b\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bd2e70c36bd4711831d846593db4fee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing peft model\n",
      "trainable params: 3670016 || all params: 6176956416 || trainable%: 0.05941463324063059\n"
     ]
    }
   ],
   "source": [
    "import hcgf\n",
    "gl = hcgf.GlmLora(\"THUDM/chatglm-6b\", load_in_8bit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c7283271",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Switch to inference mode...\n"
     ]
    }
   ],
   "source": [
    "gl.load_pretrained(\"./output/ckpt/lora-ckpt-last-38.pt\").eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6ddd9083",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('我是一休小和尚。 问题', [('你是谁？', '我是一休小和尚。 问题')])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gl.chat(\"你是谁？\", temperature=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4ab1969",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}