04_Data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1c55279f",
   "metadata": {},
   "source": [
    "# Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfc9f672",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import datasets\n",
    "\n",
    "from pprint import pprint\n",
    "from transformers import AutoTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed223760",
   "metadata": {},
   "source": [
    "## Tokenizing text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f247cd15",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/pythia-70m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37f55e08",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"Hi, how are you?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f143c0f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_text = tokenizer(text)[\"input_ids\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec714c33",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50f31a63",
   "metadata": {},
   "outputs": [],
   "source": [
    "decoded_text = tokenizer.decode(encoded_text)\n",
    "print(\"Decoded tokens back into text: \", decoded_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa452dec",
   "metadata": {},
   "source": [
    "## Tokenize multiple texts at once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04882f2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_texts = [\"Hi, how are you?\", \"I'm good\", \"Yes\"]\n",
    "encoded_texts = tokenizer(list_texts)\n",
    "print(\"Encoded several texts: \", encoded_texts[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf2e59a4",
   "metadata": {},
   "source": [
    "## Padding and truncation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09fdbb16",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pad_token = tokenizer.eos_token \n",
    "encoded_texts_longest = tokenizer(list_texts, padding=True)\n",
    "print(\"Using padding: \", encoded_texts_longest[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c210247b",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_texts_truncation = tokenizer(list_texts, max_length=3, truncation=True)\n",
    "print(\"Using truncation: \", encoded_texts_truncation[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41eaf7ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.truncation_side = \"left\"\n",
    "encoded_texts_truncation_left = tokenizer(list_texts, max_length=3, truncation=True)\n",
    "print(\"Using left-side truncation: \", encoded_texts_truncation_left[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80c5d71e",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_texts_both = tokenizer(list_texts, max_length=3, truncation=True, padding=True)\n",
    "print(\"Using both padding and truncation: \", encoded_texts_both[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00d09d35",
   "metadata": {},
   "source": [
    "## Prepare instruction dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42b18547",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "filename = \"lamini_docs.jsonl\"\n",
    "instruction_dataset_df = pd.read_json(filename, lines=True)\n",
    "examples = instruction_dataset_df.to_dict()\n",
    "\n",
    "if \"question\" in examples and \"answer\" in examples:\n",
    "  text = examples[\"question\"][0] + examples[\"answer\"][0]\n",
    "elif \"instruction\" in examples and \"response\" in examples:\n",
    "  text = examples[\"instruction\"][0] + examples[\"response\"][0]\n",
    "elif \"input\" in examples and \"output\" in examples:\n",
    "  text = examples[\"input\"][0] + examples[\"output\"][0]\n",
    "else:\n",
    "  text = examples[\"text\"][0]\n",
    "\n",
    "prompt_template = \"\"\"### Question:\n",
    "{question}\n",
    "\n",
    "### Answer:\"\"\"\n",
    "\n",
    "num_examples = len(examples[\"question\"])\n",
    "finetuning_dataset = []\n",
    "for i in range(num_examples):\n",
    "  question = examples[\"question\"][i]\n",
    "  answer = examples[\"answer\"][i]\n",
    "  text_with_prompt_template = prompt_template.format(question=question)\n",
    "  finetuning_dataset.append({\"question\": text_with_prompt_template, \"answer\": answer})\n",
    "\n",
    "from pprint import pprint\n",
    "print(\"One datapoint in the finetuning dataset:\")\n",
    "pprint(finetuning_dataset[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ab0789b",
   "metadata": {},
   "source": [
    "## Tokenize a single example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4c320ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = finetuning_dataset[0][\"question\"] + finetuning_dataset[0][\"answer\"]\n",
    "tokenized_inputs = tokenizer(\n",
    "    text,\n",
    "    return_tensors=\"np\",\n",
    "    padding=True\n",
    ")\n",
    "print(tokenized_inputs[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1674bc66",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = 2048\n",
    "max_length = min(\n",
    "    tokenized_inputs[\"input_ids\"].shape[1],\n",
    "    max_length,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c1db47b",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_inputs = tokenizer(\n",
    "    text,\n",
    "    return_tensors=\"np\",\n",
    "    truncation=True,\n",
    "    max_length=max_length\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fdaaa14",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_inputs[\"input_ids\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4c8c54",
   "metadata": {},
   "source": [
    "## Tokenize the instruction dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b8d596b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_function(examples):\n",
    "    if \"question\" in examples and \"answer\" in examples:\n",
    "      text = examples[\"question\"][0] + examples[\"answer\"][0]\n",
    "    elif \"input\" in examples and \"output\" in examples:\n",
    "      text = examples[\"input\"][0] + examples[\"output\"][0]\n",
    "    else:\n",
    "      text = examples[\"text\"][0]\n",
    "\n",
    "    tokenizer.pad_token = tokenizer.eos_token\n",
    "    tokenized_inputs = tokenizer(\n",
    "        text,\n",
    "        return_tensors=\"np\",\n",
    "        padding=True,\n",
    "    )\n",
    "\n",
    "    max_length = min(\n",
    "        tokenized_inputs[\"input_ids\"].shape[1],\n",
    "        2048\n",
    "    )\n",
    "    tokenizer.truncation_side = \"left\"\n",
    "    tokenized_inputs = tokenizer(\n",
    "        text,\n",
    "        return_tensors=\"np\",\n",
    "        truncation=True,\n",
    "        max_length=max_length\n",
    "    )\n",
    "\n",
    "    return tokenized_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ef040ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "finetuning_dataset_loaded = datasets.load_dataset(\"json\", data_files=filename, split=\"train\")\n",
    "\n",
    "tokenized_dataset = finetuning_dataset_loaded.map(\n",
    "    tokenize_function,\n",
    "    batched=True,\n",
    "    batch_size=1,\n",
    "    drop_last_batch=True\n",
    ")\n",
    "\n",
    "print(tokenized_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd17145f",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_dataset = tokenized_dataset.add_column(\"labels\", tokenized_dataset[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4918320f",
   "metadata": {},
   "source": [
    "## Prepare test/train splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa77e06e",
   "metadata": {},
   "outputs": [],
   "source": [
    "split_dataset = tokenized_dataset.train_test_split(test_size=0.1, shuffle=True, seed=123)\n",
    "print(split_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c73ea8f8",
   "metadata": {},
   "source": [
    "### Some datasets for you to try"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3c9fea2",
   "metadata": {},
   "outputs": [],
   "source": [
    "finetuning_dataset_path = \"lamini/lamini_docs\"\n",
    "finetuning_dataset = datasets.load_dataset(finetuning_dataset_path)\n",
    "print(finetuning_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc5cb28e",
   "metadata": {},
   "outputs": [],
   "source": [
    "taylor_swift_dataset = \"lamini/taylor_swift\"\n",
    "bts_dataset = \"lamini/bts\"\n",
    "open_llms = \"lamini/open_llms\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f28cef5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_swiftie = datasets.load_dataset(taylor_swift_dataset)\n",
    "print(dataset_swiftie[\"train\"][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f4b5741",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is how to push your own dataset to your Huggingface hub\n",
    "# !pip install huggingface_hub\n",
    "# !huggingface-cli login\n",
    "# split_dataset.push_to_hub(dataset_path_hf)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}