0.0. Prepare_dataset_v2(Final).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "\"\"\"\n",
    "Format Dataset: (list of lists) \n",
    "    [\n",
    "        [context_0, context_1, ..., context_N, response], # sample_0\n",
    "        [context_0, context_1, ..., context_N, response], # sample_1\n",
    "        ...,\n",
    "        [context_0, context_1, ..., context_N, response] # sample_L\n",
    "    ]\n",
    "\"\"\""
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'\\nFormat Dataset: (list of lists) \\n    [\\n        [context_0, context_1, ..., context_N, response], # sample_0\\n        [context_0, context_1, ..., context_N, response], # sample_1\\n        ...,\\n        [context_0, context_1, ..., context_N, response] # sample_L\\n    ]\\n'"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "import pandas as pd\n",
    "# tokenizer\n",
    "from transformers import AutoTokenizer\n",
    "import config as cfg"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "# Parameters\n",
    "# concatenate N_PAIRS+1 pairs of sentences (user1: txt, user2: text), the last sentence would be a sentence from the interest user\n",
    "N_PAIRS = 7 # use 7 sentences like input (context) and the next sentence like a target\n",
    "TEST_SIZE = 0.1\n",
    "\n",
    "\n",
    "# Paths\n",
    "#\"Dataset/Yoda/yoda-corpus.csv\"\n",
    "#\"Dataset/Rick/RickAndMortyScripts.csv\"\n",
    "PATH_DATASET = \"Dataset/Rick/RickAndMortyScripts.csv\"\n",
    "\n",
    "# Instance Model & Tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(cfg.output_dir)\n",
    "\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "df = pd.read_csv(PATH_DATASET)\n",
    "\n",
    "# Prepare datset setting\n",
    "## the dataframe should be to have this columns ----> [name, line]\n",
    "## Define name of character to emulate hsi personality -----> characheter_principal_name \n",
    "\n",
    "print(len(df))\n",
    "df.head(20)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "1905\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>season no.</th>\n",
       "      <th>episode no.</th>\n",
       "      <th>episode name</th>\n",
       "      <th>name</th>\n",
       "      <th>line</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>Morty! You gotta come on. Jus'... you gotta co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>What, Rick? What’s going on?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>I got a surprise for you, Morty.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>It's the middle of the night. What are you tal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>Come on, I got a surprise for you.  Come on, h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>Ow! Ow! You're tugging me too hard!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>We gotta go, gotta get outta here, come on. Go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>What do you think of this... flying vehicle, M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>Yeah, Rick... I-it's great. Is this the surprise?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>Morty. I had to... I had to do it. I had— I ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>What?! A bomb?!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>We're gonna drop it down there just get a whol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>T-t-that's absolutely crazy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>Come on, Morty. Just take it easy, Morty. It's...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>Jessica? From my math class?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>When I drop the bomb you know, I want you to h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>Ohh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>And Jessica's gonna be Eve.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Morty</td>\n",
       "      <td>Whhhh-wha?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilot</td>\n",
       "      <td>Rick</td>\n",
       "      <td>And so that's the surprise, Morty.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index  season no.  episode no. episode name   name  \\\n",
       "0       0           1            1        Pilot   Rick   \n",
       "1       1           1            1        Pilot  Morty   \n",
       "2       2           1            1        Pilot   Rick   \n",
       "3       3           1            1        Pilot  Morty   \n",
       "4       4           1            1        Pilot   Rick   \n",
       "5       5           1            1        Pilot  Morty   \n",
       "6       6           1            1        Pilot   Rick   \n",
       "7       7           1            1        Pilot   Rick   \n",
       "8       8           1            1        Pilot  Morty   \n",
       "9       9           1            1        Pilot   Rick   \n",
       "10     10           1            1        Pilot  Morty   \n",
       "11     11           1            1        Pilot   Rick   \n",
       "12     12           1            1        Pilot  Morty   \n",
       "13     13           1            1        Pilot   Rick   \n",
       "14     14           1            1        Pilot  Morty   \n",
       "15     15           1            1        Pilot   Rick   \n",
       "16     16           1            1        Pilot  Morty   \n",
       "17     17           1            1        Pilot   Rick   \n",
       "18     18           1            1        Pilot  Morty   \n",
       "19     19           1            1        Pilot   Rick   \n",
       "\n",
       "                                                 line  \n",
       "0   Morty! You gotta come on. Jus'... you gotta co...  \n",
       "1                        What, Rick? What’s going on?  \n",
       "2                    I got a surprise for you, Morty.  \n",
       "3   It's the middle of the night. What are you tal...  \n",
       "4   Come on, I got a surprise for you.  Come on, h...  \n",
       "5                 Ow! Ow! You're tugging me too hard!  \n",
       "6   We gotta go, gotta get outta here, come on. Go...  \n",
       "7   What do you think of this... flying vehicle, M...  \n",
       "8   Yeah, Rick... I-it's great. Is this the surprise?  \n",
       "9   Morty. I had to... I had to do it. I had— I ha...  \n",
       "10                                    What?! A bomb?!  \n",
       "11  We're gonna drop it down there just get a whol...  \n",
       "12                       T-t-that's absolutely crazy!  \n",
       "13  Come on, Morty. Just take it easy, Morty. It's...  \n",
       "14                       Jessica? From my math class?  \n",
       "15  When I drop the bomb you know, I want you to h...  \n",
       "16                                             Ohh...  \n",
       "17                        And Jessica's gonna be Eve.  \n",
       "18                                         Whhhh-wha?  \n",
       "19                 And so that's the surprise, Morty.  "
      ]
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Format data\n",
    "## add special tokens"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "source": [
    "MIN_SENTENCES = 7\n",
    "CHARACTER_PRINCIPAL_NAME = 'Rick'\n",
    "\n",
    "# identify all id sentences of the principañ character\n",
    "id_principal_character_speak = df[df.name==CHARACTER_PRINCIPAL_NAME].index.tolist()\n",
    "Samples = []\n",
    "\n",
    "#ix = id_principal_character_speak[5]\n",
    "for ix in id_principal_character_speak:\n",
    "    if ix-MIN_SENTENCES<0:\n",
    "        #ix_ant=None\n",
    "        continue\n",
    "    else:\n",
    "        ix_ant = ix-MIN_SENTENCES\n",
    "\n",
    "\n",
    "    Line=[]\n",
    "    for i in range(ix_ant,ix+1):\n",
    "        line = df.line[i]\n",
    "        name = df.name[i]\n",
    "\n",
    "        # identify if the character that is specaking is the bot or a user\n",
    "        name = cfg.bot_token if name==CHARACTER_PRINCIPAL_NAME else cfg.user_token\n",
    "        line = name + ' ' + line\n",
    "\n",
    "        # concatenate sentences\n",
    "        Line.append(line)\n",
    "    \n",
    "        #sample = cfg.bos_token +' ' + ' '.join(Line) + ' ' + cfg.eos_token\n",
    "    Samples.append(Line)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "source": [
    "columns = [f\"context_{i}\" for i in range(MIN_SENTENCES)] + ['response']\n",
    "df_prep = pd.DataFrame(Samples,columns=[columns])\n",
    "df_prep.head()"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context_0</th>\n",
       "      <th>context_1</th>\n",
       "      <th>context_2</th>\n",
       "      <th>context_3</th>\n",
       "      <th>context_4</th>\n",
       "      <th>context_5</th>\n",
       "      <th>context_6</th>\n",
       "      <th>response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;|bot|&gt; Morty! You gotta come on. Jus'... you ...</td>\n",
       "      <td>&lt;|user|&gt; What, Rick? What’s going on?</td>\n",
       "      <td>&lt;|bot|&gt; I got a surprise for you, Morty.</td>\n",
       "      <td>&lt;|user|&gt; It's the middle of the night. What ar...</td>\n",
       "      <td>&lt;|bot|&gt; Come on, I got a surprise for you.  Co...</td>\n",
       "      <td>&lt;|user|&gt; Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>&lt;|bot|&gt; We gotta go, gotta get outta here, com...</td>\n",
       "      <td>&lt;|bot|&gt; What do you think of this... flying ve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;|bot|&gt; I got a surprise for you, Morty.</td>\n",
       "      <td>&lt;|user|&gt; It's the middle of the night. What ar...</td>\n",
       "      <td>&lt;|bot|&gt; Come on, I got a surprise for you.  Co...</td>\n",
       "      <td>&lt;|user|&gt; Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>&lt;|bot|&gt; We gotta go, gotta get outta here, com...</td>\n",
       "      <td>&lt;|bot|&gt; What do you think of this... flying ve...</td>\n",
       "      <td>&lt;|user|&gt; Yeah, Rick... I-it's great. Is this t...</td>\n",
       "      <td>&lt;|bot|&gt; Morty. I had to... I had to do it. I h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;|bot|&gt; Come on, I got a surprise for you.  Co...</td>\n",
       "      <td>&lt;|user|&gt; Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>&lt;|bot|&gt; We gotta go, gotta get outta here, com...</td>\n",
       "      <td>&lt;|bot|&gt; What do you think of this... flying ve...</td>\n",
       "      <td>&lt;|user|&gt; Yeah, Rick... I-it's great. Is this t...</td>\n",
       "      <td>&lt;|bot|&gt; Morty. I had to... I had to do it. I h...</td>\n",
       "      <td>&lt;|user|&gt; What?! A bomb?!</td>\n",
       "      <td>&lt;|bot|&gt; We're gonna drop it down there just ge...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;|bot|&gt; We gotta go, gotta get outta here, com...</td>\n",
       "      <td>&lt;|bot|&gt; What do you think of this... flying ve...</td>\n",
       "      <td>&lt;|user|&gt; Yeah, Rick... I-it's great. Is this t...</td>\n",
       "      <td>&lt;|bot|&gt; Morty. I had to... I had to do it. I h...</td>\n",
       "      <td>&lt;|user|&gt; What?! A bomb?!</td>\n",
       "      <td>&lt;|bot|&gt; We're gonna drop it down there just ge...</td>\n",
       "      <td>&lt;|user|&gt; T-t-that's absolutely crazy!</td>\n",
       "      <td>&lt;|bot|&gt; Come on, Morty. Just take it easy, Mor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;|user|&gt; Yeah, Rick... I-it's great. Is this t...</td>\n",
       "      <td>&lt;|bot|&gt; Morty. I had to... I had to do it. I h...</td>\n",
       "      <td>&lt;|user|&gt; What?! A bomb?!</td>\n",
       "      <td>&lt;|bot|&gt; We're gonna drop it down there just ge...</td>\n",
       "      <td>&lt;|user|&gt; T-t-that's absolutely crazy!</td>\n",
       "      <td>&lt;|bot|&gt; Come on, Morty. Just take it easy, Mor...</td>\n",
       "      <td>&lt;|user|&gt; Jessica? From my math class?</td>\n",
       "      <td>&lt;|bot|&gt; When I drop the bomb you know, I want ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           context_0  \\\n",
       "0  <|bot|> Morty! You gotta come on. Jus'... you ...   \n",
       "1           <|bot|> I got a surprise for you, Morty.   \n",
       "2  <|bot|> Come on, I got a surprise for you.  Co...   \n",
       "3  <|bot|> We gotta go, gotta get outta here, com...   \n",
       "4  <|user|> Yeah, Rick... I-it's great. Is this t...   \n",
       "\n",
       "                                           context_1  \\\n",
       "0              <|user|> What, Rick? What’s going on?   \n",
       "1  <|user|> It's the middle of the night. What ar...   \n",
       "2       <|user|> Ow! Ow! You're tugging me too hard!   \n",
       "3  <|bot|> What do you think of this... flying ve...   \n",
       "4  <|bot|> Morty. I had to... I had to do it. I h...   \n",
       "\n",
       "                                           context_2  \\\n",
       "0           <|bot|> I got a surprise for you, Morty.   \n",
       "1  <|bot|> Come on, I got a surprise for you.  Co...   \n",
       "2  <|bot|> We gotta go, gotta get outta here, com...   \n",
       "3  <|user|> Yeah, Rick... I-it's great. Is this t...   \n",
       "4                           <|user|> What?! A bomb?!   \n",
       "\n",
       "                                           context_3  \\\n",
       "0  <|user|> It's the middle of the night. What ar...   \n",
       "1       <|user|> Ow! Ow! You're tugging me too hard!   \n",
       "2  <|bot|> What do you think of this... flying ve...   \n",
       "3  <|bot|> Morty. I had to... I had to do it. I h...   \n",
       "4  <|bot|> We're gonna drop it down there just ge...   \n",
       "\n",
       "                                           context_4  \\\n",
       "0  <|bot|> Come on, I got a surprise for you.  Co...   \n",
       "1  <|bot|> We gotta go, gotta get outta here, com...   \n",
       "2  <|user|> Yeah, Rick... I-it's great. Is this t...   \n",
       "3                           <|user|> What?! A bomb?!   \n",
       "4              <|user|> T-t-that's absolutely crazy!   \n",
       "\n",
       "                                           context_5  \\\n",
       "0       <|user|> Ow! Ow! You're tugging me too hard!   \n",
       "1  <|bot|> What do you think of this... flying ve...   \n",
       "2  <|bot|> Morty. I had to... I had to do it. I h...   \n",
       "3  <|bot|> We're gonna drop it down there just ge...   \n",
       "4  <|bot|> Come on, Morty. Just take it easy, Mor...   \n",
       "\n",
       "                                           context_6  \\\n",
       "0  <|bot|> We gotta go, gotta get outta here, com...   \n",
       "1  <|user|> Yeah, Rick... I-it's great. Is this t...   \n",
       "2                           <|user|> What?! A bomb?!   \n",
       "3              <|user|> T-t-that's absolutely crazy!   \n",
       "4              <|user|> Jessica? From my math class?   \n",
       "\n",
       "                                            response  \n",
       "0  <|bot|> What do you think of this... flying ve...  \n",
       "1  <|bot|> Morty. I had to... I had to do it. I h...  \n",
       "2  <|bot|> We're gonna drop it down there just ge...  \n",
       "3  <|bot|> Come on, Morty. Just take it easy, Mor...  \n",
       "4  <|bot|> When I drop the bomb you know, I want ...  "
      ]
     },
     "metadata": {},
     "execution_count": 22
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "source": [
    "# save\n",
    "df_prep.to_csv(\"Rick_dataset_preproced.csv\", index=False)"
   ],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.5",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit ('base': conda)"
  },
  "interpreter": {
   "hash": "4a64d8e05436eb44142eb50eb52ba7fe11bf3b9fdb34fcbbedc3b49239359503"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}