ROCStories_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess \"ROC Stories\" for Story Completion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "DATAPATH = '/path/to/ROCStories'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ROCstory_spring2016 = pd.read_csv(os.path.join(DATAPATH, \"ROCStories__spring2016 - ROCStories_spring2016.csv\"))\n",
    "ROCstory_winter2017 = pd.read_csv(os.path.join(DATAPATH, \"ROCStories_winter2017 - ROCStories_winter2017.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ROCstory_train = pd.concat([ROCstory_spring2016, ROCstory_winter2017])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98161"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ROCstory_train[\"storyid\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stories = ROCstory_train.loc[:, \"sentence1\":\"sentence5\"].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train, Dev, Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_and_dev, test_stories = train_test_split(stories, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stories, dev_stories = train_test_split(train_and_dev, test_size=1/9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(78528, 9816, 9817)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_stories), len(dev_stories), len(test_stories)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_missing_indexes = np.random.randint(low=0, high=5, size=len(dev_stories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_stories_with_missing = []\n",
    "\n",
    "for st, mi in zip(dev_stories, dev_missing_indexes):\n",
    "    missing_sentence = st[mi]\n",
    "    remain_sentences = np.delete(st, mi)\n",
    "    \n",
    "    dev_stories_with_missing.append([remain_sentences[0], \n",
    "                                 remain_sentences[1],\n",
    "                                 remain_sentences[2],\n",
    "                                 remain_sentences[3],                                 \n",
    "                                 mi, missing_sentence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df = pd.DataFrame(dev_stories_with_missing,\n",
    "                  columns=['stories_with_missing_sentence1',\n",
    "                           'stories_with_missing_sentence2',\n",
    "                           'stories_with_missing_sentence3',\n",
    "                           'stories_with_missing_sentence4',\n",
    "                           'missing_id', 'missing_sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df.to_csv(\"./data/rocstories_completion_dev.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_missing_indexes = np.random.randint(low=0, high=5, size=len(test_stories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_stories_with_missing = []\n",
    "\n",
    "for st, mi in zip(test_stories, test_missing_indexes):\n",
    "    missing_sentence = st[mi]\n",
    "    remain_sentences = np.delete(st, mi)\n",
    "    \n",
    "    test_stories_with_missing.append([remain_sentences[0], \n",
    "                                 remain_sentences[1],\n",
    "                                 remain_sentences[2],\n",
    "                                 remain_sentences[3],                                 \n",
    "                                 mi, missing_sentence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.DataFrame(test_stories_with_missing,\n",
    "                  columns=['stories_with_missing_sentence1',\n",
    "                           'stories_with_missing_sentence2',\n",
    "                           'stories_with_missing_sentence3',\n",
    "                           'stories_with_missing_sentence4',\n",
    "                           'missing_id', 'missing_sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv(\"./data/rocstories_completion_test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.DataFrame(train_stories,\n",
    "                       columns=['sentence1',\n",
    "                                'sentence2',\n",
    "                                'sentence3',\n",
    "                                'sentence4',\n",
    "                                'sentence5'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv(\"./data/rocstories_completion_train.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load saved data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2 = pd.read_csv(\"./data/rocstories_completion_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df2 = pd.read_csv(\"./data/rocstories_completion_dev.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df2 = pd.read_csv(\"./data/rocstories_completion_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    1997\n",
       "4    1983\n",
       "2    1963\n",
       "0    1945\n",
       "1    1928\n",
       "Name: missing_id, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df2.missing_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2020\n",
       "1    1988\n",
       "3    1953\n",
       "2    1943\n",
       "4    1913\n",
       "Name: missing_id, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2.missing_id.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### mini size dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mini, train_else = train_test_split(train_df, test_size=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7852"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_mini)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mini.to_csv(\"./data/rocstories_completion_train_mini.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_mini, dev_else = train_test_split(dev_df, test_size=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "981"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dev_mini)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_mini.to_csv(\"./data/rocstories_completion_dev_mini.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "base"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}