Handling of data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook follows the plan:\n",
    "- Import the modules\n",
    "- Import the \"basic\" data (movies and characters datasets from CMU), clean it and save it\n",
    "- Extraction of the lemmatized version of the plot summaries from the corenlp processed data\n",
    "- Processing of the summaries according to the gender\n",
    "- Loading, cleaning of IMDb dataset\n",
    "- Matching CMU and IMDb datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import the modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTimeoutError\u001b[0m                              Traceback (most recent call last)",
      "File \u001b[1;32m~\\anaconda3\\lib\\socket.py:796\u001b[0m, in \u001b[0;36mcreate_connection\u001b[1;34m(address, timeout, source_address)\u001b[0m\n\u001b[0;32m    795\u001b[0m     sock\u001b[38;5;241m.\u001b[39mbind(source_address)\n\u001b[1;32m--> 796\u001b[0m \u001b[43msock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    797\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
      "\u001b[1;31mTimeoutError\u001b[0m: [WinError 10060] Une tentative de connexion a échoué car le parti connecté n’a pas répondu convenablement au-delà d’une certaine durée ou une connexion établie a échoué car l’hôte de connexion n’a pas répondu",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn [64], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Download useful packages for nltk\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mnltk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mpunkt\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m      3\u001b[0m nltk\u001b[38;5;241m.\u001b[39mdownload(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstopwords\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      4\u001b[0m nltk\u001b[38;5;241m.\u001b[39mdownload(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwordnet\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\downloader.py:777\u001b[0m, in \u001b[0;36mDownloader.download\u001b[1;34m(self, info_or_id, download_dir, quiet, force, prefix, halt_on_error, raise_on_error, print_error_to)\u001b[0m\n\u001b[0;32m    768\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mshow\u001b[39m(s, prefix2\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m    769\u001b[0m     print_to(\n\u001b[0;32m    770\u001b[0m         textwrap\u001b[38;5;241m.\u001b[39mfill(\n\u001b[0;32m    771\u001b[0m             s,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    774\u001b[0m         )\n\u001b[0;32m    775\u001b[0m     )\n\u001b[1;32m--> 777\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m msg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mincr_download(info_or_id, download_dir, force):\n\u001b[0;32m    778\u001b[0m     \u001b[38;5;66;03m# Error messages\u001b[39;00m\n\u001b[0;32m    779\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(msg, ErrorMessage):\n\u001b[0;32m    780\u001b[0m         show(msg\u001b[38;5;241m.\u001b[39mmessage)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\downloader.py:629\u001b[0m, in \u001b[0;36mDownloader.incr_download\u001b[1;34m(self, info_or_id, download_dir, force)\u001b[0m\n\u001b[0;32m    627\u001b[0m \u001b[38;5;66;03m# Look up the requested collection or package.\u001b[39;00m\n\u001b[0;32m    628\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 629\u001b[0m     info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_info_or_id\u001b[49m\u001b[43m(\u001b[49m\u001b[43minfo_or_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    630\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mOSError\u001b[39;00m, \u001b[38;5;167;01mValueError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    631\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m ErrorMessage(\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError loading \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo_or_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\downloader.py:603\u001b[0m, in \u001b[0;36mDownloader._info_or_id\u001b[1;34m(self, info_or_id)\u001b[0m\n\u001b[0;32m    601\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_info_or_id\u001b[39m(\u001b[38;5;28mself\u001b[39m, info_or_id):\n\u001b[0;32m    602\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(info_or_id, \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m--> 603\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minfo\u001b[49m\u001b[43m(\u001b[49m\u001b[43minfo_or_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    604\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    605\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m info_or_id\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\downloader.py:1009\u001b[0m, in \u001b[0;36mDownloader.info\u001b[1;34m(self, id)\u001b[0m\n\u001b[0;32m   1006\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minfo\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mid\u001b[39m):\n\u001b[0;32m   1007\u001b[0m     \u001b[38;5;124;03m\"\"\"Return the ``Package`` or ``Collection`` record for the\u001b[39;00m\n\u001b[0;32m   1008\u001b[0m \u001b[38;5;124;03m    given item.\"\"\"\u001b[39;00m\n\u001b[1;32m-> 1009\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_update_index\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1010\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mid\u001b[39m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_packages:\n\u001b[0;32m   1011\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_packages[\u001b[38;5;28mid\u001b[39m]\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\downloader.py:952\u001b[0m, in \u001b[0;36mDownloader._update_index\u001b[1;34m(self, url)\u001b[0m\n\u001b[0;32m    948\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_url \u001b[38;5;241m=\u001b[39m url \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_url\n\u001b[0;32m    950\u001b[0m \u001b[38;5;66;03m# Download the index file.\u001b[39;00m\n\u001b[0;32m    951\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_index \u001b[38;5;241m=\u001b[39m nltk\u001b[38;5;241m.\u001b[39minternals\u001b[38;5;241m.\u001b[39mElementWrapper(\n\u001b[1;32m--> 952\u001b[0m     ElementTree\u001b[38;5;241m.\u001b[39mparse(\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_url\u001b[49m\u001b[43m)\u001b[49m)\u001b[38;5;241m.\u001b[39mgetroot()\n\u001b[0;32m    953\u001b[0m )\n\u001b[0;32m    954\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_index_timestamp \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[0;32m    956\u001b[0m \u001b[38;5;66;03m# Build a dictionary of packages.\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\urllib\\request.py:222\u001b[0m, in \u001b[0;36murlopen\u001b[1;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[0;32m    220\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    221\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[1;32m--> 222\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\urllib\\request.py:525\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[1;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[0;32m    522\u001b[0m     req \u001b[38;5;241m=\u001b[39m meth(req)\n\u001b[0;32m    524\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[1;32m--> 525\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    527\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n\u001b[0;32m    528\u001b[0m meth_name \u001b[38;5;241m=\u001b[39m protocol\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_response\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\urllib\\request.py:542\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[1;34m(self, req, data)\u001b[0m\n\u001b[0;32m    539\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[0;32m    541\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[1;32m--> 542\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[0;32m    543\u001b[0m \u001b[43m                          \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    544\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[0;32m    545\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\urllib\\request.py:502\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[1;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[0;32m    500\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[0;32m    501\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[1;32m--> 502\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    503\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    504\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\urllib\\request.py:1393\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[1;34m(self, req)\u001b[0m\n\u001b[0;32m   1392\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[1;32m-> 1393\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1394\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\urllib\\request.py:1350\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[1;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[0;32m   1348\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1349\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 1350\u001b[0m         \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mselector\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1351\u001b[0m \u001b[43m                  \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhas_header\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mTransfer-encoding\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1352\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n\u001b[0;32m   1353\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m URLError(err)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\http\\client.py:1240\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[1;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[0;32m   1237\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\u001b[38;5;28mself\u001b[39m, method, url, body\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, headers\u001b[38;5;241m=\u001b[39m{}, \u001b[38;5;241m*\u001b[39m,\n\u001b[0;32m   1238\u001b[0m             encode_chunked\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[0;32m   1239\u001b[0m     \u001b[38;5;124;03m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[1;32m-> 1240\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\http\\client.py:1286\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[1;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[0;32m   1282\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(body, \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m   1283\u001b[0m     \u001b[38;5;66;03m# RFC 2616 Section 3.7.1 says that text default has a\u001b[39;00m\n\u001b[0;32m   1284\u001b[0m     \u001b[38;5;66;03m# default charset of iso-8859-1.\u001b[39;00m\n\u001b[0;32m   1285\u001b[0m     body \u001b[38;5;241m=\u001b[39m _encode(body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbody\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m-> 1286\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\http\\client.py:1235\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[1;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[0;32m   1233\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1234\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[1;32m-> 1235\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\http\\client.py:1006\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[1;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[0;32m   1004\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer)\n\u001b[0;32m   1005\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[1;32m-> 1006\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1008\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m   1009\u001b[0m \n\u001b[0;32m   1010\u001b[0m     \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n\u001b[0;32m   1011\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(message_body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mread\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m   1012\u001b[0m         \u001b[38;5;66;03m# Let file-like take precedence over byte-like.  This\u001b[39;00m\n\u001b[0;32m   1013\u001b[0m         \u001b[38;5;66;03m# is needed to allow the current position of mmap'ed\u001b[39;00m\n\u001b[0;32m   1014\u001b[0m         \u001b[38;5;66;03m# files to be taken into account.\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\http\\client.py:946\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m    944\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    945\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[1;32m--> 946\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    947\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    948\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m NotConnected()\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\http\\client.py:1402\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1399\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m   1400\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnect to a host on a given (SSL) port.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m-> 1402\u001b[0m     \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1404\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host:\n\u001b[0;32m   1405\u001b[0m         server_hostname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\http\\client.py:917\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    915\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    916\u001b[0m     \u001b[38;5;124;03m\"\"\"Connect to the host and port specified in __init__.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 917\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    918\u001b[0m \u001b[43m        \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhost\u001b[49m\u001b[43m,\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msource_address\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    919\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock\u001b[38;5;241m.\u001b[39msetsockopt(socket\u001b[38;5;241m.\u001b[39mIPPROTO_TCP, socket\u001b[38;5;241m.\u001b[39mTCP_NODELAY, \u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m    921\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host:\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\socket.py:796\u001b[0m, in \u001b[0;36mcreate_connection\u001b[1;34m(address, timeout, source_address)\u001b[0m\n\u001b[0;32m    794\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m source_address:\n\u001b[0;32m    795\u001b[0m     sock\u001b[38;5;241m.\u001b[39mbind(source_address)\n\u001b[1;32m--> 796\u001b[0m \u001b[43msock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    797\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n\u001b[0;32m    798\u001b[0m err \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Download useful packages for nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('omw-1.4')\n",
    "nltk.download('vader_lexicon')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FOLDER = 'Data/'\n",
    "\n",
    "# Original file names\n",
    "CHARACTER_DATASET = DATA_FOLDER + 'character.metadata.tsv'\n",
    "MOVIE_DATASET = DATA_FOLDER + 'movie.metadata.tsv'\n",
    "\n",
    "SUMMARIES_DATASET = DATA_FOLDER + 'plot_summaries.txt'\n",
    "NLP_FOLDER = DATA_FOLDER + 'corenlp_plot_summaries/'\n",
    "DEFAULT_COMPRESSION = 'gzip'\n",
    "\n",
    "# Pickled file names\n",
    "CHARACTER_DATASET = DATA_FOLDER + 'characters.pkl'\n",
    "MOVIE_DATASET = DATA_FOLDER + 'movies.pkl'\n",
    "SUMMARIES_DATASET = DATA_FOLDER + 'nlp_summaries.pkl'\n",
    "\n",
    "FROM_PICKLE = True # True if we load data from pickled data (already cleaned etc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to load data\n",
    "def load_metadata(path, column_names, header=None, low_memory=False):\n",
    "    return pd.read_table(path, header=header, names=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not FROM_PICKLE:\n",
    "    # Name columns\n",
    "    columns_character = ['Wikipedia_movie_ID', 'Freebase_movie_ID', 'Movie_release_date', 'Character_name', 'Actor_date_of_birth', 'Actor_gender', 'Actor_height_meters', 'Actor_ethnicity_Freebase_ID', 'Actor_name', 'Actor_age_at_movie_release', 'Freebase_character_actor_map_ID', 'Freebase_character_ID', 'Freebase_actor_ID']\n",
    "    columns_movie = ['Wikipedia_movie_ID', 'Freebase_movie_ID', 'Movie_name','Movie_release_date','Movie_box_office_revenue', 'Movie_runtime','Movie_languages','Movie_countries','Movie_genres' ]\n",
    "\n",
    "    # Load data with correct column names\n",
    "    characters = load_metadata(CHARACTER_DATASET,column_names=columns_character)\n",
    "    movies = load_metadata(MOVIE_DATASET,column_names=columns_movie)\n",
    "    \n",
    "    # Load summaries\n",
    "    with open(SUMMARIES_DATASET,'r', encoding='utf-8') as file:\n",
    "        summaries = file.readlines()\n",
    "    \n",
    "else:\n",
    "    characters = pd.read_pickle(CHARACTER_DATASET)\n",
    "    movies = pd.read_pickle(MOVIE_DATASET)\n",
    "    summaries = pd.read_pickle(SUMMARIES_DATASET) # dictionnary {id (str): summary (str)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First glimpse at the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we observe the movies dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81741\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wikipedia_movie_ID</th>\n",
       "      <th>Freebase_movie_ID</th>\n",
       "      <th>Movie_name</th>\n",
       "      <th>Movie_release_date</th>\n",
       "      <th>Movie_box_office_revenue</th>\n",
       "      <th>Movie_runtime</th>\n",
       "      <th>Movie_languages</th>\n",
       "      <th>Movie_countries</th>\n",
       "      <th>Movie_genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>975900</td>\n",
       "      <td>/m/03vyhn</td>\n",
       "      <td>Ghosts of Mars</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>14010832.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>[English Language]</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Thriller, Science Fiction, Horror, Adventure,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3196793</td>\n",
       "      <td>/m/08yl5d</td>\n",
       "      <td>Getting Away with Murder: The JonBenét Ramsey ...</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>95.0</td>\n",
       "      <td>[English Language]</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Mystery, Biographical film, Drama, Crime Drama]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wikipedia_movie_ID Freebase_movie_ID  \\\n",
       "0              975900         /m/03vyhn   \n",
       "1             3196793         /m/08yl5d   \n",
       "\n",
       "                                          Movie_name  Movie_release_date  \\\n",
       "0                                     Ghosts of Mars              2001.0   \n",
       "1  Getting Away with Murder: The JonBenét Ramsey ...              2000.0   \n",
       "\n",
       "   Movie_box_office_revenue  Movie_runtime     Movie_languages  \\\n",
       "0                14010832.0           98.0  [English Language]   \n",
       "1                       NaN           95.0  [English Language]   \n",
       "\n",
       "              Movie_countries  \\\n",
       "0  [United States of America]   \n",
       "1  [United States of America]   \n",
       "\n",
       "                                        Movie_genres  \n",
       "0  [Thriller, Science Fiction, Horror, Adventure,...  \n",
       "1   [Mystery, Biographical film, Drama, Crime Drama]  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(movies))\n",
    "movies.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we observe the characters dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "450646\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wikipedia_movie_ID</th>\n",
       "      <th>Freebase_movie_ID</th>\n",
       "      <th>Movie_release_date</th>\n",
       "      <th>Character_name</th>\n",
       "      <th>Actor_date_of_birth</th>\n",
       "      <th>Actor_gender</th>\n",
       "      <th>Actor_height_meters</th>\n",
       "      <th>Actor_ethnicity_Freebase_ID</th>\n",
       "      <th>Actor_name</th>\n",
       "      <th>Actor_age_at_movie_release</th>\n",
       "      <th>Freebase_character_actor_map_ID</th>\n",
       "      <th>Freebase_character_ID</th>\n",
       "      <th>Freebase_actor_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>975900</td>\n",
       "      <td>/m/03vyhn</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>Akooshay</td>\n",
       "      <td>1958.0</td>\n",
       "      <td>F</td>\n",
       "      <td>1.62</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Wanda De Jesus</td>\n",
       "      <td>42.0</td>\n",
       "      <td>/m/0bgchxw</td>\n",
       "      <td>/m/0bgcj3x</td>\n",
       "      <td>/m/03wcfv7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>975900</td>\n",
       "      <td>/m/03vyhn</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>Lieutenant Melanie Ballard</td>\n",
       "      <td>1974.0</td>\n",
       "      <td>F</td>\n",
       "      <td>1.78</td>\n",
       "      <td>/m/044038p</td>\n",
       "      <td>Natasha Henstridge</td>\n",
       "      <td>27.0</td>\n",
       "      <td>/m/0jys3m</td>\n",
       "      <td>/m/0bgchn4</td>\n",
       "      <td>/m/0346l4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Wikipedia_movie_ID Freebase_movie_ID  Movie_release_date  \\\n",
       "0             975900         /m/03vyhn              2001.0   \n",
       "1             975900         /m/03vyhn              2001.0   \n",
       "\n",
       "               Character_name  Actor_date_of_birth Actor_gender  \\\n",
       "0                    Akooshay               1958.0            F   \n",
       "1  Lieutenant Melanie Ballard               1974.0            F   \n",
       "\n",
       "  Actor_height_meters Actor_ethnicity_Freebase_ID          Actor_name  \\\n",
       "0                1.62                         NaN      Wanda De Jesus   \n",
       "1                1.78                  /m/044038p  Natasha Henstridge   \n",
       "\n",
       "  Actor_age_at_movie_release Freebase_character_actor_map_ID  \\\n",
       "0                       42.0                      /m/0bgchxw   \n",
       "1                       27.0                       /m/0jys3m   \n",
       "\n",
       "  Freebase_character_ID Freebase_actor_ID  \n",
       "0            /m/0bgcj3x        /m/03wcfv7  \n",
       "1            /m/0bgchn4         /m/0346l4  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(characters))\n",
    "characters.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also check the summaries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of plots: 42306\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('10000053',\n",
       " \"Fur trapper Jean La B te paddle he canoe through wild water towards the settlement in order to sell a load of fur . at the settlement a steamboat be landing and the trader and he foster-child Eve , arrive at the seaport to fetch mail and consumer goods . the trader explain to Eve that the ship bring `` Jailbirds ... from the east '' and that `` they husbands-to-be have bail they out and pay they fine and they passage with a guarantee of marriage '' . later , the captain be auction off one of those woman because she husband-to-be have die in the meantime . Jean La B te decide to take he chance to buy the wife , but he make he bid too late . next day , the trader 's wife , in the need to compensate for the loss of she savings , seize the opportunity to offer she foster-child for thousand dollar to the simple-minded , rough-cut trapper . she praise the quality of the shy girl and explain , that she inability to speak be cause from the shock she suffer when she have to witness how she parent be barbarously murder several year ago . La B te finally agree to buy the mute girl and take she against she will into the breathtakingly beautiful wilderness of British Columbia . here the strange couple start a difficult relationship characterize by mistrust and Eve 's dislike . Eve vehemently reject the advance of the gruff trapper . La B te take she for hunting and acquaint she with the beauty and the danger of the wilderness , but here , as well , he fail to win she trust . the lonely trapper still spend the night alone in he bed . one day , on check he trap for catch animal , La B te be threaten by a mountain lion . he successfully shoot the cat but inadvertently get he foot into he own bear trap . badly injure , he try to drag himself back to he hut , hunt by famished wolf . meanwhile , Eve be wait at the cabin and hear the distant howling of the wolf approach the hut . equip with a gun she set out in search for La B te , and together they can get rid of the wolf pack . La B te 's foot be break , so he ask Eve to bring the medicine man from the next indian village , a two day trip away . the canadian winter have already come , so Eve put on she snowshoe , and start a long , arduous walk over snow cover hill top . she finally reach the village only to find it totally desert . return empty-handed , Eve find La B te already suffer from blood poisoning . have no time to lose , he urge the terrify girl to immediately chop off he poison leg use a axe . after La B te have stun himself by gulp the last drop of rum , Eve act as command and she patient instantly pass out from pain . Eve succeed in save the trapper 's life in the following period of nursing . in that time she have learn to hunt on she own and be now capable to provide for the couple . eventually they seem to come closer , the hobble trapper no longer appear that gruesome to Eve . but she childhood trauma be still keep she from start a closer relationship and consequently , she flee with a canoe back to the settlement . here , although be welcome , she remain a outsider . even the impending marriage to a sympathetic young man can not overcome she inner barrier . on the day of marriage , she run away again in order to finally return to Jean La B te . \")"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Number of plots:', len(summaries))\n",
    "list(summaries.items())[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Problem of dates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We fix typos and absurd dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not FROM_PICKLE:\n",
    "    movies.loc[movies.Movie_release_date == '1010-12-02','Movie_release_date'] = '2010-12-02'\n",
    "    characters.loc[characters.Movie_release_date == '1010-12-02','Movie_release_date'] = '2010-12-02'\n",
    "    characters[characters.Actor_date_of_birth == '2050'] = '1971'\n",
    "    characters = characters.drop(characters[characters.Actor_date_of_birth < '1500'].index)\n",
    "    characters = characters.drop(characters[characters.Actor_date_of_birth > '2030'].index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Format of movie languages, genres and country"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the format of languages, genres, country columns to a simpler format (in terms of utilisation)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_multiple(chain,deb,step):\n",
    "    '''Split the chain of characters at each \" encountered, and keep only the element in deb +i*step'''\n",
    "    res = chain.split('\"')[deb::step]\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not FROM_PICKLE:\n",
    "    movies.loc[:,'Movie_genres'] = movies.Movie_genres.apply(format_multiple,deb=3,step=4)\n",
    "    movies.loc[:,'Movie_countries'] = movies.Movie_countries.apply(format_multiple,deb=3,step=4)\n",
    "    movies.loc[:,'Movie_languages'] = movies.Movie_languages.apply(format_multiple,deb=3,step=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13866 movies without Movie_languages (16.96% of the dataset)\n",
      "8154 movies without Movie_countries (9.98% of the dataset)\n",
      "2294 movies without Movie_genres (2.81% of the dataset)\n"
     ]
    }
   ],
   "source": [
    "keys = ['Movie_languages','Movie_countries','Movie_genres']\n",
    "for key in keys:\n",
    "    nb = len(movies[movies[key].apply(len) == 0])\n",
    "    print('{nb} movies without {key} ({percentage:.2f}% of the dataset)'.format(nb=nb,key=key, percentage=nb*100/len(movies)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Format for dates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For our study, we only keep the years from the dates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not FROM_PICKLE:\n",
    "    movies.Movie_release_date = pd.to_datetime(movies.Movie_release_date,format='%Y-%m-%d').dt.year\n",
    "    characters.Movie_release_date = pd.to_datetime(characters.Movie_release_date,format='%Y-%m-%d').dt.year\n",
    "    characters.Actor_date_of_birth = pd.to_datetime(characters.Actor_date_of_birth,format='%Y-%m-%d',utc=True,errors='coerce').dt.year"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the new dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We pickle our data in order to reuse directly the cleaned data (and load it faster)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not FROM_PICKLE:\n",
    "    DESTINATION = './Data/'\n",
    "    EXT = '.pkl'\n",
    "    to_pickle_data = [characters,movies]\n",
    "    to_pickle_name = ['characters','movies']\n",
    "    for i in range(len(to_pickle_data)):\n",
    "        to_pickle_data[i].to_pickle(DESTINATION+to_pickle_name[i]+EXT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lemmatizing the summaries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We lemmatize data (for examples *'is'* becomes *'be'*) to be able to count words better. To do so, we used the `corenlp_plot_summaries` files, and exctracted from it the lemmatized versions of the movies summaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set to True to save the data\n",
    "LEMMATIZE_SUMMARIES = False # Takes ~7 mins to run (on i7-10875H CPU)\n",
    "\n",
    "if LEMMATIZE_SUMMARIES:\n",
    "    # Imports\n",
    "    from time import time\n",
    "    import os\n",
    "    import gzip\n",
    "    import re\n",
    "\n",
    "    # Count the number of files in the directory\n",
    "    nb_files = 0\n",
    "    for filename in os.listdir(NLP_FOLDER):\n",
    "        path = os.path.join(NLP_FOLDER, filename)\n",
    "        nb_files += 1\n",
    "    print('Number of summaries:',nb_files)\n",
    "\n",
    "    ext = '.xml.gz' # Extension name\n",
    "    dico_processed_summmaries = {} # Dictionary to store the processed summaries\n",
    "    regex = r'<lemma>.*?</lemma>' # Expression to detect in the corenlp data <lemma>(word)</lemma>\n",
    "\n",
    "    deb = time() # Start timer\n",
    "    count = 0 # Counter\n",
    "\n",
    "    # Iteration over the files\n",
    "    for filename in os.listdir(NLP_FOLDER):\n",
    "        path = os.path.join(NLP_FOLDER, filename) # Path to the file\n",
    "        id_summary = path[len(NLP_FOLDER):-len(ext)] # id of the summary = filename without extension\n",
    "        summary = '' # String to store the summary\n",
    "\n",
    "        if os.path.isfile(path): # Checking if it is a file\n",
    "            with gzip.open(path, 'rb') as f: # Opening the .gz file\n",
    "                for line in f:\n",
    "                    txt = line.decode().strip() # Extract the line as txt\n",
    "                    for elt in re.finditer(regex,txt): # Find all the elements like regex\n",
    "                        summary += re.split('[><]',elt.group(0))[2].lower() + ' ' # Adding only the lemmatized word\n",
    "        \n",
    "        # Set the summary in the dictionary and increment the counter\n",
    "        dico_processed_summmaries[id_summary] = summary\n",
    "        count += 1\n",
    "\n",
    "        # Evolution of the process\n",
    "        if count%1000 == 0:\n",
    "            print('{processed}/{tot} files processed --> {perc:.1f}% ({t:.1f} seconds since deb)'.format(processed=count,tot=nb_files,perc=count/nb_files*100,t=time()-deb))\n",
    "    \n",
    "    # Pickle the file\n",
    "    with open(DATA_FOLDER + 'nlp_summaries.pkl', 'wb') as file:\n",
    "        pickle.dump(dico_processed_summmaries, file, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Separating sentences between sexes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The aim of this part is to separate sentences between sexes to do a sentimental analysis later. To do so, we check if a feminine actor or the *'she'* pronoun is present in a sentence and add them to a new file. We do the same for a male actor and the *'he'* pronoun. Note that for example the sentence *'She hates him'* will become *'she hate he'* once lemmatized, which will be put in the feminine and maculine files\n",
    "\n",
    "This approach is not perfect, since for example in the sentences 'She likes butter. Indeed, the actress loves food.', only the first one will be added. It is not perfect, but the best solution we could think of."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe with the characters\n",
    "characters_per_film = characters.copy()\n",
    "# Put the column in their correct type and lower chars\n",
    "characters_per_film['Wikipedia_movie_ID'] = characters_per_film['Wikipedia_movie_ID'].astype(int)\n",
    "characters_per_film['Character_name'] = characters_per_film['Character_name'].astype(str).apply(lambda x: x.lower())\n",
    "# Sort the dataframe by movie ID\n",
    "characters_per_film = characters_per_film.sort_values(by=['Wikipedia_movie_ID'])\n",
    "# Drio rows where the character name or the gender is empty\n",
    "characters_per_film = characters_per_film.dropna(subset=['Character_name', 'Actor_gender'])\n",
    "# Group the dataframe by movie ID\n",
    "characters_per_film = characters_per_film.groupby('Wikipedia_movie_ID')[['Wikipedia_movie_ID', 'Character_name', 'Actor_gender']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'nlp_summaries' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn [16], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Import dataframe from lemmatized summaries\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame(\u001b[38;5;28mlist\u001b[39m(\u001b[43mnlp_summaries\u001b[49m\u001b[38;5;241m.\u001b[39mitems()), columns \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mplot_lemmatized\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[0;32m      3\u001b[0m \u001b[38;5;66;03m# Put column in their correct type\u001b[39;00m\n\u001b[0;32m      4\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mint\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'nlp_summaries' is not defined"
     ]
    }
   ],
   "source": [
    "# Import dataframe from lemmatized summaries\n",
    "df = pd.DataFrame(list(nlp_summaries.items()), columns = ['id','plot_lemmatized'])\n",
    "# Put column in their correct type\n",
    "df['id'] = df['id'].astype(int)\n",
    "# Sort the dataframe by movie ID\n",
    "df = df.sort_values(by=['id'])\n",
    "# Show the first 5 rows\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set to True to save the data\n",
    "SEPARATE_SENTENCES = False # Takes ~20 mins to run (on i7-10875H CPU)\n",
    "\n",
    "if SEPARATE_SENTENCES:\n",
    "    # Imports\n",
    "    count = 0\n",
    "    dico_male = {}\n",
    "    dico_female = {}\n",
    "    regexp = nltk.tokenize.RegexpTokenizer('\\w+')\n",
    "\n",
    "    # Loop on subgroups\n",
    "    for _, group in characters_per_film:\n",
    "        # Get the movie id\n",
    "        movie_id = group['Wikipedia_movie_ID'].iloc[0]\n",
    "        female_sentences = []\n",
    "        male_sentences = []\n",
    "\n",
    "        # Check if wikipedia movie id is in the nlp summaries\n",
    "        if movie_id in df['id'].values:\n",
    "            index = df[df['id'] == movie_id].index[0] # Take the correct index\n",
    "            plot = df['plot_lemmatized'][index] # Take the correct plot\n",
    "            sentences = plot.split('.') # Split into sentences\n",
    "            # Loop on sentences\n",
    "            for sentence in sentences:\n",
    "                tokens = regexp.tokenize(sentence)\n",
    "                # Loop on characters\n",
    "                for character in group['Character_name']:\n",
    "                    # Find the sex of the character\n",
    "                    gender = group[group['Character_name'] == character].Actor_gender.values[0]\n",
    "                    # Find potential pronouns discriminative on gender\n",
    "                    he_index = any('he' in sublist for sublist in tokens)\n",
    "                    she_index = any('she' in sublist for sublist in tokens)\n",
    "                    # Check if the pronoun or actor name is in the sentence\n",
    "                    if ((character in sentence) or she_index or he_index):\n",
    "                        # Store in dictionary depending on gender of sentence (can also be in both)\n",
    "                        if ((gender == '1') or she_index):\n",
    "                            female_sentences.append(sentence)\n",
    "                        if ((gender == '0') or he_index):\n",
    "                            male_sentences.append(sentence)\n",
    "\n",
    "        # Store in dictionary and increment counter\n",
    "        dico_male[movie_id] = male_sentences\n",
    "        dico_female[movie_id] = female_sentences\n",
    "        count += 1\n",
    "\n",
    "        # Evolution of the process\n",
    "        if count%1000 == 0:\n",
    "            print('{processed} files processed'.format(processed=count))\n",
    "\n",
    "    # Pickle the file\n",
    "    with open(DATA_FOLDER + 'male_sentences.pkl', 'wb') as file:\n",
    "        pickle.dump(dico_male, file, protocol=pickle.HIGHEST_PROTOCOL)    \n",
    "    with open(DATA_FOLDER + 'female_sentences.pkl', 'wb') as file:\n",
    "        pickle.dump(dico_female, file, protocol=pickle.HIGHEST_PROTOCOL)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyse sentiments for each group"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We run it in the handling of data since it takes a long time to calculate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import male sentences\n",
    "male_sentences_dict = pd.read_pickle(DATA_FOLDER + 'male_sentences.pkl')\n",
    "# Form a dataframe\n",
    "male_sentences = pd.DataFrame(list(male_sentences_dict.items()), columns = ['id','sentences'])\n",
    "# Create a new column that reconstructs the summary from the lemmatized sentences\n",
    "male_sentences['summary'] = male_sentences['sentences'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "# Import female sentences\n",
    "female_sentences_dict = pd.read_pickle(DATA_FOLDER + 'female_sentences.pkl')\n",
    "# Form a dataframe\n",
    "female_sentences = pd.DataFrame(list(female_sentences_dict.items()), columns = ['id','sentences'])\n",
    "# Create a new column that reconstructs the summary from the lemmatized sentences\n",
    "female_sentences['summary'] = female_sentences['sentences'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "# Show the first 5 rows of male sentences\n",
    "male_sentences.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_SENTIMENTS = False # Takes ~41 mins to run (on i7-10875H CPU)\n",
    "\n",
    "if SAVE_SENTIMENTS:\n",
    "    # Use nltk Vader to get the sentiment of the sentences\n",
    "    analyzer =  nltk.sentiment.SentimentIntensityAnalyzer()\n",
    "\n",
    "    # Apply sentiments to plots\n",
    "    male_sentences['polarity'] = male_sentences['summary'].apply(lambda x: analyzer.polarity_scores(x))\n",
    "    female_sentences['polarity'] = female_sentences['summary'].apply(lambda x: analyzer.polarity_scores(x))\n",
    "\n",
    "    # Pickle the file\n",
    "    with open(DATA_FOLDER + 'male_sentiments.pkl', 'wb') as file:\n",
    "        pickle.dump(male_sentences, file, protocol=pickle.HIGHEST_PROTOCOL)    \n",
    "    with open(DATA_FOLDER + 'female_sentiments.pkl', 'wb') as file:\n",
    "        pickle.dump(female_sentences, file, protocol=pickle.HIGHEST_PROTOCOL)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Enriching the CMU dataset with IMDb dataset and movie-stats\n",
    "\n",
    "## Loading the data and first glimpse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load the most useful datasets from IMDb\n",
    "TITLE_BASICS_DATASET = DATA_FOLDER + 'title.basics.tsv.gz'\n",
    "TITLE_RATINGS_DATASET = DATA_FOLDER + 'title.ratings.tsv.gz'\n",
    "\n",
    "#Load movie-stats, a dataset generated from IMDb movies\n",
    "MOVIE_STATS = DATA_FOLDER + 'movie-stats.csv'\n",
    "\n",
    "# Merged datasets\n",
    "MERGED_CMU_STATS = DATA_FOLDER + 'merge_CMU_stats.pkl'\n",
    "MERGED_CMU_IMDB_STATS = DATA_FOLDER + 'merge_all.pkl'\n",
    "\n",
    "columns_title_basics = ['tconst', 'titleType', 'primaryTitle', 'originalTitle', 'isAdult', 'startYear', 'endYear', 'runtimeMinutes', 'genres']\n",
    "columns_ratings = ['tconstIdentifier', 'averageRating', 'numVotes']\n",
    "\n",
    "MATCHING_TABLE = DATA_FOLDER + 'matching_table.pkl'\n",
    "\n",
    "CLEAN_DATA = False # True to clean again the data, False to use the already pickled data\n",
    "MATCH_DATA = False # True to match on film names, False to use the matching table already computed\n",
    "MERGE_AGAIN = False # True to match and merge the movie_stats data to previous data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Load title_basics\n",
    "    title_basics = load_metadata(TITLE_BASICS_DATASET, column_names=columns_title_basics)\n",
    "    print(\"length of title_basics: \", len(title_basics))\n",
    "    title_basics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Load title_ratings\n",
    "    ratings = load_metadata(TITLE_RATINGS_DATASET, column_names=columns_ratings)\n",
    "    print(\"length of ratings: \", len(ratings))\n",
    "    ratings.head(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Load movie-stats\n",
    "    movie_stats = pd.read_csv(MOVIE_STATS, header = 8)\n",
    "    print(\"length of movie_stats: \", len(movie_stats))\n",
    "    movie_stats.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning the datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Create a new table with only titleType=movies (get rid of videos, tvshows, tvepisodes and short)\n",
    "    title_basics_movies = title_basics[title_basics[\"titleType\"] == \"movie\"]\n",
    "    #Remove the endYear column since movies are not concerned by thats\n",
    "    title_basics_movies_cleaned = title_basics_movies.drop(columns='endYear')\n",
    "    title_basics_movies_cleaned.replace('\\\\N',np.NaN,inplace=True) # replace \\\\N by NaN\n",
    "    # datetime format for dates\n",
    "    title_basics_movies_cleaned.startYear = pd.to_datetime(title_basics_movies_cleaned.startYear,format='%Y').dt.year \n",
    "    title_basics_movies_cleaned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Drop the first row which represents the titles of the columns\n",
    "    #Careful: execute only once, otherwise it will delete the first row each time!\n",
    "    ratings_cleaned = ratings.drop(index=ratings.index[0], axis=0) \n",
    "    print(\"length of ratings_cleaned: \", len(ratings_cleaned))\n",
    "    ratings_cleaned.replace('\\\\N',np.NaN,inplace=True) # replace \\\\N by NaN\n",
    "    #Check if there are NaN values in the dataset\n",
    "    print('Number of NaN in the ratings dataset: \\n',ratings_cleaned.isnull().sum())\n",
    "    ratings_cleaned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Check if there are NaN values in the dataset\n",
    "    print('Number of NaN in the movie-stats dataset: \\n', movie_stats.isnull().sum())\n",
    "    #Remove useless columns\n",
    "    movie_stats_cleaned = movie_stats.drop(columns=['rating', 'released'])\n",
    "    #Remove rows where budget is NaN because we use movie-stats dataset to get information on budget\n",
    "    movie_stats_cleaned.dropna(subset=['budget'], inplace=True)\n",
    "    # Convert ratings and number of votes to float/int\n",
    "    ratings_cleaned.averageRating = ratings_cleaned.averageRating.apply(float)\n",
    "    ratings_cleaned.numVotes = ratings_cleaned.numVotes.apply(int)\n",
    "    movie_stats_cleaned.head()\n",
    "    print(\"length of movie_stats_cleaned: \", len(movie_stats_cleaned))\n",
    "    print('Number of NaN in the cleaned movie-stats dataset: \\n', movie_stats_cleaned.isnull().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the cleaned dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "DESTINATION = './Data/'\n",
    "EXT = '.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Pickle the data\n",
    "    to_pickle_data = title_basics_movies_cleaned\n",
    "    to_pickle_name = 'IMDb_title_movies'\n",
    "    to_pickle_data.to_pickle(DESTINATION+to_pickle_name+EXT)\n",
    "\n",
    "if not CLEAN_DATA: # for testing part\n",
    "    # load already pickled data\n",
    "    title_basics_movies_cleaned = pd.read_pickle(\"./Data/IMDb_title_movies.pkl\")\n",
    "    title_basics_movies_cleaned.startYear = pd.to_datetime(title_basics_movies_cleaned.startYear,format='%Y').dt.year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Pickle the data\n",
    "    to_pickle_data = ratings_cleaned\n",
    "    to_pickle_name = 'IMDb_ratings'\n",
    "    to_pickle_data.to_pickle(DESTINATION+to_pickle_name+EXT)\n",
    "\n",
    "if not CLEAN_DATA: # for testing part\n",
    "    # load already pickled data\n",
    "    ratings_cleaned = pd.read_pickle(\"./Data/IMDb_ratings.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CLEAN_DATA:\n",
    "    #Pickle the data\n",
    "    to_pickle_data = movie_stats_cleaned\n",
    "    to_pickle_name = 'movie-stats_budget'\n",
    "    to_pickle_data.to_pickle(DESTINATION+to_pickle_name+EXT)\n",
    "\n",
    "if not CLEAN_DATA: # for testing part\n",
    "    # load already pickled data\n",
    "    movie_stats_cleaned = pd.read_pickle(\"./Data/movie-stats_budget.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>genre</th>\n",
       "      <th>year</th>\n",
       "      <th>score</th>\n",
       "      <th>votes</th>\n",
       "      <th>director</th>\n",
       "      <th>writer</th>\n",
       "      <th>star</th>\n",
       "      <th>country</th>\n",
       "      <th>budget</th>\n",
       "      <th>gross</th>\n",
       "      <th>company</th>\n",
       "      <th>runtime\\</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Shining</td>\n",
       "      <td>Drama</td>\n",
       "      <td>1980</td>\n",
       "      <td>8.4</td>\n",
       "      <td>927000.0</td>\n",
       "      <td>Stanley Kubrick</td>\n",
       "      <td>Stephen King</td>\n",
       "      <td>Jack Nicholson</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>19000000.0</td>\n",
       "      <td>46998772.0</td>\n",
       "      <td>Warner Bros.</td>\n",
       "      <td>146.0\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Blue Lagoon</td>\n",
       "      <td>Adventure</td>\n",
       "      <td>1980</td>\n",
       "      <td>5.8</td>\n",
       "      <td>65000.0</td>\n",
       "      <td>Randal Kleiser</td>\n",
       "      <td>Henry De Vere Stacpoole</td>\n",
       "      <td>Brooke Shields</td>\n",
       "      <td>United States</td>\n",
       "      <td>4500000.0</td>\n",
       "      <td>58853106.0</td>\n",
       "      <td>Columbia Pictures</td>\n",
       "      <td>104.0\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Star Wars: Episode V - The Empire Strikes Back</td>\n",
       "      <td>Action</td>\n",
       "      <td>1980</td>\n",
       "      <td>8.7</td>\n",
       "      <td>1200000.0</td>\n",
       "      <td>Irvin Kershner</td>\n",
       "      <td>Leigh Brackett</td>\n",
       "      <td>Mark Hamill</td>\n",
       "      <td>United States</td>\n",
       "      <td>18000000.0</td>\n",
       "      <td>538375067.0</td>\n",
       "      <td>Lucasfilm</td>\n",
       "      <td>124.0\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Airplane!</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>1980</td>\n",
       "      <td>7.7</td>\n",
       "      <td>221000.0</td>\n",
       "      <td>Jim Abrahams</td>\n",
       "      <td>Jim Abrahams</td>\n",
       "      <td>Robert Hays</td>\n",
       "      <td>United States</td>\n",
       "      <td>3500000.0</td>\n",
       "      <td>83453539.0</td>\n",
       "      <td>Paramount Pictures</td>\n",
       "      <td>88.0\\</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Caddyshack</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>1980</td>\n",
       "      <td>7.3</td>\n",
       "      <td>108000.0</td>\n",
       "      <td>Harold Ramis</td>\n",
       "      <td>Brian Doyle-Murray</td>\n",
       "      <td>Chevy Chase</td>\n",
       "      <td>United States</td>\n",
       "      <td>6000000.0</td>\n",
       "      <td>39846344.0</td>\n",
       "      <td>Orion Pictures</td>\n",
       "      <td>98.0\\</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             name      genre  year  score  \\\n",
       "0                                     The Shining      Drama  1980    8.4   \n",
       "1                                 The Blue Lagoon  Adventure  1980    5.8   \n",
       "2  Star Wars: Episode V - The Empire Strikes Back     Action  1980    8.7   \n",
       "3                                       Airplane!     Comedy  1980    7.7   \n",
       "4                                      Caddyshack     Comedy  1980    7.3   \n",
       "\n",
       "       votes         director                   writer            star  \\\n",
       "0   927000.0  Stanley Kubrick             Stephen King  Jack Nicholson   \n",
       "1    65000.0   Randal Kleiser  Henry De Vere Stacpoole  Brooke Shields   \n",
       "2  1200000.0   Irvin Kershner           Leigh Brackett     Mark Hamill   \n",
       "3   221000.0     Jim Abrahams             Jim Abrahams     Robert Hays   \n",
       "4   108000.0     Harold Ramis       Brian Doyle-Murray     Chevy Chase   \n",
       "\n",
       "          country      budget        gross             company runtime\\  \n",
       "0  United Kingdom  19000000.0   46998772.0        Warner Bros.   146.0\\  \n",
       "1   United States   4500000.0   58853106.0   Columbia Pictures   104.0\\  \n",
       "2   United States  18000000.0  538375067.0           Lucasfilm   124.0\\  \n",
       "3   United States   3500000.0   83453539.0  Paramount Pictures    88.0\\  \n",
       "4   United States   6000000.0   39846344.0      Orion Pictures    98.0\\  "
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_stats_cleaned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tconstIdentifier</th>\n",
       "      <th>averageRating</th>\n",
       "      <th>numVotes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tt0000001</td>\n",
       "      <td>5.7</td>\n",
       "      <td>1922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tt0000002</td>\n",
       "      <td>5.8</td>\n",
       "      <td>259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tt0000003</td>\n",
       "      <td>6.5</td>\n",
       "      <td>1734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tt0000004</td>\n",
       "      <td>5.6</td>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>tt0000005</td>\n",
       "      <td>6.2</td>\n",
       "      <td>2545</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  tconstIdentifier  averageRating  numVotes\n",
       "1        tt0000001            5.7      1922\n",
       "2        tt0000002            5.8       259\n",
       "3        tt0000003            6.5      1734\n",
       "4        tt0000004            5.6       174\n",
       "5        tt0000005            6.2      2545"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_cleaned.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Matching CMU and movie_stats films\n",
    "\n",
    "As movie_stats has few films, it will be computationnaly less demanding to do this matching first, and then match the resulting dataset with the IMDb one.\n",
    "\n",
    "Moreover, as their is a huge number of NaN in the `Movie_box_office_revenue` of CMU dataset, we will not delete these rows at the begining, hoping that movie_stats' column `gross` will complete the missing values.\n",
    "\n",
    "We do the same for the column concerning the release year.\n",
    "\n",
    "It allowed us to save $\\approx 600$ films."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We match the movies from one dataset to the films on the other dataset on the movie name, as the ids are different.\n",
    "\n",
    "In order to avoid mismatched pairs due to a little variation in the titles, we matched films of the same year, with almost identical titles (via Jaccard distance). We create a dictionnary that matches the index of matched films."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of NaN for Movie_box_office_revenue in CMU: 73340\n"
     ]
    }
   ],
   "source": [
    "print('Number of NaN for Movie_box_office_revenue in CMU: %d'%movies.Movie_box_office_revenue.isnull().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tconst</th>\n",
       "      <th>titleType</th>\n",
       "      <th>primaryTitle</th>\n",
       "      <th>originalTitle</th>\n",
       "      <th>isAdult</th>\n",
       "      <th>startYear</th>\n",
       "      <th>runtimeMinutes</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>tt0000009</td>\n",
       "      <td>movie</td>\n",
       "      <td>Miss Jerry</td>\n",
       "      <td>Miss Jerry</td>\n",
       "      <td>0</td>\n",
       "      <td>1894.0</td>\n",
       "      <td>45</td>\n",
       "      <td>Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5707587</th>\n",
       "      <td>tt2210499</td>\n",
       "      <td>movie</td>\n",
       "      <td>Birmingham</td>\n",
       "      <td>Birmingham</td>\n",
       "      <td>0</td>\n",
       "      <td>1896.0</td>\n",
       "      <td>61</td>\n",
       "      <td>Documentary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219951</th>\n",
       "      <td>tt0229676</td>\n",
       "      <td>movie</td>\n",
       "      <td>Reproduction of the Corbett and Fitzsimmons Fight</td>\n",
       "      <td>Reproduction of the Corbett and Fitzsimmons Fight</td>\n",
       "      <td>0</td>\n",
       "      <td>1897.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Documentary,News,Sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211832</th>\n",
       "      <td>tt0221040</td>\n",
       "      <td>movie</td>\n",
       "      <td>Buck Dance, Ute Indians</td>\n",
       "      <td>Buck Dance, Ute Indians</td>\n",
       "      <td>0</td>\n",
       "      <td>1898.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Documentary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226939</th>\n",
       "      <td>tt0236940</td>\n",
       "      <td>movie</td>\n",
       "      <td>69th Regiment Passing in Review</td>\n",
       "      <td>69th Regiment Passing in Review</td>\n",
       "      <td>0</td>\n",
       "      <td>1898.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Documentary</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            tconst titleType  \\\n",
       "9        tt0000009     movie   \n",
       "5707587  tt2210499     movie   \n",
       "219951   tt0229676     movie   \n",
       "211832   tt0221040     movie   \n",
       "226939   tt0236940     movie   \n",
       "\n",
       "                                              primaryTitle  \\\n",
       "9                                               Miss Jerry   \n",
       "5707587                                         Birmingham   \n",
       "219951   Reproduction of the Corbett and Fitzsimmons Fight   \n",
       "211832                             Buck Dance, Ute Indians   \n",
       "226939                     69th Regiment Passing in Review   \n",
       "\n",
       "                                             originalTitle isAdult  startYear  \\\n",
       "9                                               Miss Jerry       0     1894.0   \n",
       "5707587                                         Birmingham       0     1896.0   \n",
       "219951   Reproduction of the Corbett and Fitzsimmons Fight       0     1897.0   \n",
       "211832                             Buck Dance, Ute Indians       0     1898.0   \n",
       "226939                     69th Regiment Passing in Review       0     1898.0   \n",
       "\n",
       "        runtimeMinutes                  genres  \n",
       "9                   45                 Romance  \n",
       "5707587             61             Documentary  \n",
       "219951             NaN  Documentary,News,Sport  \n",
       "211832             NaN             Documentary  \n",
       "226939             NaN             Documentary  "
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "copy_IMDb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "copy_IMDb = title_basics_movies_cleaned.copy()\n",
    "copy_CMU = movies.copy()\n",
    "copy_stats = movie_stats_cleaned.copy()\n",
    "\n",
    "# sort by release date to facilitate matching\n",
    "copy_IMDb.sort_values(by='startYear',inplace=True)\n",
    "copy_CMU.sort_values(by='Movie_release_date',inplace=True)\n",
    "copy_stats.sort_values(by='year',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "common_words = {'a','an','and','the','of','at','in'}\n",
    "punctuation = {'.',',','!',';','?',''}\n",
    "def compare(df1,df2,col1_title,col2_title,col1_year,col2_year,threshold = 0.8, delta_year=1):\n",
    "        # for progression\n",
    "        N = len(df1)\n",
    "        count = 0\n",
    "        \n",
    "        matched = {}\n",
    "        # iteration over the rows of the first dataframe\n",
    "        for idx1,row1 in df1.iterrows():\n",
    "            found = False # True if a matched is found\n",
    "            # process the title\n",
    "            title1 = set(re.split('[ :,]',row1[col1_title].lower()))\n",
    "            title1 = title1.difference(punctuation)\n",
    "            y1 = row1[col1_year]\n",
    "            \n",
    "            if np.isnan(y1):\n",
    "                search = df2 # if the year is given, we search films of same year first\n",
    "            else:\n",
    "                search = df2[df2[col2_year]==y1] # otherwise, search everywhere\n",
    "            for idx2,row2 in search.iterrows():\n",
    "                title2 = set(re.split('[ :,]',row2[col2_title].lower()))\n",
    "                title2 = title2.difference(punctuation)\n",
    "                if len(title1 & title2)/(len(title1 | title2)) > threshold:\n",
    "                    found = True\n",
    "                    try:\n",
    "                        matched[idx1].append(idx2)\n",
    "                    except KeyError:\n",
    "                        matched[idx1] = [idx2]\n",
    "            # if match not found and we did not check in NaN years:\n",
    "            if not found and not np.isnan(y1):\n",
    "                for idx2,row2 in df2[df2[col2_year].isnull()].iterrows():\n",
    "                    title2 = set(re.split('[ :,]',row2[col2_title].lower()))\n",
    "                    title2 = title2.difference(punctuation)\n",
    "                    if len(title1 & title2)/(len(title1 | title2)) > threshold:\n",
    "                        try:\n",
    "                            matched[idx1].append(idx2)\n",
    "                        except KeyError:\n",
    "                            matched[idx1] = [idx2]\n",
    "                \n",
    "            count += 1\n",
    "            if count%1000 == 0:\n",
    "                print('Iter {c}/{n}'.format(c=count,n=N))\n",
    "            \n",
    "        return matched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 1000/5497\n",
      "Iter 2000/5497\n",
      "Iter 3000/5497\n",
      "Iter 4000/5497\n",
      "Iter 5000/5497\n",
      "Time of execution: 774.4327118396759\n"
     ]
    }
   ],
   "source": [
    "if True:\n",
    "    from time import time\n",
    "    deb = time()\n",
    "    matched = compare(copy_stats,copy_CMU, 'name', 'Movie_name','year', 'Movie_release_date')\n",
    "    end = time()\n",
    "    print('Time of execution:', end-deb) # 2360s\n",
    "    matched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4192"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(matched)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    # save the matching table\n",
    "    with open(MATCHING_TABLE, 'wb') as file:\n",
    "        pickle.dump(matched, file, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "else:\n",
    "    matched = pd.read_pickle(MATCHING_TABLE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 duplicates (0.45% of all matchings)\n"
     ]
    }
   ],
   "source": [
    "doublons = {}\n",
    "for match in matched:\n",
    "    if len(matched[match]) > 1:\n",
    "        doublons[match] = matched[match]\n",
    "print('{nb} duplicates ({per:.2f}% of all matchings)'.format(nb=len(doublons), per=len(doublons)/len(matched)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49203    Indiana Jones and the Last Crusade  VS  Indian...\n",
      "4856     Indiana Jones and the Last Crusade  VS  Indian...\n",
      "Name: Movie_name, dtype: object\n",
      "25299    Black Rain  VS  Black Rain\n",
      "79339    Black Rain  VS  Black Rain\n",
      "Name: Movie_name, dtype: object\n",
      "39266    Leprechaun  VS  Leprechaun\n",
      "50948    Leprechaun  VS  Leprechaun\n",
      "Name: Movie_name, dtype: object\n",
      "80575    Thumbelina  VS  Thumbelina\n",
      "22136    Thumbelina  VS  Thumbelina\n",
      "Name: Movie_name, dtype: object\n",
      "76000    Emma  VS  Emma\n",
      "14313    Emma  VS  Emma\n",
      "Name: Movie_name, dtype: object\n",
      "15592    Soldier  VS  Soldier\n",
      "67509    Soldier  VS  Soldier\n",
      "Name: Movie_name, dtype: object\n",
      "46575    Gloria  VS  Gloria\n",
      "58390    Gloria  VS  Gloria\n",
      "Name: Movie_name, dtype: object\n",
      "16826    Gossip  VS  Gossip\n",
      "73795    Gossip  VS  Gossip\n",
      "Name: Movie_name, dtype: object\n",
      "76507    Man of the House  VS  Man of the House\n",
      "1754     Man of the House  VS  Man of the House\n",
      "Name: Movie_name, dtype: object\n",
      "36490    Rendition  VS  Rendition\n",
      "34986    Rendition  VS  Rendition\n",
      "Name: Movie_name, dtype: object\n",
      "17801    Nothing But the Truth  VS  Nothing but the Truth\n",
      "10890    Nothing But the Truth  VS  Nothing but the Truth\n",
      "Name: Movie_name, dtype: object\n",
      "19405    Journey to the Center of the Earth  VS  Journe...\n",
      "77521    Journey to the Center of the Earth  VS  Journe...\n",
      "64311    Journey to the Center of the Earth  VS  Journe...\n",
      "Name: Movie_name, dtype: object\n",
      "27755    Sherlock Holmes  VS  Sherlock Holmes\n",
      "64743    Sherlock Holmes  VS  Sherlock Holmes\n",
      "Name: Movie_name, dtype: object\n",
      "47219    Cyrus  VS  Cyrus\n",
      "31198    Cyrus  VS  Cyrus\n",
      "Name: Movie_name, dtype: object\n",
      "62998    Burlesque  VS  Burlesque\n",
      "13365    Burlesque  VS  Burlesque\n",
      "Name: Movie_name, dtype: object\n",
      "21479    Anonymous  VS  Anonymous\n",
      "44851    Anonymous  VS  Anonymous\n",
      "Name: Movie_name, dtype: object\n",
      "73955    The First Time  VS  The First Time\n",
      "44765    The First Time  VS  The First Time\n",
      "Name: Movie_name, dtype: object\n",
      "18883    The Hunger Games: Catching Fire  VS  The Hunge...\n",
      "50852    The Hunger Games: Catching Fire  VS  The Hunge...\n",
      "Name: Movie_name, dtype: object\n",
      "21342    Beauty and the Beast  VS  Beauty and the Beast\n",
      "22734    Beauty and the Beast  VS  Beauty and the Beast\n",
      "Name: Movie_name, dtype: object\n"
     ]
    }
   ],
   "source": [
    "for stats,cmus in doublons.items():\n",
    "    print(copy_stats.loc[stats,'name'] + '  VS  ' + copy_CMU.loc[cmus,'Movie_name'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Many duplicated matched are juste films with the same name (and same year), so probably duplicated films in the database.\n",
    "\n",
    "Some are similar titles but the order of words is changed (e.g \"Black and White\" corresponding to \"Black and White\" and \"White and Black\")."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merging the datasets\n",
    "\n",
    "As only $0.45\\%$ of the matchings are duplicated, we will simply drop them. Finally, we add the ratings corresponding to the films (we lose only 5 films which did not have ratings)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4173\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genre</th>\n",
       "      <th>score</th>\n",
       "      <th>votes</th>\n",
       "      <th>director</th>\n",
       "      <th>writer</th>\n",
       "      <th>star</th>\n",
       "      <th>country</th>\n",
       "      <th>budget</th>\n",
       "      <th>gross</th>\n",
       "      <th>company</th>\n",
       "      <th>runtime\\</th>\n",
       "      <th>Wikipedia_movie_ID</th>\n",
       "      <th>Freebase_movie_ID</th>\n",
       "      <th>Movie_name</th>\n",
       "      <th>Movie_release_date</th>\n",
       "      <th>Movie_box_office_revenue</th>\n",
       "      <th>Movie_runtime</th>\n",
       "      <th>Movie_languages</th>\n",
       "      <th>Movie_countries</th>\n",
       "      <th>Movie_genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Drama</td>\n",
       "      <td>8.4</td>\n",
       "      <td>927000.0</td>\n",
       "      <td>Stanley Kubrick</td>\n",
       "      <td>Stephen King</td>\n",
       "      <td>Jack Nicholson</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>19000000.0</td>\n",
       "      <td>46998772.0</td>\n",
       "      <td>Warner Bros.</td>\n",
       "      <td>146.0\\</td>\n",
       "      <td>1186616</td>\n",
       "      <td>/m/04fjzv</td>\n",
       "      <td>The Shining</td>\n",
       "      <td>1980.0</td>\n",
       "      <td>44017374.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>[English Language]</td>\n",
       "      <td>[United States of America, United Kingdom]</td>\n",
       "      <td>[Horror, Supernatural, Surrealism, Psychologic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Comedy</td>\n",
       "      <td>6.2</td>\n",
       "      <td>24000.0</td>\n",
       "      <td>Howard Zieff</td>\n",
       "      <td>Nancy Meyers</td>\n",
       "      <td>Goldie Hawn</td>\n",
       "      <td>United States</td>\n",
       "      <td>10000000.0</td>\n",
       "      <td>69847348.0</td>\n",
       "      <td>Warner Bros.</td>\n",
       "      <td>109.0\\</td>\n",
       "      <td>240371</td>\n",
       "      <td>/m/01jw6z</td>\n",
       "      <td>Private Benjamin</td>\n",
       "      <td>1980.0</td>\n",
       "      <td>69847348.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>[English Language]</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Parody, Comedy-drama, Satire, Drama, Comedy, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Comedy</td>\n",
       "      <td>6.3</td>\n",
       "      <td>4300.0</td>\n",
       "      <td>Floyd Mutrux</td>\n",
       "      <td>Floyd Mutrux</td>\n",
       "      <td>Tony Danza</td>\n",
       "      <td>United States</td>\n",
       "      <td>4000000.0</td>\n",
       "      <td>10000000.0</td>\n",
       "      <td>PolyGram Filmed Entertainment</td>\n",
       "      <td>91.0\\</td>\n",
       "      <td>1595532</td>\n",
       "      <td>/m/05f8tn</td>\n",
       "      <td>The Hollywood Knights</td>\n",
       "      <td>1980.0</td>\n",
       "      <td>10000000.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>[English Language]</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Cult, Sex comedy, Comedy, Teen]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Comedy</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>Kevin Connor</td>\n",
       "      <td>Robert Jaffe</td>\n",
       "      <td>Rory Calhoun</td>\n",
       "      <td>United States</td>\n",
       "      <td>3000000.0</td>\n",
       "      <td>6342668.0</td>\n",
       "      <td>Camp Hill</td>\n",
       "      <td>101.0\\</td>\n",
       "      <td>3262031</td>\n",
       "      <td>/m/091_np</td>\n",
       "      <td>Motel Hell</td>\n",
       "      <td>1980.0</td>\n",
       "      <td>6342668.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>[English Language]</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Satire, Horror, Comedy, Slasher]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Action</td>\n",
       "      <td>7.1</td>\n",
       "      <td>9000.0</td>\n",
       "      <td>Richard Rush</td>\n",
       "      <td>Lawrence B. Marcus</td>\n",
       "      <td>Peter O'Toole</td>\n",
       "      <td>United States</td>\n",
       "      <td>3500000.0</td>\n",
       "      <td>7063886.0</td>\n",
       "      <td>Melvin Simon Productions</td>\n",
       "      <td>131.0\\</td>\n",
       "      <td>164395</td>\n",
       "      <td>/m/015wq4</td>\n",
       "      <td>The Stunt Man</td>\n",
       "      <td>1980.0</td>\n",
       "      <td>7068886.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>[English Language]</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Thriller, Comedy-drama, Comedy Thriller, Acti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    genre  score     votes         director              writer  \\\n",
       "0   Drama    8.4  927000.0  Stanley Kubrick        Stephen King   \n",
       "1  Comedy    6.2   24000.0     Howard Zieff        Nancy Meyers   \n",
       "2  Comedy    6.3    4300.0     Floyd Mutrux        Floyd Mutrux   \n",
       "3  Comedy    6.0   11000.0     Kevin Connor        Robert Jaffe   \n",
       "4  Action    7.1    9000.0     Richard Rush  Lawrence B. Marcus   \n",
       "\n",
       "             star         country      budget       gross  \\\n",
       "0  Jack Nicholson  United Kingdom  19000000.0  46998772.0   \n",
       "1     Goldie Hawn   United States  10000000.0  69847348.0   \n",
       "2      Tony Danza   United States   4000000.0  10000000.0   \n",
       "3    Rory Calhoun   United States   3000000.0   6342668.0   \n",
       "4   Peter O'Toole   United States   3500000.0   7063886.0   \n",
       "\n",
       "                         company runtime\\  Wikipedia_movie_ID  \\\n",
       "0                   Warner Bros.   146.0\\             1186616   \n",
       "1                   Warner Bros.   109.0\\              240371   \n",
       "2  PolyGram Filmed Entertainment    91.0\\             1595532   \n",
       "3                      Camp Hill   101.0\\             3262031   \n",
       "4       Melvin Simon Productions   131.0\\              164395   \n",
       "\n",
       "  Freebase_movie_ID             Movie_name  Movie_release_date  \\\n",
       "0         /m/04fjzv            The Shining              1980.0   \n",
       "1         /m/01jw6z       Private Benjamin              1980.0   \n",
       "2         /m/05f8tn  The Hollywood Knights              1980.0   \n",
       "3         /m/091_np             Motel Hell              1980.0   \n",
       "4         /m/015wq4          The Stunt Man              1980.0   \n",
       "\n",
       "   Movie_box_office_revenue  Movie_runtime     Movie_languages  \\\n",
       "0                44017374.0          143.0  [English Language]   \n",
       "1                69847348.0          109.0  [English Language]   \n",
       "2                10000000.0           91.0  [English Language]   \n",
       "3                 6342668.0          102.0  [English Language]   \n",
       "4                 7068886.0          129.0  [English Language]   \n",
       "\n",
       "                              Movie_countries  \\\n",
       "0  [United States of America, United Kingdom]   \n",
       "1                  [United States of America]   \n",
       "2                  [United States of America]   \n",
       "3                  [United States of America]   \n",
       "4                  [United States of America]   \n",
       "\n",
       "                                        Movie_genres  \n",
       "0  [Horror, Supernatural, Surrealism, Psychologic...  \n",
       "1  [Parody, Comedy-drama, Satire, Drama, Comedy, ...  \n",
       "2                   [Cult, Sex comedy, Comedy, Teen]  \n",
       "3                  [Satire, Horror, Comedy, Slasher]  \n",
       "4  [Thriller, Comedy-drama, Comedy Thriller, Acti...  "
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if MATCH_DATA:\n",
    "    for stats,cmus in matched.items():\n",
    "        if stats not in doublons:\n",
    "            copy_stats.loc[stats,'CMU_index'] = cmus[0]\n",
    "    copy_CMU['CMU_index'] = copy_CMU.index\n",
    "    copy_stats.dropna(subset=['CMU_index'],inplace=True)\n",
    "    copy_stats['CMU_index'] = copy_stats['CMU_index'].astype('int64')\n",
    "    merge_df = pd.merge(copy_stats, copy_CMU, on = 'CMU_index', how = \"inner\")\n",
    "\n",
    "    # Find the year\n",
    "    for idx,row in merge_df.iterrows():\n",
    "        if np.isnan(row['Movie_release_date']):\n",
    "            merge_df.loc[idx,'Movie_release_date'] = row['year']\n",
    "        if np.isnan(row['Movie_box_office_revenue']):\n",
    "            merge_df.loc[idx,'Movie_box_office_revenue'] = row['gross']\n",
    "    # Drop duplicated columns\n",
    "    merge_df.drop(['name', 'year','CMU_index'], axis=1, inplace=True)\n",
    "    \n",
    "    # Save the dataset\n",
    "    with open(DESTINATION + 'merge_CMU_stats.pkl', 'wb') as file:\n",
    "        pickle.dump(merge_df, file, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "else:\n",
    "    merge_df = pd.read_pickle(MERGED_CMU_STATS)\n",
    "print(len(merge_df))\n",
    "merge_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "genre                         0\n",
       "score                         0\n",
       "votes                         0\n",
       "director                      0\n",
       "writer                        0\n",
       "star                          0\n",
       "country                       0\n",
       "budget                        0\n",
       "gross                        37\n",
       "company                       2\n",
       "runtime\\                      0\n",
       "Wikipedia_movie_ID            0\n",
       "Freebase_movie_ID             0\n",
       "Movie_name                    0\n",
       "Movie_release_date            0\n",
       "Movie_box_office_revenue     20\n",
       "Movie_runtime               104\n",
       "Movie_languages               0\n",
       "Movie_countries               0\n",
       "Movie_genres                  0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge this dataframe to the IMDb one\n",
    "\n",
    "This one only dropped 8 films (not found in IMDb), and only 46 duplicates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tconst</th>\n",
       "      <th>titleType</th>\n",
       "      <th>primaryTitle</th>\n",
       "      <th>originalTitle</th>\n",
       "      <th>isAdult</th>\n",
       "      <th>startYear</th>\n",
       "      <th>runtimeMinutes</th>\n",
       "      <th>genres</th>\n",
       "      <th>IMDB_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>tt0000009</td>\n",
       "      <td>movie</td>\n",
       "      <td>Miss Jerry</td>\n",
       "      <td>Miss Jerry</td>\n",
       "      <td>0</td>\n",
       "      <td>1894.0</td>\n",
       "      <td>45</td>\n",
       "      <td>Romance</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5707587</th>\n",
       "      <td>tt2210499</td>\n",
       "      <td>movie</td>\n",
       "      <td>Birmingham</td>\n",
       "      <td>Birmingham</td>\n",
       "      <td>0</td>\n",
       "      <td>1896.0</td>\n",
       "      <td>61</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>5707587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219951</th>\n",
       "      <td>tt0229676</td>\n",
       "      <td>movie</td>\n",
       "      <td>Reproduction of the Corbett and Fitzsimmons Fight</td>\n",
       "      <td>Reproduction of the Corbett and Fitzsimmons Fight</td>\n",
       "      <td>0</td>\n",
       "      <td>1897.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Documentary,News,Sport</td>\n",
       "      <td>219951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211832</th>\n",
       "      <td>tt0221040</td>\n",
       "      <td>movie</td>\n",
       "      <td>Buck Dance, Ute Indians</td>\n",
       "      <td>Buck Dance, Ute Indians</td>\n",
       "      <td>0</td>\n",
       "      <td>1898.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>211832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226939</th>\n",
       "      <td>tt0236940</td>\n",
       "      <td>movie</td>\n",
       "      <td>69th Regiment Passing in Review</td>\n",
       "      <td>69th Regiment Passing in Review</td>\n",
       "      <td>0</td>\n",
       "      <td>1898.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>226939</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            tconst titleType  \\\n",
       "9        tt0000009     movie   \n",
       "5707587  tt2210499     movie   \n",
       "219951   tt0229676     movie   \n",
       "211832   tt0221040     movie   \n",
       "226939   tt0236940     movie   \n",
       "\n",
       "                                              primaryTitle  \\\n",
       "9                                               Miss Jerry   \n",
       "5707587                                         Birmingham   \n",
       "219951   Reproduction of the Corbett and Fitzsimmons Fight   \n",
       "211832                             Buck Dance, Ute Indians   \n",
       "226939                     69th Regiment Passing in Review   \n",
       "\n",
       "                                             originalTitle isAdult  startYear  \\\n",
       "9                                               Miss Jerry       0     1894.0   \n",
       "5707587                                         Birmingham       0     1896.0   \n",
       "219951   Reproduction of the Corbett and Fitzsimmons Fight       0     1897.0   \n",
       "211832                             Buck Dance, Ute Indians       0     1898.0   \n",
       "226939                     69th Regiment Passing in Review       0     1898.0   \n",
       "\n",
       "        runtimeMinutes                  genres  IMDB_index  \n",
       "9                   45                 Romance           9  \n",
       "5707587             61             Documentary     5707587  \n",
       "219951             NaN  Documentary,News,Sport      219951  \n",
       "211832             NaN             Documentary      211832  \n",
       "226939             NaN             Documentary      226939  "
      ]
     },
     "execution_count": 308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "copy_IMDb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [],
   "source": [
    "if MERGE_AGAIN:\n",
    "    matched = compare(merge_df,copy_IMDb, 'Movie_name', 'primaryTitle', 'Movie_release_date', 'startYear')\n",
    "    # save the matching table\n",
    "    with open(DESTINATION+'matching_table_bis.pkl', 'wb') as file:\n",
    "        pickle.dump(matched, file, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "else:\n",
    "    matched = pd.read_pickle(DESTINATION+'matching_table_bis.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46 duplicates (1.10% of all matchings)\n"
     ]
    }
   ],
   "source": [
    "doublons = {}\n",
    "for match in matched:\n",
    "    if len(matched[match]) > 1:\n",
    "        doublons[match] = matched[match]\n",
    "print('{nb} duplicates ({per:.2f}% of all matchings)'.format(nb=len(doublons), per=len(doublons)/len(matched)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4165\n",
      "4173\n",
      "626772\n"
     ]
    }
   ],
   "source": [
    "print(len(matched))\n",
    "print(len(merge_df))\n",
    "print(len(copy_IMDb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4116\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genre</th>\n",
       "      <th>score</th>\n",
       "      <th>votes</th>\n",
       "      <th>director</th>\n",
       "      <th>writer</th>\n",
       "      <th>star</th>\n",
       "      <th>country</th>\n",
       "      <th>budget</th>\n",
       "      <th>company</th>\n",
       "      <th>Wikipedia_movie_ID</th>\n",
       "      <th>...</th>\n",
       "      <th>Movie_countries</th>\n",
       "      <th>Movie_genres</th>\n",
       "      <th>IMDB_index</th>\n",
       "      <th>tconst</th>\n",
       "      <th>titleType</th>\n",
       "      <th>isAdult</th>\n",
       "      <th>genres</th>\n",
       "      <th>tconstIdentifier</th>\n",
       "      <th>averageRating</th>\n",
       "      <th>numVotes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Drama</td>\n",
       "      <td>8.4</td>\n",
       "      <td>927000.0</td>\n",
       "      <td>Stanley Kubrick</td>\n",
       "      <td>Stephen King</td>\n",
       "      <td>Jack Nicholson</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>19000000.0</td>\n",
       "      <td>Warner Bros.</td>\n",
       "      <td>1186616</td>\n",
       "      <td>...</td>\n",
       "      <td>[United States of America, United Kingdom]</td>\n",
       "      <td>[Horror, Supernatural, Surrealism, Psychologic...</td>\n",
       "      <td>79732</td>\n",
       "      <td>tt0081505</td>\n",
       "      <td>movie</td>\n",
       "      <td>0</td>\n",
       "      <td>Drama,Horror</td>\n",
       "      <td>tt0081505</td>\n",
       "      <td>8.4</td>\n",
       "      <td>1015506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Comedy</td>\n",
       "      <td>6.2</td>\n",
       "      <td>24000.0</td>\n",
       "      <td>Howard Zieff</td>\n",
       "      <td>Nancy Meyers</td>\n",
       "      <td>Goldie Hawn</td>\n",
       "      <td>United States</td>\n",
       "      <td>10000000.0</td>\n",
       "      <td>Warner Bros.</td>\n",
       "      <td>240371</td>\n",
       "      <td>...</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Parody, Comedy-drama, Satire, Drama, Comedy, ...</td>\n",
       "      <td>79606</td>\n",
       "      <td>tt0081375</td>\n",
       "      <td>movie</td>\n",
       "      <td>0</td>\n",
       "      <td>Comedy,War</td>\n",
       "      <td>tt0081375</td>\n",
       "      <td>6.2</td>\n",
       "      <td>26738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Comedy</td>\n",
       "      <td>6.3</td>\n",
       "      <td>4300.0</td>\n",
       "      <td>Floyd Mutrux</td>\n",
       "      <td>Floyd Mutrux</td>\n",
       "      <td>Tony Danza</td>\n",
       "      <td>United States</td>\n",
       "      <td>4000000.0</td>\n",
       "      <td>PolyGram Filmed Entertainment</td>\n",
       "      <td>1595532</td>\n",
       "      <td>...</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Cult, Sex comedy, Comedy, Teen]</td>\n",
       "      <td>79134</td>\n",
       "      <td>tt0080881</td>\n",
       "      <td>movie</td>\n",
       "      <td>0</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>tt0080881</td>\n",
       "      <td>6.2</td>\n",
       "      <td>4746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Comedy</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>Kevin Connor</td>\n",
       "      <td>Robert Jaffe</td>\n",
       "      <td>Rory Calhoun</td>\n",
       "      <td>United States</td>\n",
       "      <td>3000000.0</td>\n",
       "      <td>Camp Hill</td>\n",
       "      <td>3262031</td>\n",
       "      <td>...</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Satire, Horror, Comedy, Slasher]</td>\n",
       "      <td>79428</td>\n",
       "      <td>tt0081184</td>\n",
       "      <td>movie</td>\n",
       "      <td>0</td>\n",
       "      <td>Comedy,Horror,Thriller</td>\n",
       "      <td>tt0081184</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Action</td>\n",
       "      <td>7.1</td>\n",
       "      <td>9000.0</td>\n",
       "      <td>Richard Rush</td>\n",
       "      <td>Lawrence B. Marcus</td>\n",
       "      <td>Peter O'Toole</td>\n",
       "      <td>United States</td>\n",
       "      <td>3500000.0</td>\n",
       "      <td>Melvin Simon Productions</td>\n",
       "      <td>164395</td>\n",
       "      <td>...</td>\n",
       "      <td>[United States of America]</td>\n",
       "      <td>[Thriller, Comedy-drama, Comedy Thriller, Acti...</td>\n",
       "      <td>79793</td>\n",
       "      <td>tt0081568</td>\n",
       "      <td>movie</td>\n",
       "      <td>0</td>\n",
       "      <td>Action,Comedy,Drama</td>\n",
       "      <td>tt0081568</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9784</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    genre  score     votes         director              writer  \\\n",
       "0   Drama    8.4  927000.0  Stanley Kubrick        Stephen King   \n",
       "1  Comedy    6.2   24000.0     Howard Zieff        Nancy Meyers   \n",
       "2  Comedy    6.3    4300.0     Floyd Mutrux        Floyd Mutrux   \n",
       "3  Comedy    6.0   11000.0     Kevin Connor        Robert Jaffe   \n",
       "4  Action    7.1    9000.0     Richard Rush  Lawrence B. Marcus   \n",
       "\n",
       "             star         country      budget                        company  \\\n",
       "0  Jack Nicholson  United Kingdom  19000000.0                   Warner Bros.   \n",
       "1     Goldie Hawn   United States  10000000.0                   Warner Bros.   \n",
       "2      Tony Danza   United States   4000000.0  PolyGram Filmed Entertainment   \n",
       "3    Rory Calhoun   United States   3000000.0                      Camp Hill   \n",
       "4   Peter O'Toole   United States   3500000.0       Melvin Simon Productions   \n",
       "\n",
       "   Wikipedia_movie_ID  ...                             Movie_countries  \\\n",
       "0             1186616  ...  [United States of America, United Kingdom]   \n",
       "1              240371  ...                  [United States of America]   \n",
       "2             1595532  ...                  [United States of America]   \n",
       "3             3262031  ...                  [United States of America]   \n",
       "4              164395  ...                  [United States of America]   \n",
       "\n",
       "                                        Movie_genres  IMDB_index     tconst  \\\n",
       "0  [Horror, Supernatural, Surrealism, Psychologic...       79732  tt0081505   \n",
       "1  [Parody, Comedy-drama, Satire, Drama, Comedy, ...       79606  tt0081375   \n",
       "2                   [Cult, Sex comedy, Comedy, Teen]       79134  tt0080881   \n",
       "3                  [Satire, Horror, Comedy, Slasher]       79428  tt0081184   \n",
       "4  [Thriller, Comedy-drama, Comedy Thriller, Acti...       79793  tt0081568   \n",
       "\n",
       "   titleType isAdult                  genres tconstIdentifier  averageRating  \\\n",
       "0      movie       0            Drama,Horror        tt0081505            8.4   \n",
       "1      movie       0              Comedy,War        tt0081375            6.2   \n",
       "2      movie       0                  Comedy        tt0080881            6.2   \n",
       "3      movie       0  Comedy,Horror,Thriller        tt0081184            6.0   \n",
       "4      movie       0     Action,Comedy,Drama        tt0081568            7.0   \n",
       "\n",
       "  numVotes  \n",
       "0  1015506  \n",
       "1    26738  \n",
       "2     4746  \n",
       "3    12308  \n",
       "4     9784  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 314,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if MERGE_AGAIN:\n",
    "    for old,imdbs in matched.items():\n",
    "        if old not in doublons:\n",
    "            merge_df.loc[old,'IMDB_index'] = imdbs[0]\n",
    "    copy_IMDb['IMDB_index'] = copy_IMDb.index\n",
    "    merge_df.dropna(subset=['IMDB_index'],inplace=True)\n",
    "    merge_df['IMDB_index'] = merge_df['IMDB_index'].astype('int64')\n",
    "    merge_df = pd.merge(merge_df, copy_IMDb, on = 'IMDB_index', how = \"inner\")\n",
    "    \n",
    "    # Drop duplicated columns\n",
    "    merge_df.drop(['runtime\\\\','runtimeMinutes','originalTitle','primaryTitle','startYear','gross'], axis=1, inplace=True)\n",
    "    \n",
    "    \n",
    "    # Add the ratings\n",
    "    merge_df = pd.merge(merge_df, ratings_cleaned, left_on = 'tconst', right_on ='tconstIdentifier', how = \"inner\")\n",
    "    \n",
    "    # Save the results\n",
    "    with open(MERGED_CMU_IMDB_STATS, 'wb') as file:\n",
    "        pickle.dump(merge_df, file, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "else:\n",
    "    merge_df = pd.read_pickle(MERGED_CMU_IMDB_STATS)\n",
    "print(len(merge_df))\n",
    "merge_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "vscode": {
   "interpreter": {
    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}