hwk4-jupyter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.tensorflow import TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = TensorFlow(\n",
    "    entry_point=\"sentiment_training.py\",\n",
    "    role = 'arn:aws:iam::431525178180:role/hwk4-sagemaker',\n",
    "    train_instance_type='ml.p2.xlarge',\n",
    "    train_instance_count=1,\n",
    "    # output_path='s3://ai2020/hwk4/sagemaker_data/output',\n",
    "    framework_version='1.14',\n",
    "    py_version=\"py3\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "ClientError",
     "evalue": "An error occurred (AccessDeniedException) when calling the CreateTrainingJob operation: User: arn:aws:sts::431525178180:assumed-role/hwk4-sagemaker/SageMaker is not authorized to perform: sagemaker:CreateTrainingJob on resource: arn:aws:sagemaker:us-east-1:431525178180:training-job/tensorflow-training-2020-02-23-03-05-31-864 with an explicit deny",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mClientError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-36-2f7b7d07de24>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m's3://ai2020/hwk4/sagemaker_data/train/'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/sagemaker/tensorflow/estimator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, inputs, wait, logs, job_name, experiment_config, run_tensorboard_locally)\u001b[0m\n\u001b[1;32m    464\u001b[0m                 \u001b[0mtensorboard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    465\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 466\u001b[0;31m             \u001b[0mfit_super\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    467\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    468\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/sagemaker/tensorflow/estimator.py\u001b[0m in \u001b[0;36mfit_super\u001b[0;34m()\u001b[0m\n\u001b[1;32m    443\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    444\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mfit_super\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 445\u001b[0;31m             \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTensorFlow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexperiment_config\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_tensorboard_locally\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mwait\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, inputs, wait, logs, job_name, experiment_config)\u001b[0m\n\u001b[1;32m    460\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prepare_for_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 462\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_TrainingJob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_new\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexperiment_config\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    463\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mstart_new\u001b[0;34m(cls, estimator, inputs, experiment_config)\u001b[0m\n\u001b[1;32m   1013\u001b[0m             \u001b[0mtrain_args\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"enable_sagemaker_metrics\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_sagemaker_metrics\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1015\u001b[0;31m         \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mtrain_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1016\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current_job_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/sagemaker/session.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, input_mode, input_config, role, job_name, output_config, resource_config, vpc_config, hyperparameters, stop_condition, tags, metric_definitions, enable_network_isolation, image, algorithm_arn, encrypt_inter_container_traffic, train_use_spot_instances, checkpoint_s3_uri, checkpoint_local_path, experiment_config, debugger_rule_configs, debugger_hook_config, tensorboard_output_config, enable_sagemaker_metrics)\u001b[0m\n\u001b[1;32m    567\u001b[0m         \u001b[0mLOGGER\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Creating training-job with name: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    568\u001b[0m         \u001b[0mLOGGER\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"train request: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_request\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 569\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_training_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mtrain_request\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    570\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    571\u001b[0m     def process(\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    274\u001b[0m                     \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m    275\u001b[0m             \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 276\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    277\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    278\u001b[0m         \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m    584\u001b[0m             \u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Code\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0merror_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mClientError\u001b[0m: An error occurred (AccessDeniedException) when calling the CreateTrainingJob operation: User: arn:aws:sts::431525178180:assumed-role/hwk4-sagemaker/SageMaker is not authorized to perform: sagemaker:CreateTrainingJob on resource: arn:aws:sagemaker:us-east-1:431525178180:training-job/tensorflow-training-2020-02-23-03-05-31-864 with an explicit deny"
     ]
    }
   ],
   "source": [
    "estimator.fit({'train': 's3://ai2020/hwk4/sagemaker_data/train/'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'arn:aws:iam::431525178180:role/hwk4-sagemaker'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sagemaker import get_execution_role\n",
    "\n",
    "role = get_execution_role()\n",
    "role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "\n",
    "sagemaker_session = sagemaker.Session()\n",
    "bucket = sagemaker_session.default_bucket()\n",
    "role = sagemaker.get_execution_role()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sagemaker-us-east-1-431525178180'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "from sagemaker.estimator import Framework\n",
    "from sagemaker.tensorflow import TensorFlow\n",
    "\n",
    "class ScriptModeTensorFlow(Framework):\n",
    "    \"\"\"This class is temporary until the final version of Script Mode is released.\n",
    "    \"\"\"\n",
    "    \n",
    "    __framework_name__ = \"tensorflow-scriptmode\"\n",
    "    \n",
    "    create_model = TensorFlow.create_model\n",
    "    \n",
    "    def __init__(self, py_version='py3', **kwargs):\n",
    "        super(ScriptModeTensorFlow, self).__init__(**kwargs)\n",
    "        self.py_version = py_version\n",
    "        self.image_name = None\n",
    "        self.framework_version = '1.14'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "role = get_execution_role()\n",
    "bucket='ai2020'\n",
    "data_key = 'hwk4/sagemaker_data/train'\n",
    "data_location = 's3://{}/{}'.format(bucket, data_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>features</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0, 0, 7, 906, 7, 2361, 7, 364, 21, 0, 1870, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>@perrinc Hi Colleen, welcome, if you have any ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>Im Well Bored  Just Sat Eyaa With Chris And Hi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0, 128, 0, 3464, 209, 10, 366, 21, 185, 174, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>Bummer of a day...spent $360 on something I re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0, 7, 542, 7, 21, 1943, 526, 21, 4746, 0, 0, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>@kathyfoley @kieranmurphy Gosh, is she pretty,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[409, 300, 321, 12, 146, 7491, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>@anberlin i wish i could be there tonight! i g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[21, 12, 600, 31914, 832, 12, 0, 556, 294, 221...</td>\n",
       "      <td>4</td>\n",
       "      <td>@hdcgf0 OMG! another nickelback fan! Not many ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[5115, 3236, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>0</td>\n",
       "      <td>havent tweeted today</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[13327, 4169, 1083, 0, 0, 7, 344, 7, 120, 1092...</td>\n",
       "      <td>4</td>\n",
       "      <td>ooo keeping up to date with all the e3 anounce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[0, 3289, 6258, 99, 238, 4, 0, 21, 628, 347, 4...</td>\n",
       "      <td>4</td>\n",
       "      <td>Three spanish lessons in one night. Hopefully ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[0, 7, 735, 4, 0, 21, 644, 17, 21, 644, 0, 4, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>@khali_blache Egads, that is awesome. Can I vo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>[0, 1023, 151, 0, 1502, 676, 1207, 1424, 1023,...</td>\n",
       "      <td>0</td>\n",
       "      <td>So cold today... kinda feeling a bit blue from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>[462, 0, 21, 180, 784, 199, 834, 500, 0, 0, 0,...</td>\n",
       "      <td>4</td>\n",
       "      <td>@rajeshlalwani very nice.....  I am as happy t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>[4, 0, 597, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>4</td>\n",
       "      <td>@work. Someone's gotta do the job</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>[0, 770, 66, 6800, 0, 0, 9738, 0, 0, 556, 7466...</td>\n",
       "      <td>4</td>\n",
       "      <td>Looking for a job is like an adult Easter Egg ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>[511, 1319, 21, 7513, 382, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>0</td>\n",
       "      <td>had to put my dog that I grew up with to sleep</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>[0, 0, 7, 0, 4, 212, 212, 1865, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>4</td>\n",
       "      <td>@Loman_15 Ocean City, Md. has the best of the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>[3253, 439, 205, 2456, 9124, 98619, 221, 4326,...</td>\n",
       "      <td>0</td>\n",
       "      <td>@officaltila not to be rude but do you have to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>[21, 71, 683, 0, 4, 425, 248, 80463, 86, 229, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>@JimKitzmiller I love your tweets Jim. They R ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>[0, 4, 21, 189, 1744, 683, 532, 51033, 1853, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>@yat4ever Wow.  I think reading your tweets ju...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>[0, 13396, 7, 99, 5654, 128, 414, 438, 0, 0, 0...</td>\n",
       "      <td>0</td>\n",
       "      <td>According to forecast, there will be one sunny...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>[0, 1506, 12, 196, 461, 4, 222, 4, 222, 51, 51...</td>\n",
       "      <td>0</td>\n",
       "      <td>2more weeks! oh damn it. =.='' I need chiong a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>[1905, 10, 1045, 652, 5443, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>@caffeinebomb it's about £35 per head and so a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>[0, 18134, 4, 21, 1430, 133, 99, 14451, 540, 1...</td>\n",
       "      <td>4</td>\n",
       "      <td>@retroasylum Lol yeaaah. A couple people but y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>[21, 21, 21, 21, 21, 21, 4, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>@DJ_Jonas STAY AWAY FROM PIGS.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>[423, 351, 185, 19074, 4, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>@AetherPrincess i hope you dont  its really di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>[0, 11223, 1004, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>Life has no excitement anymore.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>[8189, 4, 7, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>4</td>\n",
       "      <td>@VelvetMuffin don't be a douche. or do, but OW...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>@nathalialduarte</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>[272, 125, 910, 87, 31, 21, 119231, 0, 9612, 2...</td>\n",
       "      <td>4</td>\n",
       "      <td>@BarbraSundquist thanks for the new place to f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>[0, 7, 1175, 3069, 400, 323, 395, 7, 275, 0, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>@MissDreg  that's it now, i can kiss goodbye t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8470</th>\n",
       "      <td>[4383, 4163, 12, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>0</td>\n",
       "      <td>is studying for finals!  One more Day!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8471</th>\n",
       "      <td>[0, 8423, 42170, 13967, 5500, 2312, 21, 6351, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>My dumbass wiped the micro sd card I had in my...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8472</th>\n",
       "      <td>[0, 818, 31, 476, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>0</td>\n",
       "      <td>@jstank that's sad- sorry.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8473</th>\n",
       "      <td>[146, 1550, 0, 138, 7, 51, 509, 248, 904, 6322...</td>\n",
       "      <td>0</td>\n",
       "      <td>i just got this account for 2nd time, 'cause i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8474</th>\n",
       "      <td>[19470, 7, 214, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>after some pizzas, going sleep</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8475</th>\n",
       "      <td>[0, 400, 14127, 0, 0, 400, 946, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>I'm getting your emails... you ain't getting m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8476</th>\n",
       "      <td>[91, 5345, 12620, 91, 73407, 118, 809, 4, 1958...</td>\n",
       "      <td>4</td>\n",
       "      <td>@SarahG8s lol i wasnt on about the burns lol i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8477</th>\n",
       "      <td>[2315, 7, 484408, 0, 4837, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>0</td>\n",
       "      <td>off to church, alooone.. huhu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8478</th>\n",
       "      <td>[0, 347, 1282, 0, 21, 185, 818, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>Home morning loves...... I  an really sad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8479</th>\n",
       "      <td>[272, 0, 7, 4745, 1030, 10, 4, 0, 4, 0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>thanks iPhone, you kept me up till 1 am. Assho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8480</th>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>I'm over it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8481</th>\n",
       "      <td>[58635, 0, 21, 784, 683, 1739, 146, 0, 54, 76,...</td>\n",
       "      <td>4</td>\n",
       "      <td>@kimhollandswort Thanks....I read his tweets o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8482</th>\n",
       "      <td>[0, 0, 30148, 54, 10, 2940, 0, 0, 0, 277, 0, 3...</td>\n",
       "      <td>4</td>\n",
       "      <td>@MelissaNyomi Hmmm Im followin u for 2 reasons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8483</th>\n",
       "      <td>[600, 567, 289, 21, 0, 409, 804, 0, 0, 3010, 3...</td>\n",
       "      <td>0</td>\n",
       "      <td>another amazing video from FFXIII...wish i was...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8484</th>\n",
       "      <td>[0, 540, 1369, 151, 4, 0, 735, 4, 0, 214, 701,...</td>\n",
       "      <td>0</td>\n",
       "      <td>Not sure what the plan is today. Weather is aw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8485</th>\n",
       "      <td>[196, 27460, 333, 7, 71, 9771, 12, 12, 0, 809,...</td>\n",
       "      <td>4</td>\n",
       "      <td>@bradiewebbstack  oh my fkn god, i love supern...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8486</th>\n",
       "      <td>[1234, 238, 275, 291, 238, 12, 10390, 12, 299,...</td>\n",
       "      <td>4</td>\n",
       "      <td>i had the worst night ever last night! ughh! w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8487</th>\n",
       "      <td>[0, 0, 0, 3604, 0, 5969, 0, 578, 1831, 138, 16...</td>\n",
       "      <td>4</td>\n",
       "      <td>Watching Finding Nemo with my babies  we're he...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8488</th>\n",
       "      <td>[31, 0, 0, 12, 12, 12, 0, 7, 21, 295, 0, 0, 12...</td>\n",
       "      <td>4</td>\n",
       "      <td>@teeuh0203 http://twitpic.com/3jmku - It's my ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8489</th>\n",
       "      <td>[1735, 16431, 11057, 0, 17, 0, 800, 2725, 545,...</td>\n",
       "      <td>0</td>\n",
       "      <td>@ThisIsDavina any inside scoop on a replacemen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8490</th>\n",
       "      <td>[1354, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>yay, Me and Linn and the Globe and Green Day a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8491</th>\n",
       "      <td>[21, 21, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>@afreshmusic BORED AND SLEEPY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8492</th>\n",
       "      <td>[0, 2186, 5649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>Home with a sore throat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8493</th>\n",
       "      <td>[0, 358, 7, 4739, 41, 2332, 235, 364, 17, 0, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>@rymus that's not so bad, but will security/po...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8494</th>\n",
       "      <td>[0, 676, 185, 18323, 3376, 0, 21, 66, 676, 4, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>I'm feeling really anxious this evening... I d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8495</th>\n",
       "      <td>[0, 5925, 2822, 1415, 21, 0, 166, 2880, 3639, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>@TheConeZone Hulu content not available outsid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8496</th>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 12, 0, 865, 0, 17, 0, 90, 2...</td>\n",
       "      <td>4</td>\n",
       "      <td>@LON3WOLF Yes...it's all Aero...Glass and Glas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8497</th>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>@danceaddicted ehiiiiiiii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8498</th>\n",
       "      <td>[0, 0, 701, 692, 1106, 14691, 12, 0, 4, 425, 2...</td>\n",
       "      <td>0</td>\n",
       "      <td>@alicecooper91 Muhahahaha I'z gone mad wit pow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8499</th>\n",
       "      <td>[11340, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>@achyna yeay! Okay</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               features  sentiment  \\\n",
       "0     [0, 0, 7, 906, 7, 2361, 7, 364, 21, 0, 1870, 0...          4   \n",
       "1     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12, ...          0   \n",
       "2     [0, 128, 0, 3464, 209, 10, 366, 21, 185, 174, ...          0   \n",
       "3     [0, 7, 542, 7, 21, 1943, 526, 21, 4746, 0, 0, ...          4   \n",
       "4     [409, 300, 321, 12, 146, 7491, 0, 0, 0, 0, 0, ...          0   \n",
       "5     [21, 12, 600, 31914, 832, 12, 0, 556, 294, 221...          4   \n",
       "6     [5115, 3236, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...          0   \n",
       "7     [13327, 4169, 1083, 0, 0, 7, 344, 7, 120, 1092...          4   \n",
       "8     [0, 3289, 6258, 99, 238, 4, 0, 21, 628, 347, 4...          4   \n",
       "9     [0, 7, 735, 4, 0, 21, 644, 17, 21, 644, 0, 4, ...          4   \n",
       "10    [0, 1023, 151, 0, 1502, 676, 1207, 1424, 1023,...          0   \n",
       "11    [462, 0, 21, 180, 784, 199, 834, 500, 0, 0, 0,...          4   \n",
       "12    [4, 0, 597, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...          4   \n",
       "13    [0, 770, 66, 6800, 0, 0, 9738, 0, 0, 556, 7466...          4   \n",
       "14    [511, 1319, 21, 7513, 382, 0, 0, 0, 0, 0, 0, 0...          0   \n",
       "15    [0, 0, 7, 0, 4, 212, 212, 1865, 0, 0, 0, 0, 0,...          4   \n",
       "16    [3253, 439, 205, 2456, 9124, 98619, 221, 4326,...          0   \n",
       "17    [21, 71, 683, 0, 4, 425, 248, 80463, 86, 229, ...          4   \n",
       "18    [0, 4, 21, 189, 1744, 683, 532, 51033, 1853, 0...          4   \n",
       "19    [0, 13396, 7, 99, 5654, 128, 414, 438, 0, 0, 0...          0   \n",
       "20    [0, 1506, 12, 196, 461, 4, 222, 4, 222, 51, 51...          0   \n",
       "21    [1905, 10, 1045, 652, 5443, 0, 0, 0, 0, 0, 0, ...          0   \n",
       "22    [0, 18134, 4, 21, 1430, 133, 99, 14451, 540, 1...          4   \n",
       "23    [21, 21, 21, 21, 21, 21, 4, 0, 0, 0, 0, 0, 0, ...          4   \n",
       "24    [423, 351, 185, 19074, 4, 0, 0, 0, 0, 0, 0, 0,...          0   \n",
       "25    [0, 11223, 1004, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,...          0   \n",
       "26    [8189, 4, 7, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,...          4   \n",
       "27    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...          0   \n",
       "28    [272, 125, 910, 87, 31, 21, 119231, 0, 9612, 2...          4   \n",
       "29    [0, 7, 1175, 3069, 400, 323, 395, 7, 275, 0, 0...          4   \n",
       "...                                                 ...        ...   \n",
       "8470  [4383, 4163, 12, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0...          0   \n",
       "8471  [0, 8423, 42170, 13967, 5500, 2312, 21, 6351, ...          0   \n",
       "8472  [0, 818, 31, 476, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0...          0   \n",
       "8473  [146, 1550, 0, 138, 7, 51, 509, 248, 904, 6322...          0   \n",
       "8474  [19470, 7, 214, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0...          4   \n",
       "8475  [0, 400, 14127, 0, 0, 400, 946, 0, 0, 0, 0, 0,...          0   \n",
       "8476  [91, 5345, 12620, 91, 73407, 118, 809, 4, 1958...          4   \n",
       "8477  [2315, 7, 484408, 0, 4837, 0, 0, 0, 0, 0, 0, 0...          0   \n",
       "8478  [0, 347, 1282, 0, 21, 185, 818, 0, 0, 0, 0, 0,...          0   \n",
       "8479  [272, 0, 7, 4745, 1030, 10, 4, 0, 4, 0, 0, 0, ...          0   \n",
       "8480  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...          0   \n",
       "8481  [58635, 0, 21, 784, 683, 1739, 146, 0, 54, 76,...          4   \n",
       "8482  [0, 0, 30148, 54, 10, 2940, 0, 0, 0, 277, 0, 3...          4   \n",
       "8483  [600, 567, 289, 21, 0, 409, 804, 0, 0, 3010, 3...          0   \n",
       "8484  [0, 540, 1369, 151, 4, 0, 735, 4, 0, 214, 701,...          0   \n",
       "8485  [196, 27460, 333, 7, 71, 9771, 12, 12, 0, 809,...          4   \n",
       "8486  [1234, 238, 275, 291, 238, 12, 10390, 12, 299,...          4   \n",
       "8487  [0, 0, 0, 3604, 0, 5969, 0, 578, 1831, 138, 16...          4   \n",
       "8488  [31, 0, 0, 12, 12, 12, 0, 7, 21, 295, 0, 0, 12...          4   \n",
       "8489  [1735, 16431, 11057, 0, 17, 0, 800, 2725, 545,...          0   \n",
       "8490  [1354, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...          4   \n",
       "8491  [21, 21, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...          0   \n",
       "8492  [0, 2186, 5649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...          0   \n",
       "8493  [0, 358, 7, 4739, 41, 2332, 235, 364, 17, 0, 0...          4   \n",
       "8494  [0, 676, 185, 18323, 3376, 0, 21, 66, 676, 4, ...          0   \n",
       "8495  [0, 5925, 2822, 1415, 21, 0, 166, 2880, 3639, ...          0   \n",
       "8496  [0, 0, 0, 0, 0, 0, 12, 0, 865, 0, 17, 0, 90, 2...          4   \n",
       "8497  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...          4   \n",
       "8498  [0, 0, 701, 692, 1106, 14691, 12, 0, 4, 425, 2...          0   \n",
       "8499  [11340, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...          4   \n",
       "\n",
       "                                                  tweet  \n",
       "0     @perrinc Hi Colleen, welcome, if you have any ...  \n",
       "1     Im Well Bored  Just Sat Eyaa With Chris And Hi...  \n",
       "2     Bummer of a day...spent $360 on something I re...  \n",
       "3     @kathyfoley @kieranmurphy Gosh, is she pretty,...  \n",
       "4     @anberlin i wish i could be there tonight! i g...  \n",
       "5     @hdcgf0 OMG! another nickelback fan! Not many ...  \n",
       "6                                 havent tweeted today   \n",
       "7     ooo keeping up to date with all the e3 anounce...  \n",
       "8     Three spanish lessons in one night. Hopefully ...  \n",
       "9     @khali_blache Egads, that is awesome. Can I vo...  \n",
       "10    So cold today... kinda feeling a bit blue from...  \n",
       "11    @rajeshlalwani very nice.....  I am as happy t...  \n",
       "12                   @work. Someone's gotta do the job   \n",
       "13    Looking for a job is like an adult Easter Egg ...  \n",
       "14      had to put my dog that I grew up with to sleep   \n",
       "15    @Loman_15 Ocean City, Md. has the best of the ...  \n",
       "16    @officaltila not to be rude but do you have to...  \n",
       "17    @JimKitzmiller I love your tweets Jim. They R ...  \n",
       "18    @yat4ever Wow.  I think reading your tweets ju...  \n",
       "19    According to forecast, there will be one sunny...  \n",
       "20    2more weeks! oh damn it. =.='' I need chiong a...  \n",
       "21    @caffeinebomb it's about £35 per head and so a...  \n",
       "22    @retroasylum Lol yeaaah. A couple people but y...  \n",
       "23                      @DJ_Jonas STAY AWAY FROM PIGS.   \n",
       "24    @AetherPrincess i hope you dont  its really di...  \n",
       "25                     Life has no excitement anymore.   \n",
       "26    @VelvetMuffin don't be a douche. or do, but OW...  \n",
       "27                                    @nathalialduarte   \n",
       "28    @BarbraSundquist thanks for the new place to f...  \n",
       "29    @MissDreg  that's it now, i can kiss goodbye t...  \n",
       "...                                                 ...  \n",
       "8470             is studying for finals!  One more Day!  \n",
       "8471  My dumbass wiped the micro sd card I had in my...  \n",
       "8472                        @jstank that's sad- sorry.   \n",
       "8473  i just got this account for 2nd time, 'cause i...  \n",
       "8474                    after some pizzas, going sleep   \n",
       "8475  I'm getting your emails... you ain't getting m...  \n",
       "8476  @SarahG8s lol i wasnt on about the burns lol i...  \n",
       "8477                     off to church, alooone.. huhu   \n",
       "8478         Home morning loves...... I  an really sad   \n",
       "8479  thanks iPhone, you kept me up till 1 am. Assho...  \n",
       "8480                                       I'm over it   \n",
       "8481  @kimhollandswort Thanks....I read his tweets o...  \n",
       "8482  @MelissaNyomi Hmmm Im followin u for 2 reasons...  \n",
       "8483  another amazing video from FFXIII...wish i was...  \n",
       "8484  Not sure what the plan is today. Weather is aw...  \n",
       "8485  @bradiewebbstack  oh my fkn god, i love supern...  \n",
       "8486  i had the worst night ever last night! ughh! w...  \n",
       "8487  Watching Finding Nemo with my babies  we're he...  \n",
       "8488  @teeuh0203 http://twitpic.com/3jmku - It's my ...  \n",
       "8489  @ThisIsDavina any inside scoop on a replacemen...  \n",
       "8490  yay, Me and Linn and the Globe and Green Day a...  \n",
       "8491                     @afreshmusic BORED AND SLEEPY   \n",
       "8492                        Home with a sore throat...   \n",
       "8493  @rymus that's not so bad, but will security/po...  \n",
       "8494  I'm feeling really anxious this evening... I d...  \n",
       "8495  @TheConeZone Hulu content not available outsid...  \n",
       "8496  @LON3WOLF Yes...it's all Aero...Glass and Glas...  \n",
       "8497                         @danceaddicted ehiiiiiiii   \n",
       "8498  @alicecooper91 Muhahahaha I'z gone mad wit pow...  \n",
       "8499                                @achyna yeay! Okay   \n",
       "\n",
       "[8500 rows x 3 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.read_json(f\"{data_location}/train.json\", lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://ai2020/hwk4/sagemaker_data/train'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_location"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`sentiment_training.py`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/__init__.py:1467: The name tf.estimator.inputs is deprecated. Please use tf.compat.v1.estimator.inputs instead.\n",
      "\n",
      "Preparing for training...\n",
      "Fetching train data...\n",
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/python/data/util/random_seed.py:58: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/ec2-user/SageMaker/sentiment_dataset.py:85: DatasetV1.make_one_shot_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_one_shot_iterator(dataset)`.\n",
      "Fetching validation data...\n",
      "Fetching eval data...\n",
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "Starting training...\n",
      "Train on 85 samples, validate on ? samples\n",
      "WARNING:tensorflow:OMP_NUM_THREADS is no longer used by the default Keras config. To configure the number of threads, use tf.config.threading APIs.\n",
      "Epoch 1/10\n",
      "85/85 [==============================] - 7s 85ms/step - loss: 0.6801 - acc: 0.5627 - val_loss: 0.6443 - val_acc: 0.6390\n",
      "Epoch 2/10\n",
      "85/85 [==============================] - 6s 72ms/step - loss: 0.5485 - acc: 0.7302 - val_loss: 0.5873 - val_acc: 0.6740\n",
      "Epoch 3/10\n",
      "85/85 [==============================] - 6s 72ms/step - loss: 0.3911 - acc: 0.8296 - val_loss: 0.6540 - val_acc: 0.6910\n",
      "Epoch 4/10\n",
      "85/85 [==============================] - 6s 72ms/step - loss: 0.2752 - acc: 0.8894 - val_loss: 0.7812 - val_acc: 0.6810\n",
      "Epoch 5/10\n",
      "85/85 [==============================] - 6s 72ms/step - loss: 0.1981 - acc: 0.9201 - val_loss: 0.9219 - val_acc: 0.6790\n",
      "Epoch 6/10\n",
      "85/85 [==============================] - 6s 70ms/step - loss: 0.1437 - acc: 0.9432 - val_loss: 1.0716 - val_acc: 0.6760\n",
      "Epoch 7/10\n",
      "85/85 [==============================] - 6s 71ms/step - loss: 0.1076 - acc: 0.9571 - val_loss: 1.2040 - val_acc: 0.6650\n",
      "Epoch 8/10\n",
      "85/85 [==============================] - 6s 71ms/step - loss: 0.0841 - acc: 0.9659 - val_loss: 1.3141 - val_acc: 0.6610\n",
      "Epoch 9/10\n",
      "85/85 [==============================] - 6s 72ms/step - loss: 0.0677 - acc: 0.9728 - val_loss: 1.4584 - val_acc: 0.6640\n",
      "Epoch 10/10\n",
      "85/85 [==============================] - 6s 71ms/step - loss: 0.0595 - acc: 0.9767 - val_loss: 1.5412 - val_acc: 0.6690\n",
      "Test loss:1.3393884658813477\n",
      "Test accuracy:0.6840000152587891\n",
      "INFO:tensorflow:Assets written to: s3://ai2020/hwk4/sagemaker_data/output/sentiment_model.h5/1/assets\n",
      "Model successfully saved at: s3://ai2020/hwk4/sagemaker_data/output/sentiment_model.h5\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Main sentiment training script\n",
    "\n",
    "Author pharnoux\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import boto3\n",
    "import argparse\n",
    "import sentiment_dataset as sentiment_dataset\n",
    "import sentiment_model_cnn as sentiment_model_cnn\n",
    "import config_holder as config_holder\n",
    "\n",
    "def main(args):\n",
    "    \"\"\"\n",
    "    Main training method\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    print(\"Preparing for training...\")\n",
    "\n",
    "    training_config = config_holder.ConfigHolder(args.config_file).config\n",
    "\n",
    "    training_config[\"num_epoch\"] = args.num_epoch\n",
    "\n",
    "    train_dataset = sentiment_dataset.train_input_fn(args.train, training_config)\n",
    "    validation_dataset = sentiment_dataset.validation_input_fn(args.validation, training_config)\n",
    "    eval_dataset = sentiment_dataset.eval_input_fn(args.eval, training_config)\n",
    "\n",
    "    model = sentiment_model_cnn.keras_model_fn(None, training_config)\n",
    "\n",
    "    print(\"Starting training...\")\n",
    "\n",
    "    model.fit(\n",
    "        x=train_dataset[0], y=train_dataset[1], steps_per_epoch=train_dataset[2][\"num_batches\"],\n",
    "        epochs=training_config[\"num_epoch\"],\n",
    "        validation_data=(validation_dataset[0], validation_dataset[1]),\n",
    "        validation_steps=validation_dataset[2][\"num_batches\"])\n",
    "\n",
    "    score = model.evaluate(\n",
    "        eval_dataset[0], eval_dataset[1], steps=eval_dataset[2][\"num_batches\"], verbose=0)\n",
    "\n",
    "    print(\"Test loss:{}\".format(score[0]))\n",
    "    print(\"Test accuracy:{}\".format(score[1]))\n",
    "\n",
    "    sentiment_model_cnn.save_model(model, os.path.join(args.model_output_dir, \"sentiment_model.h5\"))\n",
    "\n",
    "def get_arg_parser():\n",
    "    \"\"\"\n",
    "    Adding this method to unit test\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument(\n",
    "        \"--train\",\n",
    "        type=str,\n",
    "        required=False,\n",
    "        default=\"training_data\",\n",
    "        help=\"The directory where the training data is stored.\")\n",
    "    parser.add_argument(\n",
    "        \"--validation\",\n",
    "        type=str,\n",
    "        required=False,\n",
    "        default=\"dev_data\",\n",
    "        help=\"The directory where the validation data is stored.\")\n",
    "    parser.add_argument(\n",
    "        \"--eval\",\n",
    "        type=str,\n",
    "        required=False,\n",
    "        default=\"eval_data\",\n",
    "        help=\"The directory where the evalutaion data is stored.\")\n",
    "    parser.add_argument(\n",
    "        \"--model_output_dir\",\n",
    "        type=str,\n",
    "        required=False,\n",
    "        default=\"s3://ai2020/hwk4/sagemaker_data/output/\")\n",
    "    parser.add_argument(\n",
    "        \"--model_dir\",\n",
    "        type=str,\n",
    "        required=False,\n",
    "        default=os.environ.get(\"\"))\n",
    "    parser.add_argument(\n",
    "        \"--num_epoch\",\n",
    "        type=int,\n",
    "        default=10,\n",
    "        help=\"The number of steps to use for training.\")\n",
    "    parser.add_argument(\n",
    "        \"--config_file\",\n",
    "        type=str,\n",
    "        default=\"training_config.json\",\n",
    "        help=\"The path to the training config file.\")\n",
    "\n",
    "    return parser\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    if '-f' in sys.argv:\n",
    "        sys.argv.remove('-f')\n",
    "    PARSER = get_arg_parser()\n",
    "    # ARGS = PARSER.parse_args()\n",
    "    ARGS, unknown = PARSER.parse_known_args()\n",
    "    main(ARGS)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "SystemExit",
     "evalue": "2",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mSystemExit\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-f618e32cc72c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'-f'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0mPARSER\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_arg_parser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m     \u001b[0mARGS\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPARSER\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mARGS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/argparse.py\u001b[0m in \u001b[0;36mparse_args\u001b[0;34m(self, args, namespace)\u001b[0m\n\u001b[1;32m   1731\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0margv\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1732\u001b[0m             \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'unrecognized arguments: %s'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1733\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1734\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1735\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/argparse.py\u001b[0m in \u001b[0;36merror\u001b[0;34m(self, message)\u001b[0m\n\u001b[1;32m   2387\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_usage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2388\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'prog'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprog\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'message'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2389\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%(prog)s: error: %(message)s\\n'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/argparse.py\u001b[0m in \u001b[0;36mexit\u001b[0;34m(self, status, message)\u001b[0m\n\u001b[1;32m   2374\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2375\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_print_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2376\u001b[0;31m         \u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2377\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2378\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mSystemExit\u001b[0m: 2"
     ]
    }
   ],
   "source": [
    "%tb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tried to use sagemaker, but access denied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}