personalize_temporal_holdout/personalize_putEvents_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using the put events api with Personalize "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The notebook largely follows the basic notebook, with the additional tweak that we show how to push new customer interactions to Personalize and how this changes recommendations made for a user. If you already have a running campaign, jump to the end of the notebook where we set up the event tracker."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, os\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['AWS_DEFAULT_REGION']=\"us-east-1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "suffix = str(np.random.uniform())[4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket = \"demo-temporal-holdout-\"+   suffix        # replace with the name of your S3 bucket\n",
    "filename = \"DEMO-temporal-holdout.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "make_bucket: demo-temporal-holdout-55242\r\n"
     ]
    }
   ],
   "source": [
    "!aws s3 mb s3://{bucket}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 46104  100 46104    0     0  1000k      0 --:--:-- --:--:-- --:--:-- 1000k\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  3320  100  3320    0     0  92222      0 --:--:-- --:--:-- --:--:-- 92222\n"
     ]
    }
   ],
   "source": [
    "!curl -O https://s3-us-west-2.amazonaws.com/personalize-cli-json-models/personalize.json\n",
    "!curl -O https://s3-us-west-2.amazonaws.com/personalize-cli-json-models/personalize-runtime.json\n",
    "!aws configure add-model --service-model file://`pwd`/personalize.json --service-name personalize\n",
    "!aws configure add-model --service-model file://`pwd`/personalize-runtime.json --service-name personalize-runtime\n",
    "\n",
    "personalize = boto3.client(service_name='personalize', endpoint_url='https://personalize.us-east-1.amazonaws.com')\n",
    "personalize_runtime = boto3.client(service_name='personalize-runtime', endpoint_url='https://personalize-runtime.us-east-1.amazonaws.com')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download and process benchmark data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 5778k  100 5778k    0     0  9912k      0 --:--:-- --:--:-- --:--:-- 9912k\n",
      "Archive:  ml-1m.zip\n",
      "   creating: ml-1m/\n",
      "  inflating: ml-1m/movies.dat        \n",
      "  inflating: ml-1m/ratings.dat       \n",
      "  inflating: ml-1m/README            \n",
      "  inflating: ml-1m/users.dat         \n"
     ]
    }
   ],
   "source": [
    "!curl -O http://files.grouplens.org/datasets/movielens/ml-1m.zip\n",
    "!unzip -o ml-1m.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USER_ID</th>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th>RATING</th>\n",
       "      <th>TIMESTAMP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000207</th>\n",
       "      <td>6040</td>\n",
       "      <td>1096</td>\n",
       "      <td>4</td>\n",
       "      <td>956715648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000208</th>\n",
       "      <td>6040</td>\n",
       "      <td>1097</td>\n",
       "      <td>4</td>\n",
       "      <td>956715569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000209 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         USER_ID  ITEM_ID  RATING  TIMESTAMP\n",
       "0              1     1193       5  978300760\n",
       "1              1      661       3  978302109\n",
       "...          ...      ...     ...        ...\n",
       "1000207     6040     1096       4  956715648\n",
       "1000208     6040     1097       4  956715569\n",
       "\n",
       "[1000209 rows x 4 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('./ml-1m/ratings.dat', sep='::', names=['USER_ID','ITEM_ID','RATING','TIMESTAMP'])\n",
    "pd.set_option('display.max_rows', 5)\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract last 10% of interactions per user as hold-out tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks = data.groupby('USER_ID').TIMESTAMP.rank(pct=True, method='first')\n",
    "data = data.join((ranks>0.9).to_frame('holdout'))\n",
    "holdout = data[data['holdout']].drop('holdout', axis=1)\n",
    "data = data[~data['holdout']].drop('holdout', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique users 6040; unique items 3706\n"
     ]
    }
   ],
   "source": [
    "# data = data[data['RATING'] > 3.6]  # Use all data to predict view recommendations\n",
    "data = data[['USER_ID', 'ITEM_ID', 'TIMESTAMP']] # select columns that match the columns in the schema below\n",
    "print('unique users %d; unique items %d'%(\n",
    "    len(data['USER_ID'].unique()), len(data['ITEM_ID'].unique())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Upload data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['EVENT_TYPE']='CLICK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "USER_ID,ITEM_ID,TIMESTAMP,EVENT_TYPE\r\n",
      "1,1193,978300760,CLICK\r\n",
      "1,661,978302109,CLICK\r\n",
      "1,914,978301968,CLICK\r\n",
      "1,3408,978300275,CLICK\r\n",
      "1,2355,978824291,CLICK\r\n",
      "1,1197,978302268,CLICK\r\n",
      "1,1287,978302039,CLICK\r\n",
      "1,2804,978300719,CLICK\r\n",
      "1,594,978302268,CLICK\r\n"
     ]
    }
   ],
   "source": [
    "data.to_csv(filename, index=False)\n",
    "!head {filename}\n",
    "\n",
    "boto3.Session().resource('s3').Bucket(bucket).Object(filename).upload_file(filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"schemaArn\": \"arn:aws:personalize:us-east-1:261294318658:schema/DEMO-temporal-schema-55242\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"95bba312-0ced-4783-86af-5dd7e3295200\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 00:47:09 GMT\",\n",
      "      \"x-amzn-requestid\": \"95bba312-0ced-4783-86af-5dd7e3295200\",\n",
      "      \"content-length\": \"92\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "schema = {\n",
    "    \"type\": \"record\",\n",
    "    \"name\": \"Interactions\",\n",
    "    \"namespace\": \"com.amazonaws.personalize.schema\",\n",
    "    \"fields\": [\n",
    "        {\n",
    "            \"name\": \"USER_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"ITEM_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"TIMESTAMP\",\n",
    "            \"type\": \"long\"\n",
    "        },\n",
    "        {\n",
    "            \"name\":\"EVENT_TYPE\",\n",
    "            \"type\":\"string\"\n",
    "        }\n",
    "    ],\n",
    "    \"version\": \"1.0\"\n",
    "}\n",
    "\n",
    "create_schema_response = personalize.create_schema(\n",
    "    name = \"DEMO-temporal-schema-\"+suffix,\n",
    "    schema = json.dumps(schema)\n",
    ")\n",
    "\n",
    "schema_arn = create_schema_response['schemaArn']\n",
    "print(json.dumps(create_schema_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and Wait for Dataset Group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetGroupArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset-group/DEMO-temporal-dataset-group-55242\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"2860ce73-d23c-4622-b6b9-bcad48d40dc0\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 00:47:39 GMT\",\n",
      "      \"x-amzn-requestid\": \"2860ce73-d23c-4622-b6b9-bcad48d40dc0\",\n",
      "      \"content-length\": \"112\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_dataset_group_response = personalize.create_dataset_group(\n",
    "    name = \"DEMO-temporal-dataset-group-\"+suffix\n",
    ")\n",
    "\n",
    "dataset_group_arn = create_dataset_group_response['datasetGroupArn']\n",
    "print(json.dumps(create_dataset_group_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetGroup: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_group_response = personalize.describe_dataset_group(\n",
    "        datasetGroupArn = dataset_group_arn\n",
    "    )\n",
    "    status = describe_dataset_group_response[\"datasetGroup\"][\"status\"]\n",
    "    print(\"DatasetGroup: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset/DEMO-temporal-dataset-group-55242/INTERACTIONS\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"225b786c-05a4-40b3-8bb4-e716f26c5a6e\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 00:47:46 GMT\",\n",
      "      \"x-amzn-requestid\": \"225b786c-05a4-40b3-8bb4-e716f26c5a6e\",\n",
      "      \"content-length\": \"114\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset_type = \"INTERACTIONS\"\n",
    "create_dataset_response = personalize.create_dataset(\n",
    "    datasetType = dataset_type,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    schemaArn = schema_arn,\n",
    "    name = \"DEMO-temporal-dataset-\"+suffix\n",
    ")\n",
    "\n",
    "dataset_arn = create_dataset_response['datasetArn']\n",
    "print(json.dumps(create_dataset_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare, Create, and Wait for Dataset Import Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.client(\"s3\")\n",
    "\n",
    "policy = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Id\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Sid\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Principal\": {\n",
    "                \"Service\": \"personalize.amazonaws.com\"\n",
    "            },\n",
    "            \"Action\": [\n",
    "                \"s3:GetObject\",\n",
    "                \"s3:ListBucket\"\n",
    "            ],\n",
    "            \"Resource\": [\n",
    "                \"arn:aws:s3:::{}\".format(bucket),\n",
    "                \"arn:aws:s3:::{}/*\".format(bucket)\n",
    "            ]\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "s3.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create S3 Read-Only Access Role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from botocore.exceptions import ClientError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:iam::261294318658:role/PersonalizeS3Role-55242\n"
     ]
    }
   ],
   "source": [
    "iam = boto3.client(\"iam\")\n",
    "\n",
    "role_name = \"PersonalizeS3Role-\"+suffix\n",
    "assume_role_policy_document = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "          \"Effect\": \"Allow\",\n",
    "          \"Principal\": {\n",
    "            \"Service\": \"personalize.amazonaws.com\"\n",
    "          },\n",
    "          \"Action\": \"sts:AssumeRole\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "try:\n",
    "    create_role_response = iam.create_role(\n",
    "        RoleName = role_name,\n",
    "        AssumeRolePolicyDocument = json.dumps(assume_role_policy_document)\n",
    "    );\n",
    "\n",
    "    iam.attach_role_policy(\n",
    "        RoleName = role_name,\n",
    "        PolicyArn = \"arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess\"\n",
    "    );\n",
    "\n",
    "    role_arn = create_role_response[\"Role\"][\"Arn\"]\n",
    "except ClientError as e:\n",
    "    if e.response['Error']['Code'] == 'EntityAlreadyExists':\n",
    "        role_arn = iam.get_role(RoleName=role_name)['Role']['Arn']\n",
    "    else:\n",
    "        raise\n",
    "print(role_arn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetImportJobArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset-import-job/DEMO-temporal-dataset-import-job-55242\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"4dbc1315-67ff-4df4-88ac-7e155ab341c9\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 00:49:18 GMT\",\n",
      "      \"x-amzn-requestid\": \"4dbc1315-67ff-4df4-88ac-7e155ab341c9\",\n",
      "      \"content-length\": \"126\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_dataset_import_job_response = personalize.create_dataset_import_job(\n",
    "    jobName = \"DEMO-temporal-dataset-import-job-\"+suffix,\n",
    "    datasetArn = dataset_arn,\n",
    "    dataSource = {\n",
    "        \"dataLocation\": \"s3://{}/{}\".format(bucket, filename)\n",
    "    },\n",
    "    roleArn = role_arn\n",
    ")\n",
    "\n",
    "dataset_import_job_arn = create_dataset_import_job_response['datasetImportJobArn']\n",
    "print(json.dumps(create_dataset_import_job_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Dataset Import Job and Dataset Import Job Run to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_import_job_response = personalize.describe_dataset_import_job(\n",
    "        datasetImportJobArn = dataset_import_job_arn\n",
    "    )\n",
    "    \n",
    "    dataset_import_job = describe_dataset_import_job_response[\"datasetImportJob\"]\n",
    "    if \"latestDatasetImportJobRun\" not in dataset_import_job:\n",
    "        status = dataset_import_job[\"status\"]\n",
    "        print(\"DatasetImportJob: {}\".format(status))\n",
    "    else:\n",
    "        status = dataset_import_job[\"latestDatasetImportJobRun\"][\"status\"]\n",
    "        print(\"LatestDatasetImportJobRun: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:personalize:::recipe/aws-hrnn\n",
      "arn:aws:personalize:::recipe/aws-hrnn-coldstart\n",
      "arn:aws:personalize:::recipe/aws-hrnn-metadata\n",
      "arn:aws:personalize:::recipe/aws-personalized-ranking\n",
      "arn:aws:personalize:::recipe/aws-popularity-count\n",
      "arn:aws:personalize:::recipe/aws-sims\n"
     ]
    }
   ],
   "source": [
    "recipe_list = personalize.list_recipes()\n",
    "for recipe in recipe_list['recipes']:\n",
    "    print(recipe['recipeArn'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many recipes for different scenarios. In this example, we only have interactions data, so we will choose one from the basic recipes.\n",
    "\n",
    "| Feasible? | Recipe | Description \n",
    "|-------- | -------- |:------------\n",
    "| Y | aws-popularity-count | Calculates popularity of items based on count of events against that item in user-item interactions dataset.\n",
    "| Y | aws-hrnn | Predicts items a user will interact with. A hierarchical recurrent neural network which can model the temporal order of user-item interactions.\n",
    "| N - requires meta data | aws-hrnn-metadata | Predicts items a user will interact with. HRNN with additional features derived from contextual (user-item interaction metadata), user medata (user dataset) and item metadata (item dataset)\n",
    "| N - for bandits and requires meta data | aws-hrnn-coldstart | Predicts items a user will interact with. HRNN-metadata with with personalized exploration of new items.\n",
    "| N - for item-based queries | aws-sims | Computes items similar to a given item based on co-occurrence of item in same user history in user-item interaction dataset\n",
    "| N - for reranking a short list | aws-personalized-ranking | Reranks a list of items for a user. Trains on user-item interactions dataset. \n",
    "\n",
    "\n",
    "We (or autoML) can run all of these basic recipes and choose the best-performing model from internal metrics. We recommend comparisons, especially with popularity-baseline, to see the lifts in metrics via personalization. However, in this demo, we will pick one recipe - aws-hrnn, to focus on external evaluations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "recipe_arn = \"arn:aws:personalize:::recipe/aws-hrnn\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-temporal-solution-55242\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"88955570-16b9-495d-bdc7-45a02b29872f\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 01:04:25 GMT\",\n",
      "      \"x-amzn-requestid\": \"88955570-16b9-495d-bdc7-45a02b29872f\",\n",
      "      \"content-length\": \"98\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_response = personalize.create_solution(\n",
    "    name = \"DEMO-temporal-solution-\"+suffix,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    recipeArn = recipe_arn,\n",
    ")\n",
    "\n",
    "solution_arn = create_solution_response['solutionArn']\n",
    "print(json.dumps(create_solution_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-temporal-solution-55242/32e7870d\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"0ec7530f-c36c-4b43-9f3a-7f07df65a6a8\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 01:04:25 GMT\",\n",
      "      \"x-amzn-requestid\": \"0ec7530f-c36c-4b43-9f3a-7f07df65a6a8\",\n",
      "      \"content-length\": \"114\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_version_response = personalize.create_solution_version(\n",
    "    solutionArn = solution_arn\n",
    ")\n",
    "\n",
    "solution_version_arn = create_solution_version_response['solutionVersionArn']\n",
    "print(json.dumps(create_solution_version_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Solution Version to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SolutionVersion: CREATE PENDING\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_solution_version_response = personalize.describe_solution_version(\n",
    "        solutionVersionArn = solution_version_arn\n",
    "    )\n",
    "    status = describe_solution_version_response[\"solutionVersion\"][\"status\"]\n",
    "    print(\"SolutionVersion: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Metrics of Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-temporal-solution-55242/32e7870d\",\n",
      "  \"metrics\": {\n",
      "    \"coverage\": 0.5603,\n",
      "    \"mean_reciprocal_rank_at_25\": 0.0816,\n",
      "    \"normalized_discounted_cumulative_gain_at_10\": 0.1132,\n",
      "    \"normalized_discounted_cumulative_gain_at_25\": 0.1479,\n",
      "    \"normalized_discounted_cumulative_gain_at_5\": 0.0926,\n",
      "    \"precision_at_10\": 0.018,\n",
      "    \"precision_at_25\": 0.0129,\n",
      "    \"precision_at_5\": 0.0243\n",
      "  },\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"37cbdba9-65cc-442c-be44-e962ac466e3e\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 01:51:36 GMT\",\n",
      "      \"x-amzn-requestid\": \"37cbdba9-65cc-442c-be44-e962ac466e3e\",\n",
      "      \"content-length\": \"411\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "get_metrics_response = personalize.get_solution_metrics(\n",
    "    solutionVersionArn = solution_version_arn\n",
    ")\n",
    "\n",
    "print(json.dumps(get_metrics_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and Wait for Campaign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"campaignArn\": \"arn:aws:personalize:us-east-1:261294318658:campaign/DEMO-temporal-campaign-55242\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"731226a5-22f1-4265-be97-093f140d1f2d\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 01:51:37 GMT\",\n",
      "      \"x-amzn-requestid\": \"731226a5-22f1-4265-be97-093f140d1f2d\",\n",
      "      \"content-length\": \"98\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_campaign_response = personalize.create_campaign(\n",
    "    name = \"DEMO-temporal-campaign-\"+suffix,\n",
    "    solutionVersionArn = solution_version_arn,\n",
    "    minProvisionedTPS = 2,    \n",
    ")\n",
    "\n",
    "campaign_arn = create_campaign_response['campaignArn']\n",
    "print(json.dumps(create_campaign_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Campaign to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Campaign: CREATE PENDING\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_campaign_response = personalize.describe_campaign(\n",
    "        campaignArn = campaign_arn\n",
    "    )\n",
    "    status = describe_campaign_response[\"campaign\"][\"status\"]\n",
    "    print(\"Campaign: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate using external metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An explanation of the evaluation metrics are provided at https://docs.aws.amazon.com/personalize/latest/dg/working-with-training-metrics.html\n",
    "\n",
    "For example, suppose we recommend four items and two of them are relevant, $r=[0,1,0,1]$. In this case, the metrics are:\n",
    "\n",
    "|Name\t|Example\t|Explanation\n",
    "|:------|:----------|:----------\n",
    "|Precision@K\t|$\\frac{2}{4} = 0.5$\t|Total relevant items divided by total recommended items.\n",
    "|Mean reciprocal ranks (MRR@K)\t|${\\rm mean}(\\frac{1}{2} + \\frac{1}{4}) = 0.375$\t|Considers positional effects by computing the mean of the inverse positions of all relevant items.\n",
    "|Normalized discounted cumulative gains (NDCG@K)\t|$\\frac{\\frac{1}{\\log(1 + 2)} + \\frac{1}{\\log(1 + 4)}}{\\frac{1}{\\log(1 + 1)} + \\frac{1}{\\log(1 + 2)}} = 0.65$\t|Considers positional effects by applying inverse logarithmic weights based on the positions of relevant items, normalized by the largest possible scores from ideal recommendations.\n",
    "|Average precision (AP@K)\t|${\\rm mean}(\\frac{1}{2} + \\frac{2}{4}) = 0.5$\t|Average precision@K where K is the position of every relevant item."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These metrics are different from the internal metrics in two aspects:\n",
    "* They are evaluated at different times, which may imply different click rates. We recommend to always keep the evaluations in the same time periods to avoid temporal drifts.\n",
    "* The example external evaluations may hold out and consider multiple items as ground truth, while the internal evaluations only hold out the last item in each user-history as the ground truth. There is no absolute preference as to how many items should be held out; we recommend designing the evaluation methods that are similar to the actual use case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "import numpy as np\n",
    "from metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fad42f7f64024670a9fb295e139fa848",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6040), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "relevance = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user_id)\n",
    "    )\n",
    "    rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "    relevance.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reciprocal_rank 0.145002645054113\n",
      "precision_at_5 0.11493377483443709\n",
      "precision_at_10 0.10456953642384106\n",
      "precision_at_25 0.08750993377483444\n",
      "normalized_discounted_cumulative_gain_at_5 0.177783170055624\n",
      "normalized_discounted_cumulative_gain_at_10 0.2378382874621963\n",
      "normalized_discounted_cumulative_gain_at_25 0.3516037599383753\n"
     ]
    }
   ],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in relevance]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in relevance]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in relevance]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in relevance]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optional: slightly better results after deduplicating previous purchase histories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2ab81331b7e43eb9fdba8bda0dec03a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6040), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "rel_dedup = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user_id)\n",
    "    )\n",
    "    past_items = data[data.USER_ID == user_id].ITEM_ID.values\n",
    "    topk = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "    rec_items = [x for x in topk if x not in past_items]\n",
    "    if len(rec_items) < 25:\n",
    "        rec_items.extend([x for x in topk if x not in rec_items])\n",
    "    rec_items = rec_items[:25]    \n",
    "\n",
    "    rel_dedup.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reciprocal_rank 0.18802986426679386\n",
      "precision_at_5 0.16082781456953643\n",
      "precision_at_10 0.14276490066225164\n",
      "precision_at_25 0.08750993377483444\n",
      "normalized_discounted_cumulative_gain_at_5 0.2499630328682448\n",
      "normalized_discounted_cumulative_gain_at_10 0.32913706965420175\n",
      "normalized_discounted_cumulative_gain_at_25 0.40842287268608907\n"
     ]
    }
   ],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in rel_dedup]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in rel_dedup]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in rel_dedup]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in rel_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in rel_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in rel_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in rel_dedup]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try comparing with popularity baseline as a dummy recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "topk = data.groupby(\"ITEM_ID\").TIMESTAMP.count().sort_values(ascending=False).iloc[:100].index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dd45bfad704496cbcba8bd8b3bb302e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6040), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "rel_popular = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    rec_items = topk[:25]\n",
    "    rel_popular.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reciprocal_rank 0.05072304507652334\n",
      "precision_at_5 0.01980132450331126\n",
      "precision_at_10 0.020066225165562918\n",
      "precision_at_25 0.021735099337748344\n",
      "normalized_discounted_cumulative_gain_at_5 0.043792523162000045\n",
      "normalized_discounted_cumulative_gain_at_10 0.06552215768085591\n",
      "normalized_discounted_cumulative_gain_at_25 0.12216764300043037\n"
     ]
    }
   ],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in rel_popular]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in rel_popular]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in rel_popular]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in rel_popular]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in rel_popular]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in rel_popular]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in rel_popular]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Popularity baseline deduplicating user histories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c88bbd71ebbb47609d291aeebf4cf373",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6040), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "rel_pop_dedup = []\n",
    "for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):\n",
    "    past_items = data[data.USER_ID == user_id].ITEM_ID.values\n",
    "    rec_items = [x for x in topk if x not in past_items]\n",
    "    if len(rec_items) < 25:\n",
    "        rec_items.extend([x for x in topk if x not in rec_items])\n",
    "    rec_items = rec_items[:25]    \n",
    "    rel_pop_dedup.append([int(x in true_items.values) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reciprocal_rank 0.08990051372135084\n",
      "precision_at_5 0.06321192052980133\n",
      "precision_at_10 0.055993377483443706\n",
      "precision_at_25 0.04464238410596026\n",
      "normalized_discounted_cumulative_gain_at_5 0.10508158681030984\n",
      "normalized_discounted_cumulative_gain_at_10 0.14086321546662076\n",
      "normalized_discounted_cumulative_gain_at_25 0.21005097965654965\n"
     ]
    }
   ],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in rel_pop_dedup]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in rel_pop_dedup]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in rel_pop_dedup]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in rel_pop_dedup]))\n",
    "print('normalized_discounted_cumulatixvgive_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in rel_pop_dedup]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in rel_pop_dedup]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## we now set up an event tracker where we push new interactions for a user, and see changes in recommendations made by the system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!curl -O https://s3-us-west-2.amazonaws.com/personalize-cli-json-models/personalize-events.json\n",
    "!aws configure add-model --service-name personalize-events --service-model file://`pwd`/personalize-events.json\n",
    "    personalize_events = boto3.client(service_name='personalize-events', endpoint_url='https://personalize-events.us-east-1.amazonaws.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:personalize:us-east-1:261294318658:event-tracker/efcbc840\n",
      "c9fcae63-c431-4617-b84f-3311e91f158c\n"
     ]
    }
   ],
   "source": [
    "response = personalize.create_event_tracker(\n",
    " name='demoClickTracker-'+suffix,\n",
    " datasetGroupArn=dataset_group_arn,\n",
    "    roleArn=role_arn\n",
    ")\n",
    "print(response['eventTrackerArn'])\n",
    "print(response['trackingId'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"eventTrackerArn\": \"arn:aws:personalize:us-east-1:261294318658:event-tracker/efcbc840\",\n",
      "  \"trackingId\": \"c9fcae63-c431-4617-b84f-3311e91f158c\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"2116462c-ecad-41ca-9317-6587bb2a395b\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 08 Jun 2019 02:03:57 GMT\",\n",
      "      \"x-amzn-requestid\": \"2116462c-ecad-41ca-9317-6587bb2a395b\",\n",
      "      \"content-length\": \"139\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(response,indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "event_tracker_arn = response['eventTrackerArn']\n",
    "tracking_id = response['trackingId']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## we pick two users (user 10 and user 15 in this case) and see what recommendations we get"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:7: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Silence of the Lambs, The (1991)',\n",
       "       'Star Wars: Episode IV - A New Hope (1977)',\n",
       "       'Sixth Sense, The (1999)',\n",
       "       'Star Wars: Episode V - The Empire Strikes Back (1980)',\n",
       "       'Lost World: Jurassic Park, The (1997)', 'American Beauty (1999)',\n",
       "       'Ace Ventura: Pet Detective (1994)', \"Schindler's List (1993)\",\n",
       "       'Raiders of the Lost Ark (1981)', 'Wizard of Oz, The (1939)',\n",
       "       'Casablanca (1942)', 'Fargo (1996)', 'Back to the Future (1985)',\n",
       "       'Godfather, The (1972)', 'Best in Show (2000)',\n",
       "       'Austin Powers: The Spy Who Shagged Me (1999)',\n",
       "       'Back to the Future Part III (1990)',\n",
       "       'Mission: Impossible 2 (2000)', 'Almost Famous (2000)',\n",
       "       'L.A. Confidential (1997)', 'Braveheart (1995)',\n",
       "       'Citizen Kane (1941)', 'Shawshank Redemption, The (1994)',\n",
       "       'Shakespeare in Love (1998)', 'Alien (1979)'], dtype=object)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user = 10\n",
    "rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user)\n",
    "    )\n",
    "rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "movies = pd.read_csv('./ml-1m/movies.dat',sep='::',names=['ITEM_ID','Name','Genre']) \n",
    "movies=movies.set_index('ITEM_ID')\n",
    "seen_items1 = data[data['USER_ID']==user]['ITEM_ID']\n",
    "seen_items1 = seen_items1.values\n",
    "movies.loc[rec_items]['Name'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Titan A.E. (2000)', 'Return to Me (2000)', 'Space Cowboys (2000)',\n",
       "       'Shaft (2000)', 'Mission: Impossible 2 (2000)', 'Gladiator (2000)',\n",
       "       'Mission: Impossible (1996)', 'Frequency (2000)',\n",
       "       'Shanghai Noon (2000)', 'Scary Movie (2000)',\n",
       "       'Me, Myself and Irene (2000)', 'Rules of Engagement (2000)',\n",
       "       'Perfect Storm, The (2000)', 'U-571 (2000)',\n",
       "       'Usual Suspects, The (1995)', \"Schindler's List (1993)\",\n",
       "       'Fargo (1996)', 'Remember the Titans (2000)',\n",
       "       'Gone in 60 Seconds (2000)',\n",
       "       'Ghost Dog: The Way of the Samurai (1999)',\n",
       "       'Romeo Must Die (2000)', 'Erin Brockovich (2000)',\n",
       "       'Pitch Black (2000)', 'What Lies Beneath (2000)',\n",
       "       'Maltese Falcon, The (1941)'], dtype=object)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user = 15\n",
    "rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user)\n",
    "    )\n",
    "rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "seen_items2 = data[data['USER_ID']==user]['ITEM_ID']\n",
    "seen_items2 = seen_items2.values\n",
    "\n",
    "movies.loc[rec_items]['Name'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## now we enter items from the user 10 as clicks for user 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = int(time.time())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(seen_items1)):\n",
    "\n",
    "    personalize_events.put_events(\n",
    "     trackingId = tracking_id,\n",
    "     userId= '15',\n",
    "     sessionId = '1',\n",
    "     \n",
    "     eventList = [{\n",
    "     'sentAt': start+i*1000,\n",
    "     'properties': \"{\\\"itemId\\\":\\\"\"+  str(i) + \"\\\"}\",\n",
    "    'eventType' : 'CLICK',\n",
    "     }]\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## we see that the recommendations for user 15 and now similar to the ones we got for user 10, as the model learns the 'change in tastes'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Titanic (1997)', 'Mask, The (1994)', 'High Fidelity (2000)',\n",
       "       'Mission: Impossible 2 (2000)', 'Ghost (1990)',\n",
       "       \"There's Something About Mary (1998)\", 'Gladiator (2000)',\n",
       "       'Chicken Run (2000)', 'American Pie (1999)',\n",
       "       'Talented Mr. Ripley, The (1999)',\n",
       "       'Thomas Crown Affair, The (1999)', 'Big Lebowski, The (1998)',\n",
       "       'League of Their Own, A (1992)', 'Mission: Impossible (1996)',\n",
       "       'Kingpin (1996)', 'Perfect Storm, The (2000)', 'Scream (1996)',\n",
       "       'Mystery Men (1999)', 'Forrest Gump (1994)',\n",
       "       'Seven (Se7en) (1995)',\n",
       "       'Austin Powers: The Spy Who Shagged Me (1999)',\n",
       "       'Ace Ventura: Pet Detective (1994)', 'My Cousin Vinny (1992)',\n",
       "       'Chinatown (1974)', 'Clear and Present Danger (1994)'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user = 15\n",
    "rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user)\n",
    "    )\n",
    "rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "seen_items2 = data[data['USER_ID']==user]['ITEM_ID']\n",
    "seen_items2 = seen_items2.values\n",
    "\n",
    "movies.loc[rec_items]['Name'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the predictions are not exactly the same, since the recommendations depend on the entire history not just the last few items."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}