1.2-data-processing-users.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4a6ab610-d302-4ab0-ba0a-6bc3ca3c5e4c",
   "metadata": {},
   "source": [
    "## Collaborative Recommender \n",
    "based on user behaviour \n",
    "\n",
    "Use implicit package for python\n",
    "https://implicit.readthedocs.io/en/latest/models.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "2b6bb936-3405-46b7-9795-694d6c6945b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.neighbors import NearestNeighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "189df4e5-60de-4b8f-9a9a-247d680a5640",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('data/users_projects_0-100.csv')\n",
    "df2 = pd.read_csv('data/users_projects_100-300.csv')\n",
    "df3 = pd.read_csv('data/users_projects_300-500.csv')\n",
    "df4 = pd.read_csv('data/users_projects_500-700.csv')\n",
    "df5 = pd.read_csv('data/users_projects_700-900.csv')\n",
    "df6 = pd.read_csv('data/users_projects_900-1100.csv')\n",
    "df7 = pd.read_csv('data/users_projects_1100-1300.csv')\n",
    "df8 = pd.read_csv('data/users_projects_1300-1500.csv')\n",
    "df9 = pd.read_csv('data/users_projects_1500-1700.csv')# this one will need to be redone\n",
    "df10 = pd.read_csv('data/users_projects_1700-1900.csv')\n",
    "df11 = pd.read_csv('data/users_projects_1887-1900.csv')\n",
    "df12 = pd.read_csv('data/users_projects_1900-2100.csv')\n",
    "df13 = pd.read_csv('data/users_projects_2100-2300.csv')\n",
    "df14 = pd.read_csv('data/users_projects_2300-2500.csv')\n",
    "df15 = pd.read_csv('data/users_projects_2500-2700.csv')\n",
    "df16 = pd.read_csv('data/users_projects_2700-2900.csv')\n",
    "df17 = pd.read_csv('data/users_projects_2900-2962.csv')\n",
    "\n",
    "pattern_details = pd.read_csv('data/consolidated_patterns.csv', low_memory=False)\n",
    "# this needs to be repaced with cleaned data?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38022834-0427-4e6c-b691-fe5f53b9ba3b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "id": "ef984fed-39af-4572-a5db-65c335cef2e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((343032, 4),\n",
       " (386356, 4),\n",
       " (248913, 4),\n",
       " (210382, 4),\n",
       " (171374, 4),\n",
       " (161808, 4),\n",
       " (281092, 4),\n",
       " (279575, 4),\n",
       " (596740, 4),\n",
       " (813227, 4),\n",
       " (35527, 4),\n",
       " (366425, 4),\n",
       " (298682, 4),\n",
       " (252212, 4),\n",
       " (220635, 4),\n",
       " (185410, 4),\n",
       " (44375, 4))"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df1.shape, df2.shape, df3.shape, df4.shape, \n",
    " df5.shape, df6.shape, df7.shape, df8.shape,\n",
    " df9.shape, df10.shape, df11.shape,df12.shape,\n",
    " df13.shape, df14.shape, df15.shape,df16.shape, df17.shape\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "74e78ae9-6f99-4422-be71-5cac8e0192a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4895765, 4)"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For full set\n",
    "df = df1.append(df2, ignore_index=True)\n",
    "df = df.append(df3, ignore_index=True)\n",
    "df = df.append(df4, ignore_index=True)\n",
    "df = df.append(df5, ignore_index=True)\n",
    "df = df.append(df6, ignore_index=True)\n",
    "df = df.append(df7, ignore_index=True)\n",
    "df = df.append(df8, ignore_index=True)\n",
    "df = df.append(df9, ignore_index=True)\n",
    "df = df.append(df10, ignore_index=True)\n",
    "df = df.append(df11, ignore_index=True)\n",
    "df = df.append(df12, ignore_index=True)\n",
    "df = df.append(df13, ignore_index=True)\n",
    "df = df.append(df14, ignore_index=True)\n",
    "df = df.append(df15, ignore_index=True)\n",
    "df = df.append(df16, ignore_index=True)\n",
    "df = df.append(df17, ignore_index=True)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "564411ef-0f16-48cf-9f7b-cf26213150bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # use these plots from https://towardsdatascience.com/building-and-testing-recommender-systems-with-surprise-step-by-step-d4ba702ef80b\n",
    "# # Number of ratings per user\n",
    "# data = df.groupby('userID')['bookRating'].count().clip(upper=50)\n",
    "\n",
    "# # Create trace\n",
    "# trace = go.Histogram(x = data.values,\n",
    "#                      name = 'Ratings',\n",
    "#                      xbins = dict(start = 0,\n",
    "#                                   end = 50,\n",
    "#                                   size = 2))\n",
    "# # Create layout\n",
    "# layout = go.Layout(title = 'Distribution Of Number of Ratings Per User (Clipped at 50)',\n",
    "#                    xaxis = dict(title = 'Ratings Per User'),\n",
    "#                    yaxis = dict(title = 'Count'),\n",
    "#                    bargap = 0.2)\n",
    "\n",
    "# # Create plot\n",
    "# fig = go.Figure(data=[trace], layout=layout)\n",
    "# iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "9375a132-159d-4120-b729-d3bc74c07910",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean data and save\n",
    "\n",
    "# remove accidental headers\n",
    "df = df.drop(df[df['user_id']=='user_id'].index)\n",
    "\n",
    "# remove douplicates\n",
    "df = df.drop_duplicates()\n",
    "\n",
    "# reset index\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "\n",
    "# df.to_csv('data/user_projects_cleaned.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "560fa045-acb6-4fd5-a41a-7814cd6c58d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4478010, 4)"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "960d2a86-4f86-468c-8d53-35d3479506ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4478010, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>pattern_ids</th>\n",
       "      <th>date_completed</th>\n",
       "      <th>project_id</th>\n",
       "      <th>has_knit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5153610</td>\n",
       "      <td>88</td>\n",
       "      <td>2021/11/19</td>\n",
       "      <td>29203938</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3183664</td>\n",
       "      <td>88</td>\n",
       "      <td>2021/11/18</td>\n",
       "      <td>29195691</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3183664</td>\n",
       "      <td>88</td>\n",
       "      <td>2021/11/17</td>\n",
       "      <td>29190583</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2102666</td>\n",
       "      <td>88</td>\n",
       "      <td>2021/11/17</td>\n",
       "      <td>29177512</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3183664</td>\n",
       "      <td>88</td>\n",
       "      <td>2021/11/16</td>\n",
       "      <td>29187197</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id pattern_ids date_completed project_id  has_knit\n",
       "0  5153610          88     2021/11/19   29203938         1\n",
       "1  3183664          88     2021/11/18   29195691         1\n",
       "2  3183664          88     2021/11/17   29190583         1\n",
       "3  2102666          88     2021/11/17   29177512         1\n",
       "4  3183664          88     2021/11/16   29187197         1"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add 'has_knit' column is place of rating (using implicit rather than explicit rating)\n",
    "df['has_knit'] = 1\n",
    "print(df.shape)\n",
    "df.head()\n",
    "\n",
    "# rename uid and iid as per utility matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96ff935a-58c1-4617-85e9-91fcb01a6526",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3250978, 5)"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop douplicate knits per pattern\n",
    "df = df.drop_duplicates(['user_id', 'pattern_ids'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "855ee7b6-ec46-48f7-a70b-c1faf66532c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>pattern_ids</th>\n",
       "      <th>date_completed</th>\n",
       "      <th>project_id</th>\n",
       "      <th>has_knit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>61820</th>\n",
       "      <td>17244</td>\n",
       "      <td>302</td>\n",
       "      <td>298</td>\n",
       "      <td>302</td>\n",
       "      <td>302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273983</th>\n",
       "      <td>698879</td>\n",
       "      <td>261</td>\n",
       "      <td>254</td>\n",
       "      <td>261</td>\n",
       "      <td>261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253153</th>\n",
       "      <td>616629</td>\n",
       "      <td>260</td>\n",
       "      <td>258</td>\n",
       "      <td>260</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296299</th>\n",
       "      <td>79380</td>\n",
       "      <td>256</td>\n",
       "      <td>105</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111658</th>\n",
       "      <td>24654</td>\n",
       "      <td>239</td>\n",
       "      <td>229</td>\n",
       "      <td>239</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254489</th>\n",
       "      <td>6218491</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254488</th>\n",
       "      <td>6218465</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160108</th>\n",
       "      <td>342361</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85382</th>\n",
       "      <td>204944</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338750</th>\n",
       "      <td>9999967</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>338751 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  pattern_ids  date_completed  project_id  has_knit\n",
       "61820     17244          302             298         302       302\n",
       "273983   698879          261             254         261       261\n",
       "253153   616629          260             258         260       260\n",
       "296299    79380          256             105         256       256\n",
       "111658    24654          239             229         239       239\n",
       "...         ...          ...             ...         ...       ...\n",
       "254489  6218491            1               1           1         1\n",
       "254488  6218465            1               1           1         1\n",
       "160108   342361            1               0           1         1\n",
       "85382    204944            1               1           1         1\n",
       "338750  9999967            1               1           1         1\n",
       "\n",
       "[338751 rows x 5 columns]"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how many users, how many patterns?\n",
    "pattern_count_per_user = df.groupby(\"user_id\", as_index=False).count().sort_values('has_knit', ascending = False)\n",
    "pattern_count_per_user\n",
    "\n",
    "# uh oh - how many are 0?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5961b1a-3d92-4174-9374-39f2538bd870",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pattern_ids</th>\n",
       "      <th>date_completed</th>\n",
       "      <th>project_id</th>\n",
       "      <th>has_knit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>345059.000000</td>\n",
       "      <td>345059.000000</td>\n",
       "      <td>345059.000000</td>\n",
       "      <td>345059.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>12.977520</td>\n",
       "      <td>10.615164</td>\n",
       "      <td>12.977520</td>\n",
       "      <td>12.977520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>22.736905</td>\n",
       "      <td>21.070589</td>\n",
       "      <td>22.736905</td>\n",
       "      <td>22.736905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>14.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         pattern_ids  date_completed     project_id       has_knit\n",
       "count  345059.000000   345059.000000  345059.000000  345059.000000\n",
       "mean       12.977520       10.615164      12.977520      12.977520\n",
       "std        22.736905       21.070589      22.736905      22.736905\n",
       "min         1.000000        0.000000       1.000000       1.000000\n",
       "25%         2.000000        1.000000       2.000000       2.000000\n",
       "50%         4.000000        3.000000       4.000000       4.000000\n",
       "75%        14.000000       11.000000      14.000000      14.000000\n",
       "max       720.000000      720.000000     720.000000     720.000000"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern_count_per_user.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "c3ab4bf4-bc4f-4b15-a92d-37f188296bb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmMAAAFlCAYAAACnee/9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAcIElEQVR4nO3df4xe1X3n8fendkLIDwiQAbE2qomwsgtoQ4KF6GYVdeO0OE0V8wdIEynFWrnyCrHdZHelymyljfqHpbBalS7SgoRCiqFtwHWTxUpKN8g0qlZCJkNCCoZ4mQQCXhw8DYTQVpCafveP50zyeHg889jYnPnxfklX9z7f55zrc4/A+viee2dSVUiSJKmPX+o9AEmSpJXMMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdre49gBP1vve9r9atW9d7GJIkSQt65JFH/raqJkZ9t2TD2Lp165iamuo9DEmSpAUl+eGxvnOZUpIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdGcYkSZI6MozNY932r/cegiRJWuYMY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSOxgpjSf5jkv1JHk/y5STvSHJ2kgeSPNX2Zw21vzHJdJIDSa4aql+e5LH23S1J0uqnJbm31fclWXfSr1SSJGkRWjCMJVkD/AdgQ1VdCqwCJoHtwN6qWg/sbZ9JcnH7/hJgE3BrklXtdLcB24D1bdvU6luBl6rqIuBm4KaTcnWSJEmL3LjLlKuB05OsBt4JPA9sBna273cCV7fjzcA9VfVaVT0NTANXJDkfOKOqHqqqAu6a02f2XLuBjbN3zSRJkpazBcNYVf0/4L8DzwKHgJer6hvAeVV1qLU5BJzbuqwBnhs6xcFWW9OO59aP6lNVR4CXgXNO7JIkSZKWjnGWKc9icOfqQuCfAe9K8pn5uoyo1Tz1+frMHcu2JFNJpmZmZuYfuCRJ0hIwzjLlx4Gnq2qmqv4R+Arwr4AX2tIjbX+4tT8IXDDUfy2DZc2D7Xhu/ag+bSn0TODFuQOpqturakNVbZiYmBjvCiVJkhaxccLYs8CVSd7ZnuPaCDwJ7AG2tDZbgPva8R5gsr0heSGDB/UfbkuZryS5sp3nujl9Zs91DfBge65MkiRpWVu9UIOq2pdkN/Bt4AjwHeB24N3AriRbGQS2a1v7/Ul2AU+09jdU1evtdNcDdwKnA/e3DeAO4O4k0wzuiE2elKuTJEla5BYMYwBV9Xng83PKrzG4Szaq/Q5gx4j6FHDpiPqrtDAnSZK0kvgT+CVJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1ZBiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktTRgmEsyQeSPDq0/TTJ55KcneSBJE+1/VlDfW5MMp3kQJKrhuqXJ3msfXdLkrT6aUnubfV9SdadkquVJElaZBYMY1V1oKouq6rLgMuBfwC+CmwH9lbVemBv+0ySi4FJ4BJgE3BrklXtdLcB24D1bdvU6luBl6rqIuBm4KaTcnWSJEmL3PEuU24Evl9VPwQ2AztbfSdwdTveDNxTVa9V1dPANHBFkvOBM6rqoaoq4K45fWbPtRvYOHvXTJIkaTk73jA2CXy5HZ9XVYcA2v7cVl8DPDfU52CrrWnHc+tH9amqI8DLwDnHOTZJkqQlZ+wwluTtwKeAP1uo6YhazVOfr8/cMWxLMpVkamZmZoFhSJIkLX7Hc2fsE8C3q+qF9vmFtvRI2x9u9YPABUP91gLPt/raEfWj+iRZDZwJvDh3AFV1e1VtqKoNExMTxzF0SZKkxel4wtin+cUSJcAeYEs73gLcN1SfbG9IXsjgQf2H21LmK0mubM+DXTenz+y5rgEebM+VSZIkLWurx2mU5J3ArwH/bqj8BWBXkq3As8C1AFW1P8ku4AngCHBDVb3e+lwP3AmcDtzfNoA7gLuTTDO4Izb5Jq5JkiRpyRgrjFXVPzDngfqq+jGDtytHtd8B7BhRnwIuHVF/lRbmJEmSVhJ/Ar8kSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1ZBiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1NFYYS/LeJLuTfC/Jk0l+JcnZSR5I8lTbnzXU/sYk00kOJLlqqH55ksfad7ckSaufluTeVt+XZN1Jv1JJkqRFaNw7Y/8D+Muq+ufAB4Enge3A3qpaD+xtn0lyMTAJXAJsAm5Nsqqd5zZgG7C+bZtafSvwUlVdBNwM3PQmr0uSJGlJWDCMJTkD+ChwB0BV/ayqfgJsBna2ZjuBq9vxZuCeqnqtqp4GpoErkpwPnFFVD1VVAXfN6TN7rt3Axtm7ZpIkScvZOHfG3g/MAH+U5DtJvpjkXcB5VXUIoO3Pbe3XAM8N9T/Yamva8dz6UX2q6gjwMnDO3IEk2ZZkKsnUzMzMmJcoSZK0eI0TxlYDHwZuq6oPAX9PW5I8hlF3tGqe+nx9ji5U3V5VG6pqw8TExPyjliRJWgLGCWMHgYNVta993s0gnL3Qlh5p+8ND7S8Y6r8WeL7V146oH9UnyWrgTODF470YSZKkpWbBMFZVPwKeS/KBVtoIPAHsAba02hbgvna8B5hsb0heyOBB/YfbUuYrSa5sz4NdN6fP7LmuAR5sz5VJkiQta6vHbPc7wJ8keTvwA+DfMghyu5JsBZ4FrgWoqv1JdjEIbEeAG6rq9Xae64E7gdOB+9sGg5cD7k4yzeCO2OSbvC5JkqQlYawwVlWPAhtGfLXxGO13ADtG1KeAS0fUX6WFOUmSpJXEn8AvSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1ZBiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjsYKY0meSfJYkkeTTLXa2UkeSPJU25811P7GJNNJDiS5aqh+eTvPdJJbkqTVT0tyb6vvS7LuJF+nJEnSonQ8d8b+TVVdVlUb2uftwN6qWg/sbZ9JcjEwCVwCbAJuTbKq9bkN2Aasb9umVt8KvFRVFwE3Azed+CVJkiQtHW9mmXIzsLMd7wSuHqrfU1WvVdXTwDRwRZLzgTOq6qGqKuCuOX1mz7Ub2Dh710ySJGk5GzeMFfCNJI8k2dZq51XVIYC2P7fV1wDPDfU92Gpr2vHc+lF9quoI8DJwztxBJNmWZCrJ1MzMzJhDlyRJWrxWj9nuI1X1fJJzgQeSfG+etqPuaNU89fn6HF2ouh24HWDDhg1v+F6SJGmpGevOWFU93/aHga8CVwAvtKVH2v5wa34QuGCo+1rg+VZfO6J+VJ8kq4EzgReP/3IkSZKWlgXDWJJ3JXnP7DHw68DjwB5gS2u2BbivHe8BJtsbkhcyeFD/4baU+UqSK9vzYNfN6TN7rmuAB9tzZZIkScvaOMuU5wFfbc/Trwb+tKr+Msm3gF1JtgLPAtcCVNX+JLuAJ4AjwA1V9Xo71/XAncDpwP1tA7gDuDvJNIM7YpMn4dokSZIWvQXDWFX9APjgiPqPgY3H6LMD2DGiPgVcOqL+Ki3MSZIkrST+BH5JkqSODGOSJEkdGcYkSZI6MoxJkiR1ZBiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1NHYYS7IqyXeSfK19PjvJA0meavuzhtremGQ6yYEkVw3VL0/yWPvuliRp9dOS3Nvq+5KsO4nXKEmStGgdz52xzwJPDn3eDuytqvXA3vaZJBcDk8AlwCbg1iSrWp/bgG3A+rZtavWtwEtVdRFwM3DTCV2NJEnSEjNWGEuyFvgk8MWh8mZgZzveCVw9VL+nql6rqqeBaeCKJOcDZ1TVQ1VVwF1z+syeazewcfaumSRJ0nI27p2xPwR+F/inodp5VXUIoO3PbfU1wHND7Q622pp2PLd+VJ+qOgK8DJwz7kVIkiQtVQuGsSS/CRyuqkfGPOeoO1o1T32+PnPHsi3JVJKpmZmZMYcjSZK0eI1zZ+wjwKeSPAPcA3wsyR8DL7SlR9r+cGt/ELhgqP9a4PlWXzuiflSfJKuBM4EX5w6kqm6vqg1VtWFiYmKsC5QkSVrMFgxjVXVjVa2tqnUMHsx/sKo+A+wBtrRmW4D72vEeYLK9IXkhgwf1H25Lma8kubI9D3bdnD6z57qm/RlvuDMmSZK03Kx+E32/AOxKshV4FrgWoKr2J9kFPAEcAW6oqtdbn+uBO4HTgfvbBnAHcHeSaQZ3xCbfxLgkSZKWjOMKY1X1TeCb7fjHwMZjtNsB7BhRnwIuHVF/lRbmJEmSVhJ/Ar8kSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1ZBiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1tGAYS/KOJA8n+W6S/Ul+v9XPTvJAkqfa/qyhPjcmmU5yIMlVQ/XLkzzWvrslSVr9tCT3tvq+JOtOwbVKkiQtOuPcGXsN+FhVfRC4DNiU5EpgO7C3qtYDe9tnklwMTAKXAJuAW5Osaue6DdgGrG/bplbfCrxUVRcBNwM3vflLkyRJWvwWDGM18Hft49vaVsBmYGer7wSubsebgXuq6rWqehqYBq5Icj5wRlU9VFUF3DWnz+y5dgMbZ++aSZIkLWdjPTOWZFWSR4HDwANVtQ84r6oOAbT9ua35GuC5oe4HW21NO55bP6pPVR0BXgbOGTGObUmmkkzNzMyMdYGSJEmL2VhhrKper6rLgLUM7nJdOk/zUXe0ap76fH3mjuP2qtpQVRsmJiYWGLUkSdLid1xvU1bVT4BvMnjW64W29EjbH27NDgIXDHVbCzzf6mtH1I/qk2Q1cCbw4vGMTZIkaSka523KiSTvbcenAx8HvgfsAba0ZluA+9rxHmCyvSF5IYMH9R9uS5mvJLmyPQ923Zw+s+e6BniwPVcmSZK0rK0eo835wM72RuQvAbuq6mtJHgJ2JdkKPAtcC1BV+5PsAp4AjgA3VNXr7VzXA3cCpwP3tw3gDuDuJNMM7ohNnoyLkyRJWuwWDGNV9TfAh0bUfwxsPEafHcCOEfUp4A3Pm1XVq7QwJ0mStJL4E/glSZI6MoxJkiR1ZBiTJEnqyDC2gHXbv957CJIkaRkzjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw5gkSVJHhjFJkqSODGOSJEkdGcYkSZI6MoxJkiR1ZBiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjpaMIwluSDJXyV5Msn+JJ9t9bOTPJDkqbY/a6jPjUmmkxxIctVQ/fIkj7XvbkmSVj8tyb2tvi/JulNwrZIkSYvOOHfGjgD/uar+BXAlcEOSi4HtwN6qWg/sbZ9p300ClwCbgFuTrGrnug3YBqxv26ZW3wq8VFUXATcDN52Eaztp1m3/eu8hSJKkZWrBMFZVh6rq2+34FeBJYA2wGdjZmu0Erm7Hm4F7quq1qnoamAauSHI+cEZVPVRVBdw1p8/suXYDG2fvmkmSJC1nx/XMWFs+/BCwDzivqg7BILAB57Zma4DnhrodbLU17Xhu/ag+VXUEeBk4Z8Sfvy3JVJKpmZmZ4xm6JEnSojR2GEvybuDPgc9V1U/nazqiVvPU5+tzdKHq9qraUFUbJiYmFhqyJEnSojdWGEvyNgZB7E+q6iut/EJbeqTtD7f6QeCCoe5rgedbfe2I+lF9kqwGzgRePN6LkSRJWmrGeZsywB3Ak1X1B0Nf7QG2tOMtwH1D9cn2huSFDB7Uf7gtZb6S5Mp2zuvm9Jk91zXAg+25MkmSpGVt9RhtPgL8FvBYkkdb7b8AXwB2JdkKPAtcC1BV+5PsAp5g8CbmDVX1eut3PXAncDpwf9tgEPbuTjLN4I7Y5Ju7LEmSpKVhwTBWVf+H0c90AWw8Rp8dwI4R9Sng0hH1V2lhTpIkaSXxJ/BLkiR1ZBiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHVkGJMkSerIMCZJktSRYUySJKkjw9iY1m3/eu8hSJKkZcgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI4MY5IkSR0ZxiRJkjoyjEmSJHW0YBhL8qUkh5M8PlQ7O8kDSZ5q+7OGvrsxyXSSA0muGqpfnuSx9t0tSdLqpyW5t9X3JVl3kq/xpPFnjUmSpJNtnDtjdwKb5tS2A3uraj2wt30mycXAJHBJ63NrklWtz23ANmB922bPuRV4qaouAm4GbjrRi5EkSVpqFgxjVfXXwItzypuBne14J3D1UP2eqnqtqp4GpoErkpwPnFFVD1VVAXfN6TN7rt3Axtm7ZpIkScvdiT4zdl5VHQJo+3NbfQ3w3FC7g622ph3PrR/Vp6qOAC8D55zguCRJkpaUk/0A/6g7WjVPfb4+bzx5si3JVJKpmZmZExyiJEnS4nGiYeyFtvRI2x9u9YPABUPt1gLPt/raEfWj+iRZDZzJG5dFAaiq26tqQ1VtmJiYOMGhS5IkLR4nGsb2AFva8RbgvqH6ZHtD8kIGD+o/3JYyX0lyZXse7Lo5fWbPdQ3wYHuubFHyjUpJknQyrV6oQZIvA78KvC/JQeDzwBeAXUm2As8C1wJU1f4ku4AngCPADVX1ejvV9QzezDwduL9tAHcAdyeZZnBHbPKkXJkkSdISsGAYq6pPH+OrjcdovwPYMaI+BVw6ov4qLcxJkiStNP4EfkmSpI4MYyfA58YkSdLJYhiTJEnqyDAmSZLUkWFMkiSpI8OYJElSR4YxSZKkjgxjJ8g3KiVJ0slgGJMkSerIMCZJktSRYexNcKlSkiS9WYYxSZKkjgxjkiRJHRnGJEmSOjKMSZIkdWQYOwl8kF+SJJ0ow5gkSVJHhjFJkqSODGOSJEkdGcZOIp8dkyRJx8swdpIZyCRJ0vEwjJ0CBjJJkjQuw9gpYiCTJEnjMIydQuu2f91QJkmS5mUYkyRJ6sgw9hbxDpkkSRrFMPYWc+lSkiQNM4x1YiiTJEmwiMJYkk1JDiSZTrK993jeSrOhzIAmSdLKs7r3AACSrAL+J/BrwEHgW0n2VNUTfUfWx9xA9swXPsm67V/nmS98stOIJEnSqbIowhhwBTBdVT8ASHIPsBlYkWFsPsNBbTakDTO4SZK0tCyWZco1wHNDnw+2mt6Eucufw8ug4353PG2G96PGIUmS3ihV1XsMJLkWuKqqfrt9/i3giqr6nTnttgHb2scPAAdO8dDeB/ztKf4zliLn5dicm9Gcl2NzbkZzXo7NuTm2xTw3v1xVE6O+WCzLlAeBC4Y+rwWen9uoqm4Hbn+rBpVkqqo2vFV/3lLhvBybczOa83Jszs1ozsuxOTfHtlTnZrEsU34LWJ/kwiRvByaBPZ3HJEmSdMotijtjVXUkyb8H/jewCvhSVe3vPCxJkqRTblGEMYCq+gvgL3qPY463bEl0iXFejs25Gc15OTbnZjTn5dicm2NbknOzKB7glyRJWqkWyzNjkiRJK5JhbISV/KuZAJJ8KcnhJI8P1c5O8kCSp9r+rKHvbmxzdSDJVX1GfeoluSDJXyV5Msn+JJ9tdecmeUeSh5N8t83N77f6ip8bGPyWkSTfSfK19tl5AZI8k+SxJI8mmWq1FT83Sd6bZHeS77W/b37FeYEkH2j/rcxuP03yuWUxN1XlNrQxeIHg+8D7gbcD3wUu7j2ut3gOPgp8GHh8qPbfgO3teDtwUzu+uM3RacCFbe5W9b6GUzQv5wMfbsfvAf5vu37nBgK8ux2/DdgHXOnc/Hx+/hPwp8DX2mfnZXC9zwDvm1Nb8XMD7AR+ux2/HXiv8/KGOVoF/Aj45eUwN94Ze6Of/2qmqvoZMPurmVaMqvpr4MU55c0M/oKg7a8eqt9TVa9V1dPANIM5XHaq6lBVfbsdvwI8yeA3RTg3A3/XPr6tbYVzQ5K1wCeBLw6VV/y8zGNFz02SMxj8g/gOgKr6WVX9hBU+LyNsBL5fVT9kGcyNYeyN/NVMo51XVYdgEEqAc1t9Rc5XknXAhxjcAXJu+PlS3KPAYeCBqnJuBv4Q+F3gn4ZqzstAAd9I8kj7DSvg3LwfmAH+qC1tfzHJu3Be5poEvtyOl/zcGMbeKCNqvnJ6bCtuvpK8G/hz4HNV9dP5mo6oLdu5qarXq+oyBr9B44okl87TfEXMTZLfBA5X1SPjdhlRW3bzMuQjVfVh4BPADUk+Ok/blTI3qxk8JnJbVX0I+HsGS2/HslLm5efaD4f/FPBnCzUdUVuUc2MYe6OxfjXTCvRCkvMB2v5wq6+o+UryNgZB7E+q6iut7NwMaUsq3wQ24dx8BPhUkmcYPPLwsSR/jPMCQFU93/aHga8yWEJa6XNzEDjY7iwD7GYQzlb6vAz7BPDtqnqhfV7yc2MYeyN/NdNoe4At7XgLcN9QfTLJaUkuBNYDD3cY3ymXJAye43iyqv5g6CvnJplI8t52fDrwceB7rPC5qaobq2ptVa1j8HfJg1X1GVb4vAAkeVeS98weA78OPM4Kn5uq+hHwXJIPtNJG4AlW+LzM8Wl+sUQJy2Fuer9BsBg34DcYvCn3feD3eo+nw/V/GTgE/CODf1lsBc4B9gJPtf3ZQ+1/r83VAeATvcd/CuflXzO4xf03wKNt+w3npgD+JfCdNjePA/+11Vf83Axd76/yi7cpV/y8MHg26rtt2z/7d61zUwCXAVPt/6f/BZzlvPz8Wt8J/Bg4c6i25OfGn8AvSZLUkcuUkiRJHRnGJEmSOjKMSZIkdWQYkyRJ6sgwJkmS1JFhTJIkqSPDmCRJUkeGMUmSpI7+P/FOXNvioWsgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = pattern_count_per_user['pattern_ids']\n",
    "fig, ax = plt.subplots(figsize = (10,6))\n",
    "\n",
    "# histogram\n",
    "ax.hist(y,bins=1000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "15bd3d74-cf23-456d-87ed-17406b455c19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "323603"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern_count_per_user_gt_40 = pattern_count_per_user[pattern_count_per_user['pattern_ids'] >40]\n",
    "# pattern_count_per_user.shape\n",
    "indices_to_drop = pattern_count_per_user[pattern_count_per_user['pattern_ids'] <=40].index.to_list()\n",
    "len(indices_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "a720a9bd-c0f5-41e3-b791-a0e58e7c08e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAFlCAYAAAA+gTZIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUeUlEQVR4nO3db4hl933f8c+3u7b8L66laiVUrcjKsLiVTBu7i3DqYkIVKjkJXj0RrMHpUhQERU3tthBWDdT0wYJbSkgLlUHYTrfEtVAVt1qstI3YxIRCsbqy5FqrtapN5EhbrbWbhNRuHsiR8u2DOSI3s7M7s3PnNzN39vWC4Z77u+fO+emHZN4+58y91d0BAGCcv7DVEwAA2OkEFwDAYIILAGAwwQUAMJjgAgAYTHABAAy2e6snsJrrr7++9+3bt9XTAABY1dNPP/373b1n+fi2D659+/bl5MmTWz0NAIBVVdXvrTTukiIAwGCCCwBgMMEFADCY4AIAGExwAQAMJrgAAAYTXAAAg60aXFX1pao6X1XPzYxdV1VPVtWL0+O1M689WFVnquqFqrprZvxvVNW3p9f+TVXVxv/jAABsP2s5w/Xvkty9bOxIkhPdvT/Jiel5quq2JIeS3D6956Gq2jW95/NJ7k+yf/pZ/jsBAHakVYOru387yR8uGz6Y5Ni0fSzJPTPjj3T36939UpIzSe6oqpuSvLe7/0d3d5J/P/MeAIAdbb33cN3Y3eeSZHq8YRq/OckrM/udncZunraXj6+oqu6vqpNVdfLChQvrnCIAwPaw0TfNr3RfVl9mfEXd/XB3H+juA3v2XPT9jwAAC2W9wfXadJkw0+P5afxskltm9tub5NVpfO8K4wAAO956g+t4ksPT9uEkj8+MH6qqa6rq1izdHP/UdNnxB1X1kemvE//uzHu23L4jT2z1FACAHWz3ajtU1VeS/ESS66vqbJLPJvlckker6r4kLye5N0m6+1RVPZrk+SRvJHmgu9+cftXfz9JfPL4zyX+ZfgAAdrxVg6u7P3mJl+68xP5HkxxdYfxkkg9e0ewAAHYAnzQPADCY4AIAGExwAQAMJrgAAAYTXAAAgwkuAIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrgAAAYTXAAAgwkuAIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrgAAAYTXAAAgwkuAIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrgAAAYTXAAAgwkuAIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrgAAAYTXAAAgwkuAIDBBBcAwGCCa8a+I09s9RQAgB1IcE3EFgAwiuACABhMcAEADCa4AAAGE1wAAIMJLgCAwQQXAMBgggsAYDDBBQAwmOACABhMcAEADCa4AAAGE1wAAIMJLgCAweYKrqr6R1V1qqqeq6qvVNU7quq6qnqyql6cHq+d2f/BqjpTVS9U1V3zTx8AYPtbd3BV1c1J/mGSA939wSS7khxKciTJie7en+TE9DxVddv0+u1J7k7yUFXtmm/6AADb37yXFHcneWdV7U7yriSvJjmY5Nj0+rEk90zbB5M80t2vd/dLSc4kuWPO4wMAbHvrDq7u/j9J/lWSl5OcS/J/u/s3ktzY3eemfc4luWF6y81JXpn5FWensYtU1f1VdbKqTl64cGG9UwQA2BbmuaR4bZbOWt2a5C8neXdVfepyb1lhrFfasbsf7u4D3X1gz549650iAMC2MM8lxZ9M8lJ3X+juP0ny1SR/M8lrVXVTkkyP56f9zya5Zeb9e7N0CRIAYEebJ7heTvKRqnpXVVWSO5OcTnI8yeFpn8NJHp+2jyc5VFXXVNWtSfYneWqO4wMALITd631jd3+jqh5L8s0kbyR5JsnDSd6T5NGqui9LUXbvtP+pqno0yfPT/g9095tzzh8AYNtbd3AlSXd/Nslnlw2/nqWzXSvtfzTJ0XmOCQCwaHzSPADAYIILAGAwwQUAMJjgAgAYTHABAAwmuFaw78gTWz0FAGAHEVwAAIMJLgCAwQQXAMBgggsAYDDBBQAwmOACABhMcAEADCa4AAAGE1wAAIMJLgCAwQQXAMBgggsAYDDBBQAwmOACABhMcAEADCa4LmHfkSe2egoAwA4huAAABhNcAACDCS4AgMEEFwDAYIILAGAwwQUAMJjgAgAYTHABAAwmuAAABhNcAACDCS4AgMEEFwDAYIILAGAwwQUAMJjgAgAYTHABAAwmuAAABhNcAACDCS4AgMEEFwDAYIILAGAwwQUAMJjgAgAYTHABAAwmuAAABhNcAACDCS4AgMEE1yr2HXliq6cAACw4wQUAMJjgAgAYTHABAAw2V3BV1fuq6rGq+k5Vna6qH6+q66rqyap6cXq8dmb/B6vqTFW9UFV3zT/9zeE+LgBgHvOe4frXSf5rd/+VJH89yekkR5Kc6O79SU5Mz1NVtyU5lOT2JHcneaiqds15fACAbW/dwVVV703ysSRfTJLu/mF3/1GSg0mOTbsdS3LPtH0wySPd/Xp3v5TkTJI71nt8AIBFMc8ZrvcnuZDkV6rqmar6QlW9O8mN3X0uSabHG6b9b07yysz7z05jAAA72jzBtTvJh5N8vrs/lOSPM10+vIRaYaxX3LHq/qo6WVUnL1y4MMcUAQC23jzBdTbJ2e7+xvT8sSwF2GtVdVOSTI/nZ/a/Zeb9e5O8utIv7u6Hu/tAdx/Ys2fPHFMEANh66w6u7v5ekleq6gPT0J1Jnk9yPMnhaexwksen7eNJDlXVNVV1a5L9SZ5a7/EBABbF7jnf//NJvlxVb0/yu0n+XpYi7tGqui/Jy0nuTZLuPlVVj2Ypyt5I8kB3vznn8QEAtr25gqu7n01yYIWX7rzE/keTHJ3nmAAAi8YnzQMADCa4AAAGE1wAAIMJLgCAwQQXAMBgggsAYDDBBQAwmOACABhMcAEADCa4AAAGE1wAAIMJLgCAwQQXAMBggusK7DvyxFZPAQBYQIILAGAwwQUAMJjgAgAYTHABAAwmuAAABhNcAACDCS4AgMEEFwDAYILrCvnwUwDgSgkuAIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrgAAAYTXAAAgwkuAIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrjWad+RJ7Z6CgDAghBcAACDCS4AgMEEFwDAYIILAGAwwQUAMJjgAgAYTHBtAB8RAQBcjuACABhMcAEADCa4AAAGE1xzcv8WALAawQUAMJjgAgAYTHABAAwmuAAABhNcAACDzR1cVbWrqp6pqq9Nz6+rqier6sXp8dqZfR+sqjNV9UJV3TXvsQEAFsFGnOH6dJLTM8+PJDnR3fuTnJiep6puS3Ioye1J7k7yUFXt2oDjAwBsa3MFV1XtTfLTSb4wM3wwybFp+1iSe2bGH+nu17v7pSRnktwxz/EBABbBvGe4fjnJLyT505mxG7v7XJJMjzdM4zcneWVmv7PT2EWq6v6qOllVJy9cuDDnFAEAtta6g6uqfibJ+e5+eq1vWWGsV9qxux/u7gPdfWDPnj3rnSIAwLawe473fjTJJ6rqp5K8I8l7q+pXk7xWVTd197mquinJ+Wn/s0lumXn/3iSvznF8AICFsO4zXN39YHfv7e59WboZ/je7+1NJjic5PO12OMnj0/bxJIeq6pqqujXJ/iRPrXvmAAALYp4zXJfyuSSPVtV9SV5Ocm+SdPepqno0yfNJ3kjyQHe/OeD4AADbyoYEV3d/PcnXp+0/SHLnJfY7muToRhwTAGBR+KR5AIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrg20L4jT2z1FACAbUhwAQAMJrgAAAYTXAAAgwkuAIDBBNcGc+M8ALCc4AIAGExwAQAMJrgAAAYTXAAAgwkuAIDBBBcAwGCCCwBgMMEFADCY4AIAGExwAQAMJrgAAAYTXIP4TkUA4C2CCwBgMMEFADCY4BrIZUUAIBFcAADDCS4AgMEEFwDAYIJrE7iXCwCuboILAGAwwQUAMJjgAgAYTHABAAwmuDaZG+gB4OojuAAABhNcAACDCS4AgMEEFwDAYIJrk7hZHgCuXoILAGAwwQUAMJjgAgAYTHABAAwmuAAABhNcAACDCS4AgMEEFwDAYIILAGAwwQUAMJjg2gZ87Q8A7GyCCwBgMMEFADDYuoOrqm6pqt+qqtNVdaqqPj2NX1dVT1bVi9PjtTPvebCqzlTVC1V110b8AwAAbHfznOF6I8k/6e6/muQjSR6oqtuSHElyorv3JzkxPc/02qEktye5O8lDVbVrnsnvBO7fAoCdb93B1d3nuvub0/YPkpxOcnOSg0mOTbsdS3LPtH0wySPd/Xp3v5TkTJI71nv8RSe0AODqsSH3cFXVviQfSvKNJDd297lkKcqS3DDtdnOSV2bednYaW+n33V9VJ6vq5IULFzZiigAAW2bu4Kqq9yT5tSSf6e7vX27XFcZ6pR27++HuPtDdB/bs2TPvFAEAttRcwVVVb8tSbH25u786Db9WVTdNr9+U5Pw0fjbJLTNv35vk1XmODwCwCOb5K8VK8sUkp7v7l2ZeOp7k8LR9OMnjM+OHquqaqro1yf4kT633+DuR+7oAYGfaPcd7P5rkZ5N8u6qencb+aZLPJXm0qu5L8nKSe5Oku09V1aNJns/SXzg+0N1vznF8AICFsO7g6u7/npXvy0qSOy/xnqNJjq73mAAAi8gnzQMADCa4AAAGE1wAAIMJLgCAwQQXAMBggmub8plcALBzCC4AgMEEFwDAYIJrG3NZEQB2BsG1zYgsANh5BBcAwGCCCwBgMMEFADCY4FoA7usCgMUmuBaE6AKAxSW4AAAGE1wAAIMJLgCAwQQXAMBgggsAYDDBBQAwmOACABhMcC0Yn8cFAItHcC2Q5bH11nMRBgDbm+BaUCILABaH4AIAGExwAQAMJrh2GJcaAWD7EVwAAIMJrh3CmS0A2L4E1w50qY+PAAC2huACABhMcF0lnOUCgK0juHYwkQUA24PgAgAYTHBdZZz1AoDNJ7gAAAYTXFc5Z7wAYDzBBQAwmOBiRc58AcDGEVxXobdiSlQBwOYQXFzSbJCJMwBYP8HFRcQVAGwswcWfI7YAYOMJLgCAwQQXq7rUWS9nwwBgbQQXSdYeT1cSX4IMAJYILq7Ypf56cbWPmxBgAFytBBcbSlQBwMUEF5tuvVEm5gBYVIILAGAwwcWmWH52arXnlxpbzz4AsNUEF0OtFkSrfX3QWm/AF14AbGebHlxVdXdVvVBVZ6rqyGYfn+1vtfBa719Bzvs6AKzXpgZXVe1K8m+TfDzJbUk+WVW3beYc2DnWc/ZrI27Yd6kTgCu12We47khyprt/t7t/mOSRJAc3eQ5cBS4XPMvj6XKfKzbPWbH1vnYl+wCwGDY7uG5O8srM87PTGAyxEZcfV/pA17V+4OtstF0q4FbaZ6XwW0scXmpea9m+kjmuZV1W+r1rmeOVWO3Ddleb15X+vnkjeCO+0UGIbzxrujZXsk7W9GLV3Zt3sKp7k9zV3T83Pf/ZJHd0988v2+/+JPdPTz+Q5IVNm+RiuD7J72/1JBaEtVo7a7U21mntrNXaWau1WYR1+tHu3rN8cPcmT+Jskltmnu9N8urynbr74SQPb9akFk1VnezuA1s9j0VgrdbOWq2NdVo7a7V21mptFnmdNvuS4v9Msr+qbq2qtyc5lOT4Js8BAGBTbeoZru5+o6r+QZL/lmRXki9196nNnAMAwGbb7EuK6e5fT/Lrm33cHcbl1rWzVmtnrdbGOq2dtVo7a7U2C7tOm3rTPADA1chX+wAADCa4tqGq+lJVna+q52bGrquqJ6vqxenx2pnXHpy+KumFqrpra2a9+arqlqr6rao6XVWnqurT07i1Wqaq3lFVT1XVt6a1+ufTuLVaQVXtqqpnqupr03PrtIKq+m5Vfbuqnq2qk9OYtVpBVb2vqh6rqu9M/5v149bqYlX1genfp7d+vl9Vn9kRa9XdfrbZT5KPJflwkudmxv5lkiPT9pEk/2Lavi3Jt5Jck+TWJL+TZNdW/zNs0jrdlOTD0/aPJPnf03pYq4vXqpK8Z9p+W5JvJPmItbrkev3jJP8hydem59Zp5XX6bpLrl41Zq5XX6liSn5u2357kfdZq1TXbleR7SX50J6yVM1zbUHf/dpI/XDZ8MEv/wWZ6vGdm/JHufr27X0pyJktfobTjdfe57v7mtP2DJKez9M0F1mqZXvL/pqdvm3461uoiVbU3yU8n+cLMsHVaO2u1TFW9N0v/R/qLSdLdP+zuP4q1Ws2dSX6nu38vO2CtBNfiuLG7zyVLoZHkhmnc1yUlqap9ST6UpTM31moF02WyZ5OcT/Jkd1urlf1ykl9I8qczY9ZpZZ3kN6rq6ekbQhJrtZL3J7mQ5FemS9VfqKp3x1qt5lCSr0zbC79Wgmvx1QpjV9WfnlbVe5L8WpLPdPf3L7frCmNXzVp195vd/WNZ+oaHO6rqg5fZ/apcq6r6mSTnu/vptb5lhbEdv04zPtrdH07y8SQPVNXHLrPv1bxWu7N0m8jnu/tDSf44S5fFLuVqXqskyfTh6J9I8h9X23WFsW25VoJrcbxWVTclyfR4fhpf09cl7VRV9bYsxdaXu/ur07C1uozpUsbXk9wda7XcR5N8oqq+m+SRJH+7qn411mlF3f3q9Hg+yX/K0qUca3Wxs0nOTmeVk+SxLAWYtbq0jyf5Zne/Nj1f+LUSXIvjeJLD0/bhJI/PjB+qqmuq6tYk+5M8tQXz23RVVVm6J+J0d//SzEvWapmq2lNV75u235nkJ5N8J9bqz+nuB7t7b3fvy9LljN/s7k/FOl2kqt5dVT/y1naSv5PkuViri3T395K8UlUfmIbuTPJ8rNXlfDJ/djkx2QlrtdV37fu5+CdL/5KdS/InWar3+5L8pSQnkrw4PV43s/8vZukvM15I8vGtnv8mrtPfytKp4/+V5Nnp56es1Ypr9deSPDOt1XNJ/tk0bq0uvWY/kT/7K0XrdPH6vD9Lfx32rSSnkvyitbrsev1YkpPTf4P/Ocm11uqSa/WuJH+Q5C/OjC38WvmkeQCAwVxSBAAYTHABAAwmuAAABhNcAACDCS4AgMEEFwDAYIILAGAwwQUAMNj/B8jNfCX+kqMkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = pattern_count_per_user_gt_40['pattern_ids']\n",
    "fig, ax = plt.subplots(figsize = (10,6))\n",
    "\n",
    "# histogram\n",
    "ax.hist(y,bins=1000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "6a3c4bac-cd73-4e1c-a1ca-1db5dfc1d104",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3250978, 5)"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = df[~df.index.isin(indices_to_drop)]\n",
    "# merge metadata\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "id": "09cd6e25-631d-42a6-82ed-3fb83abc8ec6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pattern_ids</th>\n",
       "      <th>date_completed</th>\n",
       "      <th>project_id</th>\n",
       "      <th>has_knit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>27661.000000</td>\n",
       "      <td>27661.000000</td>\n",
       "      <td>27661.000000</td>\n",
       "      <td>27661.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>73.736416</td>\n",
       "      <td>65.853368</td>\n",
       "      <td>73.736416</td>\n",
       "      <td>73.736416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>39.116392</td>\n",
       "      <td>38.951095</td>\n",
       "      <td>39.116392</td>\n",
       "      <td>39.116392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>41.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>49.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>49.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>61.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>61.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>84.000000</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>84.000000</td>\n",
       "      <td>84.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        pattern_ids  date_completed    project_id      has_knit\n",
       "count  27661.000000    27661.000000  27661.000000  27661.000000\n",
       "mean      73.736416       65.853368     73.736416     73.736416\n",
       "std       39.116392       38.951095     39.116392     39.116392\n",
       "min       41.000000        0.000000     41.000000     41.000000\n",
       "25%       49.000000       43.000000     49.000000     49.000000\n",
       "50%       61.000000       55.000000     61.000000     61.000000\n",
       "75%       84.000000       77.000000     84.000000     84.000000\n",
       "max      720.000000      720.000000    720.000000    720.000000"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern_count_per_user_gt_40.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "ccceb989-5d78-4805-a57c-a973bb8aa40f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['88', '88', '88', ..., '449489', '449489', '449489'], dtype=object)"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.pattern_ids.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "5bf47625-6d76-4d01-b71f-14c59493943e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2963, 24)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pattern_id</th>\n",
       "      <th>name</th>\n",
       "      <th>name_permalink</th>\n",
       "      <th>favorites_count</th>\n",
       "      <th>projects_count</th>\n",
       "      <th>difficulty_average</th>\n",
       "      <th>difficulty_count</th>\n",
       "      <th>rating_average</th>\n",
       "      <th>queued_projects_count</th>\n",
       "      <th>rating_count</th>\n",
       "      <th>...</th>\n",
       "      <th>pattern_attributes</th>\n",
       "      <th>yardage_max</th>\n",
       "      <th>yardage</th>\n",
       "      <th>generally_available</th>\n",
       "      <th>gauge</th>\n",
       "      <th>gauge_divisor</th>\n",
       "      <th>free</th>\n",
       "      <th>downloadable</th>\n",
       "      <th>categories</th>\n",
       "      <th>yarn_weight_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>88</td>\n",
       "      <td>Calorimetry</td>\n",
       "      <td>calorimetry</td>\n",
       "      <td>29838</td>\n",
       "      <td>19426</td>\n",
       "      <td>1.816424</td>\n",
       "      <td>7294.0</td>\n",
       "      <td>4.225295</td>\n",
       "      <td>7895</td>\n",
       "      <td>6618.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['unisex', 'teen', 'adult', 'fitted', 'reversi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>93.0</td>\n",
       "      <td>2006/12/01 00:00:00 -0500</td>\n",
       "      <td>20.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>['headband', 'headwear', 'accessories']</td>\n",
       "      <td>Aran (8 wpi)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3156</td>\n",
       "      <td>Basic Ribbed Socks</td>\n",
       "      <td>basic-ribbed-socks</td>\n",
       "      <td>17795</td>\n",
       "      <td>10987</td>\n",
       "      <td>2.435581</td>\n",
       "      <td>2670.0</td>\n",
       "      <td>4.455242</td>\n",
       "      <td>3519</td>\n",
       "      <td>2480.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['unisex', 'adult', 'top-cuff-down', 'heel-fla...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>370.0</td>\n",
       "      <td>2006/04/01 00:00:00 -0500</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>['mid-calf', 'socks', 'feet-legs', 'accessories']</td>\n",
       "      <td>Fingering (14 wpi)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3636</td>\n",
       "      <td>#29 Classic Socks- baby, child &amp; adult socks</td>\n",
       "      <td>29-classic-socks--baby-child--adult-socks</td>\n",
       "      <td>3483</td>\n",
       "      <td>6019</td>\n",
       "      <td>2.851249</td>\n",
       "      <td>921.0</td>\n",
       "      <td>4.493304</td>\n",
       "      <td>463</td>\n",
       "      <td>896.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['unisex', 'baby', 'toddler', 'child', 'teen',...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007/05/18 07:01:30 -0400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>['mid-calf', 'socks', 'feet-legs', 'accessories']</td>\n",
       "      <td>Sport (12 wpi)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27077</td>\n",
       "      <td>Jacques Cousteau Hat</td>\n",
       "      <td>jacques-cousteau-hat</td>\n",
       "      <td>22539</td>\n",
       "      <td>7325</td>\n",
       "      <td>1.865684</td>\n",
       "      <td>2442.0</td>\n",
       "      <td>4.267587</td>\n",
       "      <td>4534</td>\n",
       "      <td>2246.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['male', 'ribbed', 'written-pattern', 'in-the-...</td>\n",
       "      <td>234.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>2006/03/01 00:00:00 -0500</td>\n",
       "      <td>22.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>['beanie-toque', 'hat', 'accessories']</td>\n",
       "      <td>DK (11 wpi)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29869</td>\n",
       "      <td>Sock Patterns for Regia 4ply</td>\n",
       "      <td>sock-patterns-for-regia-4ply</td>\n",
       "      <td>1190</td>\n",
       "      <td>2469</td>\n",
       "      <td>2.853333</td>\n",
       "      <td>300.0</td>\n",
       "      <td>4.223776</td>\n",
       "      <td>219</td>\n",
       "      <td>286.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['unisex', 'adult', 'top-cuff-down', 'heel-fla...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>460.0</td>\n",
       "      <td>2007/10/27 18:20:38 -0400</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>['mid-calf', 'socks', 'feet-legs', 'accessories']</td>\n",
       "      <td>Fingering (14 wpi)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   pattern_id                                          name  \\\n",
       "0          88                                   Calorimetry   \n",
       "1        3156                            Basic Ribbed Socks   \n",
       "2        3636  #29 Classic Socks- baby, child & adult socks   \n",
       "3       27077                          Jacques Cousteau Hat   \n",
       "4       29869                  Sock Patterns for Regia 4ply   \n",
       "\n",
       "                              name_permalink  favorites_count  projects_count  \\\n",
       "0                                calorimetry            29838           19426   \n",
       "1                         basic-ribbed-socks            17795           10987   \n",
       "2  29-classic-socks--baby-child--adult-socks             3483            6019   \n",
       "3                       jacques-cousteau-hat            22539            7325   \n",
       "4               sock-patterns-for-regia-4ply             1190            2469   \n",
       "\n",
       "   difficulty_average  difficulty_count  rating_average  \\\n",
       "0            1.816424            7294.0        4.225295   \n",
       "1            2.435581            2670.0        4.455242   \n",
       "2            2.851249             921.0        4.493304   \n",
       "3            1.865684            2442.0        4.267587   \n",
       "4            2.853333             300.0        4.223776   \n",
       "\n",
       "   queued_projects_count  rating_count  ...  \\\n",
       "0                   7895        6618.0  ...   \n",
       "1                   3519        2480.0  ...   \n",
       "2                    463         896.0  ...   \n",
       "3                   4534        2246.0  ...   \n",
       "4                    219         286.0  ...   \n",
       "\n",
       "                                  pattern_attributes yardage_max yardage  \\\n",
       "0  ['unisex', 'teen', 'adult', 'fitted', 'reversi...         NaN    93.0   \n",
       "1  ['unisex', 'adult', 'top-cuff-down', 'heel-fla...         NaN   370.0   \n",
       "2  ['unisex', 'baby', 'toddler', 'child', 'teen',...         NaN     NaN   \n",
       "3  ['male', 'ribbed', 'written-pattern', 'in-the-...       234.0   234.0   \n",
       "4  ['unisex', 'adult', 'top-cuff-down', 'heel-fla...         NaN   460.0   \n",
       "\n",
       "         generally_available gauge  gauge_divisor   free downloadable  \\\n",
       "0  2006/12/01 00:00:00 -0500  20.0            4.0   True         True   \n",
       "1  2006/04/01 00:00:00 -0500  32.0            4.0   True         True   \n",
       "2  2007/05/18 07:01:30 -0400   NaN            1.0  False         True   \n",
       "3  2006/03/01 00:00:00 -0500  22.0            4.0   True         True   \n",
       "4  2007/10/27 18:20:38 -0400  30.0            4.0  False        False   \n",
       "\n",
       "                                          categories  yarn_weight_description  \n",
       "0            ['headband', 'headwear', 'accessories']             Aran (8 wpi)  \n",
       "1  ['mid-calf', 'socks', 'feet-legs', 'accessories']       Fingering (14 wpi)  \n",
       "2  ['mid-calf', 'socks', 'feet-legs', 'accessories']           Sport (12 wpi)  \n",
       "3             ['beanie-toque', 'hat', 'accessories']              DK (11 wpi)  \n",
       "4  ['mid-calf', 'socks', 'feet-legs', 'accessories']       Fingering (14 wpi)  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# In order to pull out pattern details after we get similar patterns, need the metadata \n",
    "#(merge df with pattern_details)\n",
    "\n",
    "metadata = pattern_details[pattern_details['pattern_id'].isin(df.pattern_ids.values)]\n",
    "# REINDEX??\n",
    "print(metadata.shape)\n",
    "metadata.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0575b576-7513-445f-9e6f-f78560a00d2a",
   "metadata": {},
   "source": [
    "#### Check Sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "2146c1b5-bf58-4c10-b2a4-f2504c461c23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of users: 338751\n",
      "Number of models: 2961\n",
      "Sparsity: 0.324%\n"
     ]
    }
   ],
   "source": [
    "n_users = df.user_id.unique().shape[0]\n",
    "n_items = df.pattern_ids.unique().shape[0]\n",
    "\n",
    "print('Number of users: {}'.format(n_users))\n",
    "print('Number of models: {}'.format(n_items))\n",
    "print('Sparsity: {:4.3f}%'.format(float(df.shape[0]) / float(n_users*n_items) * 100))\n",
    "\n",
    "# Woot - this is above the 0.1 percent threshold - performance generally degrades the more sparse the matrix is "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07fb45e1-4ace-4720-8874-ac5b83f4d73d",
   "metadata": {},
   "source": [
    "### Unique counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7caafed7-369f-4961-9d6e-09ea6cfa4df6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print('No. of Unique Users    :', reviews.userID.nunique())\n",
    "# print('No. of Unique Business :', reviews.businessID.nunique())\n",
    "# print('No. of Unique Ratings  :', reviews.rating.nunique())\n",
    "\n",
    "\n",
    "# look here for bar charts I should include here too\n",
    "# https://bmanohar16.github.io/blog/recsys-evaluation-in-surprise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "id": "213020f4-ae74-4ae0-8902-7de62752a7a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3250978, 5)"
      ]
     },
     "execution_count": 315,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "08f4f5d7-71a6-41a1-88a7-ceb0e0b43c66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(290270, 1700)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# user_project_map = df.pivot_table(index='user_id', columns='pattern_ids', values='has_knit')\n",
    "# user_project_map.head()                    \n",
    "# user_project_map.shape\n",
    "# # user_project_map.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "id": "eaa75e7b-895f-4f7f-8593-01907dcdd739",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/less_sparse_users_patterns.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25254646-00ef-44f3-8610-a34c5fc54727",
   "metadata": {},
   "source": [
    "##  Clean up patterns Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b1dfe0e-dec3-4f6d-a43a-c20efe4e5c56",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/consolidated_patterns.csv', low_memory=False)\n",
    "\n",
    "# Need to save and reload clean data!! \n",
    "df = clean_out_bad_data(df)\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "\n",
    "df.to_csv('data/patterns_cleaned.csv', index=False)\n",
    "\n",
    "# # INCLUDE WEIGHTED AVERAGE SCORE RATHER THAN RATING?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test_env",
   "language": "python",
   "name": "test_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}