data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JngFMhySY4n1"
   },
   "source": [
    "https://github.com/microsoft/recommenders/blob/main/examples/01_prepare_data/data_split.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 769,
     "status": "ok",
     "timestamp": 1652636104003,
     "user": {
      "displayName": "권유진",
      "userId": "17694849594474805342"
     },
     "user_tz": -540
    },
    "id": "UrfB16nFcmkQ"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "executionInfo": {
     "elapsed": 2712,
     "status": "ok",
     "timestamp": 1652636106711,
     "user": {
      "displayName": "권유진",
      "userId": "17694849594474805342"
     },
     "user_tz": -540
    },
    "id": "hRNVe8GocjUw"
   },
   "outputs": [],
   "source": [
    "transaction = pd.read_csv('book_transactions.csv', encoding='cp949')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 456,
     "status": "ok",
     "timestamp": 1652636131486,
     "user": {
      "displayName": "권유진",
      "userId": "17694849594474805342"
     },
     "user_tz": -540
    },
    "id": "ZqS4UmsAd787",
    "outputId": "19d7685a-cb4a-4e97-872b-baa868a0c313",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = transaction[['회원번호', '책제목']]\n",
    "ids = data.groupby('회원번호').책제목.count().reset_index().query(\"책제목 > 5\").회원번호.unique()\n",
    "data = data.query('회원번호 in @ids')\n",
    "\n",
    "id2user = dict(enumerate(data.회원번호.unique()))\n",
    "user2id = {j:i for i, j in id2user.items()}\n",
    "id2book = dict(enumerate(data.책제목.unique()))\n",
    "book2id = {j:i for i, j in id2book.items()}\n",
    "#id2cat = dict(enumerate(data.카테고리.unique()))\n",
    "#cat2id = {j:i for i, j in id2cat.items()}\n",
    "\n",
    "data.회원번호 = data.회원번호.map(lambda x: user2id[x])\n",
    "data.책제목 = data.책제목.map(lambda x: book2id[x])\n",
    "data = data.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data_cat = transaction.groupby('책제목')['카테고리'].unique().map(lambda x: x[0]).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data_cat['책제목'] = data_cat.책제목.map(lambda x: book2id[x])\n",
    "#data_cat['카테고리'] = data_cat.카테고리.map(lambda x: cat2id[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data_cat = dict(data_cat.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "executionInfo": {
     "elapsed": 1302,
     "status": "ok",
     "timestamp": 1652636134150,
     "user": {
      "displayName": "권유진",
      "userId": "17694849594474805342"
     },
     "user_tz": -540
    },
    "id": "u6m61kAoSU7I"
   },
   "outputs": [],
   "source": [
    "data.to_csv('main_data_yes24.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "id": "K4RGtPKbz7P9"
   },
   "outputs": [],
   "source": [
    "#data_cat = transaction[['회원번호', '카테고리']]\n",
    "#data_cat.회원번호 = data_cat.회원번호.map(lambda x: user2id[x])\n",
    "#data_cat.카테고리 = data_cat.카테고리.map(lambda x: cat2id[x])\n",
    "#data_cat = data_cat.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_tr = pd.read_csv('main_data_yes24.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(203652, 2)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_tr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = data_tr.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "items = data.groupby('회원번호')['책제목'].unique()\n",
    "test_data = items.map(lambda x: x[-1]).reset_index().values\n",
    "train_data = train_data.append(pd.DataFrame(test_data, columns=['회원번호', '책제목'])).drop_duplicates(keep=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.to_csv('main_data_train_yes24.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_num = data_tr.책제목.nunique()\n",
    "item_list = set(range(item_num))\n",
    "def noninteracted(x):\n",
    "    lst = list(item_list - set(x))\n",
    "    sampled = np.random.choice(lst, 99)\n",
    "    result = x[-1].reshape(1).tolist() + sampled.tolist()\n",
    "    return result\n",
    "res = items.map(noninteracted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('main_data_test_yes24.npy', res.reset_index().values, allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_raw = np.load('main_data_test_yes24.npy', allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = []\n",
    "for data in test_data_raw:\n",
    "    u = data[0]\n",
    "    i = data[1]\n",
    "    for item in i:\n",
    "        test_data.append([u, int(item)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12124 60210\n"
     ]
    }
   ],
   "source": [
    "print(data_tr.회원번호.nunique(), data_tr.책제목.nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ml-1m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>914</td>\n",
       "      <td>3</td>\n",
       "      <td>978301968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3408</td>\n",
       "      <td>4</td>\n",
       "      <td>978300275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2355</td>\n",
       "      <td>5</td>\n",
       "      <td>978824291</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID  MovieID  Rating  Timestamp\n",
       "0        1     1193       5  978300760\n",
       "1        1      661       3  978302109\n",
       "2        1      914       3  978301968\n",
       "3        1     3408       4  978300275\n",
       "4        1     2355       5  978824291"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_names = ['MovieID', 'Title', 'Genres']\n",
    "movies = pd.read_csv(\"./ml-1m/movies.dat\", sep = \"::\", names = column_names, encoding='latin-1', engine='python')\n",
    "\n",
    "column_names = ['User_ID', 'Gender', 'Age', 'Occupation', 'Zip-code']\n",
    "users = pd.read_csv(\"./ml-1m/users.dat\", sep = \"::\", names = column_names, engine='python')\n",
    "\n",
    "column_names = ['User_ID', 'MovieID', 'Rating', 'Timestamp']\n",
    "ratings = pd.read_csv(\"./ml-1m/ratings.dat\", sep = \"::\", names = column_names, engine='python')\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2user = dict(enumerate(users.User_ID.unique()))\n",
    "user2id = {j:i for i, j in id2user.items()}\n",
    "\n",
    "id2item = dict(enumerate(movies.MovieID.unique()))\n",
    "item2id = {j:i for i, j in id2item.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings['User_ID'] = ratings['User_ID'].map(lambda x: user2id[x]) \n",
    "ratings['MovieID'] = ratings['MovieID'].map(lambda x: item2id[x]) \n",
    "users['User_ID'] = users['User_ID'].map(lambda x: user2id[x]) \n",
    "movies['MovieID'] = movies['MovieID'].map(lambda x: item2id[x]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "users['aux'] = users['Occupation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "users['aux'] = users[['Gender', 'Age', 'Occupation']].apply(lambda x: ','.join(map(lambda x: str(x),x)), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies[['MovieID', 'Genres']].to_csv('ml_movies.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies.Genres = movies.Genres.map(lambda x: x.split('|')[0])\n",
    "movies[['MovieID', 'Genres']].to_csv('ml_movies2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "users[['User_ID', 'aux']].to_csv('ml_users2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies[['MovieID', 'Genres']].Genres.nunique()#, users[['User_ID', 'aux']].aux.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = ratings.sort_values(['User_ID', 'Timestamp'])[['User_ID', 'MovieID']].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    6040.000000\n",
       "mean      165.597517\n",
       "std       192.747029\n",
       "min        20.000000\n",
       "25%        44.000000\n",
       "50%        96.000000\n",
       "75%       208.000000\n",
       "max      2314.000000\n",
       "Name: User_ID, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings['User_ID'].value_counts().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_copy = ratings.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "items = ratings.groupby('User_ID')['MovieID'].unique()\n",
    "test_data = items.map(lambda x: x[-1]).reset_index().values\n",
    "train_data = ratings_copy.append(pd.DataFrame(test_data, columns=['User_ID', 'MovieID'])).drop_duplicates(keep=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>MovieID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000203</th>\n",
       "      <td>6039</td>\n",
       "      <td>229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000204</th>\n",
       "      <td>6039</td>\n",
       "      <td>2848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000205</th>\n",
       "      <td>6039</td>\n",
       "      <td>1852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000206</th>\n",
       "      <td>6039</td>\n",
       "      <td>1726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000207</th>\n",
       "      <td>6039</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>994169 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         User_ID  MovieID\n",
       "0              0     3117\n",
       "1              0     1250\n",
       "2              0     1672\n",
       "3              0     1009\n",
       "4              0     2271\n",
       "...          ...      ...\n",
       "1000203     6039      229\n",
       "1000204     6039     2848\n",
       "1000205     6039     1852\n",
       "1000206     6039     1726\n",
       "1000207     6039      159\n",
       "\n",
       "[994169 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0,   47],\n",
       "       [   1, 1848],\n",
       "       [   2, 2012],\n",
       "       ...,\n",
       "       [6037, 1166],\n",
       "       [6038, 1234],\n",
       "       [6039, 1203]], dtype=int64)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.to_csv('main_data_train_ml.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_num = ratings_copy.MovieID.nunique()\n",
    "item_list = set(range(item_num))\n",
    "def noninteracted(x):\n",
    "    lst = list(item_list - set(x))\n",
    "    sampled = np.random.choice(lst, 99)\n",
    "    result = x[-1].reshape(1).tolist() + sampled.tolist()\n",
    "    return result\n",
    "res = items.map(noninteracted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('main_data_test_ml.npy', res.reset_index().values, allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6039, 3882)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.User_ID.max(), ratings.MovieID.max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Amazon Book Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\KwonYujin\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3444: DtypeWarning: Columns (3) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "rating_am = pd.read_csv('./amazon_book_dataset/Ratings.csv')\n",
    "users_am = pd.read_csv('./amazon_book_dataset/Users.csv')\n",
    "items_am = pd.read_csv('./amazon_book_dataset/Books.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    105283.000000\n",
       "mean         10.920851\n",
       "std          90.562825\n",
       "min           1.000000\n",
       "25%           1.000000\n",
       "50%           1.000000\n",
       "75%           4.000000\n",
       "max       13602.000000\n",
       "Name: ISBN, dtype: float64"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_am.groupby('User-ID').ISBN.count().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = rating_am.groupby('User-ID').ISBN.count().reset_index().query('ISBN > 5')['User-ID'].values\n",
    "rating_am = rating_am.query('`User-ID` in @ids').reset_index(drop=True)\n",
    "users_am = users_am.query('`User-ID` in @ids').reset_index(drop=True)\n",
    "\n",
    "books = rating_am['ISBN'].unique()\n",
    "items_am = items_am.query('ISBN in @books').reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2user = dict(enumerate(users_am['User-ID'].unique()))\n",
    "user2id = {j:i for i, j in id2user.items()}\n",
    "\n",
    "id2item = dict(enumerate(rating_am.ISBN.unique()))\n",
    "item2id = {j:i for i, j in id2item.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "rating_am['User-ID'] = rating_am['User-ID'].map(lambda x: user2id[x]) \n",
    "rating_am['ISBN'] = rating_am['ISBN'].map(lambda x: item2id[x]) \n",
    "users_am['User-ID'] = users_am['User-ID'].map(lambda x: user2id[x]) \n",
    "items_am['ISBN'] = items_am['ISBN'].map(lambda x: item2id[x]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_copy = rating_am.copy()[['User-ID', 'ISBN']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "items = rating_am.groupby('User-ID')['ISBN'].unique()\n",
    "test_data = items.map(lambda x: x[-1]).reset_index().values\n",
    "train_data = ratings_copy.append(pd.DataFrame(test_data, columns=['User-ID', 'ISBN'])).drop_duplicates(keep=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[    0,  8158],\n",
       "       [    1,  8164],\n",
       "       [    2,  8171],\n",
       "       ...,\n",
       "       [19714,  8113],\n",
       "       [19715,  8136],\n",
       "       [19716,  1758]], dtype=int64)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.to_csv('main_data_train_am.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_num = ratings_copy.ISBN.nunique()\n",
    "item_list = set(range(item_num))\n",
    "def noninteracted(x):\n",
    "    lst = list(item_list - set(x))\n",
    "    sampled = np.random.choice(lst, 99)\n",
    "    result = x[-1].reshape(1).tolist() + sampled.tolist()\n",
    "    return result\n",
    "res = items.map(noninteracted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('main_data_test_am.npy', res.reset_index().values, allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19716, 316098)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_am['User-ID'].max(), rating_am.ISBN.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_am.Age.fillna(users_am.Age.mode()[0], inplace= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    19717.000000\n",
       "mean        33.951057\n",
       "std         12.807987\n",
       "min          0.000000\n",
       "25%         29.000000\n",
       "50%         29.000000\n",
       "75%         38.000000\n",
       "max        244.000000\n",
       "Name: Age, dtype: float64"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_am.Age.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "def age(x):\n",
    "    if x < 10:\n",
    "        return '10대 미만'\n",
    "    else:\n",
    "        return str(x)[0]+'0대'\n",
    "\n",
    "users_am['Age'] = users_am.Age.map(age)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User-ID</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>40대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19712</th>\n",
       "      <td>19712</td>\n",
       "      <td>30대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19713</th>\n",
       "      <td>19713</td>\n",
       "      <td>10대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19714</th>\n",
       "      <td>19714</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19715</th>\n",
       "      <td>19715</td>\n",
       "      <td>30대</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19716</th>\n",
       "      <td>19716</td>\n",
       "      <td>20대</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19717 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       User-ID  Age\n",
       "0            0  20대\n",
       "1            1  20대\n",
       "2            2  20대\n",
       "3            3  20대\n",
       "4            4  40대\n",
       "...        ...  ...\n",
       "19712    19712  30대\n",
       "19713    19713  10대\n",
       "19714    19714  20대\n",
       "19715    19715  30대\n",
       "19716    19716  20대\n",
       "\n",
       "[19717 rows x 2 columns]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_am[['User-ID', 'Age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISBN</th>\n",
       "      <th>Publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8141</td>\n",
       "      <td>HarperFlamingo Canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8142</td>\n",
       "      <td>HarperPerennial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8143</td>\n",
       "      <td>Farrar Straus Giroux</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8144</td>\n",
       "      <td>W. W. Norton &amp;amp; Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8145</td>\n",
       "      <td>Putnam Pub Group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256721</th>\n",
       "      <td>315861</td>\n",
       "      <td>Random House Childrens Pub (Mm)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256722</th>\n",
       "      <td>316009</td>\n",
       "      <td>Dutton Books</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256723</th>\n",
       "      <td>316064</td>\n",
       "      <td>HarperSanFrancisco</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256724</th>\n",
       "      <td>316066</td>\n",
       "      <td>Oxford University Press</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256725</th>\n",
       "      <td>316075</td>\n",
       "      <td>McGraw-Hill Humanities/Social Sciences/Languages</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>256726 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          ISBN                                         Publisher\n",
       "0         8141                             HarperFlamingo Canada\n",
       "1         8142                                   HarperPerennial\n",
       "2         8143                              Farrar Straus Giroux\n",
       "3         8144                        W. W. Norton &amp; Company\n",
       "4         8145                                  Putnam Pub Group\n",
       "...        ...                                               ...\n",
       "256721  315861                   Random House Childrens Pub (Mm)\n",
       "256722  316009                                      Dutton Books\n",
       "256723  316064                                HarperSanFrancisco\n",
       "256724  316066                           Oxford University Press\n",
       "256725  316075  McGraw-Hill Humanities/Social Sciences/Languages\n",
       "\n",
       "[256726 rows x 2 columns]"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items_am[['ISBN', 'Publisher']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISBN</th>\n",
       "      <th>Book-Title</th>\n",
       "      <th>Book-Author</th>\n",
       "      <th>Year-Of-Publication</th>\n",
       "      <th>Publisher</th>\n",
       "      <th>Image-URL-S</th>\n",
       "      <th>Image-URL-M</th>\n",
       "      <th>Image-URL-L</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8141</td>\n",
       "      <td>Clara Callan</td>\n",
       "      <td>Richard Bruce Wright</td>\n",
       "      <td>2001</td>\n",
       "      <td>HarperFlamingo Canada</td>\n",
       "      <td>http://images.amazon.com/images/P/0002005018.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0002005018.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0002005018.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8142</td>\n",
       "      <td>Decision in Normandy</td>\n",
       "      <td>Carlo D'Este</td>\n",
       "      <td>1991</td>\n",
       "      <td>HarperPerennial</td>\n",
       "      <td>http://images.amazon.com/images/P/0060973129.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0060973129.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0060973129.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8143</td>\n",
       "      <td>Flu: The Story of the Great Influenza Pandemic...</td>\n",
       "      <td>Gina Bari Kolata</td>\n",
       "      <td>1999</td>\n",
       "      <td>Farrar Straus Giroux</td>\n",
       "      <td>http://images.amazon.com/images/P/0374157065.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0374157065.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0374157065.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8144</td>\n",
       "      <td>The Mummies of Urumchi</td>\n",
       "      <td>E. J. W. Barber</td>\n",
       "      <td>1999</td>\n",
       "      <td>W. W. Norton &amp;amp; Company</td>\n",
       "      <td>http://images.amazon.com/images/P/0393045218.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0393045218.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0393045218.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8145</td>\n",
       "      <td>The Kitchen God's Wife</td>\n",
       "      <td>Amy Tan</td>\n",
       "      <td>1991</td>\n",
       "      <td>Putnam Pub Group</td>\n",
       "      <td>http://images.amazon.com/images/P/0399135782.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0399135782.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0399135782.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256721</th>\n",
       "      <td>315861</td>\n",
       "      <td>There's a Bat in Bunk Five</td>\n",
       "      <td>Paula Danziger</td>\n",
       "      <td>1988</td>\n",
       "      <td>Random House Childrens Pub (Mm)</td>\n",
       "      <td>http://images.amazon.com/images/P/0440400988.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0440400988.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0440400988.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256722</th>\n",
       "      <td>316009</td>\n",
       "      <td>From One to One Hundred</td>\n",
       "      <td>Teri Sloat</td>\n",
       "      <td>1991</td>\n",
       "      <td>Dutton Books</td>\n",
       "      <td>http://images.amazon.com/images/P/0525447644.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0525447644.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0525447644.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256723</th>\n",
       "      <td>316064</td>\n",
       "      <td>Lily Dale : The True Story of the Town that Ta...</td>\n",
       "      <td>Christine Wicker</td>\n",
       "      <td>2004</td>\n",
       "      <td>HarperSanFrancisco</td>\n",
       "      <td>http://images.amazon.com/images/P/006008667X.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/006008667X.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/006008667X.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256724</th>\n",
       "      <td>316066</td>\n",
       "      <td>Republic (World's Classics)</td>\n",
       "      <td>Plato</td>\n",
       "      <td>1996</td>\n",
       "      <td>Oxford University Press</td>\n",
       "      <td>http://images.amazon.com/images/P/0192126040.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0192126040.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0192126040.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256725</th>\n",
       "      <td>316075</td>\n",
       "      <td>A Guided Tour of Rene Descartes' Meditations o...</td>\n",
       "      <td>Christopher  Biffle</td>\n",
       "      <td>2000</td>\n",
       "      <td>McGraw-Hill Humanities/Social Sciences/Languages</td>\n",
       "      <td>http://images.amazon.com/images/P/0767409752.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0767409752.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0767409752.0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>256726 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          ISBN                                         Book-Title  \\\n",
       "0         8141                                       Clara Callan   \n",
       "1         8142                               Decision in Normandy   \n",
       "2         8143  Flu: The Story of the Great Influenza Pandemic...   \n",
       "3         8144                             The Mummies of Urumchi   \n",
       "4         8145                             The Kitchen God's Wife   \n",
       "...        ...                                                ...   \n",
       "256721  315861                         There's a Bat in Bunk Five   \n",
       "256722  316009                            From One to One Hundred   \n",
       "256723  316064  Lily Dale : The True Story of the Town that Ta...   \n",
       "256724  316066                        Republic (World's Classics)   \n",
       "256725  316075  A Guided Tour of Rene Descartes' Meditations o...   \n",
       "\n",
       "                 Book-Author Year-Of-Publication  \\\n",
       "0       Richard Bruce Wright                2001   \n",
       "1               Carlo D'Este                1991   \n",
       "2           Gina Bari Kolata                1999   \n",
       "3            E. J. W. Barber                1999   \n",
       "4                    Amy Tan                1991   \n",
       "...                      ...                 ...   \n",
       "256721        Paula Danziger                1988   \n",
       "256722            Teri Sloat                1991   \n",
       "256723      Christine Wicker                2004   \n",
       "256724                 Plato                1996   \n",
       "256725   Christopher  Biffle                2000   \n",
       "\n",
       "                                               Publisher  \\\n",
       "0                                  HarperFlamingo Canada   \n",
       "1                                        HarperPerennial   \n",
       "2                                   Farrar Straus Giroux   \n",
       "3                             W. W. Norton &amp; Company   \n",
       "4                                       Putnam Pub Group   \n",
       "...                                                  ...   \n",
       "256721                   Random House Childrens Pub (Mm)   \n",
       "256722                                      Dutton Books   \n",
       "256723                                HarperSanFrancisco   \n",
       "256724                           Oxford University Press   \n",
       "256725  McGraw-Hill Humanities/Social Sciences/Languages   \n",
       "\n",
       "                                              Image-URL-S  \\\n",
       "0       http://images.amazon.com/images/P/0002005018.0...   \n",
       "1       http://images.amazon.com/images/P/0060973129.0...   \n",
       "2       http://images.amazon.com/images/P/0374157065.0...   \n",
       "3       http://images.amazon.com/images/P/0393045218.0...   \n",
       "4       http://images.amazon.com/images/P/0399135782.0...   \n",
       "...                                                   ...   \n",
       "256721  http://images.amazon.com/images/P/0440400988.0...   \n",
       "256722  http://images.amazon.com/images/P/0525447644.0...   \n",
       "256723  http://images.amazon.com/images/P/006008667X.0...   \n",
       "256724  http://images.amazon.com/images/P/0192126040.0...   \n",
       "256725  http://images.amazon.com/images/P/0767409752.0...   \n",
       "\n",
       "                                              Image-URL-M  \\\n",
       "0       http://images.amazon.com/images/P/0002005018.0...   \n",
       "1       http://images.amazon.com/images/P/0060973129.0...   \n",
       "2       http://images.amazon.com/images/P/0374157065.0...   \n",
       "3       http://images.amazon.com/images/P/0393045218.0...   \n",
       "4       http://images.amazon.com/images/P/0399135782.0...   \n",
       "...                                                   ...   \n",
       "256721  http://images.amazon.com/images/P/0440400988.0...   \n",
       "256722  http://images.amazon.com/images/P/0525447644.0...   \n",
       "256723  http://images.amazon.com/images/P/006008667X.0...   \n",
       "256724  http://images.amazon.com/images/P/0192126040.0...   \n",
       "256725  http://images.amazon.com/images/P/0767409752.0...   \n",
       "\n",
       "                                              Image-URL-L  \n",
       "0       http://images.amazon.com/images/P/0002005018.0...  \n",
       "1       http://images.amazon.com/images/P/0060973129.0...  \n",
       "2       http://images.amazon.com/images/P/0374157065.0...  \n",
       "3       http://images.amazon.com/images/P/0393045218.0...  \n",
       "4       http://images.amazon.com/images/P/0399135782.0...  \n",
       "...                                                   ...  \n",
       "256721  http://images.amazon.com/images/P/0440400988.0...  \n",
       "256722  http://images.amazon.com/images/P/0525447644.0...  \n",
       "256723  http://images.amazon.com/images/P/006008667X.0...  \n",
       "256724  http://images.amazon.com/images/P/0192126040.0...  \n",
       "256725  http://images.amazon.com/images/P/0767409752.0...  \n",
       "\n",
       "[256726 rows x 8 columns]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items_am"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    15575.000000\n",
       "mean        16.483082\n",
       "std        129.512730\n",
       "min          1.000000\n",
       "25%          1.000000\n",
       "50%          1.000000\n",
       "75%          4.000000\n",
       "max       7506.000000\n",
       "Name: ISBN, dtype: float64"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items_am.groupby('Publisher')['ISBN'].count().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    15575.000000\n",
       "mean        16.483082\n",
       "std        129.512730\n",
       "min          1.000000\n",
       "25%          1.000000\n",
       "50%          1.000000\n",
       "75%          4.000000\n",
       "max       7506.000000\n",
       "Name: Year-Of-Publication, dtype: float64"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items_am.groupby('Publisher')['Year-Of-Publication'].count().describe()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyOj7hfTZ1vV3I/Kt4i8D/dp",
   "collapsed_sections": [],
   "machine_shape": "hm",
   "mount_file_id": "19jgbstTjrn2n8b0V9_lm6F6jgqnmzj5s",
   "name": "data.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}