Handle Categorical Features Many Categories Count or Frequency Encoding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "641d0451",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count or Frequency Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8a3b5300",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "4c55fa01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>y</th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X8</th>\n",
       "      <th>...</th>\n",
       "      <th>X375</th>\n",
       "      <th>X376</th>\n",
       "      <th>X377</th>\n",
       "      <th>X378</th>\n",
       "      <th>X379</th>\n",
       "      <th>X380</th>\n",
       "      <th>X382</th>\n",
       "      <th>X383</th>\n",
       "      <th>X384</th>\n",
       "      <th>X385</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>130.81</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>a</td>\n",
       "      <td>d</td>\n",
       "      <td>u</td>\n",
       "      <td>j</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>88.53</td>\n",
       "      <td>k</td>\n",
       "      <td>t</td>\n",
       "      <td>av</td>\n",
       "      <td>e</td>\n",
       "      <td>d</td>\n",
       "      <td>y</td>\n",
       "      <td>l</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>76.26</td>\n",
       "      <td>az</td>\n",
       "      <td>w</td>\n",
       "      <td>n</td>\n",
       "      <td>c</td>\n",
       "      <td>d</td>\n",
       "      <td>x</td>\n",
       "      <td>j</td>\n",
       "      <td>x</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>80.62</td>\n",
       "      <td>az</td>\n",
       "      <td>t</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>d</td>\n",
       "      <td>x</td>\n",
       "      <td>l</td>\n",
       "      <td>e</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>78.02</td>\n",
       "      <td>az</td>\n",
       "      <td>v</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>d</td>\n",
       "      <td>h</td>\n",
       "      <td>d</td>\n",
       "      <td>n</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 378 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       y  X0 X1  X2 X3 X4 X5 X6 X8  ...  X375  X376  X377  X378  X379  \\\n",
       "0   0  130.81   k  v  at  a  d  u  j  o  ...     0     0     1     0     0   \n",
       "1   6   88.53   k  t  av  e  d  y  l  o  ...     1     0     0     0     0   \n",
       "2   7   76.26  az  w   n  c  d  x  j  x  ...     0     0     0     0     0   \n",
       "3   9   80.62  az  t   n  f  d  x  l  e  ...     0     0     0     0     0   \n",
       "4  13   78.02  az  v   n  f  d  h  d  n  ...     0     0     0     0     0   \n",
       "\n",
       "   X380  X382  X383  X384  X385  \n",
       "0     0     0     0     0     0  \n",
       "1     0     0     0     0     0  \n",
       "2     0     1     0     0     0  \n",
       "3     0     0     0     0     0  \n",
       "4     0     0     0     0     0  \n",
       "\n",
       "[5 rows x 378 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"mercedes.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8e717c4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4209, 565)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(data).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "38c546db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data[\"X1\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "31792fd0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data[\"X2\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d1bff2b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ID', 'y', 'X0', 'X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'X8',\n",
       "       ...\n",
       "       'X375', 'X376', 'X377', 'X378', 'X379', 'X380', 'X382', 'X383', 'X384',\n",
       "       'X385'],\n",
       "      dtype='object', length=378)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns[0:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b1b24819",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets count the categories/levels in the categorical variable X2 and convert the same into a dict (re-mapping purpose)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2f3b1d22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'as': 1659,\n",
       " 'ae': 496,\n",
       " 'ai': 415,\n",
       " 'm': 367,\n",
       " 'ak': 265,\n",
       " 'r': 153,\n",
       " 'n': 137,\n",
       " 's': 94,\n",
       " 'f': 87,\n",
       " 'e': 81,\n",
       " 'aq': 63,\n",
       " 'ay': 54,\n",
       " 'a': 47,\n",
       " 't': 29,\n",
       " 'i': 25,\n",
       " 'k': 25,\n",
       " 'b': 21,\n",
       " 'ao': 20,\n",
       " 'z': 19,\n",
       " 'ag': 19,\n",
       " 'd': 18,\n",
       " 'ac': 13,\n",
       " 'g': 12,\n",
       " 'y': 11,\n",
       " 'ap': 11,\n",
       " 'x': 10,\n",
       " 'aw': 8,\n",
       " 'h': 6,\n",
       " 'at': 6,\n",
       " 'q': 5,\n",
       " 'al': 5,\n",
       " 'an': 5,\n",
       " 'ah': 4,\n",
       " 'av': 4,\n",
       " 'p': 4,\n",
       " 'au': 3,\n",
       " 'o': 1,\n",
       " 'ar': 1,\n",
       " 'l': 1,\n",
       " 'aa': 1,\n",
       " 'am': 1,\n",
       " 'c': 1,\n",
       " 'j': 1,\n",
       " 'af': 1}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"X2\"].value_counts().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b706a235",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will now replace each levels in X2 with it's count value - mapping "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "86d2d63d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'as': 1659,\n",
       " 'ae': 496,\n",
       " 'ai': 415,\n",
       " 'm': 367,\n",
       " 'ak': 265,\n",
       " 'r': 153,\n",
       " 'n': 137,\n",
       " 's': 94,\n",
       " 'f': 87,\n",
       " 'e': 81,\n",
       " 'aq': 63,\n",
       " 'ay': 54,\n",
       " 'a': 47,\n",
       " 't': 29,\n",
       " 'i': 25,\n",
       " 'k': 25,\n",
       " 'b': 21,\n",
       " 'ao': 20,\n",
       " 'z': 19,\n",
       " 'ag': 19,\n",
       " 'd': 18,\n",
       " 'ac': 13,\n",
       " 'g': 12,\n",
       " 'y': 11,\n",
       " 'ap': 11,\n",
       " 'x': 10,\n",
       " 'aw': 8,\n",
       " 'h': 6,\n",
       " 'at': 6,\n",
       " 'q': 5,\n",
       " 'al': 5,\n",
       " 'an': 5,\n",
       " 'ah': 4,\n",
       " 'av': 4,\n",
       " 'p': 4,\n",
       " 'au': 3,\n",
       " 'o': 1,\n",
       " 'ar': 1,\n",
       " 'l': 1,\n",
       " 'aa': 1,\n",
       " 'am': 1,\n",
       " 'c': 1,\n",
       " 'j': 1,\n",
       " 'af': 1}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x2_count_levels = data.X2.value_counts().to_dict()\n",
    "x2_count_levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4c1f40c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.X2 = data.X2.map(x2_count_levels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3977dfab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          6\n",
       "1          4\n",
       "2        137\n",
       "3        137\n",
       "4        137\n",
       "        ... \n",
       "4204    1659\n",
       "4205      29\n",
       "4206     153\n",
       "4207      81\n",
       "4208     496\n",
       "Name: X2, Length: 4209, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.X2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "8921d52f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>y</th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X8</th>\n",
       "      <th>...</th>\n",
       "      <th>X375</th>\n",
       "      <th>X376</th>\n",
       "      <th>X377</th>\n",
       "      <th>X378</th>\n",
       "      <th>X379</th>\n",
       "      <th>X380</th>\n",
       "      <th>X382</th>\n",
       "      <th>X383</th>\n",
       "      <th>X384</th>\n",
       "      <th>X385</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>130.81</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>6</td>\n",
       "      <td>a</td>\n",
       "      <td>d</td>\n",
       "      <td>u</td>\n",
       "      <td>j</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>88.53</td>\n",
       "      <td>k</td>\n",
       "      <td>t</td>\n",
       "      <td>4</td>\n",
       "      <td>e</td>\n",
       "      <td>d</td>\n",
       "      <td>y</td>\n",
       "      <td>l</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>76.26</td>\n",
       "      <td>az</td>\n",
       "      <td>w</td>\n",
       "      <td>137</td>\n",
       "      <td>c</td>\n",
       "      <td>d</td>\n",
       "      <td>x</td>\n",
       "      <td>j</td>\n",
       "      <td>x</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>80.62</td>\n",
       "      <td>az</td>\n",
       "      <td>t</td>\n",
       "      <td>137</td>\n",
       "      <td>f</td>\n",
       "      <td>d</td>\n",
       "      <td>x</td>\n",
       "      <td>l</td>\n",
       "      <td>e</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>78.02</td>\n",
       "      <td>az</td>\n",
       "      <td>v</td>\n",
       "      <td>137</td>\n",
       "      <td>f</td>\n",
       "      <td>d</td>\n",
       "      <td>h</td>\n",
       "      <td>d</td>\n",
       "      <td>n</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 378 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       y  X0 X1   X2 X3 X4 X5 X6 X8  ...  X375  X376  X377  X378  X379  \\\n",
       "0   0  130.81   k  v    6  a  d  u  j  o  ...     0     0     1     0     0   \n",
       "1   6   88.53   k  t    4  e  d  y  l  o  ...     1     0     0     0     0   \n",
       "2   7   76.26  az  w  137  c  d  x  j  x  ...     0     0     0     0     0   \n",
       "3   9   80.62  az  t  137  f  d  x  l  e  ...     0     0     0     0     0   \n",
       "4  13   78.02  az  v  137  f  d  h  d  n  ...     0     0     0     0     0   \n",
       "\n",
       "   X380  X382  X383  X384  X385  \n",
       "0     0     0     0     0     0  \n",
       "1     0     0     0     0     0  \n",
       "2     0     1     0     0     0  \n",
       "3     0     0     0     0     0  \n",
       "4     0     0     0     0     0  \n",
       "\n",
       "[5 rows x 378 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9c6592b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e79ce94",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09ab855b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2ab19dd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "331b6a8c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d4f5fca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db1a9a4b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee0a0709",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}