data_preprocessing_python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "37puETfgRzzg"
   },
   "source": [
    "# Data Preprocessing Tools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EoRP98MpR-qj"
   },
   "source": [
    "## Importing the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N-qiINBQSK2g"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RopL7tUZSQkT"
   },
   "source": [
    "## Importing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WwEPNDWySTKm"
   },
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('Data.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Age</th>\n",
       "      <th>Salary</th>\n",
       "      <th>Purchased</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>France</td>\n",
       "      <td>44.0</td>\n",
       "      <td>72000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Spain</td>\n",
       "      <td>27.0</td>\n",
       "      <td>48000.0</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Germany</td>\n",
       "      <td>30.0</td>\n",
       "      <td>54000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Spain</td>\n",
       "      <td>38.0</td>\n",
       "      <td>61000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Germany</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Country   Age   Salary Purchased\n",
       "0   France  44.0  72000.0        No\n",
       "1    Spain  27.0  48000.0       Yes\n",
       "2  Germany  30.0  54000.0        No\n",
       "3    Spain  38.0  61000.0        No\n",
       "4  Germany  40.0      NaN       Yes"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 4)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10 entries, 0 to 9\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Non-Null Count  Dtype  \n",
      "---  ------     --------------  -----  \n",
      " 0   Country    10 non-null     object \n",
      " 1   Age        9 non-null      float64\n",
      " 2   Salary     9 non-null      float64\n",
      " 3   Purchased  10 non-null     object \n",
      "dtypes: float64(2), object(2)\n",
      "memory usage: 448.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 188
    },
    "colab_type": "code",
    "id": "hCsz2yCebe1R",
    "outputId": "1e4cc568-4e51-4b38-9d46-4aa3f15204be"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Country', 'Age', 'Salary', 'Purchased'], dtype='object')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Country      0\n",
       "Age          1\n",
       "Salary       1\n",
       "Purchased    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "eYrOQ43XcJR3",
    "outputId": "e0873b2a-3b08-4bab-ef0d-15b88858ca44"
   },
   "outputs": [],
   "source": [
    "X = dataset.iloc[:, :-1].values\n",
    "y = dataset.iloc[:, -1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['France', 44.0, 72000.0],\n",
       "       ['Spain', 27.0, 48000.0],\n",
       "       ['Germany', 30.0, 54000.0],\n",
       "       ['Spain', 38.0, 61000.0],\n",
       "       ['Germany', 40.0, nan],\n",
       "       ['France', 35.0, 58000.0],\n",
       "       ['Spain', nan, 52000.0],\n",
       "       ['France', 48.0, 79000.0],\n",
       "       ['Germany', 50.0, 83000.0],\n",
       "       ['France', 37.0, 67000.0]], dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['No', 'Yes', 'No', 'No', 'Yes', 'Yes', 'No', 'Yes', 'No', 'Yes'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nhfKXNxlSabC"
   },
   "source": [
    "## Taking care of missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "c93k7ipkSexq"
   },
   "outputs": [],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n",
    "imputer.fit(X[:, 1:3])\n",
    "X[:, 1:3] = imputer.transform(X[:, 1:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 188
    },
    "colab_type": "code",
    "id": "3UgLdMS_bjq_",
    "outputId": "254af4e0-681e-47f5-aaa7-b9c6f43258e9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['France' 44.0 72000.0]\n",
      " ['Spain' 27.0 48000.0]\n",
      " ['Germany' 30.0 54000.0]\n",
      " ['Spain' 38.0 61000.0]\n",
      " ['Germany' 40.0 63777.77777777778]\n",
      " ['France' 35.0 58000.0]\n",
      " ['Spain' 38.77777777777778 52000.0]\n",
      " ['France' 48.0 79000.0]\n",
      " ['Germany' 50.0 83000.0]\n",
      " ['France' 37.0 67000.0]]\n"
     ]
    }
   ],
   "source": [
    "print(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CriG6VzVSjcK"
   },
   "source": [
    "## Encoding categorical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Germany</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Germany</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Germany</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Country\n",
       "0   France\n",
       "1    Spain\n",
       "2  Germany\n",
       "3    Spain\n",
       "4  Germany\n",
       "5   France\n",
       "6    Spain\n",
       "7   France\n",
       "8  Germany\n",
       "9   France"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[[\"Country\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AhSpdQWeSsFh"
   },
   "source": [
    "### Encoding the Independent Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5hwuVddlSwVi"
   },
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [0])], remainder='passthrough')\n",
    "X = np.array(ct.fit_transform(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 188
    },
    "colab_type": "code",
    "id": "f7QspewyeBfx",
    "outputId": "5b35feef-7fe2-46ef-ce70-80495f94f4ed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.0 0.0 0.0 44.0 72000.0]\n",
      " [0.0 0.0 1.0 27.0 48000.0]\n",
      " [0.0 1.0 0.0 30.0 54000.0]\n",
      " [0.0 0.0 1.0 38.0 61000.0]\n",
      " [0.0 1.0 0.0 40.0 63777.77777777778]\n",
      " [1.0 0.0 0.0 35.0 58000.0]\n",
      " [0.0 0.0 1.0 38.77777777777778 52000.0]\n",
      " [1.0 0.0 0.0 48.0 79000.0]\n",
      " [0.0 1.0 0.0 50.0 83000.0]\n",
      " [1.0 0.0 0.0 37.0 67000.0]]\n"
     ]
    }
   ],
   "source": [
    "print(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DXh8oVSITIc6"
   },
   "source": [
    "### Encoding the Dependent Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XgHCShVyTOYY"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "y = le.fit_transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['No', 'Yes', 'No', 'No', 'Yes', 'Yes', 'No', 'Yes', 'No', 'Yes'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "FyhY8-gPpFCa",
    "outputId": "7f76ef29-5423-4c3e-cf69-45fbc366a997"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 0 0 1 1 0 1 0 1]\n"
     ]
    }
   ],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qb_vcgm3qZKW"
   },
   "source": [
    "## Splitting the dataset into the Training set and Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pXgA6CzlqbCl"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 154
    },
    "colab_type": "code",
    "id": "GuwQhFdKrYTM",
    "outputId": "de1e527f-c229-4daf-e7c5-ea9d2485148d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.0 0.0 1.0 38.77777777777778 52000.0]\n",
      " [0.0 1.0 0.0 40.0 63777.77777777778]\n",
      " [1.0 0.0 0.0 44.0 72000.0]\n",
      " [0.0 0.0 1.0 38.0 61000.0]\n",
      " [0.0 0.0 1.0 27.0 48000.0]\n",
      " [1.0 0.0 0.0 48.0 79000.0]\n",
      " [0.0 1.0 0.0 50.0 83000.0]\n",
      " [1.0 0.0 0.0 35.0 58000.0]]\n"
     ]
    }
   ],
   "source": [
    "print(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "TUrX_Tvcrbi4",
    "outputId": "9a041a9b-2642-4828-fa2f-a431d7d77631"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.0 1.0 0.0 30.0 54000.0]\n",
      " [1.0 0.0 0.0 37.0 67000.0]]\n"
     ]
    }
   ],
   "source": [
    "print(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "pSMHiIsWreQY",
    "outputId": "5afe91e0-9244-4bf5-ec1b-e3e092b85c08"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 0 0 1 1 0 1]\n"
     ]
    }
   ],
   "source": [
    "print(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "I_tW7H56rgtW",
    "outputId": "2a93f141-2a99-4a69-eec5-c82a3bb8d36b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1]\n"
     ]
    }
   ],
   "source": [
    "print(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TpGqbS4TqkIR"
   },
   "source": [
    "## Feature Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AxjSUXFQqo-3"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "sc = StandardScaler()\n",
    "X_train[:, 3:] = sc.fit_transform(X_train[:, 3:])\n",
    "X_test[:, 3:] = sc.transform(X_test[:, 3:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 154
    },
    "colab_type": "code",
    "id": "DWPET8ZdlMnu",
    "outputId": "dea86927-5124-4e2a-e974-2804df9a913c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.0 0.0 1.0 -0.19159184384578545 -1.0781259408412425]\n",
      " [0.0 1.0 0.0 -0.014117293757057777 -0.07013167641635372]\n",
      " [1.0 0.0 0.0 0.566708506533324 0.633562432710455]\n",
      " [0.0 0.0 1.0 -0.30453019390224867 -0.30786617274297867]\n",
      " [0.0 0.0 1.0 -1.9018011447007988 -1.420463615551582]\n",
      " [1.0 0.0 0.0 1.1475343068237058 1.232653363453549]\n",
      " [0.0 1.0 0.0 1.4379472069688968 1.5749910381638885]\n",
      " [1.0 0.0 0.0 -0.7401495441200351 -0.5646194287757332]]\n"
     ]
    }
   ],
   "source": [
    "print(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "sTXykB_QlRjE",
    "outputId": "b68f0cfc-d07c-48cb-80d0-6800028c41f9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.0 1.0 0.0 -1.4661817944830124 -0.9069571034860727]\n",
      " [1.0 0.0 0.0 -0.44973664397484414 0.2056403393225306]]\n"
     ]
    }
   ],
   "source": [
    "print(X_test)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "data_preprocessing_tools.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}