Sample Code/Day_11_SampleCode.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"類別資料與缺值處理.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"5uxdeW8LlGtO"},"source":["範例題目:<br>\r\n","1. 認識類別資料\r\n","2. 實做缺值處理方法與應用函式\r\n"]},{"cell_type":"markdown","metadata":{"id":"X2pH-Mu6lQVz"},"source":["範例重點:<br>\r\n","1. 類別資料，有分順序型與一般型，使用的編碼方式不同\r\n","2. 缺失值有很多處理方式，在這邊簡單介紹2種常見的方式"]},{"cell_type":"code","metadata":{"id":"Tqp3yFkXlPfC"},"source":["import pandas as pd"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":142},"id":"365x9GdjkZBU","executionInfo":{"status":"ok","timestamp":1606481914752,"user_tz":-480,"elapsed":2481,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"27fe50fb-39ec-424f-8b46-552462fd9daa"},"source":["df = pd.DataFrame([['green', 'M', 'male', 'short'],\n","          ['red', 'L', 'female', 'normal'],\n","          ['blue', 'XL', 'male', 'long']])\n","df.columns =['color', 'size', 'sex', 'lenght']\n","df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>color</th>\n","      <th>size</th>\n","      <th>sex</th>\n","      <th>lenght</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>green</td>\n","      <td>M</td>\n","      <td>male</td>\n","      <td>short</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>red</td>\n","      <td>L</td>\n","      <td>female</td>\n","      <td>normal</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>blue</td>\n","      <td>XL</td>\n","      <td>male</td>\n","      <td>long</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   color size     sex  lenght\n","0  green    M    male   short\n","1    red    L  female  normal\n","2   blue   XL    male    long"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":142},"id":"S_xdZBoLvKsI","executionInfo":{"status":"ok","timestamp":1606481914753,"user_tz":-480,"elapsed":2471,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"7651df33-e881-413e-951d-f54b63adf52a"},"source":["#順序性類別資料，編碼也需要有順序性，將類別資料依序編碼由0到n-1，其中n為類別總數，因此類別之間會有順序關係0<1<2<….\n","from sklearn.preprocessing import LabelEncoder\n","df['size_label'] = LabelEncoder().fit_transform(df['size'].values)\n","df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>color</th>\n","      <th>size</th>\n","      <th>sex</th>\n","      <th>lenght</th>\n","      <th>size_label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>green</td>\n","      <td>M</td>\n","      <td>male</td>\n","      <td>short</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>red</td>\n","      <td>L</td>\n","      <td>female</td>\n","      <td>normal</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>blue</td>\n","      <td>XL</td>\n","      <td>male</td>\n","      <td>long</td>\n","      <td>2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   color size     sex  lenght  size_label\n","0  green    M    male   short           1\n","1    red    L  female  normal           0\n","2   blue   XL    male    long           2"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"p3az-ibkxKEF","executionInfo":{"status":"ok","timestamp":1606482051931,"user_tz":-480,"elapsed":984,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"695e4162-f3dc-4f92-d4cc-b2c361dfd8fc"},"source":["#排序依照python內建順序，可以藉由ord()查看內建順序\r\n","ord('L'),ord('M')"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(76, 77)"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":142},"id":"-5NGETtHkaVO","executionInfo":{"status":"ok","timestamp":1606482203879,"user_tz":-480,"elapsed":1016,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"19bfccdf-fc15-47b0-a686-540293a062cf"},"source":["#get_dummies()把資料表中的每個類別 對應的欄位，經過One-hot Encoding(一位有效編碼)\n","#One-hot Encoding(一位有效編碼) 是沒有順序性的編碼\n","pf = pd.get_dummies(df[['color']])\n","df = pd.concat([df, pf], axis=1)\n","df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>color</th>\n","      <th>size</th>\n","      <th>sex</th>\n","      <th>lenght</th>\n","      <th>size_label</th>\n","      <th>color_blue</th>\n","      <th>color_green</th>\n","      <th>color_red</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>green</td>\n","      <td>M</td>\n","      <td>male</td>\n","      <td>short</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>red</td>\n","      <td>L</td>\n","      <td>female</td>\n","      <td>normal</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>blue</td>\n","      <td>XL</td>\n","      <td>male</td>\n","      <td>long</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   color size     sex  lenght  size_label  color_blue  color_green  color_red\n","0  green    M    male   short           1           0            1          0\n","1    red    L  female  normal           0           0            0          1\n","2   blue   XL    male    long           2           1            0          0"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"eXuVJEjcklK9","executionInfo":{"status":"ok","timestamp":1606483846756,"user_tz":-480,"elapsed":589,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"8418587e-8361-4023-e9c9-86717270547d"},"source":["temp_data = pd.DataFrame([['2020-11-01', 24.8], \n","              ['2020-11-02', 24.8],\n","              ['2020-11-03', None],\n","              ['2020-11-04', 25]],columns=['date','current_temp'])\n","temp_data"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>date</th>\n","      <th>current_temp</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2020-11-01</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2020-11-02</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2020-11-03</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2020-11-04</td>\n","      <td>25.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         date  current_temp\n","0  2020-11-01          24.8\n","1  2020-11-02          24.8\n","2  2020-11-03           NaN\n","3  2020-11-04          25.0"]},"metadata":{"tags":[]},"execution_count":15}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"4B6h9VoQ6bnh","executionInfo":{"status":"ok","timestamp":1606484087932,"user_tz":-480,"elapsed":568,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"fba66d61-992b-40d6-d216-12cdf6cf8ff5"},"source":["#以0填補\n","temp_data.fillna(0)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>date</th>\n","      <th>current_temp</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2020-11-01</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2020-11-02</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2020-11-03</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2020-11-04</td>\n","      <td>25.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         date  current_temp\n","0  2020-11-01          24.8\n","1  2020-11-02          24.8\n","2  2020-11-03           0.0\n","3  2020-11-04          25.0"]},"metadata":{"tags":[]},"execution_count":21}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"mND2DX_26WS9","executionInfo":{"status":"ok","timestamp":1606484071535,"user_tz":-480,"elapsed":788,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"2aec3981-1fe6-44ce-de3c-04997a7a31b8"},"source":["#以該欄位所有資料的算術平均數做填補\n","temp_data.fillna(temp_data.current_temp.mean())"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>date</th>\n","      <th>current_temp</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2020-11-01</td>\n","      <td>24.800000</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2020-11-02</td>\n","      <td>24.800000</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2020-11-03</td>\n","      <td>24.866667</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2020-11-04</td>\n","      <td>25.000000</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         date  current_temp\n","0  2020-11-01     24.800000\n","1  2020-11-02     24.800000\n","2  2020-11-03     24.866667\n","3  2020-11-04     25.000000"]},"metadata":{"tags":[]},"execution_count":20}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"-znesYczlWxV","executionInfo":{"status":"ok","timestamp":1606484285678,"user_tz":-480,"elapsed":882,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"1f1596dc-7a0b-44fb-c64f-cf129855fcb5"},"source":["#以該欄位所有資料的中位數做填補\n","temp_data.fillna(temp_data.current_temp.median())"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>date</th>\n","      <th>current_temp</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2020-11-01</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2020-11-02</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2020-11-03</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2020-11-04</td>\n","      <td>25.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         date  current_temp\n","0  2020-11-01          24.8\n","1  2020-11-02          24.8\n","2  2020-11-03          24.8\n","3  2020-11-04          25.0"]},"metadata":{"tags":[]},"execution_count":24}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"jKRhofKgojLJ","executionInfo":{"status":"ok","timestamp":1606484483835,"user_tz":-480,"elapsed":948,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"44d07ef4-8261-4260-d271-49d0486d1cb7"},"source":["#運用參數method=‘ffill’即可填補前一列數值\r\n","temp_data.fillna(method='ffill')"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>date</th>\n","      <th>current_temp</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2020-11-01</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2020-11-02</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2020-11-03</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2020-11-04</td>\n","      <td>25.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         date  current_temp\n","0  2020-11-01          24.8\n","1  2020-11-02          24.8\n","2  2020-11-03          24.8\n","3  2020-11-04          25.0"]},"metadata":{"tags":[]},"execution_count":26}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"3avR-gbSowrl","executionInfo":{"status":"ok","timestamp":1606484494001,"user_tz":-480,"elapsed":874,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"69069ab9-cb82-460d-f4dc-24b942991f9d"},"source":["#method=‘bfill’填補後一列數值\r\n","temp_data.fillna(method='bfill')"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>date</th>\n","      <th>current_temp</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2020-11-01</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2020-11-02</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2020-11-03</td>\n","      <td>25.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2020-11-04</td>\n","      <td>25.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         date  current_temp\n","0  2020-11-01          24.8\n","1  2020-11-02          24.8\n","2  2020-11-03          25.0\n","3  2020-11-04          25.0"]},"metadata":{"tags":[]},"execution_count":27}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"Dp1d6r0j8AMC","executionInfo":{"status":"ok","timestamp":1606485231887,"user_tz":-480,"elapsed":1001,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"afb9ff6d-bd6e-48c4-dc7d-4b4513813036"},"source":["#內差法補值\r\n","temp_data.interpolate()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>date</th>\n","      <th>current_temp</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>2020-11-01</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2020-11-02</td>\n","      <td>24.8</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2020-11-03</td>\n","      <td>24.9</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>2020-11-04</td>\n","      <td>25.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         date  current_temp\n","0  2020-11-01          24.8\n","1  2020-11-02          24.8\n","2  2020-11-03          24.9\n","3  2020-11-04          25.0"]},"metadata":{"tags":[]},"execution_count":28}]},{"cell_type":"code","metadata":{"id":"__8P6Gw6-0T3"},"source":[""],"execution_count":null,"outputs":[]}]}