try_multi_label_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy\n",
    "from scipy.io import arff\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data, meta = scipy.io.arff.loadarff('yeast/yeast-train.arff')\n",
    "df = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset: MultiLabelData\n",
       "\tAtt1's type is numeric\n",
       "\tAtt2's type is numeric\n",
       "\tAtt3's type is numeric\n",
       "\tAtt4's type is numeric\n",
       "\tAtt5's type is numeric\n",
       "\tAtt6's type is numeric\n",
       "\tAtt7's type is numeric\n",
       "\tAtt8's type is numeric\n",
       "\tAtt9's type is numeric\n",
       "\tAtt10's type is numeric\n",
       "\tAtt11's type is numeric\n",
       "\tAtt12's type is numeric\n",
       "\tAtt13's type is numeric\n",
       "\tAtt14's type is numeric\n",
       "\tAtt15's type is numeric\n",
       "\tAtt16's type is numeric\n",
       "\tAtt17's type is numeric\n",
       "\tAtt18's type is numeric\n",
       "\tAtt19's type is numeric\n",
       "\tAtt20's type is numeric\n",
       "\tAtt21's type is numeric\n",
       "\tAtt22's type is numeric\n",
       "\tAtt23's type is numeric\n",
       "\tAtt24's type is numeric\n",
       "\tAtt25's type is numeric\n",
       "\tAtt26's type is numeric\n",
       "\tAtt27's type is numeric\n",
       "\tAtt28's type is numeric\n",
       "\tAtt29's type is numeric\n",
       "\tAtt30's type is numeric\n",
       "\tAtt31's type is numeric\n",
       "\tAtt32's type is numeric\n",
       "\tAtt33's type is numeric\n",
       "\tAtt34's type is numeric\n",
       "\tAtt35's type is numeric\n",
       "\tAtt36's type is numeric\n",
       "\tAtt37's type is numeric\n",
       "\tAtt38's type is numeric\n",
       "\tAtt39's type is numeric\n",
       "\tAtt40's type is numeric\n",
       "\tAtt41's type is numeric\n",
       "\tAtt42's type is numeric\n",
       "\tAtt43's type is numeric\n",
       "\tAtt44's type is numeric\n",
       "\tAtt45's type is numeric\n",
       "\tAtt46's type is numeric\n",
       "\tAtt47's type is numeric\n",
       "\tAtt48's type is numeric\n",
       "\tAtt49's type is numeric\n",
       "\tAtt50's type is numeric\n",
       "\tAtt51's type is numeric\n",
       "\tAtt52's type is numeric\n",
       "\tAtt53's type is numeric\n",
       "\tAtt54's type is numeric\n",
       "\tAtt55's type is numeric\n",
       "\tAtt56's type is numeric\n",
       "\tAtt57's type is numeric\n",
       "\tAtt58's type is numeric\n",
       "\tAtt59's type is numeric\n",
       "\tAtt60's type is numeric\n",
       "\tAtt61's type is numeric\n",
       "\tAtt62's type is numeric\n",
       "\tAtt63's type is numeric\n",
       "\tAtt64's type is numeric\n",
       "\tAtt65's type is numeric\n",
       "\tAtt66's type is numeric\n",
       "\tAtt67's type is numeric\n",
       "\tAtt68's type is numeric\n",
       "\tAtt69's type is numeric\n",
       "\tAtt70's type is numeric\n",
       "\tAtt71's type is numeric\n",
       "\tAtt72's type is numeric\n",
       "\tAtt73's type is numeric\n",
       "\tAtt74's type is numeric\n",
       "\tAtt75's type is numeric\n",
       "\tAtt76's type is numeric\n",
       "\tAtt77's type is numeric\n",
       "\tAtt78's type is numeric\n",
       "\tAtt79's type is numeric\n",
       "\tAtt80's type is numeric\n",
       "\tAtt81's type is numeric\n",
       "\tAtt82's type is numeric\n",
       "\tAtt83's type is numeric\n",
       "\tAtt84's type is numeric\n",
       "\tAtt85's type is numeric\n",
       "\tAtt86's type is numeric\n",
       "\tAtt87's type is numeric\n",
       "\tAtt88's type is numeric\n",
       "\tAtt89's type is numeric\n",
       "\tAtt90's type is numeric\n",
       "\tAtt91's type is numeric\n",
       "\tAtt92's type is numeric\n",
       "\tAtt93's type is numeric\n",
       "\tAtt94's type is numeric\n",
       "\tAtt95's type is numeric\n",
       "\tAtt96's type is numeric\n",
       "\tAtt97's type is numeric\n",
       "\tAtt98's type is numeric\n",
       "\tAtt99's type is numeric\n",
       "\tAtt100's type is numeric\n",
       "\tAtt101's type is numeric\n",
       "\tAtt102's type is numeric\n",
       "\tAtt103's type is numeric\n",
       "\tClass1's type is nominal, range is ('0', '1')\n",
       "\tClass2's type is nominal, range is ('0', '1')\n",
       "\tClass3's type is nominal, range is ('0', '1')\n",
       "\tClass4's type is nominal, range is ('0', '1')\n",
       "\tClass5's type is nominal, range is ('0', '1')\n",
       "\tClass6's type is nominal, range is ('0', '1')\n",
       "\tClass7's type is nominal, range is ('0', '1')\n",
       "\tClass8's type is nominal, range is ('0', '1')\n",
       "\tClass9's type is nominal, range is ('0', '1')\n",
       "\tClass10's type is nominal, range is ('0', '1')\n",
       "\tClass11's type is nominal, range is ('0', '1')\n",
       "\tClass12's type is nominal, range is ('0', '1')\n",
       "\tClass13's type is nominal, range is ('0', '1')\n",
       "\tClass14's type is nominal, range is ('0', '1')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Att1</th>\n",
       "      <th>Att2</th>\n",
       "      <th>Att3</th>\n",
       "      <th>Att4</th>\n",
       "      <th>Att5</th>\n",
       "      <th>Att6</th>\n",
       "      <th>Att7</th>\n",
       "      <th>Att8</th>\n",
       "      <th>Att9</th>\n",
       "      <th>Att10</th>\n",
       "      <th>...</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.093700</td>\n",
       "      <td>0.139771</td>\n",
       "      <td>0.062774</td>\n",
       "      <td>0.007698</td>\n",
       "      <td>0.083873</td>\n",
       "      <td>-0.119156</td>\n",
       "      <td>0.073305</td>\n",
       "      <td>0.005510</td>\n",
       "      <td>0.027523</td>\n",
       "      <td>0.043477</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.022711</td>\n",
       "      <td>-0.050504</td>\n",
       "      <td>-0.035691</td>\n",
       "      <td>-0.065434</td>\n",
       "      <td>-0.084316</td>\n",
       "      <td>-0.378560</td>\n",
       "      <td>0.038212</td>\n",
       "      <td>0.085770</td>\n",
       "      <td>0.182613</td>\n",
       "      <td>-0.055544</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.090407</td>\n",
       "      <td>0.021198</td>\n",
       "      <td>0.208712</td>\n",
       "      <td>0.102752</td>\n",
       "      <td>0.119315</td>\n",
       "      <td>0.041729</td>\n",
       "      <td>-0.021728</td>\n",
       "      <td>0.019603</td>\n",
       "      <td>-0.063853</td>\n",
       "      <td>-0.053756</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.085235</td>\n",
       "      <td>0.009540</td>\n",
       "      <td>-0.013228</td>\n",
       "      <td>0.094063</td>\n",
       "      <td>-0.013592</td>\n",
       "      <td>-0.030719</td>\n",
       "      <td>-0.116062</td>\n",
       "      <td>-0.131674</td>\n",
       "      <td>-0.165448</td>\n",
       "      <td>-0.123053</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.088765</td>\n",
       "      <td>-0.026743</td>\n",
       "      <td>0.002075</td>\n",
       "      <td>-0.043819</td>\n",
       "      <td>-0.005465</td>\n",
       "      <td>0.004306</td>\n",
       "      <td>-0.055865</td>\n",
       "      <td>-0.071484</td>\n",
       "      <td>-0.159025</td>\n",
       "      <td>-0.111348</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 117 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Att1      Att2      Att3      Att4      Att5      Att6      Att7  \\\n",
       "0  0.093700  0.139771  0.062774  0.007698  0.083873 -0.119156  0.073305   \n",
       "1 -0.022711 -0.050504 -0.035691 -0.065434 -0.084316 -0.378560  0.038212   \n",
       "2 -0.090407  0.021198  0.208712  0.102752  0.119315  0.041729 -0.021728   \n",
       "3 -0.085235  0.009540 -0.013228  0.094063 -0.013592 -0.030719 -0.116062   \n",
       "4 -0.088765 -0.026743  0.002075 -0.043819 -0.005465  0.004306 -0.055865   \n",
       "\n",
       "       Att8      Att9     Att10   ...     Class5  Class6  Class7  Class8  \\\n",
       "0  0.005510  0.027523  0.043477   ...          0       0       0       0   \n",
       "1  0.085770  0.182613 -0.055544   ...          0       0       1       1   \n",
       "2  0.019603 -0.063853 -0.053756   ...          0       0       0       0   \n",
       "3 -0.131674 -0.165448 -0.123053   ...          0       0       0       0   \n",
       "4 -0.071484 -0.159025 -0.111348   ...          0       0       0       0   \n",
       "\n",
       "   Class9  Class10  Class11  Class12  Class13  Class14  \n",
       "0       0        0        0        0        0        0  \n",
       "1       0        0        0        1        1        0  \n",
       "2       0        0        0        1        1        0  \n",
       "3       0        0        0        1        1        1  \n",
       "4       0        0        0        0        0        0  \n",
       "\n",
       "[5 rows x 117 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Att1       float64\n",
       "Att2       float64\n",
       "Att3       float64\n",
       "Att4       float64\n",
       "Att5       float64\n",
       "Att6       float64\n",
       "Att7       float64\n",
       "Att8       float64\n",
       "Att9       float64\n",
       "Att10      float64\n",
       "Att11      float64\n",
       "Att12      float64\n",
       "Att13      float64\n",
       "Att14      float64\n",
       "Att15      float64\n",
       "Att16      float64\n",
       "Att17      float64\n",
       "Att18      float64\n",
       "Att19      float64\n",
       "Att20      float64\n",
       "Att21      float64\n",
       "Att22      float64\n",
       "Att23      float64\n",
       "Att24      float64\n",
       "Att25      float64\n",
       "Att26      float64\n",
       "Att27      float64\n",
       "Att28      float64\n",
       "Att29      float64\n",
       "Att30      float64\n",
       "            ...   \n",
       "Att88      float64\n",
       "Att89      float64\n",
       "Att90      float64\n",
       "Att91      float64\n",
       "Att92      float64\n",
       "Att93      float64\n",
       "Att94      float64\n",
       "Att95      float64\n",
       "Att96      float64\n",
       "Att97      float64\n",
       "Att98      float64\n",
       "Att99      float64\n",
       "Att100     float64\n",
       "Att101     float64\n",
       "Att102     float64\n",
       "Att103     float64\n",
       "Class1      object\n",
       "Class2      object\n",
       "Class3      object\n",
       "Class4      object\n",
       "Class5      object\n",
       "Class6      object\n",
       "Class7      object\n",
       "Class8      object\n",
       "Class9      object\n",
       "Class10     object\n",
       "Class11     object\n",
       "Class12     object\n",
       "Class13     object\n",
       "Class14     object\n",
       "Length: 117, dtype: object"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "117"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class1</th>\n",
       "      <th>Class2</th>\n",
       "      <th>Class3</th>\n",
       "      <th>Class4</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Class1 Class2 Class3 Class4 Class5 Class6 Class7 Class8 Class9 Class10  \\\n",
       "0      0      0      1      1      0      0      0      0      0       0   \n",
       "1      0      0      0      0      0      0      1      1      0       0   \n",
       "2      0      1      1      0      0      0      0      0      0       0   \n",
       "3      0      0      1      1      0      0      0      0      0       0   \n",
       "4      1      1      0      0      0      0      0      0      0       0   \n",
       "\n",
       "  Class11 Class12 Class13 Class14  \n",
       "0       0       0       0       0  \n",
       "1       0       1       1       0  \n",
       "2       0       1       1       0  \n",
       "3       0       1       1       1  \n",
       "4       0       0       0       0  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_df = df.ix[:,103:117]\n",
    "labels_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Class1     float64\n",
       "Class2     float64\n",
       "Class3     float64\n",
       "Class4     float64\n",
       "Class5     float64\n",
       "Class6     float64\n",
       "Class7     float64\n",
       "Class8     float64\n",
       "Class9     float64\n",
       "Class10    float64\n",
       "Class11    float64\n",
       "Class12    float64\n",
       "Class13    float64\n",
       "Class14    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(1,15):\n",
    "    class_name = 'Class'+str(i)\n",
    "    labels_df[class_name] = labels_df[class_name].astype('float64')\n",
    "    \n",
    "labels_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class1</th>\n",
       "      <th>Class2</th>\n",
       "      <th>Class3</th>\n",
       "      <th>Class4</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.507232</td>\n",
       "      <td>-0.309405</td>\n",
       "      <td>-0.313830</td>\n",
       "      <td>-0.257625</td>\n",
       "      <td>-0.016337</td>\n",
       "      <td>-0.040895</td>\n",
       "      <td>-0.038894</td>\n",
       "      <td>0.005224</td>\n",
       "      <td>-0.116104</td>\n",
       "      <td>-0.113541</td>\n",
       "      <td>-0.142461</td>\n",
       "      <td>-0.131753</td>\n",
       "      <td>-0.081242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class2</th>\n",
       "      <td>0.507232</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.216886</td>\n",
       "      <td>-0.334276</td>\n",
       "      <td>-0.285610</td>\n",
       "      <td>-0.100926</td>\n",
       "      <td>-0.121585</td>\n",
       "      <td>-0.076037</td>\n",
       "      <td>0.053730</td>\n",
       "      <td>0.042503</td>\n",
       "      <td>-0.020448</td>\n",
       "      <td>-0.112380</td>\n",
       "      <td>-0.101452</td>\n",
       "      <td>-0.103496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class3</th>\n",
       "      <td>-0.309405</td>\n",
       "      <td>0.216886</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.435189</td>\n",
       "      <td>-0.310254</td>\n",
       "      <td>-0.284435</td>\n",
       "      <td>-0.240820</td>\n",
       "      <td>-0.180264</td>\n",
       "      <td>-0.005892</td>\n",
       "      <td>0.043173</td>\n",
       "      <td>-0.039968</td>\n",
       "      <td>0.048252</td>\n",
       "      <td>0.056433</td>\n",
       "      <td>0.146345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class4</th>\n",
       "      <td>-0.313830</td>\n",
       "      <td>-0.334276</td>\n",
       "      <td>0.435189</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.151685</td>\n",
       "      <td>-0.330798</td>\n",
       "      <td>-0.292469</td>\n",
       "      <td>-0.280890</td>\n",
       "      <td>-0.122692</td>\n",
       "      <td>-0.149018</td>\n",
       "      <td>-0.182061</td>\n",
       "      <td>0.199855</td>\n",
       "      <td>0.208159</td>\n",
       "      <td>0.160734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class5</th>\n",
       "      <td>-0.257625</td>\n",
       "      <td>-0.285610</td>\n",
       "      <td>-0.310254</td>\n",
       "      <td>0.151685</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.521967</td>\n",
       "      <td>-0.068450</td>\n",
       "      <td>-0.055528</td>\n",
       "      <td>-0.028495</td>\n",
       "      <td>-0.105581</td>\n",
       "      <td>-0.139253</td>\n",
       "      <td>0.119842</td>\n",
       "      <td>0.130416</td>\n",
       "      <td>-0.039531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class6</th>\n",
       "      <td>-0.016337</td>\n",
       "      <td>-0.100926</td>\n",
       "      <td>-0.284435</td>\n",
       "      <td>-0.330798</td>\n",
       "      <td>0.521967</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.426211</td>\n",
       "      <td>0.284887</td>\n",
       "      <td>0.008103</td>\n",
       "      <td>-0.022680</td>\n",
       "      <td>0.104809</td>\n",
       "      <td>0.045314</td>\n",
       "      <td>0.027331</td>\n",
       "      <td>-0.056094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class7</th>\n",
       "      <td>-0.040895</td>\n",
       "      <td>-0.121585</td>\n",
       "      <td>-0.240820</td>\n",
       "      <td>-0.292469</td>\n",
       "      <td>-0.068450</td>\n",
       "      <td>0.426211</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.770091</td>\n",
       "      <td>0.006746</td>\n",
       "      <td>-0.062572</td>\n",
       "      <td>0.096359</td>\n",
       "      <td>0.096621</td>\n",
       "      <td>0.071787</td>\n",
       "      <td>-0.054690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class8</th>\n",
       "      <td>-0.038894</td>\n",
       "      <td>-0.076037</td>\n",
       "      <td>-0.180264</td>\n",
       "      <td>-0.280890</td>\n",
       "      <td>-0.055528</td>\n",
       "      <td>0.284887</td>\n",
       "      <td>0.770091</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.445439</td>\n",
       "      <td>-0.060176</td>\n",
       "      <td>0.019236</td>\n",
       "      <td>0.030030</td>\n",
       "      <td>0.034797</td>\n",
       "      <td>-0.043823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class9</th>\n",
       "      <td>0.005224</td>\n",
       "      <td>0.053730</td>\n",
       "      <td>-0.005892</td>\n",
       "      <td>-0.122692</td>\n",
       "      <td>-0.028495</td>\n",
       "      <td>0.008103</td>\n",
       "      <td>0.006746</td>\n",
       "      <td>0.445439</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.143646</td>\n",
       "      <td>-0.023400</td>\n",
       "      <td>-0.123043</td>\n",
       "      <td>-0.116908</td>\n",
       "      <td>-0.008542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class10</th>\n",
       "      <td>-0.116104</td>\n",
       "      <td>0.042503</td>\n",
       "      <td>0.043173</td>\n",
       "      <td>-0.149018</td>\n",
       "      <td>-0.105581</td>\n",
       "      <td>-0.022680</td>\n",
       "      <td>-0.062572</td>\n",
       "      <td>-0.060176</td>\n",
       "      <td>0.143646</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.781024</td>\n",
       "      <td>-0.025295</td>\n",
       "      <td>-0.028544</td>\n",
       "      <td>-0.041319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class11</th>\n",
       "      <td>-0.113541</td>\n",
       "      <td>-0.020448</td>\n",
       "      <td>-0.039968</td>\n",
       "      <td>-0.182061</td>\n",
       "      <td>-0.139253</td>\n",
       "      <td>0.104809</td>\n",
       "      <td>0.096359</td>\n",
       "      <td>0.019236</td>\n",
       "      <td>-0.023400</td>\n",
       "      <td>0.781024</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.046081</td>\n",
       "      <td>-0.001408</td>\n",
       "      <td>-0.046468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class12</th>\n",
       "      <td>-0.142461</td>\n",
       "      <td>-0.112380</td>\n",
       "      <td>0.048252</td>\n",
       "      <td>0.199855</td>\n",
       "      <td>0.119842</td>\n",
       "      <td>0.045314</td>\n",
       "      <td>0.096621</td>\n",
       "      <td>0.030030</td>\n",
       "      <td>-0.123043</td>\n",
       "      <td>-0.025295</td>\n",
       "      <td>0.046081</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979002</td>\n",
       "      <td>0.002733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class13</th>\n",
       "      <td>-0.131753</td>\n",
       "      <td>-0.101452</td>\n",
       "      <td>0.056433</td>\n",
       "      <td>0.208159</td>\n",
       "      <td>0.130416</td>\n",
       "      <td>0.027331</td>\n",
       "      <td>0.071787</td>\n",
       "      <td>0.034797</td>\n",
       "      <td>-0.116908</td>\n",
       "      <td>-0.028544</td>\n",
       "      <td>-0.001408</td>\n",
       "      <td>0.979002</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.004889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class14</th>\n",
       "      <td>-0.081242</td>\n",
       "      <td>-0.103496</td>\n",
       "      <td>0.146345</td>\n",
       "      <td>0.160734</td>\n",
       "      <td>-0.039531</td>\n",
       "      <td>-0.056094</td>\n",
       "      <td>-0.054690</td>\n",
       "      <td>-0.043823</td>\n",
       "      <td>-0.008542</td>\n",
       "      <td>-0.041319</td>\n",
       "      <td>-0.046468</td>\n",
       "      <td>0.002733</td>\n",
       "      <td>0.004889</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Class1    Class2    Class3    Class4    Class5    Class6    Class7  \\\n",
       "Class1   1.000000  0.507232 -0.309405 -0.313830 -0.257625 -0.016337 -0.040895   \n",
       "Class2   0.507232  1.000000  0.216886 -0.334276 -0.285610 -0.100926 -0.121585   \n",
       "Class3  -0.309405  0.216886  1.000000  0.435189 -0.310254 -0.284435 -0.240820   \n",
       "Class4  -0.313830 -0.334276  0.435189  1.000000  0.151685 -0.330798 -0.292469   \n",
       "Class5  -0.257625 -0.285610 -0.310254  0.151685  1.000000  0.521967 -0.068450   \n",
       "Class6  -0.016337 -0.100926 -0.284435 -0.330798  0.521967  1.000000  0.426211   \n",
       "Class7  -0.040895 -0.121585 -0.240820 -0.292469 -0.068450  0.426211  1.000000   \n",
       "Class8  -0.038894 -0.076037 -0.180264 -0.280890 -0.055528  0.284887  0.770091   \n",
       "Class9   0.005224  0.053730 -0.005892 -0.122692 -0.028495  0.008103  0.006746   \n",
       "Class10 -0.116104  0.042503  0.043173 -0.149018 -0.105581 -0.022680 -0.062572   \n",
       "Class11 -0.113541 -0.020448 -0.039968 -0.182061 -0.139253  0.104809  0.096359   \n",
       "Class12 -0.142461 -0.112380  0.048252  0.199855  0.119842  0.045314  0.096621   \n",
       "Class13 -0.131753 -0.101452  0.056433  0.208159  0.130416  0.027331  0.071787   \n",
       "Class14 -0.081242 -0.103496  0.146345  0.160734 -0.039531 -0.056094 -0.054690   \n",
       "\n",
       "           Class8    Class9   Class10   Class11   Class12   Class13   Class14  \n",
       "Class1  -0.038894  0.005224 -0.116104 -0.113541 -0.142461 -0.131753 -0.081242  \n",
       "Class2  -0.076037  0.053730  0.042503 -0.020448 -0.112380 -0.101452 -0.103496  \n",
       "Class3  -0.180264 -0.005892  0.043173 -0.039968  0.048252  0.056433  0.146345  \n",
       "Class4  -0.280890 -0.122692 -0.149018 -0.182061  0.199855  0.208159  0.160734  \n",
       "Class5  -0.055528 -0.028495 -0.105581 -0.139253  0.119842  0.130416 -0.039531  \n",
       "Class6   0.284887  0.008103 -0.022680  0.104809  0.045314  0.027331 -0.056094  \n",
       "Class7   0.770091  0.006746 -0.062572  0.096359  0.096621  0.071787 -0.054690  \n",
       "Class8   1.000000  0.445439 -0.060176  0.019236  0.030030  0.034797 -0.043823  \n",
       "Class9   0.445439  1.000000  0.143646 -0.023400 -0.123043 -0.116908 -0.008542  \n",
       "Class10 -0.060176  0.143646  1.000000  0.781024 -0.025295 -0.028544 -0.041319  \n",
       "Class11  0.019236 -0.023400  0.781024  1.000000  0.046081 -0.001408 -0.046468  \n",
       "Class12  0.030030 -0.123043 -0.025295  0.046081  1.000000  0.979002  0.002733  \n",
       "Class13  0.034797 -0.116908 -0.028544 -0.001408  0.979002  1.000000  0.004889  \n",
       "Class14 -0.043823 -0.008542 -0.041319 -0.046468  0.002733  0.004889  1.000000  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Here, I'm trying to check the correlations between labels\n",
    "\n",
    "labels_df.apply(lambda s: labels_df.corrwith(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# you can also generate a random multi-label dataset on your own\n",
    "\n",
    "from sklearn.datasets import make_multilabel_classification\n",
    "\n",
    "X, y = make_multilabel_classification(sparse = True, n_labels = 7,\n",
    "return_indicator = 'sparse', allow_unlabeled = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2    3    4    5    6    7    8    9    10   11   12   13   14  \\\n",
       "0  2.0  1.0  1.0  6.0  0.0  4.0  2.0  4.0  3.0  1.0  1.0  5.0  3.0  4.0  4.0   \n",
       "1  1.0  3.0  2.0  2.0  3.0  3.0  3.0  6.0  1.0  2.0  2.0  0.0  1.0  5.0  3.0   \n",
       "2  0.0  3.0  3.0  2.0  2.0  6.0  1.0  1.0  4.0  1.0  2.0  0.0  1.0  3.0  5.0   \n",
       "3  5.0  3.0  1.0  3.0  3.0  2.0  2.0  4.0  2.0  1.0  1.0  4.0  0.0  0.0  2.0   \n",
       "4  1.0  6.0  4.0  2.0  7.0  5.0  1.0  7.0  3.0  1.0  0.0  0.0  4.0  1.0  2.0   \n",
       "\n",
       "    15   16   17   18   19  \n",
       "0  3.0  3.0  4.0  2.0  3.0  \n",
       "1  1.0  3.0  1.0  2.0  4.0  \n",
       "2  2.0  2.0  5.0  3.0  1.0  \n",
       "3  5.0  3.0  4.0  2.0  1.0  \n",
       "4  0.0  3.0  2.0  5.0  1.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix_df = pd.SparseDataFrame([ pd.SparseSeries(X[i].toarray().ravel()) \n",
    "                                   for i in np.arange(X.shape[0])])\n",
    "matrix_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Start Multi-Label Experiments From HERE\n",
    "\n",
    "training_data, train_meta = scipy.io.arff.loadarff('yeast/yeast-train.arff')\n",
    "train_df = pd.DataFrame(training_data)\n",
    "\n",
    "testing_data, test_meta = scipy.io.arff.loadarff('yeast/yeast-test.arff')\n",
    "test_df = pd.DataFrame(testing_data)\n",
    "\n",
    "# sparse: If True, returns a sparse matrix, where sparse matrix means a matrix having a large number of zero elements.\n",
    "# n_labels:  The average number of labels for each instance.\n",
    "# return_indicator: If ‘sparse’ return Y in the sparse binary indicator format.\n",
    "# allow_unlabeled: If True, some instances might not belong to any class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Att1</th>\n",
       "      <th>Att2</th>\n",
       "      <th>Att3</th>\n",
       "      <th>Att4</th>\n",
       "      <th>Att5</th>\n",
       "      <th>Att6</th>\n",
       "      <th>Att7</th>\n",
       "      <th>Att8</th>\n",
       "      <th>Att9</th>\n",
       "      <th>Att10</th>\n",
       "      <th>...</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.093700</td>\n",
       "      <td>0.139771</td>\n",
       "      <td>0.062774</td>\n",
       "      <td>0.007698</td>\n",
       "      <td>0.083873</td>\n",
       "      <td>-0.119156</td>\n",
       "      <td>0.073305</td>\n",
       "      <td>0.005510</td>\n",
       "      <td>0.027523</td>\n",
       "      <td>0.043477</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.022711</td>\n",
       "      <td>-0.050504</td>\n",
       "      <td>-0.035691</td>\n",
       "      <td>-0.065434</td>\n",
       "      <td>-0.084316</td>\n",
       "      <td>-0.378560</td>\n",
       "      <td>0.038212</td>\n",
       "      <td>0.085770</td>\n",
       "      <td>0.182613</td>\n",
       "      <td>-0.055544</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.090407</td>\n",
       "      <td>0.021198</td>\n",
       "      <td>0.208712</td>\n",
       "      <td>0.102752</td>\n",
       "      <td>0.119315</td>\n",
       "      <td>0.041729</td>\n",
       "      <td>-0.021728</td>\n",
       "      <td>0.019603</td>\n",
       "      <td>-0.063853</td>\n",
       "      <td>-0.053756</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.085235</td>\n",
       "      <td>0.009540</td>\n",
       "      <td>-0.013228</td>\n",
       "      <td>0.094063</td>\n",
       "      <td>-0.013592</td>\n",
       "      <td>-0.030719</td>\n",
       "      <td>-0.116062</td>\n",
       "      <td>-0.131674</td>\n",
       "      <td>-0.165448</td>\n",
       "      <td>-0.123053</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.088765</td>\n",
       "      <td>-0.026743</td>\n",
       "      <td>0.002075</td>\n",
       "      <td>-0.043819</td>\n",
       "      <td>-0.005465</td>\n",
       "      <td>0.004306</td>\n",
       "      <td>-0.055865</td>\n",
       "      <td>-0.071484</td>\n",
       "      <td>-0.159025</td>\n",
       "      <td>-0.111348</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 117 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Att1      Att2      Att3      Att4      Att5      Att6      Att7  \\\n",
       "0  0.093700  0.139771  0.062774  0.007698  0.083873 -0.119156  0.073305   \n",
       "1 -0.022711 -0.050504 -0.035691 -0.065434 -0.084316 -0.378560  0.038212   \n",
       "2 -0.090407  0.021198  0.208712  0.102752  0.119315  0.041729 -0.021728   \n",
       "3 -0.085235  0.009540 -0.013228  0.094063 -0.013592 -0.030719 -0.116062   \n",
       "4 -0.088765 -0.026743  0.002075 -0.043819 -0.005465  0.004306 -0.055865   \n",
       "\n",
       "       Att8      Att9     Att10   ...     Class5  Class6  Class7  Class8  \\\n",
       "0  0.005510  0.027523  0.043477   ...          0       0       0       0   \n",
       "1  0.085770  0.182613 -0.055544   ...          0       0       1       1   \n",
       "2  0.019603 -0.063853 -0.053756   ...          0       0       0       0   \n",
       "3 -0.131674 -0.165448 -0.123053   ...          0       0       0       0   \n",
       "4 -0.071484 -0.159025 -0.111348   ...          0       0       0       0   \n",
       "\n",
       "   Class9  Class10  Class11  Class12  Class13  Class14  \n",
       "0       0        0        0        0        0        0  \n",
       "1       0        0        0        1        1        0  \n",
       "2       0        0        0        1        1        0  \n",
       "3       0        0        0        1        1        1  \n",
       "4       0        0        0        0        0        0  \n",
       "\n",
       "[5 rows x 117 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Att1</th>\n",
       "      <th>Att2</th>\n",
       "      <th>Att3</th>\n",
       "      <th>Att4</th>\n",
       "      <th>Att5</th>\n",
       "      <th>Att6</th>\n",
       "      <th>Att7</th>\n",
       "      <th>Att8</th>\n",
       "      <th>Att9</th>\n",
       "      <th>Att10</th>\n",
       "      <th>...</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.004168</td>\n",
       "      <td>-0.170975</td>\n",
       "      <td>-0.156748</td>\n",
       "      <td>-0.142151</td>\n",
       "      <td>0.058781</td>\n",
       "      <td>0.026851</td>\n",
       "      <td>0.197719</td>\n",
       "      <td>0.041850</td>\n",
       "      <td>0.066938</td>\n",
       "      <td>-0.056617</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.103956</td>\n",
       "      <td>0.011879</td>\n",
       "      <td>-0.098986</td>\n",
       "      <td>-0.054501</td>\n",
       "      <td>-0.007970</td>\n",
       "      <td>0.049113</td>\n",
       "      <td>-0.030580</td>\n",
       "      <td>-0.077933</td>\n",
       "      <td>-0.080529</td>\n",
       "      <td>-0.016267</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.509949</td>\n",
       "      <td>0.401709</td>\n",
       "      <td>0.293799</td>\n",
       "      <td>0.087714</td>\n",
       "      <td>0.011686</td>\n",
       "      <td>-0.006411</td>\n",
       "      <td>-0.006255</td>\n",
       "      <td>0.013646</td>\n",
       "      <td>-0.040666</td>\n",
       "      <td>-0.024447</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.119092</td>\n",
       "      <td>0.004412</td>\n",
       "      <td>-0.002262</td>\n",
       "      <td>0.072254</td>\n",
       "      <td>0.044512</td>\n",
       "      <td>-0.051467</td>\n",
       "      <td>0.074686</td>\n",
       "      <td>-0.007670</td>\n",
       "      <td>0.079438</td>\n",
       "      <td>0.062184</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.042037</td>\n",
       "      <td>0.007054</td>\n",
       "      <td>-0.069483</td>\n",
       "      <td>0.081015</td>\n",
       "      <td>-0.048207</td>\n",
       "      <td>0.089446</td>\n",
       "      <td>-0.004947</td>\n",
       "      <td>0.064456</td>\n",
       "      <td>-0.133387</td>\n",
       "      <td>0.068878</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 117 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Att1      Att2      Att3      Att4      Att5      Att6      Att7  \\\n",
       "0  0.004168 -0.170975 -0.156748 -0.142151  0.058781  0.026851  0.197719   \n",
       "1 -0.103956  0.011879 -0.098986 -0.054501 -0.007970  0.049113 -0.030580   \n",
       "2  0.509949  0.401709  0.293799  0.087714  0.011686 -0.006411 -0.006255   \n",
       "3  0.119092  0.004412 -0.002262  0.072254  0.044512 -0.051467  0.074686   \n",
       "4  0.042037  0.007054 -0.069483  0.081015 -0.048207  0.089446 -0.004947   \n",
       "\n",
       "       Att8      Att9     Att10   ...     Class5  Class6  Class7  Class8  \\\n",
       "0  0.041850  0.066938 -0.056617   ...          0       0       1       1   \n",
       "1 -0.077933 -0.080529 -0.016267   ...          0       0       0       0   \n",
       "2  0.013646 -0.040666 -0.024447   ...          0       0       0       0   \n",
       "3 -0.007670  0.079438  0.062184   ...          0       0       0       0   \n",
       "4  0.064456 -0.133387  0.068878   ...          1       1       0       0   \n",
       "\n",
       "   Class9  Class10  Class11  Class12  Class13  Class14  \n",
       "0       0        0        0        1        1        0  \n",
       "1       0        0        0        0        0        0  \n",
       "2       0        0        0        1        1        0  \n",
       "3       0        0        0        0        0        0  \n",
       "4       0        0        0        0        0        0  \n",
       "\n",
       "[5 rows x 117 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = train_df.iloc[:,:103]\n",
    "Y_train = train_df.iloc[:,103:]\n",
    "\n",
    "X_test = test_df.iloc[:,:103]\n",
    "Y_test = test_df.iloc[:,103:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Class1     float64\n",
       "Class2     float64\n",
       "Class3     float64\n",
       "Class4     float64\n",
       "Class5     float64\n",
       "Class6     float64\n",
       "Class7     float64\n",
       "Class8     float64\n",
       "Class9     float64\n",
       "Class10    float64\n",
       "Class11    float64\n",
       "Class12    float64\n",
       "Class13    float64\n",
       "Class14    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(1,15):\n",
    "    class_name = 'Class'+str(i)\n",
    "    Y_train[class_name] = Y_train[class_name].astype('float64')\n",
    "    \n",
    "Y_train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Class1     float64\n",
       "Class2     float64\n",
       "Class3     float64\n",
       "Class4     float64\n",
       "Class5     float64\n",
       "Class6     float64\n",
       "Class7     float64\n",
       "Class8     float64\n",
       "Class9     float64\n",
       "Class10    float64\n",
       "Class11    float64\n",
       "Class12    float64\n",
       "Class13    float64\n",
       "Class14    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(1,15):\n",
    "    class_name = 'Class'+str(i)\n",
    "    Y_test[class_name] = Y_test[class_name].astype('float64')\n",
    "    \n",
    "Y_test.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Att1</th>\n",
       "      <th>Att2</th>\n",
       "      <th>Att3</th>\n",
       "      <th>Att4</th>\n",
       "      <th>Att5</th>\n",
       "      <th>Att6</th>\n",
       "      <th>Att7</th>\n",
       "      <th>Att8</th>\n",
       "      <th>Att9</th>\n",
       "      <th>Att10</th>\n",
       "      <th>...</th>\n",
       "      <th>Att94</th>\n",
       "      <th>Att95</th>\n",
       "      <th>Att96</th>\n",
       "      <th>Att97</th>\n",
       "      <th>Att98</th>\n",
       "      <th>Att99</th>\n",
       "      <th>Att100</th>\n",
       "      <th>Att101</th>\n",
       "      <th>Att102</th>\n",
       "      <th>Att103</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.004168</td>\n",
       "      <td>-0.170975</td>\n",
       "      <td>-0.156748</td>\n",
       "      <td>-0.142151</td>\n",
       "      <td>0.058781</td>\n",
       "      <td>0.026851</td>\n",
       "      <td>0.197719</td>\n",
       "      <td>0.041850</td>\n",
       "      <td>0.066938</td>\n",
       "      <td>-0.056617</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006166</td>\n",
       "      <td>-0.012976</td>\n",
       "      <td>-0.014259</td>\n",
       "      <td>-0.015024</td>\n",
       "      <td>-0.010747</td>\n",
       "      <td>0.000411</td>\n",
       "      <td>-0.032056</td>\n",
       "      <td>-0.018312</td>\n",
       "      <td>0.030126</td>\n",
       "      <td>0.124722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.103956</td>\n",
       "      <td>0.011879</td>\n",
       "      <td>-0.098986</td>\n",
       "      <td>-0.054501</td>\n",
       "      <td>-0.007970</td>\n",
       "      <td>0.049113</td>\n",
       "      <td>-0.030580</td>\n",
       "      <td>-0.077933</td>\n",
       "      <td>-0.080529</td>\n",
       "      <td>-0.016267</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007680</td>\n",
       "      <td>0.027719</td>\n",
       "      <td>-0.085811</td>\n",
       "      <td>0.111123</td>\n",
       "      <td>0.050541</td>\n",
       "      <td>0.027565</td>\n",
       "      <td>-0.063569</td>\n",
       "      <td>-0.041471</td>\n",
       "      <td>-0.079758</td>\n",
       "      <td>0.017161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.509949</td>\n",
       "      <td>0.401709</td>\n",
       "      <td>0.293799</td>\n",
       "      <td>0.087714</td>\n",
       "      <td>0.011686</td>\n",
       "      <td>-0.006411</td>\n",
       "      <td>-0.006255</td>\n",
       "      <td>0.013646</td>\n",
       "      <td>-0.040666</td>\n",
       "      <td>-0.024447</td>\n",
       "      <td>...</td>\n",
       "      <td>0.096277</td>\n",
       "      <td>-0.044932</td>\n",
       "      <td>-0.089470</td>\n",
       "      <td>-0.009162</td>\n",
       "      <td>-0.012010</td>\n",
       "      <td>0.308378</td>\n",
       "      <td>-0.028053</td>\n",
       "      <td>0.026710</td>\n",
       "      <td>-0.066565</td>\n",
       "      <td>-0.122352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.119092</td>\n",
       "      <td>0.004412</td>\n",
       "      <td>-0.002262</td>\n",
       "      <td>0.072254</td>\n",
       "      <td>0.044512</td>\n",
       "      <td>-0.051467</td>\n",
       "      <td>0.074686</td>\n",
       "      <td>-0.007670</td>\n",
       "      <td>0.079438</td>\n",
       "      <td>0.062184</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.083809</td>\n",
       "      <td>0.200354</td>\n",
       "      <td>-0.075716</td>\n",
       "      <td>0.196605</td>\n",
       "      <td>0.152758</td>\n",
       "      <td>-0.028484</td>\n",
       "      <td>-0.074207</td>\n",
       "      <td>-0.089227</td>\n",
       "      <td>-0.049913</td>\n",
       "      <td>-0.043893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.042037</td>\n",
       "      <td>0.007054</td>\n",
       "      <td>-0.069483</td>\n",
       "      <td>0.081015</td>\n",
       "      <td>-0.048207</td>\n",
       "      <td>0.089446</td>\n",
       "      <td>-0.004947</td>\n",
       "      <td>0.064456</td>\n",
       "      <td>-0.133387</td>\n",
       "      <td>0.068878</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.060467</td>\n",
       "      <td>0.044351</td>\n",
       "      <td>-0.057209</td>\n",
       "      <td>0.028047</td>\n",
       "      <td>0.029661</td>\n",
       "      <td>-0.050026</td>\n",
       "      <td>0.023248</td>\n",
       "      <td>-0.061539</td>\n",
       "      <td>-0.035160</td>\n",
       "      <td>0.067834</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Att1      Att2      Att3      Att4      Att5      Att6      Att7  \\\n",
       "0  0.004168 -0.170975 -0.156748 -0.142151  0.058781  0.026851  0.197719   \n",
       "1 -0.103956  0.011879 -0.098986 -0.054501 -0.007970  0.049113 -0.030580   \n",
       "2  0.509949  0.401709  0.293799  0.087714  0.011686 -0.006411 -0.006255   \n",
       "3  0.119092  0.004412 -0.002262  0.072254  0.044512 -0.051467  0.074686   \n",
       "4  0.042037  0.007054 -0.069483  0.081015 -0.048207  0.089446 -0.004947   \n",
       "\n",
       "       Att8      Att9     Att10    ...        Att94     Att95     Att96  \\\n",
       "0  0.041850  0.066938 -0.056617    ...     0.006166 -0.012976 -0.014259   \n",
       "1 -0.077933 -0.080529 -0.016267    ...     0.007680  0.027719 -0.085811   \n",
       "2  0.013646 -0.040666 -0.024447    ...     0.096277 -0.044932 -0.089470   \n",
       "3 -0.007670  0.079438  0.062184    ...    -0.083809  0.200354 -0.075716   \n",
       "4  0.064456 -0.133387  0.068878    ...    -0.060467  0.044351 -0.057209   \n",
       "\n",
       "      Att97     Att98     Att99    Att100    Att101    Att102    Att103  \n",
       "0 -0.015024 -0.010747  0.000411 -0.032056 -0.018312  0.030126  0.124722  \n",
       "1  0.111123  0.050541  0.027565 -0.063569 -0.041471 -0.079758  0.017161  \n",
       "2 -0.009162 -0.012010  0.308378 -0.028053  0.026710 -0.066565 -0.122352  \n",
       "3  0.196605  0.152758 -0.028484 -0.074207 -0.089227 -0.049913 -0.043893  \n",
       "4  0.028047  0.029661 -0.050026  0.023248 -0.061539 -0.035160  0.067834  \n",
       "\n",
       "[5 rows x 103 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class1</th>\n",
       "      <th>Class2</th>\n",
       "      <th>Class3</th>\n",
       "      <th>Class4</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class1  Class2  Class3  Class4  Class5  Class6  Class7  Class8  Class9  \\\n",
       "0     0.0     0.0     0.0     0.0     0.0     0.0     1.0     1.0     0.0   \n",
       "1     0.0     0.0     1.0     1.0     0.0     0.0     0.0     0.0     0.0   \n",
       "2     0.0     1.0     1.0     0.0     0.0     0.0     0.0     0.0     0.0   \n",
       "3     0.0     0.0     1.0     1.0     0.0     0.0     0.0     0.0     0.0   \n",
       "4     0.0     0.0     1.0     1.0     1.0     1.0     0.0     0.0     0.0   \n",
       "\n",
       "   Class10  Class11  Class12  Class13  Class14  \n",
       "0      0.0      0.0      1.0      1.0      0.0  \n",
       "1      0.0      0.0      0.0      0.0      0.0  \n",
       "2      0.0      0.0      1.0      1.0      0.0  \n",
       "3      0.0      0.0      0.0      0.0      0.0  \n",
       "4      0.0      0.0      0.0      0.0      0.0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10359869138495092"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Method 1 - Problem Transformation - Binary Relavance\n",
    "## Simply treat each label independently\n",
    "\n",
    "from skmultilearn.problem_transform import BinaryRelevance\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "classifier = BinaryRelevance(GaussianNB())\n",
    "\n",
    "# train\n",
    "classifier.fit(X_train, Y_train)\n",
    "\n",
    "# predict\n",
    "predictions = classifier.predict(X_test)\n",
    "accuracy_score(Y_test,predictions)   # 0.10359869138495092, very low accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class1</th>\n",
       "      <th>Class2</th>\n",
       "      <th>Class3</th>\n",
       "      <th>Class4</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.507232</td>\n",
       "      <td>-0.309405</td>\n",
       "      <td>-0.313830</td>\n",
       "      <td>-0.257625</td>\n",
       "      <td>-0.016337</td>\n",
       "      <td>-0.040895</td>\n",
       "      <td>-0.038894</td>\n",
       "      <td>0.005224</td>\n",
       "      <td>-0.116104</td>\n",
       "      <td>-0.113541</td>\n",
       "      <td>-0.142461</td>\n",
       "      <td>-0.131753</td>\n",
       "      <td>-0.081242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class2</th>\n",
       "      <td>0.507232</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.216886</td>\n",
       "      <td>-0.334276</td>\n",
       "      <td>-0.285610</td>\n",
       "      <td>-0.100926</td>\n",
       "      <td>-0.121585</td>\n",
       "      <td>-0.076037</td>\n",
       "      <td>0.053730</td>\n",
       "      <td>0.042503</td>\n",
       "      <td>-0.020448</td>\n",
       "      <td>-0.112380</td>\n",
       "      <td>-0.101452</td>\n",
       "      <td>-0.103496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class3</th>\n",
       "      <td>-0.309405</td>\n",
       "      <td>0.216886</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.435189</td>\n",
       "      <td>-0.310254</td>\n",
       "      <td>-0.284435</td>\n",
       "      <td>-0.240820</td>\n",
       "      <td>-0.180264</td>\n",
       "      <td>-0.005892</td>\n",
       "      <td>0.043173</td>\n",
       "      <td>-0.039968</td>\n",
       "      <td>0.048252</td>\n",
       "      <td>0.056433</td>\n",
       "      <td>0.146345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class4</th>\n",
       "      <td>-0.313830</td>\n",
       "      <td>-0.334276</td>\n",
       "      <td>0.435189</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.151685</td>\n",
       "      <td>-0.330798</td>\n",
       "      <td>-0.292469</td>\n",
       "      <td>-0.280890</td>\n",
       "      <td>-0.122692</td>\n",
       "      <td>-0.149018</td>\n",
       "      <td>-0.182061</td>\n",
       "      <td>0.199855</td>\n",
       "      <td>0.208159</td>\n",
       "      <td>0.160734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class5</th>\n",
       "      <td>-0.257625</td>\n",
       "      <td>-0.285610</td>\n",
       "      <td>-0.310254</td>\n",
       "      <td>0.151685</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.521967</td>\n",
       "      <td>-0.068450</td>\n",
       "      <td>-0.055528</td>\n",
       "      <td>-0.028495</td>\n",
       "      <td>-0.105581</td>\n",
       "      <td>-0.139253</td>\n",
       "      <td>0.119842</td>\n",
       "      <td>0.130416</td>\n",
       "      <td>-0.039531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class6</th>\n",
       "      <td>-0.016337</td>\n",
       "      <td>-0.100926</td>\n",
       "      <td>-0.284435</td>\n",
       "      <td>-0.330798</td>\n",
       "      <td>0.521967</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.426211</td>\n",
       "      <td>0.284887</td>\n",
       "      <td>0.008103</td>\n",
       "      <td>-0.022680</td>\n",
       "      <td>0.104809</td>\n",
       "      <td>0.045314</td>\n",
       "      <td>0.027331</td>\n",
       "      <td>-0.056094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class7</th>\n",
       "      <td>-0.040895</td>\n",
       "      <td>-0.121585</td>\n",
       "      <td>-0.240820</td>\n",
       "      <td>-0.292469</td>\n",
       "      <td>-0.068450</td>\n",
       "      <td>0.426211</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.770091</td>\n",
       "      <td>0.006746</td>\n",
       "      <td>-0.062572</td>\n",
       "      <td>0.096359</td>\n",
       "      <td>0.096621</td>\n",
       "      <td>0.071787</td>\n",
       "      <td>-0.054690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class8</th>\n",
       "      <td>-0.038894</td>\n",
       "      <td>-0.076037</td>\n",
       "      <td>-0.180264</td>\n",
       "      <td>-0.280890</td>\n",
       "      <td>-0.055528</td>\n",
       "      <td>0.284887</td>\n",
       "      <td>0.770091</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.445439</td>\n",
       "      <td>-0.060176</td>\n",
       "      <td>0.019236</td>\n",
       "      <td>0.030030</td>\n",
       "      <td>0.034797</td>\n",
       "      <td>-0.043823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class9</th>\n",
       "      <td>0.005224</td>\n",
       "      <td>0.053730</td>\n",
       "      <td>-0.005892</td>\n",
       "      <td>-0.122692</td>\n",
       "      <td>-0.028495</td>\n",
       "      <td>0.008103</td>\n",
       "      <td>0.006746</td>\n",
       "      <td>0.445439</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.143646</td>\n",
       "      <td>-0.023400</td>\n",
       "      <td>-0.123043</td>\n",
       "      <td>-0.116908</td>\n",
       "      <td>-0.008542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class10</th>\n",
       "      <td>-0.116104</td>\n",
       "      <td>0.042503</td>\n",
       "      <td>0.043173</td>\n",
       "      <td>-0.149018</td>\n",
       "      <td>-0.105581</td>\n",
       "      <td>-0.022680</td>\n",
       "      <td>-0.062572</td>\n",
       "      <td>-0.060176</td>\n",
       "      <td>0.143646</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.781024</td>\n",
       "      <td>-0.025295</td>\n",
       "      <td>-0.028544</td>\n",
       "      <td>-0.041319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class11</th>\n",
       "      <td>-0.113541</td>\n",
       "      <td>-0.020448</td>\n",
       "      <td>-0.039968</td>\n",
       "      <td>-0.182061</td>\n",
       "      <td>-0.139253</td>\n",
       "      <td>0.104809</td>\n",
       "      <td>0.096359</td>\n",
       "      <td>0.019236</td>\n",
       "      <td>-0.023400</td>\n",
       "      <td>0.781024</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.046081</td>\n",
       "      <td>-0.001408</td>\n",
       "      <td>-0.046468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class12</th>\n",
       "      <td>-0.142461</td>\n",
       "      <td>-0.112380</td>\n",
       "      <td>0.048252</td>\n",
       "      <td>0.199855</td>\n",
       "      <td>0.119842</td>\n",
       "      <td>0.045314</td>\n",
       "      <td>0.096621</td>\n",
       "      <td>0.030030</td>\n",
       "      <td>-0.123043</td>\n",
       "      <td>-0.025295</td>\n",
       "      <td>0.046081</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979002</td>\n",
       "      <td>0.002733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class13</th>\n",
       "      <td>-0.131753</td>\n",
       "      <td>-0.101452</td>\n",
       "      <td>0.056433</td>\n",
       "      <td>0.208159</td>\n",
       "      <td>0.130416</td>\n",
       "      <td>0.027331</td>\n",
       "      <td>0.071787</td>\n",
       "      <td>0.034797</td>\n",
       "      <td>-0.116908</td>\n",
       "      <td>-0.028544</td>\n",
       "      <td>-0.001408</td>\n",
       "      <td>0.979002</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.004889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class14</th>\n",
       "      <td>-0.081242</td>\n",
       "      <td>-0.103496</td>\n",
       "      <td>0.146345</td>\n",
       "      <td>0.160734</td>\n",
       "      <td>-0.039531</td>\n",
       "      <td>-0.056094</td>\n",
       "      <td>-0.054690</td>\n",
       "      <td>-0.043823</td>\n",
       "      <td>-0.008542</td>\n",
       "      <td>-0.041319</td>\n",
       "      <td>-0.046468</td>\n",
       "      <td>0.002733</td>\n",
       "      <td>0.004889</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Class1    Class2    Class3    Class4    Class5    Class6    Class7  \\\n",
       "Class1   1.000000  0.507232 -0.309405 -0.313830 -0.257625 -0.016337 -0.040895   \n",
       "Class2   0.507232  1.000000  0.216886 -0.334276 -0.285610 -0.100926 -0.121585   \n",
       "Class3  -0.309405  0.216886  1.000000  0.435189 -0.310254 -0.284435 -0.240820   \n",
       "Class4  -0.313830 -0.334276  0.435189  1.000000  0.151685 -0.330798 -0.292469   \n",
       "Class5  -0.257625 -0.285610 -0.310254  0.151685  1.000000  0.521967 -0.068450   \n",
       "Class6  -0.016337 -0.100926 -0.284435 -0.330798  0.521967  1.000000  0.426211   \n",
       "Class7  -0.040895 -0.121585 -0.240820 -0.292469 -0.068450  0.426211  1.000000   \n",
       "Class8  -0.038894 -0.076037 -0.180264 -0.280890 -0.055528  0.284887  0.770091   \n",
       "Class9   0.005224  0.053730 -0.005892 -0.122692 -0.028495  0.008103  0.006746   \n",
       "Class10 -0.116104  0.042503  0.043173 -0.149018 -0.105581 -0.022680 -0.062572   \n",
       "Class11 -0.113541 -0.020448 -0.039968 -0.182061 -0.139253  0.104809  0.096359   \n",
       "Class12 -0.142461 -0.112380  0.048252  0.199855  0.119842  0.045314  0.096621   \n",
       "Class13 -0.131753 -0.101452  0.056433  0.208159  0.130416  0.027331  0.071787   \n",
       "Class14 -0.081242 -0.103496  0.146345  0.160734 -0.039531 -0.056094 -0.054690   \n",
       "\n",
       "           Class8    Class9   Class10   Class11   Class12   Class13   Class14  \n",
       "Class1  -0.038894  0.005224 -0.116104 -0.113541 -0.142461 -0.131753 -0.081242  \n",
       "Class2  -0.076037  0.053730  0.042503 -0.020448 -0.112380 -0.101452 -0.103496  \n",
       "Class3  -0.180264 -0.005892  0.043173 -0.039968  0.048252  0.056433  0.146345  \n",
       "Class4  -0.280890 -0.122692 -0.149018 -0.182061  0.199855  0.208159  0.160734  \n",
       "Class5  -0.055528 -0.028495 -0.105581 -0.139253  0.119842  0.130416 -0.039531  \n",
       "Class6   0.284887  0.008103 -0.022680  0.104809  0.045314  0.027331 -0.056094  \n",
       "Class7   0.770091  0.006746 -0.062572  0.096359  0.096621  0.071787 -0.054690  \n",
       "Class8   1.000000  0.445439 -0.060176  0.019236  0.030030  0.034797 -0.043823  \n",
       "Class9   0.445439  1.000000  0.143646 -0.023400 -0.123043 -0.116908 -0.008542  \n",
       "Class10 -0.060176  0.143646  1.000000  0.781024 -0.025295 -0.028544 -0.041319  \n",
       "Class11  0.019236 -0.023400  0.781024  1.000000  0.046081 -0.001408 -0.046468  \n",
       "Class12  0.030030 -0.123043 -0.025295  0.046081  1.000000  0.979002  0.002733  \n",
       "Class13  0.034797 -0.116908 -0.028544 -0.001408  0.979002  1.000000  0.004889  \n",
       "Class14 -0.043823 -0.008542 -0.041319 -0.046468  0.002733  0.004889  1.000000  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Method 1 - Problem Transformation - Classifier Chains\n",
    "## Each round predict 1 label column, and put it in feature columns for the next round of label prediction\n",
    "\n",
    "## For this method, in order to achive higher accuracy, better to have higher corrlation between labels\n",
    "## So try the labels correlation check as I did here\n",
    "\n",
    "Y_train.apply(lambda s: Y_train.corrwith(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class1</th>\n",
       "      <th>Class2</th>\n",
       "      <th>Class3</th>\n",
       "      <th>Class4</th>\n",
       "      <th>Class5</th>\n",
       "      <th>Class6</th>\n",
       "      <th>Class7</th>\n",
       "      <th>Class8</th>\n",
       "      <th>Class9</th>\n",
       "      <th>Class10</th>\n",
       "      <th>Class11</th>\n",
       "      <th>Class12</th>\n",
       "      <th>Class13</th>\n",
       "      <th>Class14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.530037</td>\n",
       "      <td>-0.253155</td>\n",
       "      <td>-0.293891</td>\n",
       "      <td>-0.273892</td>\n",
       "      <td>-0.090011</td>\n",
       "      <td>-0.153002</td>\n",
       "      <td>-0.148226</td>\n",
       "      <td>-0.057984</td>\n",
       "      <td>-0.091622</td>\n",
       "      <td>-0.073865</td>\n",
       "      <td>-0.133661</td>\n",
       "      <td>-0.135088</td>\n",
       "      <td>-0.040910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class2</th>\n",
       "      <td>0.530037</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.236645</td>\n",
       "      <td>-0.355494</td>\n",
       "      <td>-0.284067</td>\n",
       "      <td>-0.128472</td>\n",
       "      <td>-0.197395</td>\n",
       "      <td>-0.129462</td>\n",
       "      <td>0.091478</td>\n",
       "      <td>0.084874</td>\n",
       "      <td>-0.007370</td>\n",
       "      <td>-0.131636</td>\n",
       "      <td>-0.124922</td>\n",
       "      <td>-0.047932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class3</th>\n",
       "      <td>-0.253155</td>\n",
       "      <td>0.236645</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.416407</td>\n",
       "      <td>-0.340189</td>\n",
       "      <td>-0.284757</td>\n",
       "      <td>-0.212490</td>\n",
       "      <td>-0.153393</td>\n",
       "      <td>0.058039</td>\n",
       "      <td>0.113071</td>\n",
       "      <td>-0.016304</td>\n",
       "      <td>0.005845</td>\n",
       "      <td>0.016443</td>\n",
       "      <td>0.140966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class4</th>\n",
       "      <td>-0.293891</td>\n",
       "      <td>-0.355494</td>\n",
       "      <td>0.416407</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.107819</td>\n",
       "      <td>-0.329391</td>\n",
       "      <td>-0.259612</td>\n",
       "      <td>-0.261487</td>\n",
       "      <td>-0.151287</td>\n",
       "      <td>-0.129381</td>\n",
       "      <td>-0.142475</td>\n",
       "      <td>0.159624</td>\n",
       "      <td>0.167864</td>\n",
       "      <td>0.159937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class5</th>\n",
       "      <td>-0.273892</td>\n",
       "      <td>-0.284067</td>\n",
       "      <td>-0.340189</td>\n",
       "      <td>0.107819</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.598581</td>\n",
       "      <td>0.029576</td>\n",
       "      <td>0.008569</td>\n",
       "      <td>-0.046228</td>\n",
       "      <td>-0.095993</td>\n",
       "      <td>-0.070301</td>\n",
       "      <td>0.164892</td>\n",
       "      <td>0.172030</td>\n",
       "      <td>-0.019682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class6</th>\n",
       "      <td>-0.090011</td>\n",
       "      <td>-0.128472</td>\n",
       "      <td>-0.284757</td>\n",
       "      <td>-0.329391</td>\n",
       "      <td>0.598581</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.398406</td>\n",
       "      <td>0.222871</td>\n",
       "      <td>-0.028150</td>\n",
       "      <td>-0.016785</td>\n",
       "      <td>0.122053</td>\n",
       "      <td>0.086804</td>\n",
       "      <td>0.063857</td>\n",
       "      <td>-0.067171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class7</th>\n",
       "      <td>-0.153002</td>\n",
       "      <td>-0.197395</td>\n",
       "      <td>-0.212490</td>\n",
       "      <td>-0.259612</td>\n",
       "      <td>0.029576</td>\n",
       "      <td>0.398406</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.732111</td>\n",
       "      <td>-0.035740</td>\n",
       "      <td>-0.054122</td>\n",
       "      <td>0.089097</td>\n",
       "      <td>0.128627</td>\n",
       "      <td>0.101206</td>\n",
       "      <td>-0.032685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class8</th>\n",
       "      <td>-0.148226</td>\n",
       "      <td>-0.129462</td>\n",
       "      <td>-0.153393</td>\n",
       "      <td>-0.261487</td>\n",
       "      <td>0.008569</td>\n",
       "      <td>0.222871</td>\n",
       "      <td>0.732111</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.431449</td>\n",
       "      <td>-0.046145</td>\n",
       "      <td>0.009393</td>\n",
       "      <td>0.022942</td>\n",
       "      <td>0.029194</td>\n",
       "      <td>-0.038793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class9</th>\n",
       "      <td>-0.057984</td>\n",
       "      <td>0.091478</td>\n",
       "      <td>0.058039</td>\n",
       "      <td>-0.151287</td>\n",
       "      <td>-0.046228</td>\n",
       "      <td>-0.028150</td>\n",
       "      <td>-0.035740</td>\n",
       "      <td>0.431449</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.244059</td>\n",
       "      <td>-0.012136</td>\n",
       "      <td>-0.160897</td>\n",
       "      <td>-0.155884</td>\n",
       "      <td>-0.037074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class10</th>\n",
       "      <td>-0.091622</td>\n",
       "      <td>0.084874</td>\n",
       "      <td>0.113071</td>\n",
       "      <td>-0.129381</td>\n",
       "      <td>-0.095993</td>\n",
       "      <td>-0.016785</td>\n",
       "      <td>-0.054122</td>\n",
       "      <td>-0.046145</td>\n",
       "      <td>0.244059</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.763278</td>\n",
       "      <td>-0.067295</td>\n",
       "      <td>-0.062639</td>\n",
       "      <td>-0.040046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class11</th>\n",
       "      <td>-0.073865</td>\n",
       "      <td>-0.007370</td>\n",
       "      <td>-0.016304</td>\n",
       "      <td>-0.142475</td>\n",
       "      <td>-0.070301</td>\n",
       "      <td>0.122053</td>\n",
       "      <td>0.089097</td>\n",
       "      <td>0.009393</td>\n",
       "      <td>-0.012136</td>\n",
       "      <td>0.763278</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.010742</td>\n",
       "      <td>-0.048346</td>\n",
       "      <td>-0.039803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class12</th>\n",
       "      <td>-0.133661</td>\n",
       "      <td>-0.131636</td>\n",
       "      <td>0.005845</td>\n",
       "      <td>0.159624</td>\n",
       "      <td>0.164892</td>\n",
       "      <td>0.086804</td>\n",
       "      <td>0.128627</td>\n",
       "      <td>0.022942</td>\n",
       "      <td>-0.160897</td>\n",
       "      <td>-0.067295</td>\n",
       "      <td>-0.010742</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.985657</td>\n",
       "      <td>-0.037370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class13</th>\n",
       "      <td>-0.135088</td>\n",
       "      <td>-0.124922</td>\n",
       "      <td>0.016443</td>\n",
       "      <td>0.167864</td>\n",
       "      <td>0.172030</td>\n",
       "      <td>0.063857</td>\n",
       "      <td>0.101206</td>\n",
       "      <td>0.029194</td>\n",
       "      <td>-0.155884</td>\n",
       "      <td>-0.062639</td>\n",
       "      <td>-0.048346</td>\n",
       "      <td>0.985657</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.035603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class14</th>\n",
       "      <td>-0.040910</td>\n",
       "      <td>-0.047932</td>\n",
       "      <td>0.140966</td>\n",
       "      <td>0.159937</td>\n",
       "      <td>-0.019682</td>\n",
       "      <td>-0.067171</td>\n",
       "      <td>-0.032685</td>\n",
       "      <td>-0.038793</td>\n",
       "      <td>-0.037074</td>\n",
       "      <td>-0.040046</td>\n",
       "      <td>-0.039803</td>\n",
       "      <td>-0.037370</td>\n",
       "      <td>-0.035603</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Class1    Class2    Class3    Class4    Class5    Class6    Class7  \\\n",
       "Class1   1.000000  0.530037 -0.253155 -0.293891 -0.273892 -0.090011 -0.153002   \n",
       "Class2   0.530037  1.000000  0.236645 -0.355494 -0.284067 -0.128472 -0.197395   \n",
       "Class3  -0.253155  0.236645  1.000000  0.416407 -0.340189 -0.284757 -0.212490   \n",
       "Class4  -0.293891 -0.355494  0.416407  1.000000  0.107819 -0.329391 -0.259612   \n",
       "Class5  -0.273892 -0.284067 -0.340189  0.107819  1.000000  0.598581  0.029576   \n",
       "Class6  -0.090011 -0.128472 -0.284757 -0.329391  0.598581  1.000000  0.398406   \n",
       "Class7  -0.153002 -0.197395 -0.212490 -0.259612  0.029576  0.398406  1.000000   \n",
       "Class8  -0.148226 -0.129462 -0.153393 -0.261487  0.008569  0.222871  0.732111   \n",
       "Class9  -0.057984  0.091478  0.058039 -0.151287 -0.046228 -0.028150 -0.035740   \n",
       "Class10 -0.091622  0.084874  0.113071 -0.129381 -0.095993 -0.016785 -0.054122   \n",
       "Class11 -0.073865 -0.007370 -0.016304 -0.142475 -0.070301  0.122053  0.089097   \n",
       "Class12 -0.133661 -0.131636  0.005845  0.159624  0.164892  0.086804  0.128627   \n",
       "Class13 -0.135088 -0.124922  0.016443  0.167864  0.172030  0.063857  0.101206   \n",
       "Class14 -0.040910 -0.047932  0.140966  0.159937 -0.019682 -0.067171 -0.032685   \n",
       "\n",
       "           Class8    Class9   Class10   Class11   Class12   Class13   Class14  \n",
       "Class1  -0.148226 -0.057984 -0.091622 -0.073865 -0.133661 -0.135088 -0.040910  \n",
       "Class2  -0.129462  0.091478  0.084874 -0.007370 -0.131636 -0.124922 -0.047932  \n",
       "Class3  -0.153393  0.058039  0.113071 -0.016304  0.005845  0.016443  0.140966  \n",
       "Class4  -0.261487 -0.151287 -0.129381 -0.142475  0.159624  0.167864  0.159937  \n",
       "Class5   0.008569 -0.046228 -0.095993 -0.070301  0.164892  0.172030 -0.019682  \n",
       "Class6   0.222871 -0.028150 -0.016785  0.122053  0.086804  0.063857 -0.067171  \n",
       "Class7   0.732111 -0.035740 -0.054122  0.089097  0.128627  0.101206 -0.032685  \n",
       "Class8   1.000000  0.431449 -0.046145  0.009393  0.022942  0.029194 -0.038793  \n",
       "Class9   0.431449  1.000000  0.244059 -0.012136 -0.160897 -0.155884 -0.037074  \n",
       "Class10 -0.046145  0.244059  1.000000  0.763278 -0.067295 -0.062639 -0.040046  \n",
       "Class11  0.009393 -0.012136  0.763278  1.000000 -0.010742 -0.048346 -0.039803  \n",
       "Class12  0.022942 -0.160897 -0.067295 -0.010742  1.000000  0.985657 -0.037370  \n",
       "Class13  0.029194 -0.155884 -0.062639 -0.048346  0.985657  1.000000 -0.035603  \n",
       "Class14 -0.038793 -0.037074 -0.040046 -0.039803 -0.037370 -0.035603  1.000000  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_test.apply(lambda s: Y_test.corrwith(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.092693565976008724"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Not very high correlation, let's check the accuracy from Classifier Chains\n",
    "\n",
    "from skmultilearn.problem_transform import ClassifierChain\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "\n",
    "classifier = ClassifierChain(GaussianNB())\n",
    "\n",
    "# train\n",
    "classifier.fit(X_train, Y_train)\n",
    "\n",
    "# predict\n",
    "predictions = classifier.predict(X_test)\n",
    "accuracy_score(Y_test,predictions)   # 0.092693565976008724, even lower..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18647764449291168"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Method 1 - Problem Transformation - Label Powerset\n",
    "## Group rows with exactly same label value set (such as 2 rows all have labels (0,1,0,1)), together\n",
    "## Then give each label set a value, so that it will become 1-label prediction problem\n",
    "## But, this method may suffer from data imbalance when it comes to real world dataset\n",
    "\n",
    "from skmultilearn.problem_transform import LabelPowerset\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "\n",
    "classifier = LabelPowerset(GaussianNB())\n",
    "\n",
    "# train\n",
    "classifier.fit(X_train, Y_train)\n",
    "\n",
    "# predict\n",
    "predictions = classifier.predict(X_test)\n",
    "accuracy_score(Y_test,predictions)   # 0.18647764449291168, a little higher..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16684841875681569"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Method 2 - Adapt Algorithms\n",
    "## Scikit-Learn package: http://scikit.ml/api/api/skmultilearn.adapt.html#module-skmultilearn.adapt\n",
    "\n",
    "from skmultilearn.adapt import MLkNN\n",
    "\n",
    "# Scikit-learn Adapt Algorithms requires Dense/Sparse matrix as data input\n",
    "X_train_matrix = scipy.sparse.csr_matrix(X_train.values)\n",
    "Y_train_matrix = scipy.sparse.csr_matrix(Y_train.values)\n",
    "\n",
    "X_test_matrix = scipy.sparse.csr_matrix(X_test.values)\n",
    "Y_test_matrix = scipy.sparse.csr_matrix(Y_test.values)\n",
    "\n",
    "classifier = MLkNN(k=10)\n",
    "\n",
    "# train\n",
    "classifier.fit(X_train_matrix, Y_train_matrix)\n",
    "\n",
    "# predict\n",
    "predictions = classifier.predict(X_test_matrix)\n",
    "accuracy_score(Y_test_matrix,predictions)   # 0.16684841875681569"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18102508178844057"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# increase k\n",
    "classifier = MLkNN(k=20)\n",
    "\n",
    "# train\n",
    "classifier.fit(X_train_matrix, Y_train_matrix)\n",
    "\n",
    "# predict\n",
    "predictions = classifier.predict(X_test_matrix)\n",
    "accuracy_score(Y_test_matrix,predictions)   # 0.18102508178844057"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10032715376226826"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from skmultilearn.adapt import BRkNNaClassifier\n",
    "\n",
    "classifier = BRkNNaClassifier(k=10)\n",
    "\n",
    "# train\n",
    "classifier.fit(X_train_matrix, Y_train_matrix)\n",
    "\n",
    "# predict\n",
    "predictions = classifier.predict(X_test_matrix)\n",
    "accuracy_score(Y_test_matrix,predictions)   # 0.10032715376226826"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named graph_tool.all",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-5a37a197d9a0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensemble\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomForestClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mskmultilearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproblem_transform\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLabelPowerset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mskmultilearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcluster\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mIGraphLabelCooccurenceClusterer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mskmultilearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensemble\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLabelSpacePartitioningClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mbuild/bdist.macosx-10.7-x86_64/egg/skmultilearn/cluster/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mbuild/bdist.macosx-10.7-x86_64/egg/skmultilearn/cluster/graphtool.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named graph_tool.all"
     ]
    }
   ],
   "source": [
    "# Method 3 - Ensembling Multi-Label\n",
    "## Scikit-Learn emsembling: http://scikit.ml/api/classify.html#ensemble-approaches\n",
    "## tools to install in order to get \"graph_tool.all\": https://gist.github.com/v-pravin/949fc18d58a560cf85d2\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from skmultilearn.problem_transform import LabelPowerset\n",
    "from skmultilearn.cluster import IGraphLabelCooccurenceClusterer\n",
    "from skmultilearn.ensemble import LabelSpacePartitioningClassifier\n",
    "\n",
    "# construct base forest classifier\n",
    "base_classifier = RandomForestClassifier()\n",
    "\n",
    "# setup problem transformation approach with sparse matrices for random forest\n",
    "problem_transform_classifier = LabelPowerset(classifier=base_classifier,\n",
    "    require_dense=[False, False])\n",
    "\n",
    "# partition the label space using fastgreedy community detection\n",
    "# on a weighted label co-occurrence graph with self-loops allowed\n",
    "clusterer = IGraphLabelCooccurenceClusterer('fastgreedy', weighted=True,\n",
    "    include_self_edges=True)\n",
    "\n",
    "# setup the ensemble metaclassifier\n",
    "classifier = LabelSpacePartitioningClassifier(problem_transform_classifier, clusterer)\n",
    "\n",
    "# train\n",
    "classifier.fit(X_train_matrix, Y_train_matrix)\n",
    "\n",
    "# predict\n",
    "predictions = classifier.predict(X_test_matrix)\n",
    "accuracy_score(Y_test_matrix,predictions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}