features_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# BIO-SELECT - Marigliano\n",
    "## Features selection using several algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"./img/project_pipeline.png\",width=700px>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn import neighbors, datasets\n",
    "import pandas as pd\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_classif\n",
    "from sklearn.multiclass import OneVsOneClassifier, OneVsRestClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "from utils.ConfusionMatrix import ConfusionMatrix\n",
    "\n",
    "import itertools\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "import math\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# set float precision at 2 digits\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "# set the random seed for reproducibility\n",
    "#np.random.seed(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N_FEATURES_ALGORITHM = 1000\n",
    "GROUP_NAME = \"golub_11052017\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from datasets.EGEOD22619.EGEOD22619Dataset import EGEOD22619Dataset\n",
    "from datasets.MILE.MileDataset import MileDataset\n",
    "from datasets.Golub99.GolubDataset import GolubDataset\n",
    "from datasets.GSE13425.GSE13425Dataset import GSE13425Dataset\n",
    "from datasets.nci60.NCI60Dataset import NCI60Dataset\n",
    "\n",
    "from datasets.DatasetEncoder import DatasetEncoder\n",
    "from datasets.DatasetSplitter import DatasetSplitter\n",
    "from datasets.DatasetLoader import DatasetLoader\n",
    "from datasets.DatasetBalancer import DatasetBalancer\n",
    "\n",
    "# Choose a dataset\n",
    "#ds = MileDataset()\n",
    "ds = GolubDataset()\n",
    "#ds = GSE13425Dataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset transformation\n",
    "The dataset needs some transformations such as encoding the outputs as float (necessary for scikit learn), normalization, ...\n",
    "\n",
    "Used transformation:\n",
    "* dataset splitting (train, test[, validation])\n",
    "* encode outputs\n",
    "* normalization\n",
    "* classes merging (MILE contains 18 original classes that have been combined in 6 to have more balanced classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# encode Dataset string classes into numbers\n",
    "ds_encoder = DatasetEncoder(ds)\n",
    "ds = ds_encoder.encode()\n",
    "\n",
    "ds = DatasetSplitter(ds, test_size=0.4)\n",
    "\n",
    "ds_balancer = DatasetBalancer(ds)\n",
    "ds = ds_balancer.balance()\n",
    "\n",
    "X = ds.get_X()\n",
    "y = ds.get_y()\n",
    "\n",
    "X_train = ds.get_X_train()\n",
    "y_train = ds.get_y_train()\n",
    "X_test = ds.get_X_test()\n",
    "y_test = ds.get_y_test()\n",
    "\n",
    "class_names = range(len(set(ds.get_y())))\n",
    "\n",
    "print(\"Number of genes: %d\" % len(X_train[0]))\n",
    "print(\"Dataset samples: %d\" % len(y))\n",
    "print(\"Train set size %d\" % len(X_train))\n",
    "print(\"Test set size %d\" % len(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the dataset split using Pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(ds, open(\"%s.pkl\" % GROUP_NAME, \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Algorithms\n",
    "Run the chosen algorithms and save them and their output subset of features using cPickle into files. They can be used later to display some graphs and to be analyzed\n",
    "\n",
    "Used algorithms:\n",
    "* ExtraTrees\n",
    "* Random Forest\n",
    "* SVM\n",
    "* SVM RFE\n",
    "* GAANN\n",
    "* ReliefF\n",
    "* Fisher Score / F Value\n",
    "* CFS - Correlation-based Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from algorithms.Algorithm import NotSupportedException\n",
    "from algorithms.ExtraTreesAlgorithm import ExtraTreesAlgorithm\n",
    "from algorithms.ReliefFAlgorithm import ReliefFAlgorithm\n",
    "from algorithms.FisherScoreAlgorithm import FisherScoreAlgorithm\n",
    "from algorithms.FValueAlgorithm import FValueAlgorithm\n",
    "from algorithms.SVMAlgorithm import SVMAlgorithm\n",
    "from algorithms.GAANNAlgorithm import GAANNAlgorithm\n",
    "from algorithms.GridSearchableAlgorithm import GridSearchableAlgorithm\n",
    "from algorithms.SVMRFEAlgorithm import SVMRFEAlgorithm\n",
    "from algorithms.CFSAlgorithm import CFSAlgorithm\n",
    "from algorithms.MRMRAlgorithm import MRMRAlgorithm\n",
    "from algorithms.DeterministAlgorithm import DeterministAlgorithm\n",
    "\n",
    "from utils.AlgorithmListsUtils import *\n",
    "\n",
    "# the main idea here is to prepare all the algorithms in a list of tuple.\n",
    "# Then in a loop each algorithm will be runned and directly freed from memory\n",
    "# The goal is to keep the algorithm as less time as possible in memory\n",
    "algorithms = []\n",
    "\n",
    "\n",
    "# ExtraTrees\n",
    "eta_grid = [{\n",
    "        'n_estimators': np.arange(10, 1000, 300), \n",
    "        'criterion': [\"gini\", \"entropy\"], \n",
    "        'max_features': [\"auto\"],\n",
    "        'n_jobs': [2]\n",
    "    }]\n",
    "\n",
    "eta = (ExtraTreesAlgorithm, {\n",
    "        \"dataset\": ds,\n",
    "        \"n\": N_FEATURES_ALGORITHM,\n",
    "        \"gridsearch_params\": eta_grid\n",
    "    })\n",
    "algorithms.append(eta)\n",
    "\n",
    "\n",
    "# ReliefF\n",
    "rff = (ReliefFAlgorithm, {\n",
    "        \"dataset\": ds,\n",
    "        \"n\": N_FEATURES_ALGORITHM\n",
    "    })\n",
    "algorithms.append(rff)\n",
    "\n",
    "\n",
    "# Fisher score\n",
    "fsa = (FisherScoreAlgorithm, {\n",
    "        \"dataset\": ds,\n",
    "        \"n\": N_FEATURES_ALGORITHM\n",
    "    })\n",
    "algorithms.append(fsa)\n",
    "\n",
    "\n",
    "# F-Value\n",
    "fva = (FValueAlgorithm, {\n",
    "        \"dataset\": ds,\n",
    "        \"n\": N_FEATURES_ALGORITHM\n",
    "    })\n",
    "algorithms.append(fva)\n",
    "\n",
    "\n",
    "# SVM\n",
    "#FIXME: grid search for SVM always returns the first set of parameters, like all params give the same performance\n",
    "svm_grid_params = [{\n",
    "        'kernel':['linear'],\n",
    "        'C':[200, 0.1, 1, 10, 100, 1000],\n",
    "        'gamma' : [1e-2, 1e-3, 1e-4, 1e-5],\n",
    "        'tol' : [1e-2, 1e-3, 1e-4, 1e-5],\n",
    "        'cache_size':[1024],\n",
    "        'n_jobs': [2]\n",
    "    }]\n",
    "#%time svm_gs = SVMAlgorithm(ds, N_FEATURES_ALGORITHM, svm_grid_params)\n",
    "#algorithms.append(svm_gs)\n",
    "#print(\"Best params \\n\\t%s\" % svm_gs.best_params)\n",
    "\n",
    "\n",
    "svm = (SVMAlgorithm, {\n",
    "        \"dataset\": ds,\n",
    "        \"n\": N_FEATURES_ALGORITHM\n",
    "    })\n",
    "algorithms.append(svm)\n",
    "\n",
    "\n",
    "# GA ANN, commented because it does not give meaningful features\n",
    "#%time gaanna = GAANNAlgorithm(ds, n=300)\n",
    "#algorithms.append(gaanna)\n",
    "\n",
    "\n",
    "## SVM Forward, takes too long, was replaced by SVM-RFE\n",
    "#from algorithms.SVMForwardAlgorithm import SVMForwardAlgorithm\n",
    "#svm_forward = (SVMForwardAlgorithm, {\n",
    "#        \"dataset\": ds,\n",
    "#        \"n\" : N_FEATURES_ALGORITHM\n",
    "#    })\n",
    "#algorithms.append(svm_forward)\n",
    "\n",
    "\n",
    "# SVM-RFE\n",
    "svm_rfe = (SVMRFEAlgorithm, {\n",
    "        \"dataset\": ds,\n",
    "        \"n\": N_FEATURES_ALGORITHM\n",
    "    })\n",
    "algorithms.append(svm_rfe)\n",
    "\n",
    "\n",
    "# CFS\n",
    "#cfs = (CFSAlgorithm, {\n",
    "#        \"dataset\": ds,\n",
    "#        \"n\": None # CFS gives its list\n",
    "#    })\n",
    "#algorithms.append(cfs)\n",
    "\n",
    "\n",
    "# MRMR\n",
    "#mrmr = (MRMRAlgorithm, {\n",
    "#        \"dataset\": ds,\n",
    "#        \"n\": N_FEATURES_ALGORITHM\n",
    "#    })\n",
    "#algorithms.append(mrmr)\n",
    "\n",
    "\n",
    "subsets = {}\n",
    "\n",
    "plt.figure(figsize=(12, 8))\n",
    "\n",
    "# the number of subplot is defined by the number of algorithm whose are able to provide a confusion matrix\n",
    "n_subplots = len([_ for a in algorithms if isinstance(a, GridSearchableAlgorithm)])\n",
    "cols = 3\n",
    "rows = max(1, int(math.ceil(n_subplots / cols)))\n",
    "i = 1\n",
    "\n",
    "for alg in algorithms:\n",
    "    alg_class = alg[0]\n",
    "    alg_kwargs = alg[1]\n",
    "    alg_name = None\n",
    "    \n",
    "    print(\"Running %s...\" % alg_class.__name__)\n",
    "    \n",
    "    if issubclass(alg_class, DeterministAlgorithm):\n",
    "        # This algorithm is determinist, so it will be run once\n",
    "        %time alg_instance = alg_class(**alg_kwargs)\n",
    "        alg_name = alg_instance.name\n",
    "        subsets[alg_name] = {\"features\": [], \"features_by_rank\": [], \"features_by_score\": []}\n",
    "        \n",
    "        feats = alg_instance.get_best_features()\n",
    "        feats = [(f, 1) for f in feats] # assign the same weight for all features\n",
    "        subsets[alg_name][\"features\"] = feats\n",
    "        \n",
    "        try:\n",
    "            r = alg_instance.get_best_features_by_rank()\n",
    "            # reverse the rank to have the best features with a higher score appear first\n",
    "            rank_tuples = [(v, 1.0/(1.0+k)) for k, v in enumerate(r)]\n",
    "            subsets[alg_name][\"features_by_rank\"] = rank_tuples\n",
    "\n",
    "            subsets[alg_name][\"features_by_score\"] = alg_instance.get_best_features_by_score()\n",
    "        except NotSupportedException:\n",
    "            pass\n",
    "\n",
    "    else:\n",
    "        # The algorithm is not determinist, so we run it multiple times and take a kind of \"average\" of the lists.\n",
    "        # This will increase the stability of the returned lists.\n",
    "        list_of_lists = []\n",
    "        list_of_ranks = []\n",
    "        list_of_scores = []\n",
    "\n",
    "        for _ in range(3):\n",
    "            # instanciate and run the algorithm\n",
    "            %time alg_instance = alg_class(**alg_kwargs)\n",
    "\n",
    "\n",
    "            # retrieve features for this run\n",
    "            feats = alg_instance.get_best_features()\n",
    "            list_of_lists.extend(feats)\n",
    "\n",
    "\n",
    "            try:\n",
    "                r = alg_instance.get_best_features_by_rank()\n",
    "                rank_tuples = [(v, k) for k, v in enumerate(r)]\n",
    "                list_of_ranks.extend(rank_tuples)\n",
    "                list_of_scores.extend(alg_instance.get_best_features_by_score())\n",
    "            except NotSupportedException:\n",
    "                pass\n",
    "\n",
    "\n",
    "        alg_name = alg_instance.name\n",
    "        subsets[alg_name] = {\"features\": [], \"features_by_rank\": [], \"features_by_score\": []}\n",
    "\n",
    "        # Compute the \"average\" of the returned lists\n",
    "        feats = compute_most_popular_features(list_of_lists)[:N_FEATURES_ALGORITHM]\n",
    "        subsets[alg_name][\"features\"].extend(feats)\n",
    "\n",
    "        if len(list_of_ranks) > 0:\n",
    "            feats_ranks = compute_score_of_lists(list_of_ranks, higher_is_better=False)[:N_FEATURES_ALGORITHM]\n",
    "            subsets[alg_name][\"features_by_rank\"].extend(feats_ranks)\n",
    "\n",
    "        if len(list_of_scores) > 0:\n",
    "            feats_scores = compute_score_of_lists(list_of_scores, higher_is_better=True)[:N_FEATURES_ALGORITHM]\n",
    "            subsets[alg_name][\"features_by_score\"].extend(feats_scores)\n",
    "    \n",
    "    \n",
    "    \n",
    "    # print the score of the algorithm, if provided by the latter\n",
    "    try:\n",
    "        print(\"[%s] score: %.3f\" % (alg_name, alg_instance.get_score()))\n",
    "    except NotSupportedException:\n",
    "        pass\n",
    "    \n",
    "    # show the confusion matrix, if supported by the latter\n",
    "    try:\n",
    "        cm = alg_instance.get_confusion_matrix()\n",
    "        plt.subplot(rows, cols, i)\n",
    "        ConfusionMatrix.plot(cm, class_names, title=\"Confusion matrix [%s]\" % alg_name)\n",
    "        i += 1\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    \n",
    "    print(\"\") # for readability\n",
    "    \n",
    "    # the algorithm is freed at the end of the loop, but the lists are kept\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The features have been stored into the subsets dictionnary.\n",
    "# subsets[alg_name][F] where alg_name is the name of the algorithm \n",
    "# and F can take the following values : \"features\", \"features_by_rank\", \"features_by_score\"\n",
    "subsets[\"ExtraTrees\"][\"features_by_score\"][:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To use Limma algorithm, please see the notebook called 'features_selection_limma.ipynb'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the features lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from utils.CSVFeaturesExporter import CSVFeaturesExporter\n",
    "\n",
    "features_exporter = CSVFeaturesExporter(subsets, group_name=GROUP_NAME)\n",
    "features_exporter.export()\n",
    "\n",
    "# CSV have been exported to \"./outputs\" folder"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}