deep_avg_net.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using Deep Averaging Networks for malware classification\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we will experiment with the concept of Deep Averaging Networks in our malware classification setting.\n",
    "\n",
    "Let's start by loading some packages necessary for the experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gio/.virtualenvs/malwords/lib/python3.5/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from classification import cla_action, cla_dan\n",
    "from utilities import constants, evaluation\n",
    "from preprocessing import pp_action\n",
    "import plotly.graph_objs as go\n",
    "import plotly.offline as ply\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "config = json.load(open('config.json', 'r'))\n",
    "ply.init_notebook_mode(connected=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data selection\n",
    "\n",
    "Select a subset of the original dataset. Then the selected subset will be split into a training, development and  test set.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please choose the subset of data to workon on:\n",
      "l for all labeled samples\n",
      "k for samples of families mydoom, gepys, lamer, neshta, bladabindi, flystudio, eorezo\n",
      "s for a small balanced subset\n",
      "f for a single family\n",
      "b for a balanced subset of samples\n",
      "q to quit\n",
      "b\n",
      "\n",
      "Would you like to compute the Jensen-Shannon distance matrix for the chosen data? [y/n]\n",
      "n\n",
      "\n",
      "20007 train samples belonging to 65 malware families\n",
      "Malware family:      multiplug       Number of samples:  748  \n",
      "Malware family:     installcore      Number of samples:  724  \n",
      "Malware family:       firseria       Number of samples:  720  \n",
      "Malware family:      outbrowse       Number of samples:  712  \n",
      "Malware family:       virlock        Number of samples:  707  \n",
      "Malware family:      loadmoney       Number of samples:  706  \n",
      "Malware family:        sality        Number of samples:  703  \n",
      "Malware family:      browsefox       Number of samples:  701  \n",
      "Malware family:       allaple        Number of samples:  698  \n",
      "Malware family:         mira         Number of samples:  695  \n",
      "Malware family:      softpulse       Number of samples:  694  \n",
      "Malware family:        vobfus        Number of samples:  692  \n",
      "Malware family:        sytro         Number of samples:  685  \n",
      "Malware family:        parite        Number of samples:  679  \n",
      "Malware family:        upatre        Number of samples:  678  \n",
      "Malware family:        virut         Number of samples:  677  \n",
      "Malware family:         klez         Number of samples:  541  \n",
      "Malware family:        dinwod        Number of samples:  517  \n",
      "Malware family:      vtflooder       Number of samples:  413  \n",
      "Malware family:        expiro        Number of samples:  375  \n",
      "Malware family:       icloader       Number of samples:  310  \n",
      "Malware family:        wapomi        Number of samples:  301  \n",
      "Malware family:        ramnit        Number of samples:  299  \n",
      "Malware family:        ircbot        Number of samples:  271  \n",
      "Malware family:    installmonster    Number of samples:  256  \n",
      "Malware family:         zbot         Number of samples:  254  \n",
      "Malware family:        autoit        Number of samples:  248  \n",
      "Malware family:        domaiq        Number of samples:  243  \n",
      "Malware family:        ipamor        Number of samples:  242  \n",
      "Malware family:    downloadguide     Number of samples:  236  \n",
      "Malware family:     installerex      Number of samples:  225  \n",
      "Malware family:      bladabindi      Number of samples:  187  \n",
      "Malware family:         delf         Number of samples:  184  \n",
      "Malware family:        ibryte        Number of samples:  182  \n",
      "Malware family:        eorezo        Number of samples:  174  \n",
      "Malware family:        neshta        Number of samples:  162  \n",
      "Malware family:        vilsel        Number of samples:  161  \n",
      "Malware family:      crossrider      Number of samples:  155  \n",
      "Malware family:         zusy         Number of samples:  148  \n",
      "Malware family:      amonetize       Number of samples:  147  \n",
      "Malware family:        cosmu         Number of samples:  138  \n",
      "Malware family:        loring        Number of samples:  133  \n",
      "Malware family:     airinstaller     Number of samples:  131  \n",
      "Malware family:    downloadadmin     Number of samples:  128  \n",
      "Malware family:        urelas        Number of samples:  123  \n",
      "Malware family:      convertad       Number of samples:  121  \n",
      "Malware family:        simbot        Number of samples:  115  \n",
      "Malware family:        yuner         Number of samples:  113  \n",
      "Malware family:  downloadassistant   Number of samples:  110  \n",
      "Malware family:        imali         Number of samples:  107  \n",
      "Malware family:       dealply        Number of samples:  106  \n",
      "Malware family:       gamarue        Number of samples:  105  \n",
      "Malware family:        lydra         Number of samples:  104  \n",
      "Malware family:        mydoom        Number of samples:  104  \n",
      "Malware family:        memery        Number of samples:   99  \n",
      "Malware family:      softcnapp       Number of samples:   97  \n",
      "Malware family:       dlhelper       Number of samples:   95  \n",
      "Malware family:        picsys        Number of samples:   93  \n",
      "Malware family:        lamer         Number of samples:   87  \n",
      "Malware family:        berbew        Number of samples:   84  \n",
      "Malware family:       bundlore       Number of samples:   77  \n",
      "Malware family:       filetour       Number of samples:   76  \n",
      "Malware family:        shipup        Number of samples:   75  \n",
      "Malware family:        hotbar        Number of samples:   72  \n",
      "Malware family:         chir         Number of samples:   64  \n",
      "\n",
      "4288 dev samples belonging to 65 malware families\n",
      "Malware family:        vobfus        Number of samples:  171  \n",
      "Malware family:        upatre        Number of samples:  169  \n",
      "Malware family:        virut         Number of samples:  166  \n",
      "Malware family:         mira         Number of samples:  163  \n",
      "Malware family:        sytro         Number of samples:  162  \n",
      "Malware family:        sality        Number of samples:  154  \n",
      "Malware family:       virlock        Number of samples:  152  \n",
      "Malware family:       allaple        Number of samples:  148  \n",
      "Malware family:        parite        Number of samples:  147  \n",
      "Malware family:      softpulse       Number of samples:  146  \n",
      "Malware family:       firseria       Number of samples:  143  \n",
      "Malware family:      browsefox       Number of samples:  141  \n",
      "Malware family:      outbrowse       Number of samples:  132  \n",
      "Malware family:      loadmoney       Number of samples:  129  \n",
      "Malware family:      multiplug       Number of samples:  128  \n",
      "Malware family:     installcore      Number of samples:  115  \n",
      "Malware family:         klez         Number of samples:  111  \n",
      "Malware family:        dinwod        Number of samples:  111  \n",
      "Malware family:        expiro        Number of samples:   90  \n",
      "Malware family:        ramnit        Number of samples:   78  \n",
      "Malware family:        wapomi        Number of samples:   77  \n",
      "Malware family:      vtflooder       Number of samples:   76  \n",
      "Malware family:        ircbot        Number of samples:   67  \n",
      "Malware family:       icloader       Number of samples:   63  \n",
      "Malware family:         zbot         Number of samples:   59  \n",
      "Malware family:        ipamor        Number of samples:   57  \n",
      "Malware family:        autoit        Number of samples:   55  \n",
      "Malware family:    installmonster    Number of samples:   55  \n",
      "Malware family:     installerex      Number of samples:   53  \n",
      "Malware family:        eorezo        Number of samples:   51  \n",
      "Malware family:    downloadguide     Number of samples:   48  \n",
      "Malware family:        ibryte        Number of samples:   45  \n",
      "Malware family:        domaiq        Number of samples:   44  \n",
      "Malware family:      bladabindi      Number of samples:   40  \n",
      "Malware family:        simbot        Number of samples:   39  \n",
      "Malware family:        loring        Number of samples:   39  \n",
      "Malware family:         delf         Number of samples:   34  \n",
      "Malware family:        vilsel        Number of samples:   33  \n",
      "Malware family:         zusy         Number of samples:   32  \n",
      "Malware family:      softcnapp       Number of samples:   31  \n",
      "Malware family:      amonetize       Number of samples:   30  \n",
      "Malware family:      crossrider      Number of samples:   30  \n",
      "Malware family:    downloadadmin     Number of samples:   29  \n",
      "Malware family:        cosmu         Number of samples:   28  \n",
      "Malware family:        urelas        Number of samples:   27  \n",
      "Malware family:        imali         Number of samples:   26  \n",
      "Malware family:        memery        Number of samples:   24  \n",
      "Malware family:        neshta        Number of samples:   24  \n",
      "Malware family:       dealply        Number of samples:   23  \n",
      "Malware family:     airinstaller     Number of samples:   22  \n",
      "Malware family:        lamer         Number of samples:   22  \n",
      "Malware family:        berbew        Number of samples:   21  \n",
      "Malware family:        lydra         Number of samples:   21  \n",
      "Malware family:        mydoom        Number of samples:   21  \n",
      "Malware family:        yuner         Number of samples:   20  \n",
      "Malware family:         chir         Number of samples:   19  \n",
      "Malware family:       gamarue        Number of samples:   18  \n",
      "Malware family:  downloadassistant   Number of samples:   18  \n",
      "Malware family:       bundlore       Number of samples:   18  \n",
      "Malware family:        shipup        Number of samples:   17  \n",
      "Malware family:        picsys        Number of samples:   17  \n",
      "Malware family:      convertad       Number of samples:   17  \n",
      "Malware family:       filetour       Number of samples:   16  \n",
      "Malware family:       dlhelper       Number of samples:   13  \n",
      "Malware family:        hotbar        Number of samples:   13  \n",
      "\n",
      "4287 test samples belonging to 65 malware families\n",
      "Malware family:        parite        Number of samples:  174  \n",
      "Malware family:      loadmoney       Number of samples:  165  \n",
      "Malware family:     installcore      Number of samples:  161  \n",
      "Malware family:      softpulse       Number of samples:  160  \n",
      "Malware family:      browsefox       Number of samples:  158  \n",
      "Malware family:        virut         Number of samples:  157  \n",
      "Malware family:      outbrowse       Number of samples:  156  \n",
      "Malware family:       allaple        Number of samples:  154  \n",
      "Malware family:        sytro         Number of samples:  153  \n",
      "Malware family:        upatre        Number of samples:  153  \n",
      "Malware family:        sality        Number of samples:  143  \n",
      "Malware family:         mira         Number of samples:  142  \n",
      "Malware family:       virlock        Number of samples:  141  \n",
      "Malware family:        vobfus        Number of samples:  137  \n",
      "Malware family:       firseria       Number of samples:  124  \n",
      "Malware family:      multiplug       Number of samples:  124  \n",
      "Malware family:        dinwod        Number of samples:  112  \n",
      "Malware family:         klez         Number of samples:   97  \n",
      "Malware family:        expiro        Number of samples:   79  \n",
      "Malware family:      vtflooder       Number of samples:   74  \n",
      "Malware family:        wapomi        Number of samples:   71  \n",
      "Malware family:    installmonster    Number of samples:   63  \n",
      "Malware family:        ramnit        Number of samples:   62  \n",
      "Malware family:       icloader       Number of samples:   61  \n",
      "Malware family:         zbot         Number of samples:   56  \n",
      "Malware family:        ipamor        Number of samples:   54  \n",
      "Malware family:        autoit        Number of samples:   54  \n",
      "Malware family:        ircbot        Number of samples:   51  \n",
      "Malware family:    downloadguide     Number of samples:   49  \n",
      "Malware family:        domaiq        Number of samples:   45  \n",
      "Malware family:        loring        Number of samples:   44  \n",
      "Malware family:     installerex      Number of samples:   44  \n",
      "Malware family:        neshta        Number of samples:   44  \n",
      "Malware family:      bladabindi      Number of samples:   42  \n",
      "Malware family:        eorezo        Number of samples:   41  \n",
      "Malware family:      amonetize       Number of samples:   38  \n",
      "Malware family:        urelas        Number of samples:   37  \n",
      "Malware family:        cosmu         Number of samples:   36  \n",
      "Malware family:        vilsel        Number of samples:   36  \n",
      "Malware family:     airinstaller     Number of samples:   32  \n",
      "Malware family:      convertad       Number of samples:   31  \n",
      "Malware family:         delf         Number of samples:   31  \n",
      "Malware family:    downloadadmin     Number of samples:   30  \n",
      "Malware family:        ibryte        Number of samples:   30  \n",
      "Malware family:         zusy         Number of samples:   29  \n",
      "Malware family:        simbot        Number of samples:   29  \n",
      "Malware family:      softcnapp       Number of samples:   28  \n",
      "Malware family:      crossrider      Number of samples:   28  \n",
      "Malware family:        lamer         Number of samples:   27  \n",
      "Malware family:       dealply        Number of samples:   25  \n",
      "Malware family:       gamarue        Number of samples:   24  \n",
      "Malware family:        lydra         Number of samples:   22  \n",
      "Malware family:        hotbar        Number of samples:   22  \n",
      "Malware family:        memery        Number of samples:   22  \n",
      "Malware family:        imali         Number of samples:   21  \n",
      "Malware family:        mydoom        Number of samples:   21  \n",
      "Malware family:       bundlore       Number of samples:   20  \n",
      "Malware family:  downloadassistant   Number of samples:   18  \n",
      "Malware family:         chir         Number of samples:   18  \n",
      "Malware family:        picsys        Number of samples:   17  \n",
      "Malware family:        yuner         Number of samples:   16  \n",
      "Malware family:       dlhelper       Number of samples:   16  \n",
      "Malware family:        shipup        Number of samples:   14  \n",
      "Malware family:        berbew        Number of samples:   13  \n",
      "Malware family:       filetour       Number of samples:   11  \n",
      "\n",
      "\n",
      "         family  fam_num  selected    train     dev    test\n",
      "count     58180  58180.0   28582.0  20007.0  4288.0  4287.0\n",
      "unique     1270   1270.0       NaN      NaN     NaN     NaN\n",
      "top     allaple     42.0       NaN      NaN     NaN     NaN\n",
      "freq      10569  10569.0       NaN      NaN     NaN     NaN\n",
      "mean        NaN      NaN       1.0      1.0     1.0     1.0\n",
      "std         NaN      NaN       0.0      0.0     0.0     0.0\n",
      "min         NaN      NaN       1.0      1.0     1.0     1.0\n",
      "25%         NaN      NaN       1.0      1.0     1.0     1.0\n",
      "50%         NaN      NaN       1.0      1.0     1.0     1.0\n",
      "75%         NaN      NaN       1.0      1.0     1.0     1.0\n",
      "max         NaN      NaN       1.0      1.0     1.0     1.0\n"
     ]
    }
   ],
   "source": [
    "samples_data = pp_action.pre_process(config)\n",
    "pp_action.split_show_data(samples_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = samples_data.index[samples_data['train'] == 1].tolist()\n",
    "x_dev = samples_data.index[samples_data['dev'] == 1].tolist()\n",
    "x_test = samples_data.index[samples_data['test'] == 1].tolist()\n",
    "y_train = samples_data.fam_num[samples_data['train'] == 1].tolist()\n",
    "y_dev = samples_data.fam_num[samples_data['dev'] == 1].tolist()\n",
    "y_test = samples_data.fam_num[samples_data['test'] == 1].tolist()\n",
    "y_test_fam = samples_data.family[samples_data['test'] == 1].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Feature extraction\n",
    "\n",
    "Since the DAN required a very considerable amount fo time to train with the full dataset, we will try reducing the dimensionality.\n",
    "\n",
    "To achieve this we will use the Principal Component Analysis in order to operate on the sparse vectros."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xm_train_e = np.loadtxt('data/matrix/pca_1024_20007_tr.txt')\n",
    "xm_dev_e = np.loadtxt('data/matrix/pca_1024_4288_dv.txt')\n",
    "xm_test_e = np.loadtxt('data/matrix/pca_1024_4287_te.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature selection\n",
    "\n",
    "An alternative to feature extraction, which creates a new -artificial- set of features, is feature selection. With feature selection we mean a method which tries to isolate the most important features for a specific learning task, among the natural features of the dataset.\n",
    "\n",
    "We will attempt to select the most relevant features by using random forest classifiers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xm_train_s = np.loadtxt('data/matrix/rfc_1024_20007_tr.txt')\n",
    "xm_dev_s = np.loadtxt('data/matrix/rfc_1024_4288_dv.txt')\n",
    "xm_test_s = np.loadtxt('data/matrix/rfc_1024_4287_te.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification\n",
    "\n",
    "Now we can try classification with both data sets.\n",
    "\n",
    "First with extracted features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predicted, model, modifier = cla_dan.classify(xm_train_e, xm_dev_e, xm_test_e, y_train, y_dev, y_test, config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "evaluation.evaluate_classification(model[0], y_test_fam, y_predicted, model[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now with selected features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predicted, model, modifier = cla_dan.classify(xm_train_s, xm_dev_s, xm_test_s, y_train, y_dev, y_test, config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "evaluation.evaluate_classification(model[0], y_test_fam, y_predicted, model[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Let's try with a higher number of features selected with the random forest classifier method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "xm_train_s2 = np.loadtxt('data/matrix/rfc_2048_20007_tr.txt')\n",
    "xm_dev_s2 = np.loadtxt('data/matrix/rfc_2048_4288_dv.txt')\n",
    "xm_test_s2 = np.loadtxt('data/matrix/rfc_2048_4287_te.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (2048, 20007)\n",
      "Y_train shape: (65, 20007)\n",
      "X_dev shape: (2048, 4288)\n",
      "Y_dev shape: (65, 4288)\n",
      "X_test shape: (2048, 4287)\n",
      "Y_test shape: (65, 4287)\n",
      "Cost after epoch 0: 1.437555\n",
      "Train Accuracy: 0.880291\n",
      "Dev Accuracy: 0.868237\n",
      "Learning Rate: 0.000999682\n",
      "\n",
      "Cost after epoch 100: 0.047710\n",
      "Train Accuracy: 0.988303\n",
      "Dev Accuracy: 0.948694\n",
      "Learning Rate: 0.000968352\n",
      "\n",
      "Cost after epoch 200: 0.037709\n",
      "Train Accuracy: 0.991502\n",
      "Dev Accuracy: 0.945429\n",
      "Learning Rate: 0.000938004\n",
      "\n",
      "Cost after epoch 300: 0.031411\n",
      "Train Accuracy: 0.993202\n",
      "Dev Accuracy: 0.944963\n",
      "Learning Rate: 0.000908608\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_predicted, model, modifier = cla_dan.classify(xm_train_s2, xm_dev_s2, xm_test_s2, y_train, y_dev, y_test, config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluation.evaluate_classification(model[0], y_test_fam, y_predicted, model[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}