Speech-transfer-learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "np.random.seed(1969)\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(1969)\n",
    "\n",
    "from scipy import signal\n",
    "import glob\n",
    "import re\n",
    "import pandas as pd\n",
    "import gc\n",
    "from scipy.io import wavfile\n",
    "\n",
    "from keras import optimizers, losses, activations, models\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import GRU, Convolution2D, Dense, Input, Flatten, Dropout, MaxPooling2D, BatchNormalization, Conv3D, ConvLSTM2D, concatenate, merge, ZeroPadding2D\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.layers.pooling import AveragePooling2D, GlobalAveragePooling2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.utils import multi_gpu_model\n",
    "import keras.backend as K\n",
    "\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from python_speech_features import mfcc\n",
    "from python_speech_features import delta\n",
    "from python_speech_features import logfbank\n",
    "\n",
    "import fnmatch\n",
    "import random\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load local modules, custom methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utilities.utilities import list_wavs_fname, pad_audio, chop_audio, label_transform_audio, label_transform_speech, test_data_generator\n",
    "from utilities.wav_utilities import read\n",
    "from DenseNet.custom_layers import *\n",
    "from DenseNet.DenseNet import DenseNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_on_model(model, epochs_, lr_, decay_, train_x, train_y, val_x, val_y, BATCH_SIZE=256):\n",
    "    sgd = optimizers.SGD(lr=lr_, decay=decay_, momentum=0.9, nesterov=True)\n",
    "    model.compile(optimizer=sgd,loss='binary_crossentropy',metrics=['categorical_accuracy'])\n",
    "    model.fit(train_x, train_y, batch_size=BATCH_SIZE, validation_data=(val_x, val_y), epochs=epochs_, shuffle=True, verbose=1, callbacks=[])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load UrbanSound8K dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "urban = pd.read_csv(\"UrbanSound8K/metadata/UrbanSound8K.csv\")\n",
    "classes = urban[\"class\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "root_path = r'UrbanSound8K'\n",
    "out_path = r'UrbanSound8K'\n",
    "model_path = r'.'\n",
    "train_data_path = os.path.join(r\"UrbanSound8K\", 'audio')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matches = []   #.wav file paths in UrbanSound8K dataset\n",
    "for root, dirnames, filenames in os.walk(train_data_path):\n",
    "    for filename in fnmatch.filter(filenames, '*.wav'):\n",
    "        matches.append(os.path.join(root, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "matches[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"                                                                                                                                                       \n",
    "Contributions from:                                                                                                                                       \n",
    "Luis Andre Dutra e Silva                                                                                                                                  \n",
    "https://www.kaggle.com/mindcool/lb-0-77-keras-gru-with-filter-banks-features                                                                              \n",
    "\"\"\"\n",
    "\n",
    "new_sample_rate=16000\n",
    "y_train = []\n",
    "x_train = np.zeros((8732,199,26),np.float32)\n",
    "G = []\n",
    "ix = 0\n",
    "for match in tqdm(matches):\n",
    "    s_ = read(match)\n",
    "    if s_ == None:\n",
    "        continue\n",
    "    sample_rate = s_[0]\n",
    "    samples = s_[1]\n",
    "    samples = pad_audio(samples)\n",
    "    if len(samples) > 16000:\n",
    "        n_samples = chop_audio(samples)\n",
    "    else:\n",
    "        n_samples = [samples]\n",
    "    ok = 0\n",
    "    for samples in n_samples:\n",
    "        filter_banks = logfbank(samples)\n",
    "        filter_banks -= (np.mean(filter_banks, axis=0) + 1e-8)\n",
    "#         print(filter_banks.shape)\n",
    "        if filter_banks.shape != (199,26):\n",
    "            continue\n",
    "        ok = 1\n",
    "        x_train[ix,:,:] = filter_banks\n",
    "        break\n",
    "    if ok == 0:\n",
    "        continue\n",
    "    m = re.search('/([^/]+)$', match)\n",
    "    if m:\n",
    "        found = m.group(0)\n",
    "    y_train.append(list(urban.loc[urban['slice_file_name'] == found[1:]][\"class\"])[0])\n",
    "    ix += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = np.expand_dims(x_train, axis=-1)\n",
    "\n",
    "y_train = np.array(y_train)\n",
    "y_train = label_transform_audio(y_train)\n",
    "y_train = np.array(y_train)\n",
    "\n",
    "x_train = x_train[:len(y_train)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "index = range(len(x_train))\n",
    "random.shuffle(index)\n",
    "num_train = 7000\n",
    "train = index[:num_train]\n",
    "val = index[num_train:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model on UrbanSound8K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model1 = DenseNet(input_shape=(199, 26, 1),classes=10)\n",
    "parallel_model = multi_gpu_model(model1, gpus=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_on_model(parallel_model, epochs_=70, lr_=0.001, decay_=1e-7, \n",
    "               train_x = x_train[train], train_y = y_train[train], val_x=x_train[val], val_y=y_train[val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_on_model(parallel_model, epochs_=30, lr_=0.0001, decay_=1e-8, \n",
    "               train_x = x_train[train], train_y = y_train[train], val_x=x_train[val], val_y=y_train[val])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Speech Command dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "root_path = r'.'\n",
    "out_path = r'.'\n",
    "model_path = r'.'\n",
    "train_data_path = os.path.join(r\".\", 'input', 'train', 'audio')\n",
    "test_data_path = os.path.join(r\".\", 'input', 'test', 'audio')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"                                                                                                                                                       \n",
    "Contributions from:                                                                                                                                       \n",
    "Luis Andre Dutra e Silva                                                                                                                                  \n",
    "https://www.kaggle.com/mindcool/lb-0-77-keras-gru-with-filter-banks-features                                                                              \n",
    "\"\"\"\n",
    "\n",
    "fpaths = glob.glob(os.path.join(train_data_path, r'*/*' + \"wav\"))\n",
    "labels, fnames = list_wavs_fname(fpaths)\n",
    "new_sample_rate=16000\n",
    "y_train = []\n",
    "x_train = np.zeros((64727,99,26),np.float32)\n",
    "G = []\n",
    "ix = 0\n",
    "for label, fname in tqdm(zip(labels, fnames)):\n",
    "    sample_rate, samples = wavfile.read(os.path.join(train_data_path, label, fname))\n",
    "    samples = pad_audio(samples)\n",
    "    if len(samples) > 16000:\n",
    "        n_samples = chop_audio(samples)\n",
    "    else:\n",
    "        n_samples = [samples]\n",
    "    for samples in n_samples:\n",
    "        filter_banks = logfbank(samples)\n",
    "        filter_banks -= (np.mean(filter_banks, axis=0) + 1e-8)\n",
    "        x_train[ix,:,:] = filter_banks\n",
    "    y_train.append(label)\n",
    "    group = fname.split('_')[0]\n",
    "    G.append(group)\n",
    "    ix += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = np.expand_dims(x_train, axis=-1)\n",
    "\n",
    "y_train = np.array(y_train)\n",
    "y_train = label_transform_speech(y_train)\n",
    "label_index = y_train.columns.values\n",
    "y_train = np.array(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "index = range(len(x_train))\n",
    "random.shuffle(index)\n",
    "num_train = 58000\n",
    "train = index[:num_train]\n",
    "val = index[num_train:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load pretrained model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_model = DenseNet(input_shape=(99, 26, 1),classes=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(new_model.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in tqdm(range(1,len(new_model.layers))):\n",
    "    if len(model1.layers[i].get_weights()):\n",
    "        new_model.layers[i].set_weights(model1.layers[i].get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parallel_model = multi_gpu_model(new_model, gpus=4)\n",
    "parallel_model.compile(optimizer=sgd,loss='binary_crossentropy',metrics=['categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train new model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_on_model(parallel_model, epochs_=70, lr_=0.002, decay_=1e-7, \n",
    "               train_x = x_train[train], train_y = y_train[train], val_x=x_train[val], val_y=y_train[val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_on_model(parallel_model, epochs_=30, lr_=0.0002, decay_=1e-8, \n",
    "               train_x = x_train[train], train_y = y_train[train], val_x=x_train[val], val_y=y_train[val])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kaggle Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gc.collect()\n",
    "\n",
    "index = []\n",
    "results = []\n",
    "probs = []\n",
    "for fnames, imgs in tqdm(test_data_generator(batch=32)):\n",
    "    li = list(imgs.shape)\n",
    "    li.append(1)\n",
    "    imgs = imgs.reshape(tuple(li))\n",
    "    predicts = model1.predict(imgs)\n",
    "    probs.extend(predicts)\n",
    "    predicts = np.argmax(predicts, axis=1)\n",
    "    predicts = [label_index[p] for p in predicts]\n",
    "    index.extend(fnames)\n",
    "    results.extend(predicts)\n",
    "\n",
    "df = pd.DataFrame(columns=['fname', 'label'])\n",
    "df['fname'] = index\n",
    "df['label'] = results\n",
    "df.to_csv(os.path.join(out_path, 'submit.csv'), index=False)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}