Supervised_Pretraining.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "W-YuikCiP53N"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import hickle as hkl\n",
    "import sklearn.manifold\n",
    "import copy\n",
    "import __main__ as main\n",
    "import argparse\n",
    "import pandas as pd\n",
    "from tabulate import tabulate\n",
    "from tensorflow import keras\n",
    "\n",
    "seed = 1\n",
    "tf.random.set_seed(seed)\n",
    "np.random.seed(seed)\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dwFHmiuFP53O"
   },
   "outputs": [],
   "source": [
    "# Library scripts\n",
    "import utils \n",
    "import training\n",
    "import model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experimentSetting = 'LODO'\n",
    "\n",
    "testingDataset = 'UCI'\n",
    "# 'HHAR','MobiAct','MotionSense','RealWorld_Waist','UCI','PAMAP'\n",
    "evaluationType = 'group'\n",
    "# 'subject','group'\n",
    "\n",
    "architecture = \"ispl\"\n",
    "# \"mobilehart_xs,hart,ispl,deepConvlstm\"\n",
    "\n",
    "# 1024,2048\n",
    "finetune_epoch = 1\n",
    "\n",
    "finetune_batch_size = 128\n",
    "\n",
    "loss = 'Adam'\n",
    "# 'LARS', 'Adam', 'SGD'\n",
    "SSL_LR = 5e-4\n",
    "\n",
    "input_shape = (128,6)\n",
    "\n",
    "SSL_batch_size = 128\n",
    "\n",
    "SSL_epochs = 200\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_fit_args(parser):\n",
    "    \"\"\"\n",
    "    parser : argparse.ArgumentParser\n",
    "    return a parser added with args required by fit\n",
    "    \"\"\"\n",
    "    # Training settings\n",
    "    parser.add_argument('--architecture', type=str, default=architecture, \n",
    "                help='The choice of model architecture')  \n",
    "    parser.add_argument('--testingDataset', type=str, default=testingDataset, \n",
    "            help='Left out dataset')  \n",
    "    parser.add_argument('--evaluationType', type=str, default=evaluationType, \n",
    "        help='Dataset group evaluation or subject by subject evaluation')  \n",
    "    parser.add_argument('--SSL_epochs', type=int, default=SSL_epochs, \n",
    "        help='SSL Epochs')  \n",
    "    parser.add_argument('--SSL_batch_size', type=int, default=SSL_batch_size, \n",
    "        help='SSL batch_size')  \n",
    "    parser.add_argument('--finetune_epoch', type=int, default=finetune_epoch, \n",
    "        help='Fine_tune Epochs')  \n",
    "    parser.add_argument('--loss', type=str, default=loss, \n",
    "        help='Specify the loss') \n",
    "    parser.add_argument('--SSL_LR', type=float, default=SSL_LR, \n",
    "        help='Specify the learning rate for the SSL techniques') \n",
    "\n",
    "    args = parser.parse_args()\n",
    "    return args\n",
    "def is_interactive():\n",
    "    return not hasattr(main, '__file__')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not is_interactive():\n",
    "    rootdir = './SSL_Project/'\n",
    "    args = add_fit_args(argparse.ArgumentParser(description='Supervised Pre-training'))\n",
    "    testingDataset = args.testingDataset\n",
    "    evaluationType = args.evaluationType\n",
    "    SSL_epochs = args.SSL_epochs\n",
    "    SSL_batch_size = args.SSL_batch_size\n",
    "    finetune_epoch = args.finetune_epoch\n",
    "    loss = args.loss\n",
    "    SSL_LR = args.SSL_LR\n",
    "    architecture =  args.architecture\n",
    "else:\n",
    "    rootdir = '../../'\n",
    "\n",
    "initWeightDir = rootdir+architecture+'_Weight.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.set_floatx('float32')\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "for gpu in gpus:\n",
    "    tf.config.experimental.set_memory_growth(gpu, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "V8oWxqFZP53R"
   },
   "outputs": [],
   "source": [
    "# Dataset Metadata\n",
    "# ACTIVITY_LABEL = []\n",
    "ACTIVITY_LABEL = ['Downstairs', 'Upstairs','Running','Sitting','Standing','Walking','Lying','Cycling','Nordic_Walking','Jumping']\n",
    "output_shape = len(ACTIVITY_LABEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataDir = rootdir+'SSL_PipelineUnion/'+experimentSetting+'/'\n",
    "projectName = 'supervised_'+str(finetune_epoch)+'/'+architecture+'_pretrain_epochs_'+str(SSL_epochs)\n",
    "testMode = False\n",
    "if(finetune_epoch < 10):\n",
    "    testMode = True\n",
    "    projectName= projectName + '/tests'\n",
    "    \n",
    "dataSetting = testingDataset\n",
    "\n",
    "working_directory = rootdir+'results/'+projectName+'/'+experimentSetting+'/'+dataSetting+'/'\n",
    "pretrained_dir = working_directory + evaluationType + '/'\n",
    "\n",
    "os.makedirs(pretrained_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasetList = [\"HHAR\",\"MobiAct\",\"MotionSense\",\"RealWorld_Waist\",\"UCI\",\"PAMAP\"] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SSLdatasetList = copy.deepcopy(datasetList)\n",
    "SSLdatasetList.remove(testingDataset)\n",
    "SSL_data = []\n",
    "SSL_label = []\n",
    "\n",
    "SSL_val_data = []\n",
    "SSL_val_label = []\n",
    "\n",
    "for datasetName in SSLdatasetList:\n",
    "    SSL_data.append(hkl.load(dataDir + 'testData/'+str(datasetName)+'_data.hkl'))\n",
    "    SSL_data.append(hkl.load(dataDir + 'fineTuneData/'+str(datasetName)+'_70_data.hkl'))\n",
    "    SSL_label.append(hkl.load(dataDir + 'testData/'+str(datasetName)+'_label.hkl'))\n",
    "    SSL_label.append(hkl.load(dataDir + 'fineTuneData/'+str(datasetName)+'_70_label.hkl'))\n",
    "\n",
    "    SSL_val_data.append(hkl.load(dataDir + 'valData/'+str(datasetName)+'_data.hkl'))\n",
    "    SSL_val_label.append(hkl.load(dataDir + 'valData/'+str(datasetName)+'_label.hkl'))\n",
    "\n",
    "SSL_data = np.vstack((np.hstack((SSL_data))))\n",
    "SSL_label = np.vstack((np.hstack((SSL_label))))\n",
    "\n",
    "SSL_val_data = np.vstack((np.hstack((SSL_val_data))))\n",
    "SSL_val_label = np.vstack((np.hstack((SSL_val_label))))\n",
    "\n",
    "testData = hkl.load(dataDir + 'testData/'+testingDataset+'_data.hkl')\n",
    "testLabel = hkl.load(dataDir + 'testData/'+testingDataset+'_label.hkl')\n",
    "\n",
    "valData = hkl.load(dataDir + 'valData/'+testingDataset+'_data.hkl')\n",
    "valLabel = hkl.load(dataDir + 'valData/'+testingDataset+'_label.hkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testData = np.asarray([np.vstack((data)) for data in testData])\n",
    "testLabel = np.asarray([np.vstack((label)) for label in testLabel])\n",
    "valData = np.asarray([np.vstack((data)) for data in valData])\n",
    "valLabel = np.asarray([np.vstack((label)) for label in valLabel])\n",
    "testDataLength = [label.shape[0] for label in testLabel]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrain_base_save_path = working_directory+architecture+\"_trained_supervised.h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(architecture == \"hart\"):\n",
    "    learningModel = model.HART(input_shape,output_shape)\n",
    "elif(architecture == \"ispl\"):\n",
    "    learningModel = model.ispl_inception(input_shape,output_shape)\n",
    "elif(architecture == \"deepConvlstm\"):\n",
    "    learningModel = model.deepConvLSTM((128,6,1),output_shape)\n",
    "else:\n",
    "    learningModel = model.mobilehart_xs(input_shape,output_shape)\n",
    "    \n",
    "if(not os.path.exists(initWeightDir)):\n",
    "    print(\"model weights initialization not found, generating one\")\n",
    "    learningModel.save_weights(initWeightDir)\n",
    "else:\n",
    "    print(\"found initialization model weights\")\n",
    "    learningModel.load_weights(initWeightDir)\n",
    "    \n",
    "totalLayer = len(learningModel.layers)\n",
    "\n",
    "if(architecture == \"ispl_inception\" or architecture == 'deepConvLSTM'):\n",
    "    feature_extraction_layer = totalLayer - 2\n",
    "else:\n",
    "    feature_extraction_layer = totalLayer - 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if(not os.path.exists(pretrain_base_save_path) or testMode):\n",
    "    optimizer = tf.keras.optimizers.Adam(5e-4)\n",
    "    learningModel.compile(\n",
    "        optimizer=optimizer, loss=keras.losses.CategoricalCrossentropy(label_smoothing=0.1), metrics=[\"accuracy\"]\n",
    "    )\n",
    "    history = learningModel.fit(x = SSL_data,y = SSL_label, validation_data = (SSL_val_data,SSL_val_label),batch_size = SSL_batch_size, epochs = SSL_epochs,verbose=2)\n",
    "\n",
    "    utils.plot_learningCurve(history,SSL_epochs,working_directory) \n",
    "    \n",
    "    intermediate_model = model.extract_intermediate_model_from_base_model(learningModel,feature_extraction_layer)\n",
    "    perplexity = 30.0\n",
    "    embeddings = intermediate_model.predict(testData, batch_size=1024,verbose=0)\n",
    "    tsne_model = sklearn.manifold.TSNE(perplexity=perplexity, verbose=0, random_state=42)\n",
    "    tsne_projections = tsne_model.fit_transform(embeddings)\n",
    "    labels_argmax = np.argmax(testLabel, axis=1)\n",
    "    unique_labels = np.unique(labels_argmax)\n",
    "    utils.projectTSNE('TSNE_Embeds',pretrained_dir,ACTIVITY_LABEL,labels_argmax,tsne_projections,unique_labels )\n",
    "    utils.projectTSNEWithShape('TSNE_Embeds_shape',pretrained_dir,ACTIVITY_LABEL,labels_argmax,tsne_projections,unique_labels )\n",
    "    hkl.dump(tsne_projections,pretrained_dir+'tsne_projections.hkl')\n",
    "    learningModel.save_weights(pretrain_base_save_path)\n",
    "else:\n",
    "    learningModel.load_weights(pretrain_base_save_path)\n",
    "    optimizer = tf.keras.optimizers.Adam(5e-4)\n",
    "    learningModel.compile(\n",
    "        optimizer=optimizer, loss=keras.losses.CategoricalCrossentropy(label_smoothing=0.1), metrics=[\"accuracy\"]\n",
    "    )\n",
    "    print(\"Pre-trained model found, skipping training of pretrained model\",flush = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TMELGS4VP53U"
   },
   "source": [
    "### Downstream tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratio = [1,5,10,70]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ratioResults = {}\n",
    "for trainingRatio in ratio:\n",
    "    print(\"Now downstreaming on \"+testingDataset+\" dataset with train ratio \"+str(trainingRatio), flush = True)\n",
    "    evaluation_dir = pretrained_dir+'ratio_'+ str(trainingRatio) + '/'\n",
    "    os.makedirs(evaluation_dir, exist_ok=True)\n",
    "    fineTuneData,fineTuneLabel = utils.loadFineTuneData(trainingRatio,testingDataset,evaluationType,dataDir)\n",
    "    evaluationsF1 = training.downStreamPipeline(fineTuneData,fineTuneLabel,valData,valLabel,testData,testLabel,pretrain_base_save_path,evaluation_dir,initWeightDir,learningModel,\n",
    "                                                output_shape = output_shape,\n",
    "                                                finetune_epoch = finetune_epoch, \n",
    "                                                finetune_batch_size = finetune_batch_size, \n",
    "                                                feature_extraction_layer = feature_extraction_layer)\n",
    "    ratioResults['ratio_'+str(trainingRatio)] = evaluationsF1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "npRatio = np.asarray(list(ratioResults.values())).T\n",
    "evaluationMethods = ['Transfer','Linear','Full','Full_Unfrozen','Full_Random']\n",
    "ratioHeaders = ['ratio_'+str(rat) for rat in ratio]\n",
    "ratioHeaders.insert(0, \"dataset\")\n",
    "for evalIndex, methods in enumerate(evaluationMethods):\n",
    "    toWriteEvaluation = {}\n",
    "    toWriteEvaluation['dataset'] = [testingDataset]\n",
    "    for ratioIndex, rat in enumerate(ratio):\n",
    "        toWriteEvaluation['ratio_'+str(rat)] = [npRatio[evalIndex][ratioIndex]]\n",
    "    tabular = tabulate(toWriteEvaluation, headers=\"keys\")\n",
    "    print(methods)\n",
    "    print(tabular)\n",
    "    print()\n",
    "    text_file = open(pretrained_dir +methods+'_report.csv',\"w\")\n",
    "    text_file.write(tabular)\n",
    "    text_file.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allTrained = True \n",
    "for methods in evaluationMethods:\n",
    "    print(\"Processing \"+str(methods) +\" report\")\n",
    "    fullReport = []\n",
    "    ratioHeaders = ['ratio_'+str(rat) for rat in ratio]\n",
    "    ratioHeaders.insert(0, \"dataset\")\n",
    "    fullReport.append(ratioHeaders)\n",
    "    if(allTrained):\n",
    "        for datasetName in datasetList:\n",
    "            checkDir = rootdir+'results/'+projectName+'/'+experimentSetting+'/'+datasetName+'/'+evaluationType+'/'+methods+\"_report.csv\"\n",
    "            if(not os.path.exists(checkDir)):\n",
    "                print(\"Dir below not found:\")\n",
    "                print(checkDir)\n",
    "                allTrained = False\n",
    "                break\n",
    "            readData = pd.read_table(checkDir, delim_whitespace=True)\n",
    "            fullReport.append(readData.to_numpy()[1])\n",
    "    else:\n",
    "        break\n",
    "    if(allTrained):\n",
    "        print(\"Generating \"+str(methods) + \" report\")\n",
    "        tabular2 = tabulate(fullReport)\n",
    "        text_file = open(rootdir+'results/'+projectName+'/'+experimentSetting+'/'+str(evaluationType)+'_'+str(methods)+'_report.csv',\"w\")\n",
    "        text_file.write(tabular2)\n",
    "        text_file.close()\n",
    "        print(tabular2)\n",
    "        print()\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "SimCLR_MotionSense.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}