MedSpecSearch LRP and User Query Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import pandas\n",
    "import pickle\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\P\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1209: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Dermatology': 'Deri ve Zührevi Hastalıkları (Cildiye)', 'Internal Medicine': 'İç Hastalıkları (Dahiliye)', 'Neurology': 'Nöroloji', 'Obstetrics & Gynecology': 'Kadın Hastalıkları ve Doğum', 'Ophthalmology': 'Göz Hastalıkları', 'Orthopaedic Surgery': 'Ortopedi ve Travmatoloji', 'Otolaryngology': 'Kulak Burun Boğaz Hastalıkları', 'Pediatrics': 'Çocuk Sağlığı ve Hastalıkları', 'Psychiatry': 'Ruh Sağlığı ve Hastalıkları', 'Radiology-Diagnostic': 'Radyoloji', 'Surgery-General': 'Genel Cerrahi', 'Urology': 'Üroloji'}\n"
     ]
    }
   ],
   "source": [
    "# Our code\n",
    "import lrp\n",
    "import EmbedHelper\n",
    "import DataLoader\n",
    "import Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'Fast Text', 2: 'Google News', 3: 'HealthTap', 4: 'Pubmed', 5: 'Glove', 6: 'iCliniq Trigram', 7: 'iCliniq default'}\n"
     ]
    }
   ],
   "source": [
    "embedDict = EmbedHelper.EmbeddingHandler.embedDict\n",
    "print(embedDict)\n",
    "configs = {\n",
    "    \"vectorSize\":300,\n",
    "    \"trainNewModel\":True,\n",
    "    \"dataColumn\":\"question\",\n",
    "    \"maxLength\":128,\n",
    "    \"batchSize\":8,\n",
    "    \"embeddingType\":embedDict[2],\n",
    "    \"ELMo\":True,\n",
    "    \"PreEmbed\":True,\n",
    "    \"restore\":True\n",
    "}\n",
    "\n",
    "inputSize = configs[\"maxLength\"]\n",
    "vectorSize = configs[\"vectorSize\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Google News\n"
     ]
    }
   ],
   "source": [
    "EmbedModel = EmbedHelper.EmbeddingHandler(configs[\"embeddingType\"], False, 300, \"Embeddings\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new data that is larger with 9800~ data instances\n",
    "trainData = np.load(\"data//icliniq//iCliniq_14K//icliniq_14k_train_questions.npy\")\n",
    "trainTarget = np.load(\"data//icliniq//iCliniq_14K//icliniq_14k_train_target.npy\")\n",
    "testData = np.load(\"data//icliniq//iCliniq_14K//icliniq_14k_test_questions.npy\")\n",
    "testTarget = np.load(\"data//icliniq//iCliniq_14K//icliniq_14k_test_target.npy\")\n",
    "\n",
    "trainData_raw = np.load(\"data//icliniq//iCliniq_14K//icliniq_14k_train_questions_raw.npy\")\n",
    "testData_raw = np.load(\"data//icliniq//iCliniq_14K//icliniq_14k_test_questions_raw.npy\")\n",
    "\n",
    "ClassDict = {}\n",
    "with open('fold0classDict.pkl', 'rb') as f:\n",
    "    ClassDict = pickle.load(f)\n",
    "outputSize = len(ClassDict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train-Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTokenLengths(token):\n",
    "    return [len(item) for item in token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluatePerformance(nnModel,sess,testData,testTarget,batchSize,uncertaintyCoef):\n",
    "    reverseClassDict = {value:key for key,value in ClassDict.items()}\n",
    "    top3 = []\n",
    "    \n",
    "    dataSize = testData.shape[0]\n",
    "    start = 0\n",
    "    end = batchSize\n",
    "    \n",
    "    totalAcc = 0\n",
    "    totalUcAcc = 0\n",
    "    totalDataRate = 0\n",
    "    \n",
    "    truth = None\n",
    "    predu = None\n",
    "    \n",
    "    testTruth = np.array([])\n",
    "    testPred = np.array([])\n",
    "    \n",
    "    testEvTrue = 0\n",
    "    testEvFail = 0\n",
    "    \n",
    "    while(start<dataSize):\n",
    "        data = np.array(testData[start:end])\n",
    "        dataClean = data\n",
    "        \n",
    "        if(configs[\"PreEmbed\"]):\n",
    "            data = EmbedModel.vectorizeBatch(data)\n",
    "        \n",
    "        outputData = np.array(testTarget[start:end])\n",
    "        cutSize = data.shape[0]\n",
    "        tokens_length = getTokenLengths(data)\n",
    "        \n",
    "        fd = {nnModel.nn_inputs:dataClean,nnModel.nn_vector_inputs:data,nnModel.nn_outputs:outputData,nnModel.isTraining:False,nnModel.token_lengths:tokens_length,\n",
    "             nnModel.uncertaintyRatio:uncertaintyCoef}\n",
    "        \n",
    "        prob, testBAcc,nnTruth,nnPrediction,nnMatch,evCor,evFail,ucAcc,dataRate = sess.run([nnModel.prob, nnModel.accuracy,nnModel.truths,nnModel.predictions\n",
    "                                                                       ,nnModel.correct_predictions,nnModel.mean_ev_succ,nnModel.mean_ev_fail,nnModel.ucAccuracy,\n",
    "                                                                                     nnModel.dataRatio]\n",
    "                                                                      ,feed_dict=fd)\n",
    "        # For top 3\n",
    "        prob = prob[0]\n",
    "        probDict = {reverseClassDict[i]:prob[i] for i in np.arange(outputSize)}\n",
    "        probMatrix = []\n",
    "        for i in range(len(prob)):\n",
    "            probMatrix.append([reverseClassDict[i], prob[i]])\n",
    "        probMatrix = sorted(probMatrix, key=lambda x: (x[1]), reverse=True)\n",
    "        top3.append(probMatrix[0:3])\n",
    "        \n",
    "        testTruth = np.append(testTruth,nnTruth,axis=0)\n",
    "        testPred = np.append(testPred,nnPrediction,axis=0)\n",
    "        testEvTrue += evCor*cutSize\n",
    "        testEvFail += evFail*cutSize \n",
    "        \n",
    "        totalAcc += testBAcc*cutSize\n",
    "        totalUcAcc += ucAcc*cutSize\n",
    "        totalDataRate += dataRate*cutSize\n",
    "        start += batchSize\n",
    "        end += batchSize\n",
    "        \n",
    "    outputs = {\n",
    "        \"Accuracy\":totalAcc/dataSize,\n",
    "        \"TotalEvidenceTrue\":testEvTrue/dataSize,\n",
    "        \"TotalEvidenceFalse\":testEvFail/dataSize,\n",
    "        \"UncertaintyAccuracy\":totalUcAcc/dataSize,\n",
    "        \"DataRate\":totalDataRate/dataSize,\n",
    "        \"Truth\":testTruth,\n",
    "        \"Prediction\":testPred,\n",
    "        \"Top3\":top3\n",
    "    }\n",
    "        \n",
    "    return outputs\n",
    "    #return (totalAcc/dataSize,testTruth,testPred,testEvTrue/dataSize,testEvFail/dataSize,totalUcAcc/dataSize,totalDataRate/dataSize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainModel(nnModel, iterations, trainData, trainTarget, testData, testTarget, configs, accList):\n",
    "    batcher = DataLoader.DataHandler.batchIterator(trainData, trainTarget, configs[\"batchSize\"])\n",
    "    sample,_ = next(batcher)\n",
    "    \n",
    "    print(\"trainData shape : \", trainData.shape)\n",
    "    print(\"testData shape : \", testData.shape)\n",
    "    print(\"trainTarget shape : \", trainTarget.shape)\n",
    "    print(\"testTarget shape : \", testTarget.shape)\n",
    "    \n",
    "    htTestAcc=0\n",
    "    fold0TestAcc = 0\n",
    "    ucAcc = 0\n",
    "    dataRate = 0\n",
    "    \n",
    "    L_test_ev_s=[]\n",
    "    L_test_ev_f=[]\n",
    "    \n",
    "    print(\"\")\n",
    "    for i in range(iterations):\n",
    "        data, target = next(batcher)\n",
    "        dataClean = data\n",
    "\n",
    "        if(configs[\"PreEmbed\"]):\n",
    "            data = EmbedModel.vectorizeBatch(data)\n",
    "\n",
    "        tokens_length = getTokenLengths(data)\n",
    "        fd = {nnModel.nn_inputs:dataClean, nnModel.nn_vector_inputs:data,nnModel.nn_outputs:target,\n",
    "              nnModel.isTraining:True,nnModel.token_lengths:tokens_length,nnModel.annealing_step:0.00005*i}\n",
    "        _, acc, los = sess.run([nnModel.train_op,nnModel.accuracy,nnModel.loss],feed_dict=fd)\n",
    "\n",
    "        if(i%20==0):\n",
    "            title = (\"[Current iteration = \"+str(i)+\" Train Acc:\"+str(acc)+\" HT Test Acc:\"+str(htTestAcc)+\" fold0Test: (\"+str(fold0TestAcc)+') ucAcc :'+str(ucAcc)\n",
    "                +\" dataRatio  :\"+str(dataRate)+' ]')\n",
    "            title = str(title)       \n",
    "            print(title, end=\"\\r\")\n",
    "\n",
    "        if(i%50000==0 and i != 0):\n",
    "            oldTestAcc = fold0TestAcc               \n",
    "            testOutputs = evaluatePerformance(nnModel, sess, testData, testTarget, configs[\"batchSize\"], 0.1)  \n",
    "            \n",
    "            fold0TestAcc = testOutputs[\"Accuracy\"]\n",
    "            fEvTrue = testOutputs[\"TotalEvidenceTrue\"]\n",
    "            fEvFail = testOutputs[\"TotalEvidenceFalse\"]\n",
    "            ucAcc = testOutputs[\"UncertaintyAccuracy\"]\n",
    "            dataRate = testOutputs[\"DataRate\"]\n",
    "            fTruth = testOutputs[\"Truth\"]\n",
    "            fPrediction = testOutputs[\"Prediction\"]\n",
    "            \n",
    "            confidences = [0.995,0.98,0.90,0.70,0.5]\n",
    "            confidenceMatrix = np.zeros(shape=[len(confidences),3])\n",
    "            for idx in range(len(confidences)):\n",
    "                testOutputs = evaluatePerformance(nnModel, sess, testData, testTarget, configs[\"batchSize\"],1-confidences[idx])\n",
    "                confidenceMatrix[idx,0] = confidences[idx]\n",
    "                confidenceMatrix[idx,1] = testOutputs[\"DataRate\"]\n",
    "                confidenceMatrix[idx,2] = testOutputs[\"UncertaintyAccuracy\"]\n",
    "            \n",
    "            L_test_ev_s.append(fEvTrue)\n",
    "            L_test_ev_f.append(fEvFail)\n",
    "            \n",
    "            if(fold0TestAcc>oldTestAcc):\n",
    "                pass\n",
    "                #saveSession(sess)\n",
    "\n",
    "            accList.append([i, acc, htTestAcc, fold0TestAcc, los, ucAcc])\n",
    "            npAccList = np.array(accList)           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fullvectorsize:  300\n",
      "(?, 126, 1, 250)\n",
      "WARNING:tensorflow:From C:\\Users\\aaaaaa\\Jupyter Notebook\\Proje NLP Turk Telekom\\Models.py:180: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "INFO:tensorflow:Restoring parameters from NNModels/icliniq14k_GoogleNews_onelayer_pad128/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "should_load = True\n",
    "model_path = \"NNModels/icliniq14k_GoogleNews_onelayer_pad128/model.ckpt\"\n",
    "\n",
    "configs[\"maxLength\"] = 128 \n",
    "inputSize = configs[\"maxLength\"]\n",
    "configs[\"batchSize\"] = 8\n",
    "# ORIGINAL PART\n",
    "nnModel = Models.PyramidCNNVShort(inputSize=inputSize, vectorSize=vectorSize, outputSize=outputSize)\n",
    "\n",
    "# MY PART\n",
    "# nnModel = Models.myModel_CNN_TEXT(inputSize=inputSize, vectorSize=vectorSize, outputSize=outputSize)\n",
    "\n",
    "sess = tf.InteractiveSession(graph=nnModel.paperGraph)\n",
    "tf.global_variables_initializer().run()\n",
    "sess.run(tf.tables_initializer())\n",
    "\n",
    "if should_load:\n",
    "    tf.train.Saver().restore(sess, model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7432911392405064"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate\n",
    "confidence = 0.9\n",
    "results = evaluatePerformance(nnModel, sess, testData, testTarget, 1, 1-confidence)\n",
    "results[\"Accuracy\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understandin NN - LRP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We get layers from output to input so that we can backpropagate.\n",
    "\n",
    "Then we calculate word importances for each word in input.\n",
    "\n",
    "In the current model there is only one conv-pool layer so the layer_count is 1. But in the medspecsearch models have 3 layers, so this model is different. We will use this model for LRP purposes.\n",
    "\n",
    "( Maybe remove stop words? )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_count = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get weights, biases and activations to use in lrp method\n",
    "weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='.*kernel.*')\n",
    "biases = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='.*bias.*')\n",
    "\n",
    "activations = []\n",
    "if layer_count == 1:\n",
    "    activations = [nnModel.cnnInput, nnModel.conv1, nnModel.blockPool, nnModel.h_pool_flat, nnModel.fc1, nnModel.scores]\n",
    "    \n",
    "elif layer_count == 3:   \n",
    "    activations = [nnModel.cnnInput, nnModel.conv1, nnModel.blockPool, nnModel.conv2, nnModel.blockPool2, nnModel.conv3,\n",
    "             nnModel.blockPool3, nnModel.h_pool_flat, nnModel.fc1, nnModel.scores]\n",
    "\n",
    "weights.reverse()\n",
    "biases.reverse()\n",
    "activations.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We have three parallel conv-pool couple.\n",
    "# We need to split this ben backpropogating\n",
    "# I was experiencing lots of bugs so I splitted it like this\n",
    "# Need a better way for this\n",
    "if layer_count == 3:\n",
    "    biases_0 = np.array(biases)[[0,1,4]]\n",
    "    weights_0 = np.array(weights)[[0,1,4]]\n",
    "    activations_0 = np.array(activations)[[0,1,2,7,8,9]]\n",
    "\n",
    "    biases_1 = np.array(biases)[[0,1,3]]\n",
    "    weights_1 = np.array(weights)[[0,1,3]]\n",
    "    activations_1 = np.array(activations)[[0,1,2,5,6,9]]\n",
    "\n",
    "    biases_2 = np.array(biases)[[0,1,2]]\n",
    "    weights_2 = np.array(weights)[[0,1,2]]\n",
    "    activations_2 = np.array(activations)[[0,1,2,3,4,9]]\n",
    "\n",
    "    biases_splitted = [biases_0, biases_1, biases_2]\n",
    "    weights_splitted = [weights_0, weights_1, weights_2]\n",
    "    activations_splitted = [activations_0, activations_1, activations_2]\n",
    "    pool_biases = [[1,126,1,1], [1,125,1,1], [1,124,1,1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To test\n",
    "batch_x = trainData[0:21]\n",
    "batch_y = trainTarget[0:21]\n",
    "batch_x = EmbedModel.vectorizeBatch(batch_x)\n",
    "batch_y = sess.run(tf.one_hot(batch_y,outputSize)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 1\n",
    "backprop_layers = lrp.lrp_layers(alpha, layer_count, activations, weights, biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_importances, results_combined = lrp.get_word_relevances(alpha, backprop_layers, layer_count, batch_x[0:1], trainData[0], sess, nnModel, activations, weights, biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('hello', -0.00826842975688232),\n",
       " ('doctor', -0.10804215329674047),\n",
       " ('i', 0.020342812725022037),\n",
       " ('have', -0.30172094642857256),\n",
       " ('burning', 0.44691338759829063),\n",
       " ('sensation', -0.005695157685182687),\n",
       " ('while', -0.006010421393746959),\n",
       " ('urinating', -0.016021350662397415),\n",
       " ('and', 0.0),\n",
       " ('a', 0.0),\n",
       " ('frequent', 0.16106620664576704),\n",
       " ('urge', -0.012891234366653742),\n",
       " ('to', 0.0),\n",
       " ('urinate', -0.702877483046213),\n",
       " ('can', 0.04179421897966783),\n",
       " ('it', 0.09591377357002948),\n",
       " ('be', 0.24346296059100556),\n",
       " ('due', -0.12945817197134266),\n",
       " ('to', 0.0),\n",
       " ('sexual', -0.08989811715872369),\n",
       " ('contact', -0.27382462013709),\n",
       " ('i', 0.020263645015194495),\n",
       " ('am', -0.007761488160500732),\n",
       " ('a', 0.0),\n",
       " ('year', 0.0011631109154077452),\n",
       " ('old', -0.0023620034744476037),\n",
       " ('male', -0.07802475127381596),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0),\n",
       " ('[None]', 0.0)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_importances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_imps_all_classes(path, default=True):\n",
    "    dir_ = \"default//\"\n",
    "    if not default:\n",
    "        dir_ = \"stemmed//\"\n",
    "        \n",
    "    files = os.listdir(path + dir_ + \"//\")\n",
    "    word_imps_all_classes = []\n",
    "    for file in files:\n",
    "        f = open(path + dir_ + \"//\" + file)\n",
    "        tmp = []\n",
    "        for line in f:\n",
    "            tmp.append(line[0:-1].split(' '))\n",
    "        tmp = tmp[1:] # remove title\n",
    "        word_imps_all_classes.append(tmp)\n",
    "    \n",
    "    return word_imps_all_classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Asking keywords to user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_user(relevant_words, index, raw_user_input):\n",
    "    print(\"Is '\" + relevant_words[index][0] + \"' keyword related with your situation?\")\n",
    "    \n",
    "    is_relevant(relevant_words, index, raw_user_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_relevant(relevant_words, index, raw_user_input):\n",
    "    answer = input()\n",
    "    if answer == \"exit\": return\n",
    "    if answer == \"True\":\n",
    "        print(\"Please provide more explanation about '\" + relevant_words[index][0] + \"'\")\n",
    "        answer = input()\n",
    "        if answer == \"exit\": return\n",
    "        new_input = str(raw_user_input) + \" \" + str(answer)\n",
    "        new_input = process_user_input([new_input])\n",
    "        \n",
    "        new_results = evaluatePerformance(nnModel, sess, new_input, [0], 1, 1-confidence)\n",
    "        highest_confidence = new_results[\"Top3\"][0][0]\n",
    "        print(\"Results: \", new_results[\"Top3\"][0])\n",
    "        if highest_confidence[1] < desired_confidence:\n",
    "            ask_user(relevant_words, index + 1, new_input)\n",
    "        \n",
    "    else:\n",
    "        ask_user(relevant_words, index + 1, raw_user_input) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_user_input(user_input):\n",
    "    user_input = DataLoader.DataHandler.cleanTextData(user_input)\n",
    "    user_input = np.array(DataLoader.DataHandler.textIntoWordList(user_input, 128)[0])\n",
    "    \n",
    "    return user_input\n",
    "\n",
    "def get_relevant_words(confidence_top3, amount, tfidf_words):\n",
    "    relevant_words = []\n",
    "    for i in range(len(confidence_top3)):\n",
    "        category = confidence_top3[i][0]\n",
    "        \n",
    "        for words in tfidf_words[ClassDict[category]][0:amount]:      \n",
    "            relevant_words.append(words)\n",
    "    \n",
    "    return relevant_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets say confidence is low and we want to ask more about user's situation according to TF-IDF keywords\n",
    "raw_user_input is the original input of user\n",
    "\n",
    "user enters input until confidence reaches a high enough value\n",
    "(maybe there should be option for exiting early)\n",
    "\n",
    "In this example, confidence is 0.55 for most confident category, we want it to be higher than desired confidence value, so we keep asking about relevant words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_words = get_word_imps_all_classes(\"data//icliniq//iCliniq_14K//tfidf_results//\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n",
      "[[['Dermatology', 0.55359864], ['Otolaryngology', 0.046154037], ['Ophthalmology', 0.043179028]]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['acne', '0.335'],\n",
       " ['hair', '0.299'],\n",
       " ['cream', '0.268'],\n",
       " ['penis', '0.239'],\n",
       " ['itching', '0.226'],\n",
       " ['ear', '0.738'],\n",
       " ['tonsil', '0.203'],\n",
       " ['ent', '0.181'],\n",
       " ['hearing', '0.163'],\n",
       " ['tinnitus', '0.145'],\n",
       " ['eye', '0.842'],\n",
       " ['eyes', '0.298'],\n",
       " ['vision', '0.265'],\n",
       " ['lasik', '0.095'],\n",
       " ['drops', '0.084']]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# options:\n",
    "desired_confidence = 0.9\n",
    "\n",
    "raw_user_input = \"my hair is transparent\"\n",
    "user_input = process_user_input([raw_user_input])\n",
    "\n",
    "results = evaluatePerformance(nnModel, sess, user_input, [0], 1, 1-confidence)\n",
    "print(results[\"Accuracy\"])\n",
    "print(results[\"Top3\"])\n",
    "\n",
    "confidence_top3 = results[\"Top3\"][0]\n",
    "\n",
    "relevant_words = get_relevant_words(confidence_top3, 5, tfidf_words)\n",
    "relevant_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is 'ecg' keyword related with your situation?\n",
      "False\n",
      "Is 'dl' keyword related with your situation?\n",
      "no\n",
      "Is 'tsh' keyword related with your situation?\n",
      "no\n",
      "Is 'count' keyword related with your situation?\n",
      "no\n",
      "Is 'cholesterol' keyword related with your situation?\n",
      "no\n",
      "Is 'acne' keyword related with your situation?\n",
      "True\n",
      "Please provide more explanation about 'acne'\n",
      "i have acne\n",
      "Results:  [['Dermatology', 0.99923104], ['Otolaryngology', 7.039149e-05], ['Orthopaedic Surgery', 7.037149e-05]]\n"
     ]
    }
   ],
   "source": [
    "ask_user(relevant_words, 0, raw_user_input)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}