quantitative_hypotheses.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b9eae124",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "from scipy.ndimage import sobel, laplace\n",
    "from scipy.stats import wilcoxon\n",
    "import numpy as np\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def rescale_values(image,max_val,min_val):\n",
    "    '''\n",
    "    image - numpy array\n",
    "    max_val/min_val - float\n",
    "    '''\n",
    "    return (image-image.min())/(image.max()-image.min())*(max_val-min_val)+min_val\n",
    "\n",
    "SEED=1234\n",
    "np.random.seed(SEED)\n",
    "# torch.manual_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d70d55f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale matrix to sum to 1\n",
    "def sum_to_1(mat):\n",
    "    return mat / np.sum(mat)\n",
    "\n",
    "def rand_baseline(data: np.array):\n",
    "    rands = np.random.uniform(low=-1.0, high=1.0, size=(data.shape))\n",
    "    return sum_to_1(rands)\n",
    "\n",
    "def x_baseline(data: np.array):\n",
    "    rgb_weights = [0.2989, 0.5870, 0.1140]\n",
    "    greyscale = np.dot(data[...,:3], rgb_weights)\n",
    "    return sum_to_1(greyscale)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0e461a2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Confounder Data\n",
    "# with open('confounder_train128.pkl', 'rb') as f:\n",
    "#     confounder_train = pickle.load(f)\n",
    "#     confounder_train = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in confounder_train]\n",
    "\n",
    "# with open('confounder_val128.pkl', 'rb') as f:\n",
    "#     confounder_val = pickle.load(f)\n",
    "#     confounder_val = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in confounder_val]\n",
    "\n",
    "# with open('confounder_test128.pkl', 'rb') as f:\n",
    "#     confounder_test = pickle.load(f)\n",
    "#     confounder_test = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in confounder_test]\n",
    "\n",
    "    \n",
    "# # Suppressor Data\n",
    "# with open('suppressor_train128.pkl', 'rb') as f:\n",
    "#     supressor_train = pickle.load(f)\n",
    "#     supressor_train = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in supressor_train]\n",
    "\n",
    "# with open('suppressor_validation128.pkl', 'rb') as f:\n",
    "#     supressor_val = pickle.load(f)\n",
    "#     supressor_val = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in supressor_val]\n",
    "\n",
    "# with open('suppressor_test128.pkl', 'rb') as f:\n",
    "#     supressor_test = pickle.load(f)\n",
    "#     supressor_test = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in supressor_test]\n",
    "\n",
    "\n",
    "# # No Watermark Data\n",
    "# with open('no_mark_train128.pkl', 'rb') as f:\n",
    "#     no_mark_train = pickle.load(f)\n",
    "#     no_mark_train = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in no_mark_train]\n",
    "\n",
    "# with open('no_mark_validation128.pkl', 'rb') as f:\n",
    "#     no_mark_val = pickle.load(f)\n",
    "#     no_mark_val = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in no_mark_val]\n",
    "\n",
    "# with open('no_mark_test128.pkl', 'rb') as f:\n",
    "#     no_mark_test = pickle.load(f)\n",
    "#     no_mark_test = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in no_mark_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67c43d54",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('mark_all128.pkl', 'rb') as f:\n",
    "    watermark_dataset = pickle.load(f)\n",
    "    watermark_dataset = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in watermark_dataset]\n",
    "\n",
    "with open('no_mark_test128.pkl', 'rb') as f:\n",
    "    no_watermark_dataset = pickle.load(f)\n",
    "    no_watermark_dataset = [[rescale_values(i[0],1,0).transpose(2,0,1),i[1]] for i in no_watermark_dataset]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a09fb432",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_energy=[]\n",
    "methods = ['deconv', 'int_grads', 'shap', 'lrp', 'lrp_ab', 'laplace', 'sobel', 'x'] # lime just nan values\n",
    "# methods = ['deconv', 'int_grads', 'shap', 'lrp']\n",
    "n_test = 3000\n",
    "\n",
    "energies = {}\n",
    "\n",
    "for method in methods:\n",
    "    comb_water_conf = []\n",
    "    comb_water_sup = []\n",
    "    comb_water_no = []\n",
    "    \n",
    "    comb_no_water_conf = []\n",
    "    comb_no_water_sup = []\n",
    "    comb_no_water_no = []\n",
    "\n",
    "    # if method == 'lrp_ab':\n",
    "    #     file_suffix = 'lrp_'\n",
    "    # else:\n",
    "    file_suffix=''\n",
    "\n",
    "    energies[method] = {}\n",
    "    \n",
    "    for split in range(5):\n",
    "        # for model_ind in range(10):\n",
    "        file = open(f'./energies/energy_water_conf_pred_{file_suffix}{split}.pickle', 'rb')\n",
    "        energy_water_conf = pickle.load(file)\n",
    "\n",
    "        file = open(f'./energies/energy_water_sup_pred_{file_suffix}{split}.pickle', 'rb')\n",
    "        energy_water_sup = pickle.load(file)\n",
    "\n",
    "        file = open(f'./energies/energy_water_no_pred_{file_suffix}{split}.pickle', 'rb')\n",
    "        energy_water_no = pickle.load(file)\n",
    "\n",
    "        file = open(f'./energies/energy_no_water_conf_pred_{file_suffix}{split}.pickle', 'rb')\n",
    "        energy_no_water_conf = pickle.load(file)\n",
    "\n",
    "        file = open(f'./energies/energy_no_water_sup_pred_{file_suffix}{split}.pickle', 'rb')\n",
    "        energy_no_water_sup = pickle.load(file)\n",
    "\n",
    "        file = open(f'./energies/energy_no_water_no_pred_{file_suffix}{split}.pickle', 'rb')\n",
    "        energy_no_water_no = pickle.load(file)\n",
    "        \n",
    "        comb_water_conf.extend(energy_water_conf[method][:n_test])\n",
    "        comb_water_sup.extend(energy_water_sup[method][:n_test])\n",
    "        # comb_water_no.extend(energy_water_no[method][:n_test])\n",
    "\n",
    "        if method == 'laplace' or method == 'sobel' or method == 'x':\n",
    "            comb_water_no.extend(energy_water_sup[method][:n_test])\n",
    "            comb_no_water_conf.extend(energy_no_water_sup[method][:n_test])\n",
    "        else:\n",
    "            comb_water_no.extend(energy_water_no[method][:n_test])\n",
    "            comb_no_water_conf.extend(energy_no_water_conf[method][:n_test])\n",
    "\n",
    "        \n",
    "        # comb_no_water_conf.extend(energy_no_water_conf[method][:n_test])\n",
    "        comb_no_water_sup.extend(energy_no_water_sup[method][:n_test])\n",
    "        comb_no_water_no.extend(energy_no_water_no[method][:n_test])\n",
    "\n",
    "    energies[method]['conf'] = [comb_no_water_conf, comb_water_conf]\n",
    "    energies[method]['sup'] = [comb_no_water_sup, comb_water_sup]\n",
    "    energies[method]['no'] = [comb_no_water_no, comb_water_no]\n",
    "\n",
    "    train_energy.append([[comb_no_water_conf, comb_water_conf],\n",
    "                [comb_no_water_sup, comb_water_sup],\n",
    "                [comb_no_water_no, comb_water_no]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "46265812-5209-4cc0-8205-dad62b494649",
   "metadata": {},
   "outputs": [],
   "source": [
    "# energy_lapl_no = []\n",
    "# energy_lapl_wm = []\n",
    "\n",
    "# energy_sob_no = []\n",
    "# energy_sob_wm = []\n",
    "\n",
    "# energy_x_no = []\n",
    "# energy_x_wm = []\n",
    "\n",
    "# for split in range(10):\n",
    "#     with open(f'./energies/energy_baselines_{split}.pickle', 'rb') as f:\n",
    "#         [[lapl_no, lapl_wm], [sob_no, sob_wm], [x_no, x_wm]] = pickle.load(f)   \n",
    "\n",
    "#     energy_lapl_no.extend(lapl_no)\n",
    "#     energy_lapl_wm.extend(lapl_wm)\n",
    "    \n",
    "#     energy_sob_no.extend(sob_no)\n",
    "#     energy_sob_wm.extend(sob_wm)\n",
    "    \n",
    "#     energy_x_no.extend(x_no)\n",
    "#     energy_x_wm.extend(x_wm)\n",
    "\n",
    "\n",
    "# baselines = {\n",
    "#     'lapl': [energy_lapl_no, energy_lapl_wm],\n",
    "#     'sob': [energy_sob_no, energy_sob_wm],\n",
    "#     'x': [energy_x_no, energy_x_wm]\n",
    "# }\n",
    "\n",
    "\n",
    "# train_energy.append([ [energy_lapl_no, energy_lapl_wm], [energy_lapl_no, energy_lapl_wm], [energy_lapl_no, energy_lapl_wm] ])\n",
    "# train_energy.append([ [energy_sob_no, energy_sob_wm], [energy_sob_no, energy_sob_wm], [energy_sob_no, energy_sob_wm] ])\n",
    "# train_energy.append([ [energy_x_no, energy_x_wm], [energy_x_no, energy_x_wm], [energy_x_no, energy_x_wm] ])\n",
    "\n",
    "\n",
    "# for baseline, results in baselines.items():\n",
    "#     energies[baseline] = {}\n",
    "#     for dataset in ['conf', 'sup', 'no']:\n",
    "#         energies[baseline][dataset] = results\n",
    "# # energies['lapl']['conf'] = [comb_no_water_conf, comb_water_conf]\n",
    "# # energies[method]['sup'] = [comb_no_water_sup, comb_water_sup]\n",
    "# # energies[method]['no'] = [comb_no_water_no, comb_water_no]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e50d374",
   "metadata": {},
   "source": [
    "## Hypothesis 1\n",
    "- if the XAI methods work as expected as confounder detectors, the relative energy should differ betwen W and NO-W test images only for the confounder case, but not for the other two.\n",
    "- Using a [paired Wilcoxon signed-rank test](https://en.wikipedia.org/wiki/Wilcoxon_signed-rank_test) to test significance\n",
    "- Evaluate differences between E(W_i) and E(NoW_i) for all three models for each given XAI method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "788e4a1b-cc19-48d3-bb02-6d3251efdf7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_e(n):\n",
    "    a = '%E' % n\n",
    "    return a.split('E')[0].rstrip('0').rstrip('.') + 'E' + a.split('E')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a736fb97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deconv conf 1.018143E+08 0.0\n",
      "deconv sup 8.200056E+07 0.0\n",
      "deconv no 1.060115E+08 0.0\n",
      "int_grads conf 1.073019E+08 0.0\n",
      "int_grads sup 1.074785E+08 0.0\n",
      "int_grads no 1.073915E+08 0.0\n",
      "shap conf 1.073156E+08 0.0\n",
      "shap sup 1.070529E+08 0.0\n",
      "shap no 1.074322E+08 0.0\n",
      "lrp conf 1.07463E+08 0.0\n",
      "lrp sup 1.074079E+08 0.0\n",
      "lrp no 1.076062E+08 0.0\n",
      "lrp_ab conf 1.067605E+08 0.0\n",
      "lrp_ab sup 1.060277E+08 0.0\n",
      "lrp_ab no 1.063642E+08 0.0\n",
      "laplace conf 1.122766E+08 0.0\n",
      "laplace sup 1.122766E+08 0.0\n",
      "laplace no 1.122766E+08 0.0\n",
      "sobel conf 1.121219E+08 0.0\n",
      "sobel sup 1.121219E+08 0.0\n",
      "sobel no 1.121219E+08 0.0\n",
      "x conf 5.526684E+07 0.0\n",
      "x sup 5.526684E+07 0.0\n",
      "x no 5.526684E+07 0.0\n"
     ]
    }
   ],
   "source": [
    "for method, results in energies.items():\n",
    "    for model, result in results.items():\n",
    "        res = wilcoxon(result[1], result[0], alternative='greater')\n",
    "        print(method, model, format_e(res.statistic), res.pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d7421828",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deconv conf 0.905076089517079 0.0\n",
      "deconv sup 0.7289424628308554 0.0\n",
      "deconv no 0.9423874924994999 0.0\n",
      "int_grads conf 0.9538578838589239 0.0\n",
      "int_grads sup 0.9554282196590884 0.0\n",
      "int_grads no 0.9546550525590595 0.0\n",
      "shap conf 0.953979900882281 0.0\n",
      "shap sup 0.9516443762917528 0.0\n",
      "shap no 0.9550169566860013 0.0\n",
      "lrp conf 0.9552901571215859 0.0\n",
      "lrp sup 0.9548009689534858 0.0\n",
      "lrp no 0.9565631042069471 0.0\n",
      "lrp_ab conf 0.9490457674956109 0.0\n",
      "lrp_ab sup 0.9425313243105096 0.0\n",
      "lrp_ab no 0.9455223859368402 0.0\n",
      "laplace conf 0.998081116518879 0.0\n",
      "laplace sup 0.998081116518879 0.0\n",
      "laplace no 0.998081116518879 0.0\n",
      "sobel conf 0.996705149232171 0.0\n",
      "sobel sup 0.996705149232171 0.0\n",
      "sobel no 0.996705149232171 0.0\n",
      "x conf 0.4912935617930084 0.0\n",
      "x sup 0.4912935617930084 0.0\n",
      "x no 0.4912935617930084 0.0\n"
     ]
    }
   ],
   "source": [
    "# Effect size as rank-biserial correlation\n",
    "# aka (test statistic / total rank sum)\n",
    "for method, results in energies.items():\n",
    "    for model, result in results.items():\n",
    "        res = wilcoxon(result[1], result[0], alternative='greater')\n",
    "        rank_sum = sum(np.where(np.array(result[1]) ==  np.array(result[1]))[0])\n",
    "        print(method, model, res.statistic/rank_sum, res.pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ddeb3b72-effa-4611-9aa0-755f3c120d7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WilcoxonResult(statistic=101814272.0, pvalue=0.0)\n"
     ]
    }
   ],
   "source": [
    "test_result = energies['deconv']['conf']\n",
    "\n",
    "res = wilcoxon(test_result[1], test_result[0], alternative='greater')\n",
    "print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa35cd88-4e08-4170-8c50-758e1fe831ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "22cf4c7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle as pkl\n",
    "\n",
    "# with open('energy_laplace.pickle', 'rb') as f:\n",
    "#     lapl = pkl.load(f)\n",
    "    \n",
    "# with open('energy_sobel.pickle', 'rb') as f:\n",
    "#     sob= pkl.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d5116b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lapl_no, lapl_wm = lapl\n",
    "# sob_no, sob_wm = sob"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3c7588d",
   "metadata": {},
   "source": [
    "## Hypothesis 2\n",
    "if the XAI methods work as expected as confounder detectors, then the relative energy on W test images should be higher for confounded training data then for the other two. If this is the case - how big is the effect?\n",
    "- Meaning, how much more watermark energy is attributed by models trained on confounded data compared to models trained on unconfounded data.\n",
    "- Calculate E(Wˆtr=Conf_i) - E(Wˆtr=Supp_i) and E(Wˆtr=Conf_i) - E(Wˆtr=NoW_i) for all XAI methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2fceef25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deconv CONF - SUPP 0.8976305709269506 0.0\n",
      "deconv CONF - NO 0.13096247305375913 1.0\n",
      "deconv NO - CONF 0.8691708691690557 0.0\n",
      "int_grads CONF - SUPP 0.8824618530124231 0.0\n",
      "int_grads CONF - NO 0.02623913594239616 1.0\n",
      "int_grads NO - CONF 0.9738942062804187 0.0\n",
      "shap CONF - SUPP 0.8594591639442629 0.0\n",
      "shap CONF - NO 0.043270715825499476 1.0\n",
      "shap NO - CONF 0.9568626263973153 0.0\n",
      "lrp CONF - SUPP 0.8403469742427273 0.0\n",
      "lrp CONF - NO 0.1529426761784119 1.0\n",
      "lrp NO - CONF 0.847190666044403 0.0\n",
      "lrp_ab CONF - SUPP 0.8191147943196213 0.0\n",
      "lrp_ab CONF - NO 0.4580823254883659 1.0\n",
      "lrp_ab NO - CONF 0.542051016734449 2.664839004392391e-19\n",
      "laplace CONF - SUPP\n",
      "laplace CONF - NO\n",
      "sobel CONF - SUPP\n",
      "sobel CONF - NO\n",
      "x CONF - SUPP\n",
      "x CONF - NO\n"
     ]
    }
   ],
   "source": [
    "for method, results in energies.items():\n",
    "    #  E(Wˆtr=Conf_i) - E(Wˆtr=Supp_i) \n",
    "    try:\n",
    "        res_sup = wilcoxon(results['conf'][1], results['sup'][1], alternative='greater')\n",
    "        \n",
    "        #  E(Wˆtr=Conf_i) - E(Wˆtr=NoW_i) \n",
    "        res_no = wilcoxon(results['conf'][1], results['no'][1], alternative='greater')\n",
    "\n",
    "        res_no_2 = wilcoxon(results['no'][1], results['conf'][1], alternative='greater')\n",
    "        \n",
    "        rank_sum = sum(np.where(np.array(results['conf'][1]) ==  np.array(results['conf'][1]))[0])\n",
    "\n",
    "        print(method, 'CONF - SUPP', res_sup.statistic/rank_sum, res_sup.pvalue)\n",
    "        print(method, 'CONF - NO', res_no.statistic/rank_sum, res_no.pvalue)\n",
    "        print(method, 'NO - CONF', res_no_2.statistic/rank_sum, res_no_2.pvalue)\n",
    "    except:\n",
    "        print(method, 'CONF - SUPP')\n",
    "        print(method, 'CONF - NO')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22b44d45",
   "metadata": {},
   "source": [
    "## Coloured MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "257b0422",
   "metadata": {},
   "outputs": [],
   "source": [
    "# method = 'shap'\n",
    "\n",
    "# conf_conf = [[],[],[]]\n",
    "# conf_sup = [[],[],[]]\n",
    "# conf_no = [[],[],[]]\n",
    "\n",
    "# sup_conf = [[],[],[]]\n",
    "# sup_sup = [[],[],[]]\n",
    "# sup_no = [[],[],[]]\n",
    "\n",
    "# no_conf = [[],[],[]]\n",
    "# no_sup = [[],[],[]]\n",
    "# no_no = [[],[],[]]\n",
    "\n",
    "# for model_ind in range(5):\n",
    "#     file = open(f'energies_mnist_{model_ind}.pickle', 'rb')\n",
    "#     energies = pickle.load(file)\n",
    "    \n",
    "#     for i in range(3):\n",
    "#         conf_conf[i].extend(energies['conf']['conf'][i][method])\n",
    "#         conf_sup[i].extend(energies['conf']['sup'][i][method])\n",
    "#         conf_no[i].extend(energies['conf']['no_col'][i][method])\n",
    "        \n",
    "#         sup_conf[i].extend(energies['sup']['conf'][i][method])\n",
    "#         sup_sup[i].extend(energies['sup']['sup'][i][method])\n",
    "#         sup_no[i].extend(energies['sup']['no_col'][i][method])\n",
    "\n",
    "#         no_conf[i].extend(energies['no_col']['conf'][i][method])\n",
    "#         no_sup[i].extend(energies['no_col']['sup'][i][method])\n",
    "#         no_no[i].extend(energies['no_col']['no_col'][i][method])\n",
    "        \n",
    "        \n",
    "# combined_energies = {\n",
    "#     'conf': { # conf model on conf/sup/no data\n",
    "#         'conf': conf_conf,\n",
    "#         'sup': conf_sup, \n",
    "#         'no_col': conf_no},\n",
    "            \n",
    "#     'sup': { # sup model on conf/sup/no data\n",
    "#         'conf': sup_conf,\n",
    "#         'sup': sup_sup, \n",
    "#         'no_col': sup_no},\n",
    "\n",
    "#     'no_col': { # no_col model on conf/sup/no data\n",
    "#         'conf': no_conf,\n",
    "#         'sup': no_sup, \n",
    "#         'no_col': no_no},\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b37d30ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# wilcoxon(x=energies['lrp']['conf'][1], y=energies['lrp']['conf'][0], alternative='greater')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "debugging-conda",
   "language": "python",
   "name": "debugging-conda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}