synthetic_algorithms_comparison/step3_synthetic_algorithms_comparision.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b27d1e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import glob\n",
    "import os\n",
    "import re\n",
    "import math\n",
    "import pickle\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b774c91",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_metrics(path, method_list, folder_name):\n",
    "    print(\"Utility Metrics Folder Path: \"+ path)\n",
    "\n",
    "    single_metrics_df = pd.DataFrame()\n",
    "    multiple_metrics_df = pd.DataFrame()\n",
    "    \n",
    "    for method in method_list:\n",
    "        metrics_path = path+method+'/'\n",
    "        print('-------------')\n",
    "        print(method)\n",
    "        \n",
    "        for i in range(1000):\n",
    "            dataset_path = metrics_path+ '/' + folder_name + '_' + str(i)  + '-utility_metrics.csv'\n",
    "            data_name = dataset_path.split('/')[-1].split('-utility_metrics')[0] \n",
    "            df = pd.read_csv(dataset_path)\n",
    "            df = df.drop(columns= ['Unnamed: 0'])\n",
    "            df = df.replace({'Value': {\"{'-': \": '', '}': ''}}, regex=True) #remove special charactors in value of attribute_disclosure \n",
    "            df['Value'] = df['Value'].astype(float) \n",
    "\n",
    "            avg = df[(df['Type'] =='Multiple')]\n",
    "            avg = avg.groupby('Metric')['Value'].mean().reset_index()\n",
    "            avg['Method'] = method\n",
    "            avg['Variable'] = '-'\n",
    "            avg['Run'] = 1\n",
    "            avg['Type'] = 'Single'\n",
    "            cols = ['Method', 'Metric', 'Variable', 'Run', 'Type', 'Value']\n",
    "            avg = avg[cols]\n",
    "            df = pd.concat([df, avg], ignore_index=True)\n",
    "            df['data_name'] = data_name\n",
    "            df.loc[df['Metric'] == 'cluster_measure', 'Value'] = np.log(df['Value'])\n",
    "\n",
    "            single = df[(df['Method'] ==method) & (df['Run'] ==1) & (df['Type'] =='Single')]\n",
    "            single_metrics_df = pd.concat([single_metrics_df, single], ignore_index=True)\n",
    "            multiple = df[(df['Method'] ==method) & (df['Run'] ==1) & (df['Type'] =='Multiple')]\n",
    "            multiple_metrics_df = pd.concat([multiple_metrics_df, multiple], ignore_index=True)\n",
    "        print(\"done\")\n",
    "            \n",
    "    \n",
    "    return single_metrics_df, multiple_metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f5d0e8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# CIs for single type metrics\n",
    "def stats_summary(single_metrics):\n",
    "    stats = single_metrics.groupby(['Method','Metric'])['Value'].agg(['mean', 'count', 'std'])\n",
    "\n",
    "    ci95_hi = []\n",
    "    ci95_lo = []\n",
    "\n",
    "    for i in stats.index:\n",
    "        m, c, s = stats.loc[i]\n",
    "        ci95_hi.append(m + 1.96*s) #https://moderndive.com/8-confidence-intervals.html#se-method\n",
    "        ci95_lo.append(m - 1.96*s)\n",
    "\n",
    "    stats['ci95_hi'] = ci95_hi\n",
    "    stats['ci95_lo'] = ci95_lo\n",
    "    \n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ba9cb5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stats_plot(stats):\n",
    "    plt.figure(figsize=(12,5))\n",
    "    plt.style.use('default')  \n",
    "\n",
    "    data = stats.loc['CLGP'].sort_index(ascending=False)\n",
    "    for ci95_lo,ci95_hi,mean,y in zip(data['ci95_lo'],data['ci95_hi'], data['mean'], range(len(data))):\n",
    "        plt.plot((ci95_lo,ci95_hi,mean),(y,y,y),'ro-',color='blue',label='CLGP'if y == 0 else \"\")\n",
    "\n",
    "    data = stats.loc['MC-MedGAN'].sort_index(ascending=False)\n",
    "    for ci95_lo,ci95_hi,mean,y in zip(data['ci95_lo'],data['ci95_hi'], data['mean'], range(len(data))):\n",
    "        plt.plot((ci95_lo,ci95_hi,mean),(y,y,y),'ro-',color='green',label='MC-MedGAN'if y == 0 else \"\")\n",
    "\n",
    "    data = stats.loc['MPoM'].sort_index(ascending=False)\n",
    "    for ci95_lo,ci95_hi,mean,y in zip(data['ci95_lo'],data['ci95_hi'], data['mean'], range(len(data))):\n",
    "        plt.plot((ci95_lo,ci95_hi,mean),(y,y,y),'ro-',color='red',label='MPoM'if y == 0 else \"\")    \n",
    "\n",
    "    plt.vlines(x=[1], ymin=0, ymax=len(data), colors='grey', ls='--', lw=2)\n",
    "\n",
    "    yvalues = ['Comb-CrCl (1)','Log-Cluster ↓','Supp. Coverage ↑','CrCl-RS (1)','PCD ↓']\n",
    "    plt.xticks(np.arange(-6, 4, 1.0))\n",
    "    plt.yticks(range(len(data)),yvalues[::-1])\n",
    "    plt.legend(title='Method', loc='upper left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1625dce",
   "metadata": {},
   "outputs": [],
   "source": [
    "def boxplots_single(single_metrics):\n",
    "    plt.figure(figsize=(16,10))\n",
    "    plt.style.use('default')  \n",
    "\n",
    "    ax = sns.boxplot(x = 'Metric',y = 'Value',hue ='Method', notch=True,\n",
    "                     data = single_metrics[single_metrics['Metric'].isin(['pairwise_correlation_difference', 'cross_classification', 'cca_accuracy', 'coverage'])]\n",
    "    )\n",
    "    ax.set(xlabel='Metrix', ylabel='Value')\n",
    "\n",
    "    plt.savefig(path+'boxplot_singletype_metrics.png')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1cdd7ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# box plot for multiple type metrics\n",
    "def boxplots_metrics(multiple_metrics):\n",
    "    plt.style.use('seaborn') #seaborn\n",
    "\n",
    "    figure, axis = plt.subplots(2, 2, figsize=(16,10))\n",
    "\n",
    "\n",
    "    axa = sns.boxplot(ax = axis[0, 0],\n",
    "                x = 'Variable',\n",
    "                y = 'Value',\n",
    "                hue = 'Method',\n",
    "                notch=True,    \n",
    "                data = multiple_metrics[(multiple_metrics['Metric'] =='cross_classification')]\n",
    "               )\n",
    "    axa.set(xlabel='', \n",
    "            xticklabels = '',\n",
    "            ylabel='CrCl-RS'\n",
    "            )\n",
    "\n",
    "    axb = sns.boxplot(ax = axis[0, 1],\n",
    "                x = 'Variable',\n",
    "                y = 'Value',\n",
    "                hue = 'Method',\n",
    "                notch=True,      \n",
    "                data = multiple_metrics[(multiple_metrics['Metric'] =='cca_accuracy')]\n",
    "               )\n",
    "    axb.set(xlabel='', \n",
    "            xticklabels = '',\n",
    "            ylabel='Comb-CrCl'\n",
    "            )\n",
    "\n",
    "    axc = sns.boxplot(ax = axis[1, 0],\n",
    "                x = 'Variable',\n",
    "                y = 'Value',\n",
    "                hue = 'Method',\n",
    "                notch=True, \n",
    "                data = multiple_metrics[(multiple_metrics['Metric'] =='kl_divergence')]\n",
    "               )\n",
    "    axc.set(xlabel='', \n",
    "            ylabel='KL'\n",
    "            )\n",
    "    axc.tick_params(labelrotation=90)\n",
    "\n",
    "    axd = sns.boxplot(ax = axis[1, 1],\n",
    "                x = 'Variable',\n",
    "                y = 'Value',\n",
    "                hue = 'Method',\n",
    "                notch=True,        \n",
    "                data = multiple_metrics[(multiple_metrics['Metric'] =='coverage')]\n",
    "               )\n",
    "    axd.set(xlabel='', \n",
    "            ylabel='Supp.Coverage'\n",
    "            )\n",
    "    axd.tick_params(labelrotation=90)\n",
    "    #plt.xticks(rotation=45)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(path+'boxplot_multipletype_metrics.png')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be507810",
   "metadata": {},
   "source": [
    "## Breast survival 2010 -2015 - step1_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cf13090",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Put all utility-metric results in same folder based on different synthetic method\n",
    "#Set appropriate folder_name, synthetic method and path\n",
    "folder_name = 'breast_survival_1000samples'\n",
    "method_list = ['CLGP','MC-MedGAN','MPoM']\n",
    "path = '/Users/Results/'\n",
    "\n",
    "single_metrics, multiple_metrics = read_metrics(path, method_list, folder_name)\n",
    "single_metrics = single_metrics[single_metrics['Metric'].isin(['pairwise_correlation_difference', 'cluster_measure', 'cross_classification', 'cca_accuracy', 'coverage'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9cdb4db",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = stats_summary(single_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8415e04f",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_plot(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c915407",
   "metadata": {},
   "outputs": [],
   "source": [
    "boxplots_metrics(multiple_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c168589",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}