Stat_test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Stat_test.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "j25mauLggnzG",
        "PSxUz9CGgcxw",
        "KVqiCJ4XrvrK",
        "z_6IB_Fowiep",
        "OVzmkl-DMEkH",
        "qotz46ZTzjSW",
        "MGZe0mBbLDPq"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "hO8F201M8hvL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3888c9f2-e009-4058-afea-f1f83daefc3a"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6RD9_2bqmb0v",
        "outputId": "334c7ace-00c2-4e34-d527-b428c1fca223"
      },
      "source": [
        "!pip install statannot"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting statannot\n",
            "  Downloading statannot-0.2.3-py3-none-any.whl (10 kB)\n",
            "Installing collected packages: statannot\n",
            "Successfully installed statannot-0.2.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tk8M7Ptm9YX5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "acb26e35-f7ff-4808-c8c5-c8f3955d32fb"
      },
      "source": [
        "import scipy.io as sio\n",
        "import numpy as np\n",
        "from scipy import stats\n",
        "import pandas as pd\n",
        "from statsmodels.stats.anova import anova_lm\n",
        "from statsmodels.formula.api import ols\n",
        "from statsmodels.stats.multicomp import MultiComparison\n",
        "from scipy.stats import ttest_ind\n",
        "import matplotlib.pyplot as plt\n",
        "from statannot import add_stat_annotation\n",
        "import seaborn as sns\n",
        "from google.colab import files as FILE"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n8IDhEkpK_Jh"
      },
      "source": [
        "# ML"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BQrfC5E_LVkF"
      },
      "source": [
        "## PD vs NC"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oj_HNCK2Lzkl"
      },
      "source": [
        "### Full data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l2wIlET7Ljfi"
      },
      "source": [
        "Firstly, we would like to see whether there is any significant differences in the means of PD vs NC classification for spectral, CSP and IIR features. We can use one-way ANOVA."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ceEsd-LJyIrg",
        "outputId": "b083148f-e90e-4204-f7b9-ff423f35d95b"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "ml_pdnc_full_spec_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_Full.mat', 'wtd_f1_list')\n",
        "ml_pdnc_full_csp_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_PDNC_CSP_Full.mat', 'wtd_f1_list')\n",
        "ml_pdnc_full_iir_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/IIR/ML_PDNC_IIR_Full.mat', 'wtd_f1_list')\n",
        "\n",
        "print(ml_pdnc_full_spec_f1)\n",
        "print(ml_pdnc_full_csp_f1)\n",
        "print(ml_pdnc_full_iir_f1)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.96959464 0.96959397 0.97432423 0.96891881 0.96351351 0.95878373\n",
            "  0.97432235 0.97162007 0.96351151 0.96013253]]\n",
            "[[0.88313734 0.87156936 0.87554928 0.8857453  0.88301818 0.871489\n",
            "  0.8802999  0.86806252 0.87072389 0.87759891]]\n",
            "[[0.65597438 0.65824913 0.65246971 0.65052087 0.63918777 0.66208408\n",
            "  0.67044856 0.65688522 0.65480616 0.65960265]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A3XMge_eXb8I",
        "outputId": "fedaa007-fe43-4b85-b614-5f68bc8a5468"
      },
      "source": [
        "# One-way ANOVA between PD vs NC full data\n",
        "from scipy.stats import f_oneway\n",
        "\n",
        "stat, p = f_oneway(ml_pdnc_full_spec_f1[0], ml_pdnc_full_csp_f1[0], ml_pdnc_full_iir_f1[0])\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "\tprint('Probably the same distribution')\n",
        "else:\n",
        "\tprint('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=5695.267, p=0.000\n",
            "Probably different distributions\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sw6TSeFVX7Y6"
      },
      "source": [
        "One-way ANOVA test says that the samples are significantly different. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yBrvYtvHkDJp",
        "outputId": "fbacee01-6cba-448c-d0be-59f5bd2f3c42"
      },
      "source": [
        "# Create a dataframe of f-scores and its corresponding features for full data\n",
        "\n",
        "ml_pdnc_full_spec_df = pd.DataFrame({'wtd_F_score': ml_pdnc_full_spec_f1[0],'Features': len(ml_pdnc_full_spec_f1[0])*['spectral']})\n",
        "ml_pdnc_full_csp_df = pd.DataFrame({'wtd_F_score': ml_pdnc_full_csp_f1[0],'Features': len(ml_pdnc_full_csp_f1[0])*['csp']})\n",
        "ml_pdnc_full_iir_df = pd.DataFrame({'wtd_F_score': ml_pdnc_full_iir_f1[0],'Features': len(ml_pdnc_full_iir_f1[0])*['iir']})\n",
        "\n",
        "ml_pdnc_full_df = pd.concat([ml_pdnc_full_spec_df, ml_pdnc_full_csp_df, ml_pdnc_full_iir_df], ignore_index=True)\n",
        "\n",
        "#print(pdnc_full_df)\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_pdnc_full_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_pdnc_full_model = ols(ml_pdnc_full_formula, ml_pdnc_full_df).fit()\n",
        "ml_pdnc_aov_table = anova_lm(ml_pdnc_full_model, typ=2)\n",
        "print(ml_pdnc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.513037   2.0  5695.267193  3.517097e-36\n",
            "Residual     0.001216  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "piNmmS2lrXTs",
        "outputId": "6fd2d7ea-0737-43d0-c0b7-322f29515bcc"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "ml_pdnc_mc = MultiComparison(ml_pdnc_full_df['wtd_F_score'], ml_pdnc_full_df['Features'])\n",
        "print(ml_pdnc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   csp      iir  -0.2207 0.001 -0.2281 -0.2133   True\n",
            "   csp spectral   0.0907 0.001  0.0833  0.0982   True\n",
            "   iir spectral   0.3114 0.001   0.304  0.3188   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vDPRcA4Ussis"
      },
      "source": [
        "The above table shows that there is statistical significance between pairwise features for PD vs NC Full data. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qQWQaOBkvF3N"
      },
      "source": [
        "### Individual Emotions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e86slXgTwlnB",
        "outputId": "e62b5795-c4e2-49b4-82e0-1f723f411e5e"
      },
      "source": [
        "# Create a DataFrame\n",
        "ml_pdnc_emot_df = pd.DataFrame()\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "for i in features:\n",
        "  for j in emotions:\n",
        "    if i == \"Spectral\":\n",
        "      ml_pdnc_emot_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(i)+'/ML_PDNC_'+str(i).lower()+'_'+str(j)+'.mat', 'wtd_f1_list')\n",
        "    else:\n",
        "      ml_pdnc_emot_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(i)+'/ML_PDNC_'+str(i)+'_'+str(j)+'.mat', 'wtd_f1_list')\n",
        "    data = pd.DataFrame({'wtd_F_score': ml_pdnc_emot_f1[0],'Features': len(ml_pdnc_emot_f1[0])*[str(i)], 'Emotions': len(ml_pdnc_emot_f1[0])*[str(j)]})\n",
        "    ml_pdnc_emot_df = ml_pdnc_emot_df.append(data, ignore_index=True)\n",
        "     \n",
        "print(ml_pdnc_emot_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "     wtd_F_score  Features Emotions\n",
            "0       0.950197  Spectral       E1\n",
            "1       0.975095  Spectral       E1\n",
            "2       0.975103  Spectral       E1\n",
            "3       0.983402  Spectral       E1\n",
            "4       0.975104  Spectral       E1\n",
            "..           ...       ...      ...\n",
            "175     0.535852       IIR       E6\n",
            "176     0.561481       IIR       E6\n",
            "177     0.630036       IIR       E6\n",
            "178     0.571848       IIR       E6\n",
            "179     0.543925       IIR       E6\n",
            "\n",
            "[180 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "klcvDxEl6EUT",
        "outputId": "6a9d0514-f671-4651-c9ae-de5b715eb75b"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api\n",
        "\n",
        "ml_pdnc_emot_formula = 'wtd_F_score ~ C(Features)+C(Emotions)+C(Features):C(Emotions)'\n",
        "ml_pdnc_emot_model = ols(ml_pdnc_emot_formula, ml_pdnc_emot_df).fit()\n",
        "ml_pdnc_emot_aov_table = anova_lm(ml_pdnc_emot_model, typ=2)\n",
        "print(ml_pdnc_emot_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                           sum_sq     df            F         PR(>F)\n",
            "C(Features)              4.940924    2.0  6146.423664  1.771353e-153\n",
            "C(Emotions)              0.006460    5.0     3.214252   8.555292e-03\n",
            "C(Features):C(Emotions)  0.012086   10.0     3.007067   1.645751e-03\n",
            "Residual                 0.065113  162.0          NaN            NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IMkVAAjHbakD",
        "outputId": "137e38a5-c882-493f-c632-41f5d2be4ff0"
      },
      "source": [
        "# One way Anova test for individual emotions:\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for i in emotions:\n",
        "  temp_df = ml_pdnc_emot_df[ml_pdnc_emot_df['Emotions']==i]\n",
        "  #del temp_df['Emotions']\n",
        "  ml_pdnc_formula = 'wtd_F_score ~ C(Features)'\n",
        "  ml_pdnc_model = ols(ml_pdnc_formula, temp_df).fit()\n",
        "  ml_pdnc_aov_table = anova_lm(ml_pdnc_model, typ=2)\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(ml_pdnc_aov_table)\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for E1 \n",
            "\n",
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.901221   2.0  1312.482477  1.274224e-27\n",
            "Residual     0.009270  27.0          NaN           NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E2 \n",
            "\n",
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.829298   2.0  1140.396165  8.322523e-27\n",
            "Residual     0.009817  27.0          NaN           NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E3 \n",
            "\n",
            "               sum_sq    df           F        PR(>F)\n",
            "C(Features)  0.748071   2.0  700.843412  5.392135e-24\n",
            "Residual     0.014410  27.0         NaN           NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E4 \n",
            "\n",
            "               sum_sq    df           F        PR(>F)\n",
            "C(Features)  0.807523   2.0  1032.21926  3.143417e-26\n",
            "Residual     0.010561  27.0         NaN           NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E5 \n",
            "\n",
            "               sum_sq    df         F        PR(>F)\n",
            "C(Features)  0.731634   2.0  871.8102  2.976577e-25\n",
            "Residual     0.011329  27.0       NaN           NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E6 \n",
            "\n",
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.935263   2.0  1298.172458  1.475241e-27\n",
            "Residual     0.009726  27.0          NaN           NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p5jN3c9TEAHM",
        "outputId": "ac8fde07-aa4c-48cc-f0e7-dfb19b6e1b22"
      },
      "source": [
        "# Tukey HSD test for individual emotions\n",
        "\n",
        "for i in emotions:\n",
        "  temp_df = ml_pdnc_emot_df[ml_pdnc_emot_df['Emotions']==i]\n",
        "  ml_pdnc_mc = MultiComparison(temp_df['wtd_F_score'], temp_df['Features'])\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(ml_pdnc_mc.tukeyhsd())\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for E1 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.3404 0.001 -0.3609 -0.3198   True\n",
            "   CSP Spectral   0.0495 0.001   0.029  0.0701   True\n",
            "   IIR Spectral   0.3899 0.001  0.3694  0.4105   True\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E2 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.3149 0.001 -0.3361 -0.2938   True\n",
            "   CSP Spectral   0.0662 0.001   0.045  0.0873   True\n",
            "   IIR Spectral   0.3811 0.001    0.36  0.4022   True\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E3 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05  \n",
            "======================================================\n",
            "group1  group2  meandiff p-adj   lower   upper  reject\n",
            "------------------------------------------------------\n",
            "   CSP      IIR  -0.3133  0.001 -0.3389 -0.2877   True\n",
            "   CSP Spectral   0.0398 0.0018  0.0142  0.0654   True\n",
            "   IIR Spectral   0.3531  0.001  0.3275  0.3787   True\n",
            "------------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E4 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.3197 0.001 -0.3416 -0.2978   True\n",
            "   CSP Spectral    0.051 0.001  0.0291   0.073   True\n",
            "   IIR Spectral   0.3707 0.001  0.3488  0.3927   True\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E5 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.3048 0.001 -0.3276 -0.2821   True\n",
            "   CSP Spectral   0.0477 0.001   0.025  0.0704   True\n",
            "   IIR Spectral   0.3525 0.001  0.3298  0.3752   True\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E6 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.3511 0.001 -0.3722 -0.3301   True\n",
            "   CSP Spectral   0.0431 0.001  0.0221  0.0641   True\n",
            "   IIR Spectral   0.3942 0.001  0.3732  0.4153   True\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RdYDpN263VD"
      },
      "source": [
        "The above results shows that for any emotions, all features are significantly different."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yLoLVX3leiiH",
        "outputId": "0d730f82-ee58-4965-fcbd-d60a91667ce1"
      },
      "source": [
        "# One way Anova test for individual features:\n",
        "\n",
        "for i in features:\n",
        "  temp_df = ml_pdnc_emot_df[ml_pdnc_emot_df['Features']==i]\n",
        "  #del temp_df['Emotions']\n",
        "  ml_pdnc_formula = 'wtd_F_score ~ C(Emotions)'\n",
        "  ml_pdnc_model = ols(ml_pdnc_formula, temp_df).fit()\n",
        "  ml_pdnc_aov_table = anova_lm(ml_pdnc_model, typ=2)\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(ml_pdnc_aov_table)\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for Spectral \n",
            "\n",
            "               sum_sq    df        F    PR(>F)\n",
            "C(Emotions)  0.002505   5.0  2.51875  0.040296\n",
            "Residual     0.010742  54.0      NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for CSP \n",
            "\n",
            "               sum_sq    df        F    PR(>F)\n",
            "C(Emotions)  0.005172   5.0  2.58029  0.036455\n",
            "Residual     0.021650  54.0      NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for IIR \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.010868   5.0  3.587154  0.007129\n",
            "Residual     0.032722  54.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ez1_16TpLscJ",
        "outputId": "cea6f72a-3779-4d30-8386-e93ab71c2a38"
      },
      "source": [
        "# Tukey HSD test for individual features\n",
        "\n",
        "for i in features:\n",
        "  temp_df = ml_pdnc_emot_df[ml_pdnc_emot_df['Features']==i]\n",
        "  ml_pdnc_mc = MultiComparison(temp_df['wtd_F_score'], temp_df['Emotions'])\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(ml_pdnc_mc.tukeyhsd())\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for Spectral \n",
            "\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0067 0.8902 -0.0254 0.0119  False\n",
            "    E1     E3  -0.0093 0.6634 -0.0279 0.0094  False\n",
            "    E1     E4  -0.0116 0.4483 -0.0303  0.007  False\n",
            "    E1     E5  -0.0216 0.0143 -0.0402 -0.003   True\n",
            "    E1     E6  -0.0114 0.4675 -0.0301 0.0072  False\n",
            "    E2     E3  -0.0025    0.9 -0.0212 0.0161  False\n",
            "    E2     E4  -0.0049    0.9 -0.0236 0.0137  False\n",
            "    E2     E5  -0.0149  0.189 -0.0335 0.0038  False\n",
            "    E2     E6  -0.0047    0.9 -0.0234 0.0139  False\n",
            "    E3     E4  -0.0024    0.9  -0.021 0.0163  False\n",
            "    E3     E5  -0.0123 0.3818  -0.031 0.0063  False\n",
            "    E3     E6  -0.0022    0.9 -0.0208 0.0165  False\n",
            "    E4     E5    -0.01 0.6008 -0.0286 0.0087  False\n",
            "    E4     E6   0.0002    0.9 -0.0184 0.0188  False\n",
            "    E5     E6   0.0102 0.5829 -0.0085 0.0288  False\n",
            "---------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for CSP \n",
            "\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0233 0.1132 -0.0498 0.0031  False\n",
            "    E1     E3   0.0005    0.9  -0.026 0.0269  False\n",
            "    E1     E4  -0.0131 0.6644 -0.0396 0.0133  False\n",
            "    E1     E5  -0.0197 0.2522 -0.0462 0.0067  False\n",
            "    E1     E6   -0.005    0.9 -0.0315 0.0215  False\n",
            "    E2     E3   0.0238 0.1009 -0.0027 0.0503  False\n",
            "    E2     E4   0.0102 0.8476 -0.0162 0.0367  False\n",
            "    E2     E5   0.0036    0.9 -0.0229 0.0301  False\n",
            "    E2     E6   0.0183 0.3292 -0.0081 0.0448  False\n",
            "    E3     E4  -0.0136 0.6355   -0.04 0.0129  False\n",
            "    E3     E5  -0.0202 0.2299 -0.0467 0.0062  False\n",
            "    E3     E6  -0.0055    0.9 -0.0319  0.021  False\n",
            "    E4     E5  -0.0066    0.9 -0.0331 0.0198  False\n",
            "    E4     E6   0.0081    0.9 -0.0183 0.0346  False\n",
            "    E5     E6   0.0147 0.5626 -0.0117 0.0412  False\n",
            "---------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for IIR \n",
            "\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2   0.0021    0.9 -0.0304  0.0346  False\n",
            "    E1     E3   0.0276  0.141  -0.005  0.0601  False\n",
            "    E1     E4   0.0076    0.9  -0.025  0.0401  False\n",
            "    E1     E5   0.0158 0.6822 -0.0167  0.0483  False\n",
            "    E1     E6  -0.0157 0.6848 -0.0483  0.0168  False\n",
            "    E2     E3   0.0254 0.2075 -0.0071   0.058  False\n",
            "    E2     E4   0.0054    0.9 -0.0271   0.038  False\n",
            "    E2     E5   0.0137 0.7905 -0.0188  0.0462  False\n",
            "    E2     E6  -0.0179 0.5764 -0.0504  0.0147  False\n",
            "    E3     E4    -0.02 0.4652 -0.0525  0.0125  False\n",
            "    E3     E5  -0.0118  0.888 -0.0443  0.0208  False\n",
            "    E3     E6  -0.0433 0.0031 -0.0758 -0.0108   True\n",
            "    E4     E5   0.0082    0.9 -0.0243  0.0408  False\n",
            "    E4     E6  -0.0233 0.2944 -0.0558  0.0092  False\n",
            "    E5     E6  -0.0315 0.0624 -0.0641   0.001  False\n",
            "----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kXqurqzwMCU8"
      },
      "source": [
        "The above results show that:\n",
        "\n",
        "\n",
        "1.   For spectral features, only the pair E1 and E5 are significantly different.\n",
        "2.   For CSP features, none of the emotions are signicantly different.\n",
        "3. For IIR features, only the pair E3 and E6 are significantly different.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "toYNYaVMMpAQ"
      },
      "source": [
        "## HV vs LV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K5cIGXj0MxOD",
        "outputId": "48f05d56-5c15-4050-a349-2e8fed0f911b"
      },
      "source": [
        "# Create a DataFrame\n",
        "\n",
        "ml_hvlv_df = pd.DataFrame()\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for i in Data:\n",
        "  for j in features:\n",
        "      if j == \"Spectral\":\n",
        "        ml_hvlv_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(j)+'/ML_HVLV_'+str(j).lower()+'_'+str(i)+'.mat', 'wtd_f1_list')\n",
        "      else:\n",
        "        ml_hvlv_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(j)+'/ML_HVLV_'+str(j)+'_'+str(i)+'.mat', 'wtd_f1_list')\n",
        "      data = pd.DataFrame({'wtd_F_score': ml_hvlv_f1[0],'Features': len(ml_hvlv_f1[0])*[str(j)], 'Data': len(ml_hvlv_f1[0])*[str(i)]})\n",
        "      ml_hvlv_df = ml_hvlv_df.append(data, ignore_index=True)\n",
        "     \n",
        "print(ml_hvlv_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    wtd_F_score  Features  Data\n",
            "0      0.797863  Spectral  Full\n",
            "1      0.766904  Spectral  Full\n",
            "2      0.770240  Spectral  Full\n",
            "3      0.774627  Spectral  Full\n",
            "4      0.778947  Spectral  Full\n",
            "..          ...       ...   ...\n",
            "85     0.552898       IIR    NC\n",
            "86     0.552689       IIR    NC\n",
            "87     0.560022       IIR    NC\n",
            "88     0.551980       IIR    NC\n",
            "89     0.563320       IIR    NC\n",
            "\n",
            "[90 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tQ8zfmB4PIYY",
        "outputId": "ca03b2d0-f993-4597-aefd-49c19b59ddb1"
      },
      "source": [
        "# FULL DATA\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_hvlv_full_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_hvlv_full_model = ols(ml_hvlv_full_formula, ml_hvlv_df[ml_hvlv_df['Data']=='Full']).fit()\n",
        "ml_hvlv_aov_table = anova_lm(ml_hvlv_full_model, typ=2)\n",
        "print(ml_hvlv_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.391836   2.0  2350.869926  5.180213e-31\n",
            "Residual     0.002250  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "npb-lUoNQJi3"
      },
      "source": [
        "Thers is a significant difference among the features. Let's find out for what paires."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DH4sTE75Qcwq",
        "outputId": "bdeddd6b-fd1d-4add-8172-be436410607d"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "ml_hvlv_full_mc = MultiComparison(ml_hvlv_df[ml_hvlv_df['Data']=='Full']['wtd_F_score'], ml_hvlv_df[ml_hvlv_df['Data']=='Full']['Features'])\n",
        "print(ml_hvlv_full_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.2583 0.001 -0.2684 -0.2482   True\n",
            "   CSP Spectral  -0.0357 0.001 -0.0458 -0.0256   True\n",
            "   IIR Spectral   0.2226 0.001  0.2125  0.2327   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7Iky4anIVQ5G"
      },
      "source": [
        "Tukey post-hoc results show that there is a significatn difference across all the features pairwise."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ODwhhQvWVoOJ",
        "outputId": "9be9d8dd-8ab8-48e1-904b-49b9a597adac"
      },
      "source": [
        "# PD DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_hvlv_pd_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_hvlv_pd_model = ols(ml_hvlv_pd_formula, ml_hvlv_df[ml_hvlv_df['Data']=='PD']).fit()\n",
        "ml_hvlv_pd_aov_table = anova_lm(ml_hvlv_pd_model, typ=2)\n",
        "print(ml_hvlv_pd_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df           F        PR(>F)\n",
            "C(Features)  0.437019   2.0  969.053477  7.289491e-26\n",
            "Residual     0.006088  27.0         NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dMIt0H3HWAfH"
      },
      "source": [
        "There is a significant difference across features."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gkKx1Ry2WJrU",
        "outputId": "8cc0a034-823a-4c58-d087-a21cf784bbcf"
      },
      "source": [
        "# Comparison of means using Tukey HSD for PD data\n",
        "\n",
        "ml_hvlv_pd_mc = MultiComparison(ml_hvlv_df[ml_hvlv_df['Data']=='PD']['wtd_F_score'], ml_hvlv_df[ml_hvlv_df['Data']=='PD']['Features'])\n",
        "print(ml_hvlv_pd_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.2903 0.001 -0.3069 -0.2736   True\n",
            "   CSP Spectral  -0.0966 0.001 -0.1132 -0.0799   True\n",
            "   IIR Spectral   0.1937 0.001  0.1771  0.2104   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UX5BkMO0WTxx"
      },
      "source": [
        "Okay. All are significantly different."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UVVoDLwBWjNE",
        "outputId": "ce86b0f5-231d-4739-de5e-589cc726a501"
      },
      "source": [
        "# NC DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_hvlv_nc_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_hvlv_nc_model = ols(ml_hvlv_nc_formula, ml_hvlv_df[ml_hvlv_df['Data']=='NC']).fit()\n",
        "ml_hvlv_nc_aov_table = anova_lm(ml_hvlv_nc_model, typ=2)\n",
        "print(ml_hvlv_nc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.485920   2.0  1382.905798  6.336489e-28\n",
            "Residual     0.004744  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L9EvOMSWWxyF"
      },
      "source": [
        "Umm. Features are significant for NC data too!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p4xlR-xKW80o",
        "outputId": "6abd5dde-926d-4373-e688-be01b980e157"
      },
      "source": [
        "# Comparison of means using Tukey HSD for NC data\n",
        "\n",
        "ml_hvlv_nc_mc = MultiComparison(ml_hvlv_df[ml_hvlv_df['Data']=='NC']['wtd_F_score'], ml_hvlv_df[ml_hvlv_df['Data']=='NC']['Features'])\n",
        "print(ml_hvlv_nc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.2878 0.001 -0.3025 -0.2731   True\n",
            "   CSP Spectral  -0.0402 0.001 -0.0548 -0.0255   True\n",
            "   IIR Spectral   0.2477 0.001   0.233  0.2623   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OViXSiqDXYVD"
      },
      "source": [
        "Features are pairwise significant for NC data as well. Let us check feature-wise."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "05EoysF9X2Vn",
        "outputId": "daf6e45e-aec4-406e-88ba-e7f7ca1b7554"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api for PD and NC data\n",
        "\n",
        "ml_hvlv_formula = 'wtd_F_score ~ C(Features)+C(Data)+C(Features):C(Data)'\n",
        "ml_hvlv_model = ols(ml_hvlv_formula, ml_hvlv_df[ml_hvlv_df['Data'].isin(['PD', 'NC'])]).fit()\n",
        "ml_hvlv_aov_table = anova_lm(ml_hvlv_model, typ=2)\n",
        "print(ml_hvlv_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                       sum_sq    df            F        PR(>F)\n",
            "C(Features)          0.912776   2.0  2275.250765  7.390958e-53\n",
            "C(Data)              0.005619   1.0    28.011446  2.254996e-06\n",
            "C(Features):C(Data)  0.010164   2.0    25.335495  1.735085e-08\n",
            "Residual             0.010832  54.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eAJ4EvOacj5w",
        "outputId": "60b370c6-345f-41ac-c5a4-810a6ebd4df1"
      },
      "source": [
        "# T-test for individual features\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "dummy_df = ml_hvlv_df[ml_hvlv_df['Data'].isin(['PD', 'NC'])]\n",
        "for i in features:\n",
        "  print(\"-----\", i, \"-----\")\n",
        "  dummy_df_2 = dummy_df[dummy_df['Features']==i]\n",
        "  d1=(dummy_df_2[dummy_df_2['Data']=='PD'])\n",
        "  d2=(dummy_df_2[dummy_df_2['Data']=='NC'])\n",
        "\n",
        "  stat, p = ttest_ind(d1['wtd_F_score'].to_numpy(), d2['wtd_F_score'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "\t  print('Probably the same distribution')\n",
        "  else:\n",
        "\t  print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Spectral -----\n",
            "stat=-6.349, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- CSP -----\n",
            "stat=0.058, p=0.954\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "----- IIR -----\n",
            "stat=-0.490, p=0.630\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j25mauLggnzG"
      },
      "source": [
        "## HA vs LA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S3wKgKaYg0EM",
        "outputId": "d8af2b45-659b-4aac-c34e-e479b31450fe"
      },
      "source": [
        "# Create a DataFrame\n",
        "\n",
        "ml_hala_df = pd.DataFrame()\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "data = [\"Full\", \"PD\", \"NC\"]\n",
        "for i in data:\n",
        "  for j in features:\n",
        "      if j == \"Spectral\":\n",
        "        ml_hala_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(j)+'/ML_HALA_'+str(j).lower()+'_'+str(i)+'.mat', 'wtd_f1_list')\n",
        "      else:\n",
        "        ml_hala_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(j)+'/ML_HALA_'+str(j)+'_'+str(i)+'.mat', 'wtd_f1_list')\n",
        "      data = pd.DataFrame({'wtd_F_score': ml_hala_f1[0],'Features': len(ml_hala_f1[0])*[str(j)], 'Data': len(ml_hala_f1[0])*[str(i)]})\n",
        "      ml_hala_df = ml_hala_df.append(data, ignore_index=True)\n",
        "     \n",
        "print(ml_hala_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    wtd_F_score  Features  Data\n",
            "0      0.920984  Spectral  Full\n",
            "1      0.915029  Spectral  Full\n",
            "2      0.908885  Spectral  Full\n",
            "3      0.923365  Spectral  Full\n",
            "4      0.903057  Spectral  Full\n",
            "..          ...       ...   ...\n",
            "85     0.761836       IIR    NC\n",
            "86     0.761836       IIR    NC\n",
            "87     0.761836       IIR    NC\n",
            "88     0.761836       IIR    NC\n",
            "89     0.761487       IIR    NC\n",
            "\n",
            "[90 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z5Dh87-dhMyK",
        "outputId": "ea1c9ac1-6f28-4e20-9a89-7706f901e82d"
      },
      "source": [
        "# FULL DATA\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_hala_full_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_hala_full_model = ols(ml_hala_full_formula, ml_hala_df[ml_hala_df['Data']=='Full']).fit()\n",
        "ml_hala_aov_table = anova_lm(ml_hala_full_model, typ=2)\n",
        "print(ml_hala_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.173121   2.0  3449.412616  2.999475e-33\n",
            "Residual     0.000678  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eUK8Vuq6hi2R",
        "outputId": "ca8b84d2-e812-4e75-eb0d-24a904e0089b"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "ml_hala_full_mc = MultiComparison(ml_hala_df[ml_hala_df['Data']=='Full']['wtd_F_score'], ml_hala_df[ml_hala_df['Data']=='Full']['Features'])\n",
        "print(ml_hala_full_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.1675 0.001 -0.1731 -0.1619   True\n",
            "   CSP Spectral  -0.0136 0.001 -0.0191  -0.008   True\n",
            "   IIR Spectral   0.1539 0.001  0.1484  0.1595   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wII6EwCih08D",
        "outputId": "e2b3624a-5f19-4829-ca2c-16773adac27c"
      },
      "source": [
        "# PD DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_hala_pd_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_hala_pd_model = ols(ml_hala_pd_formula, ml_hala_df[ml_hala_df['Data']=='PD']).fit()\n",
        "ml_hala_pd_aov_table = anova_lm(ml_hala_pd_model, typ=2)\n",
        "print(ml_hala_pd_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.183318   2.0  1332.944331  1.036245e-27\n",
            "Residual     0.001857  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "erpcO4DliFEH",
        "outputId": "76b6dd55-a838-459f-dcc4-7c9fb097f368"
      },
      "source": [
        "# Comparison of means using Tukey HSD for PD data\n",
        "\n",
        "ml_hala_pd_mc = MultiComparison(ml_hala_df[ml_hala_df['Data']=='PD']['wtd_F_score'], ml_hala_df[ml_hala_df['Data']=='PD']['Features'])\n",
        "print(ml_hala_pd_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.1719 0.001 -0.1811 -0.1627   True\n",
            "   CSP Spectral  -0.0128 0.005  -0.022 -0.0036   True\n",
            "   IIR Spectral    0.159 0.001  0.1499  0.1682   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5dHddpGwieMy",
        "outputId": "dedbb1d8-ce35-4b16-f6a1-5435ab8f0691"
      },
      "source": [
        "# NC DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_hala_nc_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_hala_nc_model = ols(ml_hala_nc_formula, ml_hala_df[ml_hala_df['Data']=='NC']).fit()\n",
        "ml_hala_nc_aov_table = anova_lm(ml_hala_nc_model, typ=2)\n",
        "print(ml_hala_nc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  0.188884   2.0  2278.523703  7.880395e-31\n",
            "Residual     0.001119  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lpGRzabzitr6",
        "outputId": "477c3613-40d3-4529-ecf9-b70273797f47"
      },
      "source": [
        "# Comparison of means using Tukey HSD for NC data\n",
        "\n",
        "ml_hala_nc_mc = MultiComparison(ml_hala_df[ml_hala_df['Data']=='NC']['wtd_F_score'], ml_hala_df[ml_hala_df['Data']=='NC']['Features'])\n",
        "print(ml_hala_nc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.1816 0.001 -0.1887 -0.1744   True\n",
            "   CSP Spectral  -0.0307 0.001 -0.0378 -0.0235   True\n",
            "   IIR Spectral   0.1509 0.001  0.1437   0.158   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3oUk6vQ1i8a7",
        "outputId": "3f51fd7f-d529-4ca4-9c1a-c0972e5bed6a"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api for PD and NC data\n",
        "\n",
        "ml_hala_formula = 'wtd_F_score ~ C(Features)+C(Data)+C(Features):C(Data)'\n",
        "ml_hala_model = ols(ml_hala_formula, ml_hala_df[ml_hala_df['Data'].isin(['PD', 'NC'])]).fit()\n",
        "ml_hala_aov_table = anova_lm(ml_hala_model, typ=2)\n",
        "print(ml_hala_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                       sum_sq    df            F        PR(>F)\n",
            "C(Features)          0.371402   2.0  3369.853733  2.031172e-57\n",
            "C(Data)              0.000066   1.0     1.200737  2.780376e-01\n",
            "C(Features):C(Data)  0.000800   2.0     7.257800  1.615533e-03\n",
            "Residual             0.002976  54.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TuX5Yx1MjN2g",
        "outputId": "9cbaaec9-c177-46f6-9fac-c1ded6115c10"
      },
      "source": [
        "# T-test for individual features\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "dummy_df = ml_hala_df[ml_hala_df['Data'].isin(['PD', 'NC'])]\n",
        "for i in features:\n",
        "  print(\"-----\", i, \"-----\")\n",
        "  dummy_df_2 = dummy_df[dummy_df['Features']==i]\n",
        "  d1=(dummy_df_2[dummy_df_2['Data']=='PD'])\n",
        "  d2=(dummy_df_2[dummy_df_2['Data']=='NC'])\n",
        "\n",
        "  stat, p = ttest_ind(d1['wtd_F_score'].to_numpy(), d2['wtd_F_score'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "\t  print('Probably the same distribution')\n",
        "  else:\n",
        "\t  print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Spectral -----\n",
            "stat=3.423, p=0.003\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- CSP -----\n",
            "stat=-1.524, p=0.148\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "----- IIR -----\n",
            "stat=2.102, p=0.063\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Il1oBBRgjmPG"
      },
      "source": [
        "## MultiClass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "swnzGiIpj2YD",
        "outputId": "39dd82d8-ef45-46b2-d028-4274b053e99f"
      },
      "source": [
        "# Create a DataFrame\n",
        "\n",
        "ml_multi_df = pd.DataFrame()\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "data = [\"Full\", \"PD\", \"NC\"]\n",
        "for i in data:\n",
        "  for j in features:\n",
        "      if j == \"Spectral\":\n",
        "        ml_multi_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(j)+'/ML_MultiClass_'+str(j).lower()+'_'+str(i)+'.mat', 'wtd_f1_list')\n",
        "      else:\n",
        "        ml_multi_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/'+str(j)+'/ML_MultiClass_'+str(j)+'_'+str(i)+'.mat', 'wtd_f1_list')\n",
        "      data = pd.DataFrame({'wtd_F_score': ml_multi_f1[0],'Features': len(ml_multi_f1[0])*[str(j)], 'Data': len(ml_multi_f1[0])*[str(i)]})\n",
        "      ml_multi_df = ml_multi_df.append(data, ignore_index=True)\n",
        "     \n",
        "print(ml_multi_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    wtd_F_score  Features  Data\n",
            "0      0.643762  Spectral  Full\n",
            "1      0.635823  Spectral  Full\n",
            "2      0.654581  Spectral  Full\n",
            "3      0.632369  Spectral  Full\n",
            "4      0.644625  Spectral  Full\n",
            "..          ...       ...   ...\n",
            "85     0.198164       IIR    NC\n",
            "86     0.209688       IIR    NC\n",
            "87     0.207745       IIR    NC\n",
            "88     0.205831       IIR    NC\n",
            "89     0.215441       IIR    NC\n",
            "\n",
            "[90 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_7UVVfJOlnlT",
        "outputId": "d0a6469f-4bb5-430b-e2c7-05d6d82b1642"
      },
      "source": [
        "# FULL DATA\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_multi_full_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_multi_full_model = ols(ml_multi_full_formula, ml_multi_df[ml_multi_df['Data']=='Full']).fit()\n",
        "ml_multi_aov_table = anova_lm(ml_multi_full_model, typ=2)\n",
        "print(ml_multi_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  1.670215   2.0  9133.041011  6.060911e-39\n",
            "Residual     0.002469  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AJMo7BFemDO1",
        "outputId": "17d23233-4571-4e99-fd7e-e78a8dc38b45"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "ml_multi_full_mc = MultiComparison(ml_multi_df[ml_multi_df['Data']=='Full']['wtd_F_score'], ml_multi_df[ml_multi_df['Data']=='Full']['Features'])\n",
        "print(ml_multi_full_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.5339 0.001 -0.5445 -0.5233   True\n",
            "   CSP Spectral  -0.0751 0.001 -0.0857 -0.0645   True\n",
            "   IIR Spectral   0.4587 0.001  0.4481  0.4693   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aSs8uSAOmUbo",
        "outputId": "2ea8429c-56e0-479e-98d1-aaf8433d67f6"
      },
      "source": [
        "# PD DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_multi_pd_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_multi_pd_model = ols(ml_multi_pd_formula, ml_multi_df[ml_multi_df['Data']=='PD']).fit()\n",
        "ml_multi_pd_aov_table = anova_lm(ml_multi_pd_model, typ=2)\n",
        "print(ml_multi_pd_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  1.914334   2.0  4113.914786  2.804473e-34\n",
            "Residual     0.006282  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XCU54iRYmQjC",
        "outputId": "d208b7f2-5b7b-4014-9b6b-8b53af30e71c"
      },
      "source": [
        "# Comparison of means using Tukey HSD for PD data\n",
        "\n",
        "ml_multi_pd_mc = MultiComparison(ml_multi_df[ml_multi_df['Data']=='PD']['wtd_F_score'], ml_multi_df[ml_multi_df['Data']=='PD']['Features'])\n",
        "print(ml_multi_pd_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.5905 0.001 -0.6074 -0.5736   True\n",
            "   CSP Spectral  -0.1351 0.001  -0.152 -0.1182   True\n",
            "   IIR Spectral   0.4554 0.001  0.4385  0.4723   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WjzM1urxnCJT",
        "outputId": "a504b6ad-ef89-4731-a3d7-bf1e898017f8"
      },
      "source": [
        "# NC DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "ml_multi_nc_formula = 'wtd_F_score ~ C(Features)'\n",
        "ml_multi_nc_model = ols(ml_multi_nc_formula, ml_multi_df[ml_multi_df['Data']=='NC']).fit()\n",
        "ml_multi_nc_aov_table = anova_lm(ml_multi_nc_model, typ=2)\n",
        "print(ml_multi_nc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df            F        PR(>F)\n",
            "C(Features)  1.726515   2.0  3286.544874  5.747619e-33\n",
            "Residual     0.007092  27.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxdBIfjWnWN_",
        "outputId": "99b89c84-2c07-4934-bf6d-7fac9a679aba"
      },
      "source": [
        "# Comparison of means using Tukey HSD for NC data\n",
        "\n",
        "ml_multi_nc_mc = MultiComparison(ml_multi_df[ml_multi_df['Data']=='NC']['wtd_F_score'], ml_multi_df[ml_multi_df['Data']=='NC']['Features'])\n",
        "print(ml_multi_nc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj  lower   upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.5371 0.001 -0.5551 -0.5192   True\n",
            "   CSP Spectral  -0.0622 0.001 -0.0801 -0.0442   True\n",
            "   IIR Spectral    0.475 0.001   0.457  0.4929   True\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7iAHGZhLnloi",
        "outputId": "79228eb4-1042-4fde-d57b-7e66c6a838c0"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api for PD and NC data\n",
        "\n",
        "ml_multi_formula = 'wtd_F_score ~ C(Features)+C(Data)+C(Features):C(Data)'\n",
        "ml_multi_model = ols(ml_multi_formula, ml_multi_df[ml_multi_df['Data'].isin(['PD', 'NC'])]).fit()\n",
        "ml_multi_aov_table = anova_lm(ml_multi_model, typ=2)\n",
        "print(ml_multi_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                       sum_sq    df            F        PR(>F)\n",
            "C(Features)          3.626597   2.0  7321.577940  1.816906e-66\n",
            "C(Data)              0.008309   1.0    33.551032  3.663274e-07\n",
            "C(Features):C(Data)  0.014253   2.0    28.774124  3.112700e-09\n",
            "Residual             0.013374  54.0          NaN           NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UUD1-VVWnywn",
        "outputId": "47d97011-67d2-44f1-d487-79eb2fd5034b"
      },
      "source": [
        "# T-test for individual features\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "dummy_df = ml_multi_df[ml_multi_df['Data'].isin(['PD', 'NC'])]\n",
        "for i in features:\n",
        "  print(\"-----\", i, \"-----\")\n",
        "  dummy_df_2 = dummy_df[dummy_df['Features']==i]\n",
        "  d1=(dummy_df_2[dummy_df_2['Data']=='PD'])\n",
        "  d2=(dummy_df_2[dummy_df_2['Data']=='NC'])\n",
        "\n",
        "  stat, p = ttest_ind(d1['wtd_F_score'].to_numpy(), d2['wtd_F_score'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "\t  print('Probably the same distribution')\n",
        "  else:\n",
        "\t  print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Spectral -----\n",
            "stat=-6.417, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- CSP -----\n",
            "stat=2.815, p=0.012\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- IIR -----\n",
            "stat=-6.029, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JFSzr1oKz-B5",
        "outputId": "c1f1812f-89ba-49c3-a586-f065d4647a42"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "ml_conf_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for j in Data:\n",
        "  print(\"\\n #####\", j, \"####\")\n",
        "  ml_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_MultiClass_CSP_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(ml_conf, axis=0), '\\n')\n",
        "  l = 0\n",
        "  for k in emotions:\n",
        "    data = pd.DataFrame({'Accuracy': [i[l][l] for i in ml_conf],'Emotions': len(ml_conf)*[str(k)]})\n",
        "    ml_conf_df = ml_conf_df.append(data, ignore_index=True)\n",
        "    l = l+1\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  ml_formula = 'Accuracy ~ C(Emotions)'\n",
        "  ml_model = ols(ml_formula, ml_conf_df).fit()\n",
        "  ml_aov_table = anova_lm(ml_model, typ=2)\n",
        "  print(ml_aov_table)\n",
        "  \n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  ml_mc = MultiComparison(ml_conf_df['Accuracy'], ml_conf_df['Emotions'])\n",
        "  print(ml_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " ##### Full ####\n",
            "[[0.79723326 0.07484135 0.03805091 0.02550035 0.03510635 0.02926778]\n",
            " [0.07264382 0.73611315 0.06905976 0.06068685 0.02675697 0.03473944]\n",
            " [0.05916291 0.10014124 0.65608775 0.07730637 0.04181327 0.06548845]\n",
            " [0.02166148 0.07205221 0.07830394 0.70678596 0.06122407 0.05997234]\n",
            " [0.03434929 0.03880926 0.04043033 0.06466469 0.70331723 0.11842922]\n",
            " [0.03930763 0.0565494  0.04451245 0.05571566 0.09302972 0.71088514]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df          F        PR(>F)\n",
            "C(Emotions)  0.108301   5.0  25.848595  3.221020e-13\n",
            "Residual     0.045250  54.0        NaN           NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0611  0.001 -0.0994 -0.0229   True\n",
            "    E1     E3  -0.1411  0.001 -0.1794 -0.1029   True\n",
            "    E1     E4  -0.0904  0.001 -0.1287 -0.0522   True\n",
            "    E1     E5  -0.0939  0.001 -0.1322 -0.0557   True\n",
            "    E1     E6  -0.0863  0.001 -0.1246 -0.0481   True\n",
            "    E2     E3    -0.08  0.001 -0.1183 -0.0418   True\n",
            "    E2     E4  -0.0293 0.2262 -0.0676  0.0089  False\n",
            "    E2     E5  -0.0328 0.1324  -0.071  0.0055  False\n",
            "    E2     E6  -0.0252 0.3861 -0.0635   0.013  False\n",
            "    E3     E4   0.0507 0.0033  0.0125  0.0889   True\n",
            "    E3     E5   0.0472 0.0075   0.009  0.0855   True\n",
            "    E3     E6   0.0548 0.0012  0.0165   0.093   True\n",
            "    E4     E5  -0.0035    0.9 -0.0417  0.0348  False\n",
            "    E4     E6   0.0041    0.9 -0.0341  0.0423  False\n",
            "    E5     E6   0.0076    0.9 -0.0307  0.0458  False\n",
            "----------------------------------------------------\n",
            "\n",
            " ##### PD ####\n",
            "[[0.80772409 0.0629972  0.04364846 0.02768207 0.03611345 0.02183473]\n",
            " [0.07059355 0.7673871  0.06735484 0.03530323 0.02406452 0.03529677]\n",
            " [0.05657314 0.07678393 0.74767839 0.05903005 0.03039453 0.02953995]\n",
            " [0.02412534 0.05741047 0.06987603 0.757073   0.04825758 0.04325758]\n",
            " [0.03895883 0.03975872 0.03411356 0.06411618 0.70775636 0.11529635]\n",
            " [0.03342857 0.03577778 0.03423492 0.05569524 0.08746032 0.75340317]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df          F        PR(>F)\n",
            "C(Emotions)  0.135503    5.0  17.244471  1.115784e-12\n",
            "Residual     0.179157  114.0        NaN           NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0507 0.0013 -0.0871 -0.0144   True\n",
            "    E1     E3  -0.1006  0.001 -0.1369 -0.0643   True\n",
            "    E1     E4  -0.0705  0.001 -0.1069 -0.0342   True\n",
            "    E1     E5  -0.0969  0.001 -0.1333 -0.0606   True\n",
            "    E1     E6  -0.0703  0.001 -0.1067  -0.034   True\n",
            "    E2     E3  -0.0499 0.0017 -0.0862 -0.0135   True\n",
            "    E2     E4  -0.0198 0.5976 -0.0562  0.0165  False\n",
            "    E2     E5  -0.0462 0.0046 -0.0826 -0.0099   True\n",
            "    E2     E6  -0.0196 0.6074 -0.0559  0.0167  False\n",
            "    E3     E4     0.03 0.1662 -0.0063  0.0664  False\n",
            "    E3     E5   0.0037    0.9 -0.0327    0.04  False\n",
            "    E3     E6   0.0303 0.1603 -0.0061  0.0666  False\n",
            "    E4     E5  -0.0264 0.2921 -0.0627  0.0099  False\n",
            "    E4     E6   0.0002    0.9 -0.0361  0.0366  False\n",
            "    E5     E6   0.0266 0.2834 -0.0097  0.0629  False\n",
            "----------------------------------------------------\n",
            "\n",
            " ##### NC ####\n",
            "[[0.83261019 0.07162534 0.02332645 0.02248623 0.02497245 0.02497934]\n",
            " [0.0707619  0.77741587 0.05247619 0.0572254  0.02304127 0.01907937]\n",
            " [0.06007691 0.09816978 0.65395243 0.10067654 0.03638371 0.05074064]\n",
            " [0.02837535 0.08672269 0.09256303 0.69810224 0.04751401 0.04672269]\n",
            " [0.02175484 0.04266452 0.03463226 0.07164516 0.72462581 0.10467742]\n",
            " [0.03638211 0.04933779 0.04042748 0.04846577 0.08413978 0.74124705]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df          F        PR(>F)\n",
            "C(Emotions)  0.292637    5.0  33.036621  1.300202e-23\n",
            "Residual     0.308257  174.0        NaN           NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0522  0.001 -0.0835 -0.0209   True\n",
            "    E1     E3  -0.1266  0.001 -0.1579 -0.0953   True\n",
            "    E1     E4  -0.0919  0.001 -0.1232 -0.0606   True\n",
            "    E1     E5  -0.1006  0.001 -0.1319 -0.0693   True\n",
            "    E1     E6  -0.0773  0.001 -0.1087  -0.046   True\n",
            "    E2     E3  -0.0744  0.001 -0.1057 -0.0431   True\n",
            "    E2     E4  -0.0397 0.0046  -0.071 -0.0083   True\n",
            "    E2     E5  -0.0484  0.001 -0.0797 -0.0171   True\n",
            "    E2     E6  -0.0251 0.1947 -0.0564  0.0062  False\n",
            "    E3     E4   0.0347 0.0201  0.0034  0.0661   True\n",
            "    E3     E5    0.026 0.1649 -0.0053  0.0573  False\n",
            "    E3     E6   0.0493  0.001   0.018  0.0806   True\n",
            "    E4     E5  -0.0088    0.9 -0.0401  0.0226  False\n",
            "    E4     E6   0.0145  0.737 -0.0168  0.0458  False\n",
            "    E5     E6   0.0233 0.2709  -0.008  0.0546  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5asKrOgEhyKr",
        "outputId": "d0a05a4d-e312-46a2-90f1-77b4eef48630"
      },
      "source": [
        "# conf mat T-tests\n",
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "ml_conf_df = pd.DataFrame()\n",
        "Data = [\"PD\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for j in Data:\n",
        "  print(\"\\n #####\", j, \"####\")\n",
        "  ml_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_MultiClass_CSP_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(ml_conf, axis=0), '\\n')\n",
        "  l = 0\n",
        "  for k in emotions:\n",
        "    data = pd.DataFrame({'Accuracy': [i[l][l] for i in ml_conf],'Emotions': len(ml_conf)*[str(k)]})\n",
        "    ml_conf_df = ml_conf_df.append(data, ignore_index=True)\n",
        "    l = l+1\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  ml_formula = 'Accuracy ~ C(Emotions)'\n",
        "  ml_model = ols(ml_formula, ml_conf_df).fit()\n",
        "  ml_aov_table = anova_lm(ml_model, typ=2)\n",
        "  print(ml_aov_table)\n",
        "  \n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  ml_mc = MultiComparison(ml_conf_df['Accuracy'], ml_conf_df['Emotions'])\n",
        "  print(ml_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " ##### PD ####\n",
            "[[0.80772409 0.0629972  0.04364846 0.02768207 0.03611345 0.02183473]\n",
            " [0.07059355 0.7673871  0.06735484 0.03530323 0.02406452 0.03529677]\n",
            " [0.05657314 0.07678393 0.74767839 0.05903005 0.03039453 0.02953995]\n",
            " [0.02412534 0.05741047 0.06987603 0.757073   0.04825758 0.04325758]\n",
            " [0.03895883 0.03975872 0.03411356 0.06411618 0.70775636 0.11529635]\n",
            " [0.03342857 0.03577778 0.03423492 0.05569524 0.08746032 0.75340317]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.052054   5.0  8.683719  0.000004\n",
            "Residual     0.064740  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0403 0.1138 -0.0861  0.0054  False\n",
            "    E1     E3    -0.06 0.0037 -0.1058 -0.0143   True\n",
            "    E1     E4  -0.0507 0.0219 -0.0964 -0.0049   True\n",
            "    E1     E5     -0.1  0.001 -0.1457 -0.0542   True\n",
            "    E1     E6  -0.0543 0.0112 -0.1001 -0.0086   True\n",
            "    E2     E3  -0.0197 0.7733 -0.0655   0.026  False\n",
            "    E2     E4  -0.0103    0.9 -0.0561  0.0354  False\n",
            "    E2     E5  -0.0596 0.0041 -0.1054 -0.0139   True\n",
            "    E2     E6   -0.014    0.9 -0.0597  0.0318  False\n",
            "    E3     E4   0.0094    0.9 -0.0364  0.0551  False\n",
            "    E3     E5  -0.0399 0.1207 -0.0857  0.0058  False\n",
            "    E3     E6   0.0057    0.9   -0.04  0.0515  False\n",
            "    E4     E5  -0.0493 0.0276 -0.0951 -0.0036   True\n",
            "    E4     E6  -0.0037    0.9 -0.0494  0.0421  False\n",
            "    E5     E6   0.0456 0.0508 -0.0001  0.0914  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c6rQSZc-ijZs",
        "outputId": "55c9dd77-be2d-4bd9-e159-90c780343f0b"
      },
      "source": [
        "# conf mat T-tests\n",
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "ml_conf_df = pd.DataFrame()\n",
        "Data = [\"NC\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for j in Data:\n",
        "  print(\"\\n #####\", j, \"####\")\n",
        "  ml_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_MultiClass_CSP_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(ml_conf, axis=0), '\\n')\n",
        "  l = 0\n",
        "  for k in emotions:\n",
        "    data = pd.DataFrame({'Accuracy': [i[l][l] for i in ml_conf],'Emotions': len(ml_conf)*[str(k)]})\n",
        "    ml_conf_df = ml_conf_df.append(data, ignore_index=True)\n",
        "    l = l+1\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  ml_formula = 'Accuracy ~ C(Emotions)'\n",
        "  ml_model = ols(ml_formula, ml_conf_df).fit()\n",
        "  ml_aov_table = anova_lm(ml_model, typ=2)\n",
        "  print(ml_aov_table)\n",
        "  \n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  ml_mc = MultiComparison(ml_conf_df['Accuracy'], ml_conf_df['Emotions'])\n",
        "  print(ml_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " ##### NC ####\n",
            "[[0.83261019 0.07162534 0.02332645 0.02248623 0.02497245 0.02497934]\n",
            " [0.0707619  0.77741587 0.05247619 0.0572254  0.02304127 0.01907937]\n",
            " [0.06007691 0.09816978 0.65395243 0.10067654 0.03638371 0.05074064]\n",
            " [0.02837535 0.08672269 0.09256303 0.69810224 0.04751401 0.04672269]\n",
            " [0.02175484 0.04266452 0.03463226 0.07164516 0.72462581 0.10467742]\n",
            " [0.03638211 0.04933779 0.04042748 0.04846577 0.08413978 0.74124705]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df          F        PR(>F)\n",
            "C(Emotions)  0.193499   5.0  22.536166  3.913923e-12\n",
            "Residual     0.092731  54.0        NaN           NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0552 0.0471 -0.1099 -0.0004   True\n",
            "    E1     E3  -0.1787  0.001 -0.2334 -0.1239   True\n",
            "    E1     E4  -0.1345  0.001 -0.1893 -0.0798   True\n",
            "    E1     E5   -0.108  0.001 -0.1627 -0.0532   True\n",
            "    E1     E6  -0.0914  0.001 -0.1461 -0.0366   True\n",
            "    E2     E3  -0.1235  0.001 -0.1782 -0.0687   True\n",
            "    E2     E4  -0.0793  0.001 -0.1341 -0.0246   True\n",
            "    E2     E5  -0.0528 0.0649 -0.1075   0.002  False\n",
            "    E2     E6  -0.0362 0.3844 -0.0909  0.0186  False\n",
            "    E3     E4   0.0441 0.1806 -0.0106  0.0989  False\n",
            "    E3     E5   0.0707 0.0045  0.0159  0.1254   True\n",
            "    E3     E6   0.0873  0.001  0.0325   0.142   True\n",
            "    E4     E5   0.0265 0.6842 -0.0282  0.0813  False\n",
            "    E4     E6   0.0431 0.2009 -0.0116  0.0979  False\n",
            "    E5     E6   0.0166    0.9 -0.0381  0.0714  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9xRSm_PQoRmM"
      },
      "source": [
        "# 1D-CNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1OPxBvCnywbq"
      },
      "source": [
        "## PD vs NC"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L7EgRalry0AN"
      },
      "source": [
        "### Full data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ERGP2Nnzyntk",
        "outputId": "420ecb94-1ec8-4b5a-b3a3-a34be1bf6e88"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "oneD_pdnc_full_spec_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/Spectral/CNN_1D_results_PDNC_Spectral_Full.mat', 'f1_weightedNet')\n",
        "oneD_pdnc_full_csp_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_PDNC_CSP_Full.mat', 'f1_weightedNet')\n",
        "oneD_pdnc_full_iir_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/IIR/CNN_1D_results_PDNC_IIR_Full.mat', 'f1_weightedNet')\n",
        "\n",
        "print(oneD_pdnc_full_spec_f1)\n",
        "print(oneD_pdnc_full_csp_f1)\n",
        "print(oneD_pdnc_full_iir_f1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.97634827 0.9594299  0.98175343 0.99324287 0.99459459 0.99459459\n",
            "  0.97769685 0.99594595 0.99459457 0.99662162]]\n",
            "[[0.8383849  0.85219393 0.85649712 0.88352995 0.84900738 0.88836814\n",
            "  0.88784624 0.88632038 0.89397991 0.87889339]]\n",
            "[[0.95756173 0.87557922 0.91169224 0.71929323 0.96510765 0.95062312\n",
            "  0.93684988 0.90749182 0.91844142 0.90983485]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vNUHJwZVP6GD",
        "outputId": "013bc2a6-6204-4153-832a-76f64cff70e7"
      },
      "source": [
        "# One-way ANOVA between PD vs NC full data\n",
        "from scipy.stats import f_oneway\n",
        "\n",
        "stat, p = f_oneway(oneD_pdnc_full_spec_f1[0], oneD_pdnc_full_csp_f1[0], oneD_pdnc_full_iir_f1[0])\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "\tprint('Probably the same distribution')\n",
        "else:\n",
        "\tprint('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=18.816, p=0.000\n",
            "Probably different distributions\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I3WZH1jFQXhq",
        "outputId": "769bade9-a586-4d74-d2da-bc48aed7c6e3"
      },
      "source": [
        "# Create a dataframe of f-scores and its corresponding features for full data\n",
        "\n",
        "oneD_pdnc_full_spec_df = pd.DataFrame({'f1_weightedNet': oneD_pdnc_full_spec_f1[0],'Features': len(oneD_pdnc_full_spec_f1[0])*['spectral']})\n",
        "oneD_pdnc_full_csp_df = pd.DataFrame({'f1_weightedNet': oneD_pdnc_full_csp_f1[0],'Features': len(oneD_pdnc_full_csp_f1[0])*['csp']})\n",
        "oneD_pdnc_full_iir_df = pd.DataFrame({'f1_weightedNet': oneD_pdnc_full_iir_f1[0],'Features': len(oneD_pdnc_full_iir_f1[0])*['iir']})\n",
        "\n",
        "oneD_pdnc_full_df = pd.concat([oneD_pdnc_full_spec_df, oneD_pdnc_full_csp_df, oneD_pdnc_full_iir_df], ignore_index=True)\n",
        "\n",
        "#print(pdnc_full_df)\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_pdnc_full_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_pdnc_full_model = ols(oneD_pdnc_full_formula, oneD_pdnc_full_df).fit()\n",
        "oneD_pdnc_aov_table = anova_lm(oneD_pdnc_full_model, typ=2)\n",
        "print(oneD_pdnc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df          F    PR(>F)\n",
            "C(Features)  0.069861   2.0  18.815556  0.000008\n",
            "Residual     0.050125  27.0        NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nrv9Tz8DQ7yE",
        "outputId": "cc72c7d2-3c72-4aaa-b67d-41847532c0e5"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "oneD_pdnc_mc = MultiComparison(oneD_pdnc_full_df['f1_weightedNet'], oneD_pdnc_full_df['Features'])\n",
        "print(oneD_pdnc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1  group2  meandiff p-adj  lower  upper  reject\n",
            "----------------------------------------------------\n",
            "   csp      iir   0.0337 0.2052 -0.014 0.0815  False\n",
            "   csp spectral    0.115  0.001 0.0672 0.1627   True\n",
            "   iir spectral   0.0812  0.001 0.0335  0.129   True\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oKSGMn1BT8on"
      },
      "source": [
        "CSP and IIR are **not** significantly different."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8rgzheh_lKLb",
        "outputId": "ba6f0ecf-159a-4aab-809b-5d85d4e23964"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "oneD_conf_df = pd.DataFrame()\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "for j in features:\n",
        "  print(\"-----\", j, \"----\")\n",
        "  oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(j)+'/CNN_1D_results_PDNC_'+str(j)+'_Full.mat', 'conf_mat_list')\n",
        "  print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "  oneD_conf_nc = [i[0][0] for i in oneD_conf]\n",
        "  oneD_conf_pd = [i[1][1] for i in oneD_conf]\n",
        "  stat, p = ttest_ind(oneD_conf_nc, oneD_conf_pd,equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Spectral ----\n",
            "[[0.99366104 0.00633896]\n",
            " [0.02071722 0.97928278]] \n",
            "\n",
            "stat=2.115, p=0.061\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "----- CSP ----\n",
            "[[0.83672621 0.16327379]\n",
            " [0.09324116 0.90675884]] \n",
            "\n",
            "stat=-5.053, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- IIR ----\n",
            "[[0.86027613 0.13972387]\n",
            " [0.04500414 0.95499586]] \n",
            "\n",
            "stat=-1.759, p=0.100\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H_YDSeEeNNZs",
        "outputId": "0c73ef9e-a988-4092-fbed-453be45ad604"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "oneD_conf_df = pd.DataFrame()\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "data = [\"NC\", \"PD\"]\n",
        "\n",
        "l = 0\n",
        "for i in data:\n",
        "  print(\"\\nHow well is\", i, \"classified in different features?\\n\")\n",
        "  for k in features:\n",
        "    print(10*\"-\", k, 10*\"-\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(k)+'/CNN_1D_results_PDNC_'+str(k)+'_Full.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Features': len(oneD_conf)*[str(k)]})\n",
        "    oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  oneD_formula = 'Accuracy ~ C(Features)'\n",
        "  oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "  oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "  print(oneD_aov_table)\n",
        "\n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Features'])\n",
        "  print(oneD_mc.tukeyhsd())\n",
        "  l = l+1\n",
        "  print(30*\"*\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "How well is NC classified in different features?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.99366104 0.00633896]\n",
            " [0.02071722 0.97928278]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.83672621 0.16327379]\n",
            " [0.09324116 0.90675884]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.86027613 0.13972387]\n",
            " [0.04500414 0.95499586]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.143249   2.0  9.181335  0.000908\n",
            "Residual     0.210630  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0235 0.8099 -0.0744 0.1215  False\n",
            "   CSP Spectral   0.1569 0.0013   0.059 0.2549   True\n",
            "   IIR Spectral   0.1334 0.0061  0.0355 0.2313   True\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different features?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.99366104 0.00633896]\n",
            " [0.02071722 0.97928278]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.83672621 0.16327379]\n",
            " [0.09324116 0.90675884]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.86027613 0.13972387]\n",
            " [0.04500414 0.95499586]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df          F    PR(>F)\n",
            "C(Features)  0.137775   2.0  11.128161  0.000083\n",
            "Residual     0.352852  57.0        NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1  group2  meandiff p-adj  lower  upper  reject\n",
            "----------------------------------------------------\n",
            "   CSP      IIR   0.0359 0.3267 -0.024 0.0958  False\n",
            "   CSP Spectral   0.1147  0.001 0.0549 0.1746   True\n",
            "   IIR Spectral   0.0788 0.0068  0.019 0.1387   True\n",
            "----------------------------------------------------\n",
            "******************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r_mCC9lTUWSC"
      },
      "source": [
        "### Individual Emotions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tvhbO7pgUZGK",
        "outputId": "9934bd9a-bd26-4e33-a5b2-17110196647d"
      },
      "source": [
        "# Create a DataFrame\n",
        "oneD_pdnc_emot_df = pd.DataFrame()\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "for i in features:\n",
        "  for j in emotions:\n",
        "    oneD_pdnc_emot_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(i)+'/CNN_1D_results_PDNC_'+str(i)+'_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "    data = pd.DataFrame({'f1_weightedNet': oneD_pdnc_emot_f1[0],'Features': len(oneD_pdnc_emot_f1[0])*[str(i)], 'Emotions': len(oneD_pdnc_emot_f1[0])*[str(j)]})\n",
        "    oneD_pdnc_emot_df = oneD_pdnc_emot_df.append(data, ignore_index=True)\n",
        "\n",
        "print(oneD_pdnc_emot_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "     f1_weightedNet  Features Emotions\n",
            "0          0.937746  Spectral       E1\n",
            "1          0.962629  Spectral       E1\n",
            "2          0.970944  Spectral       E1\n",
            "3          0.941698  Spectral       E1\n",
            "4          0.995850  Spectral       E1\n",
            "..              ...       ...      ...\n",
            "175        0.864170       IIR       E6\n",
            "176        1.000000       IIR       E6\n",
            "177        0.986667       IIR       E6\n",
            "178        0.995556       IIR       E6\n",
            "179        1.000000       IIR       E6\n",
            "\n",
            "[180 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JGOWae-PZOgS",
        "outputId": "2525c50e-46fb-4432-c6a1-6c64ddab2572"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_pdnc_emot_formula = 'f1_weightedNet ~ C(Features)+C(Emotions)+C(Features):C(Emotions)'\n",
        "oneD_pdnc_emot_model = ols(oneD_pdnc_emot_formula, oneD_pdnc_emot_df).fit()\n",
        "oneD_pdnc_emot_aov_table = anova_lm(oneD_pdnc_emot_model, typ=2)\n",
        "print(oneD_pdnc_emot_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                           sum_sq     df         F    PR(>F)\n",
            "C(Features)              0.009853    2.0  0.857361  0.426196\n",
            "C(Emotions)              0.041061    5.0  1.429188  0.216428\n",
            "C(Features):C(Emotions)  0.067175   10.0  1.169057  0.315336\n",
            "Residual                 0.930861  162.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XbmYL62aaQeC",
        "outputId": "fed50c26-6d7d-49c4-b44c-b401ca2fce5a"
      },
      "source": [
        "# One way Anova test for individual emotions:\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for i in emotions:\n",
        "  temp_df = oneD_pdnc_emot_df[oneD_pdnc_emot_df['Emotions']==i]\n",
        "  #del temp_df['Emotions']\n",
        "  oneD_pdnc_formula = 'f1_weightedNet ~ C(Features)'\n",
        "  oneD_pdnc_model = ols(oneD_pdnc_formula, temp_df).fit()\n",
        "  oneD_pdnc_aov_table = anova_lm(oneD_pdnc_model, typ=2)\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(oneD_pdnc_aov_table)\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for E1 \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.002615   2.0  0.845929  0.440219\n",
            "Residual     0.041729  27.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E2 \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.006812   2.0  0.561364  0.576946\n",
            "Residual     0.163811  27.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E3 \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.027403   2.0  1.558483  0.228802\n",
            "Residual     0.237373  27.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E4 \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.016025   2.0  2.235097  0.126406\n",
            "Residual     0.096791  27.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E5 \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.002098   2.0  0.247315  0.782645\n",
            "Residual     0.114509  27.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E6 \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.022075   2.0  1.077239  0.354723\n",
            "Residual     0.276647  27.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9rtvFmkAauYI",
        "outputId": "3e81b386-6f63-4a0e-8a84-772999f86b84"
      },
      "source": [
        "# Tukey HSD test for individual emotions\n",
        "\n",
        "for i in emotions:\n",
        "  temp_df = oneD_pdnc_emot_df[oneD_pdnc_emot_df['Emotions']==i]\n",
        "  oneD_pdnc_mc = MultiComparison(temp_df['f1_weightedNet'], temp_df['Features'])\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(oneD_pdnc_mc.tukeyhsd())\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for E1 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0164 0.6177 -0.0272   0.06  False\n",
            "   CSP Spectral    0.022 0.4359 -0.0216 0.0656  False\n",
            "   IIR Spectral   0.0056    0.9  -0.038 0.0492  False\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E2 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR    0.031  0.642 -0.0553 0.1174  False\n",
            "   CSP Spectral  -0.0018    0.9 -0.0882 0.0845  False\n",
            "   IIR Spectral  -0.0328 0.6122 -0.1192 0.0535  False\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E3 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0573 0.3732 -0.1612 0.0467  False\n",
            "   CSP Spectral   0.0119    0.9  -0.092 0.1159  False\n",
            "   IIR Spectral   0.0692 0.2422 -0.0347 0.1732  False\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E4 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0093    0.9 -0.0571 0.0756  False\n",
            "   CSP Spectral    0.053 0.1366 -0.0134 0.1194  False\n",
            "   IIR Spectral   0.0437 0.2494 -0.0226 0.1101  False\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E5 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0014    0.9 -0.0708 0.0736  False\n",
            "   CSP Spectral   -0.017 0.8171 -0.0892 0.0552  False\n",
            "   IIR Spectral  -0.0184 0.7896 -0.0906 0.0538  False\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for E6 \n",
            "\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0658 0.3286 -0.1781 0.0464  False\n",
            "   CSP Spectral  -0.0252 0.8322 -0.1374  0.087  False\n",
            "   IIR Spectral   0.0406 0.6379 -0.0716 0.1528  False\n",
            "-----------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7yNRPgmsbGfr"
      },
      "source": [
        "None of them are significantly different."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4TwqMum1bTDJ",
        "outputId": "e81e2fae-47ff-4359-af40-d3d8c6fab68e"
      },
      "source": [
        "# One way Anova test for individual features:\n",
        "\n",
        "for i in features:\n",
        "  temp_df = oneD_pdnc_emot_df[oneD_pdnc_emot_df['Features']==i]\n",
        "  #del temp_df['Emotions']\n",
        "  oneD_pdnc_formula = 'f1_weightedNet ~ C(Emotions)'\n",
        "  oneD_pdnc_model = ols(oneD_pdnc_formula, temp_df).fit()\n",
        "  oneD_pdnc_aov_table = anova_lm(oneD_pdnc_model, typ=2)\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(oneD_pdnc_aov_table)\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for Spectral \n",
            "\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.043642   5.0  2.915577  0.021121\n",
            "Residual     0.161661  54.0       NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for CSP \n",
            "\n",
            "               sum_sq    df        F    PR(>F)\n",
            "C(Emotions)  0.008172   5.0  2.73615  0.028284\n",
            "Residual     0.032258  54.0      NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for IIR \n",
            "\n",
            "               sum_sq    df        F    PR(>F)\n",
            "C(Emotions)  0.056421   5.0  0.82686  0.536178\n",
            "Residual     0.736942  54.0      NaN       NaN\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CNjS_CZoewtU",
        "outputId": "2b87e700-2c62-494e-b30c-01bdf98d1778"
      },
      "source": [
        "# Tukey HSD test for individual features\n",
        "\n",
        "for i in features:\n",
        "  temp_df = oneD_pdnc_emot_df[oneD_pdnc_emot_df['Features']==i]\n",
        "  oneD_pdnc_mc = MultiComparison(temp_df['f1_weightedNet'], temp_df['Emotions'])\n",
        "  print(\"Table for\", i, '\\n')\n",
        "  print(oneD_pdnc_mc.tukeyhsd())\n",
        "  print('\\n')\n",
        "  print(10*'#')\n",
        "  print('\\n\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Table for Spectral \n",
            "\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0533 0.2643 -0.1256  0.019  False\n",
            "    E1     E3  -0.0055    0.9 -0.0778 0.0668  False\n",
            "    E1     E4   0.0112    0.9 -0.0611 0.0835  False\n",
            "    E1     E5   -0.048 0.3778 -0.1203 0.0243  False\n",
            "    E1     E6  -0.0518 0.2951 -0.1241 0.0205  False\n",
            "    E2     E3   0.0478 0.3834 -0.0245 0.1201  False\n",
            "    E2     E4   0.0645 0.1058 -0.0078 0.1368  False\n",
            "    E2     E5   0.0053    0.9  -0.067 0.0776  False\n",
            "    E2     E6   0.0016    0.9 -0.0707 0.0738  False\n",
            "    E3     E4   0.0167    0.9 -0.0555  0.089  False\n",
            "    E3     E5  -0.0425 0.5119 -0.1148 0.0298  False\n",
            "    E3     E6  -0.0462 0.4213 -0.1185 0.0261  False\n",
            "    E4     E5  -0.0593 0.1673 -0.1315  0.013  False\n",
            "    E4     E6   -0.063 0.1218 -0.1353 0.0093  False\n",
            "    E5     E6  -0.0037    0.9  -0.076 0.0686  False\n",
            "---------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for CSP \n",
            "\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0295 0.0919 -0.0618 0.0028  False\n",
            "    E1     E3   0.0045    0.9 -0.0278 0.0368  False\n",
            "    E1     E4  -0.0198   0.47 -0.0521 0.0125  False\n",
            "    E1     E5   -0.009    0.9 -0.0413 0.0233  False\n",
            "    E1     E6  -0.0045    0.9 -0.0368 0.0277  False\n",
            "    E2     E3    0.034 0.0333  0.0017 0.0663   True\n",
            "    E2     E4   0.0097    0.9 -0.0226  0.042  False\n",
            "    E2     E5   0.0205 0.4316 -0.0118 0.0528  False\n",
            "    E2     E6    0.025 0.2187 -0.0073 0.0573  False\n",
            "    E3     E4  -0.0243 0.2441 -0.0566  0.008  False\n",
            "    E3     E5  -0.0136 0.7912 -0.0459 0.0187  False\n",
            "    E3     E6  -0.0091    0.9 -0.0414 0.0232  False\n",
            "    E4     E5   0.0107    0.9 -0.0216  0.043  False\n",
            "    E4     E6   0.0152 0.7055 -0.0171 0.0475  False\n",
            "    E5     E6   0.0045    0.9 -0.0278 0.0368  False\n",
            "---------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n",
            "Table for IIR \n",
            "\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0149    0.9 -0.1692 0.1395  False\n",
            "    E1     E3  -0.0692 0.7445 -0.2235 0.0852  False\n",
            "    E1     E4  -0.0269    0.9 -0.1813 0.1274  False\n",
            "    E1     E5   -0.024    0.9 -0.1784 0.1303  False\n",
            "    E1     E6  -0.0868 0.5547 -0.2411 0.0676  False\n",
            "    E2     E3  -0.0543    0.9 -0.2086 0.1001  False\n",
            "    E2     E4   -0.012    0.9 -0.1664 0.1423  False\n",
            "    E2     E5  -0.0091    0.9 -0.1635 0.1452  False\n",
            "    E2     E6  -0.0719  0.715 -0.2263 0.0824  False\n",
            "    E3     E4   0.0423    0.9 -0.1121 0.1966  False\n",
            "    E3     E5   0.0451    0.9 -0.1092 0.1995  False\n",
            "    E3     E6  -0.0176    0.9  -0.172 0.1367  False\n",
            "    E4     E5   0.0029    0.9 -0.1515 0.1572  False\n",
            "    E4     E6  -0.0599 0.8446 -0.2142 0.0945  False\n",
            "    E5     E6  -0.0628 0.8135 -0.2171 0.0916  False\n",
            "---------------------------------------------------\n",
            "\n",
            "\n",
            "##########\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eAHxYOpsm0ge",
        "outputId": "608a1bef-1856-4ff3-e698-4f689363514b"
      },
      "source": [
        "# conf mat T-tests\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "for emot in emotions:\n",
        "  print(10*\"*\", emot, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "  for j in features:\n",
        "    print(\"\\n-----\", j, \"----\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(j)+'/CNN_1D_results_PDNC_'+str(j)+'_'+str(emot)+'.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    oneD_conf_nc = [i[0][0] for i in oneD_conf]\n",
        "    oneD_conf_pd = [i[1][1] for i in oneD_conf]\n",
        "    stat, p = ttest_ind(oneD_conf_nc, oneD_conf_pd,equal_var=False)\n",
        "    print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "    if p > 0.05:\n",
        "      print('Probably the same distribution')\n",
        "    else:\n",
        "      print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** E1 **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.9785124  0.0214876 ]\n",
            " [0.04583333 0.95416667]] \n",
            "\n",
            "stat=1.357, p=0.197\n",
            "Probably the same distribution\n",
            "\n",
            "----- CSP ----\n",
            "[[0.92090909 0.07909091]\n",
            " [0.03190476 0.96809524]] \n",
            "\n",
            "stat=-3.029, p=0.011\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.94545532 0.05454468]\n",
            " [0.02242991 0.97757009]] \n",
            "\n",
            "stat=-0.811, p=0.436\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E2 **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.92712573 0.07287427]\n",
            " [0.09606299 0.90393701]] \n",
            "\n",
            "stat=0.374, p=0.713\n",
            "Probably the same distribution\n",
            "\n",
            "----- CSP ----\n",
            "[[0.93564444 0.06435556]\n",
            " [0.10575484 0.89424516]] \n",
            "\n",
            "stat=2.106, p=0.050\n",
            "Probably the same distribution\n",
            "\n",
            "----- IIR ----\n",
            "[[0.97631579 0.02368421]\n",
            " [0.07876896 0.92123104]] \n",
            "\n",
            "stat=0.918, p=0.380\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E3 **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.965     0.035    ]\n",
            " [0.0432438 0.9567562]] \n",
            "\n",
            "stat=0.454, p=0.657\n",
            "Probably the same distribution\n",
            "\n",
            "----- CSP ----\n",
            "[[0.94156815 0.05843185]\n",
            " [0.04381142 0.95618858]] \n",
            "\n",
            "stat=-0.856, p=0.403\n",
            "Probably the same distribution\n",
            "\n",
            "----- IIR ----\n",
            "[[0.85000882 0.14999118]\n",
            " [0.04676424 0.95323576]] \n",
            "\n",
            "stat=-1.072, p=0.307\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E4 **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.97167355 0.02832645]\n",
            " [0.01655647 0.98344353]] \n",
            "\n",
            "stat=-0.715, p=0.487\n",
            "Probably the same distribution\n",
            "\n",
            "----- CSP ----\n",
            "[[0.93081933 0.06918067]\n",
            " [0.08152204 0.91847796]] \n",
            "\n",
            "stat=0.681, p=0.505\n",
            "Probably the same distribution\n",
            "\n",
            "----- IIR ----\n",
            "[[0.89856352 0.10143648]\n",
            " [0.02592593 0.97407407]] \n",
            "\n",
            "stat=-1.228, p=0.248\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E5 **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.92276129 0.07723871]\n",
            " [0.08594283 0.91405717]] \n",
            "\n",
            "stat=0.407, p=0.689\n",
            "Probably the same distribution\n",
            "\n",
            "----- CSP ----\n",
            "[[0.91624516 0.08375484]\n",
            " [0.04546289 0.95453711]] \n",
            "\n",
            "stat=-2.859, p=0.011\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.93090234 0.06909766]\n",
            " [0.05019305 0.94980695]] \n",
            "\n",
            "stat=-0.282, p=0.782\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E6 **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.93172698 0.06827302]\n",
            " [0.09846894 0.90153106]] \n",
            "\n",
            "stat=0.507, p=0.618\n",
            "Probably the same distribution\n",
            "\n",
            "----- CSP ----\n",
            "[[0.93617231 0.06382769]\n",
            " [0.0564     0.9436    ]] \n",
            "\n",
            "stat=-0.345, p=0.734\n",
            "Probably the same distribution\n",
            "\n",
            "----- IIR ----\n",
            "[[0.90803571 0.09196429]\n",
            " [0.14035864 0.85964136]] \n",
            "\n",
            "stat=0.482, p=0.636\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H5_S3iiwS3T-",
        "outputId": "9cb75cb0-fc24-4294-ed59-a564ce1d693c"
      },
      "source": [
        "# How well are PD and NC classified on different features?\n",
        "\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "for emot in emotions:\n",
        "  print(10*\"*\", emot, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "  data = [\"NC\", \"PD\"]\n",
        "\n",
        "  l = 0\n",
        "  for i in data:\n",
        "    print(\"\\nHow well is\", i, \"classified in different features for\", emot, \"data?\\n\")\n",
        "    for k in features:\n",
        "      print(10*\"-\", k, 10*\"-\")\n",
        "      oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(k)+'/CNN_1D_results_PDNC_'+str(k)+'_'+str(emot)+'.mat', 'conf_mat_list')\n",
        "      print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Features': len(oneD_conf)*[str(k)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Features)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "\n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Features'])\n",
        "    print(oneD_mc.tukeyhsd())\n",
        "    l = l+1\n",
        "    print(30*\"*\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** E1 **********\n",
            "\n",
            "How well is NC classified in different features for E1 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.9785124  0.0214876 ]\n",
            " [0.04583333 0.95416667]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.92090909 0.07909091]\n",
            " [0.03190476 0.96809524]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.94545532 0.05454468]\n",
            " [0.02242991 0.97757009]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F   PR(>F)\n",
            "C(Features)  0.016711   2.0  1.442482  0.25398\n",
            "Residual     0.156400  27.0       NaN      NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0245 0.7386 -0.0598 0.1089  False\n",
            "   CSP Spectral   0.0576 0.2265 -0.0268  0.142  False\n",
            "   IIR Spectral   0.0331 0.5959 -0.0513 0.1174  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different features for E1 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.9785124  0.0214876 ]\n",
            " [0.04583333 0.95416667]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.92090909 0.07909091]\n",
            " [0.03190476 0.96809524]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.94545532 0.05454468]\n",
            " [0.02242991 0.97757009]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.005264   2.0  0.710081  0.495901\n",
            "Residual     0.211258  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR    0.017 0.6409 -0.0293 0.0633  False\n",
            "   CSP Spectral   0.0218 0.4985 -0.0245 0.0682  False\n",
            "   IIR Spectral   0.0048    0.9 -0.0415 0.0512  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** E2 **********\n",
            "\n",
            "How well is NC classified in different features for E2 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.92712573 0.07287427]\n",
            " [0.09606299 0.90393701]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.93564444 0.06435556]\n",
            " [0.10575484 0.89424516]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.97631579 0.02368421]\n",
            " [0.07876896 0.92123104]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.013821   2.0  0.914841  0.412642\n",
            "Residual     0.203956  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0407  0.553 -0.0557  0.137  False\n",
            "   CSP Spectral  -0.0085    0.9 -0.1049 0.0878  False\n",
            "   IIR Spectral  -0.0492 0.4279 -0.1455 0.0472  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different features for E2 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.92712573 0.07287427]\n",
            " [0.09606299 0.90393701]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.93564444 0.06435556]\n",
            " [0.10575484 0.89424516]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.97631579 0.02368421]\n",
            " [0.07876896 0.92123104]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.014998   2.0  0.584966  0.560434\n",
            "Residual     0.730731  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0338 0.6062 -0.0523   0.12  False\n",
            "   CSP Spectral   0.0006    0.9 -0.0856 0.0867  False\n",
            "   IIR Spectral  -0.0332 0.6155 -0.1194 0.0529  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** E3 **********\n",
            "\n",
            "How well is NC classified in different features for E3 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.965     0.035    ]\n",
            " [0.0432438 0.9567562]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.94156815 0.05843185]\n",
            " [0.04381142 0.95618858]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.85000882 0.14999118]\n",
            " [0.04676424 0.95323576]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "              sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.07385   2.0  1.271837  0.296579\n",
            "Residual     0.78389  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0916 0.4641 -0.2805 0.0973  False\n",
            "   CSP Spectral   0.0234    0.9 -0.1655 0.2123  False\n",
            "   IIR Spectral    0.115  0.303 -0.0739 0.3039  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different features for E3 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.965     0.035    ]\n",
            " [0.0432438 0.9567562]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.94156815 0.05843185]\n",
            " [0.04381142 0.95618858]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.85000882 0.14999118]\n",
            " [0.04676424 0.95323576]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.039256   2.0  1.184206  0.313403\n",
            "Residual     0.944762  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0473  0.483 -0.1452 0.0507  False\n",
            "   CSP Spectral    0.012    0.9  -0.086   0.11  False\n",
            "   IIR Spectral   0.0593 0.3204 -0.0387 0.1572  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** E4 **********\n",
            "\n",
            "How well is NC classified in different features for E4 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.97167355 0.02832645]\n",
            " [0.01655647 0.98344353]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.93081933 0.06918067]\n",
            " [0.08152204 0.91847796]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.89856352 0.10143648]\n",
            " [0.02592593 0.97407407]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.026849   2.0  1.028604  0.371094\n",
            "Residual     0.352377  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0323 0.7898 -0.1589 0.0944  False\n",
            "   CSP Spectral   0.0409 0.6938 -0.0858 0.1675  False\n",
            "   IIR Spectral   0.0731 0.3402 -0.0535 0.1998  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different features for E4 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.97167355 0.02832645]\n",
            " [0.01655647 0.98344353]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.93081933 0.06918067]\n",
            " [0.08152204 0.91847796]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.89856352 0.10143648]\n",
            " [0.02592593 0.97407407]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.030909   2.0  2.075606  0.134862\n",
            "Residual     0.424411  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0117 0.8997  -0.054 0.0773  False\n",
            "   CSP Spectral   0.0529 0.1371 -0.0128 0.1186  False\n",
            "   IIR Spectral   0.0412 0.2937 -0.0244 0.1069  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** E5 **********\n",
            "\n",
            "How well is NC classified in different features for E5 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.92276129 0.07723871]\n",
            " [0.08594283 0.91405717]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.91624516 0.08375484]\n",
            " [0.04546289 0.95453711]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.93090234 0.06909766]\n",
            " [0.05019305 0.94980695]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.001079   2.0  0.040719  0.960157\n",
            "Residual     0.357584  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1  group2  meandiff p-adj  lower  upper  reject\n",
            "----------------------------------------------------\n",
            "   CSP      IIR   0.0147   0.9 -0.1129 0.1422  False\n",
            "   CSP Spectral   0.0065   0.9 -0.1211 0.1341  False\n",
            "   IIR Spectral  -0.0081   0.9 -0.1357 0.1194  False\n",
            "----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different features for E5 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.92276129 0.07723871]\n",
            " [0.08594283 0.91405717]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.91624516 0.08375484]\n",
            " [0.04546289 0.95453711]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.93090234 0.06909766]\n",
            " [0.05019305 0.94980695]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.005297   2.0  0.320199  0.727302\n",
            "Residual     0.471514  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR    0.005    0.9 -0.0642 0.0742  False\n",
            "   CSP Spectral   -0.017 0.8073 -0.0862 0.0522  False\n",
            "   IIR Spectral  -0.0219 0.7094 -0.0912 0.0473  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** E6 **********\n",
            "\n",
            "How well is NC classified in different features for E6 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.93172698 0.06827302]\n",
            " [0.09846894 0.90153106]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.93617231 0.06382769]\n",
            " [0.0564     0.9436    ]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.90803571 0.09196429]\n",
            " [0.14035864 0.85964136]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.004576   2.0  0.114958  0.891837\n",
            "Residual     0.537341  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0281 0.8956 -0.1845 0.1283  False\n",
            "   CSP Spectral  -0.0044    0.9 -0.1608  0.152  False\n",
            "   IIR Spectral   0.0237    0.9 -0.1327 0.1801  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different features for E6 data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.93172698 0.06827302]\n",
            " [0.09846894 0.90153106]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.93617231 0.06382769]\n",
            " [0.0564     0.9436    ]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.90803571 0.09196429]\n",
            " [0.14035864 0.85964136]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.031716   2.0  0.703399  0.499145\n",
            "Residual     1.285067  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   -0.056 0.4714 -0.1703 0.0582  False\n",
            "   CSP Spectral  -0.0233 0.8645 -0.1375  0.091  False\n",
            "   IIR Spectral   0.0328 0.7505 -0.0815  0.147  False\n",
            "-----------------------------------------------------\n",
            "******************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2uOHBVjScJYH",
        "outputId": "a7052eb0-32c1-47d0-e492-8ac0d08e70af"
      },
      "source": [
        "# How well are PD and NC classified on different data?\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "for k in features:\n",
        "  print(10*\"*\", k, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  data = [\"NC\", \"PD\"]\n",
        "  emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "  l = 0\n",
        "  for i in data:\n",
        "    print(\"\\nHow well is\", i, \"classified in different data for\", k, \"features?\\n\")\n",
        "    for emot in emotions:\n",
        "      print(10*\"-\", emot, 10*\"-\")\n",
        "      oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(k)+'/CNN_1D_results_PDNC_'+str(k)+'_'+str(emot)+'.mat', 'conf_mat_list')\n",
        "      print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Emotions': len(oneD_conf)*[str(emot)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Emotions)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "\n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Emotions'])\n",
        "    print(oneD_mc.tukeyhsd())\n",
        "    l = l+1\n",
        "    print(30*\"*\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Spectral **********\n",
            "\n",
            "How well is NC classified in different data for Spectral features?\n",
            "\n",
            "---------- E1 ----------\n",
            "[[0.9785124  0.0214876 ]\n",
            " [0.04583333 0.95416667]] \n",
            "\n",
            "---------- E2 ----------\n",
            "[[0.92712573 0.07287427]\n",
            " [0.09606299 0.90393701]] \n",
            "\n",
            "---------- E3 ----------\n",
            "[[0.965     0.035    ]\n",
            " [0.0432438 0.9567562]] \n",
            "\n",
            "---------- E4 ----------\n",
            "[[0.97167355 0.02832645]\n",
            " [0.01655647 0.98344353]] \n",
            "\n",
            "---------- E5 ----------\n",
            "[[0.92276129 0.07723871]\n",
            " [0.08594283 0.91405717]] \n",
            "\n",
            "---------- E6 ----------\n",
            "[[0.93172698 0.06827302]\n",
            " [0.09846894 0.90153106]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.031051   5.0  0.961579  0.449524\n",
            "Residual     0.348747  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0514 0.6849 -0.1576 0.0548  False\n",
            "    E1     E3  -0.0135    0.9 -0.1197 0.0927  False\n",
            "    E1     E4  -0.0068    0.9  -0.113 0.0993  False\n",
            "    E1     E5  -0.0558 0.6166 -0.1619 0.0504  False\n",
            "    E1     E6  -0.0468  0.757  -0.153 0.0594  False\n",
            "    E2     E3   0.0379 0.8965 -0.0683 0.1441  False\n",
            "    E2     E4   0.0445  0.792 -0.0616 0.1507  False\n",
            "    E2     E5  -0.0044    0.9 -0.1105 0.1018  False\n",
            "    E2     E6   0.0046    0.9 -0.1016 0.1108  False\n",
            "    E3     E4   0.0067    0.9 -0.0995 0.1129  False\n",
            "    E3     E5  -0.0422 0.8282 -0.1484 0.0639  False\n",
            "    E3     E6  -0.0333    0.9 -0.1395 0.0729  False\n",
            "    E4     E5  -0.0489 0.7237 -0.1551 0.0573  False\n",
            "    E4     E6  -0.0399 0.8641 -0.1461 0.0662  False\n",
            "    E5     E6    0.009    0.9 -0.0972 0.1151  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different data for Spectral features?\n",
            "\n",
            "---------- E1 ----------\n",
            "[[0.9785124  0.0214876 ]\n",
            " [0.04583333 0.95416667]] \n",
            "\n",
            "---------- E2 ----------\n",
            "[[0.92712573 0.07287427]\n",
            " [0.09606299 0.90393701]] \n",
            "\n",
            "---------- E3 ----------\n",
            "[[0.965     0.035    ]\n",
            " [0.0432438 0.9567562]] \n",
            "\n",
            "---------- E4 ----------\n",
            "[[0.97167355 0.02832645]\n",
            " [0.01655647 0.98344353]] \n",
            "\n",
            "---------- E5 ----------\n",
            "[[0.92276129 0.07723871]\n",
            " [0.08594283 0.91405717]] \n",
            "\n",
            "---------- E6 ----------\n",
            "[[0.93172698 0.06827302]\n",
            " [0.09846894 0.90153106]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.082243    5.0  2.343076  0.045808\n",
            "Residual     0.800288  114.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0508 0.3985 -0.1276  0.026  False\n",
            "    E1     E3  -0.0055    0.9 -0.0823 0.0713  False\n",
            "    E1     E4   0.0112    0.9 -0.0656  0.088  False\n",
            "    E1     E5  -0.0479 0.4655 -0.1247 0.0289  False\n",
            "    E1     E6  -0.0497  0.424 -0.1265 0.0271  False\n",
            "    E2     E3   0.0453 0.5232 -0.0315 0.1222  False\n",
            "    E2     E4    0.062 0.1864 -0.0148 0.1388  False\n",
            "    E2     E5   0.0029    0.9 -0.0739 0.0797  False\n",
            "    E2     E6   0.0011    0.9 -0.0757 0.0779  False\n",
            "    E3     E4   0.0167    0.9 -0.0601 0.0935  False\n",
            "    E3     E5  -0.0425 0.5852 -0.1193 0.0343  False\n",
            "    E3     E6  -0.0442 0.5469 -0.1211 0.0326  False\n",
            "    E4     E5  -0.0591 0.2313  -0.136 0.0177  False\n",
            "    E4     E6  -0.0609 0.2027 -0.1377 0.0159  False\n",
            "    E5     E6  -0.0018    0.9 -0.0786  0.075  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "********** CSP **********\n",
            "\n",
            "How well is NC classified in different data for CSP features?\n",
            "\n",
            "---------- E1 ----------\n",
            "[[0.92090909 0.07909091]\n",
            " [0.03190476 0.96809524]] \n",
            "\n",
            "---------- E2 ----------\n",
            "[[0.93564444 0.06435556]\n",
            " [0.10575484 0.89424516]] \n",
            "\n",
            "---------- E3 ----------\n",
            "[[0.94156815 0.05843185]\n",
            " [0.04381142 0.95618858]] \n",
            "\n",
            "---------- E4 ----------\n",
            "[[0.93081933 0.06918067]\n",
            " [0.08152204 0.91847796]] \n",
            "\n",
            "---------- E5 ----------\n",
            "[[0.91624516 0.08375484]\n",
            " [0.04546289 0.95453711]] \n",
            "\n",
            "---------- E6 ----------\n",
            "[[0.93617231 0.06382769]\n",
            " [0.0564     0.9436    ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.004760   5.0  0.625819  0.680721\n",
            "Residual     0.082142  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2   0.0147    0.9 -0.0368 0.0663  False\n",
            "    E1     E3   0.0207  0.823 -0.0309 0.0722  False\n",
            "    E1     E4   0.0099    0.9 -0.0416 0.0614  False\n",
            "    E1     E5  -0.0047    0.9 -0.0562 0.0469  False\n",
            "    E1     E6   0.0153    0.9 -0.0363 0.0668  False\n",
            "    E2     E3   0.0059    0.9 -0.0456 0.0575  False\n",
            "    E2     E4  -0.0048    0.9 -0.0564 0.0467  False\n",
            "    E2     E5  -0.0194 0.8637 -0.0709 0.0321  False\n",
            "    E2     E6   0.0005    0.9  -0.051 0.0521  False\n",
            "    E3     E4  -0.0107    0.9 -0.0623 0.0408  False\n",
            "    E3     E5  -0.0253 0.6725 -0.0769 0.0262  False\n",
            "    E3     E6  -0.0054    0.9 -0.0569 0.0461  False\n",
            "    E4     E5  -0.0146    0.9 -0.0661  0.037  False\n",
            "    E4     E6   0.0054    0.9 -0.0462 0.0569  False\n",
            "    E5     E6   0.0199 0.8466 -0.0316 0.0715  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different data for CSP features?\n",
            "\n",
            "---------- E1 ----------\n",
            "[[0.92090909 0.07909091]\n",
            " [0.03190476 0.96809524]] \n",
            "\n",
            "---------- E2 ----------\n",
            "[[0.93564444 0.06435556]\n",
            " [0.10575484 0.89424516]] \n",
            "\n",
            "---------- E3 ----------\n",
            "[[0.94156815 0.05843185]\n",
            " [0.04381142 0.95618858]] \n",
            "\n",
            "---------- E4 ----------\n",
            "[[0.93081933 0.06918067]\n",
            " [0.08152204 0.91847796]] \n",
            "\n",
            "---------- E5 ----------\n",
            "[[0.91624516 0.08375484]\n",
            " [0.04546289 0.95453711]] \n",
            "\n",
            "---------- E6 ----------\n",
            "[[0.93617231 0.06382769]\n",
            " [0.0564     0.9436    ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df        F    PR(>F)\n",
            "C(Emotions)  0.016316    5.0  1.86674  0.105608\n",
            "Residual     0.199275  114.0      NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0296 0.2299 -0.0679 0.0088  False\n",
            "    E1     E3   0.0044    0.9 -0.0339 0.0427  False\n",
            "    E1     E4  -0.0199 0.6429 -0.0582 0.0185  False\n",
            "    E1     E5  -0.0091    0.9 -0.0474 0.0292  False\n",
            "    E1     E6  -0.0046    0.9 -0.0429 0.0337  False\n",
            "    E2     E3   0.0339 0.1144 -0.0044 0.0723  False\n",
            "    E2     E4   0.0097    0.9 -0.0286  0.048  False\n",
            "    E2     E5   0.0204 0.6173 -0.0179 0.0588  False\n",
            "    E2     E6   0.0249 0.4176 -0.0134 0.0633  False\n",
            "    E3     E4  -0.0242  0.451 -0.0626 0.0141  False\n",
            "    E3     E5  -0.0135    0.9 -0.0518 0.0248  False\n",
            "    E3     E6   -0.009    0.9 -0.0473 0.0293  False\n",
            "    E4     E5   0.0107    0.9 -0.0276 0.0491  False\n",
            "    E4     E6   0.0152  0.842 -0.0231 0.0536  False\n",
            "    E5     E6   0.0045    0.9 -0.0338 0.0428  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "********** IIR **********\n",
            "\n",
            "How well is NC classified in different data for IIR features?\n",
            "\n",
            "---------- E1 ----------\n",
            "[[0.94545532 0.05454468]\n",
            " [0.02242991 0.97757009]] \n",
            "\n",
            "---------- E2 ----------\n",
            "[[0.97631579 0.02368421]\n",
            " [0.07876896 0.92123104]] \n",
            "\n",
            "---------- E3 ----------\n",
            "[[0.85000882 0.14999118]\n",
            " [0.04676424 0.95323576]] \n",
            "\n",
            "---------- E4 ----------\n",
            "[[0.89856352 0.10143648]\n",
            " [0.02592593 0.97407407]] \n",
            "\n",
            "---------- E5 ----------\n",
            "[[0.93090234 0.06909766]\n",
            " [0.05019305 0.94980695]] \n",
            "\n",
            "---------- E6 ----------\n",
            "[[0.90803571 0.09196429]\n",
            " [0.14035864 0.85964136]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.094206   5.0  0.518919  0.760823\n",
            "Residual     1.960659  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2   0.0309    0.9 -0.2209 0.2826  False\n",
            "    E1     E3  -0.0954 0.8592 -0.3472 0.1563  False\n",
            "    E1     E4  -0.0469    0.9 -0.2987 0.2049  False\n",
            "    E1     E5  -0.0146    0.9 -0.2663 0.2372  False\n",
            "    E1     E6  -0.0374    0.9 -0.2892 0.2143  False\n",
            "    E2     E3  -0.1263 0.6555 -0.3781 0.1255  False\n",
            "    E2     E4  -0.0778    0.9 -0.3295  0.174  False\n",
            "    E2     E5  -0.0454    0.9 -0.2972 0.2064  False\n",
            "    E2     E6  -0.0683    0.9   -0.32 0.1835  False\n",
            "    E3     E4   0.0486    0.9 -0.2032 0.3003  False\n",
            "    E3     E5   0.0809    0.9 -0.1709 0.3327  False\n",
            "    E3     E6    0.058    0.9 -0.1937 0.3098  False\n",
            "    E4     E5   0.0323    0.9 -0.2194 0.2841  False\n",
            "    E4     E6   0.0095    0.9 -0.2423 0.2612  False\n",
            "    E5     E6  -0.0229    0.9 -0.2746 0.2289  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is PD classified in different data for IIR features?\n",
            "\n",
            "---------- E1 ----------\n",
            "[[0.94545532 0.05454468]\n",
            " [0.02242991 0.97757009]] \n",
            "\n",
            "---------- E2 ----------\n",
            "[[0.97631579 0.02368421]\n",
            " [0.07876896 0.92123104]] \n",
            "\n",
            "---------- E3 ----------\n",
            "[[0.85000882 0.14999118]\n",
            " [0.04676424 0.95323576]] \n",
            "\n",
            "---------- E4 ----------\n",
            "[[0.89856352 0.10143648]\n",
            " [0.02592593 0.97407407]] \n",
            "\n",
            "---------- E5 ----------\n",
            "[[0.93090234 0.06909766]\n",
            " [0.05019305 0.94980695]] \n",
            "\n",
            "---------- E6 ----------\n",
            "[[0.90803571 0.09196429]\n",
            " [0.14035864 0.85964136]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.088385    5.0  0.656798  0.656927\n",
            "Residual     3.068179  114.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0127    0.9 -0.1631 0.1376  False\n",
            "    E1     E3  -0.0599 0.8409 -0.2103 0.0905  False\n",
            "    E1     E4  -0.0252    0.9 -0.1756 0.1252  False\n",
            "    E1     E5  -0.0212    0.9 -0.1715 0.1292  False\n",
            "    E1     E6  -0.0777 0.6454 -0.2281 0.0727  False\n",
            "    E2     E3  -0.0472    0.9 -0.1975 0.1032  False\n",
            "    E2     E4  -0.0125    0.9 -0.1628 0.1379  False\n",
            "    E2     E5  -0.0084    0.9 -0.1588  0.142  False\n",
            "    E2     E6  -0.0649 0.7854 -0.2153 0.0854  False\n",
            "    E3     E4   0.0347    0.9 -0.1157 0.1851  False\n",
            "    E3     E5   0.0387    0.9 -0.1117 0.1891  False\n",
            "    E3     E6  -0.0178    0.9 -0.1682 0.1326  False\n",
            "    E4     E5    0.004    0.9 -0.1463 0.1544  False\n",
            "    E4     E6  -0.0525    0.9 -0.2029 0.0979  False\n",
            "    E5     E6  -0.0565  0.878 -0.2069 0.0939  False\n",
            "---------------------------------------------------\n",
            "******************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PSxUz9CGgcxw"
      },
      "source": [
        "## HV vs LV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mRt2hwE1ghCk",
        "outputId": "14993258-1805-492a-d216-9ada488a98c5"
      },
      "source": [
        "# Create a DataFrame\n",
        "oneD_hvlv_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "for i in features:\n",
        "  for j in Data:\n",
        "    oneD_hvlv_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(i)+'/CNN_1D_results_HVLV_'+str(i)+'_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "    data_df = pd.DataFrame({'f1_weightedNet': oneD_hvlv_f1[0],'Features': len(oneD_hvlv_f1[0])*[str(i)], 'Data': len(oneD_hvlv_f1[0])*[str(j)]})\n",
        "    oneD_hvlv_df = oneD_hvlv_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(oneD_hvlv_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Features  Data\n",
            "0         0.750797  Spectral  Full\n",
            "1         0.788473  Spectral  Full\n",
            "2         0.828239  Spectral  Full\n",
            "3         0.819656  Spectral  Full\n",
            "4         0.872778  Spectral  Full\n",
            "..             ...       ...   ...\n",
            "85        0.962180       IIR    NC\n",
            "86        0.933173       IIR    NC\n",
            "87        0.963344       IIR    NC\n",
            "88        0.961745       IIR    NC\n",
            "89        0.958404       IIR    NC\n",
            "\n",
            "[90 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EYuVtF3ohzfu",
        "outputId": "81f5f619-950b-47f5-b688-1d9495602fd9"
      },
      "source": [
        "# FULL DATA\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_hvlv_full_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_hvlv_full_model = ols(oneD_hvlv_full_formula, oneD_hvlv_df[oneD_hvlv_df['Data']=='Full']).fit()\n",
        "oneD_hvlv_aov_table = anova_lm(oneD_hvlv_full_model, typ=2)\n",
        "print(oneD_hvlv_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F   PR(>F)\n",
            "C(Features)  0.021014   2.0  1.833908  0.17914\n",
            "Residual     0.154688  27.0       NaN      NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jiDAuirvjowQ",
        "outputId": "af484aa6-665f-4ed3-a643-212bda136b2f"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "oneD_hvlv_full_mc = MultiComparison(oneD_hvlv_df[oneD_hvlv_df['Data']=='Full']['f1_weightedNet'], oneD_hvlv_df[oneD_hvlv_df['Data']=='Full']['Features'])\n",
        "print(oneD_hvlv_full_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0647 0.1546 -0.0192 0.1486  False\n",
            "   CSP Spectral   0.0354 0.5539 -0.0486 0.1193  False\n",
            "   IIR Spectral  -0.0294 0.6549 -0.1133 0.0545  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G7QZdZdAoI1j",
        "outputId": "47a08260-24c6-407b-ca11-c2c9f87b535f"
      },
      "source": [
        "# PD DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_hvlv_pd_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_hvlv_pd_model = ols(oneD_hvlv_pd_formula, oneD_hvlv_df[oneD_hvlv_df['Data']=='PD']).fit()\n",
        "oneD_hvlv_pd_aov_table = anova_lm(oneD_hvlv_pd_model, typ=2)\n",
        "print(oneD_hvlv_pd_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.057938   2.0  1.996803  0.155324\n",
            "Residual     0.391710  27.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IrC-MlLvo-VT",
        "outputId": "1a777f32-82d9-415b-cdbc-947fbf67b499"
      },
      "source": [
        "# Comparison of means using Tukey HSD for PD data\n",
        "\n",
        "oneD_hvlv_pd_mc = MultiComparison(oneD_hvlv_df[oneD_hvlv_df['Data']=='PD']['f1_weightedNet'], oneD_hvlv_df[oneD_hvlv_df['Data']=='PD']['Features'])\n",
        "print(oneD_hvlv_pd_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.1056 0.1415 -0.2391 0.0279  False\n",
            "   CSP Spectral  -0.0347 0.7824 -0.1682 0.0988  False\n",
            "   IIR Spectral   0.0709 0.4001 -0.0626 0.2044  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cm259nSVpTuz",
        "outputId": "154a6032-f81a-46f2-e155-feba8603a716"
      },
      "source": [
        "# NC DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_hvlv_nc_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_hvlv_nc_model = ols(oneD_hvlv_nc_formula, oneD_hvlv_df[oneD_hvlv_df['Data']=='NC']).fit()\n",
        "oneD_hvlv_nc_aov_table = anova_lm(oneD_hvlv_nc_model, typ=2)\n",
        "print(oneD_hvlv_nc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.012883   2.0  0.883984  0.424754\n",
            "Residual     0.196740  27.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AnWR7eubpg3S",
        "outputId": "9212cb74-8fae-4d0d-80ec-9367463e20c0"
      },
      "source": [
        "# Comparison of means using Tukey HSD for NC data\n",
        "\n",
        "oneD_hvlv_nc_mc = MultiComparison(oneD_hvlv_df[oneD_hvlv_df['Data']=='NC']['f1_weightedNet'], oneD_hvlv_df[oneD_hvlv_df['Data']=='NC']['Features'])\n",
        "print(oneD_hvlv_nc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0144    0.9 -0.0803  0.109  False\n",
            "   CSP Spectral   -0.035 0.6271 -0.1296 0.0596  False\n",
            "   IIR Spectral  -0.0493  0.413  -0.144 0.0453  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XkiRte93qj-N",
        "outputId": "c1667b55-07da-4ac9-9f53-ca548083bb50"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api for PD and NC data\n",
        "\n",
        "oneD_hvlv_formula = 'f1_weightedNet ~ C(Features)+C(Data)+C(Features):C(Data)'\n",
        "oneD_hvlv_model = ols(oneD_hvlv_formula, oneD_hvlv_df[oneD_hvlv_df['Data'].isin(['PD', 'NC'])]).fit()\n",
        "oneD_hvlv_aov_table = anova_lm(oneD_hvlv_model, typ=2)\n",
        "print(oneD_hvlv_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                       sum_sq    df         F    PR(>F)\n",
            "C(Features)          0.022743   2.0  1.043544  0.359197\n",
            "C(Data)              0.059050   1.0  5.418796  0.023698\n",
            "C(Features):C(Data)  0.048077   2.0  2.205950  0.119976\n",
            "Residual             0.588450  54.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Ly9lfRnqw2R",
        "outputId": "d4b10dfd-224f-43e5-c617-9a06a82da390"
      },
      "source": [
        "# T-test for individual features\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "dummy_df = oneD_hvlv_df[oneD_hvlv_df['Data'].isin(['PD', 'NC'])]\n",
        "for i in features:\n",
        "  print(\"-----\", i, \"-----\")\n",
        "  dummy_df_2 = dummy_df[dummy_df['Features']==i]\n",
        "  d1=(dummy_df_2[dummy_df_2['Data']=='PD'])\n",
        "  d2=(dummy_df_2[dummy_df_2['Data']=='NC'])\n",
        "\n",
        "  stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "\t  print('Probably the same distribution')\n",
        "  else:\n",
        "\t  print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Spectral -----\n",
            "stat=-0.548, p=0.591\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "----- CSP -----\n",
            "stat=-1.544, p=0.141\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "----- IIR -----\n",
            "stat=-2.101, p=0.053\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gJpDnDT9UgnA",
        "outputId": "a08ca0d0-044f-417e-dcb3-137144d83808"
      },
      "source": [
        "# conf mat T-tests\n",
        "data = [\"Full\", \"PD\", \"NC\"]\n",
        "\n",
        "for d in data:\n",
        "  print(10*\"*\", d, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "  for j in features:\n",
        "    print(\"\\n-----\", j, \"----\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(j)+'/CNN_1D_results_HVLV_'+str(j)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    oneD_conf_nc = [i[0][0] for i in oneD_conf]\n",
        "    oneD_conf_pd = [i[1][1] for i in oneD_conf]\n",
        "    stat, p = ttest_ind(oneD_conf_nc, oneD_conf_pd,equal_var=False)\n",
        "    print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "    if p > 0.05:\n",
        "      print('Probably the same distribution')\n",
        "    else:\n",
        "      print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Full **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.93188964 0.06811036]\n",
            " [0.28571831 0.71428169]] \n",
            "\n",
            "stat=5.422, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "----- CSP ----\n",
            "[[0.91204744 0.08795256]\n",
            " [0.3470833  0.6529167 ]] \n",
            "\n",
            "stat=7.215, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.9546936  0.0453064 ]\n",
            " [0.23854195 0.76145805]] \n",
            "\n",
            "stat=2.524, p=0.031\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "********** PD **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.90406891 0.09593109]\n",
            " [0.30239044 0.69760956]] \n",
            "\n",
            "stat=2.702, p=0.022\n",
            "Probably different distributions\n",
            "\n",
            "----- CSP ----\n",
            "[[0.92287143 0.07712857]\n",
            " [0.25775597 0.74224403]] \n",
            "\n",
            "stat=8.645, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.93804967 0.06195033]\n",
            " [0.50136331 0.49863669]] \n",
            "\n",
            "stat=3.551, p=0.005\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "********** NC **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.92444289 0.07555711]\n",
            " [0.27896403 0.72103597]] \n",
            "\n",
            "stat=3.027, p=0.012\n",
            "Probably different distributions\n",
            "\n",
            "----- CSP ----\n",
            "[[0.91679075 0.08320925]\n",
            " [0.1824     0.8176    ]] \n",
            "\n",
            "stat=3.725, p=0.003\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.95921606 0.04078394]\n",
            " [0.21238938 0.78761062]] \n",
            "\n",
            "stat=2.466, p=0.031\n",
            "Probably different distributions\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MXRHDYQcVF8Z",
        "outputId": "b85ec42e-9c72-4522-fb15-a3cfb6078ff5"
      },
      "source": [
        "# conf mat T-tests\n",
        "data1 = [\"Full\", \"PD\", \"NC\"]\n",
        "\n",
        "for d in data1:\n",
        "  print(10*\"*\", d, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "  data = [\"LV\", \"HV\"]\n",
        "\n",
        "  l = 0\n",
        "  for i in data:\n",
        "    print(\"\\nHow well is\", i, \"classified in different features for\", d, \"data?\\n\")\n",
        "    for k in features:\n",
        "      print(10*\"-\", k, 10*\"-\")\n",
        "      oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(k)+'/CNN_1D_results_HVLV_'+str(k)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "      print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Features': len(oneD_conf)*[str(k)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Features)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "\n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Features'])\n",
        "    print(oneD_mc.tukeyhsd())\n",
        "    l = l+1\n",
        "    print(30*\"*\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Full **********\n",
            "\n",
            "How well is LV classified in different features for Full data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.93188964 0.06811036]\n",
            " [0.28571831 0.71428169]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.91204744 0.08795256]\n",
            " [0.3470833  0.6529167 ]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.9546936  0.0453064 ]\n",
            " [0.23854195 0.76145805]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.009108   2.0  3.564009  0.042307\n",
            "Residual     0.034500  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0426  0.033   0.003 0.0823   True\n",
            "   CSP Spectral   0.0198 0.4416 -0.0198 0.0595  False\n",
            "   IIR Spectral  -0.0228 0.3424 -0.0624 0.0168  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HV classified in different features for Full data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.93188964 0.06811036]\n",
            " [0.28571831 0.71428169]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.91204744 0.08795256]\n",
            " [0.3470833  0.6529167 ]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.9546936  0.0453064 ]\n",
            " [0.23854195 0.76145805]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.057249   2.0  1.054225  0.355157\n",
            "Residual     1.547680  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0756 0.3227 -0.0498  0.201  False\n",
            "   CSP Spectral   0.0406 0.7002 -0.0848  0.166  False\n",
            "   IIR Spectral   -0.035 0.7613 -0.1604 0.0904  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** PD **********\n",
            "\n",
            "How well is LV classified in different features for PD data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.90406891 0.09593109]\n",
            " [0.30239044 0.69760956]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.92287143 0.07712857]\n",
            " [0.25775597 0.74224403]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.93804967 0.06195033]\n",
            " [0.50136331 0.49863669]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.005795   2.0  0.593865  0.559242\n",
            "Residual     0.131743  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0152 0.8728 -0.0623 0.0926  False\n",
            "   CSP Spectral  -0.0188 0.8066 -0.0962 0.0586  False\n",
            "   IIR Spectral   -0.034 0.5293 -0.1114 0.0435  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HV classified in different features for PD data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.90406891 0.09593109]\n",
            " [0.30239044 0.69760956]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.92287143 0.07712857]\n",
            " [0.25775597 0.74224403]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.93804967 0.06195033]\n",
            " [0.50136331 0.49863669]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.139044   2.0  1.206316  0.306823\n",
            "Residual     3.285010  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.1142 0.2969 -0.2969 0.0685  False\n",
            "   CSP Spectral  -0.0317    0.9 -0.2144  0.151  False\n",
            "   IIR Spectral   0.0825 0.5256 -0.1002 0.2652  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** NC **********\n",
            "\n",
            "How well is LV classified in different features for NC data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.92444289 0.07555711]\n",
            " [0.27896403 0.72103597]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.91679075 0.08320925]\n",
            " [0.1824     0.8176    ]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.95921606 0.04078394]\n",
            " [0.21238938 0.78761062]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.010225   2.0  1.546997  0.231171\n",
            "Residual     0.089233  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0424 0.2427 -0.0213 0.1062  False\n",
            "   CSP Spectral   0.0077    0.9 -0.0561 0.0714  False\n",
            "   IIR Spectral  -0.0348 0.3805 -0.0985  0.029  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HV classified in different features for NC data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.92444289 0.07555711]\n",
            " [0.27896403 0.72103597]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.91679075 0.08320925]\n",
            " [0.1824     0.8176    ]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.95921606 0.04078394]\n",
            " [0.21238938 0.78761062]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.030552   2.0  0.665082  0.518176\n",
            "Residual     1.309221  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0062    0.9 -0.1091 0.1215  False\n",
            "   CSP Spectral  -0.0445 0.6159 -0.1598 0.0709  False\n",
            "   IIR Spectral  -0.0507 0.5423  -0.166 0.0647  False\n",
            "-----------------------------------------------------\n",
            "******************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NGIcZQWQdwSl",
        "outputId": "acafcd7d-0070-4ff9-f1f8-61b96d51fc80"
      },
      "source": [
        "# How well are PD and NC classified on different data?\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "emotions = [\"Full\", \"PD\", \"NC\"]\n",
        "for k in features:\n",
        "  print(10*\"*\", k, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  data = [\"LV\", \"HV\"]\n",
        "  emotions = [\"Full\", \"PD\", \"NC\"]\n",
        "  l = 0\n",
        "  for i in data:\n",
        "    print(\"\\nHow well is\", i, \"classified in different data for\", k, \"features?\\n\")\n",
        "    for emot in emotions:\n",
        "      print(10*\"-\", emot, 10*\"-\")\n",
        "      oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(k)+'/CNN_1D_results_HVLV_'+str(k)+'_'+str(emot)+'.mat', 'conf_mat_list')\n",
        "      print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Emotions': len(oneD_conf)*[str(emot)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Emotions)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "\n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Emotions'])\n",
        "    print(oneD_mc.tukeyhsd())\n",
        "    l = l+1\n",
        "    print(30*\"*\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Spectral **********\n",
            "\n",
            "How well is LV classified in different data for Spectral features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.93188964 0.06811036]\n",
            " [0.28571831 0.71428169]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.90406891 0.09593109]\n",
            " [0.30239044 0.69760956]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.92444289 0.07555711]\n",
            " [0.27896403 0.72103597]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.004148   2.0  0.868467  0.430988\n",
            "Residual     0.064487  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC  -0.0074    0.9 -0.0616 0.0467  False\n",
            "  Full     PD  -0.0278 0.4238  -0.082 0.0264  False\n",
            "    NC     PD  -0.0204 0.6181 -0.0746 0.0338  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HV classified in different data for Spectral features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.93188964 0.06811036]\n",
            " [0.28571831 0.71428169]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.90406891 0.09593109]\n",
            " [0.30239044 0.69760956]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.92444289 0.07555711]\n",
            " [0.27896403 0.72103597]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.006498   2.0  0.106815  0.898871\n",
            "Residual     1.733672  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "==================================================\n",
            "group1 group2 meandiff p-adj  lower  upper  reject\n",
            "--------------------------------------------------\n",
            "  Full     NC  -0.0003   0.9 -0.1331 0.1324  False\n",
            "  Full     PD  -0.0222   0.9  -0.155 0.1105  False\n",
            "    NC     PD  -0.0219   0.9 -0.1546 0.1108  False\n",
            "--------------------------------------------------\n",
            "******************************\n",
            "********** CSP **********\n",
            "\n",
            "How well is LV classified in different data for CSP features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.91204744 0.08795256]\n",
            " [0.3470833  0.6529167 ]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.92287143 0.07712857]\n",
            " [0.25775597 0.74224403]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.91679075 0.08320925]\n",
            " [0.1824     0.8176    ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df        F  PR(>F)\n",
            "C(Emotions)  0.000589   2.0  0.30366  0.7406\n",
            "Residual     0.026176  27.0      NaN     NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC   0.0047    0.9 -0.0298 0.0393  False\n",
            "  Full     PD   0.0108 0.7065 -0.0237 0.0453  False\n",
            "    NC     PD   0.0061    0.9 -0.0284 0.0406  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HV classified in different data for CSP features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.91204744 0.08795256]\n",
            " [0.3470833  0.6529167 ]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.92287143 0.07712857]\n",
            " [0.25775597 0.74224403]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.91679075 0.08320925]\n",
            " [0.1824     0.8176    ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F   PR(>F)\n",
            "C(Emotions)  0.072558   2.0  2.695112  0.07614\n",
            "Residual     0.767278  57.0       NaN      NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC   0.0847 0.0626 -0.0036  0.173  False\n",
            "  Full     PD   0.0501  0.367 -0.0382 0.1384  False\n",
            "    NC     PD  -0.0346 0.6066 -0.1229 0.0537  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "********** IIR **********\n",
            "\n",
            "How well is LV classified in different data for IIR features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.9546936  0.0453064 ]\n",
            " [0.23854195 0.76145805]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.93804967 0.06195033]\n",
            " [0.50136331 0.49863669]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.95921606 0.04078394]\n",
            " [0.21238938 0.78761062]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.002485   2.0  0.203545  0.817074\n",
            "Residual     0.164814  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC   0.0045    0.9 -0.0821 0.0911  False\n",
            "  Full     PD  -0.0166 0.8783 -0.1033   0.07  False\n",
            "    NC     PD  -0.0212 0.8044 -0.1078 0.0654  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HV classified in different data for IIR features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.9546936  0.0453064 ]\n",
            " [0.23854195 0.76145805]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.93804967 0.06195033]\n",
            " [0.50136331 0.49863669]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.95921606 0.04078394]\n",
            " [0.21238938 0.78761062]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.292048   2.0  2.286037  0.110916\n",
            "Residual     3.640960  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC   0.0153    0.9  -0.177 0.2077  False\n",
            "  Full     PD  -0.1397 0.1965 -0.3321 0.0526  False\n",
            "    NC     PD  -0.1551 0.1368 -0.3474 0.0373  False\n",
            "---------------------------------------------------\n",
            "******************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KVqiCJ4XrvrK"
      },
      "source": [
        "## HA vs LA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xocSkRHbrzWp",
        "outputId": "74f1b696-4e82-446e-92bb-7cd8b792005a"
      },
      "source": [
        "# Create a DataFrame\n",
        "oneD_hala_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "for i in features:\n",
        "  for j in Data:\n",
        "    oneD_hala_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(i)+'/CNN_1D_results_HALA_'+str(i)+'_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "    data_df = pd.DataFrame({'f1_weightedNet': oneD_hala_f1[0],'Features': len(oneD_hala_f1[0])*[str(i)], 'Data': len(oneD_hala_f1[0])*[str(j)]})\n",
        "    oneD_hala_df = oneD_hala_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(oneD_hala_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Features  Data\n",
            "0         0.867426  Spectral  Full\n",
            "1         0.919566  Spectral  Full\n",
            "2         0.940210  Spectral  Full\n",
            "3         0.942940  Spectral  Full\n",
            "4         0.963786  Spectral  Full\n",
            "..             ...       ...   ...\n",
            "85        0.979954       IIR    NC\n",
            "86        0.983239       IIR    NC\n",
            "87        0.956867       IIR    NC\n",
            "88        0.984549       IIR    NC\n",
            "89        0.987815       IIR    NC\n",
            "\n",
            "[90 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aYWlzpPWsIyr",
        "outputId": "e36cd5e9-d5c4-43d5-823f-4c609e6ff14d"
      },
      "source": [
        "# FULL DATA\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_hala_full_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_hala_full_model = ols(oneD_hala_full_formula, oneD_hala_df[oneD_hala_df['Data']=='Full']).fit()\n",
        "oneD_hala_aov_table = anova_lm(oneD_hala_full_model, typ=2)\n",
        "print(oneD_hala_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F   PR(>F)\n",
            "C(Features)  0.006881   2.0  1.595362  0.22137\n",
            "Residual     0.058227  27.0       NaN      NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k-MQd3mtsVi0",
        "outputId": "f8de56a4-57ac-4c98-e25a-169abdc356dd"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "oneD_hala_full_mc = MultiComparison(oneD_hala_df[oneD_hala_df['Data']=='Full']['f1_weightedNet'], oneD_hala_df[oneD_hala_df['Data']=='Full']['Features'])\n",
        "print(oneD_hala_full_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0333 0.2607 -0.0181 0.0848  False\n",
            "   CSP Spectral   0.0307 0.3162 -0.0207 0.0822  False\n",
            "   IIR Spectral  -0.0026    0.9 -0.0541 0.0489  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nYTniq_Wsfna",
        "outputId": "07ef2df0-4026-4c99-b951-66596697eb88"
      },
      "source": [
        "# PD DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_hala_pd_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_hala_pd_model = ols(oneD_hala_pd_formula, oneD_hala_df[oneD_hala_df['Data']=='PD']).fit()\n",
        "oneD_hala_pd_aov_table = anova_lm(oneD_hala_pd_model, typ=2)\n",
        "print(oneD_hala_pd_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.005514   2.0  1.715465  0.198909\n",
            "Residual     0.043393  27.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o39s2Ak5_Ho7",
        "outputId": "08051611-8580-4ff2-f333-f8532bf7dea8"
      },
      "source": [
        "# Comparison of means using Tukey HSD for PD data\n",
        "\n",
        "oneD_hala_pd_mc = MultiComparison(oneD_hala_df[oneD_hala_df['Data']=='PD']['f1_weightedNet'], oneD_hala_df[oneD_hala_df['Data']=='PD']['Features'])\n",
        "print(oneD_hala_pd_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0201 0.5091 -0.0243 0.0646  False\n",
            "   CSP Spectral   0.0329 0.1768 -0.0115 0.0774  False\n",
            "   IIR Spectral   0.0128 0.7428 -0.0316 0.0572  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lj5-_y7__S83",
        "outputId": "fff90836-f21d-4394-eb97-f10b723b4420"
      },
      "source": [
        "# NC DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_hala_nc_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_hala_nc_model = ols(oneD_hala_nc_formula, oneD_hala_df[oneD_hala_df['Data']=='NC']).fit()\n",
        "oneD_hala_nc_aov_table = anova_lm(oneD_hala_nc_model, typ=2)\n",
        "print(oneD_hala_nc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.007427   2.0  1.342028  0.278193\n",
            "Residual     0.074707  27.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kflDrfAR_eHs",
        "outputId": "64adc5e8-fd99-4355-c2d3-2c612a5529d0"
      },
      "source": [
        "# Comparison of means using Tukey HSD for NC data\n",
        "\n",
        "oneD_hala_nc_mc = MultiComparison(oneD_hala_df[oneD_hala_df['Data']=='NC']['f1_weightedNet'], oneD_hala_df[oneD_hala_df['Data']=='NC']['Features'])\n",
        "print(oneD_hala_nc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0159 0.7634 -0.0743 0.0424  False\n",
            "   CSP Spectral  -0.0384 0.2505 -0.0967   0.02  False\n",
            "   IIR Spectral  -0.0224 0.6063 -0.0807 0.0359  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GFU519yO_o6e",
        "outputId": "21b29225-6213-445c-c6ea-7058488900d2"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api for PD and NC data\n",
        "\n",
        "oneD_hala_formula = 'f1_weightedNet ~ C(Features)+C(Data)+C(Features):C(Data)'\n",
        "oneD_hala_model = ols(oneD_hala_formula, oneD_hala_df[oneD_hala_df['Data'].isin(['PD', 'NC'])]).fit()\n",
        "oneD_hala_aov_table = anova_lm(oneD_hala_model, typ=2)\n",
        "print(oneD_hala_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                       sum_sq    df         F    PR(>F)\n",
            "C(Features)          0.000233   2.0  0.053167  0.948272\n",
            "C(Data)              0.000927   1.0  0.423650  0.517881\n",
            "C(Features):C(Data)  0.012708   2.0  2.905310  0.063331\n",
            "Residual             0.118101  54.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hsv-YCk7_zxd",
        "outputId": "dcb2591b-cf0d-46b4-b779-58c97abe611c"
      },
      "source": [
        "# T-test for individual features\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "dummy_df = oneD_hala_df[oneD_hala_df['Data'].isin(['PD', 'NC'])]\n",
        "for i in features:\n",
        "  print(\"-----\", i, \"-----\")\n",
        "  dummy_df_2 = dummy_df[dummy_df['Features']==i]\n",
        "  d1=(dummy_df_2[dummy_df_2['Data']=='PD'])\n",
        "  d2=(dummy_df_2[dummy_df_2['Data']=='NC'])\n",
        "\n",
        "  stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "\t  print('Probably the same distribution')\n",
        "  else:\n",
        "\t  print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Spectral -----\n",
            "stat=2.258, p=0.040\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- CSP -----\n",
            "stat=-4.871, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- IIR -----\n",
            "stat=0.270, p=0.790\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "se1NEY0MWBaY",
        "outputId": "e0526c96-6c6e-48c3-a46c-403831968ab6"
      },
      "source": [
        "# conf mat T-tests\n",
        "data = [\"Full\", \"PD\", \"NC\"]\n",
        "\n",
        "for d in data:\n",
        "  print(10*\"*\", d, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "  for j in features:\n",
        "    print(\"\\n-----\", j, \"----\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(j)+'/CNN_1D_results_HALA_'+str(j)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    oneD_conf_nc = [i[0][0] for i in oneD_conf]\n",
        "    oneD_conf_pd = [i[1][1] for i in oneD_conf]\n",
        "    stat, p = ttest_ind(oneD_conf_nc, oneD_conf_pd,equal_var=False)\n",
        "    print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "    if p > 0.05:\n",
        "      print('Probably the same distribution')\n",
        "    else:\n",
        "      print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Full **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.78506224 0.21493776]\n",
            " [0.02058111 0.97941889]] \n",
            "\n",
            "stat=-4.359, p=0.002\n",
            "Probably different distributions\n",
            "\n",
            "----- CSP ----\n",
            "[[0.70274756 0.29725244]\n",
            " [0.04084354 0.95915646]] \n",
            "\n",
            "stat=-7.408, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.81595177 0.18404823]\n",
            " [0.0248968  0.9751032 ]] \n",
            "\n",
            "stat=-2.214, p=0.052\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** PD **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.80916667 0.19083333]\n",
            " [0.01114989 0.98885011]] \n",
            "\n",
            "stat=-3.510, p=0.007\n",
            "Probably different distributions\n",
            "\n",
            "----- CSP ----\n",
            "[[0.72967787 0.27032213]\n",
            " [0.03625345 0.96374655]] \n",
            "\n",
            "stat=-10.599, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.77975078 0.22024922]\n",
            " [0.01791742 0.98208258]] \n",
            "\n",
            "stat=-2.512, p=0.033\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "********** NC **********\n",
            "\n",
            "----- Spectral ----\n",
            "[[0.66198347 0.33801653]\n",
            " [0.02709054 0.97290946]] \n",
            "\n",
            "stat=-3.671, p=0.005\n",
            "Probably different distributions\n",
            "\n",
            "----- CSP ----\n",
            "[[0.81840909 0.18159091]\n",
            " [0.02043339 0.97956661]] \n",
            "\n",
            "stat=-7.700, p=0.000\n",
            "Probably different distributions\n",
            "\n",
            "----- IIR ----\n",
            "[[0.80704213 0.19295787]\n",
            " [0.04079671 0.95920329]] \n",
            "\n",
            "stat=-2.509, p=0.027\n",
            "Probably different distributions\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c4px7aR5WNgW",
        "outputId": "11f399b7-59e5-44df-ec27-58fa42811a28"
      },
      "source": [
        "# conf mat T-tests\n",
        "data1 = [\"Full\", \"PD\", \"NC\"]\n",
        "\n",
        "for d in data1:\n",
        "  print(10*\"*\", d, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "  data = [\"LA\", \"HA\"]\n",
        "\n",
        "  l = 0\n",
        "  for i in data:\n",
        "    print(\"\\nHow well is\", i, \"classified in different features for\", d, \"data?\\n\")\n",
        "    for k in features:\n",
        "      print(10*\"-\", k, 10*\"-\")\n",
        "      oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(k)+'/CNN_1D_results_HALA_'+str(k)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "      print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Features': len(oneD_conf)*[str(k)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Features)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "\n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Features'])\n",
        "    print(oneD_mc.tukeyhsd())\n",
        "    l = l+1\n",
        "    print(30*\"*\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Full **********\n",
            "\n",
            "How well is LA classified in different features for Full data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.78506224 0.21493776]\n",
            " [0.02058111 0.97941889]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.70274756 0.29725244]\n",
            " [0.04084354 0.95915646]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.81595177 0.18404823]\n",
            " [0.0248968  0.9751032 ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.068484   2.0  1.265597  0.298275\n",
            "Residual     0.730507  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.1132 0.2895 -0.0691 0.2956  False\n",
            "   CSP Spectral   0.0823 0.5115    -0.1 0.2647  False\n",
            "   IIR Spectral  -0.0309    0.9 -0.2132 0.1515  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HA classified in different features for Full data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.78506224 0.21493776]\n",
            " [0.02058111 0.97941889]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.70274756 0.29725244]\n",
            " [0.04084354 0.95915646]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.81595177 0.18404823]\n",
            " [0.0248968  0.9751032 ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df       F    PR(>F)\n",
            "C(Features)  0.046514   2.0  0.9495  0.392968\n",
            "Residual     1.396145  57.0     NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0646 0.3999 -0.0545 0.1837  False\n",
            "   CSP Spectral   0.0513 0.5542 -0.0678 0.1704  False\n",
            "   IIR Spectral  -0.0133    0.9 -0.1324 0.1058  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** PD **********\n",
            "\n",
            "How well is LA classified in different features for PD data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.80916667 0.19083333]\n",
            " [0.01114989 0.98885011]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.72967787 0.27032213]\n",
            " [0.03625345 0.96374655]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.77975078 0.22024922]\n",
            " [0.01791742 0.98208258]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.032304   2.0  0.508238  0.607196\n",
            "Residual     0.858058  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0501 0.7917 -0.1476 0.2477  False\n",
            "   CSP Spectral   0.0795 0.5811 -0.1181 0.2771  False\n",
            "   IIR Spectral   0.0294    0.9 -0.1682  0.227  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HA classified in different features for PD data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.80916667 0.19083333]\n",
            " [0.01114989 0.98885011]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.72967787 0.27032213]\n",
            " [0.03625345 0.96374655]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.77975078 0.22024922]\n",
            " [0.01791742 0.98208258]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.028214   2.0  0.534701  0.588754\n",
            "Residual     1.503847  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0342 0.7645 -0.0894 0.1578  False\n",
            "   CSP Spectral   0.0523 0.5645 -0.0713 0.1759  False\n",
            "   IIR Spectral   0.0181    0.9 -0.1055 0.1417  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "********** NC **********\n",
            "\n",
            "How well is LA classified in different features for NC data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.66198347 0.33801653]\n",
            " [0.02709054 0.97290946]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.81840909 0.18159091]\n",
            " [0.02043339 0.97956661]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.80704213 0.19295787]\n",
            " [0.04079671 0.95920329]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df        F    PR(>F)\n",
            "C(Features)  0.152134   2.0  2.13317  0.137997\n",
            "Residual     0.962797  27.0      NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0114    0.9 -0.2207  0.198  False\n",
            "   CSP Spectral  -0.1564 0.1722 -0.3658 0.0529  False\n",
            "   IIR Spectral  -0.1451 0.2172 -0.3544 0.0643  False\n",
            "-----------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HA classified in different features for NC data?\n",
            "\n",
            "---------- Spectral ----------\n",
            "[[0.66198347 0.33801653]\n",
            " [0.02709054 0.97290946]] \n",
            "\n",
            "---------- CSP ----------\n",
            "[[0.81840909 0.18159091]\n",
            " [0.02043339 0.97956661]] \n",
            "\n",
            "---------- IIR ----------\n",
            "[[0.80704213 0.19295787]\n",
            " [0.04079671 0.95920329]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.074760   2.0  1.221063  0.302514\n",
            "Residual     1.744933  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR  -0.0159    0.9  -0.149 0.1173  False\n",
            "   CSP Spectral  -0.0815 0.3115 -0.2147 0.0516  False\n",
            "   IIR Spectral  -0.0657 0.4675 -0.1988 0.0675  False\n",
            "-----------------------------------------------------\n",
            "******************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rBjXI4e3h3Q6",
        "outputId": "d8a5d2fd-e4b6-493f-8298-4690a315ebe0"
      },
      "source": [
        "# How well are PD and NC classified on different data?\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "emotions = [\"Full\", \"PD\", \"NC\"]\n",
        "for k in features:\n",
        "  print(10*\"*\", k, 10*\"*\")\n",
        "  oneD_conf_df = pd.DataFrame()\n",
        "  data = [\"LA\", \"HA\"]\n",
        "  emotions = [\"Full\", \"PD\", \"NC\"]\n",
        "  l = 0\n",
        "  for i in data:\n",
        "    print(\"\\nHow well is\", i, \"classified in different data for\", k, \"features?\\n\")\n",
        "    for emot in emotions:\n",
        "      print(10*\"-\", emot, 10*\"-\")\n",
        "      oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(k)+'/CNN_1D_results_HALA_'+str(k)+'_'+str(emot)+'.mat', 'conf_mat_list')\n",
        "      print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Emotions': len(oneD_conf)*[str(emot)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Emotions)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "\n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Emotions'])\n",
        "    print(oneD_mc.tukeyhsd())\n",
        "    l = l+1\n",
        "    print(30*\"*\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Spectral **********\n",
            "\n",
            "How well is LA classified in different data for Spectral features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.78506224 0.21493776]\n",
            " [0.02058111 0.97941889]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.80916667 0.19083333]\n",
            " [0.01114989 0.98885011]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.66198347 0.33801653]\n",
            " [0.02709054 0.97290946]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F   PR(>F)\n",
            "C(Emotions)  0.124641   2.0  1.594401  0.22156\n",
            "Residual     1.055351  27.0       NaN      NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC  -0.1231 0.3599 -0.3423 0.0961  False\n",
            "  Full     PD   0.0241    0.9 -0.1951 0.2433  False\n",
            "    NC     PD   0.1472 0.2371  -0.072 0.3664  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HA classified in different data for Spectral features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.78506224 0.21493776]\n",
            " [0.02058111 0.97941889]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.80916667 0.19083333]\n",
            " [0.01114989 0.98885011]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.66198347 0.33801653]\n",
            " [0.02709054 0.97290946]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.074212   2.0  1.116665  0.334425\n",
            "Residual     1.894068  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC  -0.0648 0.5044 -0.2035 0.0739  False\n",
            "  Full     PD   0.0168    0.9 -0.1219 0.1555  False\n",
            "    NC     PD   0.0816 0.3408 -0.0572 0.2203  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "********** CSP **********\n",
            "\n",
            "How well is LA classified in different data for CSP features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.70274756 0.29725244]\n",
            " [0.04084354 0.95915646]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.72967787 0.27032213]\n",
            " [0.03625345 0.96374655]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.81840909 0.18159091]\n",
            " [0.02043339 0.97956661]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.073254   2.0  5.366171  0.010909\n",
            "Residual     0.184288  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC   0.1157 0.0112  0.0241 0.2073   True\n",
            "  Full     PD   0.0269 0.7342 -0.0647 0.1185  False\n",
            "    NC     PD  -0.0887  0.059 -0.1803 0.0029  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HA classified in different data for CSP features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.70274756 0.29725244]\n",
            " [0.04084354 0.95915646]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.72967787 0.27032213]\n",
            " [0.03625345 0.96374655]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.81840909 0.18159091]\n",
            " [0.02043339 0.97956661]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.050733   2.0  1.565377  0.217861\n",
            "Residual     0.923676  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "  Full     NC    0.068  0.218 -0.0288 0.1649  False\n",
            "  Full     PD   0.0158    0.9 -0.0811 0.1126  False\n",
            "    NC     PD  -0.0523 0.4034 -0.1491 0.0446  False\n",
            "---------------------------------------------------\n",
            "******************************\n",
            "********** IIR **********\n",
            "\n",
            "How well is LA classified in different data for IIR features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.81595177 0.18404823]\n",
            " [0.0248968  0.9751032 ]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.77975078 0.22024922]\n",
            " [0.01791742 0.98208258]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.80704213 0.19295787]\n",
            " [0.04079671 0.95920329]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.007116   2.0  0.073233  0.929568\n",
            "Residual     1.311722  27.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "==================================================\n",
            "group1 group2 meandiff p-adj  lower  upper  reject\n",
            "--------------------------------------------------\n",
            "  Full     NC  -0.0089   0.9 -0.2533 0.2354  False\n",
            "  Full     PD  -0.0362   0.9 -0.2806 0.2082  False\n",
            "    NC     PD  -0.0273   0.9 -0.2716 0.2171  False\n",
            "--------------------------------------------------\n",
            "******************************\n",
            "\n",
            "How well is HA classified in different data for IIR features?\n",
            "\n",
            "---------- Full ----------\n",
            "[[0.81595177 0.18404823]\n",
            " [0.0248968  0.9751032 ]] \n",
            "\n",
            "---------- PD ----------\n",
            "[[0.77975078 0.22024922]\n",
            " [0.01791742 0.98208258]] \n",
            "\n",
            "---------- NC ----------\n",
            "[[0.80704213 0.19295787]\n",
            " [0.04079671 0.95920329]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.002481   2.0  0.038705  0.962059\n",
            "Residual     1.827181  57.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "==================================================\n",
            "group1 group2 meandiff p-adj  lower  upper  reject\n",
            "--------------------------------------------------\n",
            "  Full     NC  -0.0124   0.9 -0.1486 0.1238  False\n",
            "  Full     PD  -0.0146   0.9 -0.1509 0.1216  False\n",
            "    NC     PD  -0.0022   0.9 -0.1385  0.134  False\n",
            "--------------------------------------------------\n",
            "******************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W89LfGnFAZbt"
      },
      "source": [
        "## MultiClass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L3lrHysqActh",
        "outputId": "ef040712-c0e3-427f-cf29-25b9e99bf6c4"
      },
      "source": [
        "# Create a DataFrame\n",
        "oneD_multi_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "for i in features:\n",
        "  for j in Data:\n",
        "    oneD_multi_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(i)+'/CNN_1D_results_MultiClass_'+str(i)+'_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "    data_df = pd.DataFrame({'f1_weightedNet': oneD_multi_f1[0],'Features': len(oneD_multi_f1[0])*[str(i)], 'Data': len(oneD_multi_f1[0])*[str(j)]})\n",
        "    oneD_multi_df = oneD_multi_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(oneD_multi_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Features  Data\n",
            "0         0.509518  Spectral  Full\n",
            "1         0.675848  Spectral  Full\n",
            "2         0.719518  Spectral  Full\n",
            "3         0.747910  Spectral  Full\n",
            "4         0.796485  Spectral  Full\n",
            "..             ...       ...   ...\n",
            "85        0.893157       IIR    NC\n",
            "86        0.917204       IIR    NC\n",
            "87        0.922704       IIR    NC\n",
            "88        0.916969       IIR    NC\n",
            "89        0.907335       IIR    NC\n",
            "\n",
            "[90 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "saZFyXhAAkrV",
        "outputId": "ae479f30-1cc3-40db-c95d-76d160760d53"
      },
      "source": [
        "# FULL DATA\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_multi_full_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_multi_full_model = ols(oneD_multi_full_formula, oneD_multi_df[oneD_multi_df['Data']=='Full']).fit()\n",
        "oneD_multi_aov_table = anova_lm(oneD_multi_full_model, typ=2)\n",
        "print(oneD_multi_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.039081   2.0  0.914746  0.412678\n",
            "Residual     0.576771  27.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VC46lPy7An18",
        "outputId": "e8eaab9a-4ea3-44ad-dc3a-057816c94702"
      },
      "source": [
        "# Comparison of means using Tukey HSD\n",
        "\n",
        "oneD_multi_full_mc = MultiComparison(oneD_multi_df[oneD_multi_df['Data']=='Full']['f1_weightedNet'], oneD_multi_df[oneD_multi_df['Data']=='Full']['Features'])\n",
        "print(oneD_multi_full_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0809 0.4437 -0.0811 0.2429  False\n",
            "   CSP Spectral   0.0714 0.5272 -0.0907 0.2334  False\n",
            "   IIR Spectral  -0.0095    0.9 -0.1716 0.1525  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-gNFZXqrArBt",
        "outputId": "11317121-0e62-4c19-bcce-41c18f88745a"
      },
      "source": [
        "# PD DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_multi_pd_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_multi_pd_model = ols(oneD_multi_pd_formula, oneD_multi_df[oneD_multi_df['Data']=='PD']).fit()\n",
        "oneD_multi_pd_aov_table = anova_lm(oneD_multi_pd_model, typ=2)\n",
        "print(oneD_multi_pd_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.009939   2.0  0.253072  0.778233\n",
            "Residual     0.530209  27.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cY93ggnzAuNv",
        "outputId": "f0312388-f998-40b4-c662-19bcb43769a7"
      },
      "source": [
        "# Comparison of means using Tukey HSD for PD data\n",
        "\n",
        "oneD_multi_pd_mc = MultiComparison(oneD_multi_df[oneD_multi_df['Data']=='PD']['f1_weightedNet'], oneD_multi_df[oneD_multi_df['Data']=='PD']['Features'])\n",
        "print(oneD_multi_pd_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "=====================================================\n",
            "group1  group2  meandiff p-adj   lower  upper  reject\n",
            "-----------------------------------------------------\n",
            "   CSP      IIR   0.0137    0.9 -0.1417  0.169  False\n",
            "   CSP Spectral  -0.0299 0.8776 -0.1853 0.1254  False\n",
            "   IIR Spectral  -0.0436 0.7532 -0.1989 0.1118  False\n",
            "-----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JeNplP5hAzlG",
        "outputId": "521729f0-a64b-402a-e603-9c60cbc9d0b8"
      },
      "source": [
        "# NC DATA\n",
        "\n",
        "# One-way ANOVA using statsmodel api\n",
        "\n",
        "oneD_multi_nc_formula = 'f1_weightedNet ~ C(Features)'\n",
        "oneD_multi_nc_model = ols(oneD_multi_nc_formula, oneD_multi_df[oneD_multi_df['Data']=='NC']).fit()\n",
        "oneD_multi_nc_aov_table = anova_lm(oneD_multi_nc_model, typ=2)\n",
        "print(oneD_multi_nc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Features)  0.002915   2.0  0.071885  0.930815\n",
            "Residual     0.547512  27.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N7dz1UllA2IY",
        "outputId": "48b90860-4680-4a61-ab9b-a1009ca4f075"
      },
      "source": [
        "# Comparison of means using Tukey HSD for NC data\n",
        "\n",
        "oneD_multi_nc_mc = MultiComparison(oneD_multi_df[oneD_multi_df['Data']=='NC']['f1_weightedNet'], oneD_multi_df[oneD_multi_df['Data']=='NC']['Features'])\n",
        "print(oneD_multi_nc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1  group2  meandiff p-adj  lower  upper  reject\n",
            "----------------------------------------------------\n",
            "   CSP      IIR   0.0226   0.9 -0.1353 0.1805  False\n",
            "   CSP Spectral   0.0039   0.9  -0.154 0.1618  False\n",
            "   IIR Spectral  -0.0187   0.9 -0.1766 0.1392  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MGhxvtOzA4f8",
        "outputId": "1cb646c5-316b-42d9-9663-8374e5c189a5"
      },
      "source": [
        "# Two-way ANOVA using statsmodel api for PD and NC data\n",
        "\n",
        "oneD_multi_formula = 'f1_weightedNet ~ C(Features)+C(Data)+C(Features):C(Data)'\n",
        "oneD_multi_model = ols(oneD_multi_formula, oneD_multi_df[oneD_multi_df['Data'].isin(['PD', 'NC'])]).fit()\n",
        "oneD_multi_aov_table = anova_lm(oneD_multi_model, typ=2)\n",
        "print(oneD_multi_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                       sum_sq    df         F    PR(>F)\n",
            "C(Features)          0.009780   2.0  0.245018  0.783556\n",
            "C(Data)              0.013085   1.0  0.655625  0.421663\n",
            "C(Features):C(Data)  0.003075   2.0  0.077030  0.925964\n",
            "Residual             1.077721  54.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DA3h3PkeA7jd",
        "outputId": "43e04b61-cb8d-467a-8618-57322773044c"
      },
      "source": [
        "# T-test for individual features\n",
        "\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "dummy_df = oneD_multi_df[oneD_multi_df['Data'].isin(['PD', 'NC'])]\n",
        "for i in features:\n",
        "  print(\"-----\", i, \"-----\")\n",
        "  dummy_df_2 = dummy_df[dummy_df['Features']==i]\n",
        "  d1=(dummy_df_2[dummy_df_2['Data']=='PD'])\n",
        "  d2=(dummy_df_2[dummy_df_2['Data']=='NC'])\n",
        "\n",
        "  stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "\t  print('Probably the same distribution')\n",
        "  else:\n",
        "\t  print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Spectral -----\n",
            "stat=0.260, p=0.798\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "----- CSP -----\n",
            "stat=2.319, p=0.034\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- IIR -----\n",
            "stat=0.346, p=0.733\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lNUU5ooTWvnE",
        "outputId": "0d9616ec-3da0-4de3-f00b-679c17568a21"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "oneD_conf_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "features = [\"Spectral\", \"CSP\", \"IIR\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for d in Data:\n",
        "  print(30*\"*\", d, 30*\"*\")\n",
        "  for f in features:\n",
        "    print(10*\"=\", f, 10*\"=\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(f)+'/CNN_1D_results_MultiClass_'+str(f)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    l = 0\n",
        "    for k in emotions:\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Emotions': len(oneD_conf)*[str(k)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "      l = l+1\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Emotions)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "    \n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Emotions'])\n",
        "    print(oneD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "****************************** Full ******************************\n",
            "========== Spectral ==========\n",
            "[[0.82282158 0.06514523 0.04896266 0.03195021 0.01950207 0.01161826]\n",
            " [0.04806373 0.79050858 0.07192708 0.0324326  0.03322151 0.02384651]\n",
            " [0.04665802 0.07041148 0.73589212 0.07123271 0.04664765 0.02915802]\n",
            " [0.02406639 0.02738589 0.06763485 0.76182573 0.0746888  0.04439834]\n",
            " [0.02326426 0.05335582 0.04129478 0.0786008  0.70439679 0.09908755]\n",
            " [0.02650731 0.03837443 0.04156942 0.05660487 0.07555054 0.76139344]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.085083   5.0  1.178027  0.332053\n",
            "Residual     0.780025  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0323    0.9 -0.1911 0.1265  False\n",
            "    E1     E3  -0.0869 0.5794 -0.2457 0.0719  False\n",
            "    E1     E4   -0.061 0.8509 -0.2198 0.0978  False\n",
            "    E1     E5  -0.1184 0.2532 -0.2772 0.0404  False\n",
            "    E1     E6  -0.0614 0.8464 -0.2202 0.0974  False\n",
            "    E2     E3  -0.0546    0.9 -0.2134 0.1042  False\n",
            "    E2     E4  -0.0287    0.9 -0.1875 0.1301  False\n",
            "    E2     E5  -0.0861  0.588 -0.2449 0.0727  False\n",
            "    E2     E6  -0.0291    0.9 -0.1879 0.1297  False\n",
            "    E3     E4   0.0259    0.9 -0.1329 0.1847  False\n",
            "    E3     E5  -0.0315    0.9 -0.1903 0.1273  False\n",
            "    E3     E6   0.0255    0.9 -0.1333 0.1843  False\n",
            "    E4     E5  -0.0574 0.8883 -0.2162 0.1014  False\n",
            "    E4     E6  -0.0004    0.9 -0.1592 0.1584  False\n",
            "    E5     E6    0.057 0.8928 -0.1018 0.2158  False\n",
            "---------------------------------------------------\n",
            "========== CSP ==========\n",
            "[[0.81774407 0.06854603 0.02717399 0.01546548 0.03720363 0.03386681]\n",
            " [0.09458645 0.73217371 0.05389482 0.0538741  0.02795378 0.03751713]\n",
            " [0.091234   0.09591826 0.63375885 0.07221269 0.03295251 0.07392369]\n",
            " [0.04747407 0.09534924 0.07953838 0.63770055 0.04206604 0.09787172]\n",
            " [0.05215979 0.04769165 0.03880599 0.06506954 0.61642125 0.17985177]\n",
            " [0.05291084 0.05171727 0.04210602 0.04808835 0.08702811 0.7181494 ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.339916    5.0  6.522836  0.000023\n",
            "Residual     1.188146  114.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0589 0.4552 -0.1525  0.0346  False\n",
            "    E1     E3  -0.1355  0.001  -0.229 -0.0419   True\n",
            "    E1     E4  -0.1205 0.0039 -0.2141 -0.0269   True\n",
            "    E1     E5  -0.1599  0.001 -0.2535 -0.0663   True\n",
            "    E1     E6  -0.0805 0.1343 -0.1741  0.0131  False\n",
            "    E2     E3  -0.0765 0.1755 -0.1701  0.0171  False\n",
            "    E2     E4  -0.0616 0.4047 -0.1552   0.032  False\n",
            "    E2     E5  -0.1009 0.0266 -0.1945 -0.0073   True\n",
            "    E2     E6  -0.0216    0.9 -0.1152   0.072  False\n",
            "    E3     E4   0.0149    0.9 -0.0786  0.1085  False\n",
            "    E3     E5  -0.0244    0.9  -0.118  0.0692  False\n",
            "    E3     E6   0.0549 0.5287 -0.0386  0.1485  False\n",
            "    E4     E5  -0.0394 0.8041 -0.1329  0.0542  False\n",
            "    E4     E6     0.04 0.7925 -0.0536  0.1336  False\n",
            "    E5     E6   0.0794 0.1454 -0.0142  0.1729  False\n",
            "----------------------------------------------------\n",
            "========== IIR ==========\n",
            "[[0.78486865 0.04739018 0.03111972 0.04461456 0.04181955 0.05018734]\n",
            " [0.03579003 0.80997427 0.02516471 0.03580757 0.04508791 0.04817551]\n",
            " [0.0489193  0.06397821 0.75262645 0.04795819 0.04092922 0.04558863]\n",
            " [0.04257766 0.06247653 0.0379459  0.76488735 0.04257979 0.04953277]\n",
            " [0.04828035 0.05497678 0.02908792 0.0429152  0.76350496 0.06123479]\n",
            " [0.03677679 0.06599607 0.02614749 0.04742183 0.06029302 0.7633648 ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.286721    5.0  2.267494  0.049869\n",
            "Residual     4.400410  174.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0309    0.9 -0.1493 0.0874  False\n",
            "    E1     E3  -0.1011 0.1415 -0.2194 0.0173  False\n",
            "    E1     E4   -0.087 0.2825 -0.2053 0.0313  False\n",
            "    E1     E5  -0.1137 0.0676  -0.232 0.0046  False\n",
            "    E1     E6  -0.0608 0.6538 -0.1792 0.0575  False\n",
            "    E2     E3  -0.0701 0.5243 -0.1885 0.0482  False\n",
            "    E2     E4  -0.0561 0.7202 -0.1744 0.0622  False\n",
            "    E2     E5  -0.0828 0.3376 -0.2011 0.0355  False\n",
            "    E2     E6  -0.0299    0.9 -0.1482 0.0884  False\n",
            "    E3     E4    0.014    0.9 -0.1043 0.1324  False\n",
            "    E3     E5  -0.0127    0.9  -0.131 0.1057  False\n",
            "    E3     E6   0.0402    0.9 -0.0781 0.1585  False\n",
            "    E4     E5  -0.0267    0.9  -0.145 0.0916  False\n",
            "    E4     E6   0.0262    0.9 -0.0922 0.1445  False\n",
            "    E5     E6   0.0529 0.7651 -0.0655 0.1712  False\n",
            "---------------------------------------------------\n",
            "****************************** PD ******************************\n",
            "========== Spectral ==========\n",
            "[[0.85666667 0.0675     0.02166667 0.02166667 0.0225     0.01      ]\n",
            " [0.02519685 0.80472441 0.07559055 0.03070866 0.03070866 0.03307087]\n",
            " [0.02998623 0.0999449  0.73849862 0.05995179 0.0382989  0.03331956]\n",
            " [0.01739669 0.03969697 0.06953857 0.71371901 0.09183884 0.06780992]\n",
            " [0.01290323 0.04354839 0.03548387 0.08387097 0.73145161 0.09274194]\n",
            " [0.01024247 0.03543932 0.03073991 0.0519935  0.10241845 0.76916635]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.416857    5.0  3.829848  0.002358\n",
            "Residual     5.093908  234.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0362 0.8746  -0.131  0.0586  False\n",
            "    E1     E3  -0.1053 0.0197 -0.2001 -0.0105   True\n",
            "    E1     E4   -0.101 0.0293 -0.1958 -0.0062   True\n",
            "    E1     E5  -0.1166 0.0065 -0.2114 -0.0218   True\n",
            "    E1     E6  -0.0675 0.3195 -0.1623  0.0273  False\n",
            "    E2     E3  -0.0692  0.293  -0.164  0.0256  False\n",
            "    E2     E4  -0.0648 0.3669 -0.1596    0.03  False\n",
            "    E2     E5  -0.0804 0.1478 -0.1752  0.0144  False\n",
            "    E2     E6  -0.0313    0.9 -0.1261  0.0635  False\n",
            "    E3     E4   0.0043    0.9 -0.0905  0.0991  False\n",
            "    E3     E5  -0.0113    0.9 -0.1061  0.0836  False\n",
            "    E3     E6   0.0378  0.846  -0.057  0.1326  False\n",
            "    E4     E5  -0.0156    0.9 -0.1104  0.0792  False\n",
            "    E4     E6   0.0335    0.9 -0.0613  0.1283  False\n",
            "    E5     E6   0.0491 0.6503 -0.0457  0.1439  False\n",
            "----------------------------------------------------\n",
            "========== CSP ==========\n",
            "[[0.85467787 0.04872549 0.03277311 0.02015406 0.02182773 0.02184174]\n",
            " [0.06344516 0.81787742 0.04729677 0.03691613 0.01603871 0.01842581]\n",
            " [0.04636092 0.05403076 0.79752884 0.05733514 0.02534539 0.01939895]\n",
            " [0.0282438  0.04320248 0.04988981 0.80710055 0.04661157 0.02495179]\n",
            " [0.03806714 0.01949253 0.0275636  0.05925125 0.75415683 0.10146866]\n",
            " [0.03666667 0.02228571 0.02948571 0.05414603 0.09320635 0.76420952]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df        F    PR(>F)\n",
            "C(Emotions)  0.448426    5.0  4.75523  0.000341\n",
            "Residual     5.544932  294.0      NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0363 0.7454 -0.1151  0.0425  False\n",
            "    E1     E3  -0.0957 0.0074 -0.1745 -0.0169   True\n",
            "    E1     E4  -0.0903 0.0143 -0.1691 -0.0115   True\n",
            "    E1     E5  -0.1134  0.001 -0.1922 -0.0346   True\n",
            "    E1     E6  -0.0721 0.0944 -0.1509  0.0067  False\n",
            "    E2     E3  -0.0594 0.2587 -0.1382  0.0194  False\n",
            "    E2     E4   -0.054 0.3652 -0.1328  0.0248  False\n",
            "    E2     E5  -0.0771 0.0594 -0.1559  0.0017  False\n",
            "    E2     E6  -0.0358 0.7561 -0.1146   0.043  False\n",
            "    E3     E4   0.0054    0.9 -0.0734  0.0842  False\n",
            "    E3     E5  -0.0177    0.9 -0.0965  0.0611  False\n",
            "    E3     E6   0.0236    0.9 -0.0552  0.1024  False\n",
            "    E4     E5  -0.0231    0.9 -0.1019  0.0557  False\n",
            "    E4     E6   0.0182    0.9 -0.0606   0.097  False\n",
            "    E5     E6   0.0413 0.6416 -0.0375  0.1201  False\n",
            "----------------------------------------------------\n",
            "========== IIR ==========\n",
            "[[0.86644168 0.04858082 0.02988058 0.01962617 0.01586189 0.01960886]\n",
            " [0.04961283 0.84128477 0.02745733 0.0221634  0.02752845 0.03195322]\n",
            " [0.07409628 0.05243343 0.79761065 0.03094692 0.02338212 0.02153059]\n",
            " [0.05818043 0.05819742 0.03965341 0.78295957 0.02126232 0.03974686]\n",
            " [0.05557432 0.04661358 0.02780727 0.03501448 0.78921332 0.04577703]\n",
            " [0.04673964 0.04943332 0.02914144 0.03444341 0.03520416 0.80503804]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.495033    5.0  4.162183  0.001094\n",
            "Residual     8.420658  354.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0344 0.8021 -0.1151  0.0462  False\n",
            "    E1     E3  -0.0912 0.0164 -0.1719 -0.0105   True\n",
            "    E1     E4  -0.0892 0.0206 -0.1699 -0.0085   True\n",
            "    E1     E5  -0.1073 0.0022  -0.188 -0.0267   True\n",
            "    E1     E6  -0.0703 0.1273  -0.151  0.0104  False\n",
            "    E2     E3  -0.0568 0.3352 -0.1375  0.0239  False\n",
            "    E2     E4  -0.0547 0.3786 -0.1354   0.026  False\n",
            "    E2     E5  -0.0729 0.1026 -0.1536  0.0078  False\n",
            "    E2     E6  -0.0359  0.773 -0.1166  0.0448  False\n",
            "    E3     E4    0.002    0.9 -0.0786  0.0827  False\n",
            "    E3     E5  -0.0161    0.9 -0.0968  0.0646  False\n",
            "    E3     E6   0.0209    0.9 -0.0598  0.1016  False\n",
            "    E4     E5  -0.0182    0.9 -0.0989  0.0625  False\n",
            "    E4     E6   0.0189    0.9 -0.0618  0.0995  False\n",
            "    E5     E6    0.037 0.7493 -0.0437  0.1177  False\n",
            "----------------------------------------------------\n",
            "****************************** NC ******************************\n",
            "========== Spectral ==========\n",
            "[[0.79752066 0.06446281 0.03636364 0.04380165 0.02561983 0.0322314 ]\n",
            " [0.04427083 0.84705063 0.03026284 0.02641715 0.03415092 0.01784763]\n",
            " [0.06916667 0.07833333 0.6725     0.06666667 0.05833333 0.055     ]\n",
            " [0.0332438  0.0531405  0.04159091 0.74396006 0.07739669 0.05066804]\n",
            " [0.03749841 0.07273016 0.03513651 0.05584762 0.71486984 0.08391746]\n",
            " [0.02699683 0.04369524 0.03890794 0.0524254  0.06191746 0.77605714]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.616335    5.0  5.554767  0.000058\n",
            "Residual     9.187164  414.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0224    0.9 -0.0945  0.0496  False\n",
            "    E1     E3   -0.096 0.0022 -0.1681  -0.024   True\n",
            "    E1     E4  -0.0841 0.0117 -0.1562  -0.012   True\n",
            "    E1     E5  -0.1038  0.001 -0.1759 -0.0317   True\n",
            "    E1     E6  -0.0633 0.1222 -0.1354  0.0088  False\n",
            "    E2     E3  -0.0736 0.0423 -0.1457 -0.0015   True\n",
            "    E2     E4  -0.0616 0.1423 -0.1337  0.0105  False\n",
            "    E2     E5  -0.0814 0.0167 -0.1535 -0.0093   True\n",
            "    E2     E6  -0.0409 0.5717  -0.113  0.0312  False\n",
            "    E3     E4    0.012    0.9 -0.0601  0.0841  False\n",
            "    E3     E5  -0.0078    0.9 -0.0799  0.0643  False\n",
            "    E3     E6   0.0327 0.7586 -0.0394  0.1048  False\n",
            "    E4     E5  -0.0197    0.9 -0.0918  0.0524  False\n",
            "    E4     E6   0.0207    0.9 -0.0513  0.0928  False\n",
            "    E5     E6   0.0405  0.581 -0.0316  0.1126  False\n",
            "----------------------------------------------------\n",
            "========== CSP ==========\n",
            "[[0.81358815 0.06991736 0.04328512 0.01829201 0.03078512 0.02413223]\n",
            " [0.03733333 0.82747302 0.05330159 0.05724444 0.01431111 0.01033651]\n",
            " [0.03465318 0.08792195 0.7210084  0.0913189  0.02619997 0.03889759]\n",
            " [0.02417367 0.09507003 0.0967577  0.71560224 0.04255602 0.02584034]\n",
            " [0.029      0.03947742 0.03947097 0.07087097 0.73745161 0.08372903]\n",
            " [0.02669814 0.05500262 0.05334382 0.05093758 0.1010556  0.71296223]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df       F    PR(>F)\n",
            "C(Emotions)  0.719174    5.0  7.1579  0.000002\n",
            "Residual     9.524814  474.0     NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0179    0.9  -0.082  0.0462  False\n",
            "    E1     E3  -0.0956  0.001 -0.1597 -0.0315   True\n",
            "    E1     E4  -0.0858  0.002   -0.15 -0.0217   True\n",
            "    E1     E5  -0.1004  0.001 -0.1645 -0.0362   True\n",
            "    E1     E6   -0.068 0.0304 -0.1321 -0.0039   True\n",
            "    E2     E3  -0.0777 0.0075 -0.1418 -0.0136   True\n",
            "    E2     E4  -0.0679 0.0308  -0.132 -0.0038   True\n",
            "    E2     E5  -0.0825 0.0035 -0.1466 -0.0183   True\n",
            "    E2     E6  -0.0501 0.2238 -0.1142   0.014  False\n",
            "    E3     E4   0.0098    0.9 -0.0543  0.0739  False\n",
            "    E3     E5  -0.0047    0.9 -0.0689  0.0594  False\n",
            "    E3     E6   0.0276 0.7971 -0.0365  0.0917  False\n",
            "    E4     E5  -0.0145    0.9 -0.0787  0.0496  False\n",
            "    E4     E6   0.0178    0.9 -0.0463   0.082  False\n",
            "    E5     E6   0.0324 0.6753 -0.0318  0.0965  False\n",
            "----------------------------------------------------\n",
            "========== IIR ==========\n",
            "[[0.75262487 0.03507475 0.06103466 0.0506371  0.03598369 0.06464492]\n",
            " [0.03875175 0.76096103 0.05292656 0.04062257 0.03527403 0.07146406]\n",
            " [0.02071945 0.03004761 0.82175101 0.05242462 0.02719097 0.04786634]\n",
            " [0.02964694 0.04074074 0.05002596 0.79715299 0.02409138 0.05834199]\n",
            " [0.03026233 0.04100822 0.04971555 0.04100032 0.76053255 0.07748104]\n",
            " [0.02869208 0.03499839 0.04405566 0.05395753 0.04934846 0.78894788]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "                sum_sq     df         F    PR(>F)\n",
            "C(Emotions)   0.578259    5.0  4.771837  0.000282\n",
            "Residual     12.942203  534.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2   -0.015    0.9 -0.0814  0.0514  False\n",
            "    E1     E3  -0.0773 0.0118 -0.1437 -0.0109   True\n",
            "    E1     E4  -0.0713 0.0268 -0.1377  -0.005   True\n",
            "    E1     E5  -0.0883 0.0022 -0.1547  -0.022   True\n",
            "    E1     E6  -0.0564 0.1475 -0.1228    0.01  False\n",
            "    E2     E3  -0.0623 0.0801 -0.1287  0.0041  False\n",
            "    E2     E4  -0.0563 0.1484 -0.1227    0.01  False\n",
            "    E2     E5  -0.0733 0.0206 -0.1397  -0.007   True\n",
            "    E2     E6  -0.0414 0.4775 -0.1078   0.025  False\n",
            "    E3     E4    0.006    0.9 -0.0604  0.0723  False\n",
            "    E3     E5   -0.011    0.9 -0.0774  0.0554  False\n",
            "    E3     E6   0.0209    0.9 -0.0455  0.0873  False\n",
            "    E4     E5   -0.017    0.9 -0.0834  0.0494  False\n",
            "    E4     E6   0.0149    0.9 -0.0514  0.0813  False\n",
            "    E5     E6   0.0319 0.7146 -0.0345  0.0983  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FQSahqzRjA-X",
        "outputId": "7c687ba3-7ca5-4a8d-8751-cdf2eb65152a"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "oneD_conf_df = pd.DataFrame()\n",
        "Data = [\"NC\"]\n",
        "features = [\"Spectral\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for d in Data:\n",
        "  print(30*\"*\", d, 30*\"*\")\n",
        "  for f in features:\n",
        "    print(10*\"=\", f, 10*\"=\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(f)+'/CNN_1D_results_MultiClass_'+str(f)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    l = 0\n",
        "    for k in emotions:\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Emotions': len(oneD_conf)*[str(k)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "      l = l+1\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Emotions)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "    \n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Emotions'])\n",
        "    print(oneD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "****************************** NC ******************************\n",
            "========== Spectral ==========\n",
            "[[0.79752066 0.06446281 0.03636364 0.04380165 0.02561983 0.0322314 ]\n",
            " [0.04427083 0.84705063 0.03026284 0.02641715 0.03415092 0.01784763]\n",
            " [0.06916667 0.07833333 0.6725     0.06666667 0.05833333 0.055     ]\n",
            " [0.0332438  0.0531405  0.04159091 0.74396006 0.07739669 0.05066804]\n",
            " [0.03749841 0.07273016 0.03513651 0.05584762 0.71486984 0.08391746]\n",
            " [0.02699683 0.04369524 0.03890794 0.0524254  0.06191746 0.77605714]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.191829   5.0  2.998001  0.018473\n",
            "Residual     0.691046  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2   0.0495    0.9 -0.0999   0.199  False\n",
            "    E1     E3   -0.125  0.151 -0.2745  0.0244  False\n",
            "    E1     E4  -0.0536 0.8938  -0.203  0.0959  False\n",
            "    E1     E5  -0.0827 0.5702 -0.2321  0.0668  False\n",
            "    E1     E6  -0.0215    0.9 -0.1709   0.128  False\n",
            "    E2     E3  -0.1746 0.0133  -0.324 -0.0251   True\n",
            "    E2     E4  -0.1031 0.3353 -0.2526  0.0464  False\n",
            "    E2     E5  -0.1322 0.1119 -0.2816  0.0173  False\n",
            "    E2     E6   -0.071 0.6999 -0.2205  0.0785  False\n",
            "    E3     E4   0.0715 0.6947  -0.078  0.2209  False\n",
            "    E3     E5   0.0424    0.9 -0.1071  0.1918  False\n",
            "    E3     E6   0.1036 0.3303 -0.0459   0.253  False\n",
            "    E4     E5  -0.0291    0.9 -0.1786  0.1204  False\n",
            "    E4     E6   0.0321    0.9 -0.1174  0.1816  False\n",
            "    E5     E6   0.0612 0.8089 -0.0883  0.2107  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9Y56ouPCjfuF",
        "outputId": "5b73fbdf-663b-4d62-cca8-14e6576adcb3"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "oneD_conf_df = pd.DataFrame()\n",
        "Data = [\"NC\"]\n",
        "features = [\"CSP\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for d in Data:\n",
        "  print(30*\"*\", d, 30*\"*\")\n",
        "  for f in features:\n",
        "    print(10*\"=\", f, 10*\"=\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(f)+'/CNN_1D_results_MultiClass_'+str(f)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    l = 0\n",
        "    for k in emotions:\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Emotions': len(oneD_conf)*[str(k)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "      l = l+1\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Emotions)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "    \n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Emotions'])\n",
        "    print(oneD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "****************************** NC ******************************\n",
            "========== CSP ==========\n",
            "[[0.81358815 0.06991736 0.04328512 0.01829201 0.03078512 0.02413223]\n",
            " [0.03733333 0.82747302 0.05330159 0.05724444 0.01431111 0.01033651]\n",
            " [0.03465318 0.08792195 0.7210084  0.0913189  0.02619997 0.03889759]\n",
            " [0.02417367 0.09507003 0.0967577  0.71560224 0.04255602 0.02584034]\n",
            " [0.029      0.03947742 0.03947097 0.07087097 0.73745161 0.08372903]\n",
            " [0.02669814 0.05500262 0.05334382 0.05093758 0.1010556  0.71296223]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "              sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.13467   5.0  4.884281  0.000929\n",
            "Residual     0.29778  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2   0.0139    0.9 -0.0842   0.112  False\n",
            "    E1     E3  -0.0926 0.0751 -0.1907  0.0055  False\n",
            "    E1     E4   -0.098 0.0505 -0.1961  0.0001  False\n",
            "    E1     E5  -0.0761  0.215 -0.1743   0.022  False\n",
            "    E1     E6  -0.1006 0.0414 -0.1987 -0.0025   True\n",
            "    E2     E3  -0.1065 0.0261 -0.2046 -0.0083   True\n",
            "    E2     E4  -0.1119 0.0167   -0.21 -0.0138   True\n",
            "    E2     E5    -0.09 0.0895 -0.1881  0.0081  False\n",
            "    E2     E6  -0.1145 0.0133 -0.2126 -0.0164   True\n",
            "    E3     E4  -0.0054    0.9 -0.1035  0.0927  False\n",
            "    E3     E5   0.0164    0.9 -0.0817  0.1146  False\n",
            "    E3     E6   -0.008    0.9 -0.1062  0.0901  False\n",
            "    E4     E5   0.0218    0.9 -0.0763    0.12  False\n",
            "    E4     E6  -0.0026    0.9 -0.1008  0.0955  False\n",
            "    E5     E6  -0.0245    0.9 -0.1226  0.0736  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KcDPs6sOjg2L",
        "outputId": "4f9c3ed1-d544-4d45-8042-99f5575ed3f3"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "oneD_conf_df = pd.DataFrame()\n",
        "Data = [\"NC\"]\n",
        "features = [\"IIR\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for d in Data:\n",
        "  print(30*\"*\", d, 30*\"*\")\n",
        "  for f in features:\n",
        "    print(10*\"=\", f, 10*\"=\")\n",
        "    oneD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/'+str(f)+'/CNN_1D_results_MultiClass_'+str(f)+'_'+str(d)+'.mat', 'conf_mat_list')\n",
        "    print(np.mean(oneD_conf, axis=0), '\\n')\n",
        "    l = 0\n",
        "    for k in emotions:\n",
        "      data_df = pd.DataFrame({'Accuracy': [i[l][l] for i in oneD_conf],'Emotions': len(oneD_conf)*[str(k)]})\n",
        "      oneD_conf_df = oneD_conf_df.append(data_df, ignore_index=True)\n",
        "      l = l+1\n",
        "\n",
        "    print('\\n--- One way Anova ---')\n",
        "\n",
        "    oneD_formula = 'Accuracy ~ C(Emotions)'\n",
        "    oneD_model = ols(oneD_formula, oneD_conf_df).fit()\n",
        "    oneD_aov_table = anova_lm(oneD_model, typ=2)\n",
        "    print(oneD_aov_table)\n",
        "    \n",
        "    print('\\n--- Tukey HSD ----')\n",
        "    # Tukey HSD test\n",
        "\n",
        "    oneD_mc = MultiComparison(oneD_conf_df['Accuracy'], oneD_conf_df['Emotions'])\n",
        "    print(oneD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "****************************** NC ******************************\n",
            "========== IIR ==========\n",
            "[[0.75262487 0.03507475 0.06103466 0.0506371  0.03598369 0.06464492]\n",
            " [0.03875175 0.76096103 0.05292656 0.04062257 0.03527403 0.07146406]\n",
            " [0.02071945 0.03004761 0.82175101 0.05242462 0.02719097 0.04786634]\n",
            " [0.02964694 0.04074074 0.05002596 0.79715299 0.02409138 0.05834199]\n",
            " [0.03026233 0.04100822 0.04971555 0.04100032 0.76053255 0.07748104]\n",
            " [0.02869208 0.03499839 0.04405566 0.05395753 0.04934846 0.78894788]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.036077   5.0  0.120677  0.987235\n",
            "Residual     3.228683  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "==================================================\n",
            "group1 group2 meandiff p-adj  lower  upper  reject\n",
            "--------------------------------------------------\n",
            "    E1     E2   0.0083   0.9 -0.3147 0.3314  False\n",
            "    E1     E3   0.0691   0.9  -0.254 0.3922  False\n",
            "    E1     E4   0.0445   0.9 -0.2786 0.3676  False\n",
            "    E1     E5   0.0079   0.9 -0.3152  0.331  False\n",
            "    E1     E6   0.0363   0.9 -0.2868 0.3594  False\n",
            "    E2     E3   0.0608   0.9 -0.2623 0.3839  False\n",
            "    E2     E4   0.0362   0.9 -0.2869 0.3593  False\n",
            "    E2     E5  -0.0004   0.9 -0.3235 0.3227  False\n",
            "    E2     E6    0.028   0.9 -0.2951 0.3511  False\n",
            "    E3     E4  -0.0246   0.9 -0.3477 0.2985  False\n",
            "    E3     E5  -0.0612   0.9 -0.3843 0.2619  False\n",
            "    E3     E6  -0.0328   0.9 -0.3559 0.2903  False\n",
            "    E4     E5  -0.0366   0.9 -0.3597 0.2865  False\n",
            "    E4     E6  -0.0082   0.9 -0.3313 0.3149  False\n",
            "    E5     E6   0.0284   0.9 -0.2947 0.3515  False\n",
            "--------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z_6IB_Fowiep"
      },
      "source": [
        "# 2D-CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wkL9GOaQd4C7",
        "outputId": "843f42ee-7111-4770-cae2-c77ed49487df"
      },
      "source": [
        "# conf mat T-tests\n",
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  #print(matContent)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "twoD_conf_df = pd.DataFrame()\n",
        "Data = [\"Full\"]\n",
        "for j in Data:\n",
        "  print(\"-----\", j, \"----\")\n",
        "  twoD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(twoD_conf, axis=0), '\\n')\n",
        "  twoD_conf_nc = [i[0][0] for i in twoD_conf]\n",
        "  twoD_conf_pd = [i[1][1] for i in twoD_conf]\n",
        "  stat, p = ttest_ind(twoD_conf_nc, twoD_conf_pd,equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Full ----\n",
            "[[0.9877193  0.0122807 ]\n",
            " [0.00947391 0.99052609]] \n",
            "\n",
            "stat=-0.587, p=0.566\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ev6GcuY4wk1s",
        "outputId": "3e695a76-f734-4b60-c5e8-6240a2efdcee"
      },
      "source": [
        "# PD vs NC Individual Emotions\n",
        "\n",
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "# Create a DataFrame\n",
        "twoD_pdnc_emot_df = pd.DataFrame()\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "for j in emotions:\n",
        "  twoD_pdnc_emot_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data = pd.DataFrame({'f1_weightedNet': twoD_pdnc_emot_f1[0],'Emotions': len(twoD_pdnc_emot_f1[0])*[str(j)]})\n",
        "  twoD_pdnc_emot_df = twoD_pdnc_emot_df.append(data, ignore_index=True)\n",
        "\n",
        "print(twoD_pdnc_emot_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet Emotions\n",
            "0         0.983401       E1\n",
            "1         0.975103       E1\n",
            "2         0.995851       E1\n",
            "3         1.000000       E1\n",
            "4         0.991701       E1\n",
            "5         0.991701       E1\n",
            "6         0.983397       E1\n",
            "7         0.975086       E1\n",
            "8         0.987551       E1\n",
            "9         0.995851       E1\n",
            "10        0.953119       E2\n",
            "11        0.933449       E2\n",
            "12        0.988281       E2\n",
            "13        1.000000       E2\n",
            "14        0.976560       E2\n",
            "15        0.988281       E2\n",
            "16        0.980469       E2\n",
            "17        0.996094       E2\n",
            "18        0.996078       E2\n",
            "19        1.000000       E2\n",
            "20        0.954347       E3\n",
            "21        0.979166       E3\n",
            "22        0.983329       E3\n",
            "23        0.983333       E3\n",
            "24        0.987498       E3\n",
            "25        1.000000       E3\n",
            "26        0.983332       E3\n",
            "27        0.995833       E3\n",
            "28        0.991666       E3\n",
            "29        0.991666       E3\n",
            "30        0.925273       E4\n",
            "31        0.983402       E4\n",
            "32        0.983402       E4\n",
            "33        0.970939       E4\n",
            "34        0.991701       E4\n",
            "35        0.975101       E4\n",
            "36        0.987551       E4\n",
            "37        0.991701       E4\n",
            "38        0.970926       E4\n",
            "39        1.000000       E4\n",
            "40        0.915867       E5\n",
            "41        0.923688       E5\n",
            "42        0.992000       E5\n",
            "43        0.971887       E5\n",
            "44        0.987951       E5\n",
            "45        0.995984       E5\n",
            "46        0.983935       E5\n",
            "47        0.983935       E5\n",
            "48        0.987952       E5\n",
            "49        0.991968       E5\n",
            "50        0.928620       E6\n",
            "51        0.976282       E6\n",
            "52        0.996047       E6\n",
            "53        0.992094       E6\n",
            "54        0.988142       E6\n",
            "55        0.988142       E6\n",
            "56        0.984189       E6\n",
            "57        1.000000       E6\n",
            "58        0.988095       E6\n",
            "59        1.000000       E6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZAcCGaa9ylSa",
        "outputId": "9a62698a-2970-4b29-8720-83025094ed45"
      },
      "source": [
        "# One way anova \n",
        "\n",
        "twoD_pdnc_formula = 'f1_weightedNet ~ C(Emotions)'\n",
        "twoD_pdnc_model = ols(twoD_pdnc_formula, twoD_pdnc_emot_df).fit()\n",
        "twoD_pdnc_aov_table = anova_lm(twoD_pdnc_model, typ=2)\n",
        "print(twoD_pdnc_aov_table)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.001372   5.0  0.677699  0.642183\n",
            "Residual     0.021858  54.0       NaN       NaN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ApqjaCWKzY3M",
        "outputId": "6703b129-1c51-4505-a704-9fc55ff8fe43"
      },
      "source": [
        "# Tukey HSD test\n",
        "\n",
        "twoD_pdnc_mc = MultiComparison(twoD_pdnc_emot_df['f1_weightedNet'], twoD_pdnc_emot_df['Emotions'])\n",
        "print(twoD_pdnc_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0067    0.9 -0.0333 0.0199  False\n",
            "    E1     E3  -0.0029    0.9 -0.0295 0.0236  False\n",
            "    E1     E4    -0.01 0.8663 -0.0365 0.0166  False\n",
            "    E1     E5  -0.0144  0.586  -0.041 0.0121  False\n",
            "    E1     E6  -0.0038    0.9 -0.0304 0.0228  False\n",
            "    E2     E3   0.0038    0.9 -0.0228 0.0304  False\n",
            "    E2     E4  -0.0032    0.9 -0.0298 0.0233  False\n",
            "    E2     E5  -0.0077    0.9 -0.0343 0.0189  False\n",
            "    E2     E6   0.0029    0.9 -0.0237 0.0295  False\n",
            "    E3     E4   -0.007    0.9 -0.0336 0.0196  False\n",
            "    E3     E5  -0.0115 0.7703 -0.0381 0.0151  False\n",
            "    E3     E6  -0.0009    0.9 -0.0274 0.0257  False\n",
            "    E4     E5  -0.0045    0.9 -0.0311 0.0221  False\n",
            "    E4     E6   0.0062    0.9 -0.0204 0.0327  False\n",
            "    E5     E6   0.0106 0.8238 -0.0159 0.0372  False\n",
            "---------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qz7RPT1szz-_",
        "outputId": "c8af36a1-730f-40c5-ebd9-08d26bf20f8e"
      },
      "source": [
        "# HVLV\n",
        "# Create a DataFrame\n",
        "twoD_hvlv_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  twoD_hvlv_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': twoD_hvlv_f1[0], 'Data': len(twoD_hvlv_f1[0])*[str(j)]})\n",
        "  twoD_hvlv_df = twoD_hvlv_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(twoD_hvlv_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Data\n",
            "0         0.749099  Full\n",
            "1         0.818357  Full\n",
            "2         0.858496  Full\n",
            "3         0.884230  Full\n",
            "4         0.900427  Full\n",
            "5         0.926848  Full\n",
            "6         0.926506  Full\n",
            "7         0.929108  Full\n",
            "8         0.949668  Full\n",
            "9         0.962126  Full\n",
            "10        0.708954    PD\n",
            "11        0.798362    PD\n",
            "12        0.814595    PD\n",
            "13        0.827588    PD\n",
            "14        0.857927    PD\n",
            "15        0.897243    PD\n",
            "16        0.898053    PD\n",
            "17        0.909734    PD\n",
            "18        0.915562    PD\n",
            "19        0.934959    PD\n",
            "20        0.763966    NC\n",
            "21        0.835731    NC\n",
            "22        0.869767    NC\n",
            "23        0.927193    NC\n",
            "24        0.933094    NC\n",
            "25        0.950044    NC\n",
            "26        0.954504    NC\n",
            "27        0.967517    NC\n",
            "28        0.967382    NC\n",
            "29        0.974322    NC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0DzEskGzz4LF",
        "outputId": "5d15d8fa-8d67-43c9-d59e-a0ef24603b6f"
      },
      "source": [
        "# T-test between PD and NC data\n",
        "\n",
        "d1=(twoD_hvlv_df[twoD_hvlv_df['Data']=='PD'])\n",
        "d2=(twoD_hvlv_df[twoD_hvlv_df['Data']=='NC'])\n",
        "\n",
        "stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=-1.867, p=0.078\n",
            "Probably the same distribution\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eKJrGVhZz7Ez",
        "outputId": "bd8def43-c6aa-4b94-d46e-c1f84a1cdb3c"
      },
      "source": [
        "# HALA\n",
        "# Create a DataFrame\n",
        "twoD_hala_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  twoD_hala_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': twoD_hala_f1[0], 'Data': len(twoD_hala_f1[0])*[str(j)]})\n",
        "  twoD_hala_df = twoD_hala_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(twoD_hala_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Data\n",
            "0         0.923685  Full\n",
            "1         0.950382  Full\n",
            "2         0.958668  Full\n",
            "3         0.972180  Full\n",
            "4         0.974731  Full\n",
            "5         0.981648  Full\n",
            "6         0.986418  Full\n",
            "7         0.982373  Full\n",
            "8         0.979208  Full\n",
            "9         0.974574  Full\n",
            "10        0.915745    PD\n",
            "11        0.964817    PD\n",
            "12        0.974068    PD\n",
            "13        0.969131    PD\n",
            "14        0.979380    PD\n",
            "15        0.979868    PD\n",
            "16        0.987882    PD\n",
            "17        0.990544    PD\n",
            "18        0.993280    PD\n",
            "19        0.993236    PD\n",
            "20        0.901344    NC\n",
            "21        0.925787    NC\n",
            "22        0.926816    NC\n",
            "23        0.953311    NC\n",
            "24        0.956215    NC\n",
            "25        0.899013    NC\n",
            "26        0.964165    NC\n",
            "27        0.943790    NC\n",
            "28        0.973764    NC\n",
            "29        0.960798    NC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TYGJ_76W0C_m",
        "outputId": "4eeacaa8-ea2f-4789-f3eb-7b91ef696732"
      },
      "source": [
        "# T-test between PD and NC data\n",
        "\n",
        "d1=(twoD_hala_df[twoD_hala_df['Data']=='PD'])\n",
        "d2=(twoD_hala_df[twoD_hala_df['Data']=='NC'])\n",
        "\n",
        "stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=3.113, p=0.006\n",
            "Probably different distributions\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6jNeisE10GQM",
        "outputId": "3f6a2305-a355-4a73-a469-c7e9f3ae4471"
      },
      "source": [
        "# MultiClass\n",
        "# Create a DataFrame\n",
        "twoD_multi_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  twoD_multi_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': twoD_multi_f1[0], 'Data': len(twoD_multi_f1[0])*[str(j)]})\n",
        "  twoD_multi_df = twoD_multi_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(twoD_multi_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Data\n",
            "0         0.620795  Full\n",
            "1         0.718294  Full\n",
            "2         0.770470  Full\n",
            "3         0.818730  Full\n",
            "4         0.841174  Full\n",
            "5         0.858135  Full\n",
            "6         0.874894  Full\n",
            "7         0.874239  Full\n",
            "8         0.897352  Full\n",
            "9         0.923447  Full\n",
            "10        0.600889    PD\n",
            "11        0.700610    PD\n",
            "12        0.800858    PD\n",
            "13        0.842152    PD\n",
            "14        0.874829    PD\n",
            "15        0.895574    PD\n",
            "16        0.914753    PD\n",
            "17        0.909314    PD\n",
            "18        0.933672    PD\n",
            "19        0.931009    PD\n",
            "20        0.692563    NC\n",
            "21        0.765450    NC\n",
            "22        0.807074    NC\n",
            "23        0.855545    NC\n",
            "24        0.905693    NC\n",
            "25        0.885380    NC\n",
            "26        0.929489    NC\n",
            "27        0.922816    NC\n",
            "28        0.925835    NC\n",
            "29        0.941749    NC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fmml2MJL0KY2",
        "outputId": "88afb180-4f88-4b2b-9f5d-0be6a6eda611"
      },
      "source": [
        "# T-test between PD and NC data\n",
        "\n",
        "d1=(twoD_multi_df[twoD_multi_df['Data']=='PD'])\n",
        "d2=(twoD_multi_df[twoD_multi_df['Data']=='NC'])\n",
        "\n",
        "stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=-0.521, p=0.609\n",
            "Probably the same distribution\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LcXIbOAHzRrp",
        "outputId": "a0a6388e-af56-4c53-8b6c-62169b277aed"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "twoD_conf_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for j in Data:\n",
        "  print(\"\\n #####\", j, \"####\")\n",
        "  twoD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(twoD_conf, axis=0), '\\n')\n",
        "  l = 0\n",
        "  for k in emotions:\n",
        "    data = pd.DataFrame({'Accuracy': [i[l][l] for i in twoD_conf],'Emotions': len(twoD_conf)*[str(k)]})\n",
        "    twoD_conf_df = twoD_conf_df.append(data, ignore_index=True)\n",
        "    l = l+1\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  twoD_formula = 'Accuracy ~ C(Emotions)'\n",
        "  twoD_model = ols(twoD_formula, twoD_conf_df).fit()\n",
        "  twoD_aov_table = anova_lm(twoD_model, typ=2)\n",
        "  print(twoD_aov_table)\n",
        "  \n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  twoD_mc = MultiComparison(twoD_conf_df['Accuracy'], twoD_conf_df['Emotions'])\n",
        "  print(twoD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " ##### Full ####\n",
            "[[0.89709544 0.03568465 0.02489627 0.01991701 0.01327801 0.00912863]\n",
            " [0.04577972 0.84271906 0.04850643 0.01565257 0.03091605 0.01642616]\n",
            " [0.04249308 0.0474879  0.7867168  0.06665284 0.03790629 0.01874308]\n",
            " [0.01950207 0.02074689 0.0560166  0.81120332 0.06348548 0.02904564]\n",
            " [0.01924177 0.02647229 0.03208193 0.07257671 0.78347309 0.06615422]\n",
            " [0.02451377 0.02293274 0.02491373 0.05219273 0.07631752 0.79912949]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.094148   5.0  1.934937  0.103587\n",
            "Residual     0.525494  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0544 0.7959 -0.1847  0.076  False\n",
            "    E1     E3  -0.1104 0.1415 -0.2407   0.02  False\n",
            "    E1     E4  -0.0859 0.3872 -0.2162 0.0444  False\n",
            "    E1     E5  -0.1136 0.1214  -0.244 0.0167  False\n",
            "    E1     E6   -0.098 0.2454 -0.2283 0.0324  False\n",
            "    E2     E3   -0.056 0.7752 -0.1863 0.0743  False\n",
            "    E2     E4  -0.0315    0.9 -0.1619 0.0988  False\n",
            "    E2     E5  -0.0592 0.7338 -0.1896 0.0711  False\n",
            "    E2     E6  -0.0436    0.9 -0.1739 0.0868  False\n",
            "    E3     E4   0.0245    0.9 -0.1059 0.1548  False\n",
            "    E3     E5  -0.0032    0.9 -0.1336 0.1271  False\n",
            "    E3     E6   0.0124    0.9 -0.1179 0.1428  False\n",
            "    E4     E5  -0.0277    0.9 -0.1581 0.1026  False\n",
            "    E4     E6  -0.0121    0.9 -0.1424 0.1183  False\n",
            "    E5     E6   0.0157    0.9 -0.1147  0.146  False\n",
            "---------------------------------------------------\n",
            "\n",
            " ##### PD ####\n",
            "[[0.93       0.04       0.01       0.01083333 0.00333333 0.00583333]\n",
            " [0.04409449 0.85748031 0.0511811  0.01181102 0.02125984 0.01417323]\n",
            " [0.02325758 0.06319559 0.80543388 0.0482989  0.03240358 0.02741047]\n",
            " [0.01157025 0.01985537 0.05294077 0.82300964 0.05789945 0.03472452]\n",
            " [0.00887097 0.02903226 0.03306452 0.05564516 0.79758065 0.07580645]\n",
            " [0.00710536 0.0276153  0.02207849 0.04573178 0.06386702 0.83360205]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.208861    5.0  3.592449  0.004702\n",
            "Residual     1.325568  114.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0634 0.4336 -0.1623  0.0354  False\n",
            "    E1     E3  -0.1175 0.0101 -0.2163 -0.0186   True\n",
            "    E1     E4  -0.0964 0.0603 -0.1953  0.0024  False\n",
            "    E1     E5   -0.123  0.006 -0.2219 -0.0242   True\n",
            "    E1     E6  -0.0972 0.0569  -0.196  0.0017  False\n",
            "    E2     E3   -0.054 0.5957 -0.1529  0.0448  False\n",
            "    E2     E4   -0.033    0.9 -0.1318  0.0659  False\n",
            "    E2     E5  -0.0596  0.503 -0.1584  0.0393  False\n",
            "    E2     E6  -0.0337    0.9 -0.1326  0.0651  False\n",
            "    E3     E4    0.021    0.9 -0.0778  0.1199  False\n",
            "    E3     E5  -0.0055    0.9 -0.1044  0.0933  False\n",
            "    E3     E6   0.0203    0.9 -0.0786  0.1191  False\n",
            "    E4     E5  -0.0266    0.9 -0.1254  0.0723  False\n",
            "    E4     E6  -0.0007    0.9 -0.0996  0.0981  False\n",
            "    E5     E6   0.0258    0.9  -0.073  0.1247  False\n",
            "----------------------------------------------------\n",
            "\n",
            " ##### NC ####\n",
            "[[0.89586777 0.02066116 0.03553719 0.01900826 0.01570248 0.01322314]\n",
            " [0.02791909 0.90539002 0.03024467 0.00930233 0.02016715 0.00697674]\n",
            " [0.04083333 0.03833333 0.82       0.055      0.02333333 0.0225    ]\n",
            " [0.01913223 0.02165289 0.05162534 0.8476584  0.03579201 0.02413912]\n",
            " [0.02231746 0.02711746 0.02545397 0.02794921 0.82860952 0.06855238]\n",
            " [0.02142857 0.01587302 0.01429841 0.01984127 0.04763175 0.88092698]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df        F    PR(>F)\n",
            "C(Emotions)  0.245692    5.0  4.63608  0.000536\n",
            "Residual     1.844246  174.0      NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0391 0.6595 -0.1157  0.0375  False\n",
            "    E1     E3  -0.1036 0.0019 -0.1802  -0.027   True\n",
            "    E1     E4  -0.0804 0.0337  -0.157 -0.0038   True\n",
            "    E1     E5  -0.1044 0.0017  -0.181 -0.0278   True\n",
            "    E1     E6  -0.0698 0.0967 -0.1464  0.0068  False\n",
            "    E2     E3  -0.0645 0.1532 -0.1411  0.0121  False\n",
            "    E2     E4  -0.0412 0.6139 -0.1178  0.0354  False\n",
            "    E2     E5  -0.0653 0.1428 -0.1419  0.0113  False\n",
            "    E2     E6  -0.0306 0.8422 -0.1072   0.046  False\n",
            "    E3     E4   0.0232    0.9 -0.0534  0.0998  False\n",
            "    E3     E5  -0.0008    0.9 -0.0774  0.0758  False\n",
            "    E3     E6   0.0338 0.7734 -0.0428  0.1104  False\n",
            "    E4     E5  -0.0241    0.9 -0.1007  0.0525  False\n",
            "    E4     E6   0.0106    0.9  -0.066  0.0872  False\n",
            "    E5     E6   0.0347 0.7555 -0.0419  0.1113  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJxOalqOkRkZ",
        "outputId": "ad199e80-5b24-49d4-b0c5-27a711d1ffc6"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "twoD_conf_df = pd.DataFrame()\n",
        "Data = [\"NC\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for j in Data:\n",
        "  print(\"\\n #####\", j, \"####\")\n",
        "  twoD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(twoD_conf, axis=0), '\\n')\n",
        "  l = 0\n",
        "  for k in emotions:\n",
        "    data = pd.DataFrame({'Accuracy': [i[l][l] for i in twoD_conf],'Emotions': len(twoD_conf)*[str(k)]})\n",
        "    twoD_conf_df = twoD_conf_df.append(data, ignore_index=True)\n",
        "    l = l+1\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  twoD_formula = 'Accuracy ~ C(Emotions)'\n",
        "  twoD_model = ols(twoD_formula, twoD_conf_df).fit()\n",
        "  twoD_aov_table = anova_lm(twoD_model, typ=2)\n",
        "  print(twoD_aov_table)\n",
        "  \n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  twoD_mc = MultiComparison(twoD_conf_df['Accuracy'], twoD_conf_df['Emotions'])\n",
        "  print(twoD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " ##### NC ####\n",
            "[[0.89586777 0.02066116 0.03553719 0.01900826 0.01570248 0.01322314]\n",
            " [0.02791909 0.90539002 0.03024467 0.00930233 0.02016715 0.00697674]\n",
            " [0.04083333 0.03833333 0.82       0.055      0.02333333 0.0225    ]\n",
            " [0.01913223 0.02165289 0.05162534 0.8476584  0.03579201 0.02413912]\n",
            " [0.02231746 0.02711746 0.02545397 0.02794921 0.82860952 0.06855238]\n",
            " [0.02142857 0.01587302 0.01429841 0.01984127 0.04763175 0.88092698]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.064656   5.0  1.556176  0.188195\n",
            "Residual     0.448719  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2   0.0095    0.9 -0.1109   0.13  False\n",
            "    E1     E3  -0.0759 0.4387 -0.1963 0.0446  False\n",
            "    E1     E4  -0.0482 0.8241 -0.1687 0.0722  False\n",
            "    E1     E5  -0.0673 0.5611 -0.1877 0.0532  False\n",
            "    E1     E6  -0.0149    0.9 -0.1354 0.1055  False\n",
            "    E2     E3  -0.0854 0.3056 -0.2058 0.0351  False\n",
            "    E2     E4  -0.0577 0.6926 -0.1782 0.0627  False\n",
            "    E2     E5  -0.0768 0.4252 -0.1972 0.0437  False\n",
            "    E2     E6  -0.0245    0.9 -0.1449  0.096  False\n",
            "    E3     E4   0.0277    0.9 -0.0928 0.1481  False\n",
            "    E3     E5   0.0086    0.9 -0.1118 0.1291  False\n",
            "    E3     E6   0.0609 0.6485 -0.0595 0.1814  False\n",
            "    E4     E5   -0.019    0.9 -0.1395 0.1014  False\n",
            "    E4     E6   0.0333    0.9 -0.0872 0.1537  False\n",
            "    E5     E6   0.0523 0.7674 -0.0681 0.1728  False\n",
            "---------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4k70iKveFlkk"
      },
      "source": [
        "# 3D-CNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4sTtnG19L_s7"
      },
      "source": [
        "## HV vs LV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wkG0hMnHFpnW",
        "outputId": "9ccb5274-364c-4af9-c3fe-e9e91fdb122c"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "# HVLV\n",
        "# Create a DataFrame\n",
        "threeD_hvlv_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  threeD_hvlv_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': threeD_hvlv_f1[0], 'Data': len(threeD_hvlv_f1[0])*[str(j)]})\n",
        "  threeD_hvlv_df = threeD_hvlv_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(threeD_hvlv_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Data\n",
            "0         0.790588  Full\n",
            "1         0.839486  Full\n",
            "2         0.887201  Full\n",
            "3         0.926583  Full\n",
            "4         0.930645  Full\n",
            "5         0.947985  Full\n",
            "6         0.954076  Full\n",
            "7         0.946948  Full\n",
            "8         0.960508  Full\n",
            "9         0.966819  Full\n",
            "10        0.739236    PD\n",
            "11        0.818206    PD\n",
            "12        0.903556    PD\n",
            "13        0.916324    PD\n",
            "14        0.948029    PD\n",
            "15        0.951466    PD\n",
            "16        0.956859    PD\n",
            "17        0.960814    PD\n",
            "18        0.968789    PD\n",
            "19        0.952308    PD\n",
            "20        0.827938    NC\n",
            "21        0.858011    NC\n",
            "22        0.901931    NC\n",
            "23        0.935342    NC\n",
            "24        0.950137    NC\n",
            "25        0.966309    NC\n",
            "26        0.945915    NC\n",
            "27        0.962103    NC\n",
            "28        0.966146    NC\n",
            "29        0.959729    NC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "56s5kuwjJJx9",
        "outputId": "8062a554-a9ff-477c-d1af-ea126476d6bf"
      },
      "source": [
        "# T-test between PD and NC data\n",
        "\n",
        "d1=(threeD_hvlv_df[threeD_hvlv_df['Data']=='PD'])\n",
        "d2=(threeD_hvlv_df[threeD_hvlv_df['Data']=='NC'])\n",
        "\n",
        "stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=-0.557, p=0.585\n",
            "Probably the same distribution\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GXOSN3895mQo",
        "outputId": "f97c36ce-214a-4846-b37b-f35a8798db63"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  print(\"-----\", j, \"----\")\n",
        "  threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "  threeD_conf_lv = [i[0][0] for i in threeD_conf]\n",
        "  threeD_conf_hv = [i[1][1] for i in threeD_conf]\n",
        "  stat, p = ttest_ind(threeD_conf_lv, threeD_conf_hv,equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Full ----\n",
            "[[0.94563892 0.05436108]\n",
            " [0.14236489 0.85763511]] \n",
            "\n",
            "stat=2.674, p=0.019\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- PD ----\n",
            "[[0.94753173 0.05246827]\n",
            " [0.15498008 0.84501992]] \n",
            "\n",
            "stat=2.279, p=0.044\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- NC ----\n",
            "[[0.94418024 0.05581976]\n",
            " [0.10503165 0.89496835]] \n",
            "\n",
            "stat=1.928, p=0.073\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fC6A0HD4HPyA",
        "outputId": "6624dd49-d7fb-4218-ea36-83703710cd85"
      },
      "source": [
        "# How well is LV classified in PD data and NC data?\n",
        "print(\"How well is LV classified in PD data and NC data?\\n\")\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "\n",
        "j = \"PD\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_lv = [i[0][0] for i in threeD_conf]\n",
        "\n",
        "j = \"NC\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_hv = [i[0][0] for i in threeD_conf]\n",
        "\n",
        "print(\"--------\")\n",
        "stat, p = ttest_ind(threeD_conf_lv, threeD_conf_hv,equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')\n",
        "print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "How well is LV classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.94753173 0.05246827]\n",
            " [0.15498008 0.84501992]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.94418024 0.05581976]\n",
            " [0.10503165 0.89496835]] \n",
            "\n",
            "--------\n",
            "stat=0.173, p=0.865\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wyyKEPR7JE2T",
        "outputId": "f59b1328-e09a-4dd8-904a-bd4139663470"
      },
      "source": [
        "# How well is HV classified in PD data and NC data?\n",
        "print(\"How well is HV classified in PD data and NC data?\\n\")\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "\n",
        "j = \"PD\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_lv = [i[1][1] for i in threeD_conf]\n",
        "\n",
        "j = \"NC\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_hv = [i[1][1] for i in threeD_conf]\n",
        "\n",
        "print(\"--------\")\n",
        "stat, p = ttest_ind(threeD_conf_lv, threeD_conf_hv,equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')\n",
        "print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "How well is HV classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.94753173 0.05246827]\n",
            " [0.15498008 0.84501992]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.94418024 0.05581976]\n",
            " [0.10503165 0.89496835]] \n",
            "\n",
            "--------\n",
            "stat=-1.042, p=0.316\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVzmkl-DMEkH"
      },
      "source": [
        "## HA vs LA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qAzBQQVAMvvr",
        "outputId": "9d6e0d80-e449-468d-f033-c4b42319281a"
      },
      "source": [
        "# HALA\n",
        "# Create a DataFrame\n",
        "threeD_hala_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  threeD_hala_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': threeD_hala_f1[0], 'Data': len(threeD_hala_f1[0])*[str(j)]})\n",
        "  threeD_hala_df = threeD_hala_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(threeD_hala_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Data\n",
            "0         0.906655  Full\n",
            "1         0.953889  Full\n",
            "2         0.973492  Full\n",
            "3         0.976169  Full\n",
            "4         0.977531  Full\n",
            "5         0.978116  Full\n",
            "6         0.982432  Full\n",
            "7         0.985726  Full\n",
            "8         0.988405  Full\n",
            "9         0.981772  Full\n",
            "10        0.906497    PD\n",
            "11        0.942261    PD\n",
            "12        0.976878    PD\n",
            "13        0.989100    PD\n",
            "14        0.989023    PD\n",
            "15        0.977965    PD\n",
            "16        0.980991    PD\n",
            "17        0.980991    PD\n",
            "18        0.991870    PD\n",
            "19        0.982039    PD\n",
            "20        0.920423    NC\n",
            "21        0.935892    NC\n",
            "22        0.967161    NC\n",
            "23        0.973185    NC\n",
            "24        0.979856    NC\n",
            "25        0.976698    NC\n",
            "26        0.971317    NC\n",
            "27        0.967822    NC\n",
            "28        0.987834    NC\n",
            "29        0.983751    NC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5jc_d3pdM_a1",
        "outputId": "a6c22c46-f382-459a-ef1c-1095619d0f59"
      },
      "source": [
        "# T-test between PD and NC data\n",
        "\n",
        "d1=(threeD_hala_df[threeD_hala_df['Data']=='PD'])\n",
        "d2=(threeD_hala_df[threeD_hala_df['Data']=='NC'])\n",
        "\n",
        "stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=0.494, p=0.628\n",
            "Probably the same distribution\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mRrIp2HfJBzs",
        "outputId": "99e29dca-711e-4403-f7f3-afc2083b9374"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  print(\"-----\", j, \"----\")\n",
        "  threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "  threeD_conf_lv = [i[0][0] for i in threeD_conf]\n",
        "  threeD_conf_hv = [i[1][1] for i in threeD_conf]\n",
        "  stat, p = ttest_ind(threeD_conf_lv, threeD_conf_hv,equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- Full ----\n",
            "[[0.87883817 0.12116183]\n",
            " [0.01065375 0.98934625]] \n",
            "\n",
            "stat=-2.986, p=0.015\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- PD ----\n",
            "[[0.9075     0.0925    ]\n",
            " [0.01599458 0.98400542]] \n",
            "\n",
            "stat=-3.594, p=0.004\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "----- NC ----\n",
            "[[0.87768595 0.12231405]\n",
            " [0.01563633 0.98436367]] \n",
            "\n",
            "stat=-3.525, p=0.006\n",
            "Probably different distributions\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "17kvCl5IJQ-f",
        "outputId": "c9911caf-fdb0-4e51-a2f8-ca40ea7dc37e"
      },
      "source": [
        "# How well is LA classified in PD data and NC data?\n",
        "print(\"How well is LA classified in PD data and NC data?\\n\")\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "\n",
        "j = \"PD\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_la = [i[0][0] for i in threeD_conf]\n",
        "\n",
        "j = \"NC\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_ha = [i[0][0] for i in threeD_conf]\n",
        "\n",
        "print(\"--------\")\n",
        "stat, p = ttest_ind(threeD_conf_la, threeD_conf_ha,equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')\n",
        "print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "How well is LA classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.9075     0.0925    ]\n",
            " [0.01599458 0.98400542]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.87768595 0.12231405]\n",
            " [0.01563633 0.98436367]] \n",
            "\n",
            "--------\n",
            "stat=0.823, p=0.423\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SJSWQIJMJdDv",
        "outputId": "16a84e3b-2124-45d3-9a67-0ec0baaac415"
      },
      "source": [
        "# How well is HA classified in PD data and NC data?\n",
        "print(\"How well is HA classified in PD data and NC data?\\n\")\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "\n",
        "j = \"PD\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_la = [i[1][1] for i in threeD_conf]\n",
        "\n",
        "j = \"NC\"\n",
        "print(\"----\", j, \"----\\n\")\n",
        "threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'conf_mat_list')\n",
        "print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "threeD_conf_ha = [i[1][1] for i in threeD_conf]\n",
        "\n",
        "print(\"--------\")\n",
        "stat, p = ttest_ind(threeD_conf_la, threeD_conf_ha,equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')\n",
        "print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "How well is HA classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.9075     0.0925    ]\n",
            " [0.01599458 0.98400542]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.87768595 0.12231405]\n",
            " [0.01563633 0.98436367]] \n",
            "\n",
            "--------\n",
            "stat=-0.048, p=0.963\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pK53Ul8eMIl8"
      },
      "source": [
        "## Multiclass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eMSepYC_NDis",
        "outputId": "90cc6d50-a66b-42ae-d207-06e75efa8f6e"
      },
      "source": [
        "# MultiClass\n",
        "# Create a DataFrame\n",
        "threeD_multi_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  threeD_multi_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_MultiClass_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': threeD_multi_f1[0], 'Data': len(threeD_multi_f1[0])*[str(j)]})\n",
        "  threeD_multi_df = threeD_multi_df.append(data_df, ignore_index=True)\n",
        "\n",
        "print(threeD_multi_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    f1_weightedNet  Data\n",
            "0         0.635037  Full\n",
            "1         0.732841  Full\n",
            "2         0.785722  Full\n",
            "3         0.844952  Full\n",
            "4         0.850072  Full\n",
            "5         0.878076  Full\n",
            "6         0.880866  Full\n",
            "7         0.898718  Full\n",
            "8         0.914754  Full\n",
            "9         0.910205  Full\n",
            "10        0.660411    PD\n",
            "11        0.716499    PD\n",
            "12        0.832643    PD\n",
            "13        0.873765    PD\n",
            "14        0.883286    PD\n",
            "15        0.917377    PD\n",
            "16        0.918708    PD\n",
            "17        0.933641    PD\n",
            "18        0.952604    PD\n",
            "19        0.948443    PD\n",
            "20        0.701125    NC\n",
            "21        0.779141    NC\n",
            "22        0.825486    NC\n",
            "23        0.880068    NC\n",
            "24        0.904665    NC\n",
            "25        0.931382    NC\n",
            "26        0.918962    NC\n",
            "27        0.901533    NC\n",
            "28        0.948449    NC\n",
            "29        0.947362    NC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dg3_PPZ8NU9u",
        "outputId": "d65b507a-2e11-4d4a-d239-583c79d518ef"
      },
      "source": [
        "# T-test between PD and NC data\n",
        "\n",
        "d1=(threeD_multi_df[threeD_multi_df['Data']=='PD'])\n",
        "d2=(threeD_multi_df[threeD_multi_df['Data']=='NC'])\n",
        "\n",
        "stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "if p > 0.05:\n",
        "  print('Probably the same distribution')\n",
        "else:\n",
        "  print('Probably different distributions')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stat=-0.247, p=0.808\n",
            "Probably the same distribution\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EVgYHn2IJWCz",
        "outputId": "07988d10-1a11-4573-c496-d3b4f9372503"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for j in Data:\n",
        "  print(\"\\n #####\", j, \"####\")\n",
        "  threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_MultiClass_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "  l = 0\n",
        "  for k in emotions:\n",
        "    data = pd.DataFrame({'Accuracy': [i[l][l] for i in threeD_conf],'Emotions': len(threeD_conf)*[str(k)]})\n",
        "    threeD_conf_df = threeD_conf_df.append(data, ignore_index=True)\n",
        "    l = l+1\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  threeD_formula = 'Accuracy ~ C(Emotions)'\n",
        "  threeD_model = ols(threeD_formula, threeD_conf_df).fit()\n",
        "  threeD_aov_table = anova_lm(threeD_model, typ=2)\n",
        "  print(threeD_aov_table)\n",
        "  \n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  threeD_mc = MultiComparison(threeD_conf_df['Accuracy'], threeD_conf_df['Emotions'])\n",
        "  print(threeD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " ##### Full ####\n",
            "[[0.8846473  0.03983402 0.02365145 0.02697095 0.01493776 0.00995851]\n",
            " [0.03674786 0.86080116 0.04536612 0.01290288 0.02971048 0.01447151]\n",
            " [0.02957296 0.05289938 0.81506224 0.05165284 0.02707123 0.02374136]\n",
            " [0.01991701 0.02033195 0.05477178 0.82282158 0.04605809 0.03609959]\n",
            " [0.01763052 0.03605462 0.02885462 0.04728675 0.80084337 0.06933012]\n",
            " [0.01227806 0.02929607 0.03205189 0.05026507 0.05976222 0.8163467 ]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.051760   5.0  1.108912  0.366588\n",
            "Residual     0.504108  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0238    0.9 -0.1515 0.1038  False\n",
            "    E1     E3  -0.0696 0.5833 -0.1972 0.0581  False\n",
            "    E1     E4  -0.0618 0.6844 -0.1895 0.0658  False\n",
            "    E1     E5  -0.0838 0.3916 -0.2115 0.0439  False\n",
            "    E1     E6  -0.0683    0.6  -0.196 0.0594  False\n",
            "    E2     E3  -0.0457 0.8939 -0.1734 0.0819  False\n",
            "    E2     E4   -0.038    0.9 -0.1656 0.0897  False\n",
            "    E2     E5    -0.06 0.7087 -0.1876 0.0677  False\n",
            "    E2     E6  -0.0445    0.9 -0.1721 0.0832  False\n",
            "    E3     E4   0.0078    0.9 -0.1199 0.1354  False\n",
            "    E3     E5  -0.0142    0.9 -0.1419 0.1134  False\n",
            "    E3     E6   0.0013    0.9 -0.1264 0.1289  False\n",
            "    E4     E5   -0.022    0.9 -0.1496 0.1057  False\n",
            "    E4     E6  -0.0065    0.9 -0.1341 0.1212  False\n",
            "    E5     E6   0.0155    0.9 -0.1122 0.1432  False\n",
            "---------------------------------------------------\n",
            "\n",
            " ##### PD ####\n",
            "[[0.92833333 0.035      0.01333333 0.0075     0.005      0.01083333]\n",
            " [0.02677165 0.87637795 0.04094488 0.00629921 0.02755906 0.02204724]\n",
            " [0.0125     0.04166667 0.84502066 0.03915978 0.02832645 0.03332645]\n",
            " [0.01570937 0.01322314 0.05126722 0.82802342 0.04629477 0.04548209]\n",
            " [0.01290323 0.02258065 0.02580645 0.03387097 0.82177419 0.08306452]\n",
            " [0.00472441 0.01575428 0.01574803 0.02285339 0.05829271 0.88262717]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F   PR(>F)\n",
            "C(Emotions)  0.120478    5.0  2.362845  0.04422\n",
            "Residual     1.162538  114.0       NaN      NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0379 0.8224 -0.1305  0.0547  False\n",
            "    E1     E3  -0.0764 0.1672  -0.169  0.0161  False\n",
            "    E1     E4  -0.0811  0.122 -0.1736  0.0115  False\n",
            "    E1     E5  -0.0952 0.0401 -0.1877 -0.0026   True\n",
            "    E1     E6   -0.057 0.4802 -0.1496  0.0356  False\n",
            "    E2     E3  -0.0385 0.8108 -0.1311   0.054  False\n",
            "    E2     E4  -0.0432 0.7284 -0.1357  0.0494  False\n",
            "    E2     E5  -0.0573 0.4749 -0.1498  0.0353  False\n",
            "    E2     E6  -0.0191    0.9 -0.1117  0.0735  False\n",
            "    E3     E4  -0.0046    0.9 -0.0972  0.0879  False\n",
            "    E3     E5  -0.0187    0.9 -0.1113  0.0738  False\n",
            "    E3     E6   0.0194    0.9 -0.0731   0.112  False\n",
            "    E4     E5  -0.0141    0.9 -0.1067  0.0785  False\n",
            "    E4     E6   0.0241    0.9 -0.0685  0.1166  False\n",
            "    E5     E6   0.0382 0.8174 -0.0544  0.1307  False\n",
            "----------------------------------------------------\n",
            "\n",
            " ##### NC ####\n",
            "[[0.91735537 0.01157025 0.01983471 0.02066116 0.0214876  0.00909091]\n",
            " [0.02487888 0.89742006 0.02950581 0.01243944 0.02487282 0.01088299]\n",
            " [0.0375     0.02833333 0.85833333 0.03916667 0.02333333 0.01333333]\n",
            " [0.01826446 0.01491047 0.03568182 0.86221074 0.0415427  0.02738981]\n",
            " [0.01672381 0.02231111 0.02548571 0.03346032 0.83988571 0.06213333]\n",
            " [0.00954921 0.01352381 0.02622222 0.03420317 0.0484381  0.86806349]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq     df         F    PR(>F)\n",
            "C(Emotions)  0.157809    5.0  3.402088  0.005869\n",
            "Residual     1.614228  174.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05 \n",
            "====================================================\n",
            "group1 group2 meandiff p-adj   lower   upper  reject\n",
            "----------------------------------------------------\n",
            "    E1     E2  -0.0319 0.7675 -0.1036  0.0398  False\n",
            "    E1     E3  -0.0706 0.0559 -0.1423   0.001  False\n",
            "    E1     E4  -0.0724  0.046 -0.1441 -0.0008   True\n",
            "    E1     E5  -0.0893 0.0057 -0.1609 -0.0176   True\n",
            "    E1     E6  -0.0544 0.2482 -0.1261  0.0172  False\n",
            "    E2     E3  -0.0387 0.6106 -0.1104  0.0329  False\n",
            "    E2     E4  -0.0405 0.5694 -0.1122  0.0312  False\n",
            "    E2     E5  -0.0574 0.1969  -0.129  0.0143  False\n",
            "    E2     E6  -0.0225    0.9 -0.0942  0.0491  False\n",
            "    E3     E4  -0.0018    0.9 -0.0735  0.0699  False\n",
            "    E3     E5  -0.0186    0.9 -0.0903   0.053  False\n",
            "    E3     E6   0.0162    0.9 -0.0555  0.0879  False\n",
            "    E4     E5  -0.0169    0.9 -0.0885  0.0548  False\n",
            "    E4     E6    0.018    0.9 -0.0537  0.0897  False\n",
            "    E5     E6   0.0348    0.7 -0.0368  0.1065  False\n",
            "----------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "45Vt_bTkkftY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "39459d8e-06db-4f89-cd39-b3bfda755291"
      },
      "source": [
        "# conf mat T-tests\n",
        "\n",
        "threeD_conf_df = pd.DataFrame()\n",
        "Data = [\"NC\"]\n",
        "emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "\n",
        "for j in Data:\n",
        "  print(\"\\n #####\", j, \"####\")\n",
        "  threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_MultiClass_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "  l = 0\n",
        "  for k in emotions:\n",
        "    data = pd.DataFrame({'Accuracy': [i[l][l] for i in threeD_conf],'Emotions': len(threeD_conf)*[str(k)]})\n",
        "    threeD_conf_df = threeD_conf_df.append(data, ignore_index=True)\n",
        "    l = l+1\n",
        "\n",
        "  print('\\n--- One way Anova ---')\n",
        "\n",
        "  threeD_formula = 'Accuracy ~ C(Emotions)'\n",
        "  threeD_model = ols(threeD_formula, threeD_conf_df).fit()\n",
        "  threeD_aov_table = anova_lm(threeD_model, typ=2)\n",
        "  print(threeD_aov_table)\n",
        "  \n",
        "  print('\\n--- Tukey HSD ----')\n",
        "  # Tukey HSD test\n",
        "\n",
        "  threeD_mc = MultiComparison(threeD_conf_df['Accuracy'], threeD_conf_df['Emotions'])\n",
        "  print(threeD_mc.tukeyhsd())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " ##### NC ####\n",
            "[[0.92809917 0.01818182 0.0214876  0.01404959 0.01157025 0.00661157]\n",
            " [0.01400194 0.92079094 0.02561773 0.01008963 0.01862282 0.01087694]\n",
            " [0.0175     0.0175     0.895      0.03583333 0.0175     0.01666667]\n",
            " [0.01742424 0.0107989  0.02573003 0.88132231 0.03898072 0.0257438 ]\n",
            " [0.01274921 0.01914286 0.02069841 0.02548571 0.88688889 0.03503492]\n",
            " [0.01033016 0.01031746 0.01192381 0.02307302 0.03659683 0.90775873]] \n",
            "\n",
            "\n",
            "--- One way Anova ---\n",
            "               sum_sq    df         F    PR(>F)\n",
            "C(Emotions)  0.017620   5.0  0.524307  0.756807\n",
            "Residual     0.362957  54.0       NaN       NaN\n",
            "\n",
            "--- Tukey HSD ----\n",
            "Multiple Comparison of Means - Tukey HSD, FWER=0.05\n",
            "===================================================\n",
            "group1 group2 meandiff p-adj   lower  upper  reject\n",
            "---------------------------------------------------\n",
            "    E1     E2  -0.0073    0.9 -0.1156  0.101  False\n",
            "    E1     E3  -0.0331    0.9 -0.1414 0.0752  False\n",
            "    E1     E4  -0.0468 0.7716 -0.1551 0.0615  False\n",
            "    E1     E5  -0.0412  0.857 -0.1495 0.0671  False\n",
            "    E1     E6  -0.0203    0.9 -0.1287  0.088  False\n",
            "    E2     E3  -0.0258    0.9 -0.1341 0.0825  False\n",
            "    E2     E4  -0.0395 0.8838 -0.1478 0.0689  False\n",
            "    E2     E5  -0.0339    0.9 -0.1422 0.0744  False\n",
            "    E2     E6   -0.013    0.9 -0.1214 0.0953  False\n",
            "    E3     E4  -0.0137    0.9  -0.122 0.0946  False\n",
            "    E3     E5  -0.0081    0.9 -0.1164 0.1002  False\n",
            "    E3     E6   0.0128    0.9 -0.0956 0.1211  False\n",
            "    E4     E5   0.0056    0.9 -0.1028 0.1139  False\n",
            "    E4     E6   0.0264    0.9 -0.0819 0.1348  False\n",
            "    E5     E6   0.0209    0.9 -0.0875 0.1292  False\n",
            "---------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DJR8Fv6GKC2Q",
        "outputId": "ad2fcc14-55be-4f2e-a639-1b4a4867e8ea"
      },
      "source": [
        "Emotions = [\"E1\", \"E2\", \"E3\", \"E4\", \"E5\", \"E6\"]\n",
        "k = 0\n",
        "for emot in emotions:\n",
        "  print(10*\"*\", emot, 10*\"*\")\n",
        "  # How well is LV classified in PD data and NC data?\n",
        "  print(\"How well is\", emot, \"classified in PD data and NC data?\\n\")\n",
        "\n",
        "  threeD_conf_df = pd.DataFrame()\n",
        "\n",
        "  j = \"PD\"\n",
        "  print(\"----\", j, \"----\\n\")\n",
        "  threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_MultiClass_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "  threeD_conf_1 = [i[k][k] for i in threeD_conf]\n",
        "\n",
        "  j = \"NC\"\n",
        "  print(\"----\", j, \"----\\n\")\n",
        "  threeD_conf = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_MultiClass_'+str(j)+'.mat', 'conf_mat_list')\n",
        "  print(np.mean(threeD_conf, axis=0), '\\n')\n",
        "  threeD_conf_2 = [i[k][k] for i in threeD_conf]\n",
        "\n",
        "  print(\"--------\")\n",
        "  stat, p = ttest_ind(threeD_conf_1, threeD_conf_2,equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')\n",
        "  k = k+1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** E1 **********\n",
            "How well is E1 classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.92833333 0.035      0.01333333 0.0075     0.005      0.01083333]\n",
            " [0.02677165 0.87637795 0.04094488 0.00629921 0.02755906 0.02204724]\n",
            " [0.0125     0.04166667 0.84502066 0.03915978 0.02832645 0.03332645]\n",
            " [0.01570937 0.01322314 0.05126722 0.82802342 0.04629477 0.04548209]\n",
            " [0.01290323 0.02258065 0.02580645 0.03387097 0.82177419 0.08306452]\n",
            " [0.00472441 0.01575428 0.01574803 0.02285339 0.05829271 0.88262717]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.91735537 0.01157025 0.01983471 0.02066116 0.0214876  0.00909091]\n",
            " [0.02487888 0.89742006 0.02950581 0.01243944 0.02487282 0.01088299]\n",
            " [0.0375     0.02833333 0.85833333 0.03916667 0.02333333 0.01333333]\n",
            " [0.01826446 0.01491047 0.03568182 0.86221074 0.0415427  0.02738981]\n",
            " [0.01672381 0.02231111 0.02548571 0.03346032 0.83988571 0.06213333]\n",
            " [0.00954921 0.01352381 0.02622222 0.03420317 0.0484381  0.86806349]] \n",
            "\n",
            "--------\n",
            "stat=0.362, p=0.722\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E2 **********\n",
            "How well is E2 classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.92833333 0.035      0.01333333 0.0075     0.005      0.01083333]\n",
            " [0.02677165 0.87637795 0.04094488 0.00629921 0.02755906 0.02204724]\n",
            " [0.0125     0.04166667 0.84502066 0.03915978 0.02832645 0.03332645]\n",
            " [0.01570937 0.01322314 0.05126722 0.82802342 0.04629477 0.04548209]\n",
            " [0.01290323 0.02258065 0.02580645 0.03387097 0.82177419 0.08306452]\n",
            " [0.00472441 0.01575428 0.01574803 0.02285339 0.05829271 0.88262717]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.91735537 0.01157025 0.01983471 0.02066116 0.0214876  0.00909091]\n",
            " [0.02487888 0.89742006 0.02950581 0.01243944 0.02487282 0.01088299]\n",
            " [0.0375     0.02833333 0.85833333 0.03916667 0.02333333 0.01333333]\n",
            " [0.01826446 0.01491047 0.03568182 0.86221074 0.0415427  0.02738981]\n",
            " [0.01672381 0.02231111 0.02548571 0.03346032 0.83988571 0.06213333]\n",
            " [0.00954921 0.01352381 0.02622222 0.03420317 0.0484381  0.86806349]] \n",
            "\n",
            "--------\n",
            "stat=-0.570, p=0.576\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E3 **********\n",
            "How well is E3 classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.92833333 0.035      0.01333333 0.0075     0.005      0.01083333]\n",
            " [0.02677165 0.87637795 0.04094488 0.00629921 0.02755906 0.02204724]\n",
            " [0.0125     0.04166667 0.84502066 0.03915978 0.02832645 0.03332645]\n",
            " [0.01570937 0.01322314 0.05126722 0.82802342 0.04629477 0.04548209]\n",
            " [0.01290323 0.02258065 0.02580645 0.03387097 0.82177419 0.08306452]\n",
            " [0.00472441 0.01575428 0.01574803 0.02285339 0.05829271 0.88262717]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.91735537 0.01157025 0.01983471 0.02066116 0.0214876  0.00909091]\n",
            " [0.02487888 0.89742006 0.02950581 0.01243944 0.02487282 0.01088299]\n",
            " [0.0375     0.02833333 0.85833333 0.03916667 0.02333333 0.01333333]\n",
            " [0.01826446 0.01491047 0.03568182 0.86221074 0.0415427  0.02738981]\n",
            " [0.01672381 0.02231111 0.02548571 0.03346032 0.83988571 0.06213333]\n",
            " [0.00954921 0.01352381 0.02622222 0.03420317 0.0484381  0.86806349]] \n",
            "\n",
            "--------\n",
            "stat=-0.275, p=0.787\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E4 **********\n",
            "How well is E4 classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.92833333 0.035      0.01333333 0.0075     0.005      0.01083333]\n",
            " [0.02677165 0.87637795 0.04094488 0.00629921 0.02755906 0.02204724]\n",
            " [0.0125     0.04166667 0.84502066 0.03915978 0.02832645 0.03332645]\n",
            " [0.01570937 0.01322314 0.05126722 0.82802342 0.04629477 0.04548209]\n",
            " [0.01290323 0.02258065 0.02580645 0.03387097 0.82177419 0.08306452]\n",
            " [0.00472441 0.01575428 0.01574803 0.02285339 0.05829271 0.88262717]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.91735537 0.01157025 0.01983471 0.02066116 0.0214876  0.00909091]\n",
            " [0.02487888 0.89742006 0.02950581 0.01243944 0.02487282 0.01088299]\n",
            " [0.0375     0.02833333 0.85833333 0.03916667 0.02333333 0.01333333]\n",
            " [0.01826446 0.01491047 0.03568182 0.86221074 0.0415427  0.02738981]\n",
            " [0.01672381 0.02231111 0.02548571 0.03346032 0.83988571 0.06213333]\n",
            " [0.00954921 0.01352381 0.02622222 0.03420317 0.0484381  0.86806349]] \n",
            "\n",
            "--------\n",
            "stat=-0.617, p=0.546\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E5 **********\n",
            "How well is E5 classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.92833333 0.035      0.01333333 0.0075     0.005      0.01083333]\n",
            " [0.02677165 0.87637795 0.04094488 0.00629921 0.02755906 0.02204724]\n",
            " [0.0125     0.04166667 0.84502066 0.03915978 0.02832645 0.03332645]\n",
            " [0.01570937 0.01322314 0.05126722 0.82802342 0.04629477 0.04548209]\n",
            " [0.01290323 0.02258065 0.02580645 0.03387097 0.82177419 0.08306452]\n",
            " [0.00472441 0.01575428 0.01574803 0.02285339 0.05829271 0.88262717]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.91735537 0.01157025 0.01983471 0.02066116 0.0214876  0.00909091]\n",
            " [0.02487888 0.89742006 0.02950581 0.01243944 0.02487282 0.01088299]\n",
            " [0.0375     0.02833333 0.85833333 0.03916667 0.02333333 0.01333333]\n",
            " [0.01826446 0.01491047 0.03568182 0.86221074 0.0415427  0.02738981]\n",
            " [0.01672381 0.02231111 0.02548571 0.03346032 0.83988571 0.06213333]\n",
            " [0.00954921 0.01352381 0.02622222 0.03420317 0.0484381  0.86806349]] \n",
            "\n",
            "--------\n",
            "stat=-0.359, p=0.724\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** E6 **********\n",
            "How well is E6 classified in PD data and NC data?\n",
            "\n",
            "---- PD ----\n",
            "\n",
            "[[0.92833333 0.035      0.01333333 0.0075     0.005      0.01083333]\n",
            " [0.02677165 0.87637795 0.04094488 0.00629921 0.02755906 0.02204724]\n",
            " [0.0125     0.04166667 0.84502066 0.03915978 0.02832645 0.03332645]\n",
            " [0.01570937 0.01322314 0.05126722 0.82802342 0.04629477 0.04548209]\n",
            " [0.01290323 0.02258065 0.02580645 0.03387097 0.82177419 0.08306452]\n",
            " [0.00472441 0.01575428 0.01574803 0.02285339 0.05829271 0.88262717]] \n",
            "\n",
            "---- NC ----\n",
            "\n",
            "[[0.91735537 0.01157025 0.01983471 0.02066116 0.0214876  0.00909091]\n",
            " [0.02487888 0.89742006 0.02950581 0.01243944 0.02487282 0.01088299]\n",
            " [0.0375     0.02833333 0.85833333 0.03916667 0.02333333 0.01333333]\n",
            " [0.01826446 0.01491047 0.03568182 0.86221074 0.0415427  0.02738981]\n",
            " [0.01672381 0.02231111 0.02548571 0.03346032 0.83988571 0.06213333]\n",
            " [0.00954921 0.01352381 0.02622222 0.03420317 0.0484381  0.86806349]] \n",
            "\n",
            "--------\n",
            "stat=0.398, p=0.695\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qotz46ZTzjSW"
      },
      "source": [
        "# Comparisons"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YObv1yKN4mQC"
      },
      "source": [
        "HV vs LV Full data: Is there a significant difference between 2D and 3D CNN?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZGOvfxmx8BiT",
        "outputId": "06f5227f-3685-4290-aabb-dfbab05d4966"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "# HVLV 2D\n",
        "twoD_hvlv_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  twoD_hvlv_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': twoD_hvlv_f1[0], 'Data': len(twoD_hvlv_f1[0])*[str(j)]})\n",
        "  twoD_hvlv_df = twoD_hvlv_df.append(data_df, ignore_index=True)\n",
        "\n",
        "#print(twoD_hvlv_df)\n",
        "\n",
        "# HVLV 3D\n",
        "# Create a DataFrame\n",
        "threeD_hvlv_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  threeD_hvlv_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': threeD_hvlv_f1[0], 'Data': len(threeD_hvlv_f1[0])*[str(j)]})\n",
        "  threeD_hvlv_df = threeD_hvlv_df.append(data_df, ignore_index=True)\n",
        "\n",
        "#print(threeD_hvlv_df)\n",
        "\n",
        "# T-test between 2D and 3D results\n",
        "\n",
        "for j in Data:\n",
        "  print(10*\"*\", j, 10*\"*\")\n",
        "  d1=(twoD_hvlv_df[twoD_hvlv_df['Data']==j])\n",
        "  d2=(threeD_hvlv_df[threeD_hvlv_df['Data']==j])\n",
        "\n",
        "  stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Full **********\n",
            "stat=-0.883, p=0.389\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** PD **********\n",
            "stat=-1.706, p=0.105\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** NC **********\n",
            "stat=-0.484, p=0.635\n",
            "Probably the same distribution\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ft2EBK_q660W"
      },
      "source": [
        "HA vs LA: Is there a significant difference between 2D and 3D CNN?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sgOKw4Do7EMK",
        "outputId": "99b32b1c-532d-4f91-9974-be953f698128"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "# HALA 2D\n",
        "twoD_hala_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  twoD_hala_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': twoD_hala_f1[0], 'Data': len(twoD_hala_f1[0])*[str(j)]})\n",
        "  twoD_hala_df = twoD_hala_df.append(data_df, ignore_index=True)\n",
        "\n",
        "#print(twoD_hvlv_df)\n",
        "\n",
        "# HALA 3D\n",
        "# Create a DataFrame\n",
        "threeD_hala_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  threeD_hala_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': threeD_hala_f1[0], 'Data': len(threeD_hala_f1[0])*[str(j)]})\n",
        "  threeD_hala_df = threeD_hala_df.append(data_df, ignore_index=True)\n",
        "\n",
        "#print(threeD_hvlv_df)\n",
        "\n",
        "# T-test between Full, 2D and 3D results\n",
        "\n",
        "for j in Data:\n",
        "  print(10*\"*\", j, 10*\"*\")\n",
        "  d1=(twoD_hala_df[twoD_hala_df['Data']==j])\n",
        "  d2=(threeD_hala_df[threeD_hala_df['Data']==j])\n",
        "\n",
        "  stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Full **********\n",
            "stat=-0.207, p=0.838\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** PD **********\n",
            "stat=0.271, p=0.789\n",
            "Probably the same distribution\n",
            "\n",
            "\n",
            "********** NC **********\n",
            "stat=-2.418, p=0.027\n",
            "Probably different distributions\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wPEXBJO59IDP"
      },
      "source": [
        "MultiClass: Is there a significant difference between 2D and 3D CNN?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EvD7j1sa8lML",
        "outputId": "c5b097de-def1-41e0-bc74-5ff8c95fbc19"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "# MultiClass 2D\n",
        "twoD_multi_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  twoD_multi_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': twoD_multi_f1[0], 'Data': len(twoD_multi_f1[0])*[str(j)]})\n",
        "  twoD_multi_df = twoD_multi_df.append(data_df, ignore_index=True)\n",
        "\n",
        "#print(twoD_multi_df)\n",
        "\n",
        "# MultiClass 3D\n",
        "# Create a DataFrame\n",
        "threeD_multi_df = pd.DataFrame()\n",
        "Data = [\"Full\", \"PD\", \"NC\"]\n",
        "for j in Data:\n",
        "  threeD_multi_f1 = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(j)+'.mat', 'f1_weightedNet')\n",
        "  data_df = pd.DataFrame({'f1_weightedNet': threeD_multi_f1[0], 'Data': len(threeD_multi_f1[0])*[str(j)]})\n",
        "  threeD_multi_df = threeD_multi_df.append(data_df, ignore_index=True)\n",
        "\n",
        "#print(threeD_multi_df)\n",
        "\n",
        "# T-test between Full, 2D and 3D results\n",
        "\n",
        "for j in Data:\n",
        "  print(10*\"*\", j, 10*\"*\")\n",
        "  d1=(twoD_multi_df[twoD_multi_df['Data']==j])\n",
        "  d2=(threeD_multi_df[threeD_multi_df['Data']==j])\n",
        "\n",
        "  stat, p = ttest_ind(d1['f1_weightedNet'].to_numpy(), d2['f1_weightedNet'].to_numpy(),equal_var=False)\n",
        "  print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "  if p > 0.05:\n",
        "    print('Probably the same distribution')\n",
        "  else:\n",
        "    print('Probably different distributions')\n",
        "  print('\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "********** Full **********\n",
            "stat=-4.981, p=0.001\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "********** PD **********\n",
            "stat=-3.654, p=0.004\n",
            "Probably different distributions\n",
            "\n",
            "\n",
            "********** NC **********\n",
            "stat=-3.801, p=0.003\n",
            "Probably different distributions\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MGZe0mBbLDPq"
      },
      "source": [
        "# Archive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BO_nm_64RmY8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 426
        },
        "outputId": "05cf0dd4-09f1-4f40-9142-cb998d984b9c"
      },
      "source": [
        "pdnc_full_2d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_Full.mat')\n",
        "pdnc_full_1d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_1D/CNN_1D_Spectral/CNN_1D_results_PDNC_spectral_Full.mat')\n",
        "\n",
        "hvlv_full_2d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_Full.mat')\n",
        "hvlv_full_1d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_1D/CNN_1D_Spectral/CNN_1D_results_HVLV_spectral_Full.mat')\n",
        "\n",
        "hala_full_2d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_Full.mat')\n",
        "hala_full_1d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_1D/CNN_1D_Spectral/CNN_1D_results_HALA_spectral_Full.mat')\n",
        "\n",
        "multi_full_2d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_Full.mat')\n",
        "multi_full_1d = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_1D/CNN_1D_Spectral/CNN_1D_results_MultiClass_spectral_Full.mat')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/scipy/io/matlab/mio.py\u001b[0m in \u001b[0;36m_open_file\u001b[0;34m(file_like, appendmat, mode)\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_like\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_Full.mat'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-61-6c1c97331dae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpdnc_full_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadmat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_Full.mat'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mpdnc_full_1d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadmat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_1D/CNN_1D_Spectral/CNN_1D_results_PDNC_spectral_Full.mat'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mhvlv_full_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadmat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_Full.mat'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mhvlv_full_1d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadmat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_1D/CNN_1D_Spectral/CNN_1D_results_HVLV_spectral_Full.mat'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/scipy/io/matlab/mio.py\u001b[0m in \u001b[0;36mloadmat\u001b[0;34m(file_name, mdict, appendmat, **kwargs)\u001b[0m\n\u001b[1;32m    214\u001b[0m     \"\"\"\n\u001b[1;32m    215\u001b[0m     \u001b[0mvariable_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'variable_names'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0m_open_file_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mappendmat\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    217\u001b[0m         \u001b[0mMR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmat_reader_factory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m         \u001b[0mmatfile_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMR\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvariable_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    110\u001b[0m         \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/scipy/io/matlab/mio.py\u001b[0m in \u001b[0;36m_open_file_context\u001b[0;34m(file_like, appendmat, mode)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mcontextmanager\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_file_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_like\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mappendmat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m     \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopened\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_like\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mappendmat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m     \u001b[0;32myield\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopened\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/scipy/io/matlab/mio.py\u001b[0m in \u001b[0;36m_open_file\u001b[0;34m(file_like, appendmat, mode)\u001b[0m\n\u001b[1;32m     43\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mappendmat\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfile_like\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'.mat'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m                 \u001b[0mfile_like\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m'.mat'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_like\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Reader needs file name or open file-like object'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_Full.mat'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 215
        },
        "id": "igZFt4LhQSoM",
        "outputId": "6340f9c5-f0df-459b-d377-b95ce9d52feb"
      },
      "source": [
        "pdnc_acc_2d = pdnc_full_2d['accNet']\n",
        "pdnc_acc_1d = pdnc_full_1d['accNet']\n",
        "\n",
        "stats.ttest_ind(pdnc_acc_2d[0],pdnc_acc_1d[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-62-521080b2fb30>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpdnc_acc_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpdnc_full_2d\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accNet'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mpdnc_acc_1d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpdnc_full_1d\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accNet'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mstats\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mttest_ind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpdnc_acc_2d\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpdnc_acc_1d\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'pdnc_full_2d' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2sQ8M0hSWpt"
      },
      "source": [
        "hvlv_f1_2d = hvlv_full_2d['f1_weightedNet']\n",
        "hvlv_f1_1d = hvlv_full_1d['f1_weightedNet']\n",
        "\n",
        "stats.ttest_ind(hvlv_f1_2d[0],hvlv_f1_1d[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w5DxbIBFSrvI"
      },
      "source": [
        "hala_f1_2d = hala_full_2d['f1_weightedNet']\n",
        "hala_f1_1d = hala_full_1d['f1_weightedNet']\n",
        "\n",
        "stats.ttest_ind(hala_f1_2d[0],hala_f1_1d[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvIcHdmnSrp-"
      },
      "source": [
        "multi_acc_2d = multi_full_2d['accNet']\n",
        "multi_acc_1d = multi_full_1d['accNet']\n",
        "\n",
        "stats.ttest_ind(multi_acc_2d[0],multi_acc_1d[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qSEfAph8rZL"
      },
      "source": [
        "pd_hala = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_PD.mat')\n",
        "nc_hala = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_NC.mat')\n",
        "\n",
        "pd_hvlv = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_PD.mat')\n",
        "nc_hvlv = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_NC.mat')\n",
        "\n",
        "pd_multi = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_PD.mat')\n",
        "nc_multi = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_NC.mat')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5_SMJGvx9zil"
      },
      "source": [
        "pd_multi.keys()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xBoqlgQH-IFo"
      },
      "source": [
        "pd_f1_hvlv = pd_hvlv['f1_weightedNet']\n",
        "nc_f1_hvlv = nc_hvlv['f1_weightedNet']\n",
        "\n",
        "stats.ttest_ind(pd_f1_hvlv[0],nc_f1_hvlv[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z08WebCKLoN-"
      },
      "source": [
        "pd_f1_hala = pd_hala['f1_weightedNet']\n",
        "nc_f1_hala = nc_hala['f1_weightedNet']\n",
        "\n",
        "stats.ttest_ind(pd_f1_hala[0],nc_f1_hala[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m1boOajvYKeW"
      },
      "source": [
        "pd_f1_multi = pd_multi['f1_weightedNet']\n",
        "nc_f1_multi = nc_multi['f1_weightedNet']\n",
        "\n",
        "stats.ttest_ind(pd_f1_multi[0],nc_f1_multi[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q6PsVkizbH67"
      },
      "source": [
        "stats.ttest_ind(pd_f1_hvlv[0],pd_f1_hala[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LhZ84IYJk32X"
      },
      "source": [
        "stats.ttest_ind(nc_f1_hvlv[0],nc_f1_hala[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lRo3yESfThzT"
      },
      "source": [
        "e1_pdnc = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_E1.mat')\n",
        "\n",
        "e2_pdnc = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_E2.mat')\n",
        "\n",
        "e3_pdnc = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_E3.mat')\n",
        "\n",
        "e4_pdnc = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_E4.mat')\n",
        "\n",
        "e5_pdnc = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_E5.mat')\n",
        "\n",
        "e6_pdnc = sio.loadmat('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_E6.mat')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jNtJcuUTUasf"
      },
      "source": [
        "e1_acc = e1_pdnc['accNet']\n",
        "e2_acc = e2_pdnc['accNet']\n",
        "e3_acc = e3_pdnc['accNet']\n",
        "e4_acc = e4_pdnc['accNet']\n",
        "e5_acc = e5_pdnc['accNet']\n",
        "e6_acc = e6_pdnc['accNet']\n",
        "\n",
        "stats.f_oneway(e1_acc[0], e2_acc[0], e3_acc[0], e4_acc[0], e5_acc[0], e6_acc[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oxFpljI18CzM"
      },
      "source": [
        "df = pd.DataFrame({'e1':e1_acc[0], 'e2':e2_acc[0], 'e3':e3_acc[0],\n",
        "                   'e4':e4_acc[0], 'e5':e5_acc[0], 'e6':e6_acc[0]})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Soao_oLEbaG3"
      },
      "source": [
        "#Plot comparing sensitivity of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "sens_hvlv_pd_m = [hv_hv_pd_ml[0], hv_hv_1pd[0], hv_hv_2pd[0], hv_hv_3pd[0]]\n",
        "sens_hvlv_pd_s = [hv_hv_pd_ml[1], hv_hv_1pd[1], hv_hv_2pd[1], hv_hv_3pd[1]]\n",
        "\n",
        "sens_hvlv_nc_m = [hv_hv_nc_ml[0], hv_hv_1nc[0], hv_hv_2nc[0], hv_hv_3nc[0]]\n",
        "sens_hvlv_nc_s = [hv_hv_nc_ml[1], hv_hv_1nc[1], hv_hv_2nc[1], hv_hv_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "p1 = plt.bar(x_pos - barWidth/2, sens_hvlv_pd_m, yerr=sens_hvlv_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "p2 = plt.bar(x_pos + barWidth/2, sens_hvlv_nc_m, yerr=sens_hvlv_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,1)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((p1[0], p2[0]), ('PD', 'NC'))\n",
        "plt.title('Sensitivity in Valence Classification')\n",
        "\n",
        "#Statistical significance\n",
        "\n",
        "# T-test between PD and NC data\n",
        "\n",
        "print(\"---ML---\")\n",
        "stat, p = ttest_ind([i[1][1] for i in hvlv_nc_ml],[i[1][1] for i in hvlv_pd_ml],equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "\n",
        "print(\"---1D---\")\n",
        "stat, p = ttest_ind([i[1][1] for i in hvlv_nc_1d],[i[1][1] for i in hvlv_pd_1d],equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "\n",
        "print(\"---2D---\")\n",
        "stat, p = ttest_ind([i[1][1] for i in hvlv_nc_2d],[i[1][1] for i in hvlv_pd_2d],equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "\n",
        "print(\"---3D---\")\n",
        "stat, p = ttest_ind([i[1][1] for i in hvlv_nc_3d],[i[1][1] for i in hvlv_pd_3d],equal_var=False)\n",
        "print('stat=%.3f, p=%.3f' % (stat, p))\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7wmDekd6bfM4"
      },
      "source": [
        "#Plot comparing specificity of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "spec_hvlv_pd_m = [lv_lv_pd_ml[0], lv_lv_1pd[0], lv_lv_2pd[0], lv_lv_3pd[0]]\n",
        "spec_hvlv_pd_s = [lv_lv_pd_ml[1], lv_lv_1pd[1], lv_lv_2pd[1], lv_lv_3pd[1]]\n",
        "\n",
        "spec_hvlv_nc_m = [lv_lv_nc_ml[0], lv_lv_1nc[0], lv_lv_2nc[0], lv_lv_3nc[0]]\n",
        "spec_hvlv_nc_s = [lv_lv_nc_ml[1], lv_lv_1nc[1], lv_lv_2nc[1], lv_lv_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "p3 = plt.bar(x_pos - barWidth/2, spec_hvlv_pd_m, yerr=spec_hvlv_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "p4 = plt.bar(x_pos + barWidth/2, spec_hvlv_nc_m, yerr=spec_hvlv_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0.5,1)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((p3[0], p4[0]), ('PD', 'NC'))\n",
        "plt.title('Specificity in Valence Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "JYuqg06Ig-t8",
        "outputId": "9cf537ad-37f8-41ff-9571-ffb4fd7276cd"
      },
      "source": [
        "#Plot comparing FNR of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "fnr_hvlv_pd_m = [hv_lv_pd_ml[0], hv_lv_1pd[0], hv_lv_2pd[0], hv_lv_3pd[0]]\n",
        "fnr_hvlv_pd_s = [hv_lv_pd_ml[1], hv_lv_1pd[1], hv_lv_2pd[1], hv_lv_3pd[1]]\n",
        "\n",
        "fnr_hvlv_nc_m = [hv_lv_nc_ml[0], hv_lv_1nc[0], hv_lv_2nc[0], hv_lv_3nc[0]]\n",
        "fnr_hvlv_nc_s = [hv_lv_nc_ml[1], hv_lv_1nc[1], hv_lv_2nc[1], hv_lv_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "q3 = plt.bar(x_pos - barWidth/2, fnr_hvlv_pd_m, yerr=fnr_hvlv_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "q4 = plt.bar(x_pos + barWidth/2, fnr_hvlv_nc_m, yerr=fnr_hvlv_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,0.5)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(ticks=[0,0.1,0.2,0.3,0.4,0.5], weight='bold')\n",
        "plt.legend((q3[0], q4[0]), ('PD', 'NC'))\n",
        "plt.title('FNR in Valence Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'FNR in Valence Classification')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYyElEQVR4nO3de5RV5Z3m8e8joGUUcEQkLYhCWuONGUdJnMQxKW1tG2NieqVhknZwBVEio50leJnW7kRsOzFt40INoQ1CMmrSMWkNTqJiSxvBTmzMgG17wVuCREtFEEUpBeTymz/2W+Z4PFW1q+pUFfXW81nrrF2193v2/tWmeM67330pRQRmZpaP3Xq7ADMzqy8Hu5lZZhzsZmaZcbCbmWXGwW5mlhkHu5lZZhzs1mGSRktqljSgh7f7ZUm/7MltdpWkNZJO7qZ1nyDpmYrvPyrpUUmbJH1V0o2SvtYN271c0oJ6r9fqx8GeiRQgm1PgtrwOkHSwpJB0T1X7H0ialb5ulLQzvWeTpGckTWltWxHxQkTsHRE7OljjSEnbJX2kxrJFkmZ3ZH27AklDJF0n6YW0/36bvt+vu7cdEf8aER+tmHUp8EBEDI6IGyLivIi4qivbSL8bTVXb/WZEnNOV9Vr3crDn5bMpcFteL1csO07SJ9t478sRsTcwBJgB3CTpo22077CIeAm4H5hcOV/SvsBpwM313F53k7Q7xc9zJPAnFPvuE8AG4OO9UNJBwJO9sF3bxTjY+49rgG+01ygK9wCvA/+5VpuKo4CB6fulkq6S9KvU47+vjR7rzVQFO/BFYFVEPC7pL1Ovd5OkVZL+tLVaJR0maYmk19NRxqSKZf9H0nck3Z3W9XDlkYKkIyve+6qky9P83Spq2CDpJ+mDp5azgNHAn0bEqojYGRHrIuKqtA+r6/24pH+TtFHSK5Lmpg8HVJgjaZ2ktyQ9LumotOy0tC82SXpJ0sVp/nu9aUm/AE4E5qYjh0PTPvjbiu2fkYZq3ko/35+k+VMkPZXWv1rSV9L8vYDFwAFVR4GzJP2gYr2fk/Rk+rmWSjq8YtkaSRdLekzSm5J+LKmhtX9Tqw8He/8xDzhU7Yz3pmD7HLAf8JsOrP/PgSnA/sDuwMWttFsE7Cfpv1fMm8zve+u/BU4AhgJXAj+Q9Ac16twLWAL8Y9rmF4F5ko6oaPbFtI7/lH6Wb6T3Dgb+BbgXOAD4Q4qeN8BfAJ8HPp2WvQF8p5Wf5WTg3ohobmV5tR0UR0P7UfTs/wj4X2nZHwOfAg5NP/skip4/wELgKxExGDgK+EX1iiPiJOBfgQvS0dqzlcslfRy4BbgE2Cdta01avA44neKIYwowR9IxEfE2MIF0NFfjKBBJhwI/Ai4EhgP3AD9v+cBKJlEc0Yyh6Cx8ucS+si5wsOflztRr2ijpzqplmymC7W9rvA+KXtnG1G4RMDMi/r0D2/5+RDwbEZuBnwBH12qUlv8TRW8XSYcAx1IENBHxTxHxcur9/hh4jtrDGqcDayLi+xGxPdV6BzCxos2iiPh1RGwHflhR0+nA2oi4NiK2RMSmiHg4LTsP+KuIaIqIrcAs4M9ajk6qDANeKbNz0s+2MiKWp3rXAN+l+AAB2AYMBg4DFBFPRcQrFcuOkDQkIt6IiEfKbrPCVOB7EbEk7duXIuLpVNfdEfHbdLS2DLiP4sO1jP8B3J3Wuw2YDewJVA773ZD+TV8Hfk4rvxtWPw72vHw+IvZJr8/XWL4AGCHpszWWvRwR+1D02m4ATurgttdWfP0OsHcbbW8GJqZD8snAP0fEOgBJZ6Xhgo3pg+Yoih5utYMozhtsrGh7JvDhEjUdSHFkUMtBwKKKdT5F0dMeUaPtBuADRxOtScMjd0laK+kt4JstP1tE/AKYS3F0sE7SfElD0lu/QHEO4neSlkn6RNltVmj1Z5Y0QdLyNCy1MW2r7MnfA4DftXwTETuBF4GRFW068rthdeBg70ci4l2KoYmrALXSZivwv4Fxkmp9ONTDLynG8M8A/idpGEbSQcBNwAXAsPRB80Qrtb4ILKv4INsnDRVML7H9F4GxbSybULXehnTit9q/AKemYaEy/gF4GjgkIoYAl1Pxs6UrWY4FjqAYkrkkzf9/EXEGxZDTnRRHRB31IlDraqQ9KI50ZgMj0j6/p6Ku9h7/+jLFh2HL+kTxIVJrf1kPcbD3P7cCDRRjnjWlD4Brga93RwFRPCv6FuDvKMZ7f54W7UURJOuhOKlH0WOv5S6KcwaTJQ1Kr49Vnrhrw13AH0i6UNIekgZLOi4tuxH4RvqQQdJwSWe0sp5bKQLzDhUncneTNEzFdd6n1Wg/GHgLaJZ0GPDeh1Cq/ThJg4C3gS3ATkm7SzpT0tA01PEWsLPEz1htITBF0h+lOkemGnYH9qDY59slTaAY72/xKjBM0tBW1vsT4DNpvYOAi4CtwEOdqNHqxMHez6Rrz78OtHalR4vvAaNbGbaph1sorij5cTpKICJWUXyg/BtFoIwDflXrzRGxiSKAvkjRa1xL8UGxR3sbTu89Bfhset9zFFeUAFwP/Ay4T9ImYDlwXCvr2UpxAvVpihO5bwG/phjGeLjGWy6mOMm8ieLI5McVy4akeW9QDG1sAP4+LZsMrEnDN+dRDDl1SET8mnRiFHgTWAYclPbFVykC+o1U388q3vc0xcnR1Wl46oCq9T5DcdT1beA1in362dQ5sF4i/6ENM7O8uMduZpaZUsEu6fh0g8FWSY9IOqZGm5abVipf19W/ZDMza0u7QzHpkrQ1FNc3/z3wVxQnRw6pfFaIpIOB5ylOPi1Ls5/p4LXQZmbWRbVuuqg2geIa3ksjYp6kDwNfAxr5/d16lVYAP4uId+pWpZmZlVYm2Mekact1qS1PehtL7WC/CVggaRUwNSKWVzeQNA2YBrDXXnsde9hhh3WoaDOz/m7lypWvRcTwWsvKBHu1mje2UFx7ewXwKHAI8C2K27g/cFNERMwH5gOMHz8+VqxY0YkyzMz6L0m/a21ZmWB/Pk1HpWnLrcKr0/j7jojYFhHrgb+p2OifA8dIaoiILZ2o28zMOqFMsC+mePrb9HTDxlSKk6lrKE6o3g2cLulc4GMUN2aMpXjQz3841M3Mela7lzumYJ4INFPclbcufV/913OepXgk53UUjyK9l/c/ac/MzHpAqTH2iHiQ4vbuapUPMFoG/Lc61WVm1mXbtm2jqamJLVv67sBBQ0MDo0aNYtCgQaXf05mTp2ZmfUJTUxODBw/m4IMPpnjwZN8SEWzYsIGmpibGjBnT/hsSP1LAzLK1ZcsWhg0b1idDHUASw4YN6/ARh4PdzLLWV0O9RWfqd7CbmWXGY+xm1m/MWfJs+406YMYph7a5fMCAAYwbN47t27dz+OGHc/PNN/OhD33ovfnbtm1j4MCBnHXWWcyYMYPddqtPX9s9djOzbrLnnnvy6KOP8sQTT7D77rtz4403vm/+k08+yZIlS1i8eDFXXnll3bbrYDcz6wEnnHACv/nNbz4wf//992f+/PnMnTuXev3hIwe7mVk32759O4sXL2bcuFq3A8HYsWPZsWMH69atq8v2PMZuZtZNNm/ezNFHHw0UPfapU6f2yHYd7GZm3aRlLL09q1evZsCAAey///512a6HYszMetH69es577zzuOCCC+p2zb177GbWb7R3eWJPaRmiabnccfLkycycObNu63ewm5l1k+bm5przd+yofjhufXkoxswsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PM+HJHM+s/Hri6vus78bI2F0ti5syZXHvttQDMnj2b5uZmZs2aBcAtt9zCNddcgyQGDhzImWeeycUXX9zlstxjNzPrJnvssQc//elPee211z6wbPHixVx33XXcd999PP744yxfvpyhQ4fWZbsOdjOzbjJw4ECmTZvGnDlzPrDs6quvZvbs2RxwwAFA8SFw7rnn1mW7DnYzs250/vnn88Mf/pA333zzffOfeOIJjj322G7ZpoPdzKwbDRkyhLPOOosbbrihx7bpYDcz62YXXnghCxcu5O23335v3pFHHsnKlSu7ZXsOdjOzbrbvvvsyadIkFi5c+N68yy67jEsuuYS1a9cC8O6777JgwYK6bM+XO5pZ/9HO5Ynd6aKLLmLu3LnvfX/aaafx6quvcvLJJxMRSOLss8+uy7Yc7GZm3aTysb0jRozgnXfeed/yKVOmMGXKlLpv10MxZmaZcbCbmWXGwW5mWYuI3i6hSzpTv4PdzLLV0NDAhg0b+my4RwQbNmygoaGhQ+/zyVMzy9aoUaNoampi/fr1vV1KpzU0NDBq1KgOvcfBbmbZGjRoEGPGjOntMnqch2LMzDJTKtglHS/pMUlbJT0i6Zg22h4uaYukkPRn9SvVzMzKaDfYJTUAdwCDgRnACOB2SQNqtBVwE7CjznWamVlJZXrsEyjCfF5EzAMWAmOAxhptpwMHAd+tV4FmZtYxZYK95czDS2nalKZjKxtJGglcTRHub7W1QknTJK2QtKIvn602M9sVdebkqVqZ/y1gBfA0sG+a92FJe1c3jIj5ETE+IsYPHz68EyWYmVlrylzu+HyatlxIOTJNV6fx9x0RsQ04EPg08FzFe78NbAR+UIdazcyshDLBvhhYB0yXtAmYCqxJr83A3cDpwBVAS/d7EjARuBZ4sK4Vm5lZm9oN9ojYImki8B3geuBJ4FyqrnyJiGUtX0s6Kn25PCJeqF+5ZmbWnlJ3nkbEg8C4GotqjrdHxCxgVqerMjOzTvOdp2ZmmXGwm5llxsFuZpYZB7vZLqqxsZHGxsbeLsP6IAe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZaZfBLuvBzaz/qRfBLuZWX/iYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMDOztArpizpJnS7VremNzh9rPOOXQTtdkZtbb3GM3M8uMg93MLDMOdjOzzDjYzcwy42C3LvFfpzLb9TjYzcwy42A3M8uMg93MLDMOdjOzzJQKdknHS3pM0lZJj0g6pkaboyQ9JWmLpI2S7pE0sv4lm5lZW9oNdkkNwB3AYGAGMAK4XdKAqqY7gduAacBPgQnAFXWt1szM2lWmxz6BIsznRcQ8YCEwBmisbBQRq4CrgXuBh9LsnXWr1MzMSinzELAxafpSmjal6Vjg/qq2pwGL0tdP0kqPXdI0ip49o0ePLltrp50/+9Zu34aZ2a6iMydP1cayX1H08K8HjgS+UqtRRMyPiPERMX748OGdKMHMzFpTJtifT9NRadpyQnS1pAZJg1oaRsT6iLgXuIhiGGZS3So1M7NSygzFLAbWAdMlbQKmAmvSazNwN3C6pMuAocDTwEkUHxqr6l+ymZm1pd0ee0RsASYCzRRDLOvS9zuqmq4HvgR8FzgV+BFwQT2LNTOz9pX6C0oR8SAwrsYiVbRZACyoU119QsvDr5YuXdqrdZiZVfKdp2ZmmXGwm5llxsFuZpYZB7uZ9Tu5/4EYB7uZWWYc7GZmmXGwm5llptR17P3OA1eXa7fxhY61P/GyztVjZtYB7rGbmWXGwW5mlhkHu5lZZhzsZmaZ8clTq2nOkmdLtWt6Y3OH2s845dBS7fyAte7jfZs/99jNzDLjYDczy4yD3cwsMx5j74Klc87t7RKsD+rt8xeWP/fYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8tMqWCXdLykxyRtlfSIpGNqtPlcWrZJ0muSvidpz/qXbGZmbWk32CU1AHcAg4EZwAjgdkkDqpr+F2AVMBNYCUwBLq1rtWZm1q4yf8x6AkWYXxoR8yR9GPga0AjcX9Hu7yLiXQBJy4HHgCPrW66ZmbWnTLCPSdOX0rQpTcdSEewtoZ6cmqYP1lqhpGnANIDRo0eXrdV2QefPvrW3SzCzKp05eao2F0pfAL4J3AP8Q602ETE/IsZHxPjhw4d3ogQzM2tNmWB/Pk1HpenINF0tqUHSoJaGkiYBtwEPAF+IiB11q9TMzEopE+yLgXXAdEnTganAmvTaDCwCkPQZ4B+BjcCPgM9LOqn+JZuZWVvaDfaI2AJMBJqB6ylCfiJQ3Rv/GDAA2A/4PkW4f72exZqZWfvKnDwlIh4ExtVYpIo2s4BZdanKzMw6zXeempllxsFuZpYZB7uZWWZKjbGb1c0DV5drt/GFjrU/8bLO1WNWR42NjQAsXbq0V+twj93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMHwJmZtmYs+TZUu2a3tjcofYzTjm00zX1BvfYzcwy42A3M8uMg93MLDMeYzfLhf+IiSXusZuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZpnxDUpmu6jzZ9/a2yVYH+Ueu5lZZhzsZmaZcbCbmWXGY+xmZu3pYw9YK9Vjl3S8pMckbZX0iKRjarTZU9L9kpolhaSL61+umZm1p91gl9QA3AEMBmYAI4DbJQ2oajoAeB24t95FmplZeWV67BMownxeRMwDFgJjgMbKRhHRHBETgbvqXaSZmZVXZox9TJq+lKZNaToWuL8zG5U0DZgGMHr06M6swjK3dM65vV2CWZ/Vmati1NWNRsT8iBgfEeOHDx/e1dWZmVmFMsH+fJqOStORabpaUoOkQfUvy8zMOqtMsC8G1gHTJU0HpgJr0mszsKiloaRzgE+lbz8u6RxJe9ezYDMza1u7wR4RW4CJQDNwPUXITwR21Gh+EzAlfT0xfb9fXSo1M7NSSt2gFBEPAuNqLFJVuy6Pv5uZWdf4kQJmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmSv0xazMza9/SOef2dgmAe+xmZtlxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmfIOSWT+zq9xE05vOn31rb5fQrdxjNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLTKlgl3S8pMckbZX0iKRjWmn3FUlNkjZL+r+ShtW3XDMza0+7wS6pAbgDGAzMAEYAt0saUNXuvwI3Ak8BVwCfAebUu2AzM2tbmR77BIownxcR84CFwBigsardl9P08oi4BngI+FL6YDAzsx5S5galMWn6Upo2pelY4P522g0EDgSeq1yhpGnAtPRts6RnOlBzt5sJ+wGv1X/Nl9d/lX2M92338b7tPrvovj2otQWdufNUXW0XEfOB+Z3Ydo+QtCIixvd2HTnyvu0+3rfdp6/t2zJDMc+n6ag0HZmmqyU1SBrURrvtwItdrtLMzEorE+yLgXXAdEnTganAmvTaDCxK7W5J029IuhT4JHBbRGypZ8FmZta2doM9BfNEoBm4niLkJwI7qtqtBM4HjgD+huIDYUad6+0pu+wwUQa8b7uP92336VP7VhHR2zWYmVkd+c5TM7PMONjNzDLTb4Nd0sGSIr2+VjF/YcX8xjSd25u19jWSbpD0atp3d1XMb9mvWyWtlXSnpKN7s9a+RtIhkh6QtEHSJklLJH2k4nc1JL0jabWk70rar7dr7kskPZz26zuSVkj6VF/ct/022Kt8WYW9gUm9XUwmbmtlfhNwDvATisdO/FLSET1WVd83kuL/7RXA94GTgQUVy+8Dvgo8TnET4DJJe/Z0kX3YQxT77yrgaProvnWww2qKu2gbKUJ9EL+/e9Y6ISK+SuvPCXozIm5Nba4A9gL+sseK6/seiohPR8TctA9fB46sWP5cRCyIiDOAJRRXqX2pNwrto2YCP6e4q34rsLNiWZ/Ztw724qFlDwNnp9edwMZeraj/WJymfeaOvt4WEe+2fC1pPLAv8GArzb1/O24osJ4iE96lOLqsZZfetw72wvcors0/nuLw1npGy2MnfM1tB0k6DPgZxY2Cf9FaszT1/i2vGfhjiiGXBop7cmrZpfetg71wG8UNV00Uh1jWM05N05W9WkUfk85JLKXoUZ4UEa+00tT7t4MiYntELImIbwO/Bk6keABYtV1633bmIWDZiYi3JJ0NbIqIndIHnl/2CUnfSl9vjogre7bCvkXSZ4Cj0rcHSjoHWJa+HyppMsUh7HTgbeBbH1yL1SLpQOABYBjw18Bxko4D1qYmh0iaCpxO0fNcBfyoN2rtaySdSnGe7SGKp9J+EngV2JCa9Jl922/vPJV0MMWDy+6OiNOrlj1BcULqRIr/RJXejIh9eqLGvkrSUuDTVbOn8Pthrm0U/1mWA7Mi4j96rrq+TVIjH/ydhPf/rm4BXgH+GfjriNhQo71VkfQxit/Rj1CcOP134FKKE/x9at/222A3M8uVx9jNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsM/8f60DzyuRY6/EAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "ajuK6xzRgKF6",
        "outputId": "01a5a461-a12e-4df9-f227-bd87353c4c47"
      },
      "source": [
        "#Plot comparing FPR of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "fpr_hvlv_pd_m = [lv_hv_pd_ml[0], lv_hv_1pd[0], lv_hv_2pd[0], lv_hv_3pd[0]]\n",
        "fpr_hvlv_pd_s = [lv_hv_pd_ml[1], lv_hv_1pd[1], lv_hv_2pd[1], lv_hv_3pd[1]]\n",
        "\n",
        "fpr_hvlv_nc_m = [lv_hv_nc_ml[0], lv_hv_1nc[0], lv_hv_2nc[0], lv_hv_3nc[0]]\n",
        "fpr_hvlv_nc_s = [lv_hv_nc_ml[1], lv_hv_1nc[1], lv_hv_2nc[1], lv_hv_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "q1 = plt.bar(x_pos - barWidth/2, fpr_hvlv_pd_m, yerr=fpr_hvlv_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "q2 = plt.bar(x_pos + barWidth/2, fpr_hvlv_nc_m, yerr=fpr_hvlv_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,0.5)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(ticks=[0,0.1,0.2,0.3,0.4,0.5], weight='bold')\n",
        "plt.legend((q1[0], q2[0]), ('PD', 'NC'))\n",
        "plt.title('FPR in Valence Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'FPR in Valence Classification')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYZ0lEQVR4nO3df5yVdZ338ddbQAcVNRAoxFUoWw3ZzCjbvK2x22pRt+1+tFiuD3mICMnamqD5yO4tsd3S21tvf0SsIlTYulq3mlsarqwrsq1hC2b+ytIbaB0TGVCU4Tf4uf+4vseOw5mZM8M5M5zvvJ+Pxzyumev7Pdf1mS/wPtf5Xj9QRGBmZvnYp68LMDOz2nKwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFu3SbpaUnNvbzPIyWFpIG9ud89Iel7kv6+jttvkzQ2fT9Y0k8kvSbp/0o6S9IDddjnSZJ+U+vtWm052DMiabWkLekffOlrVFkoltatlvTlsteFpE2p7UVJ/0fSgI72ExHjImJJD+p7VtK5FdZ/UdLy7m6vr6lwoaSn0vi1pFAd3xv7j4gDI2Jl+vEvgZHAsIiYFBG3RcQn9nQf6e/Gu8r2+e8R8cd7ul2rLwd7fv48/YMvff2+rO2QiDgQOBP4mqQ/K2t7b2r7KPBZYLcAroGFwOQK689ObY3mBuCLwIXAUODdwD3AaX1QyxHAbyNiZx/s2/YyDvZ+KCJ+DjwNHFuh7XngP4DjOnp9OuI/JX0/W9IPJd0qaWOappnQwUu/D/w3SUeUbes9wJ8At0s6TdIvJb0u6QVJszup4WBJCyS9lD5l/H3pU4akcyT9TNI1kl6VtErSxLLXDpX0XUm/T+33lLWdLulxSRskPSLpTzrY/1HABcCZEfFvEbEtIjanI+WrKvR/m6R7JbWmfd4raXRZ+zmSVqYxXCXprLT+XZIeTlMs6yT9oOw1kdqvAL4GfDZ96ppaGoOyvuMkLZb0iqSXJX0lrf+gpJ+n3/clSXMk7ZvalqaX/ypt97OSmiW1lG33GElL0uuflvSpsrbvSfq2pPvS7/WopHd29GdqteNg72fS9MGJwDjglxXajwZOAp7vxmY/BdwBHAL8GJhTqVNEtAAPURyhl5wN/DQi1gGbKI7oD6E46p0h6dMd7PN7wE7gXcD7gE8A55W1nwD8BjgUuBpYIEmp7fvA/hRjMAK4DkDS+4DvAJ8HhgE3Az+WtF+F/f93oCUiftFBfe3tA3yX4sj6j4AtpHGSdABwIzAxIoYAHwYeT6/7O+AB4G3AaOBb7TccEZcD3wR+kD6lLShvlzQE+FfgfmAUxZg9mJp3ATMpxulP0+/112m7H0l93pu2+4N22x0E/CTVNwL4G+A2SeVTNZ8Drkj1Pw98o+uhsj3lYM/PPenoaUP5kWiyDngFmA98OSIeLGt7TNIm4NfAEmBuN/b5s4j4aUTsogjN93bSdyEp2CXtA5yV1hERSyLiyYh4IyKeAG6nmBp6C0kjgVOBiyJiU0SspQjnz5V1+11E3JJqWgi8Axgp6R3AROD8iHg1InZExMPpNdOBmyPi0YjYFRELgW3Ahyr8HsOAl6oZnPS7rY+Iu9JR/UaKgCv/3d4AjpU0OCJeioin0/odFG8GoyJia0T8jO47HVgTEdembWyMiEdTXSsiYllE7IyI1RRvZruNeQc+BBwIXBUR2yPi34B7Kab6Sn4UEb9IU0S30cknQasdB3t+Ph0Rh6Sv9ke7h0bE2yLimIi4sV3b8RT/SD9LcbR7QDf2uabs+81Akzq+euVu4B2SPgQ0Uxw53wcg6QRJD6XpiteA8ymOJNs7AhgEvFR6E6MIpBGVaoqIzenbA4HDgVci4tUOtntx2RvjhtR/VIW+6yneLKoiaX9JN0v6naTXgaXAIZIGRMQminE/P/1O96VPTgCXAgJ+kaY6enLu43Dg/3VQ17vTtNCaVNc3qTzmlYwCXoiIN8rW/Q44rOzn9n83Dqy+bOspB7u9KQo/BH5OMWdbj31sBu6kmHI5G7gjIran5n+imMo5PCIOBm6iCLX2XqA4kj607E3soIgYV0UJLwBDJR3SQds3yrZ5SETsHxG3V+j7IDBaHZ9PaO9i4I+BEyLiIKA0zSGAiPiXiPg4xZvFs8Ataf2aiJgWEaMopojmquwqlSq9AIztoO0f0v6OSnV9hcpjXsnvgcPTJ6+SPwJe7GZ9VmMOdqvkKmCapLfXafsLKY5QP8Nbr4YZQnE0vVXSB4G/qvTiiHiJYl73WkkHSdpH0jsldTmFkF67iCIg3yZpkKRSyN4CnJ8+OUjSASpO6A6psJ3nKKarbk8nFPeV1CTpcyq7lLTd77YF2CBpKHB5qUHSSEl/kebatwFtFFMzSJpUdpL1VSBKbd1wL8WnpIsk7SdpiKQTyup6HWhLnxJmtHvty3T8pvAoxVH4pWkcm4E/pzjfYn3IwW67iYgnKaYKvlSnXSwFXqM4+fifZev/Gvi6pI0Unxh+2Mk2JgP7As9QBN6dVD81cjbF3PWzwFrgIoCIWA5Mozip+SrFyb5zOtnOhanvt4ENFNMd/4PihGJ71wODKc5zLKM4kVmyDzCL4gj4FYo57lLAfgB4VFIbxaeZL5Zdu16VNKf/cYrQXQM8B5ycmi+heAPdSPHG9oN2L58NLExTU2e02+72tM2J6feaC0yOiGe7U5/VnvwfbZiZ5cVH7GZmmakq2CWdKOkJSdskPSbp+Ap9Sretl39dX/uSzcysM11OxUhqAlZTnPj538D/pDjBc1S6RrjU70hgFcWVDKXrgn8TEbvdBGNmZvVTzZPyJlI8XOjSiJibrpT4KsU1yA9W6L8c+HHZtcNmZtaLqgn2MWlZuja19JyIsVQO9luA+ZKeAaZGxLL2HSRNp7jLjwMOOOD9Rx99dPsuZmbWiRUrVqyLiOGV2nrybOuObl7YRHFt7uPAURTXQt8G7PbQn4iYB8wDmDBhQixf3nBPbDUz61OSftdRWzXBviotSzdJlG4XXpnm33el5220Al8v2+lfAcdLaoqIrT2o28zMeqCaYF9EcRPHjHTjyFSKk6mrKU6o3gecLmka6WYKimma44BfOdTNzHpXl5c7pmCeRHGb8w0UIT+J4nGf5X5L8Vzt6ynuILw/9TMzs15U1Rx7RCwFKv13Xyrr8zCVH29qZtYnduzYQUtLC1u3Nu7EQVNTE6NHj2bQoEFVv6Zh/mNgM7PuamlpYciQIRx55JH84f9ZaRwRwfr162lpaWHMmDFdvyDxIwXMLFtbt25l2LBhDRnqAJIYNmxYtz9xONjNLGuNGuolPanfwW5mlhnPsZtZv3Hd4t/WdHszP/7uTtsHDBjA+PHj2blzJ8cccwwLFy5k//33f3P9jh07GDhwIJMnT2bmzJnss09tjrV9xG5mVieDBw/m8ccf56mnnmLfffflpptuesv6p59+msWLF7No0SKuuOKKmu3XwW5m1gtOOukknn/++d3Wjxgxgnnz5jFnzhxq9R8fOdjNzOps586dLFq0iPHjK90OBGPHjmXXrl2sXbu2JvvzHLuZWZ1s2bKF4447DiiO2KdOndor+3Wwm5nVSWkuvSsrV65kwIABjBgxoib79VSMmVkfam1t5fzzz+cLX/hCza659xG7mfUbXV2e2FtKUzSlyx3PPvtsZs2aVbPtO9jNzOqkra2t4vpdu9o/HLe2PBVjZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWZ8uaOZ9R8PXVnb7Z18WafNkpg1axbXXnstANdccw1tbW3Mnj0bgFtvvZWrr74aSQwcOJCzzjqLSy65ZI/L8hG7mVmd7Lffftx9992sW7dut7ZFixZx/fXX88ADD/Dkk0+ybNkyDj744Jrs18FuZlYnAwcOZPr06Vx33XW7tV155ZVcc801jBo1CijeBKZNm1aT/TrYzczq6IILLuC2227jtddee8v6p556ive///112aeD3cysjg466CAmT57MjTfe2Gv7dLCbmdXZRRddxIIFC9i0adOb68aNG8eKFSvqsj8Hu5lZnQ0dOpQzzjiDBQsWvLnusssu40tf+hJr1qwBYPv27cyfP78m+/PljmbWf3RxeWI9XXzxxcyZM+fNn0899VRefvllTjnlFCICSZx77rk12ZeD3cysTsof2zty5Eg2b978lvYpU6YwZcqUmu/XUzFmZplxsJuZZcbBbmZZi4i+LmGP9KR+B7uZZaupqYn169c3bLhHBOvXr6epqalbr/PJUzPL1ujRo2lpaaG1tbWvS+mxpqYmRo8e3a3XONjNLFuDBg1izJgxfV1Gr/NUjJlZZqoKdkknSnpC0jZJj0k6vpO+x0jaKikk/WXtSjUzs2p0GeySmoC7gCHATGAkcKekARX6CrgF2FXjOs3MrErVHLFPpAjzuRExF1gAjAGaK/SdARwB3FyrAs3MrHuqCfbSmYcX07IlLceWd5J0GHAlRbi/3tkGJU2XtFzS8kY+W21mtjfqyclTdbD+KmA58CwwNK17u6QD23eMiHkRMSEiJgwfPrwHJZiZWUequdxxVVqWLqQ8LC1Xpvn3XRGxAzgc+CjwXNlrvwVsAP6xBrWamVkVqgn2RcBaYIakjcBUYHX62gLcB5wOXA6UDr/PACYB1wJLa1qxmZl1qstgj4itkiYB3wZuAJ4GptHuypeIeLj0vaRj07fLIuK/aleumZl1pao7TyNiKTC+QlPF+faImA3M7nFVZmbWY77z1MwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMVBXskk6U9ISkbZIek3R8hT7HSvq1pK2SNkj6qaTDal+ymZl1pstgl9QE3AUMAWYCI4E7JQ1o1/UN4A5gOnA3MBG4vKbVmplZl6o5Yp9IEeZzI2IusAAYAzSXd4qIZ4ArgfuBR9LqN2pWqZmZVaWaYB+Tli+mZUtajq3Q91TgZeAW4Gk6OGKXNF3ScknLW1tbu1GumZl1pScnT9VJ239QHOHfAIwDPl+pU0TMi4gJETFh+PDhPSjBzMw6Uk2wr0rL0WlZOiG6UlKTpEGljhHRGhH3AxdTTMOcUbNKzcysKgOr6LMIWAvMkLQRmAqsTl9bgPuA0yVdBhwMPAt8jOJN45nal2xmZp3p8og9IrYCk4A2iimWtennXe26tgJnAjcDnwRuB75Qy2LNzKxr1RyxExFLgfEVmlTWZz4wv0Z1mZlZD/nOUzOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8tMVcEu6URJT0jaJukxScdX6POp1LZR0jpJ35E0uPYlm5lZZ7oMdklNwF3AEGAmMBK4U9KAdl3fCzwDzAJWAFOAS2tarZmZdWlgFX0mUoT5pRExV9Lbga8CzcCDZf3+V0RsB5C0DHgCGFfbcs3MrCvVTMWMScsX07IlLceWdyqFevLJtFxaaYOSpktaLml5a2trtbWamVkVenLyVJ02Sp8Bvgn8FPiHSn0iYl5ETIiICcOHD+9BCWZm1pFqgn1VWo5Oy8PScqWkJkmDSh0lnQHcATwEfCYidtWsUjMzq0o1wb4IWAvMkDQDmAqsTl9bgB8BSDoN+CdgA3A78GlJH6t9yWZm1pkugz0itgKTgDbgBoqQnwS0Pxr/ADAAOBT4LkW4f62WxZqZWdequSqGiFgKjK/QpLI+s4HZNanKzMx6zHeempllxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFutpdqbm6mubm5r8uwBuRgNzPLjIPdzCwzDnYzs8w42G2P1Gse2PPLZj1XVbBLOlHSE5K2SXpM0vEV+gyW9KCkNkkh6ZLal2tmZl3pMtglNQF3AUOAmcBI4E5JA9p1HQC8Atxf6yLNzKx61RyxT6QI87kRMRdYAIwBmss7RURbREwC7q11kWZmVr1qgn1MWr6Yli1pObanO5U0XdJySctbW1t7upk+53lga0T+e1s/e8vY9uTkqfZ0pxExLyImRMSE4cOH7+nmzMysTDXBviotR6flYWm5UlKTpEG1L8vMzHqqmmBfBKwFZkiaAUwFVqevLcCPSh0lnQd8JP34QUnnSTqwlgWbmVnnugz2iNgKTALagBsoQn4SsKtC91uAKen7SennQ2tSqZmZVWVgNZ0iYikwvkKT2vXb4/n3vcJDV1bXb8N/da//yZf1rB4zq6nSCc4lS5b0aR314jtPzcwy42A3M8uMg93MLDNVzbFbZUuum9bXJdTNdYt/W1W/lle3dKv/zIF3VVdAxucv6ja2H393j2uyvPiI3cwsMw52M7PMONjNzDLTL4J9b3kwj5lZb+gXwW5m1p842M3MMuNgNzPLjIPdzCwzDX2DUv1uoulxSWZ9xw+vq58GG1sfsZuZZcbBbmaWGQe7mVlm+sVs8gXXfL+vS8hWvcY25wesWf34vFvBR+xmZplxsJuZZcbBbmaWmQabOTLrP3xuyHrKR+xmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmfIOSWT/jB6zVz94ytj5iNzPLjIPdzCwzDnYzs8x4jt3M+p3cH7DmI3Yzs8xUFeySTpT0hKRtkh6TdHwH/T4vqUXSFkn/LGlYbcs1M7OudBnskpqAu4AhwExgJHCnpAHt+r0PuAn4NXA5cBpwXa0LNjOzzlVzxD6RIsznRsRcYAEwBmhu1++ctPxKRFwNPAKcmd4YzMysl1Rz8nRMWr6Yli1pORZ4sIt+A4HDgefKNyhpOjA9/dgm6TfdqLnuZsGhwLrab/krtd9kg/HY1o/Htn720rE9oqOGnlwVoz3tFxHzgHk92HevkLQ8Iib0dR058tjWj8e2fhptbKuZilmVlqPT8rC0XCmpSdKgTvrtBF7Y4yrNzKxq1QT7ImAtMEPSDGAqsDp9bQF+lPrdmpbfkHQp8GHgjojYWsuCzcysc10GewrmSUAbcANFyE8CdrXrtwK4AHgP8HWKN4SZNa63t+y100QZ8NjWj8e2fhpqbBURfV2DmZnVkO88NTPLjIPdzCwz/TbYJR0pKdLXV8vWLyhb35yWc/qy1kYj6UZJL6exu7dsfWlct0laI+keScf1Za2NRtJRkh6StF7SRkmLJb2z7O9qSNosaaWkmyUd2tc1NxJJj6Zx3SxpuaSPNOLY9ttgb+ccFQ4EzujrYjJxRwfrW4DzgB9SPHbiZ5Le02tVNb7DKP7dXg58FzgFmF/W/gBwIfAkxU2AD0sa3NtFNrBHKMbv74DjaNCxdbDDSoq7aJspQn0Qf7h71nogIi6k4+cEvRYR3099LgcOAL7ca8U1vkci4qMRMSeN4SvAuLL25yJifkT8BbCY4iq1M/ui0AY1C/gJxV3124A3ytoaZmwd7MVDyx4Fzk1f9wAb+rSi/mNRWjbMHX19LSK2l76XNAEYCiztoLvHt/sOBlopMmE7xafLSvbqsXWwF75DcW3+iRQfb613lB474Wtuu0nS0cCPKW4U/JuOuqWlx7d6bcAnKKZcmijuyalkrx5bB3vhDoobrlooPmJZ7/hkWq7o0yoaTDonsYTiiPJjEfFSB109vt0UETsjYnFEfAv4BXAyxQPA2turx9b/NR4QEa9LOhfYGBFvSLs9v+xPJV2Vvt8SEVf0boWNRdJpwLHpx8MlnQc8nH4+WNLZFB9hZwCbgKt234pVIulw4CFgGPC3wAmSTgDWpC5HSZoKnE5x5PkMcHtf1NpoJH2S4jzbIxRPpf0w8DKwPnVpmLHtt3eeSjqS4sFl90XE6e3anqI4IXUyxT+icq9FxCG9UWOjkrQE+Gi71VP4wzTXDop/LMuA2RHxq96rrrFJamb3v5Pw1r+rW4GXgH8B/jYi1lfob+1I+gDF39F3Upw4/SVwKcUJ/oYa234b7GZmufIcu5lZZhzsZmaZcbCbmWXGwW5mlhkHu5lZZhzsZmaZcbCbmWXm/wPddeoEwX39+wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "eWvFkW-TtIYu",
        "outputId": "3d956965-12b0-4d64-f75a-7edcd15d664d"
      },
      "source": [
        "#Plot comparing sensitivity and specificity within PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "sens_hvlv_pd_m = [hv_hv_pd_ml[0], hv_hv_1pd[0], hv_hv_2pd[0], hv_hv_3pd[0]]\n",
        "sens_hvlv_pd_s = [hv_hv_pd_ml[1], hv_hv_1pd[1], hv_hv_2pd[1], hv_hv_3pd[1]]\n",
        "\n",
        "spec_hvlv_pd_m = [lv_lv_pd_ml[0], lv_lv_1pd[0], lv_lv_2pd[0], lv_lv_3pd[0]]\n",
        "spec_hvlv_pd_s = [lv_lv_pd_ml[1], lv_lv_1pd[1], lv_lv_2pd[1], lv_lv_3pd[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "p5 = plt.bar(x_pos - barWidth/2, sens_hvlv_pd_m, yerr=sens_hvlv_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "p6 = plt.bar(x_pos + barWidth/2, spec_hvlv_pd_m, yerr=spec_hvlv_pd_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,1)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((p5[0], p6[0]), ('Sensitivity', 'Specificity'))\n",
        "plt.title('Sensitivity and Specificity within PD')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Sensitivity and Specificity within PD')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAflElEQVR4nO3deXxU5d338c+PpQQFN+DBBixg1Yooa4CCViIufaBWtBqpWnmCIIVqsdjWWpdKF1u1WgQrm1Vxw+UBUYv1tlAF7A3qHSjivhRQowiByhJl53f/cZ3EIcwkQ5xkkpPv+/Wa15mZc50z15yZ+c411znnGnN3REQkPhpluwIiIpJZCnYRkZhRsIuIxIyCXUQkZhTsIiIxo2AXEYkZBXsDYWalZnZkJfNfN7P8NNZzkZn9PaOVyxAzW21mp2W7HmXMbKqZXZ9we4yZrY1ei1ZVvSbRMl+LyjWu+Rqn/5hm5mZ2VIp5dfY90mC4uy61fAFOAhYDm4D/AP8N9K7Fx58B/C5D63LgqGxv06guq4HTUsxrD8wG1kfb/TWgsBbr1hTYCnT7kutZAIys5e26z2PW1OsOjAd2AqXAxuhz0i+aVwjsjuaVAquAe4Fjsvm+q4sXtdhrmZkdBMwF7gAOA9oBvwa2Z7NeDcADwIdAB6AVcDGwthYfvy2QA7xei49ZXz3q7i2ANsA/gcfNzKJ5S6J5BwOnEb4sl5rZ8dmpah2V7W+WhnYB8oCNVZS5BHgT+BR4FuiQMM+B0cC7hBbNnYBF844CFhJapOsJH5DE5Y4CRhFaRDsIrZ6/RvNXEz4ouYQPy2EJy/aI1teU0Gr6Z3T/omi9n0XrGkpoCX83Ydmm0bI9kjzPQwlfciXRc50LtE+YvwD4LeEXzRbg70DrhPkXA+8DG4BrqbzFXgp0TzGvY/Q8RgEfA2uAnyXMbwRcDfw7eqzHKmyfsl9gGwlfHoXR/TOA3wHHRNvIo3o8l/iaRNebA7dFz2cTIdCaJ9StCXAjocW6LVrPn6PX/7YKz+cpYFyS5/lr4I6E1+Uz4I8Jj7+N0Nio9DHTeB+Wv0eqKpukjuOBBxNud4mWb11xvQll5gKzsv3ZrkuXrFegoV2Ag6JwuA8YBBxaYf4Q4D2gc/TBug5YnDDfozfyIcDXCKH4f6N5DxMCrhGhdXhSheXKQmQGFbpiSAhF4Dng0oR5fwSmRteTfWiPSrh9FXt/oQwBXk2xLVoB5wIHAC2B/w88kTB/ASFMj4mCZwFwUzTvuChoTgaaAX8CdpE62OcTviC+D3ytwryyIHsYOBA4IdquZdvjCuBFQndOM2Aa8HA0rwPhS+cCQli2IvoCSdzOCY/RJMVrcmf0/NoBjYH+0WPttRwVukWAPoQvo0bR7dbA50DbJNtgYNlrEa3/38BLCfNeSVbXio+Zxvsw2XskadkkdRxPFOzR8/8j8EGy9SYscwmwNtuf7bp0UVdMLXP3zYQWngN3ASVm9pSZtY2KjAb+4O5vuvsu4PdAdzPrkLCam9x9o7t/ADwPdI/u30kImlx33+bu/6xmNWcSgoroJ/D3o/vS8SAwOOpygtCqfiBZQXff4O6z3f1zd99CaB0OqFDsXnd/x923ElrKZc/1PGCuuy9y9+3A9cCeSupVALwQlVtlZsvNrHeFMr9298/c/VVC3+0F0f2jgWvdvTh6rPHAeWbWBLgQmO/uD7v7zug5La+kHvsws0aEcLrC3T9y993uvjh6rEq5+8uEFv6p0V3fBxa4e7JupiXA0WbWivCFeDfQzsxaELb7wv2pN6nfh1+27PlmVvbrpxdwThX1+JjwS0MiCvYsiEK70N3bA8cTuj9uj2Z3ACaa2cbozf0fwAgtuTKfJFz/HGgRXb8qKvtydJTLJdWs4mygn5l9lRAAewihWCV3/5jQMj7XzA4h/Cp5KFlZMzvAzKaZ2ftmtpnQtXNIhaMxUj3XXMIHv+xxPyP8EkpVr0/d/Wp370Lo714OPJHQd0vi+ghdIrnR9Q7AnITX5E1C90Rb4AhCy/fLaE34hVXd9dwH/CC6/gNSf5FuBYoIIX4yIcgXAydSvWBP9dp82bKPufsh7v5/3H2guy+toh7tCJ8TiSjYs8zd3yL8ZC/b+fMh8MPojV12ae7ui9NY1yfufqm75wI/BCanOCSt0iE93f1TQn/2UEKL9BGPfvOmqSxoCgg7uz5KUe6nwDeAvu5+ECFsIHw5VWUNIVTDAmYHELpBquTu64FbCcGd2NI7IuH61wgtQQivyaAKr0lO9Lw+BL6ezuNWYj2hDzud9SR7HR4EhphZN0IX3hOVLL+Q0O3SA/if6Pa3CV06i/bjMeuSc0iz4dFQKNhrmZkda2Y/NbP20e0jCD/5X4yKTAV+aWZdovkHm1lBmusuKFsvYWekk7x7Yi1Q6fHThK6XYYQuj8q6YZKt6wmgJ6Fv+v5Klm1J2FG70cwOA26ook6JZgFnmtlJZvYV4DdU8n42s5vN7Hgza2JmLYExwHvuntjKvz76FdEFGA48Gt0/FbixrDvMzNqY2ZBo3kPAaWZ2frTuVmZWWTfDPtx9D3AP8CczyzWzxmbWz8yaJSm+z/Z292JCSD8AzI5a5qksJLyub7j7DqL+c2CVu5ekWCad90utirZRJzO7A8gn7BiWiIK99m0B+gIvmdlnhEB/jdB6xd3nADcDj0TdE68RujPS0TtabynhyIgr3H1lknJ3A8dFXQupWndPAUcDn7j7K5U85njgvmhd50fPYSuhO6cT8Hgly95O2Cm6nrAd/quSsntx99eBywhfOmsIX2TFlSxyADCHcFTGSkL3ylkVyiwk7Lj+B3Cru5edZDORsD3+bmZborr2jerxATCY8Pr9h9DF0y3d55HgZ8CrhID+D+E9kOzzOZHQv/+pmU1KuP8+wk7fpN0wCRYTtnlZ6/wNwq+FVK31yh4zG/pF7+/NhC+lgwjngLya1VrVMbZ/v7BF0mNmvyKcOPKDKgtnmZl1JJzs0jTaYV3vmNnJhC6ZDvvZbSYx1CTbFZD4ibpVRhCOiJEaZmZNCd1ef1GoC6TRFWNmk6LxLdzM5lZS7kQzW2Fm281smZn1zGxVpT4ws0sJOxSfcffKft5LBphZZ0L30lf54sgqaeCq7IqJ+tQcGAs87e5nJimTQzjBZSvhhIJrCafIH+3uuzNcZxERqUSVLXZ3HwtMqKLYIMIxvZPdfTJh51wnwt5qERGpRZnqY+8UTcuOVy47OuFIwhEGezGzUYRxOTjwwAN7HXvssRmqhohIw7B06dL17t4m2bya2nla6Qkm7j4dmA6Ql5fnRUVFNVQNEZF4MrP3U82r9nHsZtYs4QSKVdG07OSYstPfkx1DLSIiNajKFruZfYcvTnc/wsxGEk7kmEcY46IF8AywDhgTncAxgrAzdUHmqywiUjfl5+cDsGDBgqzWI50W+8+Bm6LrXQkjEp6YWMDdtxHGBSklnKW2DijQETEiIrWvyha7u+enmDWjQrlFhFOav7SdO3dSXFzMtm3bMrE6SZCTk0P79u1p2rRptqsikjV1pWVdU+rkmafFxcW0bNmSjh07sveoqvJluDsbNmyguLiYTp06Vb2AiNRLdXIQsG3bttGqVSuFeoaZGa1atdIvIZGYq5PBDijUa4i2q0j81dlgF5GakZ+fX97HLPFUJ/vYK5ow752Mrm/c6cekVe7GG29k5syZNG7cmEaNGjFt2jT69u2bkTr079+fxYsXs3r1ahYvXsyFF14IQFFREffffz+TJqUe9nrq1KkccMABDBs2jBkzZnDGGWeQm5ubsryINCz1ItizYcmSJcydO5dly5bRrFkz1q9fz44dO/Yq8/bbbwPwjW98Y7/Xv3hx+Ke71atXM3PmzPJgz8vLIy8vr9JlR48eXX59xowZHH/88bEL9rgftSD1zPN/SK/cxg/2r/wpv6xefaqgrpgU1qxZQ+vWrWnWLJxc27p1a3Jzc1m6dCkDBgygV69ejBgxgnXr1gEhiH7xi1/Qp08fjjnmGF54IfwF4+uvv06fPn3o3r07Xbt25d133wWgRYvwX75XX301L7zwAt27d2fChAksWLCAM888kz179tCxY0c2btxYXqejjz6atWvXMn78eG699VZmzZpFUVERF110Ed27d+fpp5/m7LPPLi8/b948zjmnqj94F5G4UbAns3kNZ3yzKx++v5Jjjvo6PxpZyMK/Pc7ODR/w4x/9kFn33snS5+dy7tlDuP1Pf4LNa2D3DnZ9vomX5z/J7b//Fb++/hrYvIapk27jilH/j+WLnqGoqIj27dvv9VA33XQT3/rWt1i+fDnjxo0rv79Ro0YMGTKEOXPmAPDSSy/RoUMH2rZtW17mvPPOIy8vj4ceeojly5czePBg3nrrLUpKwl9X3nvvvVxyySW1sMFEpC5RsKfQosWBLF34LNMn3kKb1ocxdPhopt37AK+9+Tannz2U7iedxpTp0/lk7dryZb733fDXpL26d2X1B2GAy359evH72yZx84Q/8/7779O8efO06zB06FAefTT8n/IjjzzC0KFDKy1vZlx88cU8+OCDbNy4kSVLljBoULp/lyoicaE+9ko0btyY/G/1J/9b/TnhuM7c+ZcZdDn2GyyZ/1cA3v5w/V7lm32lWbRcI3btDn+deWHB9+jbqydP/30+gwcPZtq0aQwcODCtx+/Xrx/vvfceJSUlPPHEE1x33XVVLjN8+HC++93vkpOTQ0FBAU2a6CUWaWjUYk/h7Xff491/fzE45fJXX6fzMUdTsn4DS14OwwwfefjB7CrdUOl6Vq56nyM7dWDs6JEMGTKEFStW7DW/ZcuWbNmyJemyZsY555zDlVdeSefOnWnVqtU+ZSoun5ubS25uLr/73e8YPnx42s9XROKjXjTn0j08MZNKSz/nx1ddy8ZNm2nSpAlHderI9Em3MKrwIsb+4no2bd7Crl27+MmYS+nSOfVRMY/NeYoHHp1N06ZNODz3CK655pq95nft2pXGjRvTrVs3CgsL6dGjx17zhw4dSu/evZkxY0bS9RcWFjJ69GiaN2/OkiVLaN68ORdddBElJSV07tz5y24GEamHqvzP05qW7I823nzzzeyG0uY1NbPeg75aM+ut4PLLL6dHjx6MGDEi6fysbt80DwPLH3cXAAsmXJreemvosLF6Rds2bft9OG0d3LZmttTdkx4bXS9a7JK+Xr16ceCBB3LbbbdluyoikiUK9phZunRptqsg0mCl3VKvYdp5KiISMwp2EZGYUbCLiMSM+tilTqorfZUi9VH9CPZ0R0pLV5qHGN34x4nMnDUnGrbXmHb7LfTN65mRKgwePJiZM2dyyCGHMGnSJKZMmULPnj0ZOnQob7zxBldffXXKZVMN+SsiAvUl2LNgyctFzH12PssWPRuG7d2wgR07dmZs/X/729/Kr0+ePJn58+eXDxB21llnVbpsqiF/JV5qauhi/RqKP/Wxp7Dmk3W0bnXoF8P2tmpF7lcPp+MJfbjq+t9yQr+B9DllMO/9exUAJes3cO4PRtI7fxC98wfx3y++DEBp6WcM/9FPOKHfQLp27crs2bMB6NixI+vXr2f06NGsXLmSQYMGMWHCBGbMmMHll18OwNq1aznnnHPo1q0b3bp1Kw/0VEP+nnzyySxfvrz8OZx00km88sortbPBRKTOULCncMbAAXz40ccc0/MkfnTlL1n4zyXl8w4+6CBeXfIcl48azk9+eQMAV/ziesZddin/s+AZZj9wFyN//DMAfnvLhPLyK1as2GcAsKlTp5Kbm8vzzz+/17C9AGPHjmXAgAG88sorLFu2jC5duuw1v+KQvyNGjCgfeuCdd95h27ZtdOvWLdObRkTqOAV7CsmG7Z3xUBhC94Lzzi6fLnk5nBA0f8ELXP7za+l+0mmcdUEhm7eUUlr6GfMXvMBllxaWr/fQQw9Nuw7PPfccY8aMAcJIkwcffHCl5QsKCpg7dy47d+7knnvuobCwsNLyIhJP6mOvRMVhe+97+DEgjLpYpuzqnj17eHH+XHJycrJRVQAOOOAATj/9dJ588kkee+wxnYUq0kCpxZ5CsmF7OxwRdm4++viT0fQp+vXpBYSumzum3fNF+RWvAXD6KSdz510zyu//9NNP067DqaeeypQpUwDYvXs3mzZt2mt+siF/R44cydixY+ndu/d+/ToQkfioHy32LIwul2rY3rnPzufTjZvo2v9Umn3lKzx892QAJt3yWy776TV07X8qu3bt4uT+32Tq7Tdz3c9/wmU/u4bjv3kKjZs244YbbuB73/teWnWYOHEio0aN4u6776Zx48ZMmTKFfv36lc+vOOTvuHHj6NWrFwcddJDGYhdpwOpHsGdBrx5dWTzvr0nn/fyKMdz8m73/zah1q1Y8OmPaPmVbtDiQ+6ZODDcShu1dvXp10uuFhYXlfeNt27blySef3GedpaWlADRt2pTnnntur3kff/wxe/bs4Ywzzkj53ETiasK8d9IqV/zp1v0qP66eJaW6YmLk/vvvp2/fvtx44400aqSXVqShqmffQ9m3+tWXs12FlIYNG8awYcOyXQ0RybI626zL9j87xZW2q0j81clgz8nJYcOGDQqhDHN3NmzYkNVDMkWk5tXJrpj27dtTXFxMSUlJdiqwbVPVZaojZ2PNrHd/qpCTUz4mjYjEU50M9qZNm9KpU6fsVSDTo0mWieGfAotI3VMnu2JERKT60gp2MzvRzFaY2XYzW2Zm+wxKbmbNzOwvZlZiZlvN7F9mNjDZ+kREpOZUGexmlgPMBloC44C2wCwza1yh6DBgBLAcuB7oBtyV0dqKiEiV0mmxDyKE+WR3nwzcDXQC8lOs6zVgPrAdyP7eQhGRBiadYC/bi/lRNC2OpkdWKHcfMAf4CfAv4HOgMNkKzWyUmRWZWVHWjnyRjMjPzy//px8RqRuqs/PUUtz/TeA7wEPA94HGwAxLHOM24u7T3T3P3fPatGlTjSqIiEgq6QT7qmhadvBzu2i60sxyzKxpdLsA+Aow1d0fBYqAnkDrTFVWRESqls5x7M8A64AxZraFsIN0dXTZCjwNnAmUDV5+lZl1A/oBG4D1ma2yiIhUpsoWu7tvI7TGS4GJhJAvAHZXKHonYcdqX+CPwFtAgWtcABGRWpXWmafuvgg4IcksSyizDRiZoXqJiEg16cxTEZGYUbCLiMSMgl1EJGYU7CIiMaNgFxGJmTo5HrtInE2Y905a5Yo/3bpf5cfp0ywRtdhFRGJG3/GSlFqVIvWXWuwiIjGjYBcRiRkFu4hIzKjHU0QanMtufSDbVahRarGLiMSMgl1EJGYU7CIiMaNgFxGJmQYR7Pn5+eTn52e7GiIitaJBBLuISEOiYBcRiRkFu4hIzOgEJflS4n6ih0h9pBa7iEjM1OsWu4aWFRHZl1rsIiIxo2AXEYkZBbuISMwo2EVEYkbBLiISMwp2EZGYaRAH9ukkGhFpSNRiFxGJGQW7iEjMKNhFRGKmQfSxi9RH2jck1ZVWi93MTjSzFWa23cyWmVnPFOWON7PnzGyrmW0ws1syW10REalKlS12M8sBZgNbgXHAtcAsMzva3XcnlGsO/BfQHPgV8DnQuiYqLSIiqaXTFTMIaAtc5e6Tzexw4HogH/hHQrkLgHbApcBD7r41w3UVEZE0pNMV0ymafhRNi6PpkRXKHRdNrwQ+N7MSMzs/2QrNbJSZFZlZUUlJyX5VWEREKledo2Isxf3Nouka4FxgOzDDzFpWLOju0909z93z2rRpU40qiIhIKukE+6po2j6atoumK80sx8yaVij3mLs/DrxA6G/PzUhNRUQkLen0sT8DrAPGmNkWYASwOrpsBZ4GzgQeAX4PDDezPcCphO6bf2e81iIiklKVLXZ33wYUAKXARELIFwC7K5T7GLgQaBOVew84y913ZbjOIiJSibROUHL3RcAJSWZZhXKPA49noF4iIlJNGlJARCRmFOwiIjGjYBcRiRkFu4hIzCjYRURiRsEuIhIzCnYRkZhRsIuIxIyCXUQkZhTsIiIxo2AXEYkZBbuISMwo2EVEYkbBLiISMwp2EZGYUbCLiMSMgl1EJGYU7CIiMaNgFxGJGQW7iEjMKNhFRGJGwS4iEjMKdhGRmFGwi4jEjIJdRCRmFOwiIjGjYBcRiRkFu4hIzCjYRURiRsEuIhIzCnYRkZhRsIuIxIyCXUQkZhTsIiIxo2AXEYmZtILdzE40sxVmtt3MlplZz0rKdjazbWbmZnZe5qoqIiLpqDLYzSwHmA20BMYBbYFZZtY4SVkD7gJ2Z7ieIiKSpnRa7IMIYT7Z3ScDdwOdgPwkZccAHYBpmaqgiIjsn3SCvVM0/SiaFkfTIxMLmVk74A+EcN9c2QrNbJSZFZlZUUlJyX5UV0REqlKdnaeW4v6bgCLgLeCw6L7DzaxFxYLuPt3d89w9r02bNtWogoiIpNIkjTKromn7aNoumq6M+t93u/tO4AhgAPBuwrJ3ABuBBzNQVxERSUM6wf4MsA4YY2ZbgBHA6uiyFXgaOBO4AShrfp8PFAC3AYsyWmMREalUlcHu7tvMrAC4E5gIvA5cSoUjX9x9Ydl1Mzs+uvqiu3+QueqKiEhV0mmx4+6LgBOSzEra3+7u44Hx1a6ViIhUm848FRGJGQW7iEjMKNhFRGJGwS4iEjMKdhGRmFGwi4jEjIJdRCRmFOwiIjGjYBcRiRkFu4hIzCjYRURiRsEuIhIzCnYRkZhRsIuIxIyCXUQkZhTsIiIxo2AXEYkZBbuISMwo2EVEYkbBLiISMwp2EZGYUbCLiMSMgl1EJGYU7CIiMaNgFxGJGQW7iEjMKNhFRGJGwS4iEjMKdhGRmFGwi4jEjIJdRCRmFOwiIjGjYBcRiRkFu4hIzKQV7GZ2opmtMLPtZrbMzHomKXNWNG+Lma03s3vMrHnmqywiIpWpMtjNLAeYDbQExgFtgVlm1rhC0W7AG8CVwFJgOHBVRmsrIiJVapJGmUGEML/K3Seb2eHA9UA+8I+Ecje7+w4AM3sRWAF0yWx1RUSkKul0xXSKph9F0+JoemRiobJQj3w7mi5KtkIzG2VmRWZWVFJSkm5dRUQkDdXZeWqVzjQ7F/g98DdgSrIy7j7d3fPcPa9NmzbVqIKIiKSSTrCviqbto2m7aLrSzHLMrGlZQTM7H3gEeB441913Z6ymIiKSlnSC/RlgHTDGzMYAI4DV0WUrMAfAzL4DzAQ2Ag8DZ5vZwMxXWUREKlNlsLv7NqAAKAUmEkK+AKjYGu8NNAZaA/cSwv1XmaysiIhULZ2jYnD3RcAJSWZZQpnxwPiM1EpERKpNZ56KiMSMgl1EJGYU7CIiMaNgFxGJGQW7iEjMKNhFRGJGwS4iEjMKdhGRmFGwi4jEjIJdRCRmFOwiIjGjYBcRiRkFu4hIzCjYRURiRsEuIhIzCnYRkZhRsIuIxIyCXUQkZhTsIiIxo2AXEYkZBbuISMwo2EVEYkbBLiISMwp2EZGYUbCLiMSMgl1EJGYU7CIiMaNgFxGJGQW7iEjMKNhFRGJGwS4iEjMKdhGRmFGwi4jEjIJdRCRmFOwiIjGTVrCb2YlmtsLMtpvZMjPrmaLcD82s2My2mtmTZtYqs9UVEZGqVBnsZpYDzAZaAuOAtsAsM2tcoVwPYCrwJnAD8B1gQqYrLCIilUunxT6IEOaT3X0ycDfQCcivUK4wml7j7rcAi4ELoi8GERGpJU3SKNMpmn4UTYuj6ZHAP6oo1wQ4Ang3cYVmNgoYFd0sNbO396PONe5KaA2sz/yar8n8KusZbduao21bc+rotu2QakY6wV6Rfdly7j4dmF6Nx64VZlbk7nnZrkccadvWHG3bmlPftm06XTGromn7aNoumq40sxwza1pJuV3Ah1+6liIikrZ0gv0ZYB0wxszGACOA1dFlKzAnKnd/NL3RzK4C+gOPuPu2TFZYREQqV2WwR8FcAJQCEwkhXwDsrlBuKXAZcBzwG8IXwrgM17e21NluohjQtq052rY1p15tW3P3bNdBREQySGeeiojEjIJdRCRmGmywm1lHM/Pocn3C/Xcn3J8fTf+czbrWN2Y2yczWRttubsL9Zdt1u5l9YmZPmFn3bNa1vjGzo83seTPbYGZbzGyemX094b3qZva5ma00s2lm1jrbda5PzOylaLt+bmZFZnZyfdy2DTbYKyi0oAVwfrYrExOPpLi/GBgJPEYYduKfZnZcrdWq/mtH+NzeANwLnAb8JWH+34GxwKuEkwAXmlnz2q5kPbaYsP1+C3Snnm5bBTusJJxFm08I9aZ8cfasVIO7jyX1OEGb3P2BqMwNwIHA1bVWufpvsbsPcPc/R9vwP0CXhPnvuvtf3H0IMI9wlNoF2ahoPXUl8FfCWfXbgT0J8+rNtlWwh0HLXgIuiS5PABuzWqOG45loWm/O6Ms2d99Rdt3M8oDDgEUpimv77r+DgRJCJuwg/LpMpk5vWwV7cA/h2PwTCT9vpXaUDTuhY273k5kdCzxFOFHwx6mKRVNt3/SVAmcQulxyCOfkJFOnt62CPXiEcMJVMeEnltSOb0fTpVmtRT0T7ZNYQGhRDnT3NSmKavvuJ3ff5e7z3P0O4GXgFMIAYBXV6W1bnUHAYsfdN5vZJcAWd99jts/4Zf3M7Kbo+lZ3/3Xt1rB+MbPvAMdHN48ws5HAwuj2wWZ2MeEn7BjgM+CmfdciyZjZEcDzQCvgOqCvmfUFPomKHG1mI4AzCS3PN4CHs1HX+sbMvk3Yz7aYMCptf2AtsCEqUm+2bYM989TMOhIGLnva3c+sMO81wg6pUwgfokSb3P2Q2qhjfWVmC4ABFe4ezhfdXDsJH5YXgfHu/krt1a5+M7N89n1Pwt7v1W3AGuBZ4Dp335CkvFRgZr0J79GvE3ac/gu4irCDv15t2wYb7CIicaU+dhGRmFGwi4jEjIJdRCRmFOwiIjGjYBcRiRkFu4hIzCjYRURi5n8BzPVhI8DugdEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "raxm5oTtuc0n",
        "outputId": "9f0f64ed-6113-4c18-8972-e415c51292c3"
      },
      "source": [
        "#Plot comparing FNR and FPR within PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "fnr_hvlv_pd_m = [hv_lv_pd_ml[0], hv_lv_1pd[0], hv_lv_2pd[0], hv_lv_3pd[0]]\n",
        "fnr_hvlv_pd_s = [hv_lv_pd_ml[1], hv_lv_1pd[1], hv_lv_2pd[1], hv_lv_3pd[1]]\n",
        "\n",
        "fpr_hvlv_pd_m = [lv_hv_pd_ml[0], lv_hv_1pd[0], lv_hv_2pd[0], lv_hv_3pd[0]]\n",
        "fpr_hvlv_pd_s = [lv_hv_pd_ml[1], lv_hv_1pd[1], lv_hv_2pd[1], lv_hv_3pd[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "p7 = plt.bar(x_pos - barWidth/2, fnr_hvlv_pd_m, yerr=fnr_hvlv_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "p8 = plt.bar(x_pos + barWidth/2, fpr_hvlv_pd_m, yerr=fpr_hvlv_pd_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,0.5)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((p7[0], p8[0]), ('Sensitivity', 'Specificity'))\n",
        "plt.title('FNR and FPR within PD')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'FNR and FPR within PD')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAbPUlEQVR4nO3de5wU5b3n8c+XkXVQEA2wuIA6aGRDuAoDLJrIxAQ9EG/EIKu+dEEIBwKiuOeFRknE4yUmRw+CkVu8EDGoLESTgGYXjYAR1AwcxIB4OYhxlOiAgJAAcnn2j6oZm7F7pmfoYeji+3695lXT/Txd/asSv1P9VNXTCiFgZmbJ0aihCzAzs9xysJuZJYyD3cwsYRzsZmYJ42A3M0sYB7uZWcI42C0xJC2RNKKh68iGpBmSflxN+yRJj1fTvlZSSb0UZ3nPwW4HkbRR0i5JO1N+2kgqkhQkPVul/+OSJsW/l0g6EL9mh6S3JA1rkA2pIg7KvVW2a0LctkTS7vi5zZJ+I+m/xW2zJX0et30qabGkrx1qPSGEUSGEO+L3KJFUVsvXdwohLKnLe8f/Hf8eb9OHkv5dUkHcVvHff4ekbZKWSxolyVmRR/wfy9K5KITQNOXno5S2PpLOrua1H4UQmgInAOOBX0r67/VabfaeqrJdP09pGxvX3QE4EZic0vbzuK0t8CHw8OErud50i7fp28CVwA9S2i4KITQDTgPuAW4iGdt81HCwW239HLirpk4h8izwKdA1Uz9J/0fS3yRtl7RMUqeUttmSHpS0KD6CfFXSGSnt/SWtj1/7C0CHtmkQQvgUWAB0TtO2C5gHdM+wLYXx0W7L+PGtkvZJOiF+fIek+1O27U5JxwPPAW1SPyHFq/wvkh6Lt32tpOKU99oo6Tvx75MkzcvUt4btXQ+8lGF7t4cQfgcMAf6XpC/1sSOTg91qaxrQoSJUMpHUSNLFQEvg3Wq6PgecCfxXYBXw6yrt/xO4HTgpXs9d8fpbAr8BJsbv8Z/AObXdmDR1twQuA/4jTdvxwBVk2J4Qwm7gz0C/+Kl+wPspdfUDllZ5zd+BAcSfdKp8QroYeJLoE8TvgF9UU3pt+qZu09eBb5Jme1NqfA0oi/tZHnCwWzrPxOOr2yQ9U6VtF1G43pnhtW0kbYv7PQ3cGEKoLjQeCSHsCCHsASYB3SQ1T+nydAjhtRDCPqLQrzhaHgisDSHMDyHsBe4H/lbDdl2esl3bUo6MAabGdb8ObAJuTGn7l7htB/AN4Opq3mMp0E/SMUSfVKbGjwuBXsCyGmpM9acQwrMhhP3AHKBbjvoCrJK0Ffg98BDwaA39PwK+kmXd1sAc7JbOpSGEE+OfS9O0PwS0lnRRmraPQggnEo2xTwXOy/Qmkgok3SPpPyV9BmyMm1qmdEsN638ATePf2wAfVDSEaDa7D6jevJTtOrHKuYNx8XNtQwhXhRDKU9rujbepiOgPVnXnDJYCJUAP4A1gMdGR+v8A3g0hbKmhxlRVt70w/oNxqH0BeoQQTgohnBFCmBhCOFBDLW2JhtUsDzjYrdZCCJ8TDY/cQYZx7fgI/Cagi6R0fxwgOml3CfAdoDlRcJJpnVVsAk6peCBJqY/rQwjhr8D1wBRJTTJ0W04U/IOApSGEdcCpRJ8wlmZ4zRE9xaqkXkTB/qeGrsWy42C3upoDFAL/lKlD/AfgPuAnGbo0A/YAW4DjgLtr8f6LgE6SvhcfmY4DTq7F6+skhLCYaFhiZIb2fwArgTF8EeTLgVFkDvaPgRZVhqAanKQTJF1INHb/eAjhjYauybLjYLc6icdyf0LN466PAKdmGLZ5jOjk4ofAOuCVWrz/ZmAw0eV4W4hOwL6c7esP0b8BEyQdm6F9KdAYeC3lcTMyjK/HV6Y8AWxIM/bfEH4vaQfR0NatwL8DR8T9CJYd+Ys2zMySxUfsZmYJk1WwSzpH0hpJeyStktQjTZ+KW85Tf+7PfclmZladGodi4utvNxJd5vVvRGNue4Az43HWin5FwHvADL44SfRWddcwm5lZ7lV3nWuFAUBrYEIIYZqkk4EfE12r+0Ka/qXA7+KrA8zM7DDLJtjbx8sP42XFLHSnkz7Yfwk8JGkdMDyE8KUrHSSNJL5c7Pjjj+/5ta8d8mR5ZmZHlZUrV24OIbRK15ZNsFeV6eaRvwO3AauJLj27h+gW8DOqdgwhzAJmARQXF4fS0tI6lGFmdvSS9H6mtmyC/b142S5eto2XG+Lx9/0hhL3xLdj/mvKmVwI9JBXGkyOZmdlhkE2wPwd8AoyOb1oYTnQydSPRCdVFwIWSfkA0ydGrRMM03YHXHepmZodXjZc7xsE8GNgJTCEK+cHA/ipd3yaaze5+4IfAH+J+ZmZ2GGU1xh5CWAZ0SdOklD5LiWawM7OjzN69eykrK2P3bn9Az7XCwkLatWtH48aNs35NXU6empkdpKysjGbNmlFUVEQ00ablQgiBLVu2UFZWRvv27Wt+QcxTCpjZIdu9ezctWrRwqOeYJFq0aFHrT0IOdjPLCYd6/ajLfnWwm5kljMfYzSznJi9+O6frG9+/Q4197rrrLubOnUtBQQGNGjVi5syZ9OnTJyfvf/bZZ7N8+XI2btzI8uXLufLKKwEoLS3lscceY+rUqRlfO2PGDI477jiuueYaZs+ezfnnn0+bNvU75b6D3czy3ooVK1i4cCGrVq3i2GOPZfPmzXz++ec5W//y5csB2LhxI3Pnzq0M9uLiYoqLi6t97ahRoyp/nz17Np07d673YPdQjJnlvU2bNtGyZUuOPTb6UquWLVvSpk0bVq5cSb9+/ejZsycXXHABmzZtAqCkpISbbrqJ3r1706FDB1566SUA1q5dS+/evenevTtdu3blnXfeAaBp0+g71G+++WZeeuklunfvzuTJk1myZAkXXnghBw4coKioiG3btlXWdOaZZ/Lxxx8zadIk7r33XubPn09paSlXXXUV3bt3Z9GiRVx66RdfB7x48WIGDRqUk/3hYDezvHf++efzwQcf0KFDB374wx+ydOlS9u7dy3XXXcf8+fNZuXIl1157Lbfeemvla/bt28drr73G/fffz+233w5EwybXX389q1evprS0lHbt2h30Pvfccw/f/OY3Wb16NePHj698vlGjRlxyySU8/fTTALz66qucdtpptG7durLP97//fYqLi/n1r3/N6tWrGThwIOvXr6e8vByARx99lGuvvTYn+8PBbmZ5r2nTpqxcuZJZs2bRqlUrhgwZwsyZM/nLX/5C//796d69O3feeSdlZWWVr/ne974HQM+ePdm4cSMAffv25e677+ZnP/sZ77//Pk2aNMm6hiFDhvDUU08B8OSTTzJkyJBq+0vi6quv5vHHH2fbtm2sWLGCAQMG1HLL0/MYu5klQkFBASUlJZSUlNClSxcefPBBOnXqxIoVK9L2rxi2KSgoYN++fQBceeWV9OnTh0WLFjFw4EBmzpzJeeedl9X79+3bl3fffZfy8nKeeeYZJk6cWONrhg0bxkUXXURhYSGDBw/mmGNyE8k+YjezvPfWW29VjocDrF69mo4dO1JeXl4Z7Hv37mXt2rXVrmfDhg2cfvrpjBs3jksuuYQ1a9Yc1N6sWTN27NiR9rWSGDRoEDfeeCMdO3akRYsWX+pT9fVt2rShTZs23HnnnQwbNizr7a2Jj9jNLOeyuTwxl3bu3Ml1113Htm3bOOaYY/jqV7/KrFmzGDlyJOPGjWP79u3s27ePG264gU6dOmVcz7x585gzZw6NGzfm5JNP5pZbbjmovWvXrhQUFNCtWzeGDh3KWWeddVD7kCFD6NWrF7Nnz067/qFDhzJq1CiaNGnCihUraNKkCVdddRXl5eV07NjxkPdDhRq/87S++Ys2zPLfm2++mdNgOpqMHTuWs846i+HDh2fsk27/SloZQkh7raWP2M3MGkjPnj05/vjjue+++3K6Xge7mVkDWblyZb2s1ydPzcwSxsFuZpYwDnYzs4RxsJuZJYxPnppZ7r3409yu71s/qrFLfU7bO3DgQObOncuJJ57I1KlTmT59Oj169GDIkCGsW7eOm2++OeNrM035W58c7GaW9+p72t5nn3228vdp06bx/PPPV04QdvHFF1f72kxT/tYnD8WYWd7LNG1vUVEREyZMoEuXLvTu3Zt3330XgPLyci677DJ69epFr169ePnll4HoDtZhw4bRpUsXunbtyoIFCwAoKipi8+bNjBo1ig0bNjBgwAAmT57M7NmzGTt2LAAff/wxgwYNolu3bnTr1q0y0DNN+XvuueeyevXqym34xje+weuvv56T/eFgN7O8l27a3grNmzfnjTfeYOzYsdxwww0AXH/99YwfP54///nPLFiwgBEjRgBwxx13VPZfs2bNlyYAmzFjBm3atOHFF188aNpegHHjxtGvXz9ef/11Vq1a9aWpC6pO+Tt8+PDKqQfefvttdu/eTbdu3XKyPxzsZpb30k3bWxGaV1xxReWyYkKw559/nrFjx9K9e3cuvvhiPvvsM3bu3Mnzzz/PmDFjKtd70kknZV3DH//4R0aPHg1EM0Y2b9682v6DBw9m4cKF7N27l0ceeYShQ4fWYour5zF2M0uEqtP2/upXvwKiWRcrVPx+4MABXnnlFQoLCxukVoDjjjuO/v3789vf/pZ58+bl9C5UH7GbWd5LN23vaaedBlD55RdPPfUUffv2BaKhmwceeOCg/gD9+/fnwQcfrHx+69atWdfw7W9/m+nTpwOwf/9+tm/fflB7uil/R4wYwbhx4+jVq1etPh3UxEfsZpZ7WVyemEuZpu1duHAhW7dupWvXrhx77LE88cQTAEydOpUxY8bQtWtX9u3bx7nnnsuMGTOYOHEiY8aMoXPnzhQUFHDbbbdVftNSTaZMmcLIkSN5+OGHKSgoYPr06ZV/SODLU/6OHz+enj17csIJJ+R0LnbwtL1mlgNH6rS9RUVFlJaW0rJly4YuJa2PPvqIkpIS1q9fT6NGmQdQajttr4dizMwawGOPPUafPn246667qg31uvBQjJklVsWXVB+JrrnmGq655pp6WbeP2M0sJxp6WDep6rJfHexmdsgKCwvZsmWLwz3HQghs2bKl1pdleijGzA5Zu3btKCsro7y8vKFLSZzCwsLKeWmy5WA3s0PWuHFj2rdv39BlWMxDMWZmCZNVsEs6R9IaSXskrZLUo5q+HSXtlhQkfT93pZqZWTZqDHZJhcACoBkwHmgNzJdUkKavgF8C+3Ncp5mZZSmbI/YBRGE+LYQwDXgYaA+UpOk7GjgNmJmrAs3MrHayCfaKMyIfxsuyeHl6aidJbYGfEoX7Z9WtUNJISaWSSn0W3cwst+py8lQZnr8HKAXWA1+JnztZUtOqHUMIs0IIxSGE4latWtWhBDMzyySbyx3fi5cVF1K2jZcb4vH3/SGEvcApQD/gnZTXPgBsAx7PQa1mZpaFbIL9OeATYLSkHcBwYGP8swtYBFwI3AZUHH5fDgwG7gOW5bRiMzOrVo3BHkLYLWkw8CAwBVgL/IAqV76EECq/ZFBS5/jXV0IIf81duWZmVpOs7jwNISwDuqRpSjveHkKYBEyqc1VmZlZnvvPUzCxhHOxmZgnjYDczSxgHu9kRqqSkhJKSkoYuw/KQg93MLGEc7GZmCeNgNzNLGAe7mVnCONjNzBLGwW5mljAOdjOzhDkqgt3XA5vZ0eSoCHYzs6OJg93MLGEc7GZmCeNgNzNLGAe7mVnCONjNzBLGwW5mljAOdjOzhHGwm5kljIPdzCxhHOxmZgnjYDczS5hjGrqAQzF58dtZ9SvbuqtW/cf371DnmszMGpqP2M3MEsbBbmaWMA52M7OEcbCbmSWMg90Oib+dyuzI42A3M0sYB7uZWcI42M3MEsbBbmaWMFkFu6RzJK2RtEfSKkk90vTpLOlNSbslbZP0rKS2uS/ZzMyqU2OwSyoEFgDNgPFAa2C+pIIqXQ8ATwIjgd8AA4DbclqtmZnVKJsj9gFEYT4thDANeBhoD5SkdgohrAN+CvwBWB4/fSBnlZqZWVaymQSsfbz8MF6WxcvTgReq9B0IPB3/vpYMR+ySRhId2XPqqadmW2udjbl3Tr2/h5nZkaIuJ09VTdvLREf4U4BOwD+n6xRCmBVCKA4hFLdq1aoOJZiZWSbZBPt78bJdvKw4IbpBUqGkxhUdQwjlIYQ/AP+baBjm8pxVamZmWclmKOY54BNgtKQdwHBgY/yzC1gEXCjpR0BzYD1wHtEfjXW5L9nMzKpT4xF7CGE3MBjYSTTE8kn8eH+VruXAFcBM4ALgCWBsLos1M7OaZfUNSiGEZUCXNE1K6fMQ8FCO6jIzszrynadmZgnjYDczSxgHu5lZwjjYzeyok/QviHGwm5kljIPdzCxhHOxmZgnjYDczSxgHu5lZwjjYzcwSxsFuZpYwDnYzs4TJahIwO/pMXvx2Vv3Ktu6qVf/x/TvUuSYzy46P2M3MEsbBbmaWMA52M7OE8Ri72WHm8xdW33zEbmaWMA52M7OEcbCbmSWMg93MLGEc7GZmCeNgNzNLGAe7mVnCONjNzBLGwW5mljAOdjOzhHGwm5kljIPdzCxhHOxmZgnjYDczSxgHu5lZwjjYzcwSxsFuZpYwDnYzs4TJKtglnSNpjaQ9klZJ6pGmz8Vx2w5JmyU9IqlJ7ks2M7Pq1BjskgqBBUAzYDzQGpgvqaBK127AOuBGYCUwDJiQ02rNzKxG2XyZ9QCiMJ8QQpgm6WTgx0AJ8EJKv5+FED4HkPQKsAbolNtyzcysJtkEe/t4+WG8LIuXp5MS7BWhHrsgXi5Lt0JJI4GRAKeeemq2tdoRaMy9cxq6BDOroi4nT1Vto3QZcDfwLDA9XZ8QwqwQQnEIobhVq1Z1KMHMzDLJJtjfi5ft4mXbeLlBUqGkxhUdJV0OPAm8CFwWQtifs0rNzCwr2QT7c8AnwGhJo4HhwMb4ZxfwNICk7wJzgW3AE8Clks7LfclmZladGoM9hLAbGAzsBKYQhfxgoOrReC+gAGgJPEoU7j/JZbFmZlazbE6eEkJYBnRJ06SUPpOASTmpyszM6sx3npqZJYyD3cwsYRzsZmYJ42A3M0sYB7uZWcI42M3MEsbBbmaWMA52M7OEcbCbmSWMg93MLGEc7GZmCeNgNzNLGAe7mVnCONjNzBLGwW5mljAOdjOzhMnqizbMzPLB5MVvZ9WvbOuuWvUf379DnWtqCD5iNzNLGAe7mVnCONjNzBLGwW5mljAOdjOzhHGwm5kljIPdzCxhHOxmZgnjYDczSxgHu5lZwjjYzcwSxsFuZpYwngTM7Ag15t459bLekpISAJYsWVIv67eG5yN2M7OEcbCbmSWMg93MLGEc7HZEKikpqRwLNrPaySrYJZ0jaY2kPZJWSeqRpk8TSS9I2ikpSPqX3JdrZmY1qTHYJRUCC4BmwHigNTBfUkGVrgXAp8Afcl2kmZllL5sj9gFEYT4thDANeBhoD5Skdgoh7AwhDAYW5rpIMzPLXjbB3j5efhgvy+Ll6XV9U0kjJZVKKi0vL6/raszMLI26nDzVob5pCGFWCKE4hFDcqlWrQ12dmZmlyCbY34uX7eJl23i5QVKhpMa5L8vMzOoqm2B/DvgEGC1pNDAc2Bj/7AKerugoaQRwbvywt6QRkprmsmAzM6tejcEeQtgNDAZ2AlOIQn4wsD9N918Cw+LfB8ePW+akUjMzy0pWk4CFEJYBXdI0qUq/Qx5/NzPLV0fKBGu+89TMLGEc7GZmCeNgNzNLGH/Rhh1eL/40u37b/lq7/t/6Ud3qSRLvW4v5iN3MLGEc7GZmCeNgNzNLGAf7IfCXQZjZkcjBbmaWMA52M7OEcbCbmSWMg93MLGF8g1I6vtHDzPKYj9jNzBLGwW5mljAOdjOzhPEY+yFYMvkHDV1CYnnf2hElz867+YjdzCxhHOxmZgnjYDczSxiPsZsdZXz+Ivl8xG5mljAOdjOzhHGwm5kljIPdzCxhHOxmZgnjYDczSxgHu5lZwjjYzcwSxjcomdlRZ8y9cxq6hHrlI3Yzs4RxsJuZJYyD3cwsYTzGbmaWI0fKBGs+YjczS5isgl3SOZLWSNojaZWkHhn6/bOkMkm7JP1WUovclmtmZjWpMdglFQILgGbAeKA1MF9SQZV+ZwEzgDeB24DvApNzXbCZmVUvmyP2AURhPi2EMA14GGgPlFTpNzRe3hJC+DmwHLgi/sNgZmaHSTYnT9vHyw/jZVm8PB14oYZ+xwCnAO+krlDSSGBk/HCnpLdqUXO9uxFaAptzv+Zbcr/KPON9W3+8b+vPEbpvT8vUUJerYnSo/UIIs4BZdXjvw0JSaQihuKHrSCLv2/rjfVt/8m3fZjMU8168bBcv28bLDZIKJTWupt8+4INDrtLMzLKWTbA/B3wCjJY0GhgObIx/dgFPx/0ei5d3SZoAnA08GULYncuCzcysejUGexzMg4GdwBSikB8M7K/SbyUwBvg68K9EfxDG57jew+WIHSZKAO/b+uN9W3/yat8qhNDQNZiZWQ75zlMzs4RxsJuZJcxRG+ySiiSF+OfHKc8/nPJ8Sbz8RUPWmm8kTZX0cbzvFqY8X7Ff90j6m6RnJHVvyFrzjaQzJb0oaYukHZIWSzoj5d9qkPQPSRskzZTUsqFrzieSXo336z8klUo6Nx/37VEb7FUMVaQpcHlDF5MQT2Z4vgwYAcwjmnbiT5K+ftiqyn9tif6/vQ14FPgO8FBK+/8DxgFvEN0EuFRSk8NdZB5bTrT/7gC6k6f71sEOG4juoi0hCvXGfHH3rNVBCGEcmecJ2h5CmBP3uQ04Hrj5sBWX/5aHEPqFEH4R78NPgU4p7e+EEB4KIVwCLCa6Su2Khig0T90I/J7orvo9wIGUtrzZtw72aNKyV4Fr459ngG0NWtHR47l4mTd39DW0EMLnFb9LKga+AizL0N37t/aaA+VEmfA50afLdI7ofetgjzxCdG3+OUQfb+3wqJh2wtfc1pKkrwG/I7pR8LpM3eKl92/2dgLnEw25FBLdk5POEb1vHeyRJ4luuCoj+ohlh8cF8XJlg1aRZ+JzEkuIjijPCyFsytDV+7eWQgj7QgiLQwgPAK8B3yKaAKyqI3rf+qvxgBDCZ5KuBXaEEA5IX5q/rK+ke+Lfd4UQbj+8FeYXSd8FOscPT5E0AlgaP24u6Wqij7Cjgb8D93x5LZaOpFOAF4EWwESgj6Q+wN/iLmdKGg5cSHTkuQ54oiFqzTeSLiA6z7acaFbas4GPgS1xl7zZt0ftnaeSiogmLlsUQriwSttfiE5IfYvof6JU20MIJx6OGvOVpCVAvypPD+OLYa69RP+zvAJMCiG8fviqy2+SSvjyv0k4+N/qbmAT8H+BiSGELWn6WxWSehH9Gz2D6MTpfwATiE7w59W+PWqD3cwsqTzGbmaWMA52M7OEcbCbmSWMg93MLGEc7GZmCeNgNzNLGAe7mVnC/H/z5/pbiiWEYgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "Dal7Xd1cm7Bg",
        "outputId": "9adf0f69-8862-4eab-aee1-1c265f38da24"
      },
      "source": [
        "#Plot comparing sensitivity of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "sens_hala_pd_m = [ha_ha_pd_ml[0], ha_ha_1pd[0], ha_ha_2pd[0], ha_ha_3pd[0]]\n",
        "sens_hala_pd_s = [ha_ha_pd_ml[1], ha_ha_1pd[1], ha_ha_2pd[1], ha_ha_3pd[1]]\n",
        "\n",
        "sens_hala_nc_m = [ha_ha_nc_ml[0], ha_ha_1nc[0], ha_ha_2nc[0], ha_ha_3nc[0]]\n",
        "sens_hala_nc_s = [ha_ha_nc_ml[1], ha_ha_1nc[1], ha_ha_2nc[1], ha_ha_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "a1 = plt.bar(x_pos - barWidth/2, sens_hala_pd_m, yerr=sens_hala_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "a2 = plt.bar(x_pos + barWidth/2, sens_hala_nc_m, yerr=sens_hala_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,1)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((a1[0], a2[0]), ('PD', 'NC'))\n",
        "plt.title('Sensitivity in Arousal Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Sensitivity in Arousal Classification')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAaYUlEQVR4nO3deZhdVZ3u8e9LElOYhCAhBDNcTARkFITYNKASFBsRHLoxNIhAQgiSFmwTgUe8DKFbBWkUgpDGMMigDDJoo3SuDILRS0ObgB1mwYASIKQIBDJUQoZf/7F2mZOTqjq7KqeqUivv53nOs6v2XmefX+069Z611x5KEYGZmeVji+4uwMzM6svBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQf7ZkbSUkmj2lj+pKQxJdZzrKR76lTTRyU9W491baokXSfpWxvx/JC0Yz1rqlj3er9LSQdKeq54r3xe0kxJJ3TC614p6Zx6r9dAPo+9+0j6CHARsDuwBnga+FpE/L6LXv86YH5EnF2HdQWwU0Q8v9GFdez1xwAPAN+IiO92Rw1tqbWtJb0X+BbwaaA/8DJwK3BRRCzryu0r6X7groiYVsd1jgNOioiP1Gud1jr32LuJpK2AXwI/ALYBhgHnAyu7s64e7ATgDeD4thpJ6t015ZQnaRvgv4Atgf0jYgDwSWBr4P3dUNIOwJPd8LpWLxHhRzc8gNHA4hptTiT14t8EfgXsULEsgFOA54DFwBWs2wPbEfgN8BbwOnBr1fN2BE4GVgHvAEuBXxTLXwQOAYYCTcA2Fc/9ULG+PsA44HfF/FnFepcV6/pH4AngMxXP7VM890Mt/JxjSL1ZKmo4HZhb/Ay3Ag1tbKd+wBLg6OLnGV2x7H1FbROAvxS1bgGcDfwZWAjcAAxsqZbKbVJ8/TfAbOBt4DXg+xXtbgMWFDXPAnavWHYd8K1W6v8W8DiwRRs/YwA7Fl8fDjxW1PASMLWiXQPwY2BR8b74PTCkWDYOmFdsqxeAYyvmN/8u/wSsLX73S4G+wIOk3nbza0wkvS+XAE8B+xTzv1E8v3n+3xfzdwVWkPZKl1K876u3SbHe50kf0HcBQ8u83/3Y8OEee/f5I7BG0vWSDpP0nsqFkj4HfBP4B2Aw8Fvg5qp1HAF8GPggcBRwaDH/X4F7gPcAw0l7BeuJiBnAT0i7+v0j4jNVy18h9SKPrJj9ReD2iFhV1fZjxZd7Feu6lRSWX6po9mng1Yh4rJXtUe0o4FPAyOLnG9dG238gBcZtpA/AlsaDDyIFzKHFusYBBwOjSEMfl5esaxowLSK2IvWmf1qxbCawE7Ad8Chp+5ZxCHBnRKwt2X4Zac9ka1LIT5L0+WLZCcBAYAQwiBSGTZL6AZcBh0XaIzgA+EP1iiPi/aQPwM8Uv8v19iAljQWmFq+/FfBZ0ocIpFD/aPH65wM/lvTeiHi6qOO/inVuXf26kj4OXED6vb+X9KF7S1Wz1t7vVsXB3k0i4m3gI6SeyFVAo6S7JA0pmpwCXBART0fEauA7wN6SdqhYzYURsTgi/kIaX967mL+KtDs9NCJWRMTvOljmTcAxAJJE6hHfVPK5PwY+XQw5ARwH3NiO174sIl6JiDeAX7DuZ2vJCaS9kjVFfUdL6lPVZmpELIuIJuBYUk97XkQsBc4qnlNmmGYVsKOkbSNiaUQ83LwgIq6NiCVFGE4F9pI0sMQ6BwGvlmjX/DoPRsTjEbE2IuaSPvAPqqhvEKl3vyYi5hTvNUg98T0kbRkRr0ZER4ZbTiJ1Bn4fyfMR8eeirtuK39na4sP9OdIeThnHAtdGxKPF9jsL2F/S+yratPZ+tyoO9m5UhPa4iBgO7EEa/ri0WLwDME3SYkmLSbunIo3FN1tQ8fVyUs8T4Myi7X8XZ7mc2MES7yD9cb0X+BgpGH5b5olFj///A0dK2ho4jPI9WGj9Z1uPpBGknnfzuv+DNBxxeFXTlyq+HkrqETb7M9AbGEJtE4CdgWck/V7SEUUdvSRdKOlPkt4mDd8AbFtinYtIvdRSJO0n6QFJjZLeInUCml/nRtJeyy2SXpF0kaQ+EbGMNER2CvCqpLsl7VL2NSuMIPXMW6rreEl/qHjP7kG5nx+qfifFB+4iyr3frYqDfRMREc+Qxhz3KGa9BHw5IraueGwZEQ+VWNeCiJgYEUOBLwPTWzlVrs1ToiLiTdKQzj+ShmFuiYj2nEZ1PWk4ZixpN/zldjy3rONI7+NfSFpAGkNuYMPhmMq6XyF9cDb7P8Bq0pj5MuDdzQsk9SINhaWVRDwXEceQhlu+C9xeDHN8EfgcaVhlIGlsH9IHbC33AX8vqezf402kMegRETEQuLL5dSJiVUScHxG7kYZbjqA4oBwRv4qIT5I+RJ4h7Sm210u0cEC32JO8CjgVGFQMtzzBup+/1vtmvd9JsU0Hkc4OsnZysHcTSbtI+rqk4cX3I0jDHs279lcCZ0navVg+sBjfLLPusc3rJR14DVJvu9prpDHmttxECoYv0PYwTEvr+jmwD/DPpDH3znACaTx374rHkaRhoEGtPOdmYLKkkZL6k4a5bi2GvP4INEg6vBjOOZt0ABEASV+SNLgYD19czF4LDCCd0bSI9MHwnXb8DN8njVdf3zzUJmmYpO9L+mAL7QcAb0TECkl/Q/pQaa7vYEl7Fh9Ib5OGZtZKGiLpc0VgriQdkyg7pl/pauB0Sfsq2bGouR/pfdZY1DGedZ0USO+P4ZLe1cp6bwbGS9pbUl/S9nskIl7sQI2bPQd791kC7Ac8ImkZKdCfAL4OEBE/I/UIbyl27Z8gDWeU8eFivUtJPbt/joh5LbS7Btit2HX+eSvruot0QHBBRPxPG685lRRMiyUdVfwMTaThnJHAnSVrL03S35J6eVcUeynNj7tIZ1cc08pTryUNWcwinR2yAjitqPkt4J9IAfYyqQc/v+K5nwKeLLbtNODo4ue8gTSU8DLpjJCHKak4jnAAKYQfkbQEuJ90dk1L563/E/AvRbtzWf8A7vbA7aRQf5p0dtSNpL/1KaSe8RukMflJZWusqPU24NukD/klpA/vbSLiKeB7pAPurwF7kobimv2adArlAkmvt7De+4BzSO+XV0l7BUe3tz5LfIGSdSpJ5wI7R8SXajY2s7rY5C7WsHwUF95MII2Dm1kXqTkUI+kySa8p3avil220O1DSXEkrJT0qaZ/6lmo9iaSJpANtMyNiVnfXY7Y5KTvGXn2hwHokNZDGxgYAk0mnjd1eHMCxzVBEXBUR/SLilO6uxWxzUzPYI+KrwCU1mh1GCvPpETGddFBuJOnybDMz60L1GmMfWUybzzltPotgFOno/noknUy6Vwn9+vXbd5ddOnKdhFnenn023cn4Ax/4QI9Yb0+SwzaYM2fO6xExuKVlnXXwtM2LMor7lMwAGD16dMyePbuTyjDrucaMGQPAgw8+2CPW25PksA0k/bm1ZR0+j11S3+JCAkjnAkO64RSsuwy4pXOnzcysE9XssUs6nHVXkI2QdBLpood7SfeB6E+6q91C0l3mlpBOcXuRdLvPbOXwqb+pynnbXnLvH0u1m/9mU7vaT/7kzh2uyepjU3nflumxnwFcWHz9QdL9IA6sbBARK0j3A2m+Gm8hMLa4256ZmXWhmj32iBjTyqLrqtrNIl1GvMlp96foAxeUa7f4L+1rf/BZ5dr1IJtKDyVHX7m4PXc5Nthw76aP1rLrVqvo33v9K+zPPudcAB55dG6p9W6l5aXanXdu+s+HTz9W8o4SDbXv6tzQ0MDw4cPp06f6TtSt85WnZpatXbdaxQ7bb0v/gVuT/qVAsvClFJLbjRjZ2lPXM4Q3S7Xb4qV0G5wPjCh5t+Kt2r5bc0SwaNEi5s+fz8iR5WqFHh7snTZWWXKrPHjJxHINbR3vDXUeb9sN9O8dG4R6Zyod6CVJYtCgQTQ2Nrbreb67o5llratCvbN0pP4e3WMvy2OV7dfde0NmnansEExP5T8zM9tsXDWrY5fW9KOpxfmTPza0zef1es9w9tx9F1avXsOuO+/E9Vdeyrvf/e6/zl+1Bnr37s3xxx/P5MmT2WKL+gyieCjGNkkPXjLRxzCsx9tyywb+8Lv7eOLhB3jXu/pw5bU3rjf/ySef5N5772XmzJmcf/75dXtdB7uZWRf46P778fy8FzeYv9122zFjxgwuv/xy6vWPjzwUYxvFxy96Hu8Jdb3Vq1cz875f86lPHNzi8lGjRrFmzRoWLlzIkCFDNvr1HOxmZp2kqWkFe3/kECD12Ccc39q/4a0vB7uZWSdpHkuvZd68efTq1YvtttuuLq/rMXYzs27U2NjIKaecwqmnnlq3c+7dYzezzcbEj43q0PPK3lKgrOYhmubTHY877jimTJlSt/U72M3MOsnSV55vcf6aN4t/MlfjXjEd5aEYM7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLj0x3NbLPR76F/6+AzW75tLwec1uazNHAoU079Mt/79nkAXHzZv7N02TKmnnU6ADfccAMXXXQRkujduzfHHnssp59+egdrXMc9djOzTtK3b1/uvOs/eX3Rog2Wzbz311x66aXcc889PP744zz88MMMHFj7n1uX4WA3M+skvXv34uRxX+KSK67aYNkF3/8BF198MUOHpn/W0bdvXyZOrM+dNx3sZmad6CsTx/GT2+7krbfeXm/+E089y7777tspr+lgNzPrRFttNYDjj/4Cl/3wmi57TQe7mVkn+9qkiVxz480sW778r/N233Vn5syZ0ymv52A3M+tk22zzHo76/Ge45sab/zrvrCmnccYZZ7BgwQIA3nnnHa6++uq6vJ5PdzSzzcayA87o0PP61+G2vV8/7RQuv+pHf/3+03/3CV5bsopDDjmEiEASJ5544ka/DjjYzcw6TeVte4dsN5jlC+att3z8+PGMHz++7q/roRgzs8w42M3MMuNgN7OsRUR3l7BROlK/g93MsrV0tVj61ps9NtwjgkWLFtHQ0NCu5/ngqZll6+m3+wCL6P/66xu1nje0vHajjmhYXLtJQwPDhw9v12od7GaWrVWxBXPf6rvR65nc++46VNOCg8/qlNV6KMbMLDOlgl3SgZLmSlop6VFJ+7TQpq+kqyU1SmqS9Jikj9e/ZDMza0vNYJfUANwBDAAmA0OA2yX1qmp6PDAB+ANwDrAXsOG9Ks3MrFOV6bEfRgrz6RExHbgGGAmMaWVdTwD3ASuB2kcGzMysrsoE+8hi+nIxnV9MR1W1ux74GfA14DFgOTCupRVKOlnSbEmzGxsb21WwmZm1rSMHT9XK/L8FDgd+AhwN9AKuk7RB+4iYERGjI2L04MGDO1CCmZm1pkywv1BMm0+kHFZM50lqkNSn+H4s8C7gyoi4FZgN7ANsW69izcystjLnsc8EFgKTJC0hHSB9sXg0AXcDRwDNty07U9JewP7AImDjrgwwM7N2qdljj4gVpN74UmAaKeTHAmuqml5BOrC6H/BvwDPA2Oip1/KamfVQpa48jYhZwJ4tLFJFmxXASXWqy8zMOshXnpqZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWmVLBLulASXMlrZT0qKR9Wmm3h6RfS2qStEjSRfUt18zMauldq4GkBuAOoAmYDPxf4HZJO0XEmop2WwL/D9gSOBdYDmzbGUWbmVnragY7cBgwBDgzIqZL2h44BxgD3F/R7hhgGDAR+ElENNW5VjMzK6HMUMzIYvpyMZ1fTEdVtdutmE4BlktqlHRUSyuUdLKk2ZJmNzY2tqtgMzNrW0cOnqqV+X2L6avAkcBK4DpJA6obRsSMiBgdEaMHDx7cgRLMzKw1ZYL9hWI6vJgOK6bzJDVI6lPV7qcRcSfwW9J4+9C6VGpmZqWUGWOfCSwEJklaAkwAXiweTcDdwBHALcB3gPGS1gKfIA3f/KnuVZuZWatq9tgjYgUwFlgKTCOF/FhgTVW7V4AvAoOLds8Dn42I1XWu2czM2lCmx05EzAL2bGGRqtrdCdxZh7rMzKyDfOWpmVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZaZUsEs6UNJcSSslPSppnzba7ipphaSQ9IX6lWpmZmXUDHZJDcAdwABgMjAEuF1SrxbaCrgKWFPnOs3MrKQyPfbDSGE+PSKmA9cAI4ExLbSdBOwA/LBeBZqZWfuUCfaRxfTlYjq/mI6qbCRpGHABKdzfbmuFkk6WNFvS7MbGxnaUa2ZmtXTk4KlamX8hMBt4BtimmLe9pP7VDSNiRkSMjojRgwcP7kAJZmbWmt4l2rxQTIcX02HFdF4x/r4mIlYBI4CDgOcqnvsDYDHw4zrUamZmJZQJ9pnAQmCSpCXABODF4tEE3A0cAZwHNHe/jwLGAt8DZtW1YjMza1PNYI+IFZLGAlcA04AngYlUnfkSEb9p/lrSHsWXD0fEX+pXrpmZ1VKmx05EzAL2bGFRi+PtETEVmNrhqszMrMN85amZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmSkV7JIOlDRX0kpJj0rap4U2ny2WLZH0uqRrJW1Z/5LNzKwtNYNdUgNwBzAAmAwMAW6X1Kuq6V7AU8AUYA4wHjizrtWamVlNvUu0OYwU5mdGxHRJ2wPnAGOA+yvafTci3gGQ9DAwF9i9vuWamVktZYZiRhbTl4vp/GI6qrJRc6gXDi2ms1paoaSTJc2WNLuxsbFsrWZmVkJHDp6qzYXSkcB3gP8E/r2lNhExIyJGR8TowYMHd6AEMzNrTZlgf6GYDi+mw4rpPEkNkvo0N5R0FHAL8ABwZESsqVulZmZWSplgnwksBCZJmgRMAF4sHk3AzwAkHQ7cBCwGbgY+L+nj9S/ZzMzaUjPYI2IFMBZYCkwjhfxYoLo3/mGgF7At8CNSuJ9bz2LNzKy2MmfFEBGzgD1bWKSKNlOBqXWpyszMOsxXnpqZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmSgW7pAMlzZW0UtKjkvZppd2XJc2X1CTpPyQNqm+5ZmZWS81gl9QA3AEMACYDQ4DbJfWqavch4ErgaeA84HDgknoXbGZmbSvTYz+MFObTI2I6cA0wEhhT1W5cMf1mRFwEPAQcU3wwmJlZF+ldos3IYvpyMZ1fTEcB99do1xsYATxXuUJJJwMnF98ulfRsO2rudFNgW+D1+q/5m/VfZQ/jbdt5vG07zya6bXdobUGZYK+mjW0XETOAGR147S4haXZEjO7uOnLkbdt5vG07T0/btmWGYl4opsOL6bBiOk9Sg6Q+bbRbDby00VWamVlpZYJ9JrAQmCRpEjABeLF4NAE/K9rdUEy/LelM4ADglohYUc+CzcysbTWDvQjmscBSYBop5McCa6razQG+AuwG/AvpA2FynevtKpvsMFEGvG07j7dt5+lR21YR0d01mJlZHfnKUzOzzDjYzcwys9kGu6T3SYricU7F/Gsq5o8pppd3Z609jaTLJL1WbLtfVsxv3q4rJS2Q9HNJe3dnrT2NpJ0kPSBpkaQlku6V9P6K92pIWi5pnqQfStq2u2vuSSQ9UmzX5ZJmS/pYT9y2m22wVxmnpD9wVHcXk4lbWpk/HzgJ+CnpthO/k7Rbl1XV8w0j/d2eB/wIOAS4umL5PcBXgcdJFwH+RtKWXV1kD/YQafv9K7A3PXTbOthhHukq2jGkUO/DuqtnrQMi4qu0fp+gtyLixqLNeUA/4BtdVlzP91BEHBQRlxfb8A1g94rlz0XE1RHxOeBe0llqx3RHoT3UFOAXpKvqVwJrK5b1mG3rYE83LXsEOLF4/BxY3K0VbT5mFtMec0Vfd4uId5q/ljQa2AaY1Upzb9/2Gwg0kjLhHdLeZUs26W3rYE+uJZ2bfyBp99a6RvNtJ3zObTtJ2gW4i3Sh4GmtNSum3r7lLQX+jjTk0kC6Jqclm/S2dbAnt5AuuJpP2sWyrnFoMZ3TrVX0MMUxiQdJPcqPR8SrrTT19m2niFgdEfdGxA+A/wYOJt0ArNomvW07chOw7ETE25JOBJZExFppg/uX7S/pwuLrpog4v2sr7FkkHQ7sUXw7QtJJwG+K7wdKOo60CzsJWAZcuOFarCWSRgAPAIOAs4H9JO0HLCia7CRpAnAEqef5FHBzd9Ta00g6lHSc7SHSXWkPAF4DFhVNesy23WyvPJX0PtKNy+6OiCOqlj1BOiB1MOmPqNJbEbF1V9TYU0l6EDioavZ41g1zrSL9sTwMTI2I/+m66no2SWPY8D0J679XVwCvAr8Czo6IRS20tyqSPkx6j76fdOD0MeBM0gH+HrVtN9tgNzPLlcfYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDP/C+Y1HhnWGyiOAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "RNNuYnCRnm37",
        "outputId": "72563ba0-083a-40f0-b9b8-6ef4c63ae771"
      },
      "source": [
        "#Plot comparing Specificity of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "spec_hala_pd_m = [la_la_pd_ml[0], la_la_1pd[0], la_la_2pd[0], la_la_3pd[0]]\n",
        "spec_hala_pd_s = [la_la_pd_ml[1], la_la_1pd[1], la_la_2pd[1], la_la_3pd[1]]\n",
        "\n",
        "spec_hala_nc_m = [la_la_nc_ml[0], la_la_1nc[0], la_la_2nc[0], la_la_3nc[0]]\n",
        "spec_hala_nc_s = [la_la_nc_ml[1], la_la_1nc[1], la_la_2nc[1], la_la_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "a3 = plt.bar(x_pos - barWidth/2, spec_hala_pd_m, yerr=spec_hala_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "a4 = plt.bar(x_pos + barWidth/2, spec_hala_nc_m, yerr=spec_hala_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,1)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((a3[0], a4[0]), ('PD', 'NC'))\n",
        "plt.title('Specificity in Arousal Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Specificity in Arousal Classification')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAaY0lEQVR4nO3de5xU5Z3n8c9XQNso6gpIwkUFNdEo0ShJJjGOmJhxvOQy68DoGFFE2DBmXMHLK0xixFknGlfjJcgaIkZNjCZRJmPisNE4Gjab0QSMazBKVERtRGgxKGCDXH77x3Nai6a7q7qp7up6+L5fr3qd6jpPn/rVqdPfeuo5l1ZEYGZm+dip1gWYmVl1OdjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYN8BSXpK0pjiviR9T9KfJf1W0jGSFlewjDMkPVCleip6znom6TZJV2zH74ekA6tZU8myt3ovJR0t6VlJayV9QdI8SWd1w/PeLOnSai/XQD6OvbYkfRK4GjgU2Aw8DVwQEb/roec/BrgL+EBErNuO5QRwUEQ8V7XiOvf8Y4CHga9ExDdrUUNHJN0GNEbE19qZ/z7gCuAkYHdgGfAj4OqIWNeT61fSQ8B9EXFDFZd5NnBuRHyyWsu09rnHXkOS9gB+Dnwb2BsYClwObOjBMvYDlm5PqPcSZwGvA+M7aiSpb8+UUzlJewP/CewKfDwi+gOfAfYCDqhBSfsBT9Xgea1aIsK3Gt2A0cDqDuafDfxfYCbwBvAM8OmS+XsCc4DlpB7eFUCfkvmTSN8A1gB/BI4sHl8KHA9MBNaTvimsJX2ojCH1LFuWMRyYCzQBq4CZJbX9urg/HwhgXbGcvwMWAZ8tWU4/4DXgw228ztbPuRS4CHiyeN0/Aho6WE+7Fa/xNOBtYHTJvP2L2iYCLxW17gR8DXgRWAncAezZVi2l66u4/1FgAfAmsAL4Vkm7nwCvFjXPBw4tmXcbcEU79V8B/AHYqYPXGMCBxf2Tgd8XNbwMzChp1wD8oHivVgO/AwaXvGdLinX1AnBGG+/l88AWoLl4L3cBHiH1tsttV18pfr/l8b8pHj+Erbez1W2tk2K5z5E+oO8DhrR6/V8Cni1e100UIw6+bXtzj722/gRslnS7pBMl/Zc22nyM9McyELgMmFv08CD9YWwCDgQ+DPwVcC6ApLHADFIPdg/gc6Q/9ndExBzSH8t/RsTuEXFZ6XxJfUjfKF4kBeRQ4O7WBUbEXxZ3Dy+W8yNSWH6xpNlJwPKI+H3Hq+Qd44C/BkYAHyKFT3v+KykwfgL8gtR7b+1YUsCcUCzrbOA4YCRp6GNmhXXdANwQEXuQetM/Lpk3DzgI2Ad4HLizwmUeD8yNiC0Vtl9Hel/3IoX8FElfKOadRfrAHw4MIL2/zZJ2A24EToz0jeATwBOtFxwRB5A+AD9bvJdbfXsss109DxxTPP/lwA8kvS8inmbr7Wyv1s8r6VPAlaT3/X2kba71tnYK8BHS9jCO9F5aGxzsNRQRbwKfJPVGvgs0SbpP0uCSZiuB6yNiYxGYi4GTizYnkcbj10XESuA6Uq8VUsBfHRG/i+S5iHixkyV+FBgCXFw8x/qI+HWFv/sD4KRiuAngTOD7nXjuGyPilYh4HfgZcEQHbc8CfhQRm4EfAqdJ6teqzYziNTQDZ5B62ksiYi0wvfidSoZpNgIHShoYEWsj4tGWGRFxa0SsKcJwBnC4pD0rWOYA0reuikTEIxHxh4jYEhFPkvaRHFtS3wBS735zRCwstjNIPfHDJO0aEcsjoivDLe1uVxHxk+I921Jsq8+StqFKnAHcGhGPF+tvOvBxSfuXtLkqIlZHxEuk/SkdbRM7NAd7jUXE0xFxdkQMAw4jBen1JU2WRUTpHu4Xizb7kYY3lktaLWk18B1SbxFSj+357SxvOPBiRGzq7C9GxCukYaRTJe0FnEjlPVhIQxot3iL1qrchaTip592y7H8jDUec3KrpyyX3h5DWY4sXgb7AYMqbCLwfeEbS7ySdUtTRR9JVkp6X9CZp+AbSN61yVpF6qRWR9DFJD0tqkvQGqTfc8jzfJ31ruVvSK5KultQv0j6UvyvaLpd0v6SDK33OEu1uV5LGS3qiZHs8jMpeP7R6T4oP3FWkb4ktKtomzMHeq0TEM6ThlcNKHh4qSSU/7wu8QgqqDcDAiNiruO0REYcW7V5m+3e8vQzsux07HG8nDceMJX0NX7ad9bTlTNJ2/DNJr5LGkBvYdjim9MPxFdIHY4t9SUNaK0jDHO9pmVEMRw16ZyERz0bE6aQP0G8C9xTDHH8PfJ40rLInaegKoPS9a88vgb+RVOnf4w9JY9DDI2JP4OaW5ym+2V0eER8kDbecQrFDOSJ+ERGfIX2IPEP6lthZbW5XkvYrlvdlYEAx3LKId19/ucPvtnpPinU6gLTvyDrJwV5Dkg6WdKGkYcXPw4HTgUdLmu0DnC+pXzG+eQjw7xGxHHgAuFbSHpJ2knSApJav5LcAF0k6qjhW/cDij68zfksaIrhK0m6SGiQd3U7bFaTx6lI/BY4E/jtpzL07nEUazz2i5HYqaRhoQDu/cxcwVdIISbsD3yAN5Wwi7fdokHRyMZzzNdIORAAkfVHSoGI8fHXx8BagP+mDdhXpg+EbnXgN3yKNV9/e8h5JGirpW5I+1Eb7/sDrEbFe0kdJHyot9R0naVTxgfQmaWhmi6TBkj5fBOYG0j6JSsf0S7W3Xe1GCu+moo4JbN1BWQEMk7RzO8u9C5gg6QhJu5DW32MRsbQLNe7wHOy1tYa0c/QxSetIgb4IuLCkzWOkHXKvAf8C/G1EtOysGg/sTDoC4c/APRRf6SPiJ0X7HxbP81PSIZUVK8asP0vaOfsS0Ej6Ot+WGaRgWi1pXPH7zcC9pB2gczvz3JWQ9BekXt5NEfFqye0+0tEVp7fzq7eShizmk44OWQ/8Y1HzG8A/kAJsGakH31jyu38NPCVpLWlH6mnF67yDNJSwjPR+lH44d6jYj/AJUgg/JmkN8BDp6Jq2jlv/B+Cfi3ZfZ+sduO8lbQdvko5c+VXxWncCppF6xq+TxuSnVFpjSa1tblcR8UfgWtJhmyuAUaShuBb/QTqE8lVJr7Wx3F8Cl5K2l+WkbwWntW5nlfEJSr1YDid1SPo68P6I+GLZxmZWFb3uZA3LR3FY5kTSOLiZ9ZCyQzGSbpS0QulaFT/voN3Rkp6UtEHS45KOrG6pVk8kTSLtaJsXEfNrXY/ZjqTsUIykG0k7Rc4H7o+IU9po00A6vKsZ+J/AV0k7aA4qxmnNzKyHlO2xR8T5pBNfOnIi6RjgWRExi3Sa+wjS6dlmZtaDqjXGPqKYthxz2nIUwUjS3v2tSJoMTAbYbbfdjjr44K6cJ2GWt8WL05WMP/CBD9S4EuuNFi5c+FpEDGprXnftPO3wpIyImA3MBhg9enQsWLCgm8owq19jxowB4JFHHqlpHdY7SWr3EiFdPo5d0i7FiQSQjgUGGFZMW04DXtLV5ZuZWdeU7bFLOpl3zyAbLulc0kkPD5KuA7E76ap2K0lXmVtDOsRtKelyn2Zm1oMq6bFfDFxV3P8Q6XoQW51WHhHrSdcDaTkbbyUw1kfEmJn1vLI99ogY086s21q1m086jXi7bdy4kcbGRtavX1+NxdVEQ0MDw4YNo1+/1lePNTPrXr3yzNPGxkb69+/P/vvvz9YXNqwPEcGqVatobGxkxIgR5X/BzKyKeuVFwNavX8+AAQPqMtQBJDFgwIC6/sZhlrMxY8a8c9RRPSy3s3plsAN1G+ot6r1+M6tfvTbYzcysa3rlGHtr1z34p6oub+pn3l+2TZ8+fRg1ahSbNm3ikEMO4fbbb+c973nPO49v3LiRvn37Mn78eKZOncpOO/kz0sx6B6dRO3bddVeeeOIJFi1axM4778zNN9+81eNPPfUUDz74IPPmzePyyy+vcbVmZu9ysFfgmGOO4bnntv1HNvvssw+zZ89m5syZ+B+WmFlv4WAvY9OmTcybN49Ro9o+RH/kyJFs3ryZlStX9nBlZmZtq4sx9lpobm7miCOOAFKPfeLEiTWuyMysMg72drSMpZezZMkS+vTpwz777NMDVZmZleehmO3Q1NTEl770Jb785S/7uHUz6zXqosdeyeGJPaVliKblcMczzzyTadOm1bosM7N31EWw18LatWvbfHzzZl+w0sx6Nw/FmJllxj1265Vy/rdwlZ5J3fjn5k61701DlrXidZu4x25mlhkHu5lZZhzsZmaZcbCbmWWmPnaePnxldZd33PSyTSQxbdo0rr32WgCuueYa1q5dy4wZMwC44447uPrqq5FE3759OeOMM7jooouqW6eZWRfUR7DXwC677MLcuXOZPn06AwcO3GrevHnzuP7663nggQcYMmQIGzZs4I477qhRpWadk/MRR92m0s7l6pc6176CTmZXeCimHX379mXy5Mlcd91128y78sorueaaaxgyZAiQPgQmTZrU0yWambXJwd6B8847jzvvvJM33nhjq8cXLVrEUUcdVaOqzMw65mDvwB577MH48eO58cYba12KmVnFHOxlXHDBBcyZM4d169a989ihhx7KwoULa1iVmVn7HOxl7L333owbN445c+a889j06dO5+OKLefXVVwF4++23ueWWW2pVopnZVurjqJhu2nNcqQsvvJCZM2e+8/NJJ53EihUrOP7444kIJHHOOefUsEIzs3fVR7DXQOllewcPHsxbb7211fwJEyYwYcKEni7LzKrgvGu+X+sSupWHYszMMuNgNzPLTK8N9oiodQnbpd7rN7P61SuDvaGhgVWrVvX6cFy8eDGLFy/e5vGIYNWqVTQ0NNSgKjPb0fXKnafDhg2jsbGRpqamWpfSoZbDHbds2bLNvIaGBoYNG9bTJZmZ9c5g79evHyNGjKja8rrrokdTpkzpluWamW2PXjkUY2ZmXVdRsEs6WtKTkjZIelzSkW202UXSLZKaJDVL+r2kT1W/ZDMz60jZYJfUANwL9AemAoOBeyT1adV0PDAReAK4FDgc+G5VqzUzs7Iq6bGfSArzWRExC5gDjADGtLOsRcAvgQ3A6uqUaWZmlapk52nLXsxlxbSxmI4EHippdztwAnBBcXsdOLutBUqaDEwG2HfffTtVsNmOIvfT3q37dGXnqdp5/C+Ak4E7gdOAPsBtkrZpHxGzI2J0RIweNGhQF0owM7P2VNJjf6GYthyUPbSYLinG3zdHxEZgLLAzcHNE/FrSJODTwECgdx+Qbl3m/5/Zi9TZ/+XM0SPX9Y5/kVlJsM8DVgJTJK0h7SBdWtyagfuBU4AlRftLJB0OfBxYBbxW3ZLNzKwjZYM9ItZLGgvcBNwAPAVMAja3anoTcAjwWeB44Gngoujt1wVoi3s+ZlbHKjrzNCLmA6PamKWSNuuBc6tUl5mZdZHPPDUzy4yD3cwsMw52M7PM9MqrO1bqugf/VFG7xj83d6r91LpeK2a2o3OP3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMz7+Yzv0lgv+dIduO+LoM+/vck1mVhn32M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLzA5xuKP/27vZu3I+TNcS99jNzDLjYDczy4yD3cwsMw52M7PMONjNzDKzQxwVY93HRxyZ9T7usZuZZcY9dutZD19ZWbvVL3Wu/XHTu1aPWYbcYzczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy0xFwS7paElPStog6XFJR7bT7jBJ/yGpWdIqSVdXt1wzMyun7CUFJDUA9wLNwFTgq8A9kg6KiM0l7XYF/jewK/B14C1gYHcUbWZm7avkWjEnAoOBSyJilqT3ApcCY4CHStqdDgwFJgF3RkRzlWs1M7MKVDIUM6KYLiumjcV0ZKt2Hyym04C3JDVJGtfWAiVNlrRA0oKmpqZOFWxmZh3rys5TtfP4LsV0OXAqsAG4TVL/1g0jYnZEjI6I0YMGDepCCWZm1p5Kgv2FYjqsmA4tpkskNUjq16rdjyNiLvB/SOPtQ6pSqZmZVaSSMfZ5wEpgiqQ1wERgaXFrBu4HTgHuBr4BTJC0Bfg0afjm+apXbWZm7SrbY4+I9cBYYC1wAynkxwKbW7V7Bfh7YFDR7jngcxGxqco1m5lZByr6D0oRMR8Y1cYstWo3F5hbhbrMzKyLfOapmVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZaaiYJd0tKQnJW2Q9LikIztoe4ik9ZJC0t9Wr1QzM6tE33INJDUA9wLNwFTgq8A9kg6KiM2t2gr4LrB5mwWZdcIj102qdQlmdauSHvuJwGBgVkTMAuYAI4AxbbSdAuwHfKdaBZqZWedUEuwjiumyYtpYTEeWNpI0FLiSFO5vdrRASZMlLZC0oKmpqRPlmplZOV3Zeap2Hr8KWAA8A+xdPPZeSbu3bhgRsyNidESMHjRoUBdKMDOz9pQdYwdeKKbDiunQYrqkGH/fHBEbgeHAscCzJb/7bWA18IMq1GpmZhWoJNjnASuBKZLWABOBpcWtGbgfOAW4DGjpfo8DxgLXAvOrWrGZmXWobLBHxHpJY4GbgBuAp4BJtDryJSJ+1XJf0mHF3Ucj4qXqlWtmZuVU0mMnIuYDo9qY1eZ4e0TMAGZ0uSozM+syn3lqZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYc7GZmmXGwm5llxsFuZpYZB7uZWWYqCnZJR0t6UtIGSY9LOrKNNp8r5q2R9JqkWyXtWv2SzcysI2WDXVIDcC/QH5gKDAbukdSnVdPDgT8C04CFwATgkqpWa2ZmZfWtoM2JpDC/JCJmSXovcCkwBniopN03I+JtAEmPAk8Ch1a3XDMzK6eSoZgRxXRZMW0spiNLG7WEeuGEYjq/rQVKmixpgaQFTU1NldZqZmYV6MrOU3U4UzoV+Abw78D/aqtNRMyOiNERMXrQoEFdKMHMzNpTSbC/UEyHFdOhxXSJpAZJ/VoaShoH3A08DJwaEZurVqmZmVWkkmCfB6wEpkiaAkwElha3ZuBfASSdDPwQWA3cBXxB0qeqX7KZmXWkbLBHxHpgLLAWuIEU8mOB1r3xjwB9gIHA90jh/vVqFmtmZuVVclQMETEfGNXGLJW0mQHMqEpVZmbWZT7z1MwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDJTUbBLOlrSk5I2SHpc0pHttPtvkholNUv6N0kDqluumZmVUzbYJTUA9wL9ganAYOAeSX1atfswcDPwNHAZcDJwXbULNjOzjlXSYz+RFOazImIWMAcYAYxp1e7sYvpPEXE18Bvg9OKDwczMekjfCtqMKKbLimljMR0JPFSmXV9gOPBs6QIlTQYmFz+ulbS4EzV3u2kwEHit+kv+p+ovss543XYfr9vu00vX7X7tzagk2FvT9raLiNnA7C48d4+QtCAiRte6jhx53XYfr9vuU2/rtpKhmBeK6bBiOrSYLpHUIKlfB+02AS9vd5VmZlaxSoJ9HrASmCJpCjARWFrcmoF/LdrdUUz/RdIlwCeAuyNifTULNjOzjpUN9iKYxwJrgRtIIT8W2Nyq3ULgPOCDwD+TPhCmVrnentJrh4ky4HXbfbxuu09drVtFRK1rMDOzKvKZp2ZmmXGwm5llZocNdkn7S4ridmnJ43NKHh9TTGfWstZ6I+lGSSuKdffzksdb1usGSa9K+qmkI2pZa72RdJCkhyWtkrRG0oOSDijZVkPSW5KWSPqOpIG1rrmeSHqsWK9vSVog6S/rcd3usMHeytlKdgfG1bqYTNzdzuONwLnAj0mXnfi1pA/2WFX1byjp7/Yy4HvA8cAtJfMfAM4H/kA6CfBXknbt6SLr2G9I6+9/AEdQp+vWwQ5LSGfRjiGFej/ePXvWuiAizqf96wS9ERHfL9pcBuwGfKXHiqt/v4mIYyNiZrEOXwcOLZn/bETcEhGfBx4kHaV2ei0KrVPTgJ+RzqrfAGwpmVc369bBni5a9hhwTnH7KbC6phXtOOYV07o5o6/WIuLtlvuSRgN7A/Pbae7123l7Ak2kTHib9O2yLb163TrYk1tJx+YfTfp6az2j5bITPua2kyQdDNxHOlHwH9trVky9fiu3Fvgr0pBLA+mcnLb06nXrYE/uJp1w1Uj6imU944RiurCmVdSZYp/EI6Qe5aciYnk7Tb1+OykiNkXEgxHxbeC3wHGkC4C11qvXbVcuApadiHhT0jnAmojYIm1z/bKPS7qquN8cEZf3bIX1RdLJwGHFj8MlnQv8qvh5T0lnkr7CTgHWAVdtuxRri6ThwMPAAOBrwMckfQx4tWhykKSJwCmknucfgbtqUWu9kXQCaT/bb0hXpf0EsAJYVTSpm3W7w555Kml/0oXL7o+IU1rNW0TaIXUc6Y+o1BsRsVdP1FivJD0CHNvq4Qm8O8y1kfTH8igwIyL+X89VV98kjWHbbRK23lbXA8uBXwBfi4hVbbS3ViR9hLSNHkDacfp74BLSDv66Wrc7bLCbmeXKY+xmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWmf8PPO4wdqniac4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "JyNzPpAjoKe0",
        "outputId": "58920e4a-d2bb-4052-e384-a58c52c0ca47"
      },
      "source": [
        "#Plot comparing FNR of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "fnr_hala_pd_m = [ha_la_pd_ml[0], ha_la_1pd[0], ha_la_2pd[0], ha_la_3pd[0]]\n",
        "fnr_hala_pd_s = [ha_la_pd_ml[1], ha_la_1pd[1], ha_la_2pd[1], ha_la_3pd[1]]\n",
        "\n",
        "fnr_hala_nc_m = [ha_la_nc_ml[0], ha_la_1nc[0], ha_la_2nc[0], ha_la_3nc[0]]\n",
        "fnr_hala_nc_s = [ha_la_nc_ml[1], ha_la_1nc[1], ha_la_2nc[1], ha_la_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "b1 = plt.bar(x_pos - barWidth/2, fnr_hala_pd_m, yerr=fnr_hala_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "b2 = plt.bar(x_pos + barWidth/2, fnr_hala_nc_m, yerr=fnr_hala_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,0.5)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(ticks=[0,0.1,0.2,0.3,0.4,0.5], weight='bold')\n",
        "plt.legend((b1[0], b2[0]), ('PD', 'NC'))\n",
        "plt.title('FNR in Valence Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'FNR in Valence Classification')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYeElEQVR4nO3dfZRV9X3v8fdHQMcoYEUkFYxCqvGJXq+SeFOvCVpTi9GYrhRuUosriBK52izBh1vTJmJtYmpxoYZQRUiqeTKpRq9Rx0oTxSYWc8FaH/ApQRJHRUYUZRSQh+/9Y//GHI9nZvYMZ2Y4v/m81jprz9n7d/b+ns3w2Xv/9sMoIjAzs3zs0t8FmJlZfTnYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42C3bpP0AUltkgb18XI/L+nnfbnMHSVptaQTe2nex0l6uuL9hyQ9ImmDpC9Kuk7Sl3thuV+StKje87X6cbBnIgXIxhS47a/9JB0oKSTdXdX+u5LmpJ8nStqePrNB0tOSpnW0rIj4bUTsGRHbulnjaElbJX2wxrTbJM3tzvx2BpKGSbpa0m/T+vt1er9Pby87Iv49Ij5UMepi4L6IGBoR10bEORFx+Y4sI/1utFQt92sRcdaOzNd6l4M9L6emwG1/vVgx7RhJf9TJZ1+MiD2BYcAs4AZJH+qkfbdFxAvAT4GpleMl7Q2cDNxYz+X1Nkm7Unyfw4E/pVh3HwXWAR/ph5IOAJ7oh+XaTsbBPnBcCXy1q0ZRuBt4FfjDWm0qjgIGp/f3S7pc0i/SHv+9neyx3khVsAOfBVZGxGOS/jrt9W6QtFLSn3VUq6RDJC2R9Go6yphSMe2fJX1T0l1pXg9VHilIOrzisy9L+lIav0tFDesk/ShteGo5A/gA8GcRsTIitkfE2oi4PK3D6no/Iuk/JK2X9JKk+WnjgArzJK2V9IakxyQdkaadnNbFBkkvSLowjX9nb1rSz4DjgfnpyOHgtA7+vmL5p6WumjfS9/vTNH6apCfT/FdJ+kIavwfQDOxXdRQ4R9J3K+b7KUlPpO91v6RDK6atlnShpEclvS7ph5KaOvo3tfpwsA8cC4CD1UV/bwq2TwH7AL/qxvz/ApgG7AvsClzYQbvbgH0k/c+KcVP53d76r4HjgOHAZcB3Jf1+jTr3AJYA30/L/CywQNJhFc0+m+bxe+m7fDV9dijwb8A9wH7AH1DseQP8FfBp4ONp2mvANzv4LicC90REWwfTq22jOBrah2LP/o+B/52m/QnwMeDg9N2nUOz5AywGvhARQ4EjgJ9VzzgiTgD+HTgvHa09Uzld0keAm4CLgL3SslanyWuBUyiOOKYB8yQdFRFvApNIR3M1jgKRdDDwA+B8YCRwN/CT9g1WMoXiiGYsxc7C50usK9sBDva83J72mtZLur1q2kaKYPv7Gp+DYq9sfWp3GzA7Iv6zG8v+dkQ8ExEbgR8BR9ZqlKb/C8XeLpIOAo6mCGgi4l8i4sW09/tD4Flqd2ucAqyOiG9HxNZU663A5Io2t0XELyNiK/C9ippOAdZExFURsSkiNkTEQ2naOcDfRERLRGwG5gB/3n50UmUE8FKZlZO+24qIWJbqXQ1cT7EBAdgCDAUOARQRT0bESxXTDpM0LCJei4iHyy6zwnTgWxGxJK3bFyLiqVTXXRHx63S0thS4l2LjWsb/Au5K890CzAV2Byq7/a5N/6avAj+hg98Nqx8He14+HRF7pdena0xfBIySdGqNaS9GxF4Ue23XAid0c9lrKn5+C9izk7Y3ApPTIflU4F8jYi2ApDNSd8H6tKE5gmIPt9oBFOcN1le0PR14f4ma9qc4MqjlAOC2ink+SbGnPapG23XAe44mOpK6R+6UtEbSG8DX2r9bRPwMmE9xdLBW0kJJw9JHP0NxDuI3kpZK+mjZZVbo8DtLmiRpWeqWWp+WVfbk737Ab9rfRMR24HlgdEWb7vxuWB042AeQiHibomvickAdtNkM/B9gvKRaG4d6+DlFH/5pwF+SumEkHQDcAJwHjEgbmsc7qPV5YGnFhmyv1FUws8TynwfGdTJtUtV8m9KJ32r/BpyUuoXK+CfgKeCgiBgGfImK75auZDkaOIyiS+aiNP7/RcRpFF1Ot1McEXXX80Ctq5F2ozjSmQuMSuv87oq6unr864sUG8P2+YliI1JrfVkfcbAPPN8Bmij6PGtKG4CrgK/0RgFRPCv6JuAfKPp7f5Im7UERJK1QnNSj2GOv5U6KcwZTJQ1Jrw9XnrjrxJ3A70s6X9JukoZKOiZNuw74atrIIGmkpNM6mM93KALzVhUncneRNELFdd4n12g/FHgDaJN0CPDORijVfoykIcCbwCZgu6RdJZ0uaXjq6ngD2F7iO1ZbDEyT9MepztGphl2B3SjW+VZJkyj6+9u9DIyQNLyD+f4I+GSa7xDgAmAz8GAParQ6cbAPMOna868AHV3p0e5bwAc66Laph5sorij5YTpKICJWUmxQ/oMiUMYDv6j14YjYQBFAn6XYa1xDsaHYrasFp89+Ajg1fe5ZiitKAK4B7gDulbQBWAYc08F8NlOcQH2K4kTuG8AvKboxHqrxkQspTjJvoDgy+WHFtGFp3GsUXRvrgH9M06YCq1P3zTkUXU7dEhG/JJ0YBV4HlgIHpHXxRYqAfi3Vd0fF556iODm6KnVP7Vc136cpjrq+AbxCsU5PTTsH1k/kP7RhZpYX77GbmWWmVLBLOjbdYLBZ0sOSjqrRpv2mlcrX1fUv2czMOtNlV0y6JG01xfXN/wj8DcXJkYMqnxUi6UDgOYqTT0vT6Ke7eS20mZntoFo3XVSbRHEN78URsUDS+4EvAxP53d16lZYDd0TEW3Wr0szMSisT7GPTsP261PYnvY2jdrDfACyStBKYHhHLqhtImgHMANhjjz2OPuSQQ7pVtJnZQLdixYpXImJkrWllgr1azRtbKK69vRR4BDgI+DrFbdzvuSkiIhYCCwEmTJgQy5cv70EZZmYDl6TfdDStTLA/l4Zj0rD9VuFVqf99W0RsiYhW4O8qFvoXwFGSmiJiUw/qNjOzHigT7M0UT3+bmW7YmE5xMnU1xQnVu4BTJJ0NfJjixoxxFA/6+S+HuplZ3+rycscUzJOBNoq78tam99V/PecZikdyXk3xKNJ7ePeT9szMrA+U6mOPiAcobu+uVvkAo6XA/6hTXWZmO2zLli20tLSwaVPjdhw0NTUxZswYhgwZUvozPTl5ambWEFpaWhg6dCgHHnggxYMnG0tEsG7dOlpaWhg7dmzXH0j8SAEzy9amTZsYMWJEQ4Y6gCRGjBjR7SMOB7uZZa1RQ71dT+p3sJuZZcZ97GY2YMxb8kzXjbph1icO7nT6oEGDGD9+PFu3buXQQw/lxhtv5H3ve98747ds2cLgwYM544wzmDVrFrvsUp99be+xm5n1kt13351HHnmExx9/nF133ZXrrrvuXeOfeOIJlixZQnNzM5dddlndlutgNzPrA8cddxy/+tWv3jN+3333ZeHChcyfP596/eEjB7uZWS/bunUrzc3NjB9f63YgGDduHNu2bWPt2rV1WZ772M3MesnGjRs58sgjgWKPffr06X2yXAe7mVkvae9L78qqVasYNGgQ++67b12W664YM7N+1NrayjnnnMN5551Xt2vuvcduZgNGV5cn9pX2Lpr2yx2nTp3K7Nmz6zZ/B7uZWS9pa2urOX7btuqH49aXu2LMzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4wvdzSzgeO+K+o7v+Mv6XSyJGbPns1VV10FwNy5c2lra2POnDkA3HTTTVx55ZVIYvDgwZx++ulceOGFO1yW99jNzHrJbrvtxo9//GNeeeWV90xrbm7m6quv5t577+Wxxx5j2bJlDB8+vC7LdbCbmfWSwYMHM2PGDObNm/eeaVdccQVz585lv/32A4qNwNlnn12X5TrYzcx60bnnnsv3vvc9Xn/99XeNf/zxxzn66KN7ZZkOdjOzXjRs2DDOOOMMrr322j5bpoPdzKyXnX/++SxevJg333zznXGHH344K1as6JXlOdjNzHrZ3nvvzZQpU1i8ePE74y655BIuuugi1qxZA8Dbb7/NokWL6rI8X+5oZgNHF5cn9qYLLriA+fPnv/P+5JNP5uWXX+bEE08kIpDEmWeeWZdlOdjNzHpJ5WN7R40axVtvvfWu6dOmTWPatGl1X667YszMMuNgNzPLjIPdzLIWEf1dwg7pSf0OdjPLVlNTE+vWrWvYcI8I1q1bR1NTU7c+55OnZpatMWPG0NLSQmtra3+X0mNNTU2MGTOmW59xsJtZtoYMGcLYsWP7u4w+564YM7PMlAp2ScdKelTSZkkPSzqqk7aHStokKST9ef1KNTOzMroMdklNwK3AUGAWMAq4RdKgGm0F3ABsq3OdZmZWUpk99kkUYb4gIhYAi4GxwMQabWcCBwDX16tAMzPrnjLB3n7m4YU0bEnDcZWNJI0GrqAI9zc6m6GkGZKWS1reyGerzcx2Rj05eaoOxn8dWA48Beydxr1f0p7VDSNiYURMiIgJI0eO7EEJZmbWkTKXOz6Xhu0XUo5Ow1Wp/31bRGwB9gc+Djxb8dlvAOuB79ahVjMzK6FMsDcDa4GZkjYA04HV6bURuAs4BbgUaN/9ngJMBq4CHqhrxWZm1qkugz0iNkmaDHwTuAZ4AjibqitfImJp+8+Sjkg/LouI39avXDMz60qpO08j4gFgfI1JNfvbI2IOMKfHVZmZWY/5zlMzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMlMq2CUdK+lRSZslPSzpqBptjpD0pKRNktZLulvS6PqXbGZmneky2CU1AbcCQ4FZwCjgFkmDqppuB24GZgA/BiYBl9a1WjMz61KZPfZJFGG+ICIWAIuBscDEykYRsRK4ArgHeDCN3l63Ss3MrJQywT42DV9Iw5Y0HFej7cnAy8ANwBN0sMcuaYak5ZKWt7a2dqNcMzPrSk9OnqqTab+g2MO/Bjgc+EKtRhGxMCImRMSEkSNH9qAEMzPrSJlgfy4Nx6Rh+wnRVZKaJA1pbxgRrRFxD3ABRTfMlLpVamZmpQwu0aYZWAvMlLQBmA6sTq+NwF3AKZIuAYYDTwEnUGw0Vta/ZDMz60yXe+wRsQmYDLRRdLGsTe+3VTVtBT4HXA+cBPwAOK+exZqZWdfK7LETEQ8A42tMUkWbRcCiOtVlZmY95DtPzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDOlgl3SsZIelbRZ0sOSjqrR5lNp2gZJr0j6lqTd61+ymZl1pstgl9QE3AoMBWYBo4BbJA2qavrfgJXAbGAFMA24uK7VmplZlwaXaDOJIswvjogFkt4PfBmYCPy0ot0/RMTbAJKWAY8Ch9e3XDMz60qZrpixafhCGrak4bjKRu2hnpyUhg/UmqGkGZKWS1re2tpatlYzMyuhJydP1elE6TPA14C7gX+q1SYiFkbEhIiYMHLkyB6UYGZmHSkT7M+l4Zg0HJ2GqyQ1SRrS3lDSFOBm4D7gMxGxrW6VmplZKWWCvRlYC8yUNBOYDqxOr43AbQCSPgl8H1gP/AD4tKQT6l+ymZl1pstgj4hNwGSgDbiGIuQnA9V74x8GBgH7AN+mCPev1LNYMzPrWpmrYoiIB4DxNSapos0cYE5dqjIzsx7znadmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZaZUsEs6VtKjkjZLeljSUTXa7C7pp5LaJIWkC+tfrpmZdaXLYJfUBNwKDAVmAaOAWyQNqmo6CHgVuKfeRZqZWXll9tgnUYT5gohYACwGxgITKxtFRFtETAburHeRZmZWXplgH5uGL6RhSxqO6+lCJc2QtFzS8tbW1p7OxszMaujJyVPt6EIjYmFETIiICSNHjtzR2ZmZWYUywf5cGo5Jw9FpuEpSk6Qh9S/LzMx6qkywNwNrgZmSZgLTgdXptRG4rb2hpLOAj6W3H5F0lqQ961mwmZl1rstgj4hNwGSgDbiGIuQnA9tqNL8BmJZ+npze71OXSs3MrJTBZRpFxAPA+BqTVNVuh/vfzcxsx/jOUzOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdtshEydOZOLEif1dhnWD/83y52A3M8uMg93MLDMOdjOzzDjYbafkfmCznnOwm5llxsFuZpYZB7uZWWZK/THrRtfeV3v//ff3ax2NZN6SZ0q1a3ltY7faz/rEwT2uyaxecs8E77GbmWXGwW5mlhkHu5lZZhzsZmaZaeiTp/19gi/3EzC94r4ryrVb/9vutT/+kp7VkxOv2363s2SC99jNzDLjYDczy4yD3cwsMw3dx17WuXO/070PuK+ytG6vWyttZ+mvbSS9dt5t8K3lCthJMsF77GZmmXGwm5llxsFuZpaZAdHH3lvun3d2f5eQrZzXbe/1A/e4JKuTneX31nvsZmaZKRXsko6V9KikzZIelnRUB+2+IKlF0kZJ/1fSiPqWa2ZmXeky2CU1AbcCQ4FZwCjgFkmDqtr9d+A64EngUuCTwLx6F2xmZp0rs8c+iSLMF0TEAmAxMBaYWNXu82n4pYi4EngQ+FzaMJiZWR9RRHTeQJoNXAWcHhHflzQDuB6YERE3VLS7AzgVGB0RL0r6PvA54OCIeLZqnjOAGenth4Cn6/WF6mQf4JX+LiJTXre9x+u29+yM6/aAiBhZa0JPzqNrR9tFxEJgYQ+W3SckLY+ICf1dR468bnuP123vabR1W6Yr5rk0HJOGo9NwlaQmSUM6abcVeH6HqzQzs9LKBHszsBaYKWkmMB1YnV4bgdtSu5vS8KuSLgb+CLg5IjbVs2AzM+tcl8Gegnky0AZcQxHyk4FtVe1WAOcChwF/R7FBmFXnevvKTttNlAGv297jddt7Gmrddnny1MzMGovvPDUzy4yD3cwsMwM22CUdKCnS68sV4xdXjJ+YhvP7s9ZGI+laSS+ndXdnxfj29bpZ0hpJt0s6sj9rbTSSDpJ0n6R1kjZIWiLpgxW/qyHpLUmrJF0vaZ/+rrmRSHoorde3JC2X9LFGXLcDNtirfF6FPYEp/V1MJm7uYHwLcBbwI4rHTvxc0mF9VlXjG03x//ZS4NvAicCiiun3Al8EHqO4CXCppN37usgG9iDF+rscOJIGXbcOdlgFjKN4RMIUYAjwQn8W1Ogi4ot0/Jyg1yPiO6nNpcAewF/3WXGN78GI+HhEzE/r8FXg8Irpz0bEoog4DVhCcZXa5/qj0AY1G/gJ8FNgM7C9YlrDrFsHe/HQsoeAM9PrdmB9v1Y0cDSnYcPc0dffIuLt9p8lTQD2Bh7ooLnXb/cNB1opMuFtiqPLWnbqdetgL3yL4tr8YykOb61vtD92wtfcdpOkQ4A7KG4U/KuOmqWh1295bcCfUHS5NFHck1PLTr1uHeyFmyluuGqhOMSyvnFSGq7o1yoaTDoncT/FHuUJEfFSB029frspIrZGxJKI+AbwS+B4igeAVdup163/mBYQEW9IOhPYEBHbpfc8v+yjkr6eft4YEZf1bYWNRdIngSPS2/0lnQUsTe+HS5pKcQg7E3gT+Pp752K1SNofuA8YAfwtcIykY4A1qclBkqYDp1Dsea4EftAftTYaSSdRnGd7ENif4rEoLwPrUpOGWbcD9s5TSQdSPLjsrog4pWra4xQnpI6n+E9U6fWI2KsvamxUku4HPl41ehq/6+baQvGfZRkwJyL+q++qa2ySJvLe30l49+/qJuAl4F+Bv42IdTXaWxVJH6b4Hf0gxYnT/wQupjjB31DrdsAGu5lZrtzHbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZpn5/wKPCE4gJw4vAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "fDRplkF9orGX",
        "outputId": "d088a55a-d8c1-44ab-eb11-7a87263256aa"
      },
      "source": [
        "#Plot comparing FPR of NC and PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "fpr_hala_pd_m = [la_ha_pd_ml[0], la_ha_1pd[0], la_ha_2pd[0], la_ha_3pd[0]]\n",
        "fpr_hala_pd_s = [la_ha_pd_ml[1], la_ha_1pd[1], la_ha_2pd[1], la_ha_3pd[1]]\n",
        "\n",
        "fpr_hala_nc_m = [la_ha_nc_ml[0], la_ha_1nc[0], la_ha_2nc[0], la_ha_3nc[0]]\n",
        "fpr_hala_nc_s = [la_ha_nc_ml[1], la_ha_1nc[1], la_ha_2nc[1], la_ha_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "b3 = plt.bar(x_pos - barWidth/2, fpr_hala_pd_m, yerr=fpr_hala_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "b4 = plt.bar(x_pos + barWidth/2, fpr_hala_nc_m, yerr=fpr_hala_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,0.5)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(ticks=[0,0.1,0.2,0.3,0.4,0.5], weight='bold')\n",
        "plt.legend((b1[0], b2[0]), ('PD', 'NC'))\n",
        "plt.title('FPR in Valence Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'FPR in Valence Classification')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYyklEQVR4nO3df5RU5Z3n8fdHQBsVNSCQII5C4q8gE2NIzIxr0mZNsqjJZE8GEscjR0SIjBkjaDwxO4mYmQTXxUUNYRQhCbpGk1XjGg2OrAOyGYNZMEbF+GuBxDYiDYry+5ff/eM+Zcqmuru6qeq2Hz6vc+rc6nufuvdbl+5PPfXcHygiMDOzfOzX3QWYmVltOdjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYLcOk7RCUmMXb/NoSSGpd1dud29I+rGkf67j+jdJGp6e95X0C0lvSPqfks6V9FAdtnmapOdqvV6rLQd7RiStlrQ1/cGXHkPKQrE0b7Wkb5S9LiRtTstelvTfJfVqbTsRMSIiFneivmclXVBh/tckLevo+rqbCpdIejrtv6YUqiO7YvsRcXBErEw//i0wGBgQEWMi4vaI+MzebiP9bnygbJv/JyKO29v1Wn052PPzufQHX3r8qWzZYRFxMHAO8G1J/6ls2YfSsk8CXwL2COAamA+MqzD/vLSsp7kB+BpwCdAfOBa4FzirG2o5Cng+InZ1w7btXcbBvg+KiF8DK4ATKyx7Efh34KTWXp96/Gek59Mk/UzSrZI2pmGaUa289DbgP0g6qmxdHwT+ErhD0lmSfivpTUkvSZrWRg2HSpon6ZX0LeOfS98yJJ0v6VeSZkh6XdIqSaPLXttf0o8k/Sktv7ds2dmSnpC0QdKjkv6yle0fA1wMnBMR/xYR2yNiS+opX1Oh/Xsk3S+pOW3zfklDy5afL2ll2oerJJ2b5n9A0iNpiGWdpJ+WvSbS8quBbwNfSt+6JpT2QVnbEZIWSnpN0quSvpnmf0zSr9P7fUXSLEn7p2VL0st/l9b7JUmNkprK1nuCpMXp9Sskfb5s2Y8l/UDSA+l9PSbp/a39m1rtONj3MWn44FRgBPDbCsuPB04DXuzAaj8P3AkcBtwHzKrUKCKagEUUPfSS84BfRsQ6YDNFj/4wil7vZElfaGWbPwZ2AR8APgx8BriwbPkpwHPA4cC1wDxJSstuAw6k2AeDgJkAkj4M/BD4CjAAuBm4T9IBFbb/H4GmiPhNK/W1tB/wI4qe9V8AW0n7SdJBwI3A6IjoB/w18ER63T8BDwHvAYYC32+54oi4Cvge8NP0LW1e+XJJ/YD/DTwIDKHYZw+nxbuBKRT76a/S+/r7tN5PpDYfSuv9aYv19gF+keobBPwDcLuk8qGaLwNXp/pfBL7b/q6yveVgz8+9qfe0obwnmqwDXgPmAt+IiIfLlj0uaTPwe2AxMLsD2/xVRPwyInZThOaH2mg7nxTskvYDzk3ziIjFEfFURLwVEU8Cd1AMDb2DpMHAmcClEbE5ItZShPOXy5r9ISJuSTXNB94HDJb0PmA0cFFEvB4ROyPikfSaScDNEfFYROyOiPnAduDjFd7HAOCVanZOem/rI+Lu1KvfSBFw5e/tLeBESX0j4pWIWJHm76T4MBgSEdsi4ld03NnAmoi4Lq1jY0Q8lupaHhFLI2JXRKym+DDbY5+34uPAwcA1EbEjIv4NuJ9iqK/k5xHxmzREdDttfBO02nGw5+cLEXFYerTs7R4eEe+JiBMi4sYWy06m+CP9EkVv96AObHNN2fMtQINaP3vlHuB9kj4ONFL0nB8AkHSKpEVpuOIN4CKKnmRLRwF9gFdKH2IUgTSoUk0RsSU9PRg4EngtIl5vZb2XlX0wbkjth1Rou57iw6Iqkg6UdLOkP0h6E1gCHCapV0RsptjvF6X39ED65gRwBSDgN2moozPHPo4E/l8rdR2bhoXWpLq+R+V9XskQ4KWIeKts3h+AI8p+bvm7cXD1ZVtnOdjtbVH4GfBrijHbemxjC3AXxZDLecCdEbEjLf4JxVDOkRFxKHATRai19BJFT/rwsg+xQyJiRBUlvAT0l3RYK8u+W7bOwyLiwIi4o0Lbh4Ghav14QkuXAccBp0TEIUBpmEMAEfGvEfFpig+LZ4Fb0vw1ETExIoZQDBHNVtlZKlV6CRjeyrJ/Sds7JtX1TSrv80r+BByZvnmV/AXwcgfrsxpzsFsl1wATJb23TuufT9FD/SLvPBumH0VvepukjwF/V+nFEfEKxbjudZIOkbSfpPdLancIIb12AUVAvkdSH0mlkL0FuCh9c5Ckg1Qc0O1XYT0vUAxX3ZEOKO4vqUHSl1V2KmmL97YV2CCpP3BVaYGkwZL+Jo21bwc2UQzNIGlM2UHW14EoLeuA+ym+JV0q6QBJ/SSdUlbXm8Cm9C1hcovXvkrrHwqPUfTCr0j7sRH4HMXxFutGDnbbQ0Q8RTFU8PU6bWIJ8AbFwcf/Wzb/74HvSNpI8Y3hZ22sYxywP/AMReDdRfVDI+dRjF0/C6wFLgWIiGXARIqDmq9THOw7v431XJLa/gDYQDHc8Z8pDii2dD3Ql+I4x1KKA5kl+wFTKXrAr1GMcZcC9qPAY5I2UXyb+VrZuetVSWP6n6YI3TXAC8DpafHlFB+gGyk+2H7a4uXTgPlpaGpsi/XuSOscnd7XbGBcRDzbkfqs9uT/aMPMLC/usZuZZaaqYJd0qqQnJW2X9Likkyu0KV22Xv64vvYlm5lZW9odipHUAKymOPDz34D/QnGA55h0jnCp3dHAKoozGUrnBT8XEXtcBGNmZvVTzZ3yRlPcXOiKiJidzpT4FsU5yA9XaL8MuK/s3GEzM+tC1QT7sDQtnZtauk/EcCoH+y3AXEnPABMiYmnLBpImUVzlx0EHHfSR448/vmUT28c991xxZ9jjjvONBM0qWb58+bqIGFhpWWfubd3axQubKc7NfQI4huJc6NuBPW76ExFzgDkAo0aNimXLetwdW63OGhsbAVi8eHG31mH2biXpD60tqybYV6Vp6SKJ0uXCK9P4++50v41m4DtlG/074GRJDRGxrRN1m5lZJ1QT7AsoLuKYnC4cmUBxMHU1xQHVB4CzJU0kXUxBMUxzEvA7h7qZWddq93THFMxjKC5zvoEi5MdQ3O6z3PMU99W+nuIKwgdTOzMz60JVjbFHxBKg0n/3pbI2j1D59qZmZt1i586dNDU1sW1bzx04aGhoYOjQofTp06fq1/SY/xjYzKyjmpqa6NevH0cffTR//n9Weo6IYP369TQ1NTFs2LD2X5D4lgJmlq1t27YxYMCAHhnqAJIYMGBAh79xONjNLGs9NdRLOlO/g93MLDMeYzezfcbMhc/XdH1TPn1sm8t79erFyJEj2bVrFyeccALz58/nwAMPfHv+zp076d27N+PGjWPKlCnst19t+trusZuZ1Unfvn154oknePrpp9l///256aab3jF/xYoVLFy4kAULFnD11VfXbLsOdjOzLnDaaafx4osv7jF/0KBBzJkzh1mzZlGr//jIwW5mVme7du1iwYIFjBxZ6XIgGD58OLt372bt2rU12Z7H2M3M6mTr1q2cdNJJQNFjnzBhQpds18FuZlYnpbH09qxcuZJevXoxaNCgmmzXQzFmZt2oubmZiy66iK9+9as1O+fePXYz22e0d3piVykN0ZROdzzvvPOYOnVqzdbvYDczq5NNmzZVnL97d8ub49aWh2LMzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4xPdzSzfcei6bVd3+lXtrlYElOnTuW6664DYMaMGWzatIlp06YBcOutt3Lttdciid69e3Puuedy+eWX73VZ7rGbmdXJAQccwD333MO6dev2WLZgwQKuv/56HnroIZ566imWLl3KoYceWpPtOtjNzOqkd+/eTJo0iZkzZ+6xbPr06cyYMYMhQ4YAxYfAxIkTa7JdB7uZWR1dfPHF3H777bzxxhvvmP/000/zkY98pC7bdLCbmdXRIYccwrhx47jxxhu7bJsOdjOzOrv00kuZN28emzdvfnveiBEjWL58eV2252A3M6uz/v37M3bsWObNm/f2vCuvvJKvf/3rrFmzBoAdO3Ywd+7cmmzPpzua2b6jndMT6+myyy5j1qxZb/985pln8uqrr3LGGWcQEUjiggsuqMm2HOxmZnVSftvewYMHs2XLlncsHz9+POPHj6/5dj0UY2aWGQe7mVlmHOxmlrWI6O4S9kpn6newm1m2GhoaWL9+fY8N94hg/fr1NDQ0dOh1PnhqZtkaOnQoTU1NNDc3d3cpndbQ0MDQoUM79BoHu5llq0+fPgwbNqy7y+hyHooxM8tMVcEu6VRJT0raLulxSSe30fYESdskhaS/rV2pZmZWjXaDXVIDcDfQD5gCDAbuktSrQlsBtwC7a1ynmZlVqZoe+2iKMJ8dEbOBecAwoLFC28nAUcDNtSrQzMw6pppgLx15eDlNm9J0eHkjSUcA0ynC/c22VihpkqRlkpb15KPVZmbvRp05eKpW5l8DLAOeBfqnee+VdHDLhhExJyJGRcSogQMHdqIEMzNrTTWnO65K09KJlEek6co0/r47InYCRwKfBF4oe+33gQ3A/6hBrWZmVoVqgn0BsBaYLGkjMAFYnR5bgQeAs4GrgFL3eywwBrgOWFLTis3MrE3tBntEbJM0BvgBcAOwAphIizNfIuKR0nNJJ6anSyPij7Ur18zM2lPVlacRsQQYWWFRxfH2iJgGTOt0VWZm1mm+8tTMLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy42A3M8tMVf+ZtVnNLJpeXbsNf+xY+9Ov7Fw9Zhlyj93MLDMOdjOzzDjYzcwy42C3vdLY2EhjY2N3l2FmZRzsZmaZcbCbmWXGwW5mlhkHu5lZZhzsZmaZcbCbmWXGwW5mlhkHu5lZZhzsZmaZqSrYJZ0q6UlJ2yU9LunkCm1OlPR7SdskbZD0S0lH1L5kMzNrS7vBLqkBuBvoB0wBBgN3SerVoulbwJ3AJOAeYDRwVU2rNTOzdlXTYx9NEeazI2I2MA8YBjSWN4qIZ4DpwIPAo2n2WzWr1MzMqlJNsA9L05fTtClNh1doeybwKnALsIJWeuySJklaJmlZc3NzB8rtHN+oyuzP/PeQv84cPFUby/6dood/AzAC+EqlRhExJyJGRcSogQMHdqIEMzNrTTXBvipNh6Zp6YDoSkkNkvqUGkZEc0Q8CFxGMQwztmaVmplZVar5P08XAGuByZI2AhOA1emxFXgAOFvSlcChwLPApyg+NJ6pfclmZtaWdnvsEbENGANsohhiWZt+3t2iaTNwDnAz8FngDuCrtSzWzMzaV02PnYhYAoyssEhlbeYCc2tUl5mZdZKvPDUzy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMVHUeu1VWupHS4sWLu7WOepi58Pmq2jW9vrVD7af4N86s7txjNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMuNgNzPLjIPdzCwzDnYzs8w42M3MMtOjb8lUtxtVffrYTtdkZtbd3GM3M8uMg93MLDMOdjOzzDjYzcwy42A3M8uMg93MLDMOdjOzzDjYzcwy06MvUKqbRdOra7fhjx1rf/qVnavHzKwD3GM3M8uMg93MLDMOdjOzzHiM3fbKxTNu6+4SrMTHhrpdY2MjAIsXL+7WOtxjNzPLTFXBLulUSU9K2i7pcUknV2jz+bRso6R1kn4oqW/tSzYzs7a0G+ySGoC7gX7AFGAwcJekXi2afgh4BpgKLAfGA1fUtFozM2tXNWPsoynC/IqImC3pvcC3gEbg4bJ2/zUidgBIWgo8CYyobblmZtaeaoZihqXpy2nalKbDyxuVQj35bJouqbRCSZMkLZO0rLm5udpazfYpjY2Nbx+MM+uIzhw8VZsLpS8C3wN+CfxLpTYRMSciRkXEqIEDB3aiBDMza001wb4qTYem6RFpulJSg6Q+pYaSxgJ3AouAL0bE7ppVamZmVakm2BcAa4HJkiYDE4DV6bEV+DmApLOAnwAbgDuAL0j6VO1LNjOztrQb7BGxDRgDbAJuoAj5MUDL3vhHgV7A4cCPKML927Us1szM2lfVlacRsQQYWWGRytpMA6bVpCozM+s0X3lqZpYZB7uZWWYc7GZmmdkn7u7oOxCa2b7EPXYzs8w42M3MMuNgNzPLzD4xxl4vi2dO7O4SzMz24B67mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7me1zGhsbaWxs7O4y6sbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZcbBbmaWGQe7mVlmHOxmZplxsJuZZaZ3dxdgZl1r8cyJ3V1Cz7NoenXtNvyxY+1Pv7Jz9bTDPXYzs8w42M3MMuNgNzPLjIPdzCwzVQW7pFMlPSlpu6THJZ1coU1fSQ9L2iQpJF1e+3LNzKw97Qa7pAbgbqAfMAUYDNwlqVeLpr2A14AHa12kmZlVr5oe+2iKMJ8dEbOBecAwoLG8UURsiogxwP21LtLMzKpXTbAPS9OX07QpTYd3dqOSJklaJmlZc3NzZ1djZmYVdObgqfZ2oxExJyJGRcSogQMH7u3qzMysTDXBvipNh6bpEWm6UlKDpD61L8vMzDqrmmBfAKwFJkuaDEwAVqfHVuDnpYaSLgQ+kX78mKQLJR1cy4LNzKxt7QZ7RGwDxgCbgBsoQn4MsLtC81uA8en5mPTz4TWp1MzMqlLVTcAiYgkwssIitWi31+PvZpD3japmLny+qnZNr2/tUPspvqWf923iK0/NzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLjYDczy4yD3cwsMw52M7PMONjNzDLTu7sLMDPrahfPuK0u6108c2Jd1ttR7rGbmWXGwW5mlhkHu5lZZhzsZmaZcbCbmWXGwW5mlhkHu5lZZhzsZmaZcbCbmWWmqmCXdKqkJyVtl/S4pJNbafcVSU2Stkr6X5IG1LZcMzNrT7vBLqkBuBvoB0wBBgN3SerVot2HgZuA3wNXAWcBM2tdsJmZta2aHvtoijCfHRGzgXnAMKCxRbvz0/SbEXEt8ChwTvpgMDOzLqKIaLuBNBW4Djg3In4iaRJwMzApIm4pa3cf8DngiIj4k6SfAOcAx0bECy3WOQmYlH48DniuVm+oRg4H1nV3EZnyvq0f79v6eTfu26MiYmClBZ25u6P2tl1EzAHmdGLbXULSsogY1d115Mj7tn68b+unp+3baoZiVqXp0DQ9Ik1XSmqQ1KeNdruAl/a6SjMzq1o1wb4AWAtMljQZmACsTo+twM9Tu1vT9LuSrgD+GrgzIrbVsmAzM2tbu8GegnkMsAm4gSLkxwC7W7RbDlwMfBD4DsUHwpQa19tV3rXDRBnwvq0f79v66VH7tt2Dp2Zm1rP4ylMzs8w42M3MMrPPBrukoyVFenyrbP68svmNaTqrO2vtaSTdKOnVtO/uL5tf2q/bJa2RdK+kk7qz1p5G0jGSFklaL2mjpIWS3l/2uxqStkhaKelmSYd3d809iaTH0n7dImmZpE/0xH27zwZ7C+ercDAwtruLycSdrcxvAi4EfkZx24lfSfpgl1XV8x1B8Xd7FfAj4Axgbtnyh4BLgKcoLgJ8RFLfri6yB3uUYv/9E3ASPXTfOthhJTCc4hYJY4E+wMvdWVBPFxGX0Pp9gt6IiNtSm6uAg4BvdFlxPd+jEfHJiJiV9uFrwIiy5S9ExNyI+BtgIcVZaud0R6E91FTgF8DDwHbgrbJlPWbfOtiLm5Y9BlyQHvcCG7q1on3HgjTtMVf0dbeI2FF6LmkU0B9Y0kpz79+OOxRopsiEHRTfLit5V+9bB3vhhxTn5p9K8fXWukbpthM+57aDJB0P3EdxoeA/tNYsTb1/q7cJ+AzFkEsDxTU5lbyr962DvXAnxQVXTRRfsaxrfDZNl3drFT1MOiaxmKJH+amIeKWVpt6/HRQRuyJiYUR8H/gNcDrFDcBaelfv287cBCw7EfGmpAuAjRHxlrTH/cv+StI16fnWiLi6ayvsWSSdBZyYfjxS0oXAI+nnQyWdR/EVdjKwGbhmz7VYJZKOBBYBA4B/BE6RdAqwJjU5RtIE4GyKnuczwB3dUWtPI+mzFMfZHgWOpLgtyqvA+tSkx+zbffbKU0lHU9y47IGIOLvFsqcpDkidTvFHVO6NiDisK2rsqSQtBj7ZYvZ4/jzMtZPij2UpMC0iftd11fVskhrZ83cS3vm7ug14BfhX4B8jYn2F9taCpI9S/I6+n+LA6W+BKygO8PeofbvPBruZWa48xm5mlhkHu5lZZhzsZmaZcbCbmWXGwW5mlhkHu5lZZhzsZmaZ+f+1lP7mnMIzRQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "MvyvH_qAsz1N",
        "outputId": "6ad8df69-b05e-454b-a570-82c788682fd7"
      },
      "source": [
        "#Plot comparing sensitivity and specificity within PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "sens_hala_pd_m = [ha_ha_pd_ml[0], ha_ha_1pd[0], ha_ha_2pd[0], ha_ha_3pd[0]]\n",
        "sens_hala_pd_s = [ha_ha_pd_ml[1], ha_ha_1pd[1], ha_ha_2pd[1], ha_ha_3pd[1]]\n",
        "\n",
        "spec_hala_pd_m = [la_la_pd_ml[0], la_la_1pd[0], la_la_2pd[0], la_la_3pd[0]]\n",
        "spec_hala_pd_s = [la_la_pd_ml[1], la_la_1pd[1], la_la_2pd[1], la_la_3pd[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "a5 = plt.bar(x_pos - barWidth/2, sens_hala_pd_m, yerr=sens_hala_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "a6 = plt.bar(x_pos + barWidth/2, spec_hala_pd_m, yerr=spec_hala_pd_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,1)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((a5[0], a6[0]), ('Sensitivity', 'Specificity'))\n",
        "plt.title('Sensitivity and Specificity within PD')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Sensitivity and Specificity within PD')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfUElEQVR4nO3de5hU1Z3u8e+PhqFRUBOawQEUMEpCuAqNHNQoMUEPxEgMIlEjDzcJBEVxJoSJOuJEjU50CBgRSFS84eVA1AzEmUgU0ICahkG8xFsQtRWxQVBQQC6/88fa3Smaqu6iqb7U4v08Tz27uvaqXatWVb+1au29V5m7IyIi8WhU3xUQEZHcUrCLiERGwS4iEhkFu4hIZBTsIiKRUbCLiERGwX6IMLNtZnZcFetfMbP+WWznIjP7Y04rlyNmts7Mvl3f9ShnZrPM7JqUv8eb2YbktWhZ3WuS3OfYpFxB7dc4+8c0Mzez4zOsa7DvkUOGu+tSxxfgVGA58AnwMfBnoE8dPv5c4PocbcuB4+u7TZO6rAO+nWFdO2ABsDFp95eBEXVYtybAdqDHQW5nCTCmjtt1v8esrdcdmArsArYBW5L/k37JuhHAnmTdNuBt4G6gU32+7xriRT32OmZmRwALgduALwNtgeuAnfVZr0PAfcB7QHugJXAxsKEOH781UAi8UoePma8edvfmQCvgWeB3ZmbJuhXJuiOBbxM+LFeaWdf6qWoDVd+fLIfaBSgGtlRTZhTwV2Az8D9A+5R1DowD3iT0aG4HLFl3PLCU0CPdSPgHSb3f8cBYQo/oC0Kv57+S9esI/yhtCP8sX06574nJ9poQek3PJrcvS7b7WbKtYYSe8HdT7tskue+JaZ7nlwgfcmXJc10ItEtZvwT4OeEbzVbgj0BRyvqLgXeATcBVVN1j3wb0zLCuQ/I8xgIfAOuBf0lZ3wiYAvwteaxHKrVP+TewLYQPjxHJ7XOB64FOSRt5Uo+nUl+T5Hoz4Nbk+XxCCLRmKXVrDNxA6LHuSLbz6+T1v7XS8/k9MCnN87wOuC3ldfkM+GXK4+8gdDaqfMws3ocV75Hqyqap41Tg/pS/uyT3L6q83ZQyC4H59f2/3ZAu9V6BQ+0CHJGEwz3AQOBLldYPBt4COif/WFcDy1PWe/JGPgo4lhCK/zdZ9yAh4BoReoenVrpfeYjMpdJQDCmhCDwFXJKy7pfArOR6un/a41P+nsy+HyiDgZcytEVLYAhwGNAC+H/AYynrlxDCtFMSPEuAm5J1X0+C5jSgKfCfwG4yB/tiwgfED4BjK60rD7IHgcOBbkm7lrfH5cBzhOGcpsBs4MFkXXvCh84FhLBsSfIBktrOKY/ROMNrcnvy/NoCBcDJyWPtcz8qDYsAJxE+jBolfxcBnwOt07TBGeWvRbL9vwHPp6x7MV1dKz9mFu/DdO+RtGXT1HEqSbAnz/+XwLvptptyn1HAhvr+325IFw3F1DF3/5TQw3PgN0CZmf3ezFonRcYBv3D3v7r7buBGoKeZtU/ZzE3uvsXd3wWeBnomt+8iBE0bd9/h7s/WsJrzCEFF8hX4B8lt2bgfGJQMOUHoVd+XrqC7b3L3Be7+ubtvJfQOT69U7G53f8PdtxN6yuXP9Txgobsvc/edwDXA3irqNRR4Jin3tpmtNrM+lcpc5+6fuftLhLHbC5LbxwFXuXtp8lhTgfPMrDFwIbDY3R90913Jc1pdRT32Y2aNCOF0ubu/7+573H158lhVcvcXCD38byU3/QBY4u7phplWACeYWUvCB+KdQFsza05o96UHUm8yvw8Ptuz5Zlb+7ac3cG419fiA8E1DEgr2epCE9gh3bwd0JQx//CpZ3R6YbmZbkjf3x4ARenLlPky5/jnQPLk+OSn7QnKUy6gaVnEB0M/M/okQAHsJoVgtd/+A0DMeYmZHEb6VPJCurJkdZmazzewdM/uUMLRzVKWjMTI91zaEf/zyx/2M8E0oU702u/sUd+9CGO9eDTyWMnZL6vYIQyJtkuvtgUdTXpO/EoYnWgPHEHq+B6OI8A2rptu5B/hhcv2HZP4g3Q6UEEL8NEKQLwdOoWbBnum1Odiyj7j7Ue7+j+5+hruvrKYebQn/J5JQsNczd3+N8JW9fOfPe8CPkjd2+aWZuy/PYlsfuvsl7t4G+BEwM8MhaVVO6enumwnj2cMIPdKHPPnOm6XyoBlK2Nn1foZy/wx8Fejr7kcQwgbCh1N11hNCNdzB7DDCMEi13H0jcAshuFN7esekXD+W0BOE8JoMrPSaFCbP6z3gK9k8bhU2Esaws9lOutfhfmCwmfUgDOE9VsX9lxKGXU4E/pL8fRZhSGfZATxmQ3IuWXY8DhUK9jpmZl8zs382s3bJ38cQvvI/lxSZBfyrmXVJ1h9pZkOz3PbQ8u0SdkY66YcnNgBVHj9NGHoZThjyqGoYJt22HgN6Ecam763ivi0IO2q3mNmXgWurqVOq+cDZZnaqmf0D8O9U8X42s5vNrKuZNTazFsB44C13T+3lX5N8i+gCjAQeTm6fBdxQPhxmZq3MbHCy7gHg22Z2frLtlmZW1TDDftx9L3AX8J9m1sbMCsysn5k1TVN8v/Z291JCSN8HLEh65pksJbyur7r7FyTj58Db7l6W4T7ZvF/qVNJGHc3sNqA/YcewJBTsdW8r0Bd43sw+IwT6y4TeK+7+KHAz8FAyPPEyYTgjG32S7W4jHBlxubuvTVPuTuDrydBCpt7d74ETgA/d/cUqHnMqcE+yrfOT57CdMJzTEfhdFff9FWGn6EZCO/x3FWX34e6vABMIHzrrCR9kpVXc5TDgUcJRGWsJwyvnVCqzlLDj+k/ALe5efpLNdEJ7/NHMtiZ17ZvU411gEOH1+5gwxNMj2+eR4l+AlwgB/THhPZDu/3M6YXx/s5nNSLn9HsJO37TDMCmWE9q8vHf+KuHbQqbeelWPWR/6Je/vTwkfSkcQzgF5qV5r1cDYgX3DFsmOmf0b4cSRH1ZbuJ6ZWQfCyS5Nkh3WecfMTiMMybQ/wGEziVDj+q6AxCcZVhlNOCJGapmZNSEMe/1WoS6QxVCMmc1I5rdwM1tYRblTzGyNme00s1Vm1iu3VZV8YGaXEHYoPuHuVX29lxwws86E4aV/4u9HVskhrtqhmGRMzYGJwCJ3PztNmULCCS7bCScUXEU4Rf4Ed9+T4zqLiEgVqu2xu/tEYFo1xQYSjumd6e4zCTvnOhL2VouISB3K1Rh7x2RZfrxy+dEJxxGOMNiHmY0lzMvB4Ycf3vtrX/tajqohEo/XX38dgK9+9at5sd18EkMbrFy5cqO7t0q3rrZ2nlZ5gom7zwHmABQXF3tJSUktVUMkf/Xv3x+AJUuW5MV280kMbWBm72RaV+Pj2M2sacoJFG8ny/KTY8pPf093DLWIiNSianvsZvYd/n66+zFmNoZwIseThDkumgNPAB8B45MTOEYTdqYuyX2VRfLbtCffyKpc6ebtB1R+0oBONa6TxCWbHvtPgJuS690JMxKeklrA3XcQ5gXZRjhL7SNgqI6IERGpe9X22N29f4ZVcyuVW0Y4pfmg7dq1i9LSUnbs2JGLzfHhh2FiuaOPPjon28tnhYWFtGvXjiZNmuRkezGMVTZUE26pbnYAqUzfhoIGeeZpaWkpLVq0oEOHDuw7q2rNNGoUvpjk8x7wXHB3Nm3aRGlpKR07dqz+DiKSlxpksO/YsSOrUN/waXY9+i927z2g8q2PKMyqXL4xM1q2bElZWaZJ/ETkYDSUb7ANdnbHXPTUZX9qV5H4Ncgee6794zEadjhQGquUmMW+/yIvgj1TaHy2s2YzrF5yWna/GXDDDTcwb948CgoKaNSoEbNnz6Zv3741eszKTj75ZJYvX866detYvnw5F154IQAlJSXce++9zJiRedrrWbNmcdhhhzF8+HDmzp3LmWeeSZs2bTKWF5FDS14Ee31YsWIFCxcuZNWqVTRt2pSNGzfyxRdf5Gz7y5eHX7pbt24d8+bNqwj24uJiiouLq7zvuHHjKq7PnTuXrl27KthFpEKDHWOvb+vXr6eoqIimTcPJtUVFRbRp04aVK1dy+umn07t3b8466yzWr18PhJ0mP/3pTznppJPo1KkTzzwTfoLxlVde4aSTTqJnz550796dN998E4DmzcNv+U6ZMoVnnnmGnj17Mm3aNJYsWcLZZ5/N3r176dChA1u2bKmo0wknnMCGDRuYOnUqt9xyC/Pnz6ekpISLLrqInj17smjRIr73ve9VlH/yySc599zqfuBdRGKjYM/gzDPP5L333qNTp078+Mc/ZunSpezatYvLLruM+fPns3LlSkaNGsVVV11VcZ/du3fzwgsv8Ktf/Yrrrgs/wThr1iwuv/xyVq9eTUlJCe3atdvncW666Sa+8Y1vsHr1aiZNmlRxe6NGjRg8eDCPPvooAM8//zzt27endevWFWXOO+88iouLeeCBB1i9ejWDBg3itddeqzjq5e6772bUqFG11kYQxipjH68UyTcK9gyaN2/OypUrmTNnDq1atWLYsGHMnj2bl19+mQEDBtCzZ0+uv/56Skv//jOb3//+9wHo3bs369atA6Bfv37ceOON3Hzzzbzzzjs0a9Ys6zoMGzaMhx8Ov6f80EMPMWzYsCrLmxkXX3wx999/P1u2bGHFihUMHJjtz6WKSCw0xl6FgoIC+vfvT//+/enWrRu33347Xbp0YcWKFWnLlw/bFBQUsHt32LF74YUX0rdvXxYtWsSgQYOYPXs2Z5xxRlaP369fP9566y3Kysp47LHHuPrqq6u9z8iRI/nud79LYWEhQ4cOpXFjvcQihxr12DN4/fXXK8bDAVavXk3nzp0pKyurCPZdu3bxyiuvVLmdtWvXctxxxzFx4kQGDx7MmjVr9lnfokULtm7dmva+Zsa5557LlVdeSefOnWnZsuV+ZSrfv02bNrRp04brr7+ekSNHZv18RSQeedGdy3Tsc7ZnktbEtm3buOyyy9iyZQuNGzfm+OOPZ86cOYwdO5aJEyfyySefsHv3bq644gq6dOmScTuPPPII9913H02aNOHoo4/mZz/72T7ru3fvTkFBAT169GDEiBGceOKJ+6wfNmwYffr0Ye7cuWm3P2LECMaNG0ezZs1YsWIFzZo146KLLqKsrIzOnTsfdDuISP7Ji2CvD7179644JDFVUVERy5bt/xvNqacQFxUVVYyxT5kyhSlTpuxXftu2bQA0adKEp556ap915aclQzj8sfLv0k6dOrXi+pAhQxgyZMg+65999lkuueSStM9LROKnYI9M7969Ofzww7n11lvruyoiUk8U7JFZuXJlfVdB6svTv8iu3JZ3D6z8N/+1ZvWJSZ61rXaeiohERsEuIhIZBbuISGQU7CIikcmPnacZdkQcXsNpez87+SdZlavNaXsHDRrEvHnzOOqoo5gxYwZ33HEHvXr1YtiwYbz66qtpD5Esl2nKXxERyJdgrwe1PW3vH/7wh4rrM2fOZPHixRUThJ1zzjlV3jfTlL8iIqChmIwyTdvboUMHJk+eTLdu3TjppJN46623ACgrK2PIkCH06dOHPn368Oc//xkIJyKNHDmSbt260b17dxYsWABAhw4d2LhxI+PGjWPt2rUMHDiQadOmMXfuXC699FIANmzYwLnnnkuPHj3o0aNHRaBnmvL3tNNOY/Xq1RXP4dRTT+XFF1+smwYTkQZDwZ5Buml7yx155JG89NJLXHrppVxxxRUAXH755UyaNIm//OUvLFiwgDFjxgDw85//vKL8mjVr9psAbNasWbRp04ann356n2l7ASZOnMjpp5/Oiy++yKpVq/abuqDylL+jR4+umHrgjTfeYMeOHfTo0SPXTSMiDZyCPYN00/aWh+YFF1xQsSyfEGzx4sVceuml9OzZk3POOYdPP/2Ubdu2sXjxYiZMmFCx3S996UtZ1+Gpp55i/PjxQJgx8sgjj6yy/NChQ1m4cCG7du3irrvuYsSIEQfwjEUkFhpjr0LlaXvvueceIMy6WK78+t69e3nuuecoLCysl7oCHHbYYQwYMIDHH3+cRx55RGehihyi1GPPIN20ve3btweo+PGLhx9+mH79+gFh6Oa2227bpzzAgAEDuP322ytu37x5c9Z1+Na3vsUdd9wBwJ49e/jkk0/2WZ9uyt8xY8YwceJE+vTpc0DfDkQkHvnRY88wn8Jn9TBt78KFC9m8eTPdu3enadOmPPjggwDMmDGDCRMm0L17d3bv3s1pp53GrFmzuPrqq5kwYQJdu3aloKCAa6+9tuKXlqozffp0xo4dy5133klBQQF33HFHxQcJ7D/l76RJk+jduzdHHHGE5mIXOYTlR7DXg0zT9gL85Cc/4eabb97ntqKiooqefKrmzZtXDOGkKp/Wt/L1ESNGVIyNt27dmscff3y/+1Y15e8HH3zA3r17OfPMM9PWXURqz5JpDWO6bA3FROTee++lb9++3HDDDTRqpJdW5FClHvsBSu1dNzTDhw9n+PDh9V0NEalnDbZbV/lXgyQ31K4i8WuQwV5YWMimTZsUQjnm7mzatKleD8kUkdrXIIdi2rVrR2lpKWVlZVWW+3T7rlp5/I+bNamV7TYEhYWFFXPSiEicGmSwN2nShI4dO1ZbbtqTb9TK408a0CmrcuU/Op36Q9YiIvWtQQ7FiIhIzWUV7GZ2ipmtMbOdZrbKzHqlKdPUzH5rZmVmtt3M/tfMzki3PRERqT3VBruZFQILgBbAJKA1MN/MCioVHQ6MBlYD1wA9gN/ktLYiIlKtbHrsAwlhPtPdZwJ3Ah2B/hm29TKwGNgJbMlNNUVEJFvZ7Dwt34v5frIsTZbHAX9KKXcPcBZwRXL5GBiRboNmNhYYC3DsscceUIVF5OA0lNPepfbUZOepZbj9/wDfAR4AfgAUAHMtdY7bhLvPcfdidy9u1apVDaogIiKZZBPsbyfL8oOf2ybLtWZWaGblB30PBf4BmOXuDwMlQC+gKFeVlUNH+Tz4InLgshmKeQL4CBhvZlsJO0jXJZftwCLgbGBtUn6ymfUA+gGbgI25rbKIiFSl2h67u+8g9Ma3AdMJIT8U2FOp6O2EHat9gV8CrwFDXfMCiIjUqazOPHX3ZUC3NKsspcwOYEyO6iUiIjWkM09FRCKjYBcRiYyCXUQkMg1ydsd69/Qvsiu35d0DK5/hR7lFRHJJPXYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcjoqBipWzriSKTWqccuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGR0uONB0K+9i0hDpB67iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkdFSMNEg64kik5tRjFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkchkFexmdoqZrTGznWa2ysx6ZSjX1cyeMrPtZrbJzP4jt9UVEZHqVDsJmJkVAguA7cAk4Cpgvpmd4O57Uso1A/4baAb8G/A5UFQblRYRkcyymd1xINAamOzuM83saOAaoD/wp5RyFwBtgUuAB9x9e47rKiIiWchmKKZjsnw/WZYmy+Mqlft6srwS+NzMyszs/HQbNLOxZlZiZiVlZWUHVGEREalaTXaeWobbmybL9cAQYCcw18xaVC7o7nPcvdjdi1u1alWDKoiISCbZBPvbybJdsmybLNeaWaGZNalU7hF3/x3wDGG8vU1OaioiIlnJZoz9CeAjYLyZbQVGA+uSy3ZgEXA28BBwIzDSzPYC3yIM3/wt57UWEZGMqu2xu/sOYCiwDZhOCPmhwJ5K5T4ALgRaJeXeAs5x9905rrOIiFQhq988dfdlQLc0q6xSud8Bv8tBvUREpIZ05qmISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hERsEuIhIZBbuISGQU7CIikVGwi4hEJqtgN7NTzGyNme00s1Vm1quKsp3NbIeZuZmdl7uqiohINqoNdjMrBBYALYBJQGtgvpkVpClrwG+APTmup4iIZCmbHvtAQpjPdPeZwJ1AR6B/mrLjgfbA7FxVUEREDkw2wd4xWb6fLEuT5XGphcysLfALQrh/WtUGzWysmZWYWUlZWdkBVFdERKpTk52nluH2m4AS4DXgy8ltR5tZ88oF3X2Ouxe7e3GrVq1qUAUREcmkcRZl3k6W7ZJl22S5Nhl/3+Puu4BjgNOBN1PuexuwBbg/B3UVEZEsZBPsTwAfAePNbCswGliXXLYDi4CzgWuB8u73+cBQ4FZgWU5rLCIiVao22N19h5kNBW4HpgOvAJdQ6cgXd19aft3MuiZXn3P3d3NXXRERqU42PXbcfRnQLc2qtOPt7j4VmFrjWomISI3pzFMRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIqNgFxGJjIJdRCQyCnYRkcgo2EVEIpNVsJvZKWa2xsx2mtkqM+uVpsw5ybqtZrbRzO4ys2a5r7KIiFSl2mA3s0JgAdACmAS0BuabWUGloj2AV4ErgZXASGByTmsrIiLVapxFmYGEMJ/s7jPN7GjgGqA/8KeUcje7+xcAZvYcsAboktvqiohIdbIZiumYLN9PlqXJ8rjUQuWhnjgrWS5Lt0EzG2tmJWZWUlZWlm1dRUQkCzXZeWpVrjQbAtwI/AG4I10Zd5/j7sXuXtyqVasaVEFERDLJJtjfTpbtkmXbZLnWzArNrEl5QTM7H3gIeBoY4u57clZTERHJSjbB/gTwETDezMYDo4F1yWU78CiAmX0HmAdsAR4EvmdmZ+S+yiIiUpVqg93ddwBDgW3AdELIDwUq98b7AAVAEXA3Idz/LZeVFRGR6mVzVAzuvgzolmaVpZSZCkzNSa1ERKTGdOapiEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIRCarYDezU8xsjZntNLNVZtYrQ7kfmVmpmW03s8fNrGVuqysiItWpNtjNrBBYALQAJgGtgflmVlCp3InALOCvwLXAd4Bpua6wiIhULZse+0BCmM9095nAnUBHoH+lciOS5c/c/T+A5cAFyQeDiIjUkcZZlOmYLN9PlqXJ8jjgT9WUawwcA7yZukEzGwuMTf7cZmavH0Cda92VUARszP2Wf5b7TeYZtW3tUdvWngbatu0zrcgm2Cuzgy3n7nOAOTV47DphZiXuXlzf9YiR2rb2qG1rT761bTZDMW8ny3bJsm2yXGtmhWbWpIpyu4H3DrqWIiKStWyC/QngI2C8mY0HRgPrkst24NGk3L3J8gYzmwycDDzk7jtyWWEREalatcGeBPNQYBswnRDyQ4E9lcqtBCYAXwf+nfCBMCnH9a0rDXaYKAJq29qjtq09edW25u71XQcREckhnXkqIhIZBbuISGQO2WA3sw5m5snlmpTb70y5vX+y/HV91jXfmNkMM9uQtN3ClNvL23WnmX1oZo+ZWc/6rGu+MbMTzOxpM9tkZlvN7Ekz+0rKe9XN7HMzW2tms82sqL7rnE/M7PmkXT83sxIzOy0f2/aQDfZKRljQHDi/visTiYcy3F4KjAEeIUw78ayZfb3OapX/2hL+b68F7ga+Dfw2Zf0fgYnAS4STAJeaWbO6rmQeW05ov58DPcnTtlWww1rCWbT9CaHehL+fPSs14O4TyTxP0Cfufl9S5lrgcGBKnVUu/y1399Pd/ddJG34MdElZ/6a7/9bdBwNPEo5Su6A+KpqnrgT+i3BW/U5gb8q6vGlbBXuYtOx5YFRyeQzYUq81OnQ8kSzz5oy++ubuX5RfN7Ni4MvAsgzF1b4H7kigjJAJXxC+XabToNtWwR7cRTg2/xTC11upG+XTTuiY2wNkZl8Dfk84UfCyTMWSpdo3e9uAMwlDLoWEc3LSadBtq2APHiKccFVK+IoldeOsZLmyXmuRZ5J9EksIPcoz3H19hqJq3wPk7rvd/Ul3vw14AfgmYQKwyhp029ZkErDouPunZjYK2Orue832m7+sn5ndlFzf7u7X1W0N84uZfQfomvx5jJmNAZYmfx9pZhcTvsKOBz4Dbtp/K5KOmR0DPA20BK4G+ppZX+DDpMgJZjYaOJvQ83wVeLA+6ppvzOwswn625YRZaU8GNgCbkiJ507aH7JmnZtaBMHHZInc/u9K6lwk7pL5J+CdK9Ym7H1UXdcxXZrYEOL3SzSP5+zDXLsI/y3PAVHd/se5ql9/MrD/7vydh3/fqDmA98D/A1e6+KU15qcTM+hDeo18h7Dj9X2AyYQd/XrXtIRvsIiKx0hi7iEhkFOwiIpFRsIuIREbBLiISGQW7iEhkFOwiIpFRsIuIROb/A68JeMzVReoQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "U5x7CSHuwDd6",
        "outputId": "c32f1f88-5bd6-4336-b1e9-e9b72057238c"
      },
      "source": [
        "#Plot comparing FNR and FPR within PD.\n",
        "labels = ['ML', '1D', '2D', '3D']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "fnr_hala_pd_m = [ha_la_pd_ml[0], ha_la_1pd[0], ha_la_2pd[0], ha_la_3pd[0]]\n",
        "fnr_hala_pd_s = [ha_la_pd_ml[1], ha_la_1pd[1], ha_la_2pd[1], ha_la_3pd[1]]\n",
        "\n",
        "fpr_hala_pd_m = [la_ha_pd_ml[0], la_ha_1pd[0], la_ha_2pd[0], la_ha_3pd[0]]\n",
        "fpr_hala_pd_s = [la_ha_pd_ml[1], la_ha_1pd[1], la_ha_2pd[1], la_ha_3pd[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "a7 = plt.bar(x_pos - barWidth/2, fnr_hala_pd_m, yerr=fnr_hala_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "a8 = plt.bar(x_pos + barWidth/2, fpr_hala_pd_m, yerr=fpr_hala_pd_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,0.5)\n",
        "plt.xticks(ticks=[0,1,2,3], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((a7[0], a8[0]), ('FNR', 'FPR'))\n",
        "plt.title('FNR and FPR within PD')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'FNR and FPR within PD')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAbJklEQVR4nO3de5hU1Z3u8e9ry7FREA1w9ACOoNEJQS5CI0dNpDRBD8RrDDLqowcEGQiI4plHTWIijtdkdFCMCHgjalA5EI0BzRxMBI2gpmEQg/dBjK1GGxSEBJDLOn/s3Z2yrequbqppavF+nqee3VVrVdVvb5q3d6299yqFEDAzs3js1dIFmJlZcTnYzcwi42A3M4uMg93MLDIOdjOzyDjYzcwi42C3aEhaKGlUS9dRCEnTJP24nvZJkh6qp32lpEyzFGclz8FuXyBptaRNkjZm3TpJ6iopSHqyTv+HJE1Kf85I2pE+Z4OkNySNaJEVqSMNyq111uuKtG2hpM3pY2sk/UrS/0jbZkr6PG37RNICSV/b2XpCCGNCCNel75GRVNXI5/cIISxsynun/45/TdfpfUn/Lqksbav5998gaZ2kxZLGSHJWlBD/Y1kup4UQ2mTdPshqGyDpuHqe+0EIoQ2wPzARuFvSPzZrtYV7tM56/SyrbXxa95HAAcDkrLafpW2dgfeBe3ddyc2md7pO3wLOAy7OajsthNAWOBS4GbiSONZ5j+Fgt8b6GXBDQ51C4kngE6BXvn6S/q+kv0haL+lZST2y2mZKulPS/HQP8kVJh2e1D5L0evrcnwPauVWDEMInwFzgqBxtm4DZQJ8861Ke7u12SO//SNI2Sfun96+TdFvWul0vaT/gKaBT9iek9CX/m6QH0nVfKaki671WS/p2+vMkSbPz9W1gfV8HnsuzvutDCE8Aw4D/LelLfWz35GC3xpoKHFkTKvlI2kvS6UAH4O16uj4FHAH8d2AZ8Ms67f8EXAscmL7ODenrdwB+BVydvsd/Acc3dmVy1N0BOBv4zxxt+wHnkmd9QgibgT8CA9OHBgLvZtU1EFhU5zl/BQaTftKp8wnpdOARkk8QTwA/r6f0xvTNXqevA98kx/pm1fgSUJX2sxLgYLdcHk/HV9dJerxO2yaScL0+z3M7SVqX9nsMuDyEUF9o3BdC2BBC2AJMAnpLapfV5bEQwkshhG0koV+ztzwEWBlCmBNC2ArcBvylgfU6J2u91mXtGQNMSet+GfgQuDyr7V/Stg3AN4AL6nmPRcBASXuTfFKZkt4vB/oDzzZQY7Y/hBCeDCFsBx4EehepL8AySZ8CvwHuAe5voP8HwFcKrNtamIPdcjkzhHBAejszR/s9wEGSTsvR9kEI4QCSMfYpwEn53kRSmaSbJf2XpM+A1WlTh6xu2WH9N6BN+nMn4L2ahpDMZvce9ZudtV4H1Dl2MCF9rHMI4fwQQnVW2y3pOnUl+YNV3zGDRUAG6Au8Aiwg2VP/n8DbIYS1DdSYre66l6d/MHa2L0DfEMKBIYTDQwhXhxB2NFBLZ5JhNSsBDnZrtBDC5yTDI9eRZ1w73QO/EugpKdcfB0gO2p0BfBtoRxKc5HvNOj4EDqm5I0nZ95tDCOHPwKXA7ZJa5+m2mCT4zwIWhRBeBf6B5BPGojzP2a2nWJXUnyTY/9DStVhhHOzWVA8C5cD/ytch/QNwK/CTPF3aAluAtcC+wI2NeP/5QA9J3033TCcABzfi+U0SQlhAMiwxOk/734ClwDj+HuSLgTHkD/aPgPZ1hqBanKT9JZ1KMnb/UAjhlZauyQrjYLcmScdyf0LD4673Af+QZ9jmAZKDi+8DrwIvNOL91wBDSU7HW0tyAPb5Qp+/k/4NuELSPnnaFwGtgJey7rclz/h6embKw8CqHGP/LeE3kjaQDG39CPh3YLe4HsEKI3/RhplZXLzHbmYWmYKCXdLxklZI2iJpmaS+OfrUXHKefbut+CWbmVl9GhyKSc+/XU1ymte/kYy5bQGOSMdZa/p1Bd4BpvH3g0Rv1HcOs5mZFV9957nWGAwcBFwRQpgq6WDgxyTn6v4uR/9K4In07AAzM9vFCgn2buny/XRZMwvdYeQO9ruBeyS9CowMIXzpTAdJo0lPF9tvv/36fe1rOz1ZnpnZHmXp0qVrQggdc7UVEux15bt45K/ANcByklPPbia5BPzwuh1DCDOAGQAVFRWhsrKyCWWYme25JL2br62QYH8nXXZJl53T5ap0/H17CGFregn2v2a96XlAX0nl6eRIZma2CxQS7E8BHwNj04sWRpIcTF1NckB1PnCqpItJJjl6kWSYpg/wskPdzGzXavB0xzSYhwIbgdtJQn4osL1O1zdJZrO7Dfg+8Nu0n5mZ7UIFjbGHEJ4FeuZoUlafRSQz2JnZHmbr1q1UVVWxebM/oBdbeXk5Xbp0oVWrVgU/pykHT83MvqCqqoq2bdvStWtXkok2rRhCCKxdu5aqqiq6devW8BNSnlLAzHba5s2bad++vUO9yCTRvn37Rn8ScrCbWVE41JtHU7arg93MLDIeYzezopu84M2ivt7EQUc22OeGG25g1qxZlJWVsddeezF9+nQGDBhQlPc/7rjjWLx4MatXr2bx4sWcd955AFRWVvLAAw8wZcqUvM+dNm0a++67LxdeeCEzZ87k5JNPplOn5p1y38FuZiVvyZIlzJs3j2XLlrHPPvuwZs0aPv/886K9/uLFiwFYvXo1s2bNqg32iooKKioq6n3umDFjan+eOXMmRx11VLMHu4dizKzkffjhh3To0IF99km+1KpDhw506tSJpUuXMnDgQPr168cpp5zChx9+CEAmk+HKK6/kmGOO4cgjj+S5554DYOXKlRxzzDH06dOHXr168dZbbwHQpk3yHepXXXUVzz33HH369GHy5MksXLiQU089lR07dtC1a1fWrVtXW9MRRxzBRx99xKRJk7jllluYM2cOlZWVnH/++fTp04f58+dz5pl//zrgBQsWcNZZZxVlezjYzazknXzyybz33nsceeSRfP/732fRokVs3bqVSy65hDlz5rB06VIuuugifvSjH9U+Z9u2bbz00kvcdtttXHvttUAybHLppZeyfPlyKisr6dKlyxfe5+abb+ab3/wmy5cvZ+LEibWP77XXXpxxxhk89thjALz44osceuihHHTQQbV9vve971FRUcEvf/lLli9fzpAhQ3j99deprq4G4P777+eiiy4qyvZwsJtZyWvTpg1Lly5lxowZdOzYkWHDhjF9+nT+9Kc/MWjQIPr06cP1119PVVVV7XO++93vAtCvXz9Wr14NwLHHHsuNN97IT3/6U959911at25dcA3Dhg3j0UcfBeCRRx5h2LBh9faXxAUXXMBDDz3EunXrWLJkCYMHD27kmufmMXYzi0JZWRmZTIZMJkPPnj2588476dGjB0uWLMnZv2bYpqysjG3btgFw3nnnMWDAAObPn8+QIUOYPn06J510UkHvf+yxx/L2229TXV3N448/ztVXX93gc0aMGMFpp51GeXk5Q4cOZe+9ixPJ3mM3s5L3xhtv1I6HAyxfvpzu3btTXV1dG+xbt25l5cqV9b7OqlWrOOyww5gwYQJnnHEGK1as+EJ727Zt2bBhQ87nSuKss87i8ssvp3v37rRv3/5Lfeo+v1OnTnTq1Inrr7+eESNGFLy+DfEeu5kVXSGnJxbTxo0bueSSS1i3bh177703X/3qV5kxYwajR49mwoQJrF+/nm3btnHZZZfRo0ePvK8ze/ZsHnzwQVq1asXBBx/MD3/4wy+09+rVi7KyMnr37s3w4cM5+uijv9A+bNgw+vfvz8yZM3O+/vDhwxkzZgytW7dmyZIltG7dmvPPP5/q6mq6d+++09uhRoPfedrc/EUbZqXvtddeK2ow7UnGjx/P0UcfzciRI/P2ybV9JS0NIeQ819J77GZmLaRfv37st99+3HrrrUV9XQe7mVkLWbp0abO8rg+emplFxsFuZhYZB7uZWWQc7GZmkfHBUzMrvmduKu7rnfiDBrs057S9Q4YMYdasWRxwwAFMmTKFu+66i759+zJs2DBeffVVrrrqqrzPzTflb3NysJtZyWvuaXuffPLJ2p+nTp3K008/XTtB2Omnn17vc/NN+ducPBRjZiUv37S9Xbt25YorrqBnz54cc8wxvP322wBUV1dz9tln079/f/r378/zzz8PJFewjhgxgp49e9KrVy/mzp0LQNeuXVmzZg1jxoxh1apVDB48mMmTJzNz5kzGjx8PwEcffcRZZ51F79696d27d22g55vy94QTTmD58uW16/CNb3yDl19+uSjbw8FuZiUv17S9Ndq1a8crr7zC+PHjueyyywC49NJLmThxIn/84x+ZO3cuo0aNAuC6666r7b9ixYovTQA2bdo0OnXqxDPPPPOFaXsBJkyYwMCBA3n55ZdZtmzZl6YuqDvl78iRI2unHnjzzTfZvHkzvXv3Lsr2cLCbWcnLNW1vTWiee+65tcuaCcGefvppxo8fT58+fTj99NP57LPP2LhxI08//TTjxo2rfd0DDzyw4Bp+//vfM3bsWCCZMbJdu3b19h86dCjz5s1j69at3HfffQwfPrwRa1w/j7GbWRTqTtv7i1/8AkhmXaxR8/OOHTt44YUXKC8vb5FaAfbdd18GDRrEr3/9a2bPnl3Uq1C9x25mJS/XtL2HHnooQO2XXzz66KMce+yxQDJ0c8cdd3yhP8CgQYO48847ax//9NNPC67hW9/6FnfddRcA27dvZ/369V9ozzXl76hRo5gwYQL9+/dv1KeDhniP3cyKr4DTE4sp37S98+bN49NPP6VXr17ss88+PPzwwwBMmTKFcePG0atXL7Zt28YJJ5zAtGnTuPrqqxk3bhxHHXUUZWVlXHPNNbXftNSQ22+/ndGjR3PvvfdSVlbGXXfdVfuHBL485e/EiRPp168f+++/f1HnYgdP22tmRbC7TtvbtWtXKisr6dChQ0uXktMHH3xAJpPh9ddfZ6+98g+gNHbaXg/FmJm1gAceeIABAwZwww031BvqTeGhGDOLVs2XVO+OLrzwQi688MJmeW3vsZtZUbT0sG6smrJdHexmttPKy8tZu3atw73IQgisXbu20adleijGzHZaly5dqKqqorq6uqVLiU55eXntvDSFcrCb2U5r1aoV3bp1a+kyLOWhGDOzyBQU7JKOl7RC0hZJyyT1radvd0mbJQVJ3yteqWZmVogGg11SOTAXaAtMBA4C5kgqy9FXwN3A9iLXaWZmBSpkj30wSZhPDSFMBe4FugGZHH3HAocC04tVoJmZNU4hwV5zROT9dFmVLg/L7iSpM3ATSbh/Vt8LShotqVJSpY+im5kVV1MOnirP4zcDlcDrwFfSxw6W1KZuxxDCjBBCRQihomPHjk0owczM8inkdMd30mXNiZSd0+WqdPx9ewhhK3AIMBB4K+u5dwDrgIeKUKuZmRWgkGB/CvgYGCtpAzASWJ3eNgHzgVOBa4Ca3e9zgKHArcCzRa3YzMzq1WCwhxA2SxoK3AncDqwELqbOmS8hhNovGZR0VPrjCyGEPxevXDMza0hBV56GEJ4FeuZoyjneHkKYBExqclVmZtZkvvLUzCwyDnYzs8g42M3MIuNgNzOLjIPdzCwyDnYzs8g42M3MIuNgNzOLjIPdzCwyDnYzs8g42M3MIuNgNzOLjIPdzCwyDnYzs8g42M3MIuNgNzOLjIPdzCwyDnYzs8g42M3MIuNgNzOLjIPdzCwyDnYzs8g42M3MIuNgNzOLjIPddkuZTIZMJtPSZZiVJAe7mVlkHOxmZpFxsJuZRcbBbmYWGQe7mVlkHOxmZpFxsJuZRcbBbmYWGQe7mVlkCgp2ScdLWiFpi6Rlkvrm6HOUpNckbZa0TtKTkjoXv2QzM6tPg8EuqRyYC7QFJgIHAXMkldXpugN4BBgN/AoYDFxT1GrNzKxBheyxDyYJ86khhKnAvUA3IJPdKYTwKnAT8FtgcfrwjqJVamZmBSkk2Luly/fTZVW6PCxH3yHAR8DdwEry7LFLGi2pUlJldXV1I8rdvXiiKjPbHTXl4KnqaXueZA//dqAH8M+5OoUQZoQQKkIIFR07dmxCCWZmlk8hwf5OuuySLmsOiK6SVC6pVU3HEEJ1COG3wP8hGYY5p2iVmplZQfYuoM9TwMfAWEkbgJHA6vS2CZgPnCrpB0A74HXgJJI/Gq8Wv2QzM6tPg3vsIYTNwFBgI8kQy8fp/e11ulYD5wLTgVOAh4HxxSzWzMwaVsgeOyGEZ4GeOZqU1ece4J4i1WVmZk3kK0/NzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMgWdx25WNM/cVFi/dX9uXP8Tf9C0eswi5D12M7PIONjNzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMp4ELBdPVGVmJcx77GZmkXGwm5lFxsFuZhYZB7uZWWQc7GZmkXGwm5lFxsFuZhYZB7uZWWQc7GZmkXGwm5lFxsFuZhYZB7uZWWQ8CZjtlhZOvrilSzArWd5jNzOLTEHBLul4SSskbZG0TFLfHH1OT9s2SFoj6T5JrYtfspmZ1afBYJdUDswF2gITgYOAOZLK6nTtDbwKXA4sBUYAVxS1WjMza1AhY+yDScL8ihDCVEkHAz8GMsDvsvr9NITwOYCkF4AVQI/ilmtmZg0pZCimW7p8P11WpcvDsjvVhHrqlHT5bK4XlDRaUqWkyurq6kJrNbMiyGQyZDKZli7DmlFTDp6q3kbpbOBG4Engrlx9QggzQggVIYSKjh07NqEEMzPLp5BgfydddkmXndPlKknlklrVdJR0DvAI8Axwdghhe9EqNTOzghQS7E8BHwNjJY0FRgKr09sm4DEASd8BZgHrgIeBMyWdVPySzcysPg0GewhhMzAU2AjcThLyQ4G6e+P9gTKgA3A/Sbj/pJjFmplZwwq68jSE8CzQM0eTsvpMAiYVpSozM2syX3lqZhYZB7uZWWQc7GZmkfHsjjvBMxCa2e7Ie+xmZpFxsJuZRcbBbmYWGQe7mVlkHOxmZpFxsJuZRcbBbmYWGQe7mVlkHOxmZpFxsJuZRcbBbmYWGQe7mVlkHOxmZkWSyWTIZDItXYaD3cwsNg52M7PIONjNzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMgUFu6TjJa2QtEXSMkl9c/RpLel3kjZKCpL+pfjlmplZQxoMdknlwFygLTAROAiYI6msTtcy4BPgt8Uu0szMClfIHvtgkjCfGkKYCtwLdAMy2Z1CCBtDCEOBecUu0szMCldIsHdLl++ny6p0eVhT31TSaEmVkiqrq6ub+jJmZpZDUw6eamffNIQwI4RQEUKo6Nix486+nJmZZSkk2N9Jl13SZed0uUpSuaRWxS/LzMyaqpBgfwr4GBgraSwwElid3jYBj9V0lDQKOCG9e4ykUZLaFLNgMzOrX4PBHkLYDAwFNgK3k4T8UGB7ju53AyPSn4em9zsUpVIzMyvI3oV0CiE8C/TM0aQ6/XZ6/N3MmuiZmwrrt+7Pjet/4g+aVk9MSmzb+spTM7PIONjNzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMg52M7PIONjNzCLjYDczi4yD3cwsMg522ymZTIZMJtPSZZhZFge7mVlkHOxmZpFxsJuZRcbBbmYWGQe7mVlkHOxmZpFxsJuZRaagL7MudTXnWS9cuLBF6yglkxe8WVC/qk83Nar/xD3iN852d7FngvfYzcwi42A3M4uMg93MLDIOdjOzyJT0oSwf4DOz3cnCyRe3dAmA99jNzKLjYDczi4yD3cwsMnvEaPK4Wx5s6RKi5W3bfGK/iKY5+LhbwnvsZmaRcbCbmUXGwW5mFpkSGzkyK30eB7bm5j12M7PIFBTsko6XtELSFknLJPXN0++fJVVJ2iTp15LaF7dcMzNrSIPBLqkcmAu0BSYCBwFzJJXV6Xc0MA14DbgG+A4wudgFm5lZ/QrZYx9MEuZTQwhTgXuBbkCmTr/h6fKHIYSfAYuBc9M/DGZmtosohFB/B+ly4Fbg/BDCLEmjgenA6BDC3Vn9ngBOAzqHED6QNAs4FzgyhPBWndccDYxO7/4j8EaxVqhIOgBrWrqISHnbNh9v2+azO27bQ0MIHXM1NOU4una2XwhhBjCjCe+9S0iqDCFUtHQdMfK2bT7ets2n1LZtIUMx76TLLumyc7pcJalcUqt6+m0D3tvpKs3MrGCFBPtTwMfAWEljgZHA6vS2CXgs7fdAurxB0hXAccAjIYTNxSzYzMzq12Cwp8E8FNgI3E4S8kOB7XX6LQXGAV8H/pXkD8LEIte7q+y2w0QR8LZtPt62zaektm2DB0/NzKy0+MpTM7PIONjNzCKzxwa7pK6SQnr7cdbj92Y9nkmXP2/JWkuNpCmSPkq33bysx2u26xZJf5H0uKQ+LVlrqZF0hKRnJK2VtEHSAkmHZ/2uBkl/k7RK0nRJHVq65lIi6cV0u/5NUqWkE0px2+6xwV7HcCXaAOe0dDGReCTP41XAKGA2ybQTf5D09V1WVenrTPL/9hrgfuDbwD1Z7f8PmAC8QnIR4CJJrXd1kSVsMcn2uw7oQ4luWwc7rAIOI5ki4RygFfB+SxZU6kIIE8g/T9D6EMKDaZ9rgP2Aq3ZZcaVvcQhhYAjh5+k2/ATokdX+VgjhnhDCGcACkrPUzm2JQkvU5cBvgN8BW4AdWW0ls20d7MmkZS8CF6W3x4F1LVrRnuOpdFkyV/S1tBDC5zU/S6oAvgI8m6e7t2/jtQOqSTLhc5JPl7ns1tvWwZ64j+Tc/ONJPt7arlEz7YTPuW0kSV8DniC5UPCSfN3Spbdv4TYCJ5MMuZSTXJOTy269bR3siUdILriqIvmIZbvGKelyaYtWUWLSYxILSfYoTwohfJinq7dvI4UQtoUQFoQQ7gBeAk4kmQCsrt162/rLtIAQwmeSLgI2hBB2SF+av+xYSTenP28KIVy7ayssLZK+AxyV3j1E0ihgUXq/naQLSD7CjgX+Ctz85VexXCQdAjwDtAeuBgZIGgD8Je1yhKSRwKkke56vAg+3RK2lRtIpJMfZFgOHkEyL8hGwNu1SMtt2j73yVFJXkonL5ocQTq3T9ieSA1InkvwnyrY+hHDArqixVElaCAys8/AI/j7MtZXkP8sLwKQQwsu7rrrSJinDl38n4Yu/q5uBD4H/AK4OIazN0d/qkNSf5Hf0cJIDp/8JXEFygL+ktu0eG+xmZrHyGLuZWWQc7GZmkXGwm5lFxsFuZhYZB7uZWWQc7GZmkXGwm5lF5v8DA64L0oUbZwYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KRPpyukgDWog"
      },
      "source": [
        "# Result - Plots"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aiSCGyqdiNbO"
      },
      "source": [
        "## Valence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HxCiNnQlQWXv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dacd57ff-4dc9-4511-91c7-8063e4aa004c"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "def specificity(metric):\n",
        "  pos_list = [i[0][0] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s] \n",
        "\n",
        "def sensitivity(metric):\n",
        "  neg_list = [i[1][1] for i in metric]\n",
        "  neg_list_m = round(np.mean(neg_list),2)\n",
        "  neg_list_s = round(np.std(neg_list),2)\n",
        "  return [neg_list_m, neg_list_s] \n",
        "\n",
        "def fpr(metric):\n",
        "  fpr_list = [i[0][1] for i in metric]\n",
        "  fpr_list_m = round(np.mean(fpr_list),2)\n",
        "  fpr_list_s = round(np.std(fpr_list),2)\n",
        "  return [fpr_list_m, fpr_list_s]\n",
        "\n",
        "def fnr(metric):\n",
        "  fnr_list = [i[1][0] for i in metric]\n",
        "  fnr_list_m = round(np.mean(fnr_list),2)\n",
        "  fnr_list_s = round(np.std(fnr_list),2)\n",
        "  return [fnr_list_m, fnr_list_s]\n",
        "\n",
        "\n",
        "data = 'PD'\n",
        "print(\"-----\", data, \"----\")\n",
        "print(\"-----3D----\")\n",
        "hvlv_pd_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_3pd = sensitivity(hvlv_pd_3d)\n",
        "lv_lv_3pd = specificity(hvlv_pd_3d)\n",
        "hv_lv_3pd = fnr(hvlv_pd_3d)\n",
        "lv_hv_3pd = fpr(hvlv_pd_3d)\n",
        "print('Sensitivity:',hv_hv_3pd[0],'+-',hv_hv_3pd[1])\n",
        "print('Specificity:',lv_lv_3pd[0],'+-',lv_lv_3pd[1])\n",
        "print('FPR:',lv_hv_3pd[0],'+-',lv_hv_3pd[1])\n",
        "print('FNR:',hv_lv_3pd[0],'+-',hv_lv_3pd[1])\n",
        "\n",
        "print(\"-----2D----\")\n",
        "hvlv_pd_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_2pd = sensitivity(hvlv_pd_2d)\n",
        "lv_lv_2pd = specificity(hvlv_pd_2d)\n",
        "hv_lv_2pd = fnr(hvlv_pd_2d)\n",
        "lv_hv_2pd = fpr(hvlv_pd_2d)\n",
        "print('Sensitivity:',hv_hv_2pd[0],'+-',hv_hv_2pd[1])\n",
        "print('Specificity:',lv_lv_2pd[0],'+-',lv_lv_2pd[1])\n",
        "print('FPR:',lv_hv_2pd[0],'+-',lv_hv_2pd[1])\n",
        "print('FNR:',hv_lv_2pd[0],'+-',hv_lv_2pd[1])\n",
        "\n",
        "print(\"-----1D----\")\n",
        "hvlv_pd_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_HVLV_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_1pd = sensitivity(hvlv_pd_1d)\n",
        "lv_lv_1pd = specificity(hvlv_pd_1d)\n",
        "hv_lv_1pd = fnr(hvlv_pd_1d)\n",
        "lv_hv_1pd = fpr(hvlv_pd_1d)\n",
        "print('Sensitivity:',hv_hv_1pd[0],'+-',hv_hv_1pd[1])\n",
        "print('Specificity:',lv_lv_1pd[0],'+-',lv_lv_1pd[1])\n",
        "print('FPR:',lv_hv_1pd[0],'+-',lv_hv_1pd[1])\n",
        "print('FNR:',hv_lv_1pd[0],'+-',hv_lv_1pd[1])\n",
        "\n",
        "print(\"-----ML----\")\n",
        "hvlv_pd_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HVLV_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_pd_ml = sensitivity(hvlv_pd_ml)\n",
        "lv_lv_pd_ml = specificity(hvlv_pd_ml)\n",
        "hv_lv_pd_ml = fnr(hvlv_pd_ml)\n",
        "lv_hv_pd_ml = fpr(hvlv_pd_ml)\n",
        "print('Sensitivity:',hv_hv_pd_ml[0],'+-',hv_hv_pd_ml[1])\n",
        "print('Specificity:',lv_lv_pd_ml[0],'+-',lv_lv_pd_ml[1])\n",
        "print('FPR:',lv_hv_pd_ml[0],'+-',lv_hv_pd_ml[1])\n",
        "print('FNR:',hv_lv_pd_ml[0],'+-',hv_lv_pd_ml[1])\n",
        "\n",
        "data = 'NC'\n",
        "print(\"-----\", data, \"----\")\n",
        "print(\"-----3D----\")\n",
        "hvlv_nc_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_3nc = sensitivity(hvlv_nc_3d)\n",
        "lv_lv_3nc = specificity(hvlv_nc_3d)\n",
        "hv_lv_3nc = fnr(hvlv_nc_3d)\n",
        "lv_hv_3nc = fpr(hvlv_nc_3d)\n",
        "print('Sensitivity:',hv_hv_3nc[0],'+-',hv_hv_3nc[1])\n",
        "print('Specificity:',lv_lv_3nc[0],'+-',lv_lv_3nc[1])\n",
        "print('FPR:',lv_hv_3nc[0],'+-',lv_hv_3nc[1])\n",
        "print('FNR:',hv_lv_3nc[0],'+-',hv_lv_3nc[1])\n",
        "\n",
        "print(\"-----2D----\")\n",
        "hvlv_nc_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_2nc = sensitivity(hvlv_nc_2d)\n",
        "lv_lv_2nc = specificity(hvlv_nc_2d)\n",
        "hv_lv_2nc = fnr(hvlv_nc_2d)\n",
        "lv_hv_2nc = fpr(hvlv_nc_2d)\n",
        "print('Sensitivity:',hv_hv_2nc[0],'+-',hv_hv_2nc[1])\n",
        "print('Specificity:',lv_lv_2nc[0],'+-',lv_lv_2nc[1])\n",
        "print('FPR:',lv_hv_2nc[0],'+-',lv_hv_2nc[1])\n",
        "print('FNR:',hv_lv_2nc[0],'+-',hv_lv_2nc[1])\n",
        "\n",
        "print(\"-----1D----\")\n",
        "hvlv_nc_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_HVLV_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_1nc = sensitivity(hvlv_nc_1d)\n",
        "lv_lv_1nc = specificity(hvlv_nc_1d)\n",
        "hv_lv_1nc = fnr(hvlv_nc_1d)\n",
        "lv_hv_1nc = fpr(hvlv_nc_1d)\n",
        "print('Sensitivity:',hv_hv_1nc[0],'+-',hv_hv_1nc[1])\n",
        "print('Specificity:',lv_lv_1nc[0],'+-',lv_lv_1nc[1])\n",
        "print('FPR:',lv_hv_1nc[0],'+-',lv_hv_1nc[1])\n",
        "print('FNR:',hv_lv_1nc[0],'+-',hv_lv_1nc[1])\n",
        "\n",
        "print(\"-----ML----\")\n",
        "hvlv_nc_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HVLV_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "hv_hv_nc_ml = sensitivity(hvlv_nc_ml)\n",
        "lv_lv_nc_ml = specificity(hvlv_nc_ml)\n",
        "hv_lv_nc_ml = fnr(hvlv_nc_ml)\n",
        "lv_hv_nc_ml = fpr(hvlv_nc_ml)\n",
        "print('Sensitivity:',hv_hv_nc_ml[0],'+-',hv_hv_nc_ml[1])\n",
        "print('Specificity:',lv_lv_nc_ml[0],'+-',lv_lv_nc_ml[1])\n",
        "print('FPR:',lv_hv_nc_ml[0],'+-',lv_hv_nc_ml[1])\n",
        "print('FNR:',hv_lv_nc_ml[0],'+-',hv_lv_nc_ml[1])\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----- PD ----\n",
            "-----3D----\n",
            "Sensitivity: 0.85 +- 0.13\n",
            "Specificity: 0.95 +- 0.04\n",
            "FPR: 0.05 +- 0.04\n",
            "FNR: 0.15 +- 0.13\n",
            "-----2D----\n",
            "Sensitivity: 0.75 +- 0.13\n",
            "Specificity: 0.92 +- 0.04\n",
            "FPR: 0.08 +- 0.04\n",
            "FNR: 0.25 +- 0.13\n",
            "-----1D----\n",
            "Sensitivity: 0.74 +- 0.06\n",
            "Specificity: 0.92 +- 0.03\n",
            "FPR: 0.08 +- 0.03\n",
            "FNR: 0.26 +- 0.06\n",
            "-----ML----\n",
            "Sensitivity: 0.67 +- 0.03\n",
            "Specificity: 0.94 +- 0.01\n",
            "FPR: 0.06 +- 0.01\n",
            "FNR: 0.33 +- 0.03\n",
            "----- NC ----\n",
            "-----3D----\n",
            "Sensitivity: 0.89 +- 0.07\n",
            "Specificity: 0.94 +- 0.04\n",
            "FPR: 0.06 +- 0.04\n",
            "FNR: 0.11 +- 0.07\n",
            "-----2D----\n",
            "Sensitivity: 0.86 +- 0.12\n",
            "Specificity: 0.94 +- 0.04\n",
            "FPR: 0.06 +- 0.04\n",
            "FNR: 0.14 +- 0.12\n",
            "-----1D----\n",
            "Sensitivity: 0.82 +- 0.08\n",
            "Specificity: 0.92 +- 0.03\n",
            "FPR: 0.08 +- 0.03\n",
            "FNR: 0.18 +- 0.08\n",
            "-----ML----\n",
            "Sensitivity: 0.73 +- 0.02\n",
            "Specificity: 0.91 +- 0.01\n",
            "FPR: 0.09 +- 0.01\n",
            "FNR: 0.27 +- 0.02\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u6SBmWVOA4_s",
        "outputId": "9f81b8fe-3b19-49a7-97e7-4ccef5097293"
      },
      "source": [
        "data1 = {'Sensitivity': [i[1][1] for i in hvlv_nc_ml], 'Specificity': [i[0][0] for i in hvlv_nc_ml],\n",
        "          'Data': len([i[1][1] for i in hvlv_nc_ml])*['NC'], 'Model': len([i[1][1] for i in hvlv_nc_ml])*['ML']}\n",
        "df1 = pd.DataFrame(data1)\n",
        "data2 = {'Sensitivity': [i[1][1] for i in hvlv_pd_ml], 'Specificity': [i[0][0] for i in hvlv_pd_ml],\n",
        "          'Data': len([i[1][1] for i in hvlv_pd_ml])*['PD'], 'Model': len([i[1][1] for i in hvlv_pd_ml])*['ML']}\n",
        "df2 = pd.DataFrame(data2)\n",
        "data3 = {'Sensitivity': [i[1][1] for i in hvlv_pd_1d], 'Specificity': [i[0][0] for i in hvlv_pd_1d],\n",
        "          'Data': len([i[1][1] for i in hvlv_pd_1d])*['PD'], 'Model': len([i[1][1] for i in hvlv_pd_1d])*['1D CNN']}\n",
        "df3 = pd.DataFrame(data3)\n",
        "data4 = {'Sensitivity': [i[1][1] for i in hvlv_nc_1d], 'Specificity': [i[0][0] for i in hvlv_nc_1d],\n",
        "          'Data': len([i[1][1] for i in hvlv_nc_1d])*['NC'], 'Model': len([i[1][1] for i in hvlv_nc_1d])*['1D CNN']}\n",
        "df4 = pd.DataFrame(data4)\n",
        "data5 = {'Sensitivity': [i[1][1] for i in hvlv_pd_2d], 'Specificity': [i[0][0] for i in hvlv_pd_2d],\n",
        "          'Data': len([i[1][1] for i in hvlv_pd_2d])*['PD'], 'Model': len([i[1][1] for i in hvlv_pd_2d])*['2D CNN']}\n",
        "df5 = pd.DataFrame(data5)\n",
        "data6 = {'Sensitivity': [i[1][1] for i in hvlv_nc_2d], 'Specificity': [i[0][0] for i in hvlv_nc_2d],\n",
        "          'Data': len([i[1][1] for i in hvlv_nc_2d])*['NC'], 'Model': len([i[1][1] for i in hvlv_nc_2d])*['2D CNN']}\n",
        "df6 = pd.DataFrame(data6)\n",
        "data7 = {'Sensitivity': [i[1][1] for i in hvlv_pd_3d], 'Specificity': [i[0][0] for i in hvlv_pd_3d],\n",
        "          'Data': len([i[1][1] for i in hvlv_pd_3d])*['PD'], 'Model': len([i[1][1] for i in hvlv_pd_3d])*['3D CNN']}\n",
        "df7 = pd.DataFrame(data7)\n",
        "data8 = {'Sensitivity': [i[1][1] for i in hvlv_nc_3d], 'Specificity': [i[0][0] for i in hvlv_nc_3d],\n",
        "          'Data': len([i[1][1] for i in hvlv_nc_3d])*['NC'], 'Model': len([i[1][1] for i in hvlv_nc_3d])*['3D CNN']}\n",
        "df8 = pd.DataFrame(data8)\n",
        "\n",
        "valence_df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], axis=0).reset_index()\n",
        "print(valence_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    index  Sensitivity  Specificity Data   Model\n",
            "0       0     0.696000     0.892116   NC      ML\n",
            "1       1     0.736000     0.917012   NC      ML\n",
            "2       2     0.736000     0.898340   NC      ML\n",
            "3       3     0.712000     0.908714   NC      ML\n",
            "4       4     0.724000     0.900415   NC      ML\n",
            "..    ...          ...          ...  ...     ...\n",
            "75      5     0.956693     0.971253   NC  3D CNN\n",
            "76      6     0.913386     0.963039   NC  3D CNN\n",
            "77      7     0.933071     0.977413   NC  3D CNN\n",
            "78      8     0.937008     0.981520   NC  3D CNN\n",
            "79      9     0.964567     0.956879   NC  3D CNN\n",
            "\n",
            "[80 rows x 5 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "id": "UFQKOZ4fQItr",
        "outputId": "a3630f75-cfd9-4b0e-a403-ba5e4d345f50"
      },
      "source": [
        "x = \"Model\"\n",
        "y = \"Sensitivity\"\n",
        "hue = \"Data\"\n",
        "sns.set_style(\"white\")\n",
        "ax = sns.barplot(data=valence_df, x=x, y=y, hue=hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.4)\n",
        "ax.legend(loc='upper left', bbox_to_anchor=(0, 1))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Sensitivity',fontsize = 13, weight='bold' )\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight = 'bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "ax.set_yticklabels([\"0.4\", \"0.5\", \"0.6\", \"0.7\", \"0.8\", \"0.9\", \"1.0\"], size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=valence_df, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"PD\"), (\"ML\", \"NC\")),\n",
        "                                 ((\"1D CNN\", \"PD\"), (\"1D CNN\", \"NC\")),\n",
        "                                 ((\"2D CNN\", \"PD\"), (\"2D CNN\", \"NC\")),\n",
        "                               ((\"3D CNN\", \"PD\"), (\"3D CNN\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "\n",
        "plt.savefig('sensitivity_valence.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_NC v.s. 1D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.059e-01 stat=2.417e+00\n",
            "ML_NC v.s. ML_PD: t-test independent samples with Bonferroni correction, P_val=1.600e-04 stat=5.393e+00\n",
            "2D CNN_NC v.s. 2D CNN_PD: t-test independent samples with Bonferroni correction, P_val=2.958e-01 stat=1.897e+00\n",
            "3D CNN_NC v.s. 3D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=1.042e+00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEJCAYAAAB/pOvWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de1xU9b7/8dcwwKAoWZiQgJU3tkUQmYqa5PGUeYFURLyglqa265Gc2mrJL8q9Te10MQ/bXSEna2vu5OAlC0vkdFGybdgx3RoamJY6CCZoqSiDwvz+MCZxQEeDGS7v5+Phw5m1vjPfz4w+ePNd67u+y2C1Wq2IiIhcxM3VBYiISMOjcBARETsKBxERsaNwEBEROwoHERGx4+7qAupCr169CAgIcHUZIiKNSkFBATk5OTXuaxLhEBAQwNq1a11dhohIoxITE1PrPh1WEhEROwoHERGxo3AQERE7TeKcQ03OnTuH2WymrKzM1aVcMy8vLwIDA/Hw8HB1KSLSzDTZcDCbzbRu3ZpbbrkFg8Hg6nKumtVqpaSkBLPZzK233urqckSkmWmyh5XKysrw9fVtlMEAYDAY8PX1bdQjHxFpvJpsOACNNhiqNPb6RaTxatLhICIi16bZhENFZaXT3y84OJj//M//tD1funQpixcvtj1ft24dUVFRREdHM3z4cJYuXVqnNYqIXKsme0L6UkY3N9bv+KHO3i8q/MoniT09PcnKymLatGnccMMN1fZt3ryZZcuWsXTpUvz8/CgvL2fdunV1Vp9ITcxmM1OnTqV79+7s2LEDPz8/3njjDdLT00lLS8NoNNK5c2cWLVrk6lLFxZrNyMEV3N3dGT16NMuWLbPbl5qaytNPP42fnx9wIUji4uKcXaI0QwcPHiQ+Pp6PPvqI1q1bs3HjRlJTU1m3bh0ZGRn85S9/cXWJ0gAoHOpZfHw8GRkZnDp1qtr2ffv2ERIS4qKqpDkLDAykW7duANx+++0UFBQQHBzMzJkz+eCDDzAajS6uUBoChUM9a9WqFcOGDWP58uWuLkUEuDBKrWI0GqmoqCA1NZVx48axZ88eYmNjOX/+vAsrbLjMZjODBw8mKSmJoUOHMnnyZMrKyli+fDlDhgwhOjqap556ytVl1gmFgxM89NBDrFmzhrNnz9q2de7cmW+//daFVYlcUFlZSWFhIREREcycOZNTp05x5swZV5fVYDWXw3IKBydo06YNgwYNYvXq1bZtjz76KK+88grHjh0DoLy8nFWrVrmqRGnGKisrmTVrFtHR0YwYMYKJEyfi4+Pj6rIarOZyWK7ZzFaqqKx0aIbR1byf0c3xbJ08eTL/+Mc/bM/vvfdeiouLmTRpElarFYPBwMiRI+usPpGaBAYGsn79etvzRx55xIXVNE6XHpazWCykpqby9ddf8/nnn5OSkkJGRgbu7o37x2vjrv4qXM0P8rp6vx07dtget23bln/961/V9o8cOVKBINLIXXxYrnv37nz00UecOXOm0Y++mk04iDRFa9euJTMz06U1DBo06LJ3FGvqqg7LnT59GqvV2mQOyykcRBqxzMxM8vPz6dq1q0v6z8/PBy5/u8mmpDkdllM4iDRyXbt2JTU11SV9T5s2zSX91hWNvGrnlNlK8+bNo0+fPgQHB/Poo4/W2m779u1ER0cTEhLCiBEjyM3NdUZ5ItJMVY28XCU/P9/l4VQbp40chgwZwrvvvlvrfovFQkJCAiaTicTERFJSUkhISCArK6vJTA0TqWsPPvhgs+6/LmjkVTOnhENSUhJms/my4ZCdnU1xcTGzZs0iPj6e4uJi3njjDbZt20bv3r2dUaZIoxMVFdWs+5f602DOOZjNZgDbQnT+/v4AHD58uE7Cofx8BZ7udTcCceT9unXrRteuXamoqKBjx4689NJLtGjRwrb9/PnzGI1Ghg8fzsMPP4xbHU+3FRG5Vg0mHC5ltVrr9P083Y0Mnl93S2JveHb4Fdt4eXnxwQcfADBjxgzS0tKYNGlSte0lJSXMmDGD06dPk5CQUGf1iciVufqwmKv7vxyXhkN5eTlw4YrDwMBAAIqKigA4evQoAEFBQa4pro7dfffd5OXl2W339fXlhRdeIDY2lunTp+vWoCJO5OrDYq7u/3KcEg6bNm2yzQgoLCxk1apV9OjRg8mTJ3PixAl27NhBZGQkvr6+rFy5Em9vb1avXk1AQAA9e/Z0Ron16vz582RnZ9OvX78a9wcFBVFRUUFJSQlt27Z1cnUiIvaccpB76dKlLFy4EIC8vDySkpL45ptvqrUxmUwkJyfj7e3NggUL8PX1JTk5uVHPVCorK2PYsGGMHDmS9u3bExsb6+qSREQc4pSRQ22zlC698KNHjx5kZGQ4oySnuPjcwuUcPnwYo9GIr6+vE6oSEbkyTY9xsePHjzNnzhzi4+N1vkFEGowGO1uprpWfr3BohtHVvN+1To2tOtxUNZV12LBhTJo0qc5qExH5vZpNONTlNQ6Ovt/FS3ZfbO/evXVai4hIXdNhJRERsaNwEBERO006HOr6Kmtna+z1i0jj1WTDwcvLi5KSkkb7A9ZqtVJSUoKXl5erSxGRZqjJnpAODAzEbDZz7NgxV5dyzby8vGzLioiIOFOTDQcPDw9uvfVWV5chItIoNdlwkKbHbDYzdepUunfvzo4dO/Dz8+ONN94gPT2dtLQ0jEYjnTt3ZtGiRa4uVaTRUzhIo3Lw4EFee+015s2bx3/8x3+wceNGUlNT+eyzz/D09OTkyZOuLlGkSWiyJ6SlaQoMDKRbt24A3H777RQUFBAcHMzMmTP54IMPGvVCjSINicJBGhVPT0/bY6PRSEVFBampqYwbN449e/YQGxvL+fPnXVihSNOgcJBGrbKyksLCQiIiIpg5cyanTp3izJkzri5LpNHTOQdp1CorK5k1axanT5/GarUyceJEfHx8XF2WSKOncJBGIzAwkPXr19ueP/LIIy6sRqRpUziIy6xdu5bMzEyX1jBo0CC7m06JiM45iAtlZmba7i3uCvn5+S4PJ5GGSiMHcamuXbuSmprqkr6nTZvmkn5FGgONHERExI5GDiLS7Hz11Vf8/e9/5/jx45w6dYrWrVtzww038PDDDxMREeHq8hoEhYO4zIMPPtis+xfXSU9PZ9++fbbnJSUllJSUkJ6ernD4lcJBXCYqKqpZ9y+uExcXx5kzZzhw4AAVFRUYjUY6duxIXFycq0trMJx2zmH79u1ER0cTEhLCiBEjyM3NtWtz7tw55s+fT58+fejevTt//vOfOXfunLNKFJFmIiIigpSUFPz9/QHw9/cnJSVFo4aLOGXkYLFYSEhIwGQykZiYSEpKCgkJCWRlZVVbKO3dd99l+fLlxMXF0aJFC5YtW0aHDh2YPHmyM8oUaTR0zFzqm1NGDtnZ2RQXFzNu3Dji4+OJjY3FbDazbdu2au2qnj/xxBPMnDkTgPfff98ZJYo0KlXHzEtKSigvL6ekpIR9+/aRnp7u6tKkiXBKOJjNZgD8/PwAbEO5w4cPV2t3ww03ALB161a++OKLaq8Vkd/ExcXRpUsX28jbaDTSpUsXHTOXOuOSE9JWq7XG7VOnTuWLL77gmWeewd3dHS8vL0wmk5OrE2n4IiIiiIiIYOLEiRQUFNiOmYvUFaeEQ2BgIABFRUUAHD16FICgoCAsFgtubm62ez5nZWWRn5+Pj48Po0aNonPnzs4oUUQaqYrKSoxurrue19X91xenhENkZCS+vr6sXLkSb29vVq9eTUBAAAEBAYSGhtK/f3+WLFnC3r17+eyzz/D39+ejjz7i1KlTOhktIpdldHNj/Y4frum1pZbztr+v9T2iwm+96tc0hgkFTok7k8lEcnIy3t7eLFiwAF9fX5KTk3GrIW3XrFnDnDlz+PHHH5kzZw4DBgxwRokiIk7TGCYUOO2cQ48ePcjIyLDbnpeXZ3vcrVs3PvvsM2eVJCLiEo3hIjxdIS0i4mSNYUJB0zuLIiIiv5vCQUSanQN7dvGPRXM5ebwYgJPHi/nHorkc2LPLxZU1HAoHEWl2tm/K5KeCQ1RWVgBQWVnBTwWH2L55o4srazh0zkFEmp3u/QdRbimj9NRJLGdLMbXwxru1D93vfcDVpTUYCgcRaXY63hZKx9tCXV1Gg6bDSiIuVFFZ6eoSGkQN0vBo5CDiQr/n6l5w3RW+0vRp5CAiInYUDiIiYsfhcPi///u/+qxDREQaEIfDYfz48TzwwAMsWbLEtuS2iIg0TQ6HQ4sWLTh48CD/9V//xYABA5g6dSqZmZmcO3euPutrEqpubrR48eJan9fWRkQatvLzFU2yf4dnK3311Vds3ryZDRs2sGnTJr744gu2bNmCj48P0dHRTJw4kQ4dOtRLkY3dhx9+yLFjx7BYLPz3f/837dq1s3sO2G0bNmyYiyuvH41hLXsRR3m6Gxk8f901vbb8eCkABcdLr/k9Njw7/JpedyUOjxxMJhMDBw5k0aJFfPrpp/Tv3x+r1covv/zCihUrGDJkCB999FG9FNnYDRs2DH9/f5YuXUr79u1rfF7TtqaqMaxlL9LcXdVspd27d/P8888zcOBANm/eDED79u2ZMGECrVu3ZuHChfVSZGOXkZFBUVERjzzyCEeOHKnxeU3bmqq4uDi6dOmC0WgEwGg00qVLlwa1lr1Ic+fwYaXhw4eTl5dnOxYeERHB+PHjGTBgAG5ubra7u4m9qKgoDAYDixcvZurUqVitVrvnQI3bmqLGsJa9SHPncDh89913tGzZkmHDhjF+/Hg6depUbX/v3r05f/58nRfYFBgMBgCmT59+2ee1bRO51IE9u9i6cZ3dktO9HxiuNYOkTjgcDs8++ywxMTF4e3vXuD8sLIywsLA6K0xEale15HSVi5ecVjhIXXD4nMOCBQts0yyr/P3vfyc2NrbOixKRy+vefxDtAjrg7dMGdw8PvH3a0C6gg5acljrj8Mjh4rn4VQ4fPkxubm6dFyUil6clp6W+XTEcEhMTbY+//PJL23Or1crnn3+Ol5dX/VXXwK1du5bMzEyX1jBo0CBiYmJc1n9FZSVGN9ct0eXq/kWaqiuGw/vvv4/BYMBgMLB//36+//77avt79+5db8U1dJmZmeTn59O1a1eX9J+fnw/g0nD4PUtOa7lpkYbriuEwYsQI4EJIdOrUidDQC0NZo9FI+/btGTNmjEMdbd++nT//+c/88MMPdOnShXnz5nH77bdXa1NeXs5f/vIXPv30U86cOUPHjh155plnGnQAde3aldTUVJf0PW3aNJf0KyJN3xXD4cUXXwTAbDYzaNAg4uPjr7oTi8VCQkICJpOJxMREUlJSSEhIICsry3YhFMC6detYvXo1ffr04Z577uGVV17hueee45NPPrnqPkVEGqrK4h+o+CEHyk5e2FB2knNfp2G8tRdubRvGaNjhg7XvvvvuNQUDQHZ2NsXFxYwbN474+HhiY2Mxm81s27atWrvKX29X2KVLF/r06YOnpyc+Pj7X1KeISENVcWgH1lPHwPrrLVqtlVhPHaPi0A7XFnaRy44cunXrxkMPPcTs2bPp1q1bjW0MBgN79uy5bCdmsxkAPz8/APz9/YELs50uPmQ0YsQItmzZwrJly1i2bBlt2rSxjVwaogcffLBZ9y8i18bYIZyKinKsljNwvgzcvTCYWmLsEO7q0mwuGw4XT1+ty+UcanuvnTt3smnTJqKjoxkwYADPP/88iYmJrFmzpkFeMRwVFdWs+79WurpXmju3trc2mMNHtblsOCxfvtz2W/7y5cuvuZPAwEAAioqKAGw3CwoKCsJiseDm5oaHh4ft/hBjxozh7rvvJj09na1bt3LixAluuOGGa+5fGhZd3dv4adn1pu+y4dCzZ0/bY29vb7vZRY6KjIzE19eXlStX4u3tzerVqwkICCAgIIDQ0FD69+/PkiVLCAoKAuCtt94iLy+PnTt30qZNG66//vpr6lcapu79B1FuKaP01EksZ0sxtfDGu7WPru5tRKqWXa9SUlJCSUkJ6enpCocmwuErpGNjY7ntttsYPXo0UVFRtGzZ0uFOTCYTycnJzJ07lwULFtC5c2deeOEF3C65eCk+Pp79+/fz+eefs3XrVjp16sTTTz/dIA8pybXT1b2NX1xcHGfOnOHAgQNUVFRgNBrp2LGjll1vQhwOB6PRSG5uLnPmzOGll15i6NChxMXFERIS4tDre/ToUeM9CvLy8myPTSYT8+fPd7QkEXERLbve9Dk8lfWLL74gKSmJkJAQSktLSU9PZ9SoUS69OldEROqHw+Fw/fXXM378eFatWsXHH39MREQEVquVvXv31md9IiLiAg4fVoIL1yV88MEHfPjhhxw+fBig2hXOIiLSNDgcDmPHjmXnzp3AhesU2rdvT2xsrO7nICLSBDkcDjt27MDd3Z1/+7d/Iy4ujnvuuUeziEREmiiHw+FPf/oTMTExtG3btj7rERGRBsDhcNDy0NJU6OpekStzysJ7Ig2Jru4VuTKXLLwn4kq6ulfkypyy8J5IQ6Kre0WuzOGF944cOUKnTp244447bNtKSko4ceJE/VUnIiIu4fAV0rNnz2b9+vXVti1ZsoTo6Og6L0pEnKf8fEWz7l9qdlVXSF+q6v4MItJ4ebobGTx/3TW9tvx4KQAFx0uv+T02PDv8ml4n9euK4VA1S8lgMLB8+XK7cw+6CY+4Uvn5CjzdXbeEi6v7F6kvVwyHqllKBoOhxhlL48aNq/uqGjjNk2849FuvSP24YjgsX74cq9XKQw89xAMPPEB8fDxwYcG9gIAA22ym5kTz5EWkqbtiOFTNWHrxxRfp3LlztdlKzZXmyYtIU3fZcFi3bp1t+qrBYGD//v3s37/frt3w4Y13aF1RWYnRzeFJW0DdzpO/lv5FROrbZcNh9uzZPPzww9xxxx3Mnj27xlVYDQZDow4Ho5sb63f8cFWvObBnF1s3rqO48MJsrcLCIsZOnETvB4Zf9b2Ro8Jvvar28vtVFv9AxQ85UHbywoayk5z7Og3jrb1wa6t/DxG4Qji0b98eHx8f22O5YPumTH4qOGR7XllZwU8Fh9i+eeNVh4M4X8WhHVhPHfttg7US66ljVBzaoXAQ+dVlw+Gzzz6r8XFz173/IMotZZSeOonlbCmmFt54t/ah+70PuLo0cYCxQzgVFeVYLWfgfBm4e2EwtcTYIdzVpYk0GNd0EVx5eTkbN26kTZs29OvXr65ravA63haqEUIj5tb2Vo0QRK7A4TOhjz/+OL169aKyspKEhASefvpppk2bxttvv12f9YmIiAs4HA55eXmEh4dTVlZGdnY2HTp0wGQysWrVKodev337dqKjowkJCWHEiBHk5ubatZk9ezbBwcHV/gwYMMDxTyMiTlFZ/APnvk6zO6lfWXx1kzuk4XI4HI4dO8ZNN93EDz/8gNVqZeHChQwZMsSh9ZUsFgsJCQmUlpaSmJhIcXExCQkJVFRUX3Br7NixvPbaa7z22ms88cQTANx2221X+ZFEpL7ZTupbKy9suOikvjQNDoeDyWTiyJEjbN++HYPBQKdOnTAajTVOb71UdnY2xcXFjBs3jvj4eGJjYzGbzWzbtq1au7CwMIYOHcrQoUNtS4GPGTPmKj+SiNQ3Y4dwDK1vBE9vcDOCpzeG1jfqpH4T4vAJ6S5dupCdnU12djZdunShRYsWFBQU4Ofnd8XXms1mAFvbqiU3Dh8+TO/eve3anz17lg8//JCbb76Zvn37OlqiiDiJTuo3fQ6PHGbMmIG/vz++vr7MnDmT8vJyCgsLiYyMvOpOr3TL0Y8++ohTp04RFxfn0MhERETqlsMjh+7du/P5559X27ZhwwaHXhsYGAj8dv+Ho0ePAhAUFITFYsHNzQ0PDw9b+7S0NDw9PYmJiXG0PBERqUNXdZ1DQUEBe/bsobS0tNr2Ky2fERkZia+vLytXrsTb25vVq1cTEBBAQEAAoaGh9O/fnyVLlgCwZ88edu/eTXR0tO4VISLiIg6HQ3p6OnPnzrWbYQRXDgeTyURycjJz585lwYIFdO7cmRdeeAG3GhacS0tLA3QiWkTElRwOh9dff53z58/TqlUr23pLV6NHjx5kZGTYbc/Ly6v2fO7cucydO/eq319EROqOw+Fw8uRJ+vbty1tvvaWTxC6mO9GJSH1zeLZS1RpKCgbXq7oTXUlJCeXl5ZSUlLBv3z7S09NdXZqINBEOjxx8fX353//9XyZPnkxoaCju7r+9tOpqZnEO3YlOROqbw+GwcuVKDAYD//znP9m6dWu1fQoH56rLO9GJiNTE4XDo0aNHfdbRbJWfr8DT3dhs+xeRhsnhcHj33Xfrs45my9PdyOD5667qNVW3ubSeLgGg4Egh9w0fc023udzwbOO9xauI1J+rurP9mTNnWLduHSkpKZSXl3PkyBHOnj1bX7VJLbQipojUN4dHDkePHmXs2LEUFhYCEBMTw/3338/DDz/MrFmz6q1AsafbXIpIfXM4HBYuXMiRI0cwmUyUl5fTrl07wsPD2bJli8LBybQipojUN4cPK+Xk5NC3b19Gjx5t23bLLbdQUFBQL4WJiIjrOBwOp06dol27dtW2nT17VhfFiYg0QQ6Hwy233MKWLVtsN+7JyMjg008/pUOHDvVWnIiIuIbD4TB27FiOHTtmu6fD008/jcVi0eqpIiJNkMPhEBsby5QpU/Dy8sJqtWIymZg0aRKjRo2qz/pERMQFHJ6tZDAYmDlzJgkJCWzcuJFWrVpx55131mdtIiLiIlcMh5SUFLZu3cpf//pXrrvuOubMmcO6dReu6L3uuut46623CAkJqfdCRUTEea54WCkrK4vjx49z3XXX8eOPP/L+++9jtVqxWq38/PPPvP76686oU0REnOiK4XDkyBGCg4MB+PLLLwEICwtj27ZtdOvWjd27d9dvhSIi4nRXDIfTp0/TunVrAHbv3o3BYGDw4MH4+Phw55138ssvv9R7kSIi4lxXDAdfX19ycnLYu3evbeRQdSK6pKTEFhwiItJ0XDEcIiIiOHDgADExMRQXF+Pr60tYWBgAubm5BAUF1XuRIiLiXFcMhz/96U/cfvvtWK1WvL29mTdvHgaDgZycHAoKCrj77rudUaeIiDjRFaey+vn5sWbNGk6ePIm3tzdG44W7hnXv3p1vvvkGk8lU70WKiIhzOXyFtI+Pjy0YANzd3WnZsmW1bZezfft2oqOjCQkJYcSIEeTm5tbYLj8/n4kTJxIaGkqvXr14+eWXHS1RRETqyFXdCe5aWSwWEhISKC0tJTExkeLiYhISEqioqKjWrqysjClTppCXl0dCQgIJCQm0bNnSGSWKiMhFHF4+4/fIzs6muLiYWbNmER8fT3FxMW+88Qbbtm2jd+/etnbr16/n6NGjzJs3j+joaLy8vJxRnoiIXMIpI4eqZb79/PwA8Pf3B+Dw4cPV2u3fvx+Ad955h7CwMCIiIvj444+dUaKIiFzEKeFwKavVWuP28vJyAG688UYWL16Mp6cniYmJnD592pnliYg0e04Jh8DAQACKiooAOHr0KABBQUFYLBbOnTtXrd3gwYMZOHAgd999N2VlZfz000/OKFNERH7llHCIjIzE19eXlStX8t5777F69WoCAgIICAggNDSUJ554AoAhQ4bg6enJ2rVrSU9PZ+vWrfj5+elucyIiTuaUcDCZTCQnJ+Pt7c2CBQvw9fUlOTkZN7fq3fv5+bFw4UKOHz/O/Pnzufnmm3nzzTdxd3fKeXMREfmV037q9ujRg4yMDLvteXl51Z4PHDiQgQMHOqssERGpgUtOSIuISMOmcBARETsKBxERsaNwEBEROwoHERGxo3AQERE7CgcREbGjcBARETsKBxERsaNwEBEROwoHERGxo3AQERE7CgcREbGjcBARETsKBxERsaNwEBEROwoHERGxo3AQERE7CgcREbGjcBARETsKBxERsaNwEBEROwoHERGxo3AQERE7TguH7du3Ex0dTUhICCNGjCA3N9eujdlsJjg4uNqf+fPnO6tEERH5lbszOrFYLCQkJGAymUhMTCQlJYWEhASysrIwGo127ceMGUPPnj0BuPXWW51RooiIXMQp4ZCdnU1xcTGzZs0iPj6e4uJi3njjDbZt20bv3r3t2oeEhDBgwABatGjhjPJEROQSTjmsZDabAfDz8wPA398fgMOHD9fY/rnnnuPOO+9k6NCh7Ny50xkliojIRZwycriU1WqtcXvLli2ZPn063bp14+DBgyxcuJCZM2fyySefOLlCEZHmzSnhEBgYCEBRUREAR48eBSAoKAiLxYKbmxseHh7ccMMNPPHEE7bXrV+/ntzcXCwWCyaTyRmliogITgqHyMhIfH19WblyJd7e3qxevZqAgAACAgIIDQ2lf//+LFmyhPT0dHbv3k1oaChms5m9e/fyhz/8QcEgIuJkTjnnYDKZSE5OxtvbmwULFuDr60tycjJubtW7v+WWW8jLy2PBggW899579OvXj+TkZGeUKCIiF3HaOYcePXqQkZFhtz0vL8/2uGfPnqSnpzurJBERqYWukBYRETsKBxERsaNwEBEROwoHERGxo3AQERE7CgcREbGjcBARETsKBxERsaNwEBEROwoHERGxo3AQERE7CgcREbGjcBARETsKBxERsaNwEBEROwoHERGxo3AQERE7CgcREbGjcBARETsKBxERsaNwEBEROwoHERGxo3AQERE7TguH7du3Ex0dTUhICCNGjCA3N7fWtvv37+eOO+4gODiYzMxMZ5UoIiK/cko4WCwWEhISKC0tJTExkeLiYhISEqioqLBra7VaSUpKwmg0OqM0ERGpgVPCITs7m+LiYsaNG0d8fDyxsbGYzWa2bdtm1/a9997jyJEjjB492hmliYhIDdyd0YnZbAbAz88PAH9/fwAOHz5M7969be2OHj3Ka6+9xquvvsq3337r8PsXFBQQExNThxU7z9tACxf2HxOz3IW9/376/n4/fYe/T2P+/goKCmrd55RwuJTVaq1x+6uvvkpISAgdO3bkyy+/BKC4uJjS0lK8vb1rfb+cnJx6qVNEpLlySjgEBgYCUFRUBFwYIQAEBQVhsVhwc3PDw8ODoqIitm3bxjObVWQAAApvSURBVMCBA22vfeGFF2jdujXDhg1zRqkiIoKTwiEyMhJfX19WrlyJt7c3q1evJiAggICAAEJDQ+nfvz9Llixh+vTpHD9+HIANGzaQmZnJ5MmT6dGjhzPKFBGRXznlhLTJZCI5ORlvb28WLFiAr68vycnJuLlV775nz54MGjSIQYMG0blzZwDCwsJo3769M8oUEZFfGay1nQAQEZFmS1dIi4iIHYWDiIjYUTg4gdlsJjg4mODgYF5//XXb9v/3//6fbXtOTg7BwcHMnTvXhZXWv3nz5tGnTx+Cg4N59NFHq+2r+i5CQkLo27cvjz/+OHv37q31vUpKSkhMTKRPnz6EhoYSHR1NVlYWAAMGDCA4OJjFixcDv/0bVPW5ePFigoODuf/++21X6k+YMIHg4GDbpIiG5scff2TChAn06tWL8PBwJk2axKFDhwBs/3+Cg4MJCwvj3//933n++ecv+1m+/fZbHnnkEe666y7uuusuxo8fz/79+6v9f62aJr527VqCg4NZunQp8Nt3NXv2bNv7BQcHExUVVY/fwO83atQowsPDCQsLIyYmhq+//hrQ91cThYOTvf/++1itVkpLS9mwYYOry3GJIUOG1LrP39+fefPmMXjwYDZv3sy4ceP4/vvv7dqVlZUxfvx41q5dS69evXjuuefo3bu33ZpdK1asoLS0tNb+Dh06xMcff3ztH8aJjh49itVqZfr06YwcOZJ//vOfPPvss9Xa3HPPPSQlJdG1a1f+53/+hwkTJlBWVmb3Xvv27WP8+PHk5OQwduxYnnnmGdq2bcuPP/5Yrd2SJUsuW9P69es5cuTI7/5szhIeHk5SUhKPP/443333HUlJSdX26/v7jUsugmuugoKCOHz4MDk5OZjNZs6fP4+fn5/tuo/mICkpCbPZzLvvvlvj/tatWzN8+HCGDx9O27ZtWbRoEampqbz88svV2mVkZHDgwAHuueceFi1aZNteWVlpe+zh4cHZs2dJS0vjgQceqLG/Vq1akZqa2qB+Y6tNeHg4K1assD3PyMiwC86bb76ZUaNGMWrUKCZPnsyXX37J+vXriY2NrdYuNTWVs2fPMmPGDKZNmwbA6NGjqaystP2watWqFV9++SW7d++utSaTycRbb73F888/X1cfs14lJiZy4sQJzGYzb775JgaDodp+fX+/0cjBiTp16kRYWBhr1qxhzZo13Hffffj4+Li6rAYrMjISoMalVKpGCP369au2/eLp0R4eHowcOZJ33nmH8vLyGvsYM2YM+fn5fPbZZ3VVdr3x9PS0Pd69ezc///zzZa8B+r3fX/fu3enatetlf/sdM2YMa9asoaSkxLEP4WKnTp2id+/ejBo1Cg8PD+bNm1dr2+b+/SkcnGzkyJFkZmbyzTffNNr1oJylapb1pb/d1batJlOmTOHEiROsXbu2xv133HEHffr0ITU19doLdbL9+/fz2GOPERAQYHdY5GK/9/szGAxMnTqVTz75hP3799fYZuzYsZhMJt555x0Hq3etli1b8vbbb5OUlITFYuGvf/1rrW2b+/encHCyoUOHYjQa8ff3p2/fvq4up0HbsmULALfffrvdvqptVWtwVbn4sBJAQEAAUVFRrFy5stZ+Hn30UXbu3MmePXt+b8n17vvvv2fixIl4enqybNky2rVrV2tbR76/qjZVLv3+hg4dSmBgIGlpaTX20bJlS8aPH3/Z77chcXd3p2/fvkyYMIHQ0FBycnJqPenc3L8/nXNwslatWrFgwQK8vb3trhAH2LlzJ6+++ioAXl5ePPHEE84usV5t2rSJ/Px8AAoLC1m1ahU9evTglltuAS4M+9etW8e3335LWloaLVu2tB3TvVhUVBRLly4lOzubGTNmEBERwYEDB/D09OSpp56q1nbatGl88MEHtdYUERHBnXfeyc6dO+vug9aDwsJCJk6cyM8//8yTTz7Jrl272LVrF0OHDrW1OXjwIKtWrWLTpk1s2bKFzp0713g+ZerUqWRlZZGcnMzJkycJCgoiJyeHIUOGEBwcbGtnNBqZMmUKc+bMqbWuiRMnNsjffC/1xRdfsGHDBsLDwykqKmLHjh20bduW66+/3tZG399vFA4ucLnZOrm5ubbjma1bt25y4bB06VLbfTzy8vJISkrixRdftIVDUVERSUlJtGnThnvvvZfp06fbllK5mJeXFytWrOCVV15h06ZNZGVlERQUxJNPPmnXtlOnTtx///22aa41efTRR3nsscfq5kPWk0OHDtmOTS9cuNC2/eJw2LJlC19//TU33ngjY8aM4cknn8TLy8vuvbp06cKKFStYtGiR7ST3H/7wB26++Wa7tjExMbz++uv89NNPNdbVpk0bRo8e3SB/wF3suuuuY9euXaxfvx5PT0/uuusuZs2aVe0Qkb6/32j5DBERsaNzDiIiYkfhICIidhQOIiJiR+EgIiJ2FA4iImJH4SDiZJeu2OmIqlU8q1aZFalvCgeRX1X9AL50KWWA5ORk274BAwa4qEIR51E4iNRg/fr1HDt2DLiwPHhDXN5ApD4pHEQu0aZNG86dO2e78nXt2rWcOHGCNm3aVGt3+vRpFi5cyKBBgwgLC+O+++5jzpw51VbYLCwsZOrUqYSHh3P//ffXeA+PyspKVq1axYgRIwgPDycyMpJZs2ZRVFRUvx9U5DIUDiKXiIyMJCAggLS0NM6cOcOyZcto2bIlDz74oK1NZWUlU6ZMITU1ldOnTxMVFYWbmxtpaWmMGTOGs2fPUllZyR//+Eeys7Px8fGhZ8+evPTSS3b9vfbaayQlJXHs2DEGDhxI586d+fDDDxkzZgynT5925kcXsVE4iFzCaDQyYcIEfv75Z5566il+/PFHRo4cWe3eG9988w07duwAICUlhfnz57NixQqMRiOHDh0iKyuLXbt28d133wHw5ptvMn/+fP72t79V66u8vNw2QgkJCcHHx4dOnTphMpkoLCy87HpQIvVJC++J1GDUqFH87W9/Y9OmTRiNRh566CHWrVtn23/xrR27du0KQLt27bj++uspLi6msLAQDw8PW5uqxQOr2lY5ceIEZ8+eBeDzzz+3q6OwsLDuPpTIVdDIQaQGrVq1Ii4uDoD77ruPoKCgavvbt29ve7xv3z4Ajh07xokTJwC46aabuOmmm2xtqm7nWbVceZXrr7/eturnyy+/TF5enu1PdnY2jzzySB1/MhHHaOQgUos//vGP9OvXj06dOtntu+uuuwgLC+Nf//oXjz32GJGRkXz99ddUVFQQGBjI/fffj5eXF127diU/P5/HHnuMe+65x+7mMJ6enowbN463336b559/ns2bN9OiRQsOHjzIN998Q1ZWFoGBgc76yCI2GjmI1OK6666jT58++Pn52e1zc3Nj6dKlTJkyhRYtWvDhhx9y7tw5Ro8ebbtJkZubGykpKfTr14+TJ0/y1VdfMWPGDLv3mjVrFi+88AKdOnVi8+bNbNiwgV9++YWHHnqo2o1oRJxJ93MQERE7GjmIiIgdhYOIiNhROIiIiB2Fg4iI2FE4iIiIHYWDiIjYUTiIiIgdhYOIiNj5/+HpyO4vNt8yAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-UJA6LQR7a_F",
        "outputId": "e8fe53a7-b0e5-4aec-8b55-0df1ed760dc7"
      },
      "source": [
        "list(ax.get_yticklabels())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Text(0, 0.4, '0.4'),\n",
              " Text(0, 0.5, '0.5'),\n",
              " Text(0, 0.6000000000000001, '0.6000000000000001'),\n",
              " Text(0, 0.7000000000000001, '0.7000000000000001'),\n",
              " Text(0, 0.8, '0.8'),\n",
              " Text(0, 0.9, '0.9'),\n",
              " Text(0, 1.0, '1.0'),\n",
              " Text(0, 1.1, '')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "id": "l0OCMnH5UYpu",
        "outputId": "df00bed9-b98d-4840-ec1b-bccaa7db40be"
      },
      "source": [
        "x = \"Model\"\n",
        "y = \"Specificity\"\n",
        "hue = \"Data\"\n",
        "ax = sns.barplot(data=valence_df, x=x, y=y, hue=hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.8)\n",
        "ax.legend(loc='upper left', bbox_to_anchor=(0,1))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Specificity',fontsize = 13, weight='bold')\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "ax.set_yticklabels([\"0.80\", \"0.85\", \"0.90\", \"0.95\", \"1.00\"], size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=valence_df, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"PD\"), (\"ML\", \"NC\")),\n",
        "                                 ((\"1D CNN\", \"PD\"), (\"1D CNN\", \"NC\")),\n",
        "                                 ((\"2D CNN\", \"PD\"), (\"2D CNN\", \"NC\")),\n",
        "                               ((\"3D CNN\", \"PD\"), (\"3D CNN\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "plt.savefig('specificity_valence.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_NC v.s. 1D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-4.614e-01\n",
            "ML_NC v.s. ML_PD: t-test independent samples with Bonferroni correction, P_val=4.051e-07 stat=-8.506e+00\n",
            "2D CNN_NC v.s. 2D CNN_PD: t-test independent samples with Bonferroni correction, P_val=6.461e-01 stat=1.460e+00\n",
            "3D CNN_NC v.s. 3D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-1.727e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEJCAYAAABohnsfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de1RVdf7/8efhcFPEG45QgGWgaPpDES3UJEetNDXFe1JkmpUuxZqiokyt1LKZcpFmyjemQW3J4I0CJ2MaU7IcNMVLamrmDUNHwHsKCOf3h8OZCNQtwjkHeD3WanXO3p+z93ufWrzO57M/e2+TxWKxICIiYoCTvQsQEZGaQ6EhIiKGKTRERMQwhYaIiBim0BAREcOc7V1Adbv33nvx9fW1dxkiIjXK8ePHyczMLLe81oeGr68vq1atsncZIiI1ypAhQypcruEpERExTKEhIiKGKTRERMSwWn9OoyJFRUVkZ2dz+fJle5dSae7u7vj5+eHi4mLvUkSkDqmToZGdnY2npyd33nknJpPJ3uXcNIvFQl5eHtnZ2bRs2dLe5YhIHVInh6cuX76Ml5dXjQwMAJPJhJeXV43uKYlIzVQnQwOosYFRqqbXLyI1U50NDRERuXkKDaC4pMTm2wsKCuKdd96xvk9ISGDevHnW9ykpKQwYMICBAwcyePBgEhISqrRGEZHKqJMnwn/P7OREWtahKtvegJAbn5x2dXUlPT2dp59+mqZNm5ZZt2HDBhITE0lISMDb25vCwkJSUlKqrD4RkcpST8NOnJ2dGTlyJImJieXWxcfH89JLL+Ht7Q1cDZgRI0bYukQRkXIUGnYUGRlJamoq58+fL7P8wIEDtG/f3k5ViYhcm0LDjho0aMCgQYNYvHixvUsRETFEoWFnTzzxBCtXruTSpUvWZYGBgfzwww92rEpEpGIKDTtr3Lgxffv2ZcWKFdZlzzzzDH/+8585deoUAIWFhSxfvtxeJYqIWGn2FFenyBqZ8XQz2zM7Gc/jsWPH8umnn1rf33///eTm5vLkk09isVgwmUwMHTq0yuoTEakshQbc1B/4qtpeVlaW9XWzZs3YsWNHmfVDhw5VUIiIw9HwlIiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMU26BwivFuDqbbbq9tm3b0rp1a4qLi7nrrruYM2cO9erVsy6/cuUKZrOZwYMHM2bMGJyqeFqwiEhlKDQAV2cz/WZV3a3Hv3ht8A3buLu789lnnwHwwgsvkJSUxJNPPllmeV5eHi+88AIXLlwgOjq6yuoTEaks/Xx1AJ07d+bIkSPllnt5efHWW2/x6aefYrFY7FCZiEhZ6mnY2ZUrV8jIyKBHjx4Vrvf396e4uJi8vDyaNWtm4+qkrsjOzmb8+PGEhoaSlZWFt7c3CxYsIDk5maSkJMxmM4GBgcydO9fepYqdKTTs5PLlywwaNAi42tMYNmyYnSuSuu7IkSO8//77zJw5kylTpvDll18SHx/PunXrcHV15dy5c/YuURyAQsNOfnvu4nqOHTuG2WzGy8vLBlVJXebn50fbtm0BaNeuHcePHycoKIgXX3yR3r1706dPHztXKI5A5zQcWH5+PtOnTycyMhKTyWTvcqSWc3V1tb42m80UFxcTHx/P6NGj2bNnD8OGDePKlSt2rFAcgXoaXJ0ia2TG081sr7JTeEuHrUqn3A4aNIgnn3yyymoTMaqkpIScnBzCwsIIDQ1lzZo1/PrrrzRs2NDepYkdKTSgSq/RMLq9394a/bf27t1bpbWIVFZJSQkxMTFcuHABi8VCVFSUAkMUGiJy9XxGWlqa9f24cePsWI04MoWGSC20atUq1q5da9ca+vbty5AhQ+xagy3VlWnLdfZEeE2/WK6m1y/Va+3atezfv99u+9+/f7/dQ8sejhw5QmRkJGvWrMHT09M6bTklJYXU1FTeeOMNe5d4y+pkT8Pd3Z28vDy8vLxq5Kwki8VCXl4e7u7u9i5FHFjr1q2Jj4+3y76ffvppu+zX3urCtOU6GRp+fn5kZ2dz6tQpe5dSae7u7vj5+dm7DBH5jd9PWy4oKCA+Pp4tW7bw9ddfs3DhQlJTU3F2rrl/em1S+cyZM/nHP/5BXl4ePXv2ZNGiRRW227p1KzNmzODQoUO0atWKmTNn0q5dOwC++uor5syZw4kTJ+jYsSOzZ8/G39+/UvW4uLjQsmXLSh+PiIgRtXHass3OaTz88MPXXV9QUEB0dDQXL14kNjaW3NxcoqOjKS4u5tSpUzz//PM0aNCAl156iR9++IFXXnnFRpWL1DyPPPIIjzzySJ3dv6MonbY8cOBAIiIiasW0ZZv0NKZOnUp2djZLliy5ZpuMjAxyc3OJiYkhMjKS3NxcFixYwObNm9m3bx+FhYU8/fTT9OvXj127dvHZZ59x9OhRWrRoYYtDEKlRBgwYUKf3bw91Zdqyw8yeys7OBsDb2xsAHx8f4Oq9l663TkREbMdhz8Zcb0qpppuKSHVyhOtcwDGvdbFrT6OwsJDCwkIA60ygEydOAHDy5Eng6vMkrrdORKSq2fs6F3Dca11s0tNYv3699T9ATk4Oy5cvp0uXLowdO5bTp0+TlZVFeHg4Xl5eLFu2DA8PD1asWIGvry/33HMPAQEB/OUvf+Hjjz8mLy+Pf/7zn4SGhup8hohUG3te5wKOe62LTXoaCQkJvPfeewDs27ePqVOnsm3btjJt3NzciIuLw8PDg9mzZ+Pl5UVcXBxms5nmzZvz/vvvc+7cOebMmcPdd9/NO++8Y4vSRUTkN2zS07jWrKnfj9V16dKF1NTUCts++OCDPPjgg1Vem4jI7znCdGFHqKEiDnsiXETEXhxhyrAj1FARh5lyKyIijk+hISIihik0RETEMIWGiIgYptAQERHDFBoiImKYQkNERAzTdRpSK2RnZzN+/HhCQ0PJysrC29ubBQsWkJycTFJSEmazmcDAQObOnWvvUkVqNPU07KT0Tr3z5s275nujy+SqI0eOEBkZyZo1a/D09OTLL78kPj6elJQUUlNTeeONN+xdokiNp56GnXz++eecOnWKgoIC/u///o/mzZuXew8YWjZo0CB7HorD8PPzo23btgC0a9eO48ePExQUxIsvvkjv3r3p06ePnSsUqfnU07CTQYMG4ePjQ0JCArfffnuF740uk6tcXV2tr81mM8XFxcTHxzN69Gj27NnDsGHDuHLlih0rFKn51NOwk9TUVE6ePMm4ceP45ZdfKnwPGFo2cOBAex6KwyopKSEnJ4ewsDBCQ0NZs2YNv/76a41/RrOIPSk07GTAgAGYTCbmzZvH+PHjsVgs5d4DhpdJeSUlJcTExHDhwgUsFgtRUVEKDJFbZLLU8r86Q4YMYdWqVfYuQ0SkRrnW3071NMQhOcIzmh3x+cwi9qYT4eKQ7P2MZkd9PrOIvamnIQ7Lns9odtTnM4vYm0Kjmmh4RURqI4VGNSkdXmndurVd9l86tFNTQ8Pez0e29/5FHJVCoxppeKXy7P18ZHvvX8RR6US4iIgYptAQERHDNDxVTew9Jm7v/YtI7aTQqCb2HhO39/5FpHYyPDz1yy+/VGcdIiJSAxgOjT59+jB27FjS0tIoLCyszppERMRBGQ6NkpISvvvuO2JiYujevTszZsxg586d1VmbiIg4GMPnNNatW8fatWtZu3Ytu3btIikpib///e8EBAQwdOhQhg8fToMGDaqzVhGRWu/f//43f/vb38jPz+f8+fN4enrStGlTxowZQ1hYmL3LM97TuP322xk7dizJycn861//omfPnlgsFg4ePMi7775L79692bRpU3XWKiJS6yUnJ3PgwAHy8vIoLCwkLy+PAwcOkJycbO/SgJu8TiM3N5f4+HjGjBnDhg0bAHB3d+ePf/wjBQUFvPnmm9VSpIhIXTFixAhatWqF2WwGrj66uFWrVowYMcLOlV1leHhq4sSJZGRkUFxcjMVi4Y477uDRRx9l6NCheHp68sEHH7Bw4cLqrFVEpFo5wtBQWFgYYWFhREVFcfz4cXx8fBzqb+tNndMwmUz06NGDxx57jPDw8DLrQ0NDGTRoUJUXKCJiK6VDQ6Xy8vLIy8sjOTnZIc4nOALDoTFmzBgiIyPx9/evcH337t3p3r17lRUmIjfPEX4p12QjRozg119/5eeff6a4uBiz2cxdd93lMENDjsDwOY309HTS0tLKLFu9ejVTpkyp8qJEpHIc/SSqowsLC2PhwoX4+PgAWIeGFLj/c1NXhJ85c6bMst27d5Oenm7o81u3bmXgwIG0b9+eiIgIdu/eXa5NUVERs2bNolu3boSGhjJjxgyKiooAyMzMJCgoqMw/f/vb34yWL1InOPpJVKn5bjg8NX/+fOvrHTt2WN9bLBbWrVuHq6vrDXdSUFBAdHQ0bm5uxMbGsnDhQqKjo0lPT7f+zw2wZMkSFi9ezIgRI6hXrx6JiYm0aNGCsWPHWttMnDiRwMBAAO6++27jRypSBzj6SVSp+QyFhslkwmQysWPHDnbs2GFdZ7FY6NChww13kpGRQW5uLjExMURGRpKbm8uCBQvYvHkzXbt2tbbbvHkzAJMmTaJJkyYkJiayevXqMqHRuXNnOnfujJub200dqIiI3LobhkaXLl0A2LJlC97e3rRo0eLqB52duf322xk/fvwNd5KdnQ2At7c3gHW88NixY2VCo2nTpgBs2rQJT0/PMp8tNW7cOACCg4OZM2cOLVu2vOH+RUSkatwwNJYsWQJAr169GDZsGBMnTrzlnVoslgqXjx8/nm+++YaXX34ZZ2dn3N3drT2KZs2aERMTQ0BAANu3b2fhwoXMmDGDxMTEW65HRESMuanrNCrLz88PgBMnTgBw8uRJAPz9/SkoKMDJyQkXFxdatmxJeno6+/fvp2HDhgwfPtx6/iIgIICAgAAA/vjHP5KUlMTBgwcrXZOISHUqvFKMq7P5xg1rWA3XDY3evXszbNgwJkyYQO/evStsYzKZ+Oqrr667k/DwcLy8vFi2bBkeHh6sWLECX19ffH19CQ4OpmfPnixatIi9e/eybt06fHx8WLNmDefPn7eez5g/fz5nz56lTZs27Nq1izNnzlyzJhERe3N1NtNvVkqlP1+YfxGA4/kXK72dL14bXOn9X8t1Q+P48ePWabbHjx+vsI3JZLrhTtzc3IiLi+PNN99k9uzZBAYG8tZbb+HkVH7G78qVK/nPf/5D8+bNmT59Or169QIgMDCQjz76iOXLl+Pm5kb//v159dVXb7jvukYXd4lIdbpuaLz99tvWIaG33377lnbUpUsXUlNTyy3ft2+f9XXbtm2vOQzWt29f+vbte0s11AW6DYLIVcUlJZgr+GEqt+a6oREREVHha3Fcug2CyFVmJyfSsg5V6rMXC65Y/13ZbQwIqZ0zOw3H8OLFi5k2bRrFxcUAXLlyhRkzZrB48eJqK66uK7xSfNOfqcrbIFRm/1J1iktK6vT+xTEZnj2VkJDA3Xffbb2C29nZmby8PNavX09UVFS1FViXVeZEWknuIYoPZWK5kAfA8V9y6DN4FOaW9+LU7OZ++VTHSTQxTr+UxREZ7mnk5+fTrFmzMssaN25Mfn5+lRcllVd8NAvL+VNg+e+vREsJlvOnKD6aZd/CRKRWMBwa3t7ebNy4kbNnzwJw9uxZNm7cSPPmzautOLl55hYhmDz/AK4e4GQGVw9Mnn/A3CLE3qWJSC1geHiqc+fOpKSk0KdPH1q2bMnhw4c5f/48gwdrCMORODVredPDUCIiRhnuaUyePJlGjRpx/vx5du7cyblz52jUqBGTJ0+uzvpERMSBGO5p+Pr6kpqayooVK8jJyeH2229n6NChGp4SEalCpZNZuHzu6oLL5yjaklSpySzVwXBoADRv3rxKblgoIiIVs05mKfWbySwOHxpRUVH07duX0aNHX3Narclk0p1mpdbQbVjqtp/37GTTlymcy88F4Fx+Lp/OfZOuDw3mrruDbVKDuUUIxcWFWAp+hSuXwdkdk1t9h5nMct3Q2Lx5M23btrW+roiRe0+J1BS6DUvdtnX9Wv5z/Kj1fUlJMf85fpStG760WWg4+mSW64bGpEmT6Nixo/W1SG1X02/D4gi/lGuy0J59KSy4zMXz5yi4dBG3eh54eDYk9P6H7F2aw7huaDRp0oT69esDcO+99+Lj44O/v79NChOxh5r+jG1H+KVck911d7C+pxu4bmi89dZbjBkzhk6dOhEVFcUTTzzBK6+8YqvaROQm2fuXss4J1X7XDQ1XV1e2b99OZmYmFouFnJwctmzZUq5d6XPERcS+7P1LWeeEar/rhkZgYCA7duxgzJgxmEwm0tPTSU9PL9PGZDKxZ8+eai1S6g79Uq3Zavo5Ibmx614RPmPGDAIDA61P2LNYLOX+KdHtk6UKlf5SzcvLo7CwkLy8PA4cOEBycrK9SxMDqvLW/OKYrtvTCA4Otj5tr02bNjzxxBPExsbapDCpm/RLVcSxGb4i/F//+heenp7VWYtIlc5eKrxSjKuzuYorrDn7F6kO1w2N+fPn07FjR+677z5Wr159zXa6hkMcUWUeYlWqMP8iAMfzL1Z6G3qIldRGNwyNMWPGcN999zF//vxrXv2t0BARqRuuGxpdunSxXsynabUiInLd0FiyZEmFr0VEpG4y/BCmgwcPsmXLFoqLiwG4cuUKW7Zs4eDBg9VWnNRsxZqOLVLrGJ499ac//QknJyfrCXFnZ2fmzJlDcXHxdU+SS91ldnIiLetQpT57seCK9d+V3caAEMe9U2hNYO/ZX/bev1TMcGgcOXKERx55pMyyu+++23odh0ht4OhPTbMlzT6TihgenqpXrx6HDpX9xXfo0CHc3d2rvCgRe7E+Nc3y36G13zw1TURuoqcREBDA999/z0svvURISAhZWVl8//33dO7cuTrrE7EpR39qmoi9GQ6NsWPH8v3335OamlpmSGrcuHHVUlhNpRvu1WyO/tQ0EXszPDzVq1cv3njjDfz8/DCbzfj7+zNjxgx69uxZjeXVPLrhnojUZoZ7GgAjR45k5MiR1VVLraAb7t0aPa5UxLEZ7mnA1ZsWRkVF8dBDD3Hy5Enmz5/Pzp07q6u2Gkm3hr41pY8rLSm5ej3Qbx9XKiL2Z7inkZGRwaRJk7BYLJhMJpo2bcrSpUs5dOgQ7733XnXWaDfFJSWYnW4qV+UW2ftxpSJyfYZDY8GCBdSrV48777yTvXv34uLiQqdOncjKqr1TEXVxmu3Z+3Glcmt0nUvtZ/hn9IEDB+jXr1+ZKbbNmzcnLy+vWgqrqX7es5NP575Zbkz+5z0axpPaT9e51H6GQ6N+/frlAmL//v00btzY0Oe3bt3KwIEDad++PREREezevbtcm6KiImbNmkW3bt0IDQ1lxowZFBUVWdfPnz+fsLAwQkJCeOWVVygoKDBavs1oTF7qMnOLEEyefwBXD3Ayg6sHJs8/6DqXWsTw8FS7du3YuHEjvr6+wNXrNrKysujVq9cNP1tQUEB0dDRubm7ExsaycOFCoqOjSU9Px2z+371llixZwuLFixkxYgT16tUjMTGRFi1aMHbsWP75z38yb948Hn74Yfz9/Vm0aBG33XYbU6ZMqcRhVx+NyUtdputcaj/DofHcc8+xadMmjhw5AsB3332Hq6srkydPvuFnMzIyyM3NJSYmhsjISHJzc1mwYAGbN2+ma9eu1nabN28Grj7UqUmTJiQmJrJ69WrGjh3LqlWrAHj99ddp2rQpn3/+OatWrXK40NCYvIjUZoZDo02bNiQlJbF06VJycnK47bbbiIyMpE2bNjf8bHZ2NgDe3t4A1umox44dKxMaTZs2BWDTpk3W55GXfjY7OxsXFxdrGx8fH7Zv305hYSGurq5GD0NERG7BTV3c17ZtW2bNmkV+fr71j3dlWCyWCpePHz+eb775hpdffhlnZ2fc3d1xc3O7qW2IiEj1MXwivLCwkFmzZhESEkL37t0JCQlh5syZhk5G+/n5AXDixAkATp48CYC/vz8FBQXWk90tW7YkPT2d5ORk0tLScHFxITAw0LqNoqIi68n4kydP4u3trV6GiIgNGe5pvPPOOyxbtsz6C//SpUt8+umnlJSUMG3atOt+Njw8HC8vL5YtW4aHhwcrVqzA19cXX19fgoOD6dmzJ4sWLWLv3r2sW7cOHx8f1qxZw/nz5xk7diwAERERrFu3jlmzZuHn50dOTg4TJky4hUMXEZGbZbinsXbtWtzc3Hj11Vf55JNPiI2NxdXVlbVr197ws25ubsTFxeHh4cHs2bPx8vIiLi4Opwqutl65ciXTp0/n8OHDTJ8+3To768EHH2TixIl8++23LFmyhEGDBvHss8/exKGKiMitMtzTsFgs9OvXj6ioKAC6du3Kjz/+yIYNGwx9vkuXLhU+5W/fvn3W123btmXdunXX3MaUKVMcbraUiEhdYrinMWzYMPbv3289/1BYWMi+fft49NFHq604ERFxLIZ7Gtu2bWPv3r3cf//9tGjRgqNHj3L27Fk8PDysvQ+TyURiYmK1FSsiIvZlODS2bt0KQH5+Pvn5+dblW7Zssb42mUxVWJqIiDgaw6ExadKk6qxDRERqgEqFxubNm7l48SIdO3akSZMm1VKYiIg4nhuGxsKFC9m0aRMffPABjRo1IjY2lpSUFAAaNWpEQkIC7dq1q/ZCRUTE/m44eyo9PZ38/HwaNWrE4cOHWb16NRaLBYvFwpkzZ5g/f74t6hQREQdww9D45ZdfCAoKAuDbb78FoEOHDmzevJm2bduya9eu6q1QREQcxg1D48KFC9Y7zu7atQuTyUS/fv1o2LAhHTt25OzZs9VepIiIOIYbhoaXlxeZmZns3bvX2tPo2LEjAHl5edZAERGR2u+GoREWFsbPP//MkCFDyM3NxcvLiw4dOgCwe/du/P39q71IERFxDDcMjT/96U+0a9cOi8WCh4cHM2fOxGQykZmZyfHjx+ncubMt6hQREQdwwym33t7erFy5knPnzuHh4WF9pndoaCjbtm275kOSRESk9jF8cV/Dhg3LftDZGWfnm3rwn4iI1HCG73IrIiKi0BAREcMUGiIiYphCQ0REDFNoiIiIYQoNERExTKEhIiKGKTRERMQwhYaIiBim0BAREcMUGiIiYphCQ0REDFNoiIiIYQoNERExTKEhIiKGKTRERMQwhYaIiBim0BAREcMUGiIiYphCQ0REDFNoiIiIYTYLja1btzJw4EDat29PREQEu3fvLtemsLCQ1157jbCwMIKDgxk8eDCbNm0CIDs7m6CgoDL/zJo1y1bli4gINgqNgoICoqOjuXjxIrGxseTm5hIdHU1xcXGZdikpKaxYsYK2bdsyZcoUfvzxR15//fUybUaNGsX777/P+++/T0REhC3KFxGR/3K2xU4yMjLIzc0lJiaGyMhIcnNzWbBgAZs3b6Zr167WdiUlJQC0atWKbt264erqSsOGDctsq3379vTq1Yt69erZonQREfkNm/Q0srOzAfD29gbAx8cHgGPHjpVpFxERwQMPPEBiYiKDBw+mXr16vP3222XavP7663Ts2JH+/fuzfft2G1QvIiKl7HIi3GKxVLh8+/btrF+/noEDBzJ37lyKi4uJjY3FYrFQv359Jk+ezIcffsjLL7/M4cOHefHFF21cuYhI3WaT4Sk/Pz8ATpw4AcDJkycB8Pf3p6CgACcnJ1xcXFi7di1FRUWMGjWKzp07k5yczKZNmzh9+jRNmzZl0qRJ1m2mpaWxe/duCgoKcHNzs8VhiIjUeTYJjfDwcLy8vFi2bBkeHh6sWLECX19ffH19CQ4OpmfPnixatAh/f38APv74Y/bt28f27dtp3LgxTZo0ITk5mV27dhEcHEx2djZ79+6lTZs2CgwRERuySWi4ubkRFxfHm2++yezZswkMDOStt97Cyans6FhkZCQHDx7k66+/ZtOmTQQEBPDSSy9hMpm48847WbFiBWlpaTg7O9OjRw9effVVW5QvIiL/ZZPQAOjSpQupqanllu/bt8/62s3N7ZrXXtxzzz0kJydXW30iInJjuiJcREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETHMZqGxdetWBg4cSPv27YmIiGD37t3l2hQWFvLaa68RFhZGcHAwgwcPZtOmTdb1SUlJhIeHExwczIQJEzh9+rStyhcREWwUGgUFBURHR3Px4kViY2PJzc0lOjqa4uLiMu1SUlJYsWIFbdu2ZcqUKfz444+8/vrrAOzZs4fp06cTEBDA5MmT2bBhA2+//bYtyhcRkf+ySWhkZGSQm5vL6NGjiYyMZNiwYWRnZ7N58+Yy7UpKSgBo1aoV3bp1w9XVlYYNGwKwatUqAJ5//nnGjx9PSEgIa9asoaCgwBaHICIigLMtdpKdnQ2At7c3AD4+PgAcO3aMrl27WttFRESwceNGEhMTSUxMpHHjxtbeREXbuHLlCjk5Odx5553X3Pfx48cZMmRIlR+TLfwVqGfH/Q8ZstiOe791+v5ujb6/W2Pv7w9u7Ts8fvx4hcttEhq/Z7FYKly+fft21q9fz8CBA+nVqxfTpk0jNjaWlStXGt7G72VmZt5SrSIi8j82GZ7y8/MD4MSJEwCcPHkSAH9/fwoKCigqKgJg7dq1FBUVMWrUKB5++GHat2/P7t27OX36tHUbpZ89efIkzs7O3HbbbbY4BBERwUY9jfDwcLy8vFi2bBkeHh6sWLECX19ffH19CQ4OpmfPnixatAh/f38APv74Y/bt28f27dtp3LgxTZo0YfDgwSxZsoS5c+fSrVs3srKy6N+/P25ubrY4BBERwUY9DTc3N+Li4vDw8GD27Nl4eXkRFxeHk1PZ3ZeeJN+5cyfvvvsud911F3Fxcfw/gewAAAoLSURBVJhMJtq3b8+0adP46aef+OCDDwgPDyc2NtYW5YuIyH+ZLEZPDoiISJ2nK8JFRMQwhYaIiBim0LCz7OxsgoKCCAoK4sMPP7Quf/XVV63LMzMzCQoK4s0337RjpdVv5syZdOvWjaCgIJ555pky60q/i/bt29O9e3cmTpzI3r17r7mtvLw8YmNj6datG8HBwQwcOJD09HQAevXqRVBQEPPmzQP+99+gdJ/z5s0jKCiIBx54wHrXgscff5ygoCDy8/Or49Bv2eHDh3n88ce59957CQkJ4cknn+To0aMA1v9/goKC6NChA71792batGnXPZYffviBcePG0alTJzp16sRjjz3GwYMHy/z/WjqdfdWqVQQFBZGQkAD877t65ZVXrNsLCgpiwIAB1fgN3Lrhw4cTEhJChw4dGDJkCFu2bAH0/f2eQsOBrF69GovFwsWLF/niiy/sXY5dPPzww9dc5+Pjw8yZM+nXrx8bNmxg9OjR/PTTT+XaXb58mccee4xVq1Zx77338vrrr9O1a9dy9ztbunQpFy9evOb+jh49yj/+8Y/KH4wNnTx5EovFwuTJkxk6dCjfffcdr732Wpk29913H1OnTqV169b8/e9/5/HHH+fy5cvltnXgwAEee+wxMjMzefTRR3n55Zdp1qwZhw8fLtNu0aJF160pLS2NX3755ZaPzVZCQkKYOnUqEydO5Mcff2Tq1Kll1uv7u8ouF/dJef7+/hw7dozMzEyys7O5cuUK3t7e1utS6oKpU6eSnZ3NkiVLKlzv6enJ4MGDGTx4MM2aNWPu3LnEx8fz7rvvlmmXmprKzz//zH333cfcuXOty0tvUwPg4uLCpUuXSEpK4qGHHqpwfw0aNCA+Pt5hfuFdT0hICEuXLrW+T01NLReod9xxB8OHD2f48OGMHTuWb7/9lrS0NIYNG1amXXx8PJcuXeKFF17g6aefBmDkyJGUlJRY/4g1aNCAb7/9ll27dl2zJjc3Nz7++GOmTZtWVYdZrWJjYzl9+jTZ2dl89NFHmEymMuv1/V2lnoaDCAgIoEOHDqxcuZKVK1fSp08f6323pLzw8HDg6jDA75X2KHr06FFm+W+neLu4uDB06FA++eQTCgsLK9zHqFGj2L9/P+vWrauqsquNq6ur9fWuXbs4c+YMXbp0uWb7W/3+QkNDad269XV/LY8aNYqVK1eSl5dn7CDs7Pz583Tt2pXhw4fj4uLCzJkzr9m2Ln9/Cg0HMnToUNauXcu2bdtq7P2ybKV0pvjvfw1ea1lFnnrqKU6fPm29Gebv/b//9//o1q0b8fHxlS/Uxg4ePMiECRPw9fUtN7zyW7f6/ZlMJsaPH89XX33FwYMHK2zz6KOP4ubmxieffGKwevuqX78+f/3rX5k6dSoFBQV88MEH12xbl78/hYYD6d+/P2azGR8fH7p3727vchzaxo0bAWjXrl25daXLvv322zLLfzs8BeDr68uAAQNYtmzZNffzzDPPsH37dvbs2XOrJVe7n376iaioKFxdXUlMTKR58+bXbGvk+yttU+r331///v3x8/MjKSmpwn3Ur1+fxx577LrfryNxdname/fuPP744wQHB5OZmXnNk911+fvTOQ0H0qBBA2bPno2Hh0e5q+Xh6g0d//KXvwDg7u7OpEmTbF1itVq/fj379+8HICcnh+XLl9OlSxfrXYzPnz9PSkoKP/zwA0lJSdSvX986ZvxbAwYMICEhgYyMDF544QXCwsL4+eefcXV15fnnny/T9umnn+azzz67Zk1hYWF07NiR7du3V92BVoOcnByioqI4c+YMzz33HDt37mTnzp3079/f2ubIkSMsX76c9evXs3HjRgIDAys8XzN+/HjS09OJi4vj3Llz+Pv7k5mZycMPP0xQUJC1ndls5qmnnmL69OnXrCsqKsrhfilX5JtvvuGLL74gJCSEEydOkJWVRbNmzWjSpIm1jb6/qxQaDuZ6s4d2795tHS/19PSsdaGRkJBgfcbKvn37mDp1Km+//bY1NE6cOMHUqVNp3Lgx999/P5MnTyYwMLDcdtzd3Vm6dCl//vOfWb9+Penp6fj7+/Pcc8+VaxsQEMADDzxgnY5bkWeeeYYJEyZUzUFWk6NHj1rHvt977z3r8t+GxsaNG9myZQt/+MMfGDVqFM899xzu7u7lttWqVSuWLl3K3LlzrSfX27Rpwx133FGu7ZAhQ/jwww/5z3/+U2FdjRs3ZuTIkQ73h+/3GjVqxM6dO0lLS8PV1ZVOnToRExNTZqhJ399Vuo2IiIgYpnMaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNEQcyO/vgGpE6V1RS+/aK1KdFBoiBpT+Yf79LasB4uLirOt69eplpwpFbEOhIXKT0tLSOHXqFHD1NuyOdpsHkeqk0BC5CY0bN6aoqMh6pe+qVas4ffo0jRs3LtPuwoULvPfee/Tt25cOHTrQp08fpk+fXuaOpTk5OYwfP56QkBAeeOCBCp+hUlJSwvLly4mIiCAkJITw8HBiYmI4ceJE9R6oyDUoNERuQnh4OL6+viQlJfHrr7+SmJhI/fr1eeSRR6xtSkpKeOqpp4iPj+fChQsMGDAAJycnkpKSGDVqFJcuXaKkpIRnn32WjIwMGjZsyD333MOcOXPK7e/9999n6tSpnDp1igcffJDAwEA+//xzRo0axYULF2x56CKAQkPkppjNZh5//HHOnDnD888/z+HDhxk6dGiZZ59s27aNrKwsABYuXMisWbNYunQpZrOZo0ePkp6ezs6dO/nxxx8B+Oijj5g1axbz588vs6/CwkJrj6Z9+/Y0bNiQgIAA3NzcyMnJue79skSqi25YKHKThg8fzvz581m/fj1ms5knnniClJQU6/rfPqKzdevWADRv3pwmTZqQm5tLTk4OLi4u1jalN10sbVvq9OnTXLp0CYCvv/66XB05OTlVd1AiBqmnIXKTGjRowIgRIwDo06cP/v7+Zdbffvvt1tcHDhwA4NSpU5w+fRqA2267jdtuu83apvSxrKW3hS/VpEkT611U3333Xfbt22f9JyMjg3HjxlXxkYncmHoaIpXw7LPP0qNHDwICAsqt69SpEx06dGDHjh1MmDCB8PBwtmzZQnFxMX5+fjzwwAO4u7vTunVr9u/fz4QJE7jvvvvKPbTH1dWV0aNH89e//pVp06axYcMG6tWrx5EjR9i2bRvp6en4+fnZ6pBFAPU0RCqlUaNGdOvWDW9v73LrnJycSEhI4KmnnqJevXp8/vnnFBUVMXLkSOvDo5ycnFi4cCE9evTg3Llz/Pvf/+aFF14ot62YmBjeeustAgIC2LBhA1988QVnz57liSeeKPOAIBFb0fM0RETEMPU0RETEMIWGiIgYptAQERHDFBoiImKYQkNERAxTaIiIiGEKDRERMUyhISIihv1/s3d2TFXLM78AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 523
        },
        "id": "0nhXTnkTVs1L",
        "outputId": "15996710-7b0d-4970-bec7-8a17fdab6c77"
      },
      "source": [
        "valence_df_new = pd.melt(valence_df[valence_df['Data']=='PD'], id_vars=['Model'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "x = \"Model\"\n",
        "y = \"Value\"\n",
        "hue = \"Test\"\n",
        "ax = sns.barplot(data=valence_df_new, x=x, y=y, hue = hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.5)\n",
        "ax.legend(loc='lower right', bbox_to_anchor=(1,0))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Sensitivity/Specificity',fontsize = 13, weight='bold')\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold');\n",
        "ax.set_yticklabels([\"0.5\", \"0.6\", \"0.7\", \"0.8\", \"0.9\", \"1.0\"], size= 10, weight='bold')\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=valence_df_new, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"Sensitivity\"), (\"ML\", \"Specificity\")),\n",
        "                                 ((\"1D CNN\", \"Sensitivity\"), (\"1D CNN\", \"Specificity\")),\n",
        "                                 ((\"2D CNN\", \"Sensitivity\"), (\"2D CNN\", \"Specificity\")),\n",
        "                               ((\"3D CNN\", \"Sensitivity\"), (\"3D CNN\", \"Specificity\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "plt.savefig('sens_and_speci_valence.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_Sensitivity v.s. 1D CNN_Specificity: t-test independent samples with Bonferroni correction, P_val=3.192e-07 stat=-8.645e+00\n",
            "ML_Sensitivity v.s. ML_Specificity: t-test independent samples with Bonferroni correction, P_val=2.091e-16 stat=-3.073e+01\n",
            "2D CNN_Sensitivity v.s. 2D CNN_Specificity: t-test independent samples with Bonferroni correction, P_val=8.861e-03 stat=-3.565e+00\n",
            "3D CNN_Sensitivity v.s. 3D CNN_Specificity: t-test independent samples with Bonferroni correction, P_val=1.403e-01 stat=-2.279e+00\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEJCAYAAAB/pOvWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVyU1f7A8c8wwCAgsnSFBEwDJBNRNBU3Mq8bJqkouIBoGqbepEUtuJL2M9SszEumqTcrl9JQ0YRSKU1Rr9t1ue64YCqIC4uJiqAwvz+MyXFYBmUYwO/79fLVzPOc5znfZy53vnOec55zFGq1Wo0QQgjxABNjByCEEKL6keQghBBChyQHIYQQOiQ5CCGE0CHJQQghhA5TYwdQGdq1a4ezs7OxwxBCiBolPT2dPXv2lLivViQHZ2dn4uPjjR2GEELUKIGBgaXuk9tKQgghdEhyEEIIoUOSgxBCCB2SHIQQQuiQ5CCEEEKHJAchhBA6JDkIIYTQIclBCCGEDkkOQgghdEhyEEIIoUOSgxBCCB2SHIQQQuioFRPvCSFEVUhLSyM8PJzWrVtz8OBBHB0dmT9/PnFxcaxcuRKlUom7uztz5swxdqiPTZKDEEJUwPnz5/nss8+IiYnhzTffZNOmTSxatIgtW7Zgbm7OjRs3jB1ipZDbSkIIUQEuLi40bdoUgGbNmpGeno6npycTJ07kxx9/RKlUGjnCyiHJQQghKsDc3FzzWqlUUlhYyKJFixg6dCjHjx9n4MCB3Lt3z4gRVg5JDkII8RiKiorIyMjA19eXiRMnkpuby+3bt40d1mOTPgchhHgMRUVFTJo0iZs3b6JWqwkLC8PGxsbYYT02SQ6i2lKr1SgUCubOncv48eNLfA/otU2hUBjzUkQt4eLiQmJioub9qFGjjBiNYUlyENXW+vXruXbtGvn5+fz73/+mfv36Ou8Bvbb17dvXmJciqqn4+Hg2btxo1Bh69epV5lrOxiJ9DgZU/Ct27ty5pb7Xd9uTqG/fvjg5ObF48WIaNGhQ4nt9t4n79PmbfJJs3LiRU6dOGa3+U6dOGT05lUahrgV/DYGBgcTHxxs7DB0//vgj165d4/r169SrV0/zy/fB94Be257EL7iEhASuXLmi+RycnJx03gN6bQsICDDmpVQb+vxNPkl/a6NHjwZg0aJFT2T9ZX53qmuB/v37GzuEUiUkJKife+45dWJiYonvK7LtSVNUVKRWq9Xqzz//vNT3+m4Tf9Hnb/JJER4erg4PD39i6y/ru1NaDgYkv3xFdaPP3+ST9Ldm7F/uxq6/rO9O6ZA2oD59+mhGzYSHh2uNtil+D+i9TYjHpe/f5JPilVdeeaLrL0uVtBxiYmL4+eefycrKokuXLixcuLDEcvv37+eDDz7g3LlzeHh4EBMTQ7Nmzco9f3VtOQghRHVW1ndnlY1W6t27d5n78/PziYiI4NatW0RFRZGZmUlERASFhYVVFKEQQohiVXJbKTo6mrS0NJYtW1ZqmeTkZDIzM5k0aRIhISFkZmYyf/589u7dS/v27asiTFHFZIy5ENVXtXnOIS0tDQBHR0cATcfsxYsXjRaTMCwZYy5E9VVtO6RrQseY/PJ9fE2aNDH6SBEhhC69k8M///lPBg4cSKtWrSqt8oKCAuD+FLguLi4AXL58Gbg/lBPA1dW10uqrbMW/fJs0aWKU+ot/ddfk5CAej/xAEYaid3KIj49n7dq1NGzYkAEDBtC3b1/NLaDybN26VfNFlpGRwapVq2jTpg0jR44kJyeHgwcP4ufnh4ODAytWrMDKyorVq1fj7OxM27ZtH+3Kqoj88hXGJD9QhKHonRz69+/Pli1bOH/+PHPmzCE2NpaOHTsyYMAAunbtipmZWanHLl68mL179wKQkpJCdHQ0M2fO1CqjUqmIjY1l2rRpzJgxA3d3dz788MNas6qS0GXsMd7Grr+yyA8UYQh6J4eZM2dy7949/vOf/7Bp0yZ+/fVXtm/fzvbt26lXrx6hoaGMGTMGU1PdU5Y2SunhXxtt2rQhISGhgpcgaqo+ffo80fULUZ1VqEPa1NQUX19fsrOzOXv2LIcOHQLg+vXrzJs3j7S0ND766CODBFodGfuXp7HrF8Zn7L8BY9cvDEfvJ6RPnDjBqlWrSExMJDc3F7Vajb29PUFBQfTq1YuZM2dy9OhRDhw4YOiYdcgT0kIIUXGVMrdS//79USgUqNVqvL29CQkJwd/fX7PYdteuXTX9CkIIIWo2vZODubk5/v7+hISE4O3trbN/4MCBdO/evVKDE0IIYRx6J4c333yTtm3b0rx5c822rKwscnJycHd3x9raGmtra4MEKYQQomrpPX3Gp59+qrWwNsDChQufqLnfhRDiSaF3ciip37r4aWYhhBC1S7m3lZo2bQrcX3xm6dKlLF26VGu/vb29YSITQghhNOUmhwdXJiup9TB06NDKj0oIIWqx3bt38+2335KdnU1ubi5169bF3t6eESNG4Ovra+zwAD2Sw9KlS1Gr1QwfPpyePXsSEhICgFKpxNnZWTO1thBCCP3ExcVx+vRpzfusrCyysrKIi4urOcmheOK7mTNn4u7urjVaSQghRMUFBwdz+/ZtUlNTKSwsRKlU8uyzzxIcHGzs0DTKTA7r1q3Dzc2N5s2bo1AoOHv2LGfPntUp169fP4MFKIQQtY2vry++vr6EhYWRnp6Ok5MTCxYsMHZYWspMDpGRkYwYMYLmzZsTGRmJQqHQKaNQKCQ5CCFELVNmcmjQoAE2Njaa10IIIZ4MZSaHLVu2lPhaGFdNGOkgxJOi4F4h5qbGW3fGUPXrPX3G9evXuX37Nk8//bRmWGtGRgaWlpbY2tpWemCidDVhpIMQTwpzUyX+09c90rEF2bcASM++9cjn2DDZMLf19X5CevTo0YwcOVLT76BQKAgPD+f11183SGBPioJ7hRU+xtvbG5VKpbVNpVKVOCGiIeoXQtR+ercczpw5Q+/evbW2tWrVip9++qnSg3qSPMqvjrsHtqDOz9falp+fz/KELfyQXrFWnKF+dQghaja9k4NSqSQzM1Nr29WrV0tcFlQYlrKhD4WFBajzb8O9O2BqgUJlibKhj7FDE0LooSjzHIXn9sCdG/c33LnB3X0rUTZuh8lTjY0b3J/0/mZv1KgR27ZtIzY2Fh8fHw4ePEhycjJeXl6GjE+UwOSpxtXmD6gmkg59YWyFFw6izr321wZ1EercaxReOFht/r+td3IICQkhMjJS50GN4uk0hKgppENfGFtNaP3rnRz69etHamoqy5YtIy8vjzp16jBs2DB5AE7UODVh6oLqTlpfj6cmtP4r1GHwzjvvMH78eLKzs7G3t8fMzMxQcQlhMDVh6oLqTlpftV+ZyWHfvn04OTnh6urKvn37tPZduHBB87pNmzaGiU6IctTWB5CqO2l91X5lJoewsDCGDx9OZGQkw4YNK3VupePHjxssQCHKUlsfQKrupPVV+5WZHNRqtdYCPyUt9lPSNiGEEDVbmcnh5MmTJb4WoiarCWPMq5Kxb40Zu35RsjKTQ/v27RkyZAgRERGEhYXRq1cvWRZU1Hg1YYx5VZJbc6IkZSaHnJwccnJyANi7dy9NmzatkqCEMKSaMMZcCGMrMzk4ODgQHx/PmTNnANi0aRMnTpzQKqNQKFiyZEm5Fe3fv58PPviAc+fO4eHhQUxMDM2aNdMqc/fuXT7++GN++ukn8vPzCQgIYPLkyTJkVlSqmjDGXAhjKzM5+Pv7s3z5cvbt24dCoeDy5ctcvnxZq0xJI5gelp+fT0REBCqViqioKBYsWEBERARJSUkolX/da1y2bBlLly4lODiYOnXqsGTJEho2bMjIkSMf8fKEEIYg/Ta1X5nJITo6mnbt2nH69Gk+//xzWrZsSadOnSpcSXJyMpmZmUyaNImQkBAyMzOZP38+e/fupX379ppye/fuBeCNN97Azs6OJUuWsHbtWkkOQlQz0m9T+5X7hHT37t3p3r07Fy5coFOnTvTp06fClaSlpQHg6OgIgJOTEwAXL17USg729vYA7Nq1i7p162odK4SoPqTfpvbTe/qMjz76qNIqLe3ZiPDwcLZv3857772HqakpFhYWOovaCCGMT/ptar8yk0PTpk01T0iXNlJJnyekXVxcADT9FVeuXAHA1dWV/Px8TExMMDMzo3HjxiQlJXHq1ClsbGwICgrC3d29whclhBDi8ej9hPTjPAnt5+eHg4MDK1aswMrKitWrV+Ps7IyzszPe3t506dKFhQsXcuLECbZs2YKTkxM//fQTubm50t8ghBBGUGZyWLp0qaZ/YOnSpY9ciUqlIjY2lmnTpjFjxgzc3d358MMPMTHRXcJ6zZo1XL16lfr16zN16lS6du36yPUKIYR4NGUmh7Zt25b4+lG0adOGhIQEne0pKSma102bNmXLli2PVY8QQpRH1qMon+5P91J8+umnjBkzhsLCQgDu3bvHuHHj+PTTTw0WnBBCGELxehRZWVkUFBSQlZXF6dOniYuLM3Zo1YbeyWHt2rVYWFhoHlozNTWlTp06rFv3aPOpCCGEsQQHB+Ph4aH5PlMqlXh4eMh6FA/Qeyhrbm4ulpaWWttUKhW5ubmVHpQQQhiSrEdRPr1bDk8//TSbN2/m/PnzAJw/f14zskgIIUTtonfLoXPnzixfvhx/f3/s7OzIyclBrVbzyiuvGDI+IYQQRqB3y2HcuHE0bNiQoqIisrKyKCoqwtXVlbFjxxoyPiGEEEagd8vB3t6e9evXk5SUxKVLl2jQoAE9evTAwsLCkPEJIYQwAr2TA9zvgH766acB5HaSEELUYnonhxs3bjBy5EiOHTuGQqGgZcuW9OzZk3HjxjF+/HhDxiiEEKKK6d3n8Pnnn3P06FHNfEsNGzbEy8uL5ORkQ8YnhBDCCPRODr/99hstW7Zk6NChmm3u7u5cuHDBIIEJIYQwHr2TQ3Z2Nu7u7piaat+Junv3bqUHJYQQwrj0Tg4uLi7s2bOH69evA3DgwAE2b96sWatBCCFE7aF3cujbty8XL17UzKwaEhJCbm4uffv2NVhwQgghjEPv5DB8+HD8/f01HdJqtZoePXoQFhZmyPiEEKJMhUVFxg6hVtJ7KKuZmRlz5szhvffe0zwEJ/MqCSGMTWliQuLBc4907K38e5r/Puo5+vjUzrW0K/QQ3JUrV0hISCAjI4MGDRoQEBCAo6OjoWITQghhJHonh//+97+Eh4dz584dzbZ58+axaNEi2rRpY5DghBBCGIfefQ4zZswgLy8PBwcHWrVqhYODA3l5ecycOdOQ8QkhhDACvVsOZ8+excvLixUrVmBmZkZBQQFDhw7lzJkzhoxPCCGEEeidHDw9PXFzc8PMzAwAc3NzPDw8NO+FEELUHnonB39/f2JjY/H09MTDw4PTp0+zceNGJk6cyL59+zTlpP9BCCFqPr2Tw6xZs1AoFMyaNUtre0xMjOa1QqHg+PHjlRedEEIIo6jQUFa1Wv1Y+4UQQtQMeieHkydPGjIOIYQQ1YjeQ1mFEEI8OcpNDufPn2ffvn2ah98OHTrEsGHDCAwM5PPPP6dI5jURQohap9zbStOnT2f//v3s2rWL/Px83njjDbKyslCr1Zw4cQIzMzPGjh1bFbEKIYSoIuW2HM6cOUPr1q0xNzdn9+7dZGZmYmFhwbBhw1CpVCQmJupV0f79+wkICMDLy4v+/ftz7NgxnTIFBQVMnjwZX19fvL296devH7t27ar4VQkhRBlSjx/muznTuJGdCcCN7Ey+mzON1OOHjRxZ9VFucsjJyaFBgwbA/S94gAEDBjB58mS6devGpUuXyq0kPz+fiIgIbt26RVRUFJmZmURERFBYWKhVbt26daxevZqmTZvy5ptvcvLkSd5///1HuS4hhCjV/q0buZp+gaKi+99BRUWFXE2/wP5tm4wcWfVRbnJQKBTcuHEDuN/foFAoaN26NQAWFhZ6VZKcnExmZiZDhw4lJCSEgQMHkpaWxt69e7XKFfdfeHh40KFDB8zNzbGxsanQBQkhRHlad+lFfeeGWNnYYmpmhpWNLfWdG9L6xZ7GDq3aKLfPoVGjRiQlJTFu3Dj27duHQqGgbdu2AFy4cAEHB4dyK0lLSwPQTO9dvA7ExYsXad++vaZc//792bFjB0uWLGHJkiXY2trKxH5CiEr37PPePPu8t7HDqNbKbTmEhYVx7949tmzZglqtplu3bjg4OJCdnc3Bgwfx8vKqcKWlPSx36NAhtm7dSkBAAHPmzKGwsJCoqCh5uE4IIapYuS2Hfv36YWtry86dO3F0dCQ0NBSAy5cvExoayosvvlhuJS4uLppj4P6iQQCurq7k5+djYmKCmZkZGzdu5O7duwwePJgXXniBuLg4du3aRU5ODvb29o98kULUNrt37+bbb78lOzub3Nxc6tati729PSNGjMDX19fY4YlaoNzksGHDBrp06UKXLl20tj///PM8//zzelXi5+eHg4MDK1aswMrKitWrV+Ps7IyzszPe3t506dKFhQsX4urqCsBXX31FSkoKhw4dwtbWFjs7u4pfmRC1WFxcHKdPn9a8z8rKIisri7i4OEkOolKUe1tp4sSJ+Pr6MnbsWNauXcv169crXIlKpSI2NhYrKytmzJiBg4MDsbGxmJhoV1/cWX348GE+/vhjnn32WWJjY1EoFBWuU4jaLDg4GA8PD5RKJQBKpRIPDw+Cg4ONHJmoLcptOezYsYNff/2VzZs3M3XqVN5//31at25Njx496Natm95rSLdp04aEhASd7SkpKZrXKpWK6dOnVyB8IZ5Mvr6++Pr6EhYWRnp6Ok5OTixYsMDYYYlapNzkYGdnR1BQEEFBQdy6dYtt27bxyy+/8NlnnzF9+nS8vLzo3r07AQEBmlFIQggharYKTdltZWVF79696d27NwUFBezcuZNffvmFr7/+WjO1hhBCiJpP7+Tw3//+lxdeeEHz3tzcnJdeeomXXnqJoqKiR+qLEEIIUT3pPWV3aGgoPXv2ZOHChZqhqJqTmJjIUFMhhKhF9E4OderU4fz58/zrX/+ia9euhIeHa55LEEIIUbvofVtp9+7dbNu2jQ0bNrB161a2b9/Ojh07sLGxISAggLCwMBo2bGjIWIUQQlQRvVsOKpWKHj16MGfOHDZv3kyXLl1Qq9X88ccfLF++nN69e/PTTz8ZMlYhhBBVpELLhB45coQpU6bQo0cPtm3bBkCDBg0YNmwYdevWZfbs2QYJUgghRNXS+7ZSv379SElJ0UyC5+vrS2hoKF27dsXExETz1LMQQoiaT+/kcPLkSSwtLenbty+hoaG4ublp7W/fvj337t2r9ABF7SOTxglR/emdHCZPnkxgYCBWVlYl7m/RogUtWrSotMBE7SWTxglR/end5zBjxgzmzp2rte3bb79l4MCBlR6UqN1k0jghqj+9Ww5qtVpn0Z2LFy9y7NixSg9K1G4yaZwQ1V+5ySEqKkrzeufOnZr3arWa3377Te91pIUQugqLilCaVGjQoBBVotzksHbtWhQKBQqFgrNnz3LmzBmt/Q+uAS2EqBiliQmJB8898vG38u9p/vuo5+nj0/iR6xe1V7nJoX///sD9JOHm5oa39/1FuZVKJQ0aNGDw4MGGjVAIIUSVKzc5zJw5E4C0tDR69epFSEiIwYMSNYfcFhGidtK7Q3rZsmWGjEPUUI9zW0RuiQhRfZWZHJo2bcrw4cOJjIykadOmJZZRKBQcP37cIMEJIYQwjjKTw4PDVx8exiqEEKL2KjM5LF26VLMu9NKlS6skICGEEMZXZnJo27at5rWVlRXNmjUzeEBCCCGMT+9hJgMHDmTAgAHExcVx+/ZtQ8YkarnU44f5bs40bmRnAnAjO5Pv5kwj9fhhI0cmhCimd3JQKpUcO3aMqVOn0rlzZ6ZMmcLRo0cNGZuopfZv3cjV9AsUFRUCUFRUyNX0C+zftsnIkQkhiumdHLZv3050dDReXl7cunWLuLg4goKCCAwMNGR8ohZq3aUX9Z0bYmVji6mZGVY2ttR3bkjrF3saOzQhxJ/0fs7Bzs6O0NBQQkNDSU1NZdq0aezevZsTJ04YMj5RCz37vDfPPu9t7DCEEGXQOznA/VlYf/zxR9avX8/FixcBNNMuCyGEqD30Tg5Dhgzh0KFDwP1nHho0aMDAgQNlPQchhKiF9E4OBw8exNTUlJdeeong4GA6deqEQqEwZGxCiFKkHj/Mrk3rdEZ8te/ZT27ZiUqhd3J45513CAwM5Kmnnnqkivbv388HH3zAuXPn8PDwICYmRue5icjISNauXau1zdnZmS1btjxSnULUVsUjvoo9OOJLkoOoDHonh9GjRz9yJfn5+URERKBSqYiKimLBggVERESQlJSk1WcxZMgQOnfuDEBqaipffPEFzz///CPXK0Rt1bpLLwry73Ar9wb5ebdQ1bHCqq6NjPgSlaZKJt5LTk4mMzOTSZMmERISQmZmJvPnz2fv3r1aiwW1aNGCFi1aADBt2jQAWS9CiBLIiC9haGU+5/DwxHul/StPWloaAI6OjgCa+ZqKRzw9LC8vj/Xr1/PMM8/QsWNH/a9GCCFEpTDKxHvlJZSffvqJ3NxcxowZI53eQghhBHpPvHfp0iXc3Nxo3ry5ZltWVhY5OTnlVuLi4gLA5cuXAbhy5QoArq6u5OfnY2JigpmZmab8ypUrMTc3l6evhRDCSPSePiMyMpLExEStbQsXLiQgIKDcY/38/HBwcGDFihV8//33rF69GmdnZ5ydnfH29uaNN97QlD1+/DhHjhyhZ8+e2NvbV+BShBBCVJbHWvy3uCVQHpVKRWxsLFZWVsyYMQMHBwdiY2MxKWHt4ZUrVwLSES2EEMZU7lDW4lFKCoWCpUuX6vQ96Pvrvk2bNiQkJOhsT0lJ0Xo/bdo0zUglIYQQxlFucijuPFYoFCV2JA8dOrTyoxJCCGFU5SaHpUuXolarGT58OD179iQkJAS4P+Ges7OzZjTTk2T37t18++23ZGdnk5ubS926dbG3t2fEiBH4+voaOzwhhHhs5SaH4hFLM2fOxN3dXWu00pMqLi6O06dPa95nZWWRlZVFXFycJAchRK1QZnJYt26dZviqQqHg7NmznD17Vqdcv379DBZgdRQcHMzt27dJTU2lsLAQpVLJs88+S3BwsLFDE0KISlFmcoiMjGTEiBE0b96cyMjIEh9IUygUT1xy8PX1xdfXl7CwMNLT03FycmLBggXGDksIISpNmcmhQYMG2NjYaF7XRoVFRShLGFIrhBBPsjKTw4NTZdfWabOVJiYkHjz3SMfeyr+n+e+jnqOPT+NHOk4IIQzpkX4yFxQUkJCQwPbt2ys7nhoh9fhhvpszTWehldTjh40cmRBCVA69k8O4ceNo164dRUVFRERE8O677zJ69Gi+/vprQ8ZXLRUvtFJUVAhoL7QihBC1gd7JISUlBR8fH+7cuUNycjINGzZEpVKxatUqQ8ZXLbXu0ov6zg2xsrHF1MwMKxtb6js3lIVWhBC1ht4rwV27dg0/Pz/OnTuHWq1m9uzZfP/992zYsMGQ8VVLstCKEKK207vloFKpuHTpEvv370ehUODm5oZSqZT1FoQQohbSu+Xg4eFBcnIyycnJeHh4UKdOHdLT0zWruwkhhKg99G45TJgwAScnJxwcHJg4cSIFBQVkZGTg5+dnyPiEEEIYgd4th9atW/Pbb79pbXsS+xuEEOJJoHdyAEhPT+f48ePcunVLa/uTNn2GEELUdnonh7i4OKZNm0ZhYaHOPkkOQghRu+idHObNm8e9e/ewtrbWzLckhBCidtI7Ody4cYOOHTvy1VdfyfBVIYSo5fQerdS5c2cASQxCCPEE0Lvl4ODgwC+//MLIkSPx9vbG1PSvQ9944w2DBCeEEMI49E4OK1asQKFQ8J///Iddu3Zp7ZPkIIQQtYveyaFNmzaGjEMIIUQ1ondyWLZsmSHjEEIIUY1UaLGf27dvs27dOhYsWEBBQQGXLl0iLy/PULEJIYQwEr1bDleuXGHIkCFkZGQAEBgYSPfu3RkxYgSTJk0yWIBCCCGqnt4th9mzZ3Pp0iXMzc0BqF+/Pj4+PuzYscNgwQkhhDAOvZPDnj176NixI4MGDdJsa9SoEenp6QYJTAghhPHonRxyc3OpX7++1ra8vDy9H4rbv38/AQEBeHl50b9/f44dO1ZiuVOnThEWFoa3tzft2rXj448/1jdEIYQQlUTv5NCoUSN27NhBWloaAAkJCWzevJmGDRuWe2x+fj4RERHcunWLqKgoMjMziYiI0JnE786dO7z22mukpKQQERFBREQElpaWFbwkIYQQj0vvDukhQ4bw/vvva9Z0ePfddwEYPHhwuccmJyeTmZnJpEmTCAkJITMzk/nz57N3717at2+vKZeYmMiVK1eIiYkhICAACwuLil6PEEKISqB3chg4cCDnz5/nu+++Iy8vDwsLC4YOHUpQUFC5xxa3NoqXFHVycgLg4sWLWsnh7NmzAHzzzTdER0djZ2fHlClT6N27t/5XJISoke7evUtaWhp37typ8LENTO4ZICL9nDhxgsi/uxq1/vJYWFjg4uKCmZmZ3ufVOzkoFAomTpxIREQEmzZtwtrampYtW+pd0YPUanWJ2wsKCgD429/+xltvvUVMTAxRUVH4+flhbW39SHUJIWqGtLQ06tatS6NGjSo8wef12/kGiqp8tpYqlBk5Rqu/ydN2Ze5Xq9VkZWWRlpZG48aN9T5vuX0OCxYsYPjw4fzxxx8ATJ06lXfffZdx48bh7+/P0aNHy63ExcUFgMuXLwP3n5kAcHV1JT8/n7t372qV8/f3p0ePHrzwwgvcuXOHq1ev6n1BQoia6c6dOzg4OMjMz5VMoVDg4OBQ4RZZuckhKSmJ7Oxs6tWrx++//87atWtRq9Wo1WquX7/OvHnzyq3Ez88PBwcHVqxYwffff8/q1atxdnbG2dkZb29vzcR9vXv3xtzcnPj4eOLi4ti1axeOjo56dXoLIWo+SQyG8Sifa7nJ4dKlS3h6egKwc+dOAFq0aMHevXtp2rQpR44cKbcSlUpFbGwsVlZWzJgxAwcHB2JjYzEx0a7e0bxm67oAABUOSURBVNGR2bNnk52dzfTp03nmmWf48ssvtaYHF0IIYXjlfuvevHmTunXrAnDkyBEUCgX+/v7Y2NjQsmVLVq9erVdFbdq0ISEhQWd7SkqK1vsePXrQo0cPvc4phKi9CouKUJroN9re1lJVbpl7hUXczL9bZplvvlrEpg0bUCpNUChMiIx+H6/m3nrFUJ5J/wjnk3n/5krGJU4cO0KXbj0BOH3yBFuSfub1iAmlHrvhx3hUFhZ07dmbXzck4tOmHQ5P/a1S4ipNucnBwcGBPXv2cOLECU3LobgjOisrS5M4hBCiMilNTEg8eK7SztfHp+zO2CP/+x87kpNZuuIHzM3NuZ6To+kPrQyfzPs3AFcvZ7Dt1yRNcvB4rikezzUt81j/voGa15s3/sQzjd0MnhzKTcu+vr6kpqYSGBhIZmYmDg4OtGjRAoBjx47h6mq8IVxCCFFZMjOvYWtnq5k/ztbOjr/Vr8+J48cZM+pVwoYOImLcGDKvXQNg7Gsj+SJ2DgMHDuT10CCOHT4EwPlzqbwzZiQRo4YxfmQIl9IuABDU6yUAliyaz/Ejh4gYNYx1q1Zw5OB+/i9yAkVFRYwa1I+bubmamEaHDCQnO4vvv/k38Su/Y+fWLZxJOcns6VOJGDWMfbt2Mm7cOE35nTt38o9//KNSPo9yk8M777xDs2bNUKvVWFlZERMTg0KhYM+ePaSnp/PCCy9USiBCCGFM7dp34MrlKwzsG8DHM2I48N//cu/uXWbPmsnMT2az9PsfCOjbjy/nzdUcU3ivkNWrVxP+xtus+PYrADauj+eVAcF8vngZny38Foe/aU87NHz0OJ5v3pLPFy+jX9AQzXYTExPadfJj945tAKQcP0p9Ryfs7B00ZTp26Yq753NMmPx/fL54GS/4diA1NZXs7GwA4uPjGTBgQKV8HuXeVnJ0dGTNmjXcuHEDKysrlEolAK1bt+bAgQOoVOXf6xNCiOrO0tKSJd+v5NDBA+zft5fJkZMY+dpozp49w/ixrwNQVFSodTuny9//DoBbE0+u/jlU37NZc+KWf0vmtat08OtCAxf9R1t2fqkbK5d8TTf/PiRv+ZXOXbuVWV6hUNC3b1/Wr19PYGAgBw8eZNasWRW99BLpPQzIxsZG+0BTUxlFJISoVZRKJa1faEPrF9rg5u7B6rgfePZZNxYvXV5ieXOz+7egTEyUFBbef0q7S7eeeDZtxr7dO/ngvXf4x4RIWrTS7w7Lc82ak5Gexh/Xc9i9YxuDhr1a7jGBgYGMHTsWc3NzevXqVWnfyxVaCU4IIWqr87+f48L585r3p0+l0LhxY65fz+HI//4HwL27d0k9e6bM81y+lI5TA2deGTCIdh39+P2h8nUsrcjLu13isQqFAt/OL/LVvFhcn2mETb16OmXqWFpqHe/o6Ej9+vX58ssvK+2WElSg5SCEEFWpsKio3BFGFXGvsKjM/bdv5zF71kxyc3MxNVXi4tqQqOgp9BswkNmzPuLmzZsUFhYyOCSEZ93cSz3P9t8289svGzBVmmJn70Bw6HCt/Y3c3FGamDB+VCh/7/Uybu5NtPZ3fqkb74x5lbci3y/x/H/v9TLzP5uFubmKT+bfHwEVEBBAdnY2bm5u+nwUepHkIISolvR9xgEqZ26lps8/z1dLlulst7WzY+HX3+ps//KrrzWv69nasviHdQAEhYQRFBKmU37VxvszWpuamjJ9jvbMEs19WmteezzXlIStu7X2D301XPO644td6fhiV639+/fv12sS1IqQ20pCCFGDBQYGkpKSQt++fSv1vNJyEEKIGiw+Pt4g55WWgxBCCB2SHIQQQuiQ5CCEEEKHJAchhBA6JDkIIaqlgnuFepe1tVSV+8/CTFnueb75ahGDB/QnJHgAoYOCOHrk8ONcgpYP3ntbM6ne+jU/MDZsEJ/GTGHPzmRWfbe0zGMn/eP+UNYrGZfY+uumSoupLDJaSQhRLZmbKvGfvq7Szrdhcj/u3C094Rh6yu4PZs3RvP55XTwxs+fyVP37k/K16+hX5rGlTfdtSNJyEEIISp+yu1/vXsz912cMDQrk1dChXLxwfwrunOxs3pvwNgMGDODt11/l+JH7U2zk3b7Nvz76kDdeDWH8yBB2btsCwKhB/fjj+nXmzZ7FlYx0PnjvbdatWsGvGxJZ8K9P/zxnFtOj32P8qFDGjwrlxNH7LZfSpvuOjBjDiRMnNNcwZMgQTp48WSmfhyQHIYSg5Cm7i1lbW/P9qniCBg1mzqcfA/DZJ7MYEjqMNWvWEDVtJnM/mQHAD8u+xsrKmi+++Y65X3+nM+nePya8h73DU0yfM09rym6ARXM/w6uFD3MXL+dfi5bQsNGzWvsfnu67e+8AzXMO586dIz8/n+eee65SPg9JDkIIwV9Tdke9PwVbOzsmR04icf2PAPTo1fvP//pz5PD9FsK+Pbv59KOZ9O3bl5h/TuL27dvk3b7Nof37eLn/XxPgWde10a2sFIcP7Kf3n6u+KZVKrKytyyzfscvf2bp1K3fv3mXNmjUEBgaWWb4ipM9BCCH+9PCU3T8nrgdAofirjOLPN0VqNYuXLsfRzoZTGTnGCBcLCws6dOjA5s2b2bBhQ6U+LS0tByGEoOQpu52ebgDAL5vujxD6JWkTzb29AWjn2564ld9ryqeePgVAy9Zt+WntGs32m7k39I6hRasX+PnH+1/whYWF3Lp5U2t/SdN9BwUFERMTQ/PmzalXwhTfj0paDkKIaqngXiEbJvertPPduXuvzP2lTdm9MzmZ3NwbhAQPwMzMnA9n3l9pbcK7kXzy0QwCAgLIyy+gmbcP/5jwHoOGvcqC2E/5x4ihmChNGDJ8FB38XtIrxvDxbzNv9kf88nMCJiYmjHvnXZ5r1lyz/+HpvvsFDcHLywtra+tKvaUEkhyEENWUuWn5zyUUM+SU3QChYSN44823tbbZ2tkxfdYn2FqqtG4r1bG05O2oKTrnKJ7S++HX3fz70M2/DwB29g5ET/9E59iypvu+cuUKarWaTp06lXeJFSK3lYQQoobasulngoODeeuttzCpwPoX+pCWgxBClGHdzxuNHUKpuvbszZgRIQY5t7QchBDVhlqtNnYItdKjfK6SHIQQ1YKFhQVZWVmSICqZWq0mKysLCwuLCh0nt5WEENWCi4sLaWlpXLt2rcLH5hWUPRLJkOqYm3L1j9vlFzSQwuuW5ZaxsLDAxcWlQueV5CCEqBbMzMxo3LjxIx2bePBcJUejvz5NG/NOJU4QWFGVOdz3QVV2W2n//v0EBATg5eVF//79OXbsmE6ZtLQ0PD09tf5Nnz69qkIUQgjxpyppOeTn5xMREYFKpSIqKooFCxYQERFBUlISSqXuWObBgwfTtm1bgEf+JSGEEOLRVUlySE5OJjMzk0mTJhESEkJmZibz589n7969tG/fXqe8l5cXXbt2pU6dOlURnhBCiIdUSXJIS0sDwNHREQAnJycALl68WGJyeP/994mOjsbd3Z3p06fTsmXLMs+fnp5e6Y+OV5WvAWOmwMDAslegqu7k83t88hk+npr8+aWnp5e6zygd0qUNVbO0tGT8+PE0bdqU8+fPM3v2bCZOnMivv/5a5vn27NljiDCFEOKJVSXJoXgI1eXLl4H7c4EAuLq6kp+fj4mJCWZmZtjb2/PGG29ojktMTOTYsWPk5+ejUqmqIlQhhBBUUXLw8/PDwcGBFStWYGVlxerVq3F2dsbZ2Rlvb2+6dOnCwoULiYuL48iRI3h7e5OWlsaJEyd47rnnJDEIIUQVq5KhrCqVitjYWKysrJgxYwYODg7ExsbqTBTVqFEjUlJSmDFjBt9//z2dO3cmNja2KkIUQgjxAIVanlUXQgjxEJlbSQghhA5JDkIIIXRIcqgCD04LMm/eX6s4/fOf/9Rs37NnD56enkybNs2IkRpeTEwMHTp0wNPTk9dff11rX/Fn4eXlRceOHRk3bhwnTpwo9VxZWVlERUXRoUMHvL29CQgIICkpCYCuXbvi6enJ3Llzgb/+Nyiuc+7cuXh6etK9e3cKCwsBGDZsGJ6enmRnZxvi0h/b77//zrBhw2jXrh0+Pj68+uqrXLhwAUDz9+Pp6UmLFi34+9//zpQpU8q8lqNHjzJq1ChatWpFq1atCA0N5ezZs1p/r8XDxOPj4/H09GTx4sXAX59VZGSk5nyenp706dPHgJ/A4wsKCsLHx4cWLVoQGBjIvn37APn8SiLJoYqtXbsWtVrNrVu32LBhg7HDMYrevXuXus/JyYmYmBj8/f3Ztm0bQ4cO5cyZMzrl7ty5Q2hoKPHx8bRr147333+f9u3b68zZtXz5cm7dulVqfRcuXODnn39+9IupQsXLQY4fP54BAwbwn//8h8mTJ2uV6dSpE9HR0TRp0oQffviBYcOGcefOHZ1znT59mtDQUPbs2cOQIUN47733eOqpp/j999+1yi1cuLDMmBITE7l06dJjX1tV8fHxITo6mnHjxnHy5Emio6O19svn9xeZlbUKubq6cvHiRfbs2UNaWhr37t3D0dFR89zHkyA6Opq0tDSWLSt5rd66devSr18/+vXrx1NPPcWcOXNYtGgRH3/8sVa5hIQEUlNT6dSpE3PmzNFsLyoq0rw2MzMjLy+PlStX0rNnzxLrs7a2ZtGiRdXqF1tpfHx8WL58ueZ9QkKCTuJ85plnCAoKIigoiJEjR7Jz504SExMZOHCgVrlFixaRl5fHhAkTGD16NACDBg2iqKhI82VlbW3Nzp07OXLkSKkxqVQqvvrqK6ZM0V0zuTqKiooiJyeHtLQ0vvzySxQKhdZ++fz+Ii2HKuTm5kaLFi1Ys2YNa9asoVu3btjY2Bg7rGrLz88PuN98f1hxC6Fz585a2x8cHm1mZsaAAQP45ptvKCgoKLGOwYMHc+rUKbZs2VJZYRuMubm55vWRI0e4fv06bdq0KbX8435+rVu3pkmTJmX++h08eDBr1qwhKytLv4swstzcXNq3b09QUBBmZmbExMSUWvZJ//wkOVSxAQMGsHHjRg4cOFBj54OqKsWjrB/+dVfatpK89tpr5OTkEB8fX+L+5s2b06FDBxYtWvTogVaxs2fPMnbsWJydnXVuizzocT8/hUJBeHg4v/76K2fPni2xzJAhQ1CpVHzzzTd6Rm9clpaWfP3110RHR5Ofn8/nn39eatkn/fOT5FDFXn75ZZRKJU5OTnTs2NHY4VRrO3bsAKBZs2Y6+4q37dy5U2v7g7eVAJydnenTpw8rVqwotZ7XX3+dQ4cOcfz48ccN2eDOnDlDWFgY5ubmLFmyhPr165daVp/Pr7hMsYc/v5dffhkXFxdWrlxZYh2WlpaEhoaW+flWJ6ampnTs2JFhw4bh7e3Nnj17Su10ftI/P+lzqGLW1tbMmDEDKysrnSfEAQ4dOsSnn34K3F/a78G5pmqDrVu3curUKQAyMjJYtWoVbdq0oVGjRsD9Zv+6des4evQoK1euxNLSUnNP90F9+vRh8eLFJCcnM2HCBHx9fUlNTcXc3Jy3335bq+zo0aP58ccfS43J19eXli1bcujQocq7UAPIyMggLCyM69ev89Zbb3H48GEOHz7Myy+/rClz/vx5Vq1axdatW9mxYwfu7u4l9qeEh4eTlJREbGwsN27cwNXVlT179tC7d288PT015ZRKJa+99hpTp04tNa6wsLBq+cv3Ydu3b2fDhg34+Phw+fJlDh48yFNPPYWdnZ2mjHx+f5HkYARljdY5duyY5n5m3bp1a11yWLx4MXv37gUgJSWF6OhoZs6cqUkOly9fJjo6GltbW1588UXGjx+Pu7u7znksLCxYvnw5n3zyCVu3biUpKQlXV1feeustnbJubm50795dM8y1JK+//jpjx46tnIs0kAsXLmjuTc+ePVuz/cHksGPHDvbt28ff/vY3Bg8ezFtvvVXiwvIeHh4sX76cOXPmaDq5n3vuOZ555hmdsoGBgcybN4+rV6+WGJetrS2DBg2qll9wD6pXrx6HDx8mMTERc3NzWrVqxaRJk7RuEcnn9xeZPkMIIYQO6XMQQgihQ5KDEEIIHZIchBBC6JDkIIQQQockByGEEDokOQhRxR6esVMfxbN4Fs8yK4ShSXIQ4k/FX8APT6UMEBsbq9nXtWtXI0UoRNWR5CBECRITE7l27Rpwf3rw6ji9gRCGJMlBiIfY2tpy9+5dzZOv8fHx5OTkYGtrq1Xu5s2bzJ49m169etGiRQu6devG1KlTtWbYzMjIIDw8HB8fH7p3717iGh5FRUWsWrWK/v374+Pjg5+fH5MmTeLy5cuGvVAhyiDJQYiH+Pn54ezszMqVK7l9+zZLlizB0tKSV155RVOmqKiI1157jUWLFnHz5k369OmDiYkJK1euZPDgweTl5VFUVMSYMWNITk7GxsaGtm3bMmvWLJ36PvvsM6Kjo7l27Ro9evTA3d2d9evXM3jwYG7evFmVly6EhiQHIR6iVCoZNmwY169f5+233+b3339nwIABWmtvHDhwgIMHDwKwYMECpk+fzvLly1EqlVy4cIGkpCQOHz7MyZMnAfjyyy+ZPn06X3zxhVZdBQUFmhaKl5cXNjY2uLm5oVKpyMjIKHM+KCEMSSbeE6IEQUFBfPHFF2zduhWlUsnw4cNZt26dZv+DSzs2adIEgPr162NnZ0dmZiYZGRmYmZlpyhRPHlhctlhOTg55eXkA/PbbbzpxZGRkVN5FCVEB0nIQogTW1tYEBwcD0K1bN1xdXbX2N2jQQPP69OnTAFy7do2cnBwAnn76aZ5++mlNmeLlPIunKy9mZ2enmfXz448/JiUlRfMvOTmZUaNGVfKVCaEfaTkIUYoxY8bQuXNn3NzcdPa1atWKFi1a8L///Y+xY8fi5+fHvn37KCwsxMXFhe7du2NhYUGTJk04deoUY8eOpVOnTjqLw5ibmzN06FC+/vprpkyZwrZt26hTpw7nz5/nwIEDJCUl4eLiUlWXLISGtByEKEW9evXo0KEDjo6OOvtMTExYvHgxr732GnXq1GH9+vXcvXuXQYMGaRYpMjExYcGCBXTu3JkbN26we/duJkyYoHOuSZMm8eGHH+Lm5sa2bdvYsGEDf/zxB8OHD9daiEaIqiTrOQghhNAhLQchhBA6JDkIIYTQIclBCCGEDkkOQgghdEhyEEIIoUOSgxBCCB2SHIQQQuiQ5CCEEELH/wNfbzXlqSaX/gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TYVMUAOUa3sX",
        "outputId": "24275ffe-156d-48c0-cacd-57ae660e6742"
      },
      "source": [
        "data = 'PD'\n",
        "print(\"-----\", data, \"----\")\n",
        "hvlv_pd_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hvlv_pd_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hvlv_pd_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_HVLV_CSP_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hvlv_pd_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HVLV_CSP_'+str(data)+'.mat', 'wtd_f1_list')\n",
        "\n",
        "data = 'NC'\n",
        "hvlv_nc_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HVLV_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hvlv_nc_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HVLV_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hvlv_nc_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_HVLV_CSP_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hvlv_nc_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HVLV_CSP_'+str(data)+'.mat', 'wtd_f1_list')\n",
        "\n",
        "data1 = {'F-Score': hvlv_nc_ml[0], 'Data': len(hvlv_nc_ml[0])*['NC'], 'Model': len(hvlv_nc_ml[0])*['ML']}\n",
        "df1 = pd.DataFrame(data1)\n",
        "data2 = {'F-Score': hvlv_pd_ml[0], 'Data': len(hvlv_pd_ml[0])*['PD'], 'Model': len(hvlv_pd_ml[0])*['ML']}\n",
        "df2 = pd.DataFrame(data2)\n",
        "data3 = {'F-Score': hvlv_nc_1d[0], 'Data': len(hvlv_nc_1d[0])*['NC'], 'Model': len(hvlv_nc_1d[0])*['1D CNN']}\n",
        "df3 = pd.DataFrame(data3)\n",
        "data4 = {'F-Score': hvlv_pd_1d[0], 'Data': len(hvlv_pd_1d[0])*['PD'], 'Model': len(hvlv_pd_1d[0])*['1D CNN']}\n",
        "df4 = pd.DataFrame(data4)\n",
        "data5 = {'F-Score': hvlv_nc_2d[0], 'Data': len(hvlv_nc_2d[0])*['NC'], 'Model': len(hvlv_nc_2d[0])*['2D CNN']}\n",
        "df5 = pd.DataFrame(data5)\n",
        "data6 = {'F-Score': hvlv_pd_2d[0], 'Data': len(hvlv_pd_2d[0])*['PD'], 'Model': len(hvlv_pd_2d[0])*['2D CNN']}\n",
        "df6 = pd.DataFrame(data6)\n",
        "data7 = {'F-Score': hvlv_nc_3d[0], 'Data': len(hvlv_nc_3d[0])*['NC'], 'Model': len(hvlv_nc_3d[0])*['3D CNN']}\n",
        "df7 = pd.DataFrame(data7)\n",
        "data8 = {'F-Score': hvlv_pd_3d[0], 'Data': len(hvlv_pd_3d[0])*['PD'], 'Model': len(hvlv_pd_3d[0])*['3D CNN']}\n",
        "df8 = pd.DataFrame(data8)\n",
        "\n",
        "valence_fscore_df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], axis=0).reset_index()\n",
        "print(valence_fscore_df)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- PD ----\n",
            "    index   F-Score Data   Model\n",
            "0       0  0.822852   NC      ML\n",
            "1       1  0.853121   NC      ML\n",
            "2       2  0.841493   NC      ML\n",
            "3       3  0.839066   NC      ML\n",
            "4       4  0.838364   NC      ML\n",
            "..    ...       ...  ...     ...\n",
            "75      5  0.951466   PD  3D CNN\n",
            "76      6  0.956859   PD  3D CNN\n",
            "77      7  0.960814   PD  3D CNN\n",
            "78      8  0.968789   PD  3D CNN\n",
            "79      9  0.952308   PD  3D CNN\n",
            "\n",
            "[80 rows x 4 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "id": "M25FWf5ueCRt",
        "outputId": "f4668f57-68dd-4e85-e2d6-3dbd41923e2d"
      },
      "source": [
        "x = \"Model\"\n",
        "y = \"F-Score\"\n",
        "hue = \"Data\"\n",
        "ax = sns.barplot(data=valence_fscore_df, x=x, y=y, hue=hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.6)\n",
        "ax.legend(loc='upper left', bbox_to_anchor=(0,1))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('F-Score',fontsize = 13, weight='bold')\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "ax.set_yticklabels([\"0.60\", \"0.70\", \"0.80\", \"0.90\", \"1.00\"], size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=valence_fscore_df, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"PD\"), (\"ML\", \"NC\")),\n",
        "                                 ((\"1D CNN\", \"PD\"), (\"1D CNN\", \"NC\")),\n",
        "                                 ((\"2D CNN\", \"PD\"), (\"2D CNN\", \"NC\")),\n",
        "                               ((\"3D CNN\", \"PD\"), (\"3D CNN\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "plt.savefig('f-score_valence.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_NC v.s. 1D CNN_PD: t-test independent samples with Bonferroni correction, P_val=5.603e-01 stat=1.544e+00\n",
            "ML_NC v.s. ML_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-5.793e-02\n",
            "2D CNN_NC v.s. 2D CNN_PD: t-test independent samples with Bonferroni correction, P_val=3.132e-01 stat=1.867e+00\n",
            "3D CNN_NC v.s. 3D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=5.571e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEJCAYAAABohnsfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de1RVdd7H8Tccbop3NDAgx1B5LMLwUqjF9HgbbyjgNVEb79pK1NRGJtQZb00zpYu0VMrpMW3k8YKaWEYtU9Ixb2mampiWenyQBC2v3M/zh3EmBHWrcM4BPq+1XMPZ+8fe372ndT7s32/v33ayWCwWREREDHC2dwEiIlJxKDRERMQwhYaIiBim0BAREcMUGiIiYpiLvQsob08//TS+vr72LkNEpEI5d+4cu3fvLrG80oeGr68vSUlJ9i5DRKRCiYqKKnW5uqdERMQwhYaIiBim0BAREcMq/ZhGafLy8jCbzWRnZ9u7lPvm4eGBn58frq6u9i5FRKqQKhkaZrOZmjVr8rvf/Q4nJyd7l3PPLBYLWVlZmM1mGjdubO9yRKQKqZLdU9nZ2Xh5eVXIwABwcnLCy8urQl8piUjFVCVDA6iwgVGkotcvIhVTlQ0NERG5dwoNoKCw0ObbCwwM5G9/+5v187Jly1i4cKH184YNG+jZsyfh4eFERESwbNmyMq1RROR+VMmB8FuZnJ1JPvBDmW2vZ8jdB6fd3NxISUlh9OjR1KtXr9i67du3s3z5cpYtW4a3tze5ubls2LChzOoTEblfutKwExcXFwYMGMDy5ctLrEtISOCVV17B29sbuBkw/fv3t3WJIiIlKDTsKDo6mk2bNnHlypViy0+cOEFQUJCdqhIRuT2Fhh3VqFGD3r1788EHH9i7FBERQxQadvbCCy+wbt06bty4YV3WpEkTvv32WztWJSJSOoWGndWpU4euXbuydu1a67IxY8bwj3/8gwsXLgCQm5vLmjVr7FWiVAFms5lu3boRFxdHjx49GD58ONnZ2XzwwQd0796d8PBwJk2aZO8yxQHo7ilu3iJr5I6ne9meydl4Hg8fPpwPP/zQ+vn3v/89mZmZDBs2DIvFgpOTE3369Cmz+kRKc/r0aebPn8+cOXOYMGECn376KQkJCWzduhU3NzcuX75s7xLFASg04J6+4MtqewcOHLD+XL9+fb755pti6/v06aOgEJvy8/OjefPmADz++OOcO3eOwMBApkyZQseOHenUqZOdKxRHoO4pEQFu3tpdxGQyUVBQQEJCAoMGDeLo0aP07duX/Px8O1bo2KpKF5+uNESkVIWFhaSnpxMaGkqrVq3YvHkz169fp1atWvYuzWFVhS4+hYaIlKqwsJCpU6dy9epVLBYLQ4cOVWDcRVXo4lNoiAh+fn4kJydbP48YMcKO1VRct3bx5eTkkJCQwN69e/niiy9YsmQJmzZtwsWl4n71VtzKReS2kpKS2LJli11r6Nq1K1FRUXatwd4qYxefTQbC58yZQ7t27QgMDGTMmDG3bbd//37Cw8MJCgoiMjKSI0eOWNd9/vnndO7cmSeeeIIhQ4Zw9uxZW5QuUiFt2bKFtLQ0u+0/LS3N7qHlCIq6+MLDw4mMjKwUXXw2u9Lo3r07K1asuO36nJwcYmJicHd3JzY2liVLlhATE0NKSgoXL15k0qRJNGnShFdeeYX58+czbdq0Ys82PIjc/ALcXExlsi2j22vevDnNmjWjoKCARx99lNdff51q1apZl+fn52MymYiIiOCPf/wjzmV8W7BUfs2aNSMhIcEu+x49erRd9mtPVaWLzyahERcXh9lsvmNopKamkpmZydSpU4mOjiYzM5N33nmHPXv2cPz4cXJzcxk9ejTdunXj8OHDbNy4kTNnzvDII488cH1uLia6zS27qcc/eTXirm08PDzYuHEjAJMnTyYxMZFhw4YVW56VlcXkyZO5evUqMTExZVafiNyZI3TvgWN28TnMn69msxnAOh24j48PAGfPnr3jusqgdevWnD59usRyLy8vZs+ezYcffojFYrFDZSJVk72798Bxu/gcdiD8Tl+SlekLND8/n9TUVJ599tlS1/v7+1NQUEBWVhb169e3cXVSUfXq1atK778s2LN7Dxy3i8+uoZGbmwvcvE3Nz88PgPPnzwOQkZEB3PzSvH79+m3XVVTZ2dn07t0buHml0bdvXztXJJVJz549q/T+pfzYJDS2bdtmvdRLT09nzZo1tGnThuHDh3Pp0iUOHDhAWFgYXl5erFq1Ck9PT9auXYuvry9PPfUUAQEBvPHGG7z33ntkZWXx2Wef0apVqzIZz7CX345d3MnZs2cxmUx4eXnZoCoRkTuzyZjGsmXLePPNNwE4fvw4cXFxfP3118XauLu7Ex8fj6enJ/PmzcPLy4v4+HhMJhMPPfQQ8+fP5/Lly7z++us89thj/O1vf7NF6XZ18eJFZs6cSXR0NE5OTvYuR6TK6NWrl9272ByhhtI4WSrTAEEpoqKiSEpKKrbs2LFj1kf9wT633IaEhBSb6bbIrbfc9u7dm2HDhpV6y+2txyEiUlZK++4EBx4It6WyDAyj2ystMOBmEIiIOCqHueVWREQcn0JDREQMq7KhUdGHcip6/SJSMVXJ0PDw8CArK6vCfvFaLBaysrLw8PCwdykiUsVUyYFwPz8/zGYzFy5csHcp983Dw8P6QKSIiK1UydBwdXWlcePG9i5DRKTCqZLdUyIicn8UGiIiYphCQ0REDFNoiIiIYVVyIFwqH7PZzKhRo2jVqhUHDhzA29ubd955h9WrV5OYmIjJZKJJkyYsWLDA3qWKVGgKDak0Tp8+zfz585kzZw4TJkzg008/JSEhga1bt+Lm5sbly5ftXaJIhafuKak0/Pz8rLP+Pv7445w7d47AwECmTJnCxo0bMZnKdmJKkapIoSGVhpubm/Vnk8lEQUEBCQkJDBo0iKNHj9K3b1/y8/PtWKFIxafQkEqrsLCQ9PR0QkNDmTJlCleuXLG+OlhE7o/GNKTSKiwsZOrUqVy9ehWLxcLQoUOpVauWvcsSqdAUGlIp+Pn5kZycbP08YsQIO1YjUnkpNMQhJSUlsWXLFrvW0LVrV6Kiouxag4ij0ZiGOKQtW7aQlpZmt/2npaXZPbREHJGuNMRhNWvWjISEBLvse/To0XbZr9jXV199xf/8z/9w8eJFrly5Qs2aNalXrx5//OMfCQ0NtXd5DkFXGiIiv1q9ejUnTpwgKyuL3NxcsrKyOHHiBKtXr7Z3aQ5DoSEi8qv+/fvTtGlT64OgJpOJpk2b0r9/fztX5jjUPSUOqVevXlV6/2IfoaGhhIaGMnToUM6dO4ePjw9Lliyxd1kORaEhDqlnz55Vev/3S33yUt4UGiKVSFGffJGsrCyysrJYvXq1QqOCcPTg15iGSCWiPvmKz9EH43WlIVKJqE++4uvfvz/Xr1/n1KlTFBQUYDKZePTRRx0m+BUaIiIOxNGDX91TIlIpFRQWVun9lxddaYhIpWRydib5wA/39bvXcvKt/3u/2+gZ0vi+fs/R6UpDREQMU2iIiIhhCg0RETFMoSEiIoYpNEQclL3vvrH3/u3h1NFDfLhgFpcvZgJw+WImHy6Yxamjh+xcmePQ3VMiDkp3/9je/m1b+OncGevnwsICfjp3hv3bP+XRx4LvaVu5+QW4uZjKukS716DQEBH5VavnupKbk821K5fJuXEN92qeeNasRavf/+Get+XmYqLb3A33XUvuxWsAnLt47b6388mrEfe9/9tRaIiI/OrRx4Lv+YqiqrHZmMb+/fsJDw8nKCiIyMhIjhw5UqJNXl4ec+fOpV27drRq1Yq//OUv5OXlWdcvWrSI0NBQQkJCmDZtGjk5ObYqv9yZzWa6detGXFwcPXr0YPjw4WRnZ/PBBx/QvXt3wsPDmTRpkr3LFJEqziahkZOTQ0xMDNeuXSM2NpbMzExiYmIoKCgo1m7FihV88MEHdOzYkT59+rBq1SpWrFgBwGeffcbChQtp27YtQ4YMYf369Q41H0tZOH36NNHR0WzevJmaNWvy6aefkpCQwIYNG9i0aRN//etf7V2iiFRxNgmN1NRUMjMzGTRoENHR0fTt2xez2cyePXuKtSv6/NJLLzFlyhQA1q9fD0BSUhIA06dP5+WXX6Zhw4bWZZWFn58fzZs3B+Dxxx/n3LlzBAYGMmXKFDZu3Gid7lpExF5sEhpmsxkAb29vAHx8fAA4e/ZssXb16tUDYNeuXXz55ZfFftdsNuPq6mpt4+PjQ0ZGBrm5ueV/ADbi5uZm/dlkMlFQUEBCQgKDBg3i6NGj9O3bl/z8fDtWKCJVnV0Gwi0WS6nLR40axZdffsmf/vQnXFxc8PDwwN3d/Z62UZkUFhaSnp5OaGgorVq1YvPmzVy/fp1atWrZuzQRqaJsEhp+fn4AnD9/HoCMjAwA/P39ycnJwdnZGVdXVxo3bkxKSgppaWnUqlWLfv360aRJE+s20tLSyMrKwsvLi4yMDLy9vYv9dV7ZFBYWMnXqVK5evYrFYmHo0KGVPjAc/VWXju7U0UPs+nRDiYfT2v4hQncFSZmwSWiEhYXh5eXFqlWr8PT0ZO3atfj6+uLr60twcDDPPfccS5cu5dixY2zduhUfHx82b97MlStXGD58OACRkZFs3bqVuXPn4ufnR3p6OuPGjbNF+Tbh5+dHcnKy9fOIESPsWI396B3XD6YsH04TKY1NQsPd3Z34+HhmzZrFvHnzaNKkCbNnz8bZueSQyrp16/jpp5946KGHmDlzJh06dACgS5cuvPjii/zrX/8iNzeX3r17M3bsWFuUf1+SkpLYsmWLXWvo2rUrUVFRdq3hXjn6qy4dXVk+nCZSGpuNabRp04ZNmzaVWH78+HHrz82bN2fr1q233caECROYMGFCudRX1rZs2UJaWhrNmjWzy/7T0tIAKlxoOPqrLh2dHk6r+Aozf6Dgh92QffnmguzL5O1NxNT4aZzr239qFz0RXo6aNWtGQkKCXfY9evRou+xXRB5MwZkDWK5c+M8CSyGWKxcoOHNAoSEiIsWZHgmhoCAXS851yM8GFw+c3KtjeiTE3qUBCg0pRwWFhZhKGbeqKvsXuR/O9Rs7xBXF7Sg0ykmvXr2q9P5BU3uLVEYKjXLSs2fPKr1/qZr0nE3ld0/X7unp6SxatIi4uDiuXLnC3r17yczMLK/aRKSCKXrOJisri9zcXLKysjhx4gSrV6+2d2lSRgyHxqlTp+jduzdvv/0269ato7CwkBEjRrB48eLyrE9EKpD+/fvTtGlT6+SaJpOJpk2b6jmbSsRwaLzxxhtcvnzZOmFg7dq1adWqFbt27Sq34qTq0TuaK7bQ0FCWLFlinZS06DkbdU1VHobHNL755hu6dOmCj4+P9R0X/v7+HDx4sNyKk6rH3tNgqE9e5M4Mh0ZeXh6urq7Fll28eLFSTxgotmfvaTA095XInRkOjUcffZQdO3YQGBgIwJIlS9i2bRvBwZqyQMqOvafB0NxXIndmeExj5MiR/PLLL+zduxeA+Ph4CgoKGDZsWLkVJ2Jr6pMXuTPDodGpUydmzpyJv78/JpMJPz8/4uLi6Ny5c3nWJyIiDsRQ91ReXh6bN2/Gz8+PlJSU8q5JpEzk5hfg5mK/96rbe/8i5cFQaLi6ujJ9+nQiIyN59tlny7smkTLh5mKi29wN9/W7uRevAXDu4rX73sYnr0bc1++JODLD3VNPPvkkZrO5PGsREQeSm19QpfcvpTN891TLli159913mTx5Mk899RTu7u7WdRER+otKKgdHfwGOLelKTUpjODSWLl2Kk5MTH3/8MR9//LF1uZOTk0LDgejhtAfj6C/AEbE3w6Hx8MMPl2cdlYa9v7T1cNqDcfQX4IjYm+HQuNO7u+U/7P2lrYfTHoyjvwBHxN7u6X0aOTk5pKSkkJ6ezsMPP0ynTp3w8PAor9rs7n7e/FaWX9o5eQW4u+qWURFxHIZD4+LFizz//POcOfOfyeT8/f1ZtWoVXl5e5VKcvd3Pm+dOnczgl+s5WCw3P1ss8Mv1HPaczCDT/d621TOk8T0PIuZ9nYTl53PWzwUFBZw4cYK4f7yDa8vz97QtDUSKyK0M/xn91ltvcfr0aZydnfHy8sLZ2ZmzZ8+ycOHC8qyvwimapbWw8Obtgr+dpdUWTI+E4FSzAbh5grMJ3DxxqtlAffIiUiYMX2ns3LmTRo0asXLlSho0aMCFCxcYPHgwO3bsKM/6Khx7z9KqPnmxJ92yXPkZDo0LFy4QHh5OgwYNAGjQoAFPPfUUmzZtKrfiKiJ7z9IqYk+6ZbnyM9w91bBhQ7Zv386xY8fIy8vj2LFjpKamWmcDFRFR92jlZ/hKo1OnTrz77rtERUUVWz5y5MgyL0pEKiZ1j1Z+hq80xo4dy5NPPonFYrH+Cw4OZsyYMeVZn4iIOBDDVxqenp6sWrWK/fv3k56eTsOGDWnVqhVOTk7lWZ+IiDgQw6Fx8uRJLl68SMuWLWndujX5+fns27ePevXqERAQUJ41ioiIgzAcGi+//DLOzs6sX7/+5i+6uPD6669TUFBgXSYiIpWb4TGN06dP88QTTxRb9thjj/Hjjz+WdU0iIuKgDIdGtWrV+OGH4tNg/PDDD5V67ikRESnOcPdUQEAA+/bt45VXXiEkJIQDBw6wb98+WrduXZ71iYiIAzEcGsOHD2ffvn1s2rSp2FPgI0aMKJfCRETE8RjunurQoQN//etf8ff3x2Qy4e/vz1/+8heee+65cixPREQcyT29T2PAgAEMGDCgvGoREREHd0+hAZCXl0dycjLXrl2jbdu2ekZDRKQKuWv31Jw5c+jcuTO//PILcHM6kT//+c/MnTuXyMhIdu3aZWhH+/fvJzw8nKCgICIjIzly5EiJNrm5ubz66quEhoYSHBxMREREse0nJiYSFhZGcHAw48aN49KlS0aPU0REysBdQ2P37t3Url2b2rVr891337Fz504sFgs1atQgNzeXJUuW3HUnOTk5xMTEcO3aNWJjY8nMzCQmJoaCgoJi7TZs2MDatWtp3rw5EyZM4LvvvmP69OkAHD16lJkzZxIQEMD48ePZvn07r7322n0etoiI3I+7hkZGRgZNmjQB4KuvvgKgffv27N27l9atW3PixIm77iQ1NZXMzEwGDRpEdHQ0ffv2xWw2s2fPnmLtCgsLAWjatCnt2rXDzc2NWrVqAZCUlATApEmTGDVqFCEhIWzevJmcnJx7OFwREXkQdw2NGzdu4O7uDsDhw4dxcnKiY8eOwM0v96tXr951J2azGQBvb28A6zs4zp49W6xdZGQknTt3Zvny5URERFCtWjXr1URp28jPzyc9Pf3uRykiImXirqHRoEEDUlNT+fzzz62vdg0JuflClYyMDOuVwL2wWCylLj948CDbtm0jPDycBQsWUFBQQGxsbKntb7cNEREpP3cNjf/+7/8mPT2d8ePH88svv+Dv70/z5s0pLCzk0KFDNG589xeu+Pn5AXD+/HngZtgA+Pv7k5OTQ15eHgBbtmwhLy+PgQMH0r17d4KCgjhy5AiXLl2ybqPodzMyMnBxcaFhw4b3cdgiInI/7nrL7cSJE7lw4QL//ve/8fb2Zu7cucDNAfL8/HyeeeaZu+4kLCwMLy8vVq1ahaenJ2vXrsXX1xdfX1+Cg4N57rnnWLp0Kf7+/gC89957HD9+nIMHD1KnTh3q1q1LREQEK1asYMGCBbRr144DBw7Qo0cPa9eZiIiUv7uGRs2aNXnrrbdKLG/btq11YPxu3N3diY+PZ9asWcybN48mTZowe/ZsnJ2LX+hER0dz8uRJvvjiC3bt2kVAQACvvPIKTk5OBAUFMWPGDJYsWcK+ffsICwsjNjbW4GGKiEhZuOeH+w4fPszJkyeJiIi4p99r06ZNsTmrihw/ftz6s7u7u/VKpjTR0dFER0ff035FRKTsGJ57qsimTZv0F76ISBV1z6EhIiJVl0JDREQMu+fQKJoTSkREqp67hkbHjh1ZvHgxAMnJyezcuVNzPomIVFF3DY1z587x888/A3Do0CE2bNhQ7kWJiIhj0piGiIgYptAQERHDDD3ct3PnTmJjYzl06BBAsec0nJycmDdvXvlUJyIiDsVQaJw8eZKTJ09aZ5YtGtewWCwKDRGRKuSuodGmTRtb1CEiIhXAXUNjxYoVtqhDREQqAA2Ei4iIYQoNERExTKEhIiKGKTRERMQwhYaIiBim0BAREcMUGiIiYphCQ0REDFNoiIiIYQoNERExTKEhIiKGKTRERMQwhYaIiBim0BAREcMUGiIiYphCQ0REDFNoiIiIYQoNERExTKEhIiKGKTRERMQwhYaIiBim0BAREcMUGiIiYphCQ0REDFNoiIiIYQoNERExTKEhIiKG2Sw09u/fT3h4OEFBQURGRnLkyJESbaZNm0ZgYGCxfx06dLCuX7RoEaGhoYSEhDBt2jRycnJsVb6IiGCj0MjJySEmJoZr164RGxtLZmYmMTExFBQUFGv3/PPPM3/+fObPn89LL70EwGOPPQbAZ599xsKFC2nbti1Dhgxh/fr1LFmyxBbli4jIr2wSGqmpqWRmZjJo0CCio6Pp27cvZrOZPXv2FGvXokULevToQY8ePbh06RIAAwcOBCApKQmA6dOn8/LLL9OwYUPrMhERsQ2bhIbZbAbA29sbAB8fHwDOnj1bavsbN27w0Ucf0ahRI9q3b2/dhqurK/Xq1bNuIyMjg9zc3PIuX0REfmWXgXCLxXLH9Zs3b+bKlSv0798fJyen+9qGiIiUPZuEhp+fHwDnz58HICMjAwB/f39ycnLIy8sr1j4xMRE3NzeioqKKbSMvL4+srCzrNry9vXFzc7PFIYiICOBii52EhYXh5eXFqlWr8PT0ZO3atfj6+uLr60twcDDPPfccS5cuBeDo0aMcPnyY8PBwa1cUQGRkJFu3bmXu3Ln4+fmRnp7OuHHjbFG+iIj8yiZXGu7u7sTHx+Pp6cm8efPw8vIiPj4eZ+eSu09MTAT+MwBepEuXLrz44ovs3LmTFStW0Lt3b8aOHWuL8kVE5Fc2udIAaNOmDZs2bSqx/Pjx48U+z5o1i1mzZpW6jQkTJjBhwoRyqU9ERO5OT4SLiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimM1CY//+/YSHhxMUFERkZCRHjhwptV1aWhpDhw4lODiYp59+mr///e/WdYmJiYSFhREcHMy4ceO4dOmSrcoXERFsFBo5OTnExMRw7do1YmNjyczMJCYmhoKCgmLtsrOzGTlyJMePHycmJoaYmBiqV68OwNGjR5k5cyYBAQGMHz+e7du389prr9mifBER+ZWLLXaSmppKZmYmU6dOJTo6mszMTN555x327NlD27Ztre2Sk5PJyMhgzpw5hIeH4+HhYV2XlJQEwKRJkwgODmbbtm1s3ryZ2bNn4+7ubovDEBGp8mwSGmazGQBvb28AfHx8ADh79myx0Dh58iQA77//PnFxcdStW5cZM2bQvXv3UreRn59Peno6v/vd726773PnzhEVFVXmx2QL/wSq2XH/UVEf2HHvD07n78Ho/D0Ye58/eLBzeO7cuVKX2yQ0bmWxWEpdnpubC0CDBg2YOHEic+bMITY2lrCwMMPbuNXu3bvvv1ARESnGJmMafn5+AJw/fx6AjIwMAPz9/cnJySEvL69Yu27dutGlSxdat25NdnY2P/30k3Vd0e9mZGTg4uJCw4YNbXEIIiKCjUIjLCwMLy8vVq1axb/+9S/Wrl2Lr68vvr6+BAcH89JLLwHQvXt33NzcSEpKYvXq1ezatQtvb28eeeQRIiIiAFiwYAHvvvsuBw4coHv37hrPEBGxIZuEhru7O/Hx8Xh6ejJv3jy8vLyIj4/H2bn47r29vXnzzTe5ePEic+fOpVGjRixevBgXFxeCgoKYMWMG33//PW+99RZhYWHExsbaonwREfmVk8Xo4ICIiFR5eiJcREQMU2iIiIhhCg07M5vNBAYGEhgYyNtvv21d/uc//9m6fPfu3QQGBjJr1iw7Vlr+5syZQ7t27QgMDGTMmDHF1hWdi6CgINq3b8+LL77IsWPHbrutrKwsYmNjadeuHcHBwYSHh5OSkgJAhw4dCAwMZOHChcB//j8o2ufChQsJDAykc+fO1lkLhgwZQmBgIBcvXiyPQ39gP/74I0OGDOHpp58mJCSEYcOGcebMGQDrfz+BgYG0aNGCjh07MmPGjDsey7fffsuIESNo2bIlLVu2ZPDgwZw8ebLYf69Ft7MnJSURGBjIsmXLgP+cq2nTplm3FxgYSM+ePcvxDDy4fv36ERISQosWLYiKimLv3r2Azt+tFBoOZP369VgsFq5du8Ynn3xi73Lsonv37rdd5+Pjw5w5c+jWrRvbt29n0KBBfP/99yXaZWdnM3jwYJKSknj66aeZPn06bdu2LTHf2cqVK7l27dpt93fmzBk+/vjj+z8YG8rIyMBisTB+/Hj69OnDv//9b1599dVibZ555hni4uJo1qwZ//u//8uQIUPIzs4usa0TJ04wePBgdu/ezfPPP8+f/vQn6tevz48//lis3dKlS+9YU3JyMv/3f//3wMdmKyEhIcTFxfHiiy/y3XffERcXV2y9zt9Ndnm4T0ry9/fn7Nmz7N69G7PZTH5+Pt7e3tbnUqqCuLg4zGYzK1asKHV9zZo1iYiIICIigvr167NgwQISEhKKTWoJsGnTJk6dOsUzzzzDggULrMsLCwutP7u6unLjxg0SExP5wx/+UOr+atSoQUJCgsP8hXcnISEhrFy50vp506ZNJQK1UaNG9OvXj379+jF8+HB27txJcnIyffv2LdYuISGBG1UnJi4AAAdESURBVDduMHnyZEaPHg3AgAEDKCwstH6J1ahRg507d3L48OHb1uTu7s57773HjBkzyuowy1VsbCyXLl3CbDazePFinJyciq3X+btJVxoOIiAggBYtWrBu3TrWrVtHp06dqFWrlr3LclhFswR8++23JdYVXVE8++yzxZb/9hZvV1dX+vTpw/vvv2+dieBWAwcOJC0tja1bt5ZV2eXGzc3N+vPhw4f5+eefadOmzW3bP+j5a9WqFc2aNbvjX8sDBw5k3bp1ZGVlGTsIO7ty5Qpt27alX79+uLq6MmfOnNu2rcrnT6HhQPr06cOWLVv4+uuvK+x8WbZSdKf4rX8N3m5ZaUaOHMmlS5esk2He6oknnqBdu3YkJCTcf6E2dvLkScaNG4evr2+J7pXfetDz5+TkxKhRo/j888+tc8bd6vnnn8fd3Z3333/fYPX2Vb16df75z38SFxdHTk4Ob7311m3bVuXzp9BwID169MBkMuHj40P79u3tXY5D27FjBwCPP/54iXVFy3bu3Fls+W+7pwB8fX3p2bMnq1atuu1+xowZw8GDBzl69OiDllzuvv/+e4YOHYqbmxvLly/noYceum1bI+evqE2RW89fjx498PPzIzExsdR9VK9encGDB9/x/DoSFxcX2rdvz5AhQwgODmb37t23HeyuyudPYxoOpEaNGsybNw9PT88ST8sDHDx4kDfeeAMADw8P6/QrlcW2bdtIS0sDID09nTVr1tCmTRvrLMZXrlxhw4YNfPvttyQmJlK9enVrn/Fv9ezZk2XLlpGamsrkyZMJDQ3l1KlTuLm5MWnSpGJtR48ezcaNG29bU2hoKE8++SQHDx4suwMtB+np6QwdOpSff/6ZiRMncujQIQ4dOkSPHj2sbU6fPs2aNWvYtm0bO3bsoEmTJqWO14waNYqUlBTi4+O5fPky/v7+7N69m+7duxMYGGhtZzKZGDlyJDNnzrxtXUOHDnW4v5RL8+WXX/LJJ58QEhLC+fPnOXDgAPXr16du3brWNjp/Nyk0HMyd7h46cuSItb+0Zs2alS40li1bxp49ewA4fvw4cXFxvPbaa9bQOH/+PHFxcdSpU4ff//73jB8/niZNmpTYjoeHBytXruQf//gH27ZtIyUlBX9/fyZOnFiibUBAAJ07d7bejluaMWPGMG7cuLI5yHJy5swZa9/3m2++aV3+29DYsWMHe/fupUGDBgwcOJCJEycWe2dNkaZNm7Jy5UoWLFhgHVz/r//6Lxo1alSibVRUFG+//TY//fRTqXXVqVOHAQMGONwX361q167NoUOHSE5Oxs3NjZYtWzJ16tRiXU06fzdpGhERETFMYxoiImKYQkNERAxTaIiIiGEKDRERMUyhISIihik0RBzIrTOgGlE0K2rRrL0i5UmhIWJA0RfzrVNWA8THx1vXdejQwU4VitiGQkPkHiUnJ3PhwgXg5jTsjjbNg0h5UmiI3IM6deqQl5dnfdI3KSmJS5cuUadOnWLtrl69yptvvknXrl1p0aIFnTp1YubMmcVmLE1PT2fUqFGEhITQuXPnUt+hUlhYyJo1a4iMjCQkJISwsDCmTp3K+fPny/dARW5DoSFyD8LCwvD19SUxMZHr16+zfPlyqlevTq9evaxtCgsLGTlyJAkJCVy9epWePXvi7OxMYmIiAwcO5MaNGxQWFjJ27FhSU1OpVasWTz31FK+//nqJ/c2fP5+4uDguXLhAly5daNKkCR999BEDBw7k6tWrtjx0EUChIXJPTCYTQ4YM4eeff2bSpEn8+OOP9OnTp9i7T77++msOHDgAwJIlS5g7dy4rV67EZDJx5swZUlJSOHToEN999x0AixcvZu7cuSxatKjYvnJzc61XNEFBQdSqVYuAgADc3d1JT0+/43xZIuVFExaK3KN+/fqxaNEitm3bhslk4oUXXmDDhg3W9b99RWezZs0AeOihh6hbty6ZmZmkp6fj6upqbVM06WJR2yKXLl3ixo0bAHzxxRcl6khPTy+7gxIxSFcaIveoRo0a9O/fH4BOnTrh7+9fbP3DDz9s/fnEiRMAXLhwgUuXLgHQsGFDGjZsaG1T9FrWomnhi9StW9c6i+rf//53jh8/bv2XmprKiBEjyvjIRO5OVxoi92Hs2LE8++yzBAQElFjXsmVLWrRowTfffMO4ceMICwtj7969FBQU4OfnR+fOnfHw8KBZs2akpaUxbtw4nnnmmRIv7XFzc2PQoEH885//ZMaMGWzfvp1q1apx+vRpvv76a1JSUvDz87PVIYsAutIQuS+1a9emXbt2eHt7l1jn7OzMsmXLGDlyJNWqVeOjjz4iLy+PAQMGWF8e5ezszJIlS3j22We5fPkyX331FZMnTy6xralTpzJ79mwCAgLYvn07n3zyCb/88gsvvPBCsRcEidiK3qchIiKG6UpDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJi2P8DQwsP1aiLpBQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7tK-fio1iLLy"
      },
      "source": [
        "## Arousal"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iLvYI0FOiPrT",
        "outputId": "72c49101-f8bb-4607-9313-562acb7afbad"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "def specificity(metric):\n",
        "  pos_list = [i[0][0] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s] \n",
        "\n",
        "def sensitivity(metric):\n",
        "  neg_list = [i[1][1] for i in metric]\n",
        "  neg_list_m = round(np.mean(neg_list),2)\n",
        "  neg_list_s = round(np.std(neg_list),2)\n",
        "  return [neg_list_m, neg_list_s] \n",
        "\n",
        "def fpr(metric):\n",
        "  fpr_list = [i[0][1] for i in metric]\n",
        "  fpr_list_m = round(np.mean(fpr_list),2)\n",
        "  fpr_list_s = round(np.std(fpr_list),2)\n",
        "  return [fpr_list_m, fpr_list_s]\n",
        "\n",
        "def fnr(metric):\n",
        "  fnr_list = [i[1][0] for i in metric]\n",
        "  fnr_list_m = round(np.mean(fnr_list),2)\n",
        "  fnr_list_s = round(np.std(fnr_list),2)\n",
        "  return [fnr_list_m, fnr_list_s]\n",
        "\n",
        "\n",
        "data = 'PD'\n",
        "print(\"-----\", data, \"----\")\n",
        "print(\"-----3D----\")\n",
        "hala_pd_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_3pd = sensitivity(hala_pd_3d)\n",
        "la_la_3pd = specificity(hala_pd_3d)\n",
        "ha_la_3pd = fnr(hala_pd_3d)\n",
        "la_ha_3pd = fpr(hala_pd_3d)\n",
        "print('Sensitivity:',ha_ha_3pd[0],'+-',ha_ha_3pd[1])\n",
        "print('Specificity:',la_la_3pd[0],'+-',la_la_3pd[1])\n",
        "print('FPR:',la_ha_3pd[0],'+-',la_ha_3pd[1])\n",
        "print('FNR:',ha_la_3pd[0],'+-',ha_la_3pd[1])\n",
        "\n",
        "print(\"-----2D----\")\n",
        "hala_pd_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_2pd = sensitivity(hala_pd_2d)\n",
        "la_la_2pd = specificity(hala_pd_2d)\n",
        "ha_la_2pd = fnr(hala_pd_2d)\n",
        "la_ha_2pd = fpr(hala_pd_2d)\n",
        "print('Sensitivity:',ha_ha_2pd[0],'+-',ha_ha_2pd[1])\n",
        "print('Specificity:',la_la_2pd[0],'+-',la_la_2pd[1])\n",
        "print('FPR:',la_ha_2pd[0],'+-',la_ha_2pd[1])\n",
        "print('FNR:',ha_la_2pd[0],'+-',ha_la_2pd[1])\n",
        "\n",
        "print(\"-----1D----\")\n",
        "hala_pd_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_HALA_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_1pd = sensitivity(hala_pd_1d)\n",
        "la_la_1pd = specificity(hala_pd_1d)\n",
        "ha_la_1pd = fnr(hala_pd_1d)\n",
        "la_ha_1pd = fpr(hala_pd_1d)\n",
        "print('Sensitivity:',ha_ha_1pd[0],'+-',ha_ha_1pd[1])\n",
        "print('Specificity:',la_la_1pd[0],'+-',la_la_1pd[1])\n",
        "print('FPR:',la_ha_1pd[0],'+-',la_ha_1pd[1])\n",
        "print('FNR:',ha_la_1pd[0],'+-',ha_la_1pd[1])\n",
        "\n",
        "print(\"-----ML----\")\n",
        "hala_pd_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HALA_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_pd_ml = sensitivity(hala_pd_ml)\n",
        "la_la_pd_ml = specificity(hala_pd_ml)\n",
        "ha_la_pd_ml = fnr(hala_pd_ml)\n",
        "la_ha_pd_ml = fpr(hala_pd_ml)\n",
        "print('Sensitivity:',ha_ha_pd_ml[0],'+-',ha_ha_pd_ml[1])\n",
        "print('Specificity:',la_la_pd_ml[0],'+-',la_la_pd_ml[1])\n",
        "print('FPR:',la_ha_pd_ml[0],'+-',la_ha_pd_ml[1])\n",
        "print('FNR:',ha_la_pd_ml[0],'+-',ha_la_pd_ml[1])\n",
        "\n",
        "data = 'NC'\n",
        "print(\"-----\", data, \"----\")\n",
        "print(\"-----3D----\")\n",
        "hala_nc_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_3nc = sensitivity(hala_nc_3d)\n",
        "la_la_3nc = specificity(hala_nc_3d)\n",
        "ha_la_3nc = fnr(hala_nc_3d)\n",
        "la_ha_3nc = fpr(hala_nc_3d)\n",
        "print('Sensitivity:',ha_ha_3nc[0],'+-',ha_ha_3nc[1])\n",
        "print('Specificity:',la_la_3nc[0],'+-',la_la_3nc[1])\n",
        "print('FPR:',la_ha_3nc[0],'+-',la_ha_3nc[1])\n",
        "print('FNR:',ha_la_3nc[0],'+-',ha_la_3nc[1])\n",
        "\n",
        "print(\"-----2D----\")\n",
        "hala_nc_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_2nc = sensitivity(hala_nc_2d)\n",
        "la_la_2nc = specificity(hala_nc_2d)\n",
        "ha_la_2nc = fnr(hala_nc_2d)\n",
        "la_ha_2nc = fpr(hala_nc_2d)\n",
        "print('Sensitivity:',ha_ha_2nc[0],'+-',ha_ha_2nc[1])\n",
        "print('Specificity:',la_la_2nc[0],'+-',la_la_2nc[1])\n",
        "print('FPR:',la_ha_2nc[0],'+-',la_ha_2nc[1])\n",
        "print('FNR:',ha_la_2nc[0],'+-',ha_la_2nc[1])\n",
        "\n",
        "print(\"-----1D----\")\n",
        "hala_nc_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/Spectral/CNN_1D_results_HALA_Spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_1nc = sensitivity(hala_nc_1d)\n",
        "la_la_1nc = specificity(hala_nc_1d)\n",
        "ha_la_1nc = fnr(hala_nc_1d)\n",
        "la_ha_1nc = fpr(hala_nc_1d)\n",
        "print('Sensitivity:',ha_ha_1nc[0],'+-',ha_ha_1nc[1])\n",
        "print('Specificity:',la_la_1nc[0],'+-',la_la_1nc[1])\n",
        "print('FPR:',la_ha_1nc[0],'+-',la_ha_1nc[1])\n",
        "print('FNR:',ha_la_1nc[0],'+-',ha_la_1nc[1])\n",
        "\n",
        "print(\"-----ML----\")\n",
        "hala_nc_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HALA_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "ha_ha_nc_ml = sensitivity(hala_nc_ml)\n",
        "la_la_nc_ml = specificity(hala_nc_ml)\n",
        "ha_la_nc_ml = fnr(hala_nc_ml)\n",
        "la_ha_nc_ml = fpr(hala_nc_ml)\n",
        "print('Sensitivity:',ha_ha_nc_ml[0],'+-',ha_ha_nc_ml[1])\n",
        "print('Specificity:',la_la_nc_ml[0],'+-',la_la_nc_ml[1])\n",
        "print('FPR:',la_ha_nc_ml[0],'+-',la_ha_nc_ml[1])\n",
        "print('FNR:',ha_la_nc_ml[0],'+-',ha_la_nc_ml[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- PD ----\n",
            "-----3D----\n",
            "Sensitivity: 0.98 +- 0.02\n",
            "Specificity: 0.91 +- 0.06\n",
            "FPR: 0.09 +- 0.06\n",
            "FNR: 0.02 +- 0.02\n",
            "-----2D----\n",
            "Sensitivity: 0.99 +- 0.01\n",
            "Specificity: 0.91 +- 0.11\n",
            "FPR: 0.09 +- 0.11\n",
            "FNR: 0.01 +- 0.01\n",
            "-----1D----\n",
            "Sensitivity: 0.96 +- 0.02\n",
            "Specificity: 0.73 +- 0.06\n",
            "FPR: 0.27 +- 0.06\n",
            "FNR: 0.04 +- 0.02\n",
            "-----ML----\n",
            "Sensitivity: 0.97 +- 0.01\n",
            "Specificity: 0.76 +- 0.06\n",
            "FPR: 0.24 +- 0.06\n",
            "FNR: 0.03 +- 0.01\n",
            "----- NC ----\n",
            "-----3D----\n",
            "Sensitivity: 0.98 +- 0.01\n",
            "Specificity: 0.88 +- 0.09\n",
            "FPR: 0.12 +- 0.09\n",
            "FNR: 0.02 +- 0.01\n",
            "-----2D----\n",
            "Sensitivity: 0.97 +- 0.03\n",
            "Specificity: 0.79 +- 0.09\n",
            "FPR: 0.21 +- 0.09\n",
            "FNR: 0.03 +- 0.03\n",
            "-----1D----\n",
            "Sensitivity: 0.97 +- 0.02\n",
            "Specificity: 0.66 +- 0.25\n",
            "FPR: 0.34 +- 0.25\n",
            "FNR: 0.03 +- 0.02\n",
            "-----ML----\n",
            "Sensitivity: 0.98 +- 0.01\n",
            "Specificity: 0.78 +- 0.05\n",
            "FPR: 0.22 +- 0.05\n",
            "FNR: 0.02 +- 0.01\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JLcYILBFcEm7",
        "outputId": "4889a0e5-5bc3-4030-de24-87fa5b2566c4"
      },
      "source": [
        "data1 = {'Sensitivity': [i[1][1] for i in hala_nc_ml], 'Specificity': [i[0][0] for i in hala_nc_ml],\n",
        "          'Data': len([i[1][1] for i in hala_nc_ml])*['NC'], 'Model': len([i[1][1] for i in hala_nc_ml])*['ML']}\n",
        "df1 = pd.DataFrame(data1)\n",
        "data2 = {'Sensitivity': [i[1][1] for i in hala_pd_ml], 'Specificity': [i[0][0] for i in hala_pd_ml],\n",
        "          'Data': len([i[1][1] for i in hala_pd_ml])*['PD'], 'Model': len([i[1][1] for i in hala_pd_ml])*['ML']}\n",
        "df2 = pd.DataFrame(data2)\n",
        "data3 = {'Sensitivity': [i[1][1] for i in hala_pd_1d], 'Specificity': [i[0][0] for i in hala_pd_1d],\n",
        "          'Data': len([i[1][1] for i in hala_pd_1d])*['PD'], 'Model': len([i[1][1] for i in hala_pd_1d])*['1D CNN']}\n",
        "df3 = pd.DataFrame(data3)\n",
        "data4 = {'Sensitivity': [i[1][1] for i in hala_nc_1d], 'Specificity': [i[0][0] for i in hala_nc_1d],\n",
        "          'Data': len([i[1][1] for i in hala_nc_1d])*['NC'], 'Model': len([i[1][1] for i in hala_nc_1d])*['1D CNN']}\n",
        "df4 = pd.DataFrame(data4)\n",
        "data5 = {'Sensitivity': [i[1][1] for i in hala_pd_2d], 'Specificity': [i[0][0] for i in hala_pd_2d],\n",
        "          'Data': len([i[1][1] for i in hala_pd_2d])*['PD'], 'Model': len([i[1][1] for i in hala_pd_2d])*['2D CNN']}\n",
        "df5 = pd.DataFrame(data5)\n",
        "data6 = {'Sensitivity': [i[1][1] for i in hala_nc_2d], 'Specificity': [i[0][0] for i in hala_nc_2d],\n",
        "          'Data': len([i[1][1] for i in hala_nc_2d])*['NC'], 'Model': len([i[1][1] for i in hala_nc_2d])*['2D CNN']}\n",
        "df6 = pd.DataFrame(data6)\n",
        "data7 = {'Sensitivity': [i[1][1] for i in hala_pd_3d], 'Specificity': [i[0][0] for i in hala_pd_3d],\n",
        "          'Data': len([i[1][1] for i in hala_pd_3d])*['PD'], 'Model': len([i[1][1] for i in hala_pd_3d])*['3D CNN']}\n",
        "df7 = pd.DataFrame(data7)\n",
        "data8 = {'Sensitivity': [i[1][1] for i in hala_nc_3d], 'Specificity': [i[0][0] for i in hala_nc_3d],\n",
        "          'Data': len([i[1][1] for i in hala_nc_3d])*['NC'], 'Model': len([i[1][1] for i in hala_nc_3d])*['3D CNN']}\n",
        "df8 = pd.DataFrame(data8)\n",
        "\n",
        "arousal_df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], axis=0).reset_index()\n",
        "print(arousal_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    index  Sensitivity  Specificity Data   Model\n",
            "0       0     0.978723     0.702479   NC      ML\n",
            "1       1     0.980392     0.800000   NC      ML\n",
            "2       2     0.978758     0.700000   NC      ML\n",
            "3       3     0.959150     0.858333   NC      ML\n",
            "4       4     0.982026     0.800000   NC      ML\n",
            "..    ...          ...          ...  ...     ...\n",
            "75      5     0.993548     0.892562   NC  3D CNN\n",
            "76      6     0.988710     0.884298   NC  3D CNN\n",
            "77      7     0.977419     0.917355   NC  3D CNN\n",
            "78      8     0.993548     0.958678   NC  3D CNN\n",
            "79      9     0.991935     0.942149   NC  3D CNN\n",
            "\n",
            "[80 rows x 5 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 554
        },
        "id": "pCCTtxfUc8JD",
        "outputId": "c1716fd7-0023-49fd-ed16-e60b8adb9b13"
      },
      "source": [
        "x = \"Model\"\n",
        "y = \"Sensitivity\"\n",
        "hue = \"Data\"\n",
        "ax = sns.barplot(data=arousal_df, x=x, y=y, hue=hue, palette = \"Blues\", capsize=0.05, ci=68)\n",
        "ax.set_ylim(bottom=0.9)\n",
        "ax.legend(loc='upper left', bbox_to_anchor=(0, 1))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold' )\n",
        "ax.set_ylabel('Sensitivity',fontsize = 13 , weight='bold')\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold');\n",
        "ax.set_yticklabels([\"0.90\", \"0.92\", \"0.94\", \"0.96\", \"0.98\", \"1.0\"], size= 10, weight='bold')\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=arousal_df, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"PD\"), (\"ML\", \"NC\")),\n",
        "                                 ((\"1D CNN\", \"PD\"), (\"1D CNN\", \"NC\")),\n",
        "                                 ((\"2D CNN\", \"PD\"), (\"2D CNN\", \"NC\")),\n",
        "                               ((\"3D CNN\", \"PD\"), (\"3D CNN\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "#plt.savefig('sensitivity_arousal.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_NC v.s. 1D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=1.073e+00\n",
            "ML_NC v.s. ML_PD: t-test independent samples with Bonferroni correction, P_val=5.130e-01 stat=1.594e+00\n",
            "2D CNN_NC v.s. 2D CNN_PD: t-test independent samples with Bonferroni correction, P_val=3.125e-01 stat=-1.868e+00\n",
            "3D CNN_NC v.s. 3D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=4.760e-02\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<matplotlib.axes._subplots.AxesSubplot at 0x7f260edadb90>,\n",
              " [<statannot.StatResult.StatResult at 0x7f260e551c10>,\n",
              "  <statannot.StatResult.StatResult at 0x7f260f082b10>,\n",
              "  <statannot.StatResult.StatResult at 0x7f260ed2bd10>,\n",
              "  <statannot.StatResult.StatResult at 0x7f260ebb2390>])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEJCAYAAABohnsfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7hVVb3/8feHi6CFV1AKJcDUo6ahbo+dtNTsZ2LaMUtPZl7ywiOamtfMfpVa5uXBQg4ioXittJOYph68nEr5lScLzaNoHTTAkryAJoJuRDbf3x9jLlgs9mWuvfdac+29P6/nWc9ca86x5vyuyWJ99xxjzDEUEZiZmeXRr+gAzMys53DSMDOz3Jw0zMwsNycNMzPLzUnDzMxyG1B0ALU2dOjQGDVqVNFhmJn1KI8//viSiBhWub7XJ41Ro0YxZ86cosMwM+tRJL3Q2npXT5mZWW5OGmZmlpuThpmZ5eakYWZmuTlpmJlZbk4aZmaWm5OGmZnl5qRhZma59fqb+/K4948LunV/h+w2usMykjj77LO56qqrAJg4cSLLly/noosuAuCWW27hyiuvRBIDBgzg6KOP5txzz+3WOM3MquUrjYIMGjSIO++8kyVLlqy3bdasWUyaNIkHH3yQp59+mt/97ndssskmBURpZrYuJ42CDBgwgPHjx/ODH/xgvW2XXXYZEydO5P3vfz+QEszJJ59c7xDNzNbTK5OGpPGS5kias3jx4qLDadNpp53Gj3/8Y5YuXbrO+rlz57LHHnsUFJWZWdt6ZdKIiOkR0RQRTcOGrTdIY8PYeOONOfbYY5k8eXLRoZiZ5dIrk0ZP8tWvfpUZM2bw1ltvrVm388478/jjjxcYlZlZ65w0Crb55ptz5JFHMmPGjDXrvv71r3Peeefx8ssvA7By5Uquv/76okI0M1vDXW7J10W2ls455xymTJmy5vXBBx/MK6+8wic/+UkiAkmccMIJBUZoZpYoIoqOoaaamprCkzCZmVVH0uMR0VS53tVTZmaWm5OGmZnl5qRhZma5OWmYmVluThpmZpabk4aZmeXm+zSAcZfe1a37m/WNwzos079/f3bZZRdWrVrFjjvuyM0338xGG220Zv27777LgAEDOPbYYznrrLPo18/53cyK51+igmy44YY8+eSTzJ07lw022IBp06ats/6ZZ57hoYceYtasWVx88cUFR2tmljhpNICPfexjPP/88+ut33LLLZk+fTpTpkyht9+EaWY9g5NGwVatWsWsWbPYZZddWt0+ZswYWlpaePXVV+scmZnZ+tymUZDm5mbGjh0LpCuNE088seCIzMw65qRRkFLbRUfmz59P//792XLLLesQlZlZ+1w91cAWL17MKaecwle+8hUkFR2OmZmvNCBfF9l6KVVblbrcHnPMMZx99tlFh2VmBjhpFGb58uWtrm9paalzJGZm+bl6yszMcnPSMDOz3Jw0zMwsNycNMzPLzUnDzMxyc9IwM7PcnDTMzCw3Jw0zM8utVyYNSeMlzZE0Z/HixUWHY2bWa/TKpBER0yOiKSKahg0bVnQ4Zma9Rq9MGmZmVhtOGmZmlpuThpmZ5eakYWbWDRYuXMiOO+7IySefzM4778yBBx5Ic3MzkydPZqeddmLXXXflC1/4QtFhdpmHRjcz6ybPPfcct912G9dddx1HHnkkM2fO5PLLL2fBggUMGjSIN954o+gQu8xXGmZm3WT06NGMHTsWgD322IOFCxey6667cvTRR/OjH/2IAQN6/t/pThrWK/SVqgFrbIMGDVrzvH///qxatYr77ruP0047jSeeeII999yTVatWFRhh1/X8tGeW6QtVA9azrF69mr/97W/sv//+7LPPPtx+++0sX76cTTfdtOjQOs1Jw3qN9qoGDjvsMA47rHHmgre+oaWlhS996UssXbqUiOCMM87o0QkDnDSsF6msGmhubua+++5j9uzZ3HPPPVx66aU8/fTTvaJe2RrPqFGjmDt37prX5557boHR1I7/91iv1RurBmpl4cKFjBs3jn322YdHH32UESNGcPfdd3Pdddcxbdo0BgwYwE477cTtt99edKh1ceedd3L//fcXHQYHHXQQhx9+eNFhrMNJw3qt3lg1UEtuE1rr/vvvZ968eWy//faFxTBv3jwAJw2zWugrVQO15DahdW2//fZMnz69sOOPHz++sGO3x0nDGlIjVA80YtVALblNyPLwfRrWkErVA0WZN29e4UmraOVtQldccQVLly5l+fLlRYdlBfOfDNawiqweaNSqgXrqy21CO+ywQ9EhNEQMrXHSMDO3CVU455xzig6hIWJojZOGWS/kNiGrFScNa0hFX5oXffyuKrrLaKN2F7Wuc9KwhlT0pXnRx+8ObhOyWnDvKTMzy81Jw8zMcnPSaBCeD8K60w477FBou0zRx7faUUQUHUNNNTU1xZw5c4oOo0MLFy7kgx/8IHPmzGHs2LEceeSRfOYzn+H8889fZ+yfvtJP3syKJenxiGiqXO8rjQbSF6aKNLOerfCkIWmypFckhaR72ym3t6SnJL0j6QlJu9czznroC1NFmlnPVnjSyLQ7SL+kwcBMYAhwFrAVcIek/nWIrTAe+8fMGk3h9R0RcYakUcAZ7RQbR0oU50fEVEnDgW8C+wG/rHWMRenLY/+YWWMqPGnkNDpbLsqWL2bLMbSSNCSNB8YDjBw5subBdQeP/WNmPUHupCHpYxHx/2oZTBXU3saImA5Mh9R7qi4RVfDYP2bWG1XTpvGIpP+V9HVJI2oWUUbSIEmlluEF2XLrbFk6/vxax9FZng/CzHqjaqqn3ga2A74LXCLpIeAG4O6IeLezAUj6NPCh7OU2kk4CHgEeAoYC7wVmAa8CEyQtA04EFgIPd/a49eCxf8yst6nmSmMo8HngZ8AK4CDgp8BLWbfZbTsZw3nA5dnzXYHrgL3LC0TECuAIYDlwNSmBHBERLZ08ppmZdULupBERKyLizoj4Aqlh+j5S28LmwFeAZyVVPc5FROwXEap43BQRoyLivWXlZkfELhGxQUTsFhGNf5u3mVkvU9V9GpKaJE0DngcOzlb/FZgMLGXtFYOZmfVC1fSe+iOp+qjUc+lXwBTgFxGxWtKrwHe6P8SeqejB2oo+vpn1TrkHLJS0mtSmcCswJSL+VLF9L+CgiLi426Psgp4yYKGZWSNpa8DCanpPnQncGBGtjmMREY8Bj3UyPjMz6wGqadOYBKxzFSHpLEm/796QzMysUVWTNMT6d2KPAfbovnDMzKyRdVg9JemGspcHlr3uBxxCuunPzMz6gDxtGscDkT12BHaq2N5rR5k1s+qcf/75vPzyyyxZsoRVq1YxYMAAhg4dyvDhw7nyyiuLDq9HaPRzmCdp3JwtjwP+BJTaMFYBLwA/rEFcZtYDvfzyyyxatGjN65aWlnVeW8ca/Rx2mDQi4ssAkkYDP4uIa2oelZn1SMOHDwfSD19LSwv9+/dn+PDha9Zbxxr9HOa+T6On8n0aZvV37LHHsmjRIkaMGMEtt9xSdDg9UtHnsFP3aUhqASZFxDnZ89ZERPSUyZzMLKdxl97V6feufP0tABa9/lan9zPrG4d1+vhWOx392Jd3s2134iMzs56u0RuhG0FHSWN/1k6tun+NYzGzHu7dP95FrFgGK95MK1a8ycr/vhUNHsLA3Rr/yqHRG6EbQbtJIyIeKXu5LCKeqHE8ZtaDxYpl0PxG2YrV0PwGPaXltNEboRtBNW0Rf5D0BGnu7Z9ExFs1isnMeigNHpISxDvLU8JQPxj0XjR4SNGh5VKqgio1Qg8fPtwN+RWqSRotpCFDpgETJd0GXBcRj9ckMjPrcXpCFZR1TTVJ433AUcAxwJ7AeOBkSX9srVuWmVmR7v3jgk6/9613Vq1ZdnY/1/zn/3T6+NC4PdCqme71tYiYEhF7kYYT+RWpR9Vu3R6VmZk1pKrur5A0BvgS6WpjTLa6rfs3zMysl8l9pSHpN8BzwLeBbUlzg38L+EDO9+8t6SlJ70h6QtLurZQZKGmSpFckLZU0VdLAbNtmkmZKel3SW5J+K+nDeeM3M+vIzOnf58bLv8Gbry8B4M3Xl3Dj5d9g5vTvFxxZ46hmPo2Pkq4q7gLGAWMi4rsR8VJHb5Q0GJgJDAHOArYC7pDUv6LoGaQZAu8GbgAmZOsAzgUOBx4Ars7i8b+kmXWbN19/jTeWvMLq1akCZfXqFt5Y8gpvvv5awZE1jmqSxoXANhHxuYh4IKobtGocKVFMjYipwAxgNLBfRbl9s+XFwAXZ8+MqYp1Dak8BKOsQbmbWNRtvvgWbDt2KAQM3oF+//gwYuAGbDt2KjTffoujQGkbuNo2IuLwLxxmdLUu3VpbuMh/DuvNxLM6WB7A2IZTeeyXwMWBi9voF1l6FrEPSeFLvLkaOHNmFsKvnYQisSP7+dc3nxp9ddAgNr6gBC9sax+oK4CDSHB6rgGZgRbbtYNZWST1DmsdjCvDZVgKaTroJkaamprrejOphCLrGP3pd4++f1Vq9BiwsdXTeOluOyJbzs/aOloh4NyLmSdoO2AX4B2nCp2ezsl/MYvhBRLwo6ULgwC7EVBMehqBr/KPXNf7+9XyNPn5XvQYsnAW8CkyQtAw4EViYPZqB+4BDJI0FDs2OeRSwCWuro+Zny0sk/ZlUtfVkF2KqCQ9D0DX+0esaf/96vkYfv6uaAQs/ADwbEWtmNJK0JTC0o4NExApJRwDXkHo+PQOcTOv3eJxAuhL5O3BqRNyTrb+YdFf6vwJHAr8BTu/o2F1R9B2lh+w2uuNCvYx/9Kyva/Txu6ppi7gJmETqvVTydVJjdGXX2fVExGxStVMllZV5krUN35XvXwJ8Pn+4ZmY9TyNUQbWnqzPubd1xkb5l5vTv8+brr613c9DGm2/hnhlm1uN1mDTKek0FcKakMyuKLMbWKN0cVFK6Oahe3PvIzGopz5VGqfooaL0H1TXdF07PV7oJaPnSf7C6pYV+/fvz3k02q9vNQe59ZGa1lCdp7E9KFr8C7mBtklgFvBARL7b1xr6o6Coo9z4ys1rqMGmUelBJ+jLwTHnvKWs8jdT7yL3PzHqfju4IP5a13WwD2EnSTpXlIsJ9Is3M+oCOrjRuAn5A6mZ7E7R6f0kATho10NnZuqBxZ/0ys56to6TxV9JwHqXnjXJTolmvV3T1nllrOrojfFRrz816K3dZNmtfp27ukzQI+BzwWkQ80L0hmRXHXZbN2pc7aUi6izSfxTDgZ8Cns/XnR8RVtQnPqtXoI2Q2OndZNmtfNVcauwKPAhuS5rZ4njSMyEmAk0aDaPQRMhtdI3VZNmtE1SSN95GGON+BdLPfF4FTgX+rQVzWSY0+QmZHPHaXWWOrJmmsIA2Pvg+pF9WfSEObr65BXNZJPb0Kquixu8ysfdUkjbnAuOwxNyLeljSKtfN+m3VZ0WN3mVn7qkkaFwA/ATYAvpb1oNqGVGVl1i1cBWXW2HInjYj4Lal6qtyO3RuOmZk1sqru05D0AWB3YJ1WVY89ZWbWN1Rzn8bJpGHRW5va1UnDGo7H7jLrftVcaXwrK/8m8AYeh8qs4bjLstVavyrKbgY8BGwWEaMiYnTpkefNkvaW9JSkdyQ9IWn3VsoMlDRJ0iuSlkqaKmlg2faPS/pDto+XW5l61qxPK3VZXr06zdJc6rL85uuvFRyZ9RbVXGnMAoZERNVXGJIGAzOBZuAs4BvAHZK2i4iWsqJnAGcC1wFvAV8F/gJcJWl4FsMS4BxSLy7fI2JWxl2WrdaqSRqvAodLehB4DHi3tCEiLungveOArYDzI2JqlgC+CewH/LKs3L7Z8mJScvgqcBxpmJJTgY2AE4HfRkRzFbGb9QmugrJaqyZpTCC1Y3wSOKBiW0dJo1SFVboRsDSv+BjWTRqLs+UBpHaT8veWZgycDOwo6a/AcRHxcOXBJI0HxgOMHDmyg9DMzCyvapLGbLqv8VttrL8COAi4GVhFqs5akW0blC3/h5SkrgN+RBo0cR0RMR2YDtDU1OQGezOzblLNzX37deE4panDSj/wI7Ll/Ky9oyUi3o2IeZK2A3YhzRj4e+DZin3cFBEPZI3ge0kaHBGlxGLWJR5a3qx91d7c9x7S5Etbk9oZtgKWRMTbHbx1FqlNZIKkZaR2iYXZoxm4DzhE0ljgUFL11VHAJsDEbB83A6cDZ2SJZSzwBycM604eWt6sfbm73EoaATwD3Ah8B9ic1LPp2x29N/thPwJYDlxNSiBHkEbJrXQC8ENge+DUiLgn28fjpMbwscBlpOqyL+aN3ywPDR4CG24K/QakYeX7DYANN+0xQ8ub1Vo1VxqXASNJbQyDIuIlSY8CnwK+1tGbI2I2qdqpksrKPMnahu/W9nEtcG0VMZtVxVVQZu2r5ua+/YEHSVcBJfOAUd0ZkJmZNa5qksYmwEsV696DhxMxM+szqkkazwEHklUfSfoi8K+kucLNzKwPqCZpTCXNE35o9vpWYDAwrbuDMjOzxlRN0rgBuJLURVakBvGrImJGLQIzM7PGU83NfQFcIOlbwOdJQ6T/d60CMzOzxtPhlYakCyX9UtJm2apppKqpu4E/S9qjlgGamVnjyFM9dTgwLCL+kd2JfTypekrAFqTJmczMrA/IkzQ+ADyVPf8/2fIx0h3hTwJ71iAuMzNrQHmSxsbA0uz5P5Puy/iPiHiD1KaxeY1iMzOzBpMnabwK7J8NJli60ig1gG/J2oRiZma9XJ6k8Svgn4DHgeGkJPJYtm0PYH5tQjMzs0aTJ2lcSEoYApYBJ0VESNqPNO7U7JpFZ2ZmDaXD+zQiYhGwp6RNgWURURrO/DfAENbOrGdmZr1cNTf3vVHxehVpSlYzM+sjqhlGxMzM+jgnDTMzy81Jw8zMcnPSMDOz3Jw0zMwst7olDUl7S3pK0juSnpC0eytlBkqaJOkVSUslTZU0sKLMJyRF9miqV/xmZlanpCFpMDCTdF/HWcBWwB2S+lcUPQM4kzTs+g3AhGxdaT8bAtOBt+sQtpmZVajXlcY4UqKYGhFTgRmkucb3qyi3b7a8GLgge35c2faLSWNd3VmzSM3MrE31Shqjs+WibPlithxTUW5xtjwA+FT5eyXtBpwOnAS00A5J4yXNkTRn8eLF7RU1M7MqFNUQrjbWXwH8HbiZVJ3VzNphSiYDPyONfzUkW7d1VvW1joiYHhFNEdE0bNiwbg3czKwvq1fSWJAtt86WI7LlfEmDS43dETEP2A74CLAzsBJ4Niu7DXAM8BxpNkGAnwNuDDczq5PcY0910SzSkOoTJC0DTgQWZo9m4D7gkGzOjkNJ1VdHAZsAE7N9TADekz0/jdQe8jXgT/X4AGZmVqekERErJB0BXANcDTwDnEzrbRMnkK5E/g6cGhH3ZPuYVSog6ZDs6a8i4rVaxm5mZmvV60qDiJgN7NLKJpWVeZK1jebt7et44Pjuis3MzPLxHeFmZpabk4aZmeXmpGFmZrk5aZiZWW5OGmZmlpuThpmZ5eakYWZmuTlpmJlZbk4aZmaWm5OGmZnl5qRhZma5OWmYmVluThpmZpabk4aZmeXmpGFmZrk5aZiZWW5OGmZmlpuThpmZ5eakYWZmuTlpmJlZbnVLGpL2lvSUpHckPSFp91bKDJQ0SdIrkpZKmippYLbtJEnPSHpb0kuSrpSkesVvZmZ1ShqSBgMzgSHAWcBWwB2S+lcUPQM4E7gbuAGYkK0D2BOYnb1+ETgPOLbmwZuZ2Rr1utIYR0oUUyNiKjADGA3sV1Fu32x5MXBB9vy4bHl6REyIiOuBS7J1O9csYjMzW0+9ksbobLkoW76YLcdUlFucLQ8APlX+3ohYWVautG12N8ZoZmYdGFDQcdtqi7gCOAi4GVgFNAMr1nmjdCZwGvDDiLi31Z1L44HxACNHjuymkM3MrF5XGguy5dbZckS2nC9pcKmxOyLmAdsBHyFVPa0Eni3tRNI5wCRSUjm1rYNFxPSIaIqIpmHDhnXrBzEz68vqdaUxC3gVmCBpGXAisDB7NAP3AYdIGgscSqq+OgrYBJgIIOmU7PlfgAeBIyUtiIjH6vQZzMz6vLpcaUTECuAIYDlwNSmBHAG0tFL8BOCHwPbAqRFxT7b+I9lyW+DHwG2k3lVmZlYndWvTiIjZwC6tbFJZmSdZ22he+f7jgeNrEZuZmeXjO8LNzCw3Jw0zM8vNScPMzHJz0jAzs9ycNMzMLDcnDTMzy81Jw8zMcnPSMDOz3Jw0zMwsNycNMzPLzUnDzMxyc9IwM7PcnDTMzCw3Jw0zM8vNScPMzHJz0jAzs9ycNMzMLDcnDTMzy81Jw8zMcnPSMDOz3Jw0zMwst7olDUl7S3pK0juSnpC0eytlBkqaJOkVSUslTZU0sGz7tyUtlrRc0k2SBtcrfjMzq1PSyH7cZwJDgLOArYA7JPWvKHoGcCZwN3ADMCFbh6TPAhcBvwQmA8cBF9YhfDMzy9TrSmMcKVFMjYipwAxgNLBfRbl9s+XFwAXZ8+Oy5fHZ8vSIuBD4G/DlGsVrZmatGFCn44zOlouy5YvZcgzpyqFkcbY8AHij4r2jgXcjolTmReAjkjaIiJXlB5M0HhifvVwu6X+7/hEKMRRYUtTB9X+LOnK38fnrGp+/rin0/EGXz+EHWltZr6RRSW2svwI4CLgZWAU0Ayuq3AcRMR2Y3pUAG4GkORHRVHQcPZXPX9f4/HVNbz1/9aqeWpAtt86WI7LlfEmDS43dETEP2A74CLAzsBJ4tmwfAyVtWbaPRZVXGWZmVjv1utKYBbwKTJC0DDgRWJg9moH7gEMkjQUOJVU9HQVsAkzM9nEz8BngakkLgG2A79YpfjMzo05JIyJWSDoCuAa4GngGOBloaaX4CaSriL8Dp0bEPdk+7pT0HeBUYDBwK/C9OoRfpB5fxVYwn7+u8fnrml55/hQRRcdgZmY9hO8INzOz3Jw0zMwsNyeNgkkaJSmyxzfL1s8oW79ftpxSZKy1JmlyNoRMSLq3YlvpXLwj6WVJd2UdJ9ra15aSbsz21yzpaUmHZ9sWZvu6KHs9qvyYki7KXj9fGrVA0sPZuqE1OwFdIGk7Sb+W9JqkZZIekrRttm2/svP3tqT5kn7Y3meRtIek+yW9mT0ekbRjxfd1v6zs8dnrc7PXpXN1U9n+QtLc2p6FrpH0WHbu3pY0R9LHs/U+f2WcNBrL8UreCxxZdDAFub2dbS8CJwH/AXwa+I2knSoLSdoQeIQ0isCvgdOB/wL2qCh6enau27It8G+5Iy/WCNL/528DNwKfBK6vKPMgaViep0k3vz6Snat1SNoZmA18ArgWOBd4mdQdvlxHw/h8UdLI6j5GoR4lnZ/vAGPx+WtdRPhR4AMYBQTwl2y5P6kH2QrSj2SQhlsJYErR8dbxfNxbsT6AuWWvL8zW3dLKPk7Mtt1fsb5ftlxIugeomfQfep1jksY4C2Ap8BTpRtKHs3VDiz5HbZy3DSpevwa8mj1f7/tD+gEM4IRW9nVrtu2CyvNXdq6WZssmUnIO4Nys3MNlZaa09u/XiI/s33ko8M/AW8Cfff7Wf/hKo3H8CXiMlDBOAO5i7VAqtr5Z2bK1O25LVxT3l6+MiNVlL1eSBsU8GxjUxjGmAbuQ7h1qaFF2k6ukJmBz0l+7benq+fsNMJf2/1qeBpxQdkNuo9uENJTRY6Tvx0ntlO2z589Jo7HcABwB7E2qYrC2lYaRaa3PeN5+5FcCw2h74Ms/kKq1vl5daMWR9E/AL0hXU6e3VzRbdvb8BXA5cBiwYxtlriVdMZ+dY3+NYDlwIKkKajBwSTtl++z5c9JoLLeTbnh8EXio4Fga3aey5eOtbCutO7B8paR1vu8R8QLwE9IQ/G35HmlYm906F2b9ZO07D5P+Sv5ERLzUTvEunz/S93UBcEobx1gO/Dvtn9+GERGrIuKhiPh34PfA/u00dvfZ81fUgIXWioh4U9IJwLKIWC2tNybjv0i6PHveHBEX1zfC2pL0aeBD2cttJJ0EPBIRz2XrNpF0DKlKYAKp3vny9ffEbcB5wDhJPyGNpLwj6a+2ynE/LweOaSumiPi1pN+REkfDkrQNqdF/C9Jn3EvSXhFR3rFgO0knAoeQftCeJZ2rSlcAnwO+I2kzYD6pXv+npDYeACKiRdKVpGqUtlwNnNPZz1Uvkj5F6nzyKGmIoo8Cr5Dahkp8/sAN4UU/aKPhN9s2l3UbwssfbxQdew3OxcOtfM7js22l1yuBl4CfAx9uZ19bAjeR6qhXkP6DH55tWwgsLys7s/zfgLUN4Z/PXh9advxGbQhv7TsSrWxrJv2IXQts0c7+9gAeIP21u5xUB79T5feV1B60iNYbcodmr6+igRpy2/i8e2b/35pJbYm/Bvb0+Vv/4WFEzMwsN7dpmJlZbk4aZmaWm5OGmZnl5qRhZma5OWmYmVluThpmDaRyBNSc7ymNinpR7SIzS5w0zHIo+2FeZ8jqbNslZdsWFhOhWX04aZhV7yhJw2HNMOwNNcyDWS05aZhV5zVgA+Ar2evjScNplw83gaQhki6T9Ods4p6/SLq2fMRSSdtI+s9s4p/nJR1ReTBJ/SSdKOmJrNyLkm6VtHXtPqJZ25w0zKozC3gBOEXSe4CzSMNE/KhUIBuY7gHgAmBj0qCIq0kD0z0qaaOszD3AONKwFQ+Thouo9D3SZEDvA+4kDYfypWw/Q2rw+cza5aRhVp0WYDJpYMCfkmZju4F15z7ZG/iX7PmhEXES8PHsvdsCh5Mm+vlwVuYzWZnPlh9I0gasHd78D8A/SEljBWlQvc915wczy8Oj3JpV73rStKqfJiWCScBxZdvLp+icCxARL0laAmyVbV9ZVubZ8rJlhgEbZc9bmwhqm84Eb9YVvtIwq1JEvAlcl738eUQsqCjy17LnOwNkDedDy7b/raxMaZ7zD7GuxaRRVQGOiQiVHqQ5wSd2/lOYdY6vNMw651LSdJ7PtrLtt6QpQ/cCfiFpFrAv0J806c7PScngadJ0sr+Q9ABrJ/YB0hSukq4hzWM+PZtv5C1SldjewAdJw7yb1Y2vNMw6ISL+ERH/FRF/b2XbalICuBJ4m9RwPRCYDnw0It7KyhxKSjybAp8gNZxX+pmnATQAAABQSURBVBownjSH/MGkiYI2I1WJLenuz2XWEc+nYWZmuflKw8zMcnPSMDOz3Jw0zMwsNycNMzPLzUnDzMxyc9IwM7PcnDTMzCw3Jw0zM8vt/wMJL3MeVven5AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "id": "4hQQdhIzdHEb",
        "outputId": "d86adb98-222d-4d1f-eacf-2acb54558e1b"
      },
      "source": [
        "x = \"Model\"\n",
        "y = \"Specificity\"\n",
        "hue = \"Data\"\n",
        "ax = sns.barplot(data=arousal_df, x=x, y=y, hue=hue, palette = \"Blues\", capsize=0.05, ci=68)\n",
        "ax.set_ylim(bottom=0.4)\n",
        "ax.legend(loc='upper left', bbox_to_anchor=(0, 1))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Specificity',fontsize = 13, weight='bold')\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold');\n",
        "ax.set_yticklabels([\"0.4\", \"0.5\", \"0.6\", \"0.7\", \"0.8\", \"0.9\", \"1.0\"], size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=arousal_df, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"PD\"), (\"ML\", \"NC\")),\n",
        "                                 ((\"1D CNN\", \"PD\"), (\"1D CNN\", \"NC\")),\n",
        "                                 ((\"2D CNN\", \"PD\"), (\"2D CNN\", \"NC\")),\n",
        "                               ((\"3D CNN\", \"PD\"), (\"3D CNN\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "plt.savefig('specificity_arousal.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_NC v.s. 1D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-7.771e-01\n",
            "ML_NC v.s. ML_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=6.178e-01\n",
            "2D CNN_NC v.s. 2D CNN_PD: t-test independent samples with Bonferroni correction, P_val=8.325e-02 stat=-2.533e+00\n",
            "3D CNN_NC v.s. 3D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-8.230e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEJCAYAAAB/pOvWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfVhUdf7/8SeM3ChhKiasA5a3ZBpIaqIm+TUrLSnvNSkyy9q6ktota9mozLu2tvKiXDOvtday5EdqlrgZV5mSZWDmXWboej8k5oApog46zO8PY5KGmxGZM9y8Htfl1XDOZ855z1mW13zO+XzO8XE4HA5EREQu4OvtAkREpO5ROIiIiAuFg4iIuFA4iIiIC4WDiIi4aOLtAmpDnz59MJvN3i5DRKReycvLIzs7u8J1DSIczGYzy5cv93YZIiL1ysiRIytdp9NKIiLiQuEgIiIuFA4iIuKiQVxzqMjZs2exWCycOXPG26XUWGBgIOHh4fj5+Xm7FBFpZBpsOFgsFoKDg7nqqqvw8fHxdjkXzeFwUFBQgMVioX379t4uR0QamQZ7WunMmTOEhITUy2AA8PHxISQkpF73fESk/mqw4QDU22AoU9/rF5H6q0GHg4iI1EyjCQd7aanh24uMjOQf//iH8+eFCxfyxhtvOH9esWIFw4YNIz4+nuHDh7Nw4cJarVFEpKYa7AXpPzL5+pKxeV+tbW9YTPUXif39/cnMzOTBBx+kVatW5datW7eORYsWsXDhQkJDQykpKWHFihW1Vp9IRSwWC5MnT6Znz55s3ryZ0NBQ5s2bR3p6OmlpaZhMJjp16sScOXO8Xap4WaPpOXhDkyZNGDduHIsWLXJZt2DBAp566ilCQ0OB80EyduxYo0uURujAgQMkJCSwatUqgoOD+eyzz1iwYAErVqxg5cqVvPDCC94uUeoAhYOHJSQksHLlSoqKisot3717N927d/dSVdKYhYeH07VrVwC6detGXl4ekZGRPPnkk3z88ceYTCYvVyh1gcLBwy677DLuvPNO3n33XW+XIgKc76WWMZlM2O12FixYwIQJE/jxxx8ZPXo0586d82KFUhcoHAxw7733smzZMk6fPu1c1qlTJ3744QcvViVyXmlpKYcPHyY2NpYnn3ySoqIiTp065e2yxMsUDgZo0aIFQ4YMYenSpc5lDz30EP/85z85evQoACUlJXz44YfeKlEasdLSUqZOnUp8fDwjRowgMTGR5s2be7ss8bJGM1rJXlrq1giji9meydf9bJ00aRLvv/++8+cbb7wRq9XKfffdh8PhwMfHh1GjRtVafSIVCQ8PJyMjw/nz/fff78VqpC5rNOFwMX/Ia2t7mzdvdr5u3bo1W7duLbd+1KhRCgS5JMuXL2f16tVerWHIkCFVPjRG6qdGEw4iDdHq1avZtWsXXbp08cr+d+3aBVT9RLGGpDHNE1E4iNRzXbp0YcGCBV7Z94MPPuiV/XrTgQMHeO2115g5cyaPPfaYc57ImjVr8Pf358SJE94usVbogrSIyEVoLPNEFA4iIhehscwT0WklkXrsjjvuaNT7rwsunCfSs2dPVq1axalTp+r9cGCFg0g9NmzYsEa9/7qgbJ7IyZMncTgcDWaeSKMJh5Jzdvyb1N65QHe217VrV7p06YLdbqdDhw689NJLNG3a1Ln83LlzmEwmhg8fzsSJE/Gt5eG2IlK7GtM8kUYTDv5NTAydVXu3xP70meHVtgkMDOTjjz8G4IknniAtLY377ruv3PKCggKeeOIJTp48SVJSUq3VJyLV0zyRyumrqkF69erFgQMHXJaHhIQwY8YM3n//fRwOhxcqE2m8yuaJeMuuXbu8Hk6VaTQ9B286d+4cWVlZDBgwoML1ERER2O12CgoKaN26tcHViTRumidSMYWDB505c4Y777wTON9zGD16tJcrEhFxj8LBgy68tlCVQ4cOYTKZCAkJMaAqEZHqGXLNYebMmfTr14/IyEgeeuihSttt2rSJ+Ph4unfvzogRI9ixY4cR5XlVYWEhzz//PAkJCfj4+Hi7HJFG5Y477vDqXA1v778qhvUcbrvtNt57771K19tsNpKSkggICCA5OZn58+eTlJREZmZmrUxHLzlnd2uE0cVsr6ZDY8tON5UNZb3zzju57777aq02EXGPt+dpeHv/VTEkHFJSUrBYLFWGQ1ZWFlarlalTp5KQkIDVamXevHnk5OTQt2/fS66hNuc4uLu9C2/ZfaGdO3fWai0iIrWtzgxltVgsAISGhgIQFhYGnD8fLyIixqoz4fBHGvMvIuI9Xg2HkpISSkpKgPPT0gHy8/MBOHLkCHB+DkBN1feAqe/1i0j9Zcg1h7Vr1zpnIR4+fJgPP/yQ3r17M2nSJI4dO8bmzZuJi4sjJCSEJUuWEBQUxNKlSzGbzVx//fU12mdgYCAFBQWEhITUy1FADoeDgoICAgMDvV2KiDRChoTDwoULycnJASA3N5eUlBRefPHFcm0CAgJITU1l+vTpzJ49m06dOjFjxowaj1QKDw/HYrFw9OjRS67fWwIDA509KhERI/k4GsC5i5EjR7J8+XJvlyEe1pie3ytihKr+dmqGtNQrjeX5vSLeVmdHK4lUpLE8v1fE2xQOUq80luf3inibwkHqtQuf3/vkk09SVFTEqVOnvF2WSL2naw5SrzXU5/eKeJvCQeqNxvT8XhFvUziI1+j5vSJ1l8LBQBqnX17Z83u7dOnilf2XzdpXOIi4UjgYTOP0y9Pze0XqJoWDwaoap3/TTTcxePBgL1coUr1vv/2W//znPxQWFlJUVERwcDCtWrVi4sSJxMbGers8qQUaymowjdOXhiA9PZ3du3dTUFBASUkJBQUF7N69m/T0dG+XJrVEPQcvu3Ccfs+ePVm1ahWnTp1qFMMxvf3sXG/vvz4bO3Ysp06dYu/evdjtdkwmEx06dGDs2LHeLk1qicLByxrzOH1vPz/X2/uvz2JjY4mNjSUxMZG8vDzCwsKYP3++t8uSWqRwMJDG6YtIfaFrDiIi4kI9h0ugSVwi0lApHC6BJnGJSE3Uh6HACodLpElcInKxyoYClykoKKCgoID09PQ6Ew665iAiYrCxY8fSuXNn58OpTCYTnTt3rlNDgdVzuATeHifv7f2LSM3Uh6HACodL4O1x8t7ev4g0XDqtJCIiLgzrOWzatIlp06axb98+OnfuzMyZM+nWrVu5NmfPnuXll19m1apV2Gw24uPjeeaZZ/Dz8zOqTJFGp+ScHf8mpka1/9ocLdRQj58h4WCz2UhKSiIgIIDk5GTmz59PUlISmZmZzgsyAO+99x7vvvsuY8eOpWnTpixatIh27doxadIkI8oUaZT8m5gYOmtFjd5bUlgMQF5hcY238ekzw2v0vktRm6OFGurxM+S0UlZWFlarlQkTJpCQkMDo0aOxWCzk5OSUa1f286OPPsqTTz4JwEcffWREiSLSiNSH0ULeZkg4WCwWAEJDQwEICwsD4NChQ+XatWrVCoANGzbw1VdflXuviEhtiY2NZf78+c6/RWWjherKHIO6wCujlRwOR4XLJ0+ezFdffcXTTz9NkyZNCAwMJCAgwODqRETEkHAIDw8HID8/H4AjR44AEBERgc1mw9fXFz8/P9q3b09mZia7du2iefPmjBkzhk6dOhlRooiIXMCQcIiLiyMkJIQlS5YQFBTE0qVLMZvNmM1moqKiGDhwIG+99RY7d+5kzZo1hIWFsWrVKoqKinQxWkQanFLrPuz7suHMb8+MP3OCsxvTMLXvg2/r9t4t7jeGXHMICAggNTWVoKAgZs+eTUhICKmpqfj6uu5+2bJlPP/88+zfv5/nn3+eQYMGGVGiiIhh7Ac34yg6Co7S8wscpTiKjmI/uNm7hV3AsGsOvXv3ZuXKlS7Lc3Nzna+7du3KmjVrjCpJRMQrTO1isNtLcNhOwbkz0CQQn4BmmNrFeLs0J90+Q0TEYL6t29eZ00eV0e0zROSilVr3cXZjmss581LrPsNrsZeWGr7PxsDtnsPPP/9M27ZtPVmLiNQTznPmZS44Z270N2KTry8Zm2sWSsW2c87/1nQbw2Lqdg+gptzuOQwePJhJkyaRkZFBSUmJJ2sSkTrO1C4Gn+ArwD8IfE3gH4RP8BV16py5XBq3ew6lpaV88803bNiwgRdeeIHbb7+dkSNHEhUV5cn6RKQOqg/nzOXSuN1zWLNmDU899RTXXnstJ0+eJC0tjXHjxjFs2DDeeecdTp486ck6RUTEQG6HQ9u2bZk0aRLp6el88cUXDBw4EIfDwZ49e3j55Ze56aab2LBhgydrFRERg1zUaCWr1cqCBQuYOHEi69atAyAwMJD/+7//w2azMX36dI8UKSIixnL7msMjjzxCVlYWdrsdh8PBlVdeyV133cWoUaMIDg7m9ddfr3PPQBURkZpxOxzWrFmDj48PAwYM4O677yYuLq7c+p49e3LnnXfWeoEiIrVt74/b2PDZCk4UWgE4UWjl/TnT6XvrcDpco0E2cBHhMHHiRBISEoiIiKhwff/+/enfv3+tFSYi4imb1q7ml7yDzp9LS+38kneQTes+Uzj8xu1rDpmZmWRkZJRb9tFHH/HYY4/VelEiIp7Uc+AQ2pjbEdS8BU38/Ahq3oI25nb0vPFWb5dWZ1zUDOlff/213LIdO3aQmZlZ60WJiHhSh2ui1EOoRrXhMHfuXOfrrVu3On92OBysWbMGf39/z1UnIiJe4VY4+Pj44OPjw9atW9m6datzncPhIDo62qMFioiI8aoNh969ewOwceNGQkNDadeu3fk3NmlC27ZtmTx5smcrFBERw1UbDu+99x4AgwYNYvTo0TzyyCMeL0pERLzrouY5iIhI41BlONx0002MHj2ahx9+mJtuuqnCNj4+Pnz++eceKU5EKvbtt9/yn//8h8LCQoqKiggODqZVq1ZMnDiR2NhYb5cnDUCV4ZCXl+ccvpqXl1dhGx8fn9qvSkSqlJ6ezu7du50/FxQUUFBQQHp6usJBakWV4fDiiy/SsWNH52sRqRvGjh3LqVOn2Lt3L3a7HZPJRIcOHRg7dqy3S5MGospwGDFiRIWva2LTpk1MmzaNffv20blzZ2bOnEm3bt3KtSkpKeGFF17giy++4NSpU3To0IGnn36avn37XtK+RRqa2NhYYmNjSUxMJC8vj7CwMN34UmqV27fPePfdd3nuueew2+0AnDt3jmnTpvHuu+9W+16bzUZSUhLFxcUkJydjtVpJSkpybqvMihUrWLp0KV27duWxxx7jp59+4tlnn73IjyQiIpfK7XBYuHAhR48exWQyAefnORQUFPD2229X+96srCysVisTJkwgISGB0aNHY7FYyMnJKdeutLQUgM6dO9OvXz/8/f1p3rz5xXweERGpBW6HQ2FhIa1bty63rEWLFhQWFlb7XovFAkBoaCgAYWFhABw6dKhcuxEjRnDzzTezaNEihg8fTtOmTXWtQ0TEC9wOh9DQUNavX8/x48cBOH78OOvXr6dNmzYXvVOHw1Hh8i1btrB27Vri4+OZM2cOdrud5OTkStuLiIhnuD0JrlevXqxYsYLBgwfTvn179u/fT1FREcOHD6/2veHh4QDk5+cDcOTIEQAiIiKw2Wz4+vri5+fH6tWrOXv2LOPHj6dXr16kp6ezYcMGjh07RqtWrWry+TxCY8xFpKFzOxymTJnCl19+yfHjx9m2bRtw/rTSlClTqn1vXFwcISEhLFmyhKCgIJYuXYrZbMZsNhMVFcXAgQN56623nA8S+ve//01ubi5btmyhRYsWtGzZsoYfzzM0xlxEGjq3w8FsNrNy5UqWLl3K4cOHadu2LaNGjXLrtFJAQACpqalMnz6d2bNn06lTJ2bMmIGvb/mzWgkJCezZs4cvv/ySDRs20LFjR5566qk6N9FOY8yltthLSzH5un12V8QwbocDQJs2bWp8473evXuzcuVKl+W5ubnO1wEBAcyaNatG2zeSt8eY67RWw2Hy9SVj874av7/Yds7535puZ1hM+xrvXxquKsMhMTGRIUOGMGHCBBITEyts4+Pjw6JFizxSnFRMp7VExNOqDIecnBy6du3qfF2RunbK52J5u1tfk/3rtJaIeFqV4fDoo4/So0cP5+uG6FK69d7q0nv7tJaINHxVhkPLli1p1qwZAH369CEsLMw5okhERBquKs9nzJgxg8zMTOD89Yf333/fkKJERMS7quw5+Pv7s2XLFrKzs3E4HBw+fJiNGze6tCt7znRjsffHbWz4bAUnCq0AnCi08v6c6fS9dTgdronycnXSGOh3UDytynDo1KkTW7duZeLEifj4+JCZmensSZTx8fHhxx9/9GiRdc2mtav5Je+g8+fSUju/5B1k07rPLvr/mCXn7Pg3MdV2ifVm/1Iztfk7KFKRKsNh2rRpPPPMM+zdu5fS0tIK73HUGO971HPgEEpsZyguOoHtdDEBTYMICm5Ozxtvveht+TcxMXTWihrVUVJYDEBeYXGNt/HpM9Xf/kTqntr8HRSpSJXhEBUV5Zy4dvXVV3PvvfeSnJxsSGF1WYdrovTtTLxKv4PiaW7PkP7iiy8IDg72ZC0ihtAMc5HqVRkOc+fOpUePHtxwww189NFHlbZrqHMg6qpS6z7s+7LhzInzC86c4OzGNEzt++DbWrdCqI5mmItUr9pwmDhxIjfccANz586tdDa0wsFY9oObcRQd/X2BoxRH0VHsBzfXi3Dw9jd3zTAXqV6V4dC7d2/npLfGNly1LjO1i8FuL8FhOwXnzkCTQHwCmmFqF+Pt0tzi7W/ummEuUr0qw+G9996r8LV4l2/r9vWih1AZfXMXqfvcvuPbnj172LhxI3a7HYBz586xceNG9uzZ47HipGGKjY1l/vz5zmeJl31z1/l+kbrD7dFKf/3rX/H19XVemG7SpAkvvfQSdru9yovVIiJS/7jdczhw4ADXXnttuWXXXHMN+/fvr+2aRNxWcs7eqPcv4ilu9xyaNm3Kvn3lb0u9b98+AgMDa70oEXdphrmIZ7gdDh07duS7777jqaeeIiYmhs2bN/Pdd9/Rq1cvT9YnIiJe4HY4TJo0ie+++46VK1eWexb0/fff75HCRETEe9y+5jBo0CBeeOEFwsPDMZlMREREMG3aNAYOHOjB8kRExBvc7jkAjBs3jnHjxnmqFhFD6PYjItW7qCfbf/HFFyQmJnLrrbdy5MgR5s6dy7Zt29x676ZNm4iPj6d79+6MGDGCHTt2uLT529/+RmRkZLl/gwYNupgSRarlvP2Io/T8ggtuPyIi57ndc8jKyuLRRx/F4XDg4+NDq1atWLx4Mfv27ePVV1+t8r02m42kpCQCAgJITk5m/vz5JCUlkZmZicn0+4Nm7rrrLgYMGADA3r17mTt3Ltdcc00NP5pIxer77UdEjOB2OMybN4+mTZty1VVXsXPnTvz8/LjuuuvYvLn6b1tZWVlYrVamTp1KQkICVquVefPmkZOTQ9++fZ3toqOjiY6OBmD69OkAjB8//mI/k0iV6vvtR0SM4PZppd27dzN06NByQ1fbtGlDQUFBte+1WCwAhIaGAjhvm3Do0KEK258+fZpPPvmEK6+8kv79+7tbooiI1BK3w6FZs2YuQbBr1y5atGhx0Tut7tGiq1atoqioiLFjx1Z6m3AREfEct08rdevWjfXr12M2m4Hz8x42b97s1gXj8PBwAPLz8wE4cuQIABEREdhsNnx9ffHz83O2T0tLw9/fn5EjR7r/SUREpNa43XN4/PHHMZlMHDhwAIfDwTfffIOfnx9Tpkyp9r1xcXGEhISwZMkSPvjgA5YuXYrZbMZsNhMVFVXuYUE//vgj27dv59Zbb6VVq1Y1+1QiInJJ3A6Hq6++mrS0NEaNGkW/fv0YNWoUaWlpXH311dW+NyAggNTUVIKCgpg9ezYhISGkpqbi6+u6+7S0NEAXokVEvOmiJsF17dqVWbNmUVhYeNHf6nv37l3uthtlcnNzy/08ffp050glERHxDrd7DiUlJcyaNYuYmBj69+9PTEwMM2fOxGazebI+ERHxArd7Dv/4xz9YsmSJc6TR6dOnef/99yktLeW5557zWIEiImI8t3sOq1evJiAggL///e+88847JCcn4+/vz+rVqz1Zn4iIeIHbPQeHw8HQoUNJTEwEoG/fvvz000+sW7fOY8WJiIh3uN1zGD16NLt27eLs2bPA+WsQubm53HXXXR4rTkREvMPtnsP333/Pzp07ufHGG2nXrh0HDx7k+PHjBAUFOXsTPj4+LFq0yGPFioiIMdwOh02bNgFQWFhIYWGhc/nGjRudr3Wri8bHXlqKqYL5KiJSv7kdDhfOYhYpY/L1JWPzvhq9t9h2zvnfmm5jWIzuririCTUKh5ycHIqLi+nRowctW7b0SGEiIuI91YbD/Pnz2bBhA6+//jqXX345ycnJrFixAoDLL7+chQsX0q1bN48XKiIixqn2ZHFmZiaFhYVcfvnl7N+/n48++giHw4HD4eDXX39l7ty5RtQpIiIGqjYcfv75ZyIjIwH4+uuvgfNPbMvJyaFr165s377dsxWKiIjhqg2HkydPEhwcDMD27dvx8fFh6NChNG/enB49enD8+HGPFykiIsaqNhxCQkLIzs5m586dzp5Djx49ACgoKHAGh4iINBzVhkNsbCx79+5l5MiRWK1WQkJCiI6OBmDHjh1ERER4vEgRETFWteHw17/+lW7duuFwOAgKCmLmzJn4+PiQnZ1NXl4evXr1MqJOERExULVDWUNDQ1m2bBknTpwgKCgIk8kEQM+ePfn+++8JCAjweJEiImIstyfBNW/evPwbmzShSZOLepCciIjUE7opjoiIuFA4iIiIC4WDiIi4MCwcNm3aRHx8PN27d2fEiBHs2LGjwna7du0iMTGRqKgo+vTpw8svv2xUiSIi8htDwsFms5GUlERxcTHJyclYrVaSkpKw2+3l2p05c4YHHniA3NxckpKSSEpKolmzZkaUKCIiFzBkuFFWVhZWq5WpU6eSkJCA1Wpl3rx55OTk0LdvX2e7jIwMjhw5wsyZM4mPjycwMNCI8kRE5A8M6TlYLBbg/JwJgLCwMAAOHTpUrt2ePXsAeOedd4iOjiY2Npb//ve/RpQoIiIX8MoFaYfDUeHykpISAK644greeOMN/P39SU5O5uTJk0aWJyLS6BkSDuHh4QDk5+cDcOTIEQAiIiKw2WycPXu2XLuhQ4dyyy230KtXL86cOcMvv/xiRJkiIvIbQ8IhLi6OkJAQlixZwgcffMDSpUsxm82YzWaioqKcjyC97bbb8Pf3Z/ny5aSnp7NhwwZCQ0Np166dEWWKiMhvDAmHgIAAUlNTCQoKYvbs2YSEhJCamoqvb/ndh4aG8uqrr1JYWMisWbO48sorefPNN3WbDhERgxn2V7d3796sXLnSZXlubm65n2+55RZuueUWo8oSL9j74zY2fLaCE4VWAE4UWnl/znT63jqcDtdEebk6EQHNkBYv2LR2Nb/kHaS09Pw8l9JSO7/kHWTTus+8XJmIlNH5GjFcz4FDKLGdobjoBLbTxQQ0DSIouDk9b7zV26WJyG8UDmK4DtdE6fSRSB2n00oiIuJC4SAiIi4UDiIi4kLhICIiLhQOIiLiQuEgIiIuFA4iIuJC4SAiIi4UDiIi4kLhICIiLhQOIiLiQuEgIiIuFA4iIuJC4SAiIi4UDiIi4kLhICIiLhQOIiLiQuEgIiIuFA4iIuLCsHDYtGkT8fHxdO/enREjRrBjxw6XNhaLhcjIyHL/Zs2aZVSJIiLymyZG7MRms5GUlERAQADJycnMnz+fpKQkMjMzMZlMLu3Hjx/P9ddfD0D79u2NKFFERC5gSDhkZWVhtVqZOnUqCQkJWK1W5s2bR05ODn379nVp3717dwYNGkTTpk2NKE9ERP7AkNNKFosFgNDQUADCwsIAOHToUIXtn332WXr06MHtt9/Oli1bjChRREQuYEjP4Y8cDkeFy5s1a8aUKVPo2rUrBw4c4NVXX+XJJ5/k888/N7hCEZHGzZBwCA8PByA/Px+AI0eOABAREYHNZsPX1xc/Pz9atWrFo48+6nxfRkYGO3bswGazERAQYESpIiKCQeEQFxdHSEgIS5YsISgoiKVLl2I2mzGbzURFRTFw4EDeeust0tPT2b59O1FRUVgsFnbu3MnVV1+tYBARMZgh1xwCAgJITU0lKCiI2bNnExISQmpqKr6+5Xd/1VVXkZuby+zZs/nggw8YMGAAqampRpQoIiIXMOyaQ+/evVm5cqXL8tzcXOfr66+/nvT0dKNKEhGRSmiGtIiIuFA4iIiIC4WDiIi4UDiIiIgLhYOIiLhQOIiIiAuFg4iIuFA4iIiIC4WDiIi4UDiIiIgLhYOIiLhQOIiIiAuFg4iIuFA4iIiIC4WDiIi4UDiIiIgLhYOIiLhQOIiIiAuFg4iIuFA4iIiIC4WDiIi4UDiIiIgLw8Jh06ZNxMfH0717d0aMGMGOHTsqbbtnzx6uvfZaIiMjWb16tVEliojIbwwJB5vNRlJSEsXFxSQnJ2O1WklKSsJut7u0dTgcpKSkYDKZjChNREQqYEg4ZGVlYbVamTBhAgkJCYwePRqLxUJOTo5L2w8++ICff/6ZcePGGVGaiIhUoIkRO7FYLACEhoYCEBYWBsChQ4fo27evs92RI0d47bXXeOWVV/jhhx/c3n5eXh4jR46sxYqN8zbQ1Iv7HznyXS/u/dLp+F06HcNLU5+PX15eXqXrDAmHP3I4HBUuf+WVV+jevTsdOnTg66+/BsBqtVJcXExQUFCl28vOzvZInSIijZUh4RAeHg5Afn4+cL6HABAREYHNZsPX1xc/Pz/y8/PJycnhlltucb53xowZBAcHc+eddxpRqoiIYFA4xMXFERISwpIlSwgKCmLp0qWYzWbMZjNRUVEMHDiQt956iylTplBYWAjAp59+yurVq5k0aRK9e/c2okwREfmNIRekAwICSE1NJSgoiNmzZxMSEkJqaiq+vuV3f/311zNkyBCGDBlCp06dAIiOjqZt27ZGlCkiIr/xcVR2AUBERBotzZAWEREXCgcREXGhcDCAxWIhMjKSyMhI/vWvf40T9f8AAAnLSURBVDmX//3vf3cuz87OJjIykunTp3uxUs+bOXMm/fr1IzIykoceeqjcurJj0b17d/r3788jjzzCzp07K91WQUEBycnJ9OvXj6ioKOLj48nMzARg0KBBREZG8sYbbwC//29Qts833niDyMhIbr75ZudM/XvuuYfIyEjnoIi6Zv/+/dxzzz306dOHmJgY7rvvPg4ePAjg/P2JjIwkOjqam266ieeee67Kz/LDDz9w//33c91113Hddddx9913s2fPnnK/r2XDxJcvX05kZCQLFy4Efj9Wf/vb35zbi4yMZNiwYR48ApduzJgxxMTEEB0dzciRI9m4cSOg41cRhYPBPvroIxwOB8XFxXz66afeLscrbrvttkrXhYWFMXPmTIYOHcq6deuYMGEC//vf/1zanTlzhrvvvpvly5fTp08fnn32Wfr27etyz67FixdTXFxc6f4OHjzIf//735p/GAMdOXIEh8PBlClTGDVqFN988w3PPPNMuTY33HADKSkpdOnShf/3//4f99xzD2fOnHHZ1u7du7n77rvJzs7mrrvu4umnn6Z169bs37+/XLu33nqrypoyMjL4+eefL/mzGSUmJoaUlBQeeeQRfvrpJ1JSUsqt1/H7nVcmwTVWERERHDp0iOzsbCwWC+fOnSM0NNQ576MxSElJwWKx8N5771W4Pjg4mOHDhzN8+HBat27NnDlzWLBgAS+//HK5ditXrmTv3r3ccMMNzJkzx7m8tLTU+drPz4/Tp0+TlpbGrbfeWuH+LrvsMhYsWFCnvrFVJiYmhsWLFzt/XrlypUtwXnnllYwZM4YxY8YwadIkvv76azIyMhg9enS5dgsWLOD06dM88cQTPPjggwCMGzeO0tJS5x+ryy67jK+//prt27dXWlNAQAD//ve/ee6552rrY3pUcnIyx44dw2Kx8Oabb+Lj41NuvY7f79RzMFDHjh2Jjo5m2bJlLFu2jMGDB9O8eXNvl1VnxcXFAVR4K5WyHsKAAQPKLb9weLSfnx+jRo3inXfeoaSkpMJ9jB8/nl27drFmzZraKttj/P39na+3b9/Or7/+WuUcoEs9fj179qRLly5VfvsdP348y5Yto6CgwL0P4WVFRUX07duXMWPG4Ofnx8yZMytt29iPn8LBYKNGjWL16tV8//339fZ+UEYpG2X9x293lS2ryAMPPMCxY8dYvnx5heuvvfZa+vXrx4IFC2peqMH27NnDww8/jNlsdjktcqFLPX4+Pj5MnjyZzz//nD179lTY5q677iIgIIB33nnHzeq9q1mzZrz99tukpKRgs9l4/fXXK23b2I+fwsFgt99+OyaTibCwMPr37+/tcuq09evXA9CtWzeXdWXLyu7BVebC00oAZrOZYcOGsWTJkkr389BDD7FlyxZ+/PHHSy3Z4/73v/+RmJiIv78/ixYtok2bNpW2def4lbUp88fjd/vttxMeHk5aWlqF+2jWrBl33313lce3LmnSpAn9+/fnnnvuISoqiuzs7EovOjf246drDga77LLLmD17NkFBQS4zxAG2bNnCK6+8AkBgYCCPPvqo0SV61Nq1a9m1axcAhw8f5sMPP6R3795cddVVwPlu/4oVK/jhhx9IS0ujWbNmznO6Fxo2bBgLFy4kKyuLJ554gtjYWPbu3Yu/vz9/+ctfyrV98MEH+fjjjyutKTY2lh49erBly5ba+6AecPjwYRITE/n11195/PHH2bZtG9u2beP22293tjlw4AAffvgha9euZf369XTq1KnC6ymTJ08mMzOT1NRUTpw4QUREBNnZ2dx2221ERkY625lMJh544AGef/75SutKTEysk998/+irr77i008/JSYmhvz8fDZv3kzr1q1p2bKls42O3+8UDl5Q1WidHTt2OM9nBgcHN7hwWLhwofM5Hrm5uaSkpPDiiy86wyE/P5+UlBRatGjBjTfeyJQpU5y3UrlQYGAgixcv5p///Cdr164lMzOTiIgIHn/8cZe2HTt25Oabb3YOc63IQw89xMMPP1w7H9JDDh486Dw3/eqrrzqXXxgO69evZ+PGjVxxxRWMHz+exx9/nMDAQJdtde7cmcWLFzNnzhznRe6rr76aK6+80qXtyJEj+de//sUvv/xSYV0tWrRg3LhxdfIP3IUuv/xytm3bRkZGBv7+/lx33XVMnTq13CkiHb/f6fYZIiLiQtccRETEhcJBRERcKBxERMSFwkFERFwoHERExIXCQcRgf7xjpzvK7uJZdpdZEU9TOIj8puwP8B9vpQyQmprqXDdo0CAvVShiHIWDSAUyMjI4evQocP724HXx9gYinqRwEPmDFi1acPbsWefM1+XLl3Ps2DFatGhRrt3Jkyd59dVXGTJkCNHR0QwePJjnn3++3B02Dx8+zOTJk4mJieHmm2+u8BkepaWlfPjhh4wYMYKYmBji4uKYOnUq+fn5nv2gIlVQOIj8QVxcHGazmbS0NE6dOsWiRYto1qwZd9xxh7NNaWkpDzzwAAsWLODkyZMMGzYMX19f0tLSGD9+PKdPn6a0tJQ///nPZGVl0bx5c66//npeeukll/299tprpKSkcPToUW655RY6derEJ598wvjx4zl58qSRH13ESeEg8gcmk4l77rmHX3/9lb/85S/s37+fUaNGlXv2xvfff8/mzZsBmD9/PrNmzWLx4sWYTCYOHjxIZmYm27Zt46effgLgzTffZNasWcydO7fcvkpKSpw9lO7du9O8eXM6duxIQEAAhw8frvJ+UCKepBvviVRgzJgxzJ07l7Vr12Iymbj33ntZsWKFc/2Fj3bs0qULAG3atKFly5ZYrVYOHz6Mn5+fs03ZzQPL2pY5duwYp0+fBuDLL790qePw4cO196FELoJ6DiIVuOyyyxg7diwAgwcPJiIiotz6tm3bOl/v3r0bgKNHj3Ls2DEA/vSnP/GnP/3J2abscZ5ltysv07JlS+ddP19++WVyc3Od/7Kysrj//vtr+ZOJuEc9B5FK/PnPf2bAgAF07NjRZd11111HdHQ0W7du5eGHHyYuLo6NGzdit9sJDw/n5ptvJjAwkC5durBr1y4efvhhbrjhBpeHw/j7+zNhwgTefvttnnvuOdatW0fTpk05cOAA33//PZmZmYSHhxv1kUWc1HMQqcTll19Ov379CA0NdVnn6+vLwoULeeCBB2jatCmffPIJZ8+eZdy4cc6HFPn6+jJ//nwGDBjAiRMn+Pbbb3niiSdctjV16lRmzJhBx44dWbduHZ9++inHjx/n3nvvLfcgGhEj6XkOIiLiQj0HERFxoXAQEREXCgcREXGhcBARERcKBxERcaFwEBERFwoHERFxoXAQEREX/x+kA9zXz2USwQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 526
        },
        "id": "QD895DcsdRnl",
        "outputId": "f961f1b9-72a2-4cb1-f042-760c9fb70093"
      },
      "source": [
        "arousal_df_new = pd.melt(arousal_df[valence_df['Data']=='PD'], id_vars=['Model'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "x = \"Model\"\n",
        "y = \"Value\"\n",
        "hue = \"Test\"\n",
        "ax = sns.barplot(data=arousal_df_new, x=x, y=y, hue = hue, palette = \"Blues\", capsize=0.05, ci=68)\n",
        "ax.set_ylim(bottom=0.6)\n",
        "ax.legend(loc='lower right', bbox_to_anchor=(1,0))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Sensitivity/Specificity',fontsize = 13, weight='bold')\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 13, weight='bold');\n",
        "ax.set_yticklabels([\"0.6\", \"0.7\", \"0.8\", \"0.9\", \"1.0\"], size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=arousal_df_new, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"Sensitivity\"), (\"ML\", \"Specificity\")),\n",
        "                                 ((\"1D CNN\", \"Sensitivity\"), (\"1D CNN\", \"Specificity\")),\n",
        "                                 ((\"2D CNN\", \"Sensitivity\"), (\"2D CNN\", \"Specificity\")),\n",
        "                               ((\"3D CNN\", \"Sensitivity\"), (\"3D CNN\", \"Specificity\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "plt.savefig('sens_and_speci_arousal.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_Sensitivity v.s. 1D CNN_Specificity: t-test independent samples with Bonferroni correction, P_val=1.449e-08 stat=1.060e+01\n",
            "ML_Sensitivity v.s. ML_Specificity: t-test independent samples with Bonferroni correction, P_val=6.295e-09 stat=1.118e+01\n",
            "2D CNN_Sensitivity v.s. 2D CNN_Specificity: t-test independent samples with Bonferroni correction, P_val=2.136e-01 stat=2.067e+00\n",
            "3D CNN_Sensitivity v.s. 3D CNN_Specificity: t-test independent samples with Bonferroni correction, P_val=8.291e-03 stat=3.594e+00\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEMCAYAAAAvaXplAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfVyN9/8H8NfpVCeVVGdTujGkESl3kbtm5i6TSDUqN2N8MfrNxpYvXzbfMKZZbrb4jk0YQowMzV1h7uZmMxJClDTdTXTfOb8/WmeOU+dcpdOpvJ6Pxx4757o+1/V5n/N4OO8+1+dOJJfL5SAiInqGnq4DICKiuofJgYiIVDA5EBGRCiYHIiJSweRAREQq9HUdQE3o3r07bG1tdR0GEVG9kpqairNnz1Z4rkEkB1tbW0RHR+s6DCKiesXHx6fSc3ysREREKpgciIhIBZMDERGpYHIgIiIVTA5ERKSCyYGIiFQwORARkQomByIiUsHkQEREKpgciIhIBZMDERGpYHIgIiIVTA5ERALJ5XIAwKpVq9S+bwgaxKqsRES1Ye/evXj06BEKCwvxv//9D02bNlV57+3treswawRbDkQvkZSUFHh6emLevHl4++23MWHCBBQUFCAyMhJDhgyBl5cXZs6cqesw6yxvb29YW1tj/fr1sLGxqfB9Q8HkQPSSSU5ORmBgIPbv34/GjRvj0KFDWLduHfbs2YN9+/bhs88+03WIdda+ffvw8OFDTJw4EQ8ePKjwfUPBx0pELxk7Ozs4OTkBANq3b4/U1FS0adMGs2bNwltvvYX+/fvrOMK6a+jQoRCJRFi1ahUmTZoEuVyu8r6hYMuB6CVjaGioeC0Wi1FaWop169YhICAA165dg6+vL0pKSnQYYd0lEokAADNmzFD7viFgciB6yclkMqSlpcHd3R2zZs1Cbm4u8vLydB0W6RiTgxYJGfYm9BiRtshkMsyePRteXl4YMWIExo4dCzMzM12HRTomkjeAXx4fHx9ER0frOgwVP/74Ix49eoScnBw0adJEMezt2fcABB1rSKMgiKhuUPfbyQ5pLfL29kZMTAzCwsKwfPlyvP322yrvAQg+9rJ5trNvxowZFb4HIOhYQ3oW/Kzo6GgcPHhQpzEMHjwYPj4+Oo2huvj9VY4tBy3at28f0tPTFS0Aa2trlfcABB3z8vLS5UfRCba8NJs8eTJu3LiB119/XSf1l9e9bt06ndT/ol7270/tb6e8ARgxYoSuQ6iQTCaTy+Vy+cqVKyt9L/TYy2rfvn3ytm3bymNiYip8X5VjDdGkSZPkkyZNemnrf1G6jl/X9av77eRjJS2qyrA3ocdeJuUtr2cnGD3/HoCgYy9jy4voRTA5UJ0ldMKR0GNEJByTA9VZbHlpNmzYsJe6/hel6/h1Xb86TA5E9djQoUNf6vpflK7j13X96nASHBERqWDL4QVwjDQRNVSCk8O///1v+Pr6onPnztqMp145ePCgzsdIA6i3yYHJlajuEpwcoqOjsXv3bjRv3hwjR46Et7c3rKystBlbvaDLCSyTJ0/WSb01hcmVqO4SnBxGjBiBo0ePIjk5GStWrEB4eDh69eqFkSNHol+/fjAwMNBmnNRAMbkS1U2Ck8OSJUtQUlKCX375BYcOHcLhw4dx4sQJnDhxAk2aNEFQUBCmTJkCfX12YxAR1XdV+iXX19eHu7s7srKykJSUhMuXLwMAcnJysGbNGqSkpODzzz/XSqB1ka7HKOu6fiJquAQnh4SEBOzYsQMxMTHIzc2FXC6HpaUl/Pz8MHjwYCxZsgSxsbEVJofQ0FD89NNPyMzMRN++fbF27doK67hw4QI+/fRT3LlzB46OjggNDUX79u2r/+m0TNdjlHVd/4vSdXLTdf1EdVmV+hxEIhHkcjlcXFwQGBgIT09PxZaD/fr1w7lz5yq9fsiQIdi0aVOl5wsLCxEcHAyJRII5c+YgIiICwcHBiI2NhVgsrsJHovpC18lN1/UT1WWCJ8EZGhpi2LBhiIqKQlRUFLy9vZX2ovX19cWRI0cqvHbevHkYP3682vvHx8cjIyMDAQEBCAwMhK+vL1JSUtQmHCIi0g7ByeH//u//EBQUBBcXF8WxzMxM3Lp1CwBgamoKW1vbageSkpICAIrhseX7Gty/f7/a9yQiouoRnByWL1+OmJgYpWNr167V2lLIXEmTiEh3BCeHin6sHz58+EKVFxUVoaioCABgZ2endM/09HQAgL29/QvVQUREVaexQ9rJyQlA2bLHkZGRiIyMVDpvaWmpsZLjx48rZqOmpaVhx44dcHNzw4QJE5CdnY1Lly7Bw8MDUqkUW7duhYmJCXbu3AlbW1t069atOp+LqEE7c+YMvv/+e2RlZSE3NxeNGzeGpaUlxo8fD3d3d12HV+fx+9NMY8tBLpcrWg3lr5/9LyAgQGMl69evR1hYGAAgMTER8+bNw8WLF5XKSCQShIeHw8TEBIsXL4ZUKkV4eDhHKhFVICoqCjdv3kRmZiaKioqQmZmJmzdvIioqSteh1Qv8/jTT2HKIjIyEXC7HuHHjMGjQIAQGBgIAxGIxbG1tFR3H6lQ2hPX5NW3c3NwU2zwSvQxKZTKI9aq+cr6LiwuuX7+OwsJCxTGJRKI0YETbMdRn/v7+yMvLw+3bt1FaWgqxWIxWrVrB399f16HVGRqTQ/ljnSVLlqB169bo0KGD1oMielmI9fQQc+lOla87duqsUmIAyuYKHfvlHF5xfaNK9xraqWWV66/v3N3d4e7ujrFjxyI1NRXW1taIiIjQdVh1itrksGfPHjg4OKBDhw4QiURISkpCUlKSSrnhw4drLUAiUtWl72AUFRbgae5jFOY/haSRCUwam6HLG4N0HVqt03XLR9f1a4va5BASEoLx48ejQ4cOCAkJqXAvXpFIxORAVMtatXNBq3ZVf4TUEFW39QUATwtLFP+v7j0aastLbXKwsbGBmZmZ4jURUUNw+9rvOH1oDx5nZQAAHmdlYMuKhegxaDiT7t/UJoejR49W+Pplx2FwRPXbheMH8WfqPcV7mawUf6bew4W4Q0wOfxO88F5OTg7y8vLQrFkzxQJ8aWlpMDY2hrm5uTZjrHPKh8GVy8zMRGZmJqKiopgciOoB9tloJjg5TJ48GY8fP1bs+SsSiTBp0iSYmppi+/btWguwLuIwuBfDlhfpGvtsNBPcxX7r1i107dpV6Vjnzp2V/oKuj0plspe6fl3gBCSiuk9wy0EsFiMjI0Pp2J9//lnvtwWtzkiHHes3IuX2P0mxtLQUN2/exDcbIpEhsarSvRrqSAd12PIiqvsE/7K3aNECcXFxCA8PR6dOnXDp0iXEx8fD2dlZm/HVSXxe+WI4AYmo7hOcHAIDAxESEqLyj7h8OY2XCZ9X/kPXE4B0XT9RQyU4OQwfPhy3b9/Gpk2bkJ+fj0aNGmHMmDGcAPeS4wQkooapSh0GH374IWbMmIGsrCxYWlrCwMBAW3FRA8YJSER1n9rkcP78eVhbW8Pe3h7nz59XOnfv3j8TSNzc3LQTHTVInIBEVPepTQ5jx47FuHHjEBISgjFjxlS6ttK1a9e0FiA1POzQJ6r71CaHZzf6KX9fURmiqmCHPlHdpzY5XL9+vcLXRPRy4yz3hk/tGMAePXpg5cqVAMoeMf3www+1EhQR1W2c5d7wqU0O2dnZyM7OBgCcO3cOycnJtRIUEdVt/v7+cHR0VOzxLhaL4ejoyFnuDYjax0pSqRTR0dG4desWAODQoUNISEhQKiMSibBx40btRUhEdQ5nuTd8apODp6cnNm/ejPPnz0MkEuHhw4d4+PChUpmKRjAREVH9pjY5zJs3D927d8fNmzexcuVKdOzYEb17966t2IiISEc0zpAeMGAABgwYgHv37qF3794YOnRobcRFREQ6JHj5jM8//1ybcRARUR2iNjk4OTkpZkg7OTlVWIYzpImIGh7BM6Q5E5qI6OWhNjlERkbC2tpa8ZqIiF5cfZhhrjY5dOvWrcLXRERUfeUzzMtlZmYiMzMTUVFRdSY5CN5Ca/ny5ZgyZQpKS0sBACUlJZg2bRqWL1+uteCIiBqi+jDDXHBy2L17N4yMjBQfRl9fH40aNcKePXu0FhwRUUPk7u6OiIgIxWP78hnmdaXVAFQhOeTm5sLY2FjpmEQiQW5ubo0HRUREuiU4OTRr1gxHjhxRLL6XnJyMo0ePKjIfERE1HIInwfXp0webN2+Gp6cnLCwskJ2dDblcjmHDhmkzPiIi0gHBLYdp06ahefPmkMlkyMzMhEwmg729PaZOnarN+IiISAcEtxwsLS2xd+9exMbG4sGDB7CxscHAgQNhZGSkzfiISMuKSkphqC9+aeuniglODkBZB3SzZs0AoMqPky5cuIBPP/0Ud+7cgaOjI0JDQ9G+fXulMsXFxVi2bBn279+PwsJCeHl5Ye7cuTAwMKhSXUQknKG+GJ6LqjfqsCjrKQAgNetpte9xYO7wal1XV+g6uWmrfsHJ4fHjx5gwYQKuXr0KkUiEjh07YtCgQZg2bRpmzJih9trCwkIEBwdDIpFgzpw5iIiIQHBwMGJjYxVDYwFg06ZNiIyMhL+/Pxo1aoSNGzeiefPmmDBhQvU/IRGRFjXU5Cq4z2HlypX4448/FOstNW/eHM7OzoiPj9d4bXx8PDIyMhAQEIDAwED4+voiJSUF586dUypX/n769OmYNWsWgLL5FUREVLsEJ4djx46hY8eOCAgIUBxr3bo17t27p/HalJQUAICVlRUAKIa/3r9/X6mcpaUlAOD06dM4ceKE0rVERFR7BD9WysrKQo8ePaCvr3xJcXFxlSutbIXXSZMm4cSJE/jkk0+gr68PIyMjSCSSKt+fiIhejOCWg52dHc6ePYucnBwAwMWLF3HkyBHY2dkJuhaAYv/p9PR0AIC9vT0KCwsVCaZly5aIjY1FVFQUYmJiYGBggNatW1ftExER0QsT3HLw9vbG8uXLFY95AgMDAQD/+te/NF7r4eEBqVSKrVu3wsTEBDt37oStrS1sbW3h4uKCvn37Yu3atUhISFDMut6/fz9yc3PZGU1EpAOCWw7jxo2Dp6enokNaLpdj4MCBGDt2rMZrJRIJwsPDYWJigsWLF0MqlSI8PBx6eqrV79q1CwsWLMDdu3exYMEC9OvXr2qfiIiojpNl3EHx+W1AweOyAwWPUXx+G2QZd3Qb2DMEtxwMDAywYsUKfPLJJ4pJcFVZV8nNzQ379u1TOZ6YmKh47eTkhKNHjwq+JxFRfVR67xLkuY/+OSCXQZ77CKX3LkHvlZa6C+wZVZoEl56ejn379iEtLQ02Njbw8vJSjEAiIiJhxM07obS0CPLCPKCkANA3gkhiDHHzTroOTUFwcvj1118xadIkFBQUKI6tWbMG69atg5ubm1aCIyJqiPReaVlnWgiVEdznsHjxYuTn50MqlaJz586QSqXIz8/HkiVLtBkfERHpgOCWQ1JSEpydnbF161YYGBigqKgIAQEBuHXrljbjIyIiHRCcHNq0aQMHBwfFIniGhoZwdHTkonhERA2Q4OTg6emJ8PBwtGnTBo6Ojrh58yYOHjyIWbNm4fz584py7H8gIqr/BCeHpUuXQiQSYenSpUrHQ0NDFa9FIhGuXbtWc9ERUZ0ky7iD0jtnVcbpi1t2r/MdrSSM4A5pAEoT4Cr6TyaTaStOIqpDFOP05X//m39mnD41DIJbDtevX9dmHERUj9SHcfr0Yqo0CY6ICKgf4/TpxWh8rJScnIzz588rJr9dvnwZY8aMgY+PD1auXMlHSUREDZDGlsOiRYtw4cIFnD59GoWFhZg+fToyMzMhl8uRkJAAAwMDTJ06tTZiJSKiWqKx5XDr1i106dIFhoaGOHPmDDIyMmBkZIQxY8ZAIpEgJiamNuIkIqJapLHlkJ2dDRsbGwDAhQsXAAAjR47E3LlzkZ2djSNHjmg3QiIiqnUaWw4ikQiPH5eNZb58+TJEIhG6dOkCADAyMtJudEREpBMak0OLFi0QGxuLadOm4fz58xCJROjWrRsA4N69e5BKpVoPkoiIapfG5DB27FiUlJTg6NGjkMvl6N+/P6RSKbKysnDp0iU4OzvXRpxERFSLNPY5DB8+HObm5jh16hSsrKwQFBQEAHj48CGCgoLwxhtvaD1IIiKqXRqTw4EDB9C3b1/07dtX6Xi7du3Qrl07bcVFREQ6pDE5zJo1C/r6+ujZsycGDhyIN998E+bm5rURGxER6YjGPoeTJ09i3rx5kMvlWLBgAXr37o1x48Zhy5YtSE9Pr40YiYiolmlsOVhYWMDPzw9+fn54+vQp4uLi8PPPP+PLL7/EokWL4OzsjAEDBsDLywvW1ta1ETMREWlZlRbeMzExwZAhQzBkyBAUFRXh1KlT+Pnnn7FhwwbF0hpERFT/CU4Ov/76K7p27ap4b2hoiDfffBNvvvkmZDIZcnJytBIgERHVPsGb/QQFBWHQoEFYu3atSl+Dnp4eLC0tazw4IiLSDcHJoVGjRkhOTsZXX32Ffv36YdKkSTh48CCKi4u1GR8REemA4MdKZ86cQVxcHA4cOIDjx4/jxIkTOHnyJMzMzODl5YWxY8eiefPm2oyViIhqieCWg0QiwcCBA7FixQocOXIEffv2hVwux19//YXNmzdjyJAh2L9/vzZjJSKiWiI4OQDAlStXMH/+fAwcOBBxcXEAABsbG4wZMwaNGzdGWFiYVoIkIqLaJfix0vDhw5GYmAi5XA4AcHd3R1BQEPr16wc9PT1IpVKEh4drLVAiIqo9gpPD9evXYWxsDG9vbwQFBcHBwUHpfI8ePVBSUlLjARIRUe0TnBzmzp0LHx8fmJiYVHje1dUVrq6uNRYYERHpjuA+h8WLF2PVqlVKx77//nv4+vrWeFBERKRbgpODXC5X9DeUu3//Pq5evVrjQRERkW5pfKw0Z84cxetTp04p3svlchw7doz7SBMRNUAak8Pu3bshEokgEomQlJSEW7duKZ3v0aOH1oIjIiLd0JgcRowYAaAsSTg4OMDFxQUAIBaLYWNjg1GjRgmq6MKFC/j0009x584dODo6IjQ0FO3bt1cqU1RUhM8++wxHjhxBXl4eWrVqhU8++YQJiIiolmlMDkuWLAEApKSkYPDgwQgMDKxyJYWFhQgODoZEIsGcOXMQERGB4OBgxMbGQiwWK8rt2bMHO3fuRM+ePdG7d2988cUX+M9//oPDhw9XuU4iIqo+wR3SmzZtqlZiAID4+HhkZGQgICAAgYGB8PX1RUpKCs6dO6dUTiaTAQAcHR3Rs2dPGBoawszMrFp1EhFR9altOTg5OWHcuHEICQmBk5NThWVEIhGuXbumtpKUlBQAgJWVFQAodoy7f/++0iOjESNG4OTJk9i4cSM2btwIc3NzRcuFiIhqj9qWw7PDV8tfV/RfVVV2zeXLl3H8+HF4eXlhxYoVKC0txZw5c6pVBxERVZ/alkNkZKTir/zIyMhqV2JnZwcAePjwIQAoNguyt7dHYWEh9PT0YGBgoNgfYtSoUejatSuioqJw+vRpZGdnczMhIqJapDY5dOvWTfHaxMREZXSRUB4eHpBKpdi6dStMTEywc+dO2NrawtbWFi4uLujbty/Wrl0Le3t7AMC3336LxMREXL58Gebm5rCwsKhWvUREVD2CO6R9fX0xcuRIREVFIS8vr0qVSCQShIeHw8TEBIsXL1as4Kqnp1x9eWf177//jmXLlqFVq1YIDw+HSCSqUn1ERPRiBC+8JxaLcfXqVSxYsABLly7F22+/DX9/fzg7Owu63s3NDfv27VM5npiYqHgtkUiwaNEioSEREZGWCG45nDhxAvPmzYOzszOePn2KqKgo+Pn5wcfHR5vxERGRDghODhYWFggKCsKOHTvw008/wd3dHXK5HAkJCdqMj4iIdEDwYyWgbF7Cjz/+iL179+L+/fsAoDTDmWrHmTNn8P333yMrKwu5ublo3LgxLC0tMX78eLi7u+s6PCJqAAQnh9GjR+Py5csAyuYp2NjYwNfXl/s56EBUVBRu3rypeJ+ZmYnMzExERUUxORBRjRCcHC5dugR9fX28+eab8Pf3R+/evTmKSEf8/f2Rl5eH27dvo7S0FGKxGK1atYK/v7+uQyOiBkJwcvjwww/h4+ODV155RZvxkADu7u5wd3fH2LFjkZqaCmtra0REROg6LCJqQAQnh8mTJ2szDiIiqkNqZeE9IiKqX9Qmh+cX3iMiopdDrSy8R0RE9YvghfcePHgABwcHdOjQQXEsMzMT2dnZ2ouOiIh0QvAM6ZCQEMTExCgdW7t2Lby8vGo8qJdJUUnpS10/EdVNVZoh/bzy/Rmo+gz1xfBctKda1xZlPQUApGY9rfY9DswdXq3riKhh05gcykcpiUQiREZGqvQ9cBMeIqKGR2NyKB+lJBKJKhyxFBAQUPNREWkR16Yi0kxjcoiMjIRcLse4ceMwaNAgBAYGAihbcM/W1lYxmomovuDaVESaaUwO5SOWlixZgtatWyuNViKqj7g2FZFmapPDnj17FMNXRSIRkpKSkJSUpFJu+HB2alL9wbWpiDRTmxxCQkIwfvx4dOjQASEhIRWuwioSiZgcapks4w5K75wFCh6XHSh4jOLz2yBu2R16r7TUbXBE1CCoTQ42NjYwMzNTvKa6ofTeJchzH/1zQC6DPPcRSu9dYnIgohqhNjkcPXq0wtekW+LmnVBaWgR5YR5QUgDoG0EkMYa4eSddh0ZEDUS1JsEVFRXh0KFDMDc3R58+fWo6JtJA75WWbCEQkVYJXj5j2rRp6N69O2QyGYKDg/Hxxx9j8uTJ2LBhgzbjIyIiHRCcHBITE9GpUycUFBQgPj4ezZs3h0QiwY4dO7QZHxER6YDg5PDo0SM0a9YMd+7cgVwuR1hYGIYMGcL1lYiIGiDByUEikeDBgwe4cOECRCIRHBwcIBaLKxzeSlRbdL2qrK7rJ9IWwR3Sjo6OiI+PR3x8PBwdHdGoUSOkpqbCyspKm/ERqcVVbYm0Q3DL4aOPPoK1tTWkUilmzZqFoqIipKWlwcPDQ5vxERGRDghuOXTp0gXHjh1TOnbgwIEaD4iIiHSvSvMcUlNTce3aNTx9+lTpOJfPICJqWAQnh6ioKCxcuBClpaodcEwOREQNi+DksGbNGpSUlMDU1FSx3hIRETVMgpPD48eP0atXL3z77bccvkpE1MAJHq1UvoYSEwPVd7KMOyg+v01lyXNZxh3dBkZUhwhuOUilUvz888+YMGECXFxcoK//z6XTp0/XSnBE2sAlz4k0E5wctm7dCpFIhF9++QWnT59WOickOVy4cAGffvop7ty5A0dHR4SGhqJ9+/ZKZUJCQrB7926lY7a2tlwunGoUlzwn0kxwcnBzc6t2JYWFhQgODoZEIsGcOXMQERGB4OBgxMbGQiwWK8qNHj1a8fjq9u3bWL16Ndq1a1fteokqwiXPiTQTnBw2bdpU7Uri4+ORkZGB2bNnIzAwEBkZGfj6669x7tw59OjRQ1HO1dUVrq6uAICFCxcCAEaNGlXteomIqHoEd0gDQF5eHvbs2YOIiAgUFRXhwYMHyM/P13hdSkoKACjWYbK2tgYA3L9/v8Ly+fn52Lt3L1577TX06tWrKiESEVENENxySE9Px+jRo5GWlgYA8PHxwYABAzB+/HjMnj27SpXK5XK15/fv34/c3FxMmTKFo6OIiHRAcMshLCwMDx48gKGhIQCgadOm6NSpE06ePKnxWjs7OwBQ7P2Qnp4OALC3t0dhYSGKi4uVym/btg2Ghobw8fERGh4REdUgwcnh7Nmz6NWrF9555x3FsRYtWiA1NVXjtR4eHpBKpdi6dSt++OEH7Ny5E7a2trC1tYWLi4vSaKdr167hypUrGDRoECwtLav4cYiIqCYITg65ublo2rSp0rH8/HxBj30kEgnCw8NhYmKCxYsXQyqVIjw8HHp6qtVv27YNADuiiYh0SXCfQ4sWLXDy5El06NABALBv3z4cOXIEDg4Ogq53c3PDvn37VI4nJiYqvV+4cKFipBIREemG4JbD6NGj8ejRI8WeDh9//DEKCwv5Fz4RUQMkuOXg6+uL5ORkbNmyBfn5+TAyMkJAQAD8/Py0GR8RvSSKi4uRkpKCgoKCKl9ro1eihYiESUhIQMhb9jqtXxMjIyPY2dnBwMBA8H0FJweRSIRZs2YhODgYhw4dgqmpKTp27Ci4IiIidVJSUtC4cWO0aNGiykPYc/IKtRSVZubGEojTsnVW/+vNLNSel8vlyMzMREpKClq2FL4ygMbkEBERgdOnT2PlypVo0qQJFixYgD17yjZjb9KkCb799ls4OzsLrpCIqCIFBQXVSgyknkgkglQqxaNHjzQXfobGPofY2FhkZWWhSZMmuHv3Lnbv3g25XA65XI6cnBysWbOm2kETET2LiUE7qvO9akwODx48QJs2bQAAp06dAlC2BtK5c+fg5OSEK1euVLlSIiKq2zQmhydPnqBx48YAgCtXrkAkEsHT0xNmZmbo2LEj/vrrL60HSUQvn1KZTHBZc2OJxv9MJZo7Y7/7dh1GjRyBQP+RCHrHD39c+f1FPoKS2e9PAgCkpz3A8cOHFMdvXk/A2pVhaq898GM0jh76CQBw+EAMMjOq9oioOjT2OUilUpw9exYJCQmKlkN5R3RmZqYicRAR1SSxnh5iLtXc7nxDO6nvjL3y2284GR+PyK3bYWhoiJzsbJWlfV7EF2v+BwD482Ea4g7Hom//QQAAx7ZOcGzrpPZaT+9/lhI6cnA/XmvpAOkrr9ZYbBXR2HJwd3fH7du34ePjg4yMDEilUsWy2levXoW9ve6GcBER1ZSMjEcwtzBXrB9nbmGBV5s2RcK1a5gy8V2MDXgHwdOmIOPvjt2p703A6vAV8PX1xb+C/HD198sAgOQ7t/HhlAkInjgGMyYE4kHKPQCA3+A3AQAb132Na1cuI3jiGOzZsRVXLl3AZyEfQSaTYeI7w/EkN1cR0+RAX2RnZeKH7/6H6G1bcOr4UdxKvI6wRQsQPHEMzp8+hWnTpinKnzp1Cu+//36NfB8ak8OHH36I9u3bQy6Xw8TEBKGhoRCJRDh79ixSU1PRtWvXGgmEiJ+yc1QAABOdSURBVEiXuvfoifSH6fD19sKyxaG4+OuvKCkuRtjSJVjyRRgif9gOL+/h+GbNKsU1pSWl2LlzJyZNn4mt338LADi4NxrDRvpj5fpN+HLt95C+qrzs0LjJ09CuQ0esXL8Jw/1GK47r6emhe28PnDkZBwBIvPYHmlpZw8JSqijTq28/tG7TFh/N/Qwr129CV/eeuH37NrKysgAA0dHRGDlyZI18HxofK1lZWWHXrl14/PgxTExMFDu3denSBRcvXoREIqmRQIiIdMnY2Bgbf9iGy5cu4sL5c5gbMhsT3puMpKRbmDH1XwAAmaxU6XFO37feAgA4vN4Gf/696nSb9h0Qtfl7ZDz6Ez09+sLGrrngGPq82R/bNm5Af8+hiD96GH369VdbXiQSwdvbG3v37oWPjw8uXbqEpUuXVvWjV0jwJDgzMzPlC/X1oa8v+HIiojpPLBajS1c3dOnqBofWjtgZtR2tWjlgfeTmCssbGpQ9gtLTE6O0tGyWdt/+g9DGqT3OnzmFTz/5EO9/FALXzsKesLRt3wFpqSn4KycbZ07G4Z0x72q8xsfHB1OnToWhoSEGDx5cY7/LVdoJjoiooUq+ewf3kpMV72/eSETLli2Rk5ONK7/9BgAoKS7G7aRbau/z8EEqrG1sMWzkO+jeywN3nyvfyNgE+fl5FV4rEong3ucNfLsmHPavtYBZkyYqZRoZGytdb2VlhaZNm+Kbb76psUdKQBVaDkREtalUJtM4wqgqSkrVD43Ny8tH2NIlyM3Nhb6+GHb2zTFn3nwMH+mLsKWf48mTJygtLcWowEC0cmhd6X1OHDuCYz8fgL5YHxaWUvgHjVM638KhNcR6epgxMQhvDX4bDq1fVzrf583++HDKu/gg5D8V3v+twW/j6y+XwtBQgi++LhsB5eXlhaysLMGrZAvB5EBEdZK4gv1eKlMTays5tWuHbzduUjlubmGBtRu+Vzn+zbcbFK+bmJtj/fayZYX8AsfCL3CsSvkdB8tWtNbX18eiFcorS3To1EXx2rGtE/YdP6N0PuDdSYrXvd7oh15v9FM6f+HChRpfBJWPlYiI6jEfHx8kJibC29u7Ru/LlgMRUT0WHR2tlfuy5UBERCqYHIiISAWTAxERqWByICIiFUwORFQnFZWUCi4rZMluIwOxxvtoc8nuTz+ZqVhUb++u7Zg69h0sD52Ps6fisWNLpNprK1vuW5s4WomI6iRDfTE8F+2psfsdmDscBcWVJxxtL9n96dIVitc/7YlGaNgqvNK0bFG+7r081F5b2XLf2sSWAxERKl+ye/iQwVj11ZcI8PPBu0EBuH+vbAnu7KwsfPLRTIwcORIz//Uurl0pW2IjPy8PX33+X0x/NxAzJgTiVNxRAMDEd4bjr5wcrAlbivS0VHz6yUzs2bEVhw/EIOKr5X/fMxOL5n2CGRODMGNiEBL+KGu5VLbcd0jwFCQkJCg+w+jRo3H9+vUa+T6YHIiIUPGS3eVMTU3xw45o+L0zCiuWLwMAfPnFUowOGoNdu3ZhzsIlWPXFYgDA9k0bYGJiitXfbcGqDVtUFt17/6NPYCl9BYtWrFFashsA1q36Es6unbBq/WZ8tW4jmrdopXT++eW+BwzxUsxzuHPnDgoLC9G2bdsa+T6YHIiI8M+S3XP+Mx/mFhaYGzIbMXt/BAAMHDzk7/974srvZS2E82fPYPnnS+Dt7Y3Qf89GXl4e8vPycPnCebw94p8F8Ewbm6lWVonfL17AkL93fROLxTAxNVVbvlfft3D8+HEUFxdj165d8PHxUVu+KtjnQET0t+eX7P4pZi8AQCT6p4zo7zcyuRzrIzfDysIMN9KydREujIyM0LNnTxw5cgQHDhyo0dnSbDkQEaHiJbutm9kAAH4+VDZC6OfYQ+jg4gIA6O7eA1HbflCUv33zBgCgY5du2L97l+L4k9zHgmNw7dwVP/1Y9gNfWlqKp0+eKJ2vaLlvPz8/hIaGokOHDmhSwRLf1cWWAxHVSUUlpTgwd3iN3a+guETt+cqW7D4VH4/c3McI9B8JAwND/HdJ2U5rH30cgi8+XwwvLy/kFxahvUsnvP/RJ3hnzLuICF+O98cHQE+sh9HjJqKnx5uCYpw0YybWhH2On3/aBz09PUz78GO0bd9Bcf755b6H+42Gs7MzTE1Na/SREsDkQER1lKG+5nkJ5bS5ZDcABI0dj+n/N1PpmLmFBRYt/QLmxhKlx0qNjI0xc858lXuUL+n9/Ov+nkPR33MoAMDCUop5i75QuVbdct/p6emQy+Xo3bu3po9YJXysRERUTx099BP8/f3xwQcfQK8K+18IwZYDEZEae346qOsQKtVv0BBMGR+olXuz5UBEdYZcLtd1CA1Sdb5XJgciqhOMjIyQmZnJBFHD5HI5MjMzYWRkVKXr+FiJiOoEOzs7pKSk4NGjR1W+Nr9I/UgkbWpkqI8//8rTXFBLSnOMNZYxMjKCnZ1dle7L5EBEdYKBgQFatmxZrWtjLt2p4WiEG+rUEh/W4AKBVVWTw32fVWuPlS5cuAAvLy84OztjxIgRuHr1aoXlbty4gbFjx8LFxQXdu3fHsmXLaitEIiL6W60kh8LCQgQHB+Pp06eYM2cOMjIyEBwcjNJS5eVzCwoK8N577yExMRHBwcEIDg6GsbHmJhMREdWsWnmsFB8fj4yMDMyePRuBgYHIyMjA119/jXPnzqFHjx6KcjExMUhPT0doaCi8vLyq3IFCREQ1o1aSQ0pKCgDAysoKAGBtbQ0AuH//vlJySEpKAgB89913mDdvHiwsLDB//nwMGTJE7f1TU1NrfOp4bdkAoJEO6/fxUb8DVV3H7+/F8Tt8MfX5+0tNTa30nE46pCsbqlZUVAQAePXVV/HBBx8gNDQUc+bMgYeHB0zVLF179uxZrcRJRPSyqpU+h/IhVA8fPgRQthYIANjb26OwsFCxFV95OU9PTwwcOBBdu3ZFQUEB/vzzz9oIk4iI/lYrycHDwwNSqRRbt27FDz/8gJ07d8LW1ha2trZwcXHB9OnTAQBDhgyBoaEhoqOjERUVhdOnT8PKygrNmzevjTCJiOhvtZIcJBIJwsPDYWJigsWLF0MqlSI8PFxloSgrKyuEhYUhKysLixYtwmuvvYZvvvkG+vqcjkFEVJtEcs5VJyKi53BtJSIiUsHkQEREKpgciIhIBZODFo0ZMwZt2rRBmzZtEBISonQuPDxcca5fv34AgJCQELRp0wZjxozRRbg1av/+/Rg1ahRcXV0Vn/N5z34/7dq1g5ubG0aMGIGVK1ciNzdXYx1//fUXvvzyS7z99ttwdXVFp06d4OXlhdWrVyvKlN+/c+fOyMzMVBwfMGAA2rRpg1WrVlWrrLbNnz8fw4YNg5ubG1xdXeHp6YmIiAilJWf4/VVu9erVGDBgADp27AhXV1cMGDAAS5cuRUFBgaJM+b+38u+vS5cu8PLywueff64Ybq9OYWEh1q1bhxEjRqBTp07o2LEjBg8ejMWLFyvmbPXr1w9t2rSBk5MTbty4obh2/PjxKr8LVSlbG5gcaklMTIxiKeKCggJs3bpVxxFpV0JCAoqKiuDk5KSxrIODA0aNGoVOnTrh5s2bWLNmDby9vdX+A01LS8OIESOwdu1apKamwsPDA4MHD4ahoSE2btyoUv7p06dKP3rqVKWstmzfvh0ikQj9+/eHi4sLbt++jRUrVmDlypUqZfn9qbp79y4cHR0xfPhw9OnTBykpKdiwYQO++uorlbJWVlYICAiAu7s7UlJS8N1332HYsGG4fv16pffPzc3FqFGjEBYWhhs3bqBbt24YOnQoLC0tERkZqZSEAEAmk2H58uWCYq9KWW3iGNFaYG5ujpycHGzevBkzZ85EdHQ0srOzFccbolmzZgEAfvzxR1y6dEltWRcXF8yfX7Yhe1JSEvz9/ZGamorQ0NBK/9oMDQ1FamoqGjdujJ07d6JFixaKcxX9oxaJRIiKisK4ceOUylakKmW1Zfv27ejYsaPifWBgIH799VccO3YMM2cqb3TP70/V8z+uU6ZMwbFjx5CcnKxS9rXXXsO8efMAlE3QHT16NFJTU/Hvf/8b0dHRFd5/1apVuHbtGvT19bFp0yZ07txZce727dsq68KJRCLExcXh7Nmz6N69u9rYq1JWm9hyqAUeHh6wtbXFtm3bkJeXh40bN8LY2BjDhg3TdWh1joODA/z9/QEAx44dQ2FhoUqZwsJCxMXFAQD8/PxUfoDatm2rco2XlxdKSkoQFhamMYaqlNWWZxMDAMUqAs2aNVN7Hb+/f5w6dQqhoaGYPn064uLi0KRJE0ycOFHtNVZWVooyV69exf379yssFxsbCwB46623lBIDALRq1QqGhoZKx4YMGQJ9fX0sW7ZM4053VSmrTUwOtUAsFmPMmDHIycnBzJkzcffuXYwcORJmZma6Dq1OKl9Gpbi4uMKWVU5OjuLH0t7eXtA9e/TogT59+iA2NlZjS6YqZWvDqlWr8Ntvv8HMzAyzZ8/WWJ7fX5nffvsNmzZtws8//wyZTIYuXbooFv9U59kd0yrblS4jIwOA8O+vZcuW8PX1xR9//IH9+/fXWFltYnKoJX5+fjA1NcXx48chFosxbtw4XYdUZ5Wv4mtgYABzc3OV8+bm5jAwMACASv+yq8isWbOgp6cnaAOpqpTVlsLCQnz00UdYvXo1rK2tsWXLFrRu3Vrjdfz+ykybNg0JCQk4fPgwevXqhaNHj2LcuHGQyWRqr3v2O3n11VcrLPPKK6+olNVk+vTpMDY2xooVKxTJuSbKaguTQy0xNTVVNPf79+8v+C+Ol01SUhKioqIAAH379oVEIlEpI5FI4OHhAQDYsWMH7t69q3KPirRt2xbDhg3DxYsXFT+glalKWW149OgRxo4di5iYGHTq1Am7du3C66+/rvE6fn9lLabyx2l6enqwt7dH165dAZQtUZ2dnV3ptenp6diwYQMAoF27dpX+Ox0wYAAA4MiRI7h48aLSuXv37ilGKz3r1VdfxbvvvouUlBSVa16krLawQ7oWTZkyBX369IGDg4PaclevXlUkknKrV69G06ZNtRlejTp8+DAOHz6s9JdV+VC8jz/+GJaWlorjv//+OxYuXIjU1FScOnUKxcXFsLW1VXQSVmTu3Lm4du0a0tLSFCNSmjRpgqSkJNy6dQvnz5+v8LoPPvgABw4cqPBZ/IuUrWl+fn5IS0uDRCJB27ZtsXbtWgBAkyZNFAtVluP3pyw9PR3Dhg1Dt27dYG1tjUePHin6WNq3bw+pVKpUPjk5GaGhoUhLS8Mvv/yCvLw8mJubY8mSJZXWMWPGDJw9exaJiYkYM2YM+vTpg6ZNm+LevXs4d+4czpw5o9LvAAATJ07Etm3blIb6VqYqZbWByaEWNWnSBD179tRY7unTp/jtt9+UjlX0l0hdlpCQgN27dysdK38/ffp0peSQlJSEO3fuwNTUFI6OjujXrx/Gjx+Pxo0bV3p/W1tb7N69G+vXr8fRo0cRFxcHsVgMOzs7tY/smjVrhqCgIKxfv17jZ6hK2ZqWlpYGoOzR0rPDnm1tbVWSA78/ZaampnBzc8PVq1dx8uRJAGWft0+fPpg2bZpK+fT0dGzZsgXGxsawtbVF79698e6776rtnzAzM8P27duxceNGHDx4EGfOnIFcLkezZs0QGBhY6S6WJiYmeP/997Fw4UKNn6MqZbWBC+8REZEK9jkQEZEKJgciIlLB5EBERCqYHIiISAWTAxERqWByICIiFUwORLWsfA+Bs2fPCr6mfO+G2twTgV5uTA5Ef6vq5kxEDRmTA1EFXrbNmYiex+RA9Bxzc3MUFxdj8+bNAKC0OdOznjx5grCwMAwePBiurq7o378/FixYoLQWTlpaGiZNmoROnTphwIABOHDggEp9MpkMO3bsUGw36eHhgdmzZ+Phw4fa/aBEajA5ED1HyOZMMpkM7733HtatW4cnT55g6NCh0NPTw7Zt2zBq1Cjk5+dDJpNhypQpiI+Ph5mZGbp164alS5eq1Pfll19i3rx5ePToEQYOHIjWrVtj7969GDVqFJ48eVKbH51IgcmB6DlCNme6ePGiYiObiIgILFq0CJs3b4ZYLMa9e/cQGxuL33//XbHl5jfffINFixap7K1cVFSkaKE4OzvDzMwMDg4OkEgkSEtLU+w4RlTbuCorUQX8/PywevVqpc2Z9uzZozj/4MEDxevyfRaaNm0KCwsLZGRkIC0tTbGhDgDFJj3P78mQnZ2N/Px8AGXbej6vfHVWotrGlgNRBTRtzmRjY6N4ffPmTQBlG/SUbyTTrFkzpf2eb926BQC4ceOG0n0sLCwUyzsvW7YMiYmJiv/i4+M17nlMpC1sORBVQt3mTJ07d4arqyt+++03TJ06FR4eHjh//jxKS0thZ2eHAQMGwMjICK+//jpu3LiBqVOnonfv3or9BcoZGhoiICAAGzZswPz58xEXF4dGjRohOTkZFy9eRGxsrNKexkS1hS0HokqUb85U0aYvenp6WL9+Pd577z00atQIe/fuRXFxMd555x1s27YNxsbG0NPTQ0REBPr06YPHjx/jzJkz+Oijj1TuNXv2bPz3v/+Fg4MD4uLicODAAfz1118YN24cLCwsauOjEqngZj9ERKSCLQciIlLB5EBERCqYHIiISAWTAxERqWByICIiFUwORESkgsmBiIhUMDkQEZGK/wd69utnqqoKZQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b_Llza1Dfqdc",
        "outputId": "0fcd96ca-39e2-4e4d-cd8a-59c92eb8de25"
      },
      "source": [
        "data = 'PD'\n",
        "print(\"-----\", data, \"----\")\n",
        "hala_pd_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hala_pd_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hala_pd_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_HALA_CSP_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hala_pd_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HALA_CSP_'+str(data)+'.mat', 'wtd_f1_list')\n",
        "\n",
        "data = 'NC'\n",
        "hala_nc_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_HALA_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hala_nc_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_HALA_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hala_nc_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/CSP/CNN_1D_results_HALA_CSP_'+str(data)+'.mat', 'f1_weightedNet')\n",
        "hala_nc_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_HALA_CSP_'+str(data)+'.mat', 'wtd_f1_list')\n",
        "\n",
        "data1 = {'F-Score': hala_nc_ml[0], 'Data': len(hala_nc_ml[0])*['NC'], 'Model': len(hala_nc_ml[0])*['ML']}\n",
        "df1 = pd.DataFrame(data1)\n",
        "data2 = {'F-Score': hala_pd_ml[0], 'Data': len(hala_pd_ml[0])*['PD'], 'Model': len(hala_pd_ml[0])*['ML']}\n",
        "df2 = pd.DataFrame(data2)\n",
        "data3 = {'F-Score': hala_nc_1d[0], 'Data': len(hala_nc_1d[0])*['NC'], 'Model': len(hala_nc_1d[0])*['1D CNN']}\n",
        "df3 = pd.DataFrame(data3)\n",
        "data4 = {'F-Score': hala_pd_1d[0], 'Data': len(hala_pd_1d[0])*['PD'], 'Model': len(hala_pd_1d[0])*['1D CNN']}\n",
        "df4 = pd.DataFrame(data4)\n",
        "data5 = {'F-Score': hala_nc_2d[0], 'Data': len(hala_nc_2d[0])*['NC'], 'Model': len(hala_nc_2d[0])*['2D CNN']}\n",
        "df5 = pd.DataFrame(data5)\n",
        "data6 = {'F-Score': hala_pd_2d[0], 'Data': len(hala_pd_2d[0])*['PD'], 'Model': len(hala_pd_2d[0])*['2D CNN']}\n",
        "df6 = pd.DataFrame(data6)\n",
        "data7 = {'F-Score': hala_nc_3d[0], 'Data': len(hala_nc_3d[0])*['NC'], 'Model': len(hala_nc_3d[0])*['3D CNN']}\n",
        "df7 = pd.DataFrame(data7)\n",
        "data8 = {'F-Score': hala_pd_3d[0], 'Data': len(hala_pd_3d[0])*['PD'], 'Model': len(hala_pd_3d[0])*['3D CNN']}\n",
        "df8 = pd.DataFrame(data8)\n",
        "\n",
        "arousal_fscore_df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], axis=0).reset_index()\n",
        "print(arousal_fscore_df)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- PD ----\n",
            "    index   F-Score Data   Model\n",
            "0       0  0.930163   NC      ML\n",
            "1       1  0.949764   NC      ML\n",
            "2       2  0.930130   NC      ML\n",
            "3       3  0.943360   NC      ML\n",
            "4       4  0.951068   NC      ML\n",
            "..    ...       ...  ...     ...\n",
            "75      5  0.977965   PD  3D CNN\n",
            "76      6  0.980991   PD  3D CNN\n",
            "77      7  0.980991   PD  3D CNN\n",
            "78      8  0.991870   PD  3D CNN\n",
            "79      9  0.982039   PD  3D CNN\n",
            "\n",
            "[80 rows x 4 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "id": "Zor5HWzMgVl6",
        "outputId": "c7574c69-5c30-41cb-ab2a-ab77ecfaa27d"
      },
      "source": [
        "x = \"Model\"\n",
        "y = \"F-Score\"\n",
        "hue = \"Data\"\n",
        "ax = sns.barplot(data=arousal_fscore_df, x=x, y=y, hue=hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.8)\n",
        "ax.legend(loc='upper left', bbox_to_anchor=(0,1))\n",
        "ax.set_xlabel('Model',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('F-Score',fontsize = 13, weight='bold')\n",
        "ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "ax.set_yticklabels([\"0.80\", \"0.85\", \"0.90\", \"0.95\", \"1.00\"], size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=arousal_fscore_df, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"PD\"), (\"ML\", \"NC\")),\n",
        "                                 ((\"1D CNN\", \"PD\"), (\"1D CNN\", \"NC\")),\n",
        "                                 ((\"2D CNN\", \"PD\"), (\"2D CNN\", \"NC\")),\n",
        "                               ((\"3D CNN\", \"PD\"), (\"3D CNN\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "plt.savefig('f-score_arousal.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "1D CNN_NC v.s. 1D CNN_PD: t-test independent samples with Bonferroni correction, P_val=4.918e-04 stat=4.871e+00\n",
            "ML_NC v.s. ML_PD: t-test independent samples with Bonferroni correction, P_val=5.793e-01 stat=1.524e+00\n",
            "2D CNN_NC v.s. 2D CNN_PD: t-test independent samples with Bonferroni correction, P_val=2.404e-02 stat=-3.113e+00\n",
            "3D CNN_NC v.s. 3D CNN_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-4.937e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEJCAYAAABohnsfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxVdf7H8RdcNsUdExSoFJRMBsUtpGLMpXFSE9xzK3ftoVijVkyY/XLpV9PoA7VS0pkx7Sc/txzBNOphSJrjloaZW5rLNTRATdwA4f7+UO4vAvWAcO9F3s/Hw0fcc773nM+5Ke97zvd7vsfJYrFYEBERMcDZ3gWIiEjlodAQERHDFBoiImKYQkNERAxTaIiIiGEu9i6goj322GP4+vrauwwRkUrlzJkz7Nixo9jy+z40fH19Wbt2rb3LEBGpVHr37l3icl2eEhERwxQaIiJimEJDREQMu+/7NEqSl5eH2Wzm+vXr9i6lzDw8PPDz88PV1dXepYhIFVIlQ8NsNlOzZk0efvhhnJyc7F1OqVksFrKysjCbzTRu3Nje5YhIFVIlL09dv34dLy+vShkYAE5OTnh5eVXqMyURqZyqZGgAlTYwClX2+kWkcqqyoSEiIqWn0ADyCwpsvr2goCD++7//2/p6yZIlzJ8/3/p63bp19OjRg549exIZGcmSJUvKtUYRkbKokh3hv2dydiZp70/ltr0eoXfvnHZzcyM5OZkxY8ZQr169Iuu2bNnC0qVLWbJkCd7e3uTm5rJu3bpyq09EpKx0pmEnLi4uDBgwgKVLlxZbFx8fzyuvvIK3tzdwM2D69+9v6xJFRIpRaNjR4MGDSUxMJDs7u8jyo0ePEhwcbKeqRERuT6FhRzVq1KBXr158/PHH9i5FRMQQhYadPf/886xZs4Zr165ZlwUGBvL999/bsSoRkZIpNOysTp06dOvWjdWrV1uXjR07lr/97W9kZGQAkJuby6pVq+xVooiIlUZPcXOIrJERT6XZnsnZeB6PGDGCTz75xPr6j3/8I5mZmQwfPhyLxYKTkxN9+vQpt/pERMpKoQGl+gVfXtvbu3ev9ef69evz3XffFVnfp08fBYWIOBxdnhIREcMUGiIiYphCQ0REDFNoiIiIYQoNERExTKEhIiKGacgtkHsjHzcXk02317x5c5o1a0Z+fj5NmjThnXfeoVq1atblN27cwGQyERkZyQsvvIBzOQ8LFhEpC4UG4OZi4s+zym/q8Y2vR961jYeHB//+978BmDx5MgkJCQwfPrzI8qysLCZPnszly5eJjo4ut/pE7qTwhtL58+czceJE62sR0OUph9C2bVtOnjxZbLmXlxczZszgk08+wWKx2KEyqYrWr1/P4sWLycnJ4aOPPmL9+vX2LkkciELDzm7cuEFqairNmjUrcb2/vz/5+flkZWXZuDKpqnr16oWPjw9LliyhUaNG9OrVy94liQNRaNjJ9evX6dWrF3369KFRo0b07dvX3iWJAJCYmMjZs2cZOXIkP//8M4mJifYuSRyI+jTs5Ld9F3dy+vRpTCYTXl5eNqhKBHr06GHt0xg9erQujRpkNpsZPXo0bdq0Ye/evXh7e/PBBx+wcuVKEhISMJlMBAYGMnfuXHuXek8UGg7s/PnzTJ8+ncGDB6sjUmym8O/axIkTi7yWuzt58iRz5sxh5syZTJo0ic8//5z4+Hg2b96Mm5sbly5dsneJ90yhwc0hskZGPJVme2Udwlt42apwyG2vXr0YPnx4udUmIhXHz8+P5s2bA9CiRQvOnDlDUFAQU6ZMoXPnznTp0sXOFd47hQaU6z0aRrf326nRf+vgwYPlWouI2I6bm5v1Z5PJRE5ODvHx8ezatYuvvvqKhQsXkpiYiItL5f3Vq45wEZEKUlBQQHp6OmFhYUyZMoXs7GyuXr1q77LuSeWNOxERB1dQUMDUqVO5fPkyFouFYcOGUatWLXuXdU+qbGhU9rtcNaJF7mTt2rVs2rTJrjV069aN3r1727UGW/Lz8yMpKcn6euTIkXaspuJUydDw8PAgKysLLy+vShkcFouFrKwsPDw87F2KOKhNmzZx5MiR2940WtGOHDkCUGlDwxFCFxwzeKtkaPj5+WE2m8nIyLB3KWXm4eGBn5+fvcsQB9asWTPi4+Ptsu8xY8bYZb/lxd6hC44bvFUyNFxdXWncuLG9yxARB2bP0AXHDV6NnhIREcOq5JmGyP3u2WefrdL7l4qj0BC5D/Xo0aNK7/9eOULoOUINJVFoiIj8jiOEniPUUBKb9GnMnDmT8PBwgoKCGDt27G3b7dmzh549exIcHExUVBQHDhywrvvyyy/p2rUrf/jDHxg6dCinT5+2RekiIvIbNusIf+aZZ+64Picnh+joaK5cuUJMTAyZmZlER0eTn59PRkYGL7/8MjVq1OCVV17h+++/57XXXrNR5SIiUsgmoREbG8sLL7xwxzapqalkZmYyaNAgBg8eTN++fTGbzezcuZMNGzaQm5vLmDFjGDp0KF27dmX37t2cOnXKFuWLiMgtDjPk1mw2A+Dt7Q2Aj48PcPMhRHdaJ1Vb4XQq8+fPv+3r27URkdJz2I7wO/3D1j96KbR+/XoyMjLIycnho48+okGDBsVeA8WW6bnXImVj1zON3NxccnNzAaxTYpw9exaAc+fOAeDv73/HdVK19erVCx8fH5YsWUKjRo1KfF3SMhEpG5ucaaSkpFjnUUlPT2fVqlW0a9eOESNGcOHCBfbu3UtERAReXl6sWLECT09PVq9eja+vL+3btycgIID33nuPxYsXk5WVxRdffEGbNm148MEHbVG+OLDExETOnTvHyJEj+fnnn0t8DRRb1rNnTztXLlJJWWxgyJAhlmbNmhX5s2bNGstTTz1ladWqlbXdzp07LT169LC0aNHC0qtXL0taWpp13eeff27p3LmzpUWLFpZBgwZZTp48aWjfUVFR5X484jgKCgosFovFMm/evNu+vl0bEbm92/3udLJY7u8Ogt69e7N27Vp7lyEiUqnc7nenw4yeEhERx6fQEBERwxQaIiJimEJDREQMc9ib+6Rqc4RnNDvi85lF7E1nGuKQCp/RbC9Hjhyxe2iJOCKdaYjDsuczmh31+cwi9qbQcBBms5nRo0fTpk0b9u7di7e3Nx988AErV64kISEBk8lEYGAgc+fOtXepIlKFKTQcyMmTJ5kzZw4zZ85k0qRJfP7558THx7N582bc3Ny4dOmSvUu0GXs/6tLe+6+s/vOf//Cvf/2L8+fPk52dTc2aNalXrx4vvPACYWFh9i5PyoFCw4H4+fnRvHlzAFq0aMGZM2cICgpiypQpdO7cmS5duti5Qtux96Mu7b3/ymrlypUcPXrU+jorK4usrCxWrlyp0LhPKDQciJubm/Vnk8lETk4O8fHx7Nq1i6+++oqFCxeSmJiIi4v+t4lj6t+/P1evXuX48ePk5+djMplo0qQJ/fv3t3dplYajn61p9JQDKygoID09nbCwMKZMmUJ2djZXr161d1kitxUWFsbChQutD0rz8fFh4cKFDvHLrrIoPFvLysoiNzeXrKwsjh49ysqVK+1dGqAzDYdWUFDA1KlTuXz5MhaLhWHDhlGrVi17lyUiFcjRz9YUGg7Cz8+PpKQk6+uRI0fasRoRsZewsDDCwsIYNmwYZ86csZ6tOQqFRgXRHc0icj9Sn0YF0R3NUtnl3siv0vuXkulMowLpjmapzNxcTPx51royvTf3/BUAzpy/UuZtbHw9skzvuxeOPnLJEehMQ0TkFkcfueQIFBoiIrf079+fpk2bYjKZgJv3SzVt2rRMI5cc4fJaRdSgy1MVxN7TUNh7/yKVUXmOXLqXy3vguJf4FBoVxN7TUNh7/1I1FWT+RP5PO+D6rXnSrl8ib1cCpsaP4Vy/sX2Lk3Khy1MiUm7yT+3Fkp0BloKbCywFWLIzyD+1176FSbnRmYaIlBvTg6Hk5+diybkKN66DiwdO7tUxPRhq81ryCwowOet7cXlTaIhIuXGu39hhLkOZnJ1J2vtTmd57JeeG9b9l3UaPUMf4HMqbYlhExIEUZP5E3q6EYv1CBZllC6/yptAQEXEgjt4vpMtTIiIOxJH6hUqi0BARueX4D2ls/3wdl85nAnDpfCafzH2LDn+KpMmjITapwZH6hUqiy1MiIrfsSdnEL2dOUVBw807qgoJ8fjlzij1bPrdzZY5DZxoiIre06diN3JzrXMm+RM61K7hX88SzZi3a/PFP9i7NYSg0RERuafJoiM0uQ1VWCg1xKJqaWsSxqU9DHIqmphZxbKUKjfT0dBYsWEBsbCzZ2dns2rWLzMxMQ+/ds2cPPXv2JDg4mKioKA4cOFCsTV5eHrNmzSI8PJw2bdrw5ptvkpeXB8COHTsICgoq8udf//pXacqXSqA8p6YWkfJn+PLU8ePHGThwINnZ2QBMnTqVkSNH0q9fP6ZNm3bH9+bk5BAdHY27uzsxMTEsXLiQ6OhokpOTrb8cAJYtW8bHH39M//79qVatGkuXLuXBBx9kxIgR1jYvvvgigYGBADz66KOlOlhxfOU5NbWIlD/DZxrvvfcely5dol69egDUrl2bNm3asH379ru+NzU1lczMTAYNGsTgwYPp27cvZrOZnTt3FmlX+HrChAlMmTIFgE8//bRIm7Zt29KlSxe6d+9O48aOO5ZZbk4YV5X3L3I/Mnym8d133/H000/j4+PDsmXLAPD392ffvn13fa/ZbAbA29sbAB8fHwBOnz5Nhw4drO0KA2n79u3UrFmzyHsLjRw5EoCQkBDeeecdBYcDK8uEcYU3V2WmnwUgPf0szw0bXqabq+7XCeNE7MlwaOTl5eHq6lpk2fnz53Fzcyv1Ti0WS4nLR48ezddff82rr76Ki4sLHh4euLu7A1C/fn2mTp1KQEAA+/btY+HChbz55pssXbq01PsXx1V4c1Wh395cpaGQd6fRZ1LRDIdGkyZN2Lp1K0FBQQAsXLiQlJQUQkLu/g/Zz88PgLNnb357PHfuHHDzTCUnJwdnZ2dcXV1p3LgxycnJHDlyhFq1atGvXz9r/0VAQAABAQEAPPXUUyQkJHDs2LFSHKpUBrq56t4Ujj4rlJWVRVZWFitXrlRoSLkwHBqjRo1iwoQJ7Nq1C4C4uDgAhg8fftf3RkRE4OXlxYoVK/D09GT16tX4+vri6+tLSEgIHTt2ZNGiRRw8eJDNmzfj4+PDhg0byM7OtnaCL1iwgF9//ZVHHnmE/fv3c/HiRTp37lyWY65Q+qZ3b3Rz1b3p378/V69e5fjx4+Tn52MymWjSpIlGn0m5MdwR3qVLF6ZPn46/vz8mkwk/Pz9iY2Pp2rXrXd/r7u5OXFwcnp6ezJ49Gy8vL+Li4nAu4alaa9asYfr06Zw4cYLp06fTqVMnAAIDA9m5cyczZsxg48aNdO/enbfeeqsUh2obus9A7CksLIyFCxda+w0LR5/pC4uUF0NnGnl5eWzYsAE/Pz+Sk5PLtKN27dqRmJhYbPnhw4etPzdv3pzNmzeX+P5u3brRrVu3Mu3blvRNT0TuZ4bONFxdXZk2bRpffPFFRdfjUMoyZLM8v+lpyKiIOBrDfRqtWrUqNvz1fqdnDIuIFGU4NFq3bs1HH33E5MmTad++vXUoLEBkZGSFFFcZ2fshLuqIF5GKZDg0Fi1ahJOTE5999hmfffaZdbmTk5NC4zfsfZ+BhlzeP/ILCjCVMFhExJ4Mh0ajRo0qso77hr3vM1BH/P1Dl0fFERkOjduNapKi7H2fgSb8E5GKVKqHMOXk5JCcnEx6ejqNGjWiS5cueHh4VFRtVV7ujXzcXEx3b3if7l9EHI/h0Dh//jzPPfccp079//V6f39/VqxYgZeXV4UUV9W5uZj486x1ZXpv7vkrAJw5f6XM29j4uvqqRKQow71s8+bN4+TJkzg7O+Pl5YWzszOnT59m/vz5FVmflFJB5k/k7UqA65duLrh+ibxdCRRklu26tojIbxkOjW3btvHQQw+RkpLC1q1bSUlJ4cEHH2Tr1q0VWZ+UUv6pvViyM8By68ZASwGW7AzyT+21b2FiE8d/SOOTuW8VG/J9/Ic0O1cm9wvDoZGRkUH79u154IEHAHjggQdo37694ce9im2YHgzFqeYD4OYJziZw88Sp5gOYHgy1d2liA4VDvgsK8oGiQ75FyoPhPo2GDRuyZcsWDh48SGBgID/++COpqanW6TLEMTjXb4xzfQ2VrKrsPeRb7n+GQ6NLly589NFH9O7du8jyUaNGlXtRIvZS2e+ot/eQb7n/Gb48NW7cOFq1aoXFYrH+CQkJYezYsRVZn4hNaWp7kTszfKbh6enJihUr2LNnD+np6TRs2JA2bdrg5ORUkfWJ2JTuqBe5M8OhcezYMc6fP0/r1q1p27YtN27cYPfu3dSrV8/6GFaRyk531IvcmeHQ+Mtf/oKzszOffvrpzTe6uPDOO++Qn59vXSbiSOx9R7u99y9SEQyHxsmTJ3n22WeLLHv00UdLfBqfiCPQHfUi5c9wR3i1atX46aeidxX/9NNPmntKRKQKMXymERAQwO7du3nllVcIDQ1l79697N69m7Zt21ZkfSIi4kAMh8aIESPYvXs3iYmJRS5JjRw5skIKE7GHgsyfyP9pR7G5u0yNH9NNkyKU4vJUp06d+K//+i/8/f0xmUz4+/vz5ptv0rFjxwosT8S2NHeXyJ2V6nkaAwYMYMCAARVVi4jdmR4MJT8/F0vOVbhxHVw8cHKvrrm7RG4pVWgA5OXlkZSUxJUrV+jQoYPu0ZD7iubuErmzu4bGzJkz2bJlC6tXr6Z27dqMGzeOb775BgBXV1cWLVpEhw4dKrxQERGxv7v2aezYsYPatWtTu3ZtDh06xLZt27BYLNSoUYPc3FzdLSsiUoXcNTTOnTtHYGAgcHMGUIDHH3+cXbt20bZtW44ePVqxFYqIiMO4a2hcu3YNd3d3APbv34+TkxOdO3cGoGnTply+fLliKxQREYdx19B44IEHSE1N5csvv7Q+2jU09OZIknPnzlGrVq2KrVBERBzGXUPjqaeeIj09nYkTJ/Lrr7/i7+9P8+bNKSgoIC0tjcaNNdJERKSquOvoqZdeeomMjAy++eYbvL29mTVrFnCzg/zGjRs88cQTFV6kiIg4hruGRs2aNZk3b16x5R06dLB2jIuISNVgeBqRQvv372fdurJNFS0iIpVbqUMjMTGRmJiYiqhFREQcXKlDo6z27NlDz549CQ4OJioqigMHDhRrk5eXx6xZswgPD6dNmza8+eab5OXlWdcvWLCAsLAwQkNDee2118jJybFV+SIigo1CIycnh+joaK5cuUJMTAyZmZlER0eTn59fpN2yZcv4+OOP6dy5M3369GHFihUsW7YMgC+++IL58+fToUMHhg4dyqeffqq70UVEbKzUoRESEkJkZOkeY5mamkpmZiaDBg1i8ODB9O3bF7PZzM6dO4u0K3w9YcIEpkyZAmB9/vjatWsBmDZtGn/5y19o2LChdZmIiNjGXUOjc+fOfPjhhwAkJSWxbds23n777VLtxGw2A+Dt7Q2Aj48PAKdPny7Srl69egBs376dr7/+ush7zWYzrq6u1jY+Pj6cO3eO3NzcUtUiIiJld9fQOHPmDBcvXgQgLS2tXEZOWSyWEpePHj2aBg0a8OqrrxIdHY2Hh4d1ChOj2xARkYpjkz4NPz8/AM6ePQvcnH4EwN/fn5ycHGtnd+PGjUlOTmblypUkJSXh6upqnSzRz8+PvLw8srKyrNvw9vbGzc3NFocgIiKU4SFMZREREYGXlxcrVqzA09OT1atX4+vri6+vLyEhIXTs2JFFixZx8OBBNm/ejI+PDxs2bCA7O5sRI0YAEBUVxebNm5k1axZ+fn6kp6czfvx4W5QvIiK3GAqNbdu2ERMTQ1paGkCR+zScnJyYPXv2Hd/v7u5OXFwcb731FrNnzyYwMJAZM2bg7Fz8RGfNmjX88ssvNGjQgOnTp9OpUycAnn76aV588UX+53/+h9zcXHr16sW4ceMMH6iIiNw7Q6Fx7Ngxjh07Zu1HKOzXsFgshkIDoF27diQmJhZbfvjwYevPzZs3Z/PmzbfdxqRJk5g0aZKRkkVEpALcNTTatWtnizpERKQSuGtoFN5cJyIiYrNpREREpPJTaIiIiGEKDRERMUyhISIihik0RETEMIWGiIgYptAQERHDFBoiImKYQkNERAxTaIiIiGEKDRERMUyhISIihik0RETEMIWGiIgYptAQERHDFBoiImKYQkNERAxTaIiIiGEKDRERMUyhISIihik0RETEMIWGiIgYptAQERHDFBoiImKYQkNERAxTaIiIiGEKDRERMUyhISIihik0RETEMIWGiIgYptAQERHDFBoiImKYzUJjz5499OzZk+DgYKKiojhw4ECxNrm5ubz++uuEhYUREhJCZGQk27dvB8BsNhMUFFTkz6xZs2xVvoiIYKPQyMnJITo6mitXrhATE0NmZibR0dHk5+cXabdu3TpWr15N8+bNmTRpEocOHWLatGlF2gwcOJA5c+YwZ84coqKibFG+iIjc4mKLnaSmppKZmcnUqVMZPHgwmZmZfPDBB+zcuZMOHTpY2xUUFADQtGlTwsPDcXNzo1atWkW2FRwcTKdOnahWrZotShcRkd+wyZmG2WwGwNvbGwAfHx8ATp8+XaRdVFQUXbt2ZenSpURGRlKtWjXefvvtIm2mTZtGq1at6N69O/v27bNB9SIiUsguHeEWi6XE5fv27SMlJYWePXsyd+5c8vPziYmJwWKxUL16dSZOnMj777/Pq6++yokTJ5gyZYqNKxcRqdpscnnKz88PgLNnzwJw7tw5APz9/cnJycHZ2RlXV1c2bdpEXl4eAwcOpG3btqxcuZLt27dz4cIF6tWrx4QJE6zbTEpK4sCBA+Tk5ODu7m6LwxARqfJsEhoRERF4eXmxYsUKPD09Wb16Nb6+vvj6+hISEkLHjh1ZtGgR/v7+ACxevJjDhw+zb98+6tSpQ926dVm5ciX79+8nJCQEs9nMwYMHeeSRRxQYIiI2ZJPQcHd3Jy4ujrfeeovZs2cTGBjIjBkzcHYuenVs8ODBHDt2jK+++ort27cTEBDAK6+8gpOTEw8//DCrV68mKSkJFxcXnnzySf7617/aonwREbnFJqEB0K5dOxITE4stP3z4sPVnd3f329570b59e1auXFlh9YmIyN3pjnARETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIoYpNERExDCFhoiIGKbQEBERwxQaIiJimEJDREQMU2iIiIhhCg0RETFMoSEiIobZLDT27NlDz549CQ4OJioqigMHDhRrk5uby+uvv05YWBghISFERkayfft26/qEhAQiIiIICQlh/PjxXLhwwVbli4gINgqNnJwcoqOjuXLlCjExMWRmZhIdHU1+fn6RduvWrWP16tU0b96cSZMmcejQIaZNmwbADz/8wPTp0wkICGDixIls2bKFt99+2xbli4jILTYJjdTUVDIzMxk0aBCDBw+mb9++mM1mdu7cWaRdQUEBAE2bNiU8PBw3Nzdq1aoFwNq1awF4+eWXGT16NKGhoWzYsIGcnBxbHIKIiAAuttiJ2WwGwNvbGwAfHx8ATp8+TYcOHaztoqKi2Lp1K0uXLmXp0qXUqVPHejZR0jZu3LhBeno6Dz/88G33febMGXr37l3ux2QL/wCq2XH/vXt/bMe93zt9fvdGn9+9sffnB/f2GZ45c6bE5TYJjd+zWCwlLt+3bx8pKSn07NmTTp068cYbbxATE8OaNWsMb+P3duzYcU+1iojI/7PJ5Sk/Pz8Azp49C8C5c+cA8Pf3Jycnh7y8PAA2bdpEXl4eAwcO5JlnniE4OJgDBw5w4cIF6zYK33vu3DlcXFxo2LChLQ5BRESw0ZlGREQEXl5erFixAk9PT1avXo2vry++vr6EhITQsWNHFi1ahL+/PwCLFy/m8OHD7Nu3jzp16lC3bl0iIyNZtmwZc+fOJTw8nL1799K9e3fc3d1tcQgiIoKNzjTc3d2Ji4vD09OT2bNn4+XlRVxcHM7ORXdf2EmelpbGu+++S5MmTYiLi8PJyYng4GDeeOMNfvzxR+bNm0dERAQxMTG2KF9ERG5xshjtHBARkSpPd4SLiIhhCg0RETFMoWFnZrOZoKAggoKCeP/9963L//rXv1qX79ixg6CgIN566y07VlrxZs6cSXh4OEFBQYwdO7bIusLPIjg4mMcff5wXX3yRgwcP3nZbWVlZxMTEEB4eTkhICD179iQ5ORmATp06ERQUxPz584H//39QuM/58+cTFBRE165drbMWDB06lKCgIM6fP18Rh37PTpw4wdChQ3nssccIDQ1l+PDhnDp1CsD69ycoKIiWLVvSuXNn3njjjTsey/fff8/IkSNp3bo1rVu3ZsiQIRw7dqzI39fC4exr164lKCiIJUuWAP//Wb322mvW7QUFBdGjR48K/ATuXb9+/QgNDaVly5b07t2bXbt2Afr8fk+h4UA+/fRTLBYLV65cYePGjfYuxy6eeeaZ267z8fFh5syZ/PnPf2bLli0MGjSIH3/8sVi769evM2TIENauXctjjz3GtGnT6NChQ7H5zpYvX86VK1duu79Tp07x2Weflf1gbOjcuXNYLBYmTpxInz59+Oabb3j99deLtHniiSeIjY2lWbNm/O///i9Dhw7l+vXrxbZ19OhRhgwZwo4dO3juued49dVXqV+/PidOnCjSbtGiRXesKSkpiZ9//vmej81WQkNDiY2N5cUXX+TQoUPExsYWWa/P7ya73Nwnxfn7+3P69Gl27NiB2Wzmxo0beHt7W+9LqQpiY2Mxm80sW7asxPU1a9YkMjKSyMhI6tevz9y5c4mPj+fdd98t0i4xMZHjx4/zxBNPMHfuXOvywmlqAFxdXbl27RoJCQn86U9/KtmzKrwAAAd/SURBVHF/NWrUID4+3mG+4d1JaGgoy5cvt75OTEwsFqgPPfQQ/fr1o1+/fowYMYJt27aRlJRE3759i7SLj4/n2rVrTJ48mTFjxgAwYMAACgoKrL/EatSowbZt29i/f/9ta3J3d2fx4sW88cYb5XWYFSomJoYLFy5gNpv58MMPcXJyKrJen99NOtNwEAEBAbRs2ZI1a9awZs0aunTpYp13S4qLiIgAbl4G+L3CM4onn3yyyPLfDvF2dXWlT58+/POf/yQ3N7fEfQwcOJAjR46wefPm8iq7wri5uVl/3r9/PxcvXqRdu3a3bX+vn1+bNm1o1qzZHb8tDxw4kDVr1pCVlWXsIOwsOzubDh060K9fP1xdXZk5c+Zt21blz0+h4UD69OnDpk2b+PbbbyvtfFm2UjhS/PffBm+3rCSjRo3iwoUL1skwf+8Pf/gD4eHhxMfHl71QGzt27Bjjx4/H19e32OWV37rXz8/JyYnRo0fz5ZdfcuzYsRLbPPfcc7i7u/PPf/7TYPX2Vb16df7xj38QGxtLTk4O8+bNu23bqvz5KTQcSPfu3TGZTPj4+PD444/buxyHtnXrVgBatGhRbF3hsm3bthVZ/tvLUwC+vr706NGDFStW3HY/Y8eOZd++ffzwww/3WnKF+/HHHxk2bBhubm4sXbqUBg0a3Latkc+vsE2h339+3bt3x8/Pj4SEhBL3Ub16dYYMGXLHz9eRuLi48PjjjzN06FBCQkLYsWPHbTu7q/Lnpz4NB1KjRg1mz56Np6dnsbvl4eaEju+99x4AHh4eTJgwwdYlVqiUlBSOHDkCQHp6OqtWraJdu3bWWYyzs7NZt24d33//PQkJCVSvXt16zfi3evTowZIlS0hNTWXy5MmEhYVx/Phx3NzcePnll4u0HTNmDP/+979vW1NYWBitWrVi37595XegFSA9PZ1hw4Zx8eJFXnrpJdLS0khLS6N79+7WNidPnmTVqlWkpKSwdetWAgMDS+yvGT16NMnJycTFxXHp0iX8/f3ZsWMHzzzzDEFBQdZ2JpOJUaNGMX369NvWNWzYMIf7plySr7/+mo0bNxIaGsrZs2fZu3cv9evXp27dutY2+vxuUmg4mDuNHjpw4ID1emnNmjXvu9BYsmSJ9Rkrhw8fJjY2lrffftsaGmfPniU2NpY6derwxz/+kYkTJxIYGFhsOx4eHixfvpy//e1vpKSkkJycjL+/Py+99FKxtgEBAXTt2tU6HLckY8eOZfz48eVzkBXk1KlT1mvff//7363LfxsaW7duZdeuXTzwwAMMHDiQl156CQ8Pj2Lbatq0KcuXL2fu3LnWzvVHHnmEhx56qFjb3r178/777/PLL7+UWFedOnUYMGCAw/3i+73atWuTlpZGUlISbm5utG7dmqlTpxa51KTP7yZNIyIiIoapT0NERAxTaIiIiGEKDRERMUyhISIihik0RETEMIWGiAP5/QyoRhTOilo4a69IRVJoiBhQ+Iv591NWA8TFxVnXderUyU4VitiGQkOklJKSksjIyABuTsPuaNM8iFQkhYZIKdSpU4e8vDzrnb5r167lwoUL1KlTp0i7y5cv8/e//51u3brRsmVLunTpwvTp04vMWJqens7o0aMJDQ2la9euJT5DpaCggFWrVhEVFUVoaCgRERFMnTqVs2fPVuyBityGQkOkFCIiIvD19SUhIYGrV6+ydOlSqlevzrPPPmttU1BQwKhRo4iPj+fy5cv06NEDZ2dnEhISGDhwINeuXaOgoIBx48aRmppKrVq1aN++Pe+8806x/c2ZM4fY2FgyMjJ4+umnCQwMZP369QwcOJDLly/b8tBFAIWGSKmYTCaGDh3KxYsXefnllzlx4gR9+vQp8uyTb7/9lr179wKwcOFCZs2axfLlyzGZTJw6dYrk5GTS0tI4dOgQAB9++CGzZs1iwYIFRfaVm5trPaMJDg6mVq1aBAQE4O7uTnp6+h3nyxKpKJqwUKSU+vXrx4IFC0hJScFkMvH888+zbt066/rfPqKzWbNmADRo0IC6deuSmZlJeno6rq6u1jaFky4Wti104cIFrl27BsBXX31VrI709PTyOygRg3SmIVJKNWrUoH///gB06dIFf3//IusbNWpk/fno0aMAZGRkcOHCBQAaNmxIw4YNrW0KH8taOC18obp161pnUX333Xc5fPiw9U9qaiojR44s5yMTuTudaYiUwbhx43jyyScJCAgotq5169a0bNmS7777jvHjxxMREcGuXbvIz8/Hz8+Prl274uHhQbNmzThy5Ajjx4/niSeeKPbQHjc3NwYNGsQ//vEP3njjDbZs2UK1atU4efIk3377LcnJyfj5+dnqkEUAnWmIlEnt2rUJDw/H29u72DpnZ2eWLFnCqFGjqFatGuvXrycvL48BAwZYHx7l7OzMwoULefLJJ7l06RL/+c9/mDx5crFtTZ06lRkzZhAQEMCWLVvYuHEjv/76K88//3yRBwSJ2IqepyEiIobpTENERAxTaIiIiGEKDRERMUyhISIihik0RETEMIWGiIgYptAQERHDFBoiImLY/wGvThPFBOglUAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DP1eLw6Sx661"
      },
      "source": [
        "## Multiclass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4-bfJk_3x59R",
        "outputId": "4f3860e9-48f9-4637-ecb9-e7ddf53e51a7"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "def sad_acc(metric):\n",
        "  pos_list = [i[0][0] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s] \n",
        "\n",
        "def happy_acc(metric):\n",
        "  pos_list = [i[1][1] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s]\n",
        "\n",
        "def fear_acc(metric):\n",
        "  pos_list = [i[2][2] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s]\n",
        "\n",
        "def disgust_acc(metric):\n",
        "  pos_list = [i[3][3] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s] \n",
        "\n",
        "def surprise_acc(metric):\n",
        "  pos_list = [i[4][4] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s]  \n",
        "\n",
        "def anger_acc(metric):\n",
        "  pos_list = [i[5][5] for i in metric]\n",
        "  pos_list_m = round(np.mean(pos_list),2)\n",
        "  pos_list_s = round(np.std(pos_list),2)\n",
        "  return [pos_list_m, pos_list_s]  \n",
        "\n",
        "\n",
        "data = 'PD'\n",
        "print(\"-----\", data, \"----\")\n",
        "print(\"-----3D----\")\n",
        "multi_pd_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_MultiClass_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_3pd = sad_acc(multi_pd_3d)\n",
        "happy_3pd = happy_acc(multi_pd_3d)\n",
        "fear_3pd = fear_acc(multi_pd_3d)\n",
        "disgust_3pd = disgust_acc(multi_pd_3d)\n",
        "surprise_3pd = surprise_acc(multi_pd_3d)\n",
        "anger_3pd = anger_acc(multi_pd_3d)\n",
        "print('Sad:',sad_3pd[0],'+-',sad_3pd[1])\n",
        "print('Happy:',happy_3pd[0],'+-',happy_3pd[1])\n",
        "print('Fear:',fear_3pd[0],'+-',fear_3pd[1])\n",
        "print('Disgust:',disgust_3pd[0],'+-',disgust_3pd[1])\n",
        "print('Surprise:',surprise_3pd[0],'+-',surprise_3pd[1])\n",
        "print('Anger:',anger_3pd[0],'+-',anger_3pd[1])\n",
        "\n",
        "print(\"-----2D----\")\n",
        "multi_pd_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_2pd = sad_acc(multi_pd_2d)\n",
        "happy_2pd = happy_acc(multi_pd_2d)\n",
        "fear_2pd = fear_acc(multi_pd_2d)\n",
        "disgust_2pd = disgust_acc(multi_pd_2d)\n",
        "surprise_2pd = surprise_acc(multi_pd_2d)\n",
        "anger_2pd = anger_acc(multi_pd_2d)\n",
        "print('Sad:',sad_2pd[0],'+-',sad_2pd[1])\n",
        "print('Happy:',happy_2pd[0],'+-',happy_2pd[1])\n",
        "print('Fear:',fear_2pd[0],'+-',fear_2pd[1])\n",
        "print('Disgust:',disgust_2pd[0],'+-',disgust_2pd[1])\n",
        "print('Surprise:',surprise_2pd[0],'+-',surprise_2pd[1])\n",
        "print('Anger:',anger_2pd[0],'+-',anger_2pd[1])\n",
        "\n",
        "print(\"-----1D----\")\n",
        "multi_pd_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/IIR/CNN_1D_results_MultiClass_IIR_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_1pd = sad_acc(multi_pd_1d)\n",
        "happy_1pd = happy_acc(multi_pd_1d)\n",
        "fear_1pd = fear_acc(multi_pd_1d)\n",
        "disgust_1pd = disgust_acc(multi_pd_1d)\n",
        "surprise_1pd = surprise_acc(multi_pd_1d)\n",
        "anger_1pd = anger_acc(multi_pd_1d)\n",
        "print('Sad:',sad_1pd[0],'+-',sad_1pd[1])\n",
        "print('Happy:',happy_1pd[0],'+-',happy_1pd[1])\n",
        "print('Fear:',fear_1pd[0],'+-',fear_1pd[1])\n",
        "print('Disgust:',disgust_1pd[0],'+-',disgust_1pd[1])\n",
        "print('Surprise:',surprise_1pd[0],'+-',surprise_1pd[1])\n",
        "print('Anger:',anger_1pd[0],'+-',anger_1pd[1])\n",
        "\n",
        "print(\"-----ML----\")\n",
        "multi_pd_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_MultiClass_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_pd_ml = sad_acc(multi_pd_ml)\n",
        "happy_pd_ml = happy_acc(multi_pd_ml)\n",
        "fear_pd_ml = fear_acc(multi_pd_ml)\n",
        "disgust_pd_ml = disgust_acc(multi_pd_ml)\n",
        "surprise_pd_ml = surprise_acc(multi_pd_ml)\n",
        "anger_pd_ml = anger_acc(multi_pd_ml)\n",
        "print('Sad:',sad_pd_ml[0],'+-',sad_pd_ml[1])\n",
        "print('Happy:',happy_pd_ml[0],'+-',happy_pd_ml[1])\n",
        "print('Fear:',fear_pd_ml[0],'+-',fear_pd_ml[1])\n",
        "print('Disgust:',disgust_pd_ml[0],'+-',disgust_pd_ml[1])\n",
        "print('Surprise:',surprise_pd_ml[0],'+-',surprise_pd_ml[1])\n",
        "print('Anger:',anger_pd_ml[0],'+-',anger_pd_ml[1])\n",
        "\n",
        "data = 'NC'\n",
        "print(\"-----\", data, \"----\")\n",
        "print(\"-----3D----\")\n",
        "multi_nc_3d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_3D_our/CNN_3D_results_MultiClass_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_3nc = sad_acc(multi_nc_3d)\n",
        "happy_3nc = happy_acc(multi_nc_3d)\n",
        "fear_3nc = fear_acc(multi_nc_3d)\n",
        "disgust_3nc = disgust_acc(multi_nc_3d)\n",
        "surprise_3nc = surprise_acc(multi_nc_3d)\n",
        "anger_3nc = anger_acc(multi_nc_3d)\n",
        "print('Sad:',sad_3nc[0],'+-',sad_3nc[1])\n",
        "print('Happy:',happy_3nc[0],'+-',happy_3nc[1])\n",
        "print('Fear:',fear_3nc[0],'+-',fear_3nc[1])\n",
        "print('Disgust:',disgust_3nc[0],'+-',disgust_3nc[1])\n",
        "print('Surprise:',surprise_3nc[0],'+-',surprise_3nc[1])\n",
        "print('Anger:',anger_3nc[0],'+-',anger_3nc[1])\n",
        "\n",
        "print(\"-----2D----\")\n",
        "multi_nc_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_MultiClass_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_2nc = sad_acc(multi_nc_2d)\n",
        "happy_2nc = happy_acc(multi_nc_2d)\n",
        "fear_2nc = fear_acc(multi_nc_2d)\n",
        "disgust_2nc = disgust_acc(multi_nc_2d)\n",
        "surprise_2nc = surprise_acc(multi_nc_2d)\n",
        "anger_2nc = anger_acc(multi_nc_2d)\n",
        "print('Sad:',sad_2nc[0],'+-',sad_2nc[1])\n",
        "print('Happy:',happy_2nc[0],'+-',happy_2nc[1])\n",
        "print('Fear:',fear_2nc[0],'+-',fear_2nc[1])\n",
        "print('Disgust:',disgust_2nc[0],'+-',disgust_2nc[1])\n",
        "print('Surprise:',surprise_2nc[0],'+-',surprise_2nc[1])\n",
        "print('Anger:',anger_2nc[0],'+-',anger_2nc[1])\n",
        "\n",
        "print(\"-----1D----\")\n",
        "multi_nc_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/IIR/CNN_1D_results_MultiClass_IIR_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_1nc = sad_acc(multi_nc_1d)\n",
        "happy_1nc = happy_acc(multi_nc_1d)\n",
        "fear_1nc = fear_acc(multi_nc_1d)\n",
        "disgust_1nc = disgust_acc(multi_nc_1d)\n",
        "surprise_1nc = surprise_acc(multi_nc_1d)\n",
        "anger_1nc = anger_acc(multi_nc_1d)\n",
        "print('Sad:',sad_1nc[0],'+-',sad_1nc[1])\n",
        "print('Happy:',happy_1nc[0],'+-',happy_1nc[1])\n",
        "print('Fear:',fear_1nc[0],'+-',fear_1nc[1])\n",
        "print('Disgust:',disgust_1nc[0],'+-',disgust_1nc[1])\n",
        "print('Surprise:',surprise_1nc[0],'+-',surprise_1nc[1])\n",
        "print('Anger:',anger_1nc[0],'+-',anger_1nc[1])\n",
        "\n",
        "print(\"-----ML----\")\n",
        "multi_nc_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/CSP/ML_MultiClass_CSP_'+str(data)+'.mat', 'conf_mat_list')\n",
        "sad_nc_ml = sad_acc(multi_nc_ml)\n",
        "happy_nc_ml = happy_acc(multi_nc_ml)\n",
        "fear_nc_ml = fear_acc(multi_nc_ml)\n",
        "disgust_nc_ml = disgust_acc(multi_nc_ml)\n",
        "surprise_nc_ml = surprise_acc(multi_nc_ml)\n",
        "anger_nc_ml = anger_acc(multi_nc_ml)\n",
        "print('Sad:',sad_nc_ml[0],'+-',sad_nc_ml[1])\n",
        "print('Happy:',happy_nc_ml[0],'+-',happy_nc_ml[1])\n",
        "print('Fear:',fear_nc_ml[0],'+-',fear_nc_ml[1])\n",
        "print('Disgust:',disgust_nc_ml[0],'+-',disgust_nc_ml[1])\n",
        "print('Surprise:',surprise_nc_ml[0],'+-',surprise_nc_ml[1])\n",
        "print('Anger:',anger_nc_ml[0],'+-',anger_nc_ml[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- PD ----\n",
            "-----3D----\n",
            "Sad: 0.94 +- 0.08\n",
            "Happy: 0.87 +- 0.11\n",
            "Fear: 0.85 +- 0.12\n",
            "Disgust: 0.87 +- 0.12\n",
            "Surprise: 0.87 +- 0.09\n",
            "Anger: 0.88 +- 0.11\n",
            "-----2D----\n",
            "Sad: 0.93 +- 0.05\n",
            "Happy: 0.86 +- 0.11\n",
            "Fear: 0.81 +- 0.13\n",
            "Disgust: 0.82 +- 0.14\n",
            "Surprise: 0.8 +- 0.13\n",
            "Anger: 0.83 +- 0.11\n",
            "-----1D----\n",
            "Sad: 0.87 +- 0.15\n",
            "Happy: 0.84 +- 0.18\n",
            "Fear: 0.8 +- 0.21\n",
            "Disgust: 0.78 +- 0.22\n",
            "Surprise: 0.79 +- 0.25\n",
            "Anger: 0.81 +- 0.24\n",
            "-----ML----\n",
            "Sad: 0.81 +- 0.02\n",
            "Happy: 0.77 +- 0.04\n",
            "Fear: 0.75 +- 0.03\n",
            "Disgust: 0.76 +- 0.03\n",
            "Surprise: 0.71 +- 0.04\n",
            "Anger: 0.75 +- 0.03\n",
            "----- NC ----\n",
            "-----3D----\n",
            "Sad: 0.93 +- 0.07\n",
            "Happy: 0.92 +- 0.07\n",
            "Fear: 0.89 +- 0.09\n",
            "Disgust: 0.88 +- 0.09\n",
            "Surprise: 0.89 +- 0.08\n",
            "Anger: 0.91 +- 0.06\n",
            "-----2D----\n",
            "Sad: 0.9 +- 0.08\n",
            "Happy: 0.91 +- 0.07\n",
            "Fear: 0.82 +- 0.08\n",
            "Disgust: 0.85 +- 0.08\n",
            "Surprise: 0.83 +- 0.12\n",
            "Anger: 0.88 +- 0.07\n",
            "-----1D----\n",
            "Sad: 0.75 +- 0.25\n",
            "Happy: 0.76 +- 0.27\n",
            "Fear: 0.82 +- 0.24\n",
            "Disgust: 0.8 +- 0.17\n",
            "Surprise: 0.76 +- 0.22\n",
            "Anger: 0.79 +- 0.22\n",
            "-----ML----\n",
            "Sad: 0.83 +- 0.03\n",
            "Happy: 0.78 +- 0.03\n",
            "Fear: 0.65 +- 0.03\n",
            "Disgust: 0.7 +- 0.03\n",
            "Surprise: 0.72 +- 0.06\n",
            "Anger: 0.74 +- 0.04\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nMNEsJXLhZHJ",
        "outputId": "0b70b345-9740-4935-d671-ce28d521c25a"
      },
      "source": [
        "data1 = {'Sad': [i[0][0] for i in multi_nc_ml], 'Happy': [i[1][1] for i in multi_nc_ml],\n",
        "         'Fear': [i[2][2] for i in multi_nc_ml], 'Disgust':[i[3][3] for i in multi_nc_ml],\n",
        "         'Surprise': [i[4][4] for i in multi_nc_ml], 'Anger': [i[5][5] for i in multi_nc_ml],\n",
        "          'Data': len([i[1][1] for i in multi_nc_ml])*['NC'], 'Model': len([i[1][1] for i in multi_nc_ml])*['ML']}\n",
        "df1 = pd.DataFrame(data1)\n",
        "data2 = {'Sad': [i[0][0] for i in multi_pd_ml], 'Happy': [i[1][1] for i in multi_pd_ml],\n",
        "         'Fear': [i[2][2] for i in multi_pd_ml], 'Disgust':[i[3][3] for i in multi_pd_ml],\n",
        "         'Surprise': [i[4][4] for i in multi_pd_ml], 'Anger': [i[5][5] for i in multi_pd_ml],\n",
        "          'Data': len([i[1][1] for i in multi_pd_ml])*['PD'], 'Model': len([i[1][1] for i in multi_pd_ml])*['ML']}\n",
        "df2 = pd.DataFrame(data2)\n",
        "data3 = {'Sad': [i[0][0] for i in multi_pd_1d], 'Happy': [i[1][1] for i in multi_pd_1d],\n",
        "         'Fear': [i[2][2] for i in multi_pd_1d], 'Disgust':[i[3][3] for i in multi_pd_1d],\n",
        "         'Surprise': [i[4][4] for i in multi_pd_1d], 'Anger': [i[5][5] for i in multi_pd_1d],\n",
        "          'Data': len([i[1][1] for i in multi_pd_1d])*['PD'], 'Model': len([i[1][1] for i in multi_pd_1d])*['1D CNN']}\n",
        "df3 = pd.DataFrame(data3)\n",
        "data4 = {'Sad': [i[0][0] for i in multi_nc_1d], 'Happy': [i[1][1] for i in multi_nc_1d],\n",
        "         'Fear': [i[2][2] for i in multi_nc_1d], 'Disgust':[i[3][3] for i in multi_nc_1d],\n",
        "         'Surprise': [i[4][4] for i in multi_nc_1d], 'Anger': [i[5][5] for i in multi_nc_1d],\n",
        "          'Data': len([i[1][1] for i in multi_nc_1d])*['NC'], 'Model': len([i[1][1] for i in multi_nc_1d])*['1D CNN']}\n",
        "df4 = pd.DataFrame(data4)\n",
        "data5 = {'Sad': [i[0][0] for i in multi_pd_2d], 'Happy': [i[1][1] for i in multi_pd_2d],\n",
        "         'Fear': [i[2][2] for i in multi_pd_2d], 'Disgust':[i[3][3] for i in multi_pd_2d],\n",
        "         'Surprise': [i[4][4] for i in multi_pd_2d], 'Anger': [i[5][5] for i in multi_pd_2d],\n",
        "          'Data': len([i[1][1] for i in multi_pd_2d])*['PD'], 'Model': len([i[1][1] for i in multi_pd_2d])*['2D CNN']}\n",
        "df5 = pd.DataFrame(data5)\n",
        "data6 = {'Sad': [i[0][0] for i in multi_nc_2d], 'Happy': [i[1][1] for i in multi_nc_2d],\n",
        "         'Fear': [i[2][2] for i in multi_nc_2d], 'Disgust':[i[3][3] for i in multi_nc_2d],\n",
        "         'Surprise': [i[4][4] for i in multi_nc_2d], 'Anger': [i[5][5] for i in multi_nc_2d],\n",
        "          'Data': len([i[1][1] for i in multi_nc_2d])*['NC'], 'Model': len([i[1][1] for i in multi_nc_2d])*['2D CNN']}\n",
        "df6 = pd.DataFrame(data6)\n",
        "data7 = {'Sad': [i[0][0] for i in multi_pd_3d], 'Happy': [i[1][1] for i in multi_pd_3d],\n",
        "         'Fear': [i[2][2] for i in multi_pd_3d], 'Disgust':[i[3][3] for i in multi_pd_3d],\n",
        "         'Surprise': [i[4][4] for i in multi_pd_3d], 'Anger': [i[5][5] for i in multi_pd_3d],\n",
        "          'Data': len([i[1][1] for i in multi_pd_3d])*['PD'], 'Model': len([i[1][1] for i in multi_pd_3d])*['3D CNN']}\n",
        "df7 = pd.DataFrame(data7)\n",
        "data8 = {'Sad': [i[0][0] for i in multi_nc_3d], 'Happy': [i[1][1] for i in multi_nc_3d],\n",
        "         'Fear': [i[2][2] for i in multi_nc_3d], 'Disgust':[i[3][3] for i in multi_nc_3d],\n",
        "         'Surprise': [i[4][4] for i in multi_nc_3d], 'Anger': [i[5][5] for i in multi_nc_3d],\n",
        "          'Data': len([i[1][1] for i in multi_nc_3d])*['NC'], 'Model': len([i[1][1] for i in multi_nc_3d])*['3D CNN']}\n",
        "df8 = pd.DataFrame(data8)\n",
        "\n",
        "multi_df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], axis=0).reset_index()\n",
        "print(multi_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    index       Sad     Happy      Fear  ...  Surprise     Anger  Data   Model\n",
            "0       0  0.867769  0.744000  0.680672  ...  0.800000  0.739837    NC      ML\n",
            "1       1  0.816667  0.769841  0.652542  ...  0.664000  0.707317    NC      ML\n",
            "2       2  0.783333  0.769841  0.677966  ...  0.629032  0.774194    NC      ML\n",
            "3       3  0.825000  0.769841  0.593220  ...  0.701613  0.830645    NC      ML\n",
            "4       4  0.833333  0.785714  0.635593  ...  0.758065  0.774194    NC      ML\n",
            "..    ...       ...       ...       ...  ...       ...       ...   ...     ...\n",
            "75      5  0.966942  0.930233  0.883333  ...  0.928000  0.912698    NC  3D CNN\n",
            "76      6  0.958678  0.953488  0.900000  ...  0.936000  0.920635    NC  3D CNN\n",
            "77      7  0.933884  0.961240  0.941667  ...  0.952000  0.952381    NC  3D CNN\n",
            "78      8  0.983471  0.968992  0.950000  ...  0.904000  0.968254    NC  3D CNN\n",
            "79      9  0.958678  0.976744  0.941667  ...  0.952000  0.984127    NC  3D CNN\n",
            "\n",
            "[80 rows x 9 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Hoanoh4Yr28Q",
        "outputId": "d66e3ed0-1fdb-49f9-b5c9-0ecb8143d82c"
      },
      "source": [
        "multi_df_new = pd.melt(multi_df, id_vars=['Model', 'Data'], value_vars=['Sad', 'Happy', 'Fear', 'Disgust',\n",
        "                                                                'Surprise', 'Anger'],\n",
        "        var_name='Emotions', value_name='Accuracy')\n",
        "\n",
        "x = \"Emotions\"\n",
        "y = \"Accuracy\"\n",
        "hue = \"Data\"\n",
        "\n",
        "\n",
        "fig = plt.figure(figsize=(15, 10))\n",
        "fig.subplots_adjust(hspace=0.08, wspace=0.1)\n",
        "ax = fig.add_subplot(2, 2, 1)\n",
        "plt.setp(ax, ylim=(0.5, 1.0))\n",
        "g1 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='ML'], x=x, y=y, hue=hue,\n",
        "                 ax=ax, palette = \"Blues\", capsize=0.05, ci=68, hue_order = [\"NC\", \"PD\"])\n",
        "#g1.set_ylim(bottom=0.55, top=1.0)\n",
        "#g1.legend(loc='upper left', bbox_to_anchor=(0,1))\n",
        "#g.set_xlabel('',fontsize = 13, weight='bold')\n",
        "#g1.set_ylabel('Accuracy',fontsize = 13, weight='bold')\n",
        "#g.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "#ax.set_yticklabels([\"0.80\", \"0.85\", \"0.90\", \"0.95\", \"1.00\"], size= 10, weight='bold')\n",
        "add_stat_annotation(g1, data=multi_df_new[multi_df_new['Model']=='ML'], x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"Sad\", \"PD\"), (\"Sad\", \"NC\")),\n",
        "                                 ((\"Happy\", \"PD\"), (\"Happy\", \"NC\")),\n",
        "                                 ((\"Fear\", \"PD\"), (\"Fear\", \"NC\")),\n",
        "                               ((\"Disgust\", \"PD\"), (\"Disgust\", \"NC\")),\n",
        "                               ((\"Surprise\", \"PD\"), (\"Surprise\", \"NC\")),\n",
        "                               ((\"Anger\", \"PD\"), (\"Anger\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "g1.legend_.remove()\n",
        "\n",
        "ax = fig.add_subplot(2, 2, 2)\n",
        "plt.setp(ax, ylim=(0.5, 1.0))\n",
        "g2 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='1D CNN'], x=x, y=y,\n",
        "                 hue=hue, ax=ax, palette = \"Blues\", capsize=0.05, ci=68, hue_order = [\"NC\", \"PD\"])\n",
        "#g2.set_ylim(bottom=0.55, top=1.0)\n",
        "#g2.legend(loc='upper left', bbox_to_anchor=(0,1))\n",
        "#g.set_xlabel('',fontsize = 13, weight='bold')\n",
        "#g1.set_ylabel('Accuracy',fontsize = 13, weight='bold')\n",
        "#g.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "#ax.set_yticklabels([\"0.80\", \"0.85\", \"0.90\", \"0.95\", \"1.00\"], size= 10, weight='bold')\n",
        "add_stat_annotation(g2, data=multi_df_new[multi_df_new['Model']=='1D CNN'], x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"Sad\", \"PD\"), (\"Sad\", \"NC\")),\n",
        "                                 ((\"Happy\", \"PD\"), (\"Happy\", \"NC\")),\n",
        "                                 ((\"Fear\", \"PD\"), (\"Fear\", \"NC\")),\n",
        "                               ((\"Disgust\", \"PD\"), (\"Disgust\", \"NC\")),\n",
        "                               ((\"Surprise\", \"PD\"), (\"Surprise\", \"NC\")),\n",
        "                               ((\"Anger\", \"PD\"), (\"Anger\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "g2.legend_.remove()\n",
        "\n",
        "ax = fig.add_subplot(2, 2, 3)\n",
        "plt.setp(ax, ylim=(0.5, 1.0))\n",
        "g3 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='2D CNN'], x=x, y=y,\n",
        "                 hue=hue, ax=ax, palette = \"Blues\", capsize=0.05, ci=68, hue_order = [\"NC\", \"PD\"])\n",
        "#g3.set_ylim(bottom=0.55, top=1.0)\n",
        "#g2.legend(loc='upper left', bbox_to_anchor=(0,1))\n",
        "#g.set_xlabel('',fontsize = 13, weight='bold')\n",
        "#g1.set_ylabel('Accuracy',fontsize = 13, weight='bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "#ax.set_yticklabels([\"0.80\", \"0.85\", \"0.90\", \"0.95\", \"1.00\"], size= 10, weight='bold')\n",
        "add_stat_annotation(g3, data=multi_df_new[multi_df_new['Model']=='3D CNN'], x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"Sad\", \"PD\"), (\"Sad\", \"NC\")),\n",
        "                                 ((\"Happy\", \"PD\"), (\"Happy\", \"NC\")),\n",
        "                                 ((\"Fear\", \"PD\"), (\"Fear\", \"NC\")),\n",
        "                               ((\"Disgust\", \"PD\"), (\"Disgust\", \"NC\")),\n",
        "                               ((\"Surprise\", \"PD\"), (\"Surprise\", \"NC\")),\n",
        "                               ((\"Anger\", \"PD\"), (\"Anger\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "g3.legend_.remove()\n",
        "\n",
        "#ax.set_xticklabels(ax.get_xticklabels(), rotation=90)\n",
        "ax = fig.add_subplot(2, 2, 4)\n",
        "plt.setp(ax, ylim=(0.5, 1.0))\n",
        "g4 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='3D CNN'], x=x, y=y,\n",
        "                 hue=hue, ax=ax, palette = \"Blues\", capsize=0.05, ci=68, hue_order = [\"NC\", \"PD\"])\n",
        "#g4.set_ylim(bottom=0.55, top=1.0)\n",
        "g4.legend(loc='lower right', bbox_to_anchor=(1,0))\n",
        "#g.set_xlabel('',fontsize = 13, weight='bold')\n",
        "#g1.set_ylabel('Accuracy',fontsize = 13, weight='bold')\n",
        "#g.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#_, ylabels = plt.yticks()\n",
        "#ax.set_yticklabels(ylabels, fontsize = 10, weight = 'bold');\n",
        "#g4.set_yticklabels(g4.get_yticklabels(), size= 10, weight='bold')\n",
        "#ax.set_yticklabels([\"0.80\", \"0.85\", \"0.90\", \"0.95\", \"1.00\"], size= 10, weight='bold')\n",
        "add_stat_annotation(g4, data=multi_df_new[multi_df_new['Model']=='3D CNN'], x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"Sad\", \"PD\"), (\"Sad\", \"NC\")),\n",
        "                                 ((\"Happy\", \"PD\"), (\"Happy\", \"NC\")),\n",
        "                                 ((\"Fear\", \"PD\"), (\"Fear\", \"NC\")),\n",
        "                               ((\"Disgust\", \"PD\"), (\"Disgust\", \"NC\")),\n",
        "                               ((\"Surprise\", \"PD\"), (\"Surprise\", \"NC\")),\n",
        "                               ((\"Anger\", \"PD\"), (\"Anger\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "\n",
        "props = dict(boxstyle='round', facecolor='none', edgecolor='black', alpha=0.5)\n",
        "\n",
        "# place a text box in upper left in axes coords\n",
        "g1.text(0.7, 0.96, \"ML\", transform=g1.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "g2.text(0.7, 0.96, \"1D CNN\", transform=g2.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "g3.text(0.7, 0.96, \"2D CNN\", transform=g3.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "g4.text(0.7, 0.96, \"3D CNN\", transform=g4.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "\n",
        "g1.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "g2.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "g3.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "g4.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#plt.setp(ax, ylim=(0.5, 1.0))\n",
        "\n",
        "fig.text(0.5, 0.08, 'Emotions', ha='center', fontsize = 13, weight='bold')\n",
        "fig.text(0.09, 0.5, 'Accuracy', va='center', rotation='vertical', fontsize = 13, weight='bold')\n",
        "\n",
        "plt.show()\n",
        "\n",
        "plt.savefig('multi_class.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "Happy_NC v.s. Happy_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=6.064e-01\n",
            "Sad_NC v.s. Sad_PD: t-test independent samples with Bonferroni correction, P_val=2.467e-01 stat=2.200e+00\n",
            "Fear_NC v.s. Fear_PD: t-test independent samples with Bonferroni correction, P_val=9.158e-05 stat=-5.853e+00\n",
            "Disgust_NC v.s. Disgust_PD: t-test independent samples with Bonferroni correction, P_val=3.458e-03 stat=-4.169e+00\n",
            "Surprise_NC v.s. Surprise_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=7.104e-01\n",
            "Anger_NC v.s. Anger_PD: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-6.714e-01\n",
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "Happy_PD v.s. Happy_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=7.334e-01\n",
            "Sad_PD v.s. Sad_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=1.153e+00\n",
            "Fear_PD v.s. Fear_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-2.247e-01\n",
            "Disgust_PD v.s. Disgust_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-1.518e-01\n",
            "Surprise_PD v.s. Surprise_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=2.596e-01\n",
            "Anger_PD v.s. Anger_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=1.477e-01\n",
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "Happy_PD v.s. Happy_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-1.114e+00\n",
            "Sad_PD v.s. Sad_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=3.331e-01\n",
            "Fear_PD v.s. Fear_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-8.535e-01\n",
            "Disgust_PD v.s. Disgust_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-2.716e-01\n",
            "Surprise_PD v.s. Surprise_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-4.052e-01\n",
            "Anger_PD v.s. Anger_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-7.237e-01\n",
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "Happy_PD v.s. Happy_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-1.114e+00\n",
            "Sad_PD v.s. Sad_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=3.331e-01\n",
            "Fear_PD v.s. Fear_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-8.535e-01\n",
            "Disgust_PD v.s. Disgust_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-2.716e-01\n",
            "Surprise_PD v.s. Surprise_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-4.052e-01\n",
            "Anger_PD v.s. Anger_NC: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-7.237e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAAJVCAYAAABj6TwsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde1yUdfr/8fcwgJpnUwZTNA3EPLSaGVO5YZgLAp7Q2E7aQbMtM3fNTFvXyrLarSzXLJdsPfaNZdPwgJqFeag8p7Fl/TyiAoECtiAqyMz8/nCdRAUHZZjT6/l49Ij7nnvmvm5k5jPX/TlcBpvNZhMAAAAAwCv4uToAAAAAAEDNIckDAAAAAC9CkgcAAAAAXoQkDwAAAAC8CEkeAAAAAHgRkjwAAAAA8CJOTfImTZqk2267TfHx8Zd83Gaz6ZVXXlHfvn3Vv39//fDDD84MBwAAAAC8nlOTvISEBM2ZM6fSxzds2KDMzEytWbNGL7/8sl588UVnhgMAAAAAXs+pSV7Pnj3VuHHjSh9PT0/XoEGDZDAY1K1bNxUVFeno0aPODAkAAAAAvJpL5+Tl5eUpODjYvh0cHKy8vDwXRgQAAAAAns3f1QFciYiICLVq1crVYQAAPEh2dra2bNni6jAuiXYNAFBdVbVrLk3yTCaTcnNz7du5ubkymUyXfV6rVq20ZMkSZ4YGAPAyCQkJrg6hUrRrAIDqqqpdc+lwzaioKKWmpspms2nXrl1q2LChgoKCXBkSAAAAAHg0p/bkjRs3Tlu3btXx48d15513asyYMSovL5ck3XfffYqMjNT69evVt29f1atXT6+++qozwwEAAAAAr+fUJG/69OlVPm4wGPTCCy84MwQAAAAA8CkeufAKANQUm82mw4cPKy8vTxaLpUZfOyAgQK1atVJwcLAMBkONvjYAAJUpKirSgQMHdOrUKVeH4jQGg0H169dXaGio6tWr5+pw3A5JHgCf9fPPPys5OVl169ZVSEiIAgICauy1bTabysrK9M0338jf31/333+/mjRpUmOvDwDAhSwWi5YuXaq9e/fqhhtuUIMGDbz2JqPValVmZqZWrFihnj17qk+fPl57rVeCJA+ATyopKdGiRYsUFxenTp06Oe08NptNW7du1fz58/XUU0/JaDQ67VwAAN+2evVqnT59WuPGjavRG5fu7OTJk1q4cKEaNGggs9ns6nDchktX1wQAV/nxxx/Vvn17pyZ40tnhJBEREapfv74OHjzo1HMBAHzXmTNn9J///EcDBgzwmQRPkq655hrFxsZqx44drg7FrZDkAfBJhw4d0g033FBr5wsNDdWhQ4dq7XwAAN+Sl5enpk2bqkGDBq4Opda1bt1aRUVFOnnypKtDcRskeQB8Umlpaa1O1K5Xr55KS0tr7XwAAN9S2+2aOzEYDLSzFyDJAwAAAAAvQpIHAOeZOHGiwsPD9fzzz1/02BtvvKHw8HA9/vjjkqSZM2cqPj6+tkMEAMBh27Zt0x/+8Af99re/VXh4uJYsWXLRMcOGDVN4eLjCw8PVpUsX9erVSyNGjNDSpUtls9kue46ysjLNmTNHgwYN0m9+8xvdeuutSkxMVHJyssrKyiT92r7OmjWrwnO3bNmi8PBwFRYWSpKysrIUHh6uiIgIFRcXXxTn1KlTr/RX4VNI8gDgAi1bttSqVasqjO0vLy/X0qVLdd1117kwMgAAqufkyZPq0KGD/vznP6tu3bqVHpeQkKCvvvpKX3zxhd577z11795dL7zwgkaPHl1lHdmysjKNGDFCs2fPVkJCgj7++GMtXrxYDz/8sJYsWaKdO3faj61Tp44+/PBDe0JXlVOnTikpKal6Fws7kjwAuEB4eLiuv/56rVq1yr5v3bp1CgwM1K233urCyAAAqJ7IyEiNGzdOMTEx8vOr/Kt/vXr11KJFCwUHB+umm27SU089pZkzZyo9PV2pqamVPm/+/Pnatm2b5s6dq+HDh6tTp04KCQlRbGyskpOT1blzZ/uxERERatWqld57773Lxj1s2DAtWLBAeXl51btgSCLJA4BLGjp0qBYvXmzfXrx4sRISEii0CviIrKws9evXT5MnT1ZcXJweffRRnT59WgsWLFBsbKz69++vP/3pT64OE3Cq3/72t+rQoYPWrFlT6THLly/X7bffrq5du170mJ+fX4XVPv38/DR+/HglJyfr8OHDVZ47JiZGHTp00IwZM678AnwYSR4AXEJ8fLy+//57ZWZm6tixY9q4caMSEhJcHRZQ63w52Tl06JAeeOABpaWlqWHDhvrss8+UlJSk1NRULV++XC+99JKrQwScLjQ0VEeOHKn08UOHDql9+/YOv15kZKS6d++ut99++7LHPvvss0pNTdXevXsdfn2cRZIHAJfQuHFj9e3bV4sXL9ann36qiIgI5uPBZ/lqstO6dWvdeOONkqTOnTsrOztb4eHhGj9+vJYuXSqj0ejiCAHns9lsVY5icWRhlgs9++yzWr16tb7//vsqj7v11lvVq1cvvfXWW9U+h68jyQOASgwZMkSpqalavHixhgwZ4upwAJfx1WQnMDDQ/rPRaJTFYlFSUpLuv/9+7d69W0OHDlV5ebkLIwScb//+/QoJCan08euvv14HDhyo1mvedNNN+t3vfqc33njjssc+88wzWr9+vbZv316tc/g6kjwAqMRtt92mgIAA/fLLL7r77rtdHQ7gMiQ7Z1mtVv38888ym80aP368iouLK6zCC3ibjRs3as+ePYqOjq70mPj4eH3zzTf6z3/+c9FjVqtVJ06cuOTzxo0bpx07dmjjxo1VxhAeHq5BgwY5lBDiV/6uDgAA3JXBYNCyZcskVfySe77S0lL9+OOPFfbVrVtX7dq1c3p8gKucn+z06NFDaWlpOnnypBo1auTq0JzKarXq2Wef1YkTJ2Sz2TR8+HCvv2ZfkpWVpccee0w9evTQzp07ZTKZ9N577yklJUXJyckyGo0KDQ11aC6ZOykpKbEvcmK1WpWTk6Mff/xRjRs3rjAN4dSpUzp27JgsFouOHj2qDRs2aM6cOerTp48GDBhQ6es//PDDWr9+vR599FGNGTNGPXv2VIMGDfTjjz/qn//8p/70pz8pIiLioue1bdtWiYmJWrBgwWWv4emnn7YnmmFhYdX9FfgkkjwAqML5q4JdyuHDhzVo0KAK+zp37nzJYrOAt/CFZKd169ZasWKFfXvEiBEujAa15dChQ5o+fbpeeeUVjR071j7/dO3atQoMDFRRUZGrQ6y277//XsOHD7dvz5w5UzNnztTgwYP1+uuv2/cvWbJES5YsUUBAgBo3bqyOHTvqxRdf1MCBA6uckxcYGKi5c+dq3rx5+uSTT/Tmm2+qTp06ateunRISEtS9e/dKnzt69Gh9+umnl72Gli1batiwYZozZ46DVw2D7UpmS7pYQkICX6AAXJX/+7//U48ePRQeHl4r59uyZYsKCgoUGxtbK+fDxdy57XDn2ABfkZWVpUcffdReLiApKUnl5eXasWOH6tevrz59+ujuu+9W/fr1XRzppe3fv19ff/11hYTOl7zzzjt66KGH1LRpU1eHUmuqajvoyQMAAD5nyZIlWr16tdPPExMTQ/kVD3Lh/NPS0lIlJSVp27Zt+vLLLzV79mwtX75c/v58hYZ74y8UgE/y8/OT1WqttfNZrVb5+bHWFdyLLyc6q1ev1p49e9ShQwennWPPnj2S5HbXDsd50vzT2m7X3I3FYqGdPQ9JHgCf1KRJEx07dsy+LLyzHTt2TEFBQbVyLsBRvp7odOjQQUlJSU57/VGjRjnttVE7PGn+aZMmTZSfn++TNxVPnjypsrIytx1K6wokeQB80o033qjly5fLbDZXunJmTSkpKdFPP/2kO++806nnQe2YNGmS1q1bp2uvvbbCwhzn2Gw2TZs2TevXr1fdunX1+uuvq3Pnzi6I1DEkOsBZnr7YTtOmTVW/fn3t2bNHHTt2dHU4tWrnzp264YYbGEZ7Hn4TAHxSmzZt1Lp1ay1cuFB33HGHQkJC5O/vX+UKYtVhs9lUVlamgwcPauPGjerZs6eaNGlSI68N10pISNCDDz6o55577pKPb9iwQZmZmVqzZo2+++47vfjii/r3v/9dy1EC8EWxsbH617/+pfz8fHXs2FENGzassXbN3VitVv3yyy/6/vvvlZGRcUULznhr2QyJJA+AjzIYDBowYIB27typLVu2aNmyZTVezDkgIECtWrVS79691alTpxp9bbhOz549lZWVVenj6enpGjRokAwGg7p166aioiIdPXqU4bpupqq6X550DjjGV+aftm3bVsOGDdP27du1cOFCnTp1ymWxOJvBYFCDBg3UoUMHPfLII1e8qqY3ls2QSPIA+DA/Pz/16NFDPXr0cHUo8CJ5eXkKDg62bwcHBysvL88tkzxfTnTi4+O94hzO4I29G740/7Rly5bq37+/S2PwJK1bt7bPz+/cubOys7MVHh6u8ePH28tmeCKSPAAAfBSJDirjjb0bzD/FpXhr2QzPitbNeeOdLwBA9ZhMJuXm5tq3c3NzZTKZXBgRUH3e2rsBXI4nlc2oCkleDfPGO18AAMdFRUVp0aJFiouL03fffaeGDRu65VBNoCre2ruBi9FJUZEnlc2oCu/MGsadLwDwbuPGjdPWrVt1/Phx3XnnnRozZox90Z777rtPkZGRWr9+vfr27at69erp1VdfdXHEwNXz9N4NX55/6ghf7aTw9LIZVSHJq2Hc+QIA7zZ9+vQqHzcYDHrhhRdqKRqgdnh67wbzT6tGJ4X3IdNwMk+/8wUAAHyLN/du4NK8sZPCV8pmVMZz/qU8lKff+QIAAIBv8YZOCl8qm3EpJHk1iDtfAADAnfl67wYc4y2dFL5cNoMkDwAAwEf4eu8GLkYnhXciyasG7n4BAABP58u9G76I76++yc/VAXiSc3e/nGnPnj218kYEAACA9+P7q2+iJ6+auPsFAAAAT+KL3199vTYiSR4AAICP8PUvvvAdvl4bkSQPAADAR/j6F1/AVzAnDwAAAAC8CD151cAQBwAAAHgSvr/6JpK8amCIAwAAADwJ3199E8M1AQAAAMCLOD3J27Bhg6Kjo9W3b99LLt2anZ2thx56SP3799ewYcOUm5vr7JAAAAAAwGs5NcmzWCyaOnWq5syZo7S0NK1YsUL79u2rcMxf//pXDRo0SMuXL9eTTz6pt956y5khAQAAAIBXc2qSl5GRobZt2yokJESBgYGKi4tTenp6hWP2798vs9ksSTKbzRc9DgAAAABwnFOTvLy8PAUHB9u3TSaT8vLyKhzTsWNHrVmzRpL0+eefq6SkRMePH3dmWAAAAADgtVy+8MqECRO0bds2DRo0SFu3bpXJZJLRaHR1WAAAAADgkZxaQsFkMlVYSCUvL08mk+miY959911JUklJidasWaNGjRo5MywAAAAA8FpO7cnr2rWrMjMzdeTIEZWVlSktLU1RUVEVjiksLJTVapUkJSUlaciQIc4MCQAAAAC8mlOTPH9/f02ZMkUjR45UbGys+vXrp7CwMM2YMcO+wMrWrVsVExOj6Oho5efn64knnnBmSAAAAADg1Zw6XFOSIiMjFRkZWWHf2LFj7T/HxMQoJibG2WEAAAAAgE9w+cIrAAAAAICa4/SePPiGrKwsPfbYY+rRo4d27twpk8mk9957TykpKUpOTpbRaFRoaKjefvttV4cKAAAAeDWSPNSYQ4cOafr06XrllVc0duxYffbZZ0pKStLatWsVGBiooqIiV4cI4CpwMwcAAM/AcE3UmNatW+vGG2+UJHXu3FnZ2dkKDw/X+PHjtXTpUuofAl7g0KFDeuCBB5SWlqaGDRvab+akpqZq+fLleumll1wdIgAAPo8kDzUmMDDQ/rPRaJTFYlFSUpLuv/9+7d69W0OHDlV5ebkLI7w6NptNkjRz5sxKtx3dB3gqbuYAAOD+GK4Jp7Farfr5559lNpvVo0cPpaWl6eTJkx5b7H7ZsmU6duyYSktL9cEHHygoKOiibUkO7Rs4cKArLwW4YhfezCktLVVSUpK2bdumL7/8UrNnz9by5cvl70/zAgCAq9CTB6exWq169tln1b9/fw0ePFjDhw/32ARPkgYOHKjg4GB9+OGHuu666y657eg+wFucfzNn/PjxKi4u1smTJ10dFgAAPo1bragRrVu31ooVK+zbI0aMcGE0zrF8+XLl5eVpxIgRysnJueS2JIf29e/f35WXAtSYczdzTpw4IZvN5vE3cwAA8AYkeYCD4uPjZTAYNHPmTD322GOy2WwXbUtyeB/gaXzhZg4AAN6AJA+XtWTJEq1evdrp54mJiVFCQoLTz3OlDAaDJGnMmDFVbldnH+AueJ8DAOA9SPJwWatXr9aePXvUoUMHp51jz549ksSXP7g9b60Vx/scNW3z5s2aN2+eCgsLVVxcrIYNG6pZs2Z6+OGHZTabXR0eAHg1kjw4pEOHDkpKSnLa648aNcpprw3UtEOHDmn69Ol65ZVXNHbsWHutuLVr1yowMFBFRUWuDvGK8D5HTUpJSdHevXvt2wUFBSooKFBKSgpJHgA4GatrAkA1USsOuLzExESFhYXZ3w9Go1FhYWFKTEx0cWQA4P3oyQOqwDwlXAq14oDLM5vNMpvNGj58uLKzsxUcHKzZs2e7OiwA8An05AFVODdPyZn27NlTK4kknIdacQAAwJ1wmxmXNWDAAK84x5VinhIuh1pxAADAnZDk4bLi4+O94hyoWefXCRwzZkyl297GW2vF+frNnOrYsGGDpk2bJqvVqnvuueeiGzXZ2dl6/vnnVVhYqCZNmuiNN95QcHCwi6IFAPgikjwAV2TZsmU6duyYSktL9cEHHygoKOii7YEDB7o6TDiImzmOsVgsmjp1qubOnSuTyaShQ4cqKipKoaGh9mP++te/atCgQRo8eLA2bdqkt956S2+88YYLo0Ztu7B8RPPmzTV69GiXripKSQvAt5DkAbgiAwcO1IoVK/TWW2/pzTffVFxc3EXbnoaFdnA5GRkZatu2rUJCQiRJcXFxSk9Pr5Dk7d+/X5MmTZJ0dvGR0aNHuyRWuM6F5SNycnJcXjqCkhaAb2HhFaAKAwYMcPoQs9o4hzMsX75cubm5GjFihHJyci657WlYaAeXk5eXV2HopclkUl5eXoVjOnbsqDVr1kiSPv/8c5WUlOj48eO1Gidc68LyEQEBAS4vHUFJC8C30JMHVIEhbJWLj4+3z8F77LHHKszJO7ftiVhoB1drwoQJevnll/Xpp5/qlltukclkonaij7mwfERQUJDLe8soaQH4FpI8AFfk3KIqY8aMqXIb8CYmk0m5ubn27by8PJlMpouOeffddyVJJSUlWrNmDautAgBqFcM1AQBwUNeuXZWZmakjR46orKxMaWlpioqKqnBMYWGhrFarJCkpKUlDhgxxRagAAB9GkgcAgIP8/f01ZcoUjRw5UrGxserXr5/CwsI0Y8YMpaenS5K2bt2qmJgYRUdHKz8/X0888YSLowYA+BqGawLA/1ArDo6IjIxUZGRkhX1jx461/xwTE6OYmJjaDgsAUIs2b96sWbNmKT8/3y1LkpDkOcAd690AzuSrpQRYaAcAADgiJSVFOTk5ktyzJAnDNR1wrrZMQUGBysrK7PVuAG9FKQEAAIDKJSYmKiAgQJJ7liShJ88BiYmJOnnypA4cOCCLxeIW9W4AZ6OUAAAAwKWZzWYFBQW5bUkSevIcYDabNXv2bHsBXHeodwMAAAAAl0KSBwAAAABexKeTPMv/6hh5+jkAAAAA4ByfnpNn9PPTip0HHT6+pLTc/n9Hnxffvd0VxQYAAAC4M1agd18+neQBuDTqxQEAgMs5twL9OedWoCfJcz2SPAAXoV4cAKC6LuzVcccC0ahZrEDvvkjyHHBgd4Y2fZaqosJ8SVLx8QId2J2h9p1ucnFkAAAA7uHCXh13LBCNmmU2m2U2mzV8+HBlZ2e7zQr0mzdv1qxZs5Sfn++zNxt8euEVR+1Yt1pHsw/LarVIkiyWcu1Y/5mLowIAwDXKyi1ecQ7UrMTERIWFhcloNEpyzwLR8A0pKSnKyclRWVmZCgoKtHfvXqWkpLg6rFpFT54DevSOUVnpaZUUF6n0VInqN2qiHpHRrg4LAACXCPQ3qt+0VIeOLSsskSRlF5Y4/BxJWvXnQVcUG1znwl4ddywQDd+QmJio3bt368yZMzIajWrfvr3P3WwgyXNA+043MTQTAAAAl8TwQPdiNpsVFBTk0zcbGK4JAAAAXAWGB8Ld0JPnJqgzAgAA4JkYHgh3Q0+emzi3IlVBQYHKysrsdUYAAADg3s4ND5RkHx7IjXq4Ej15boI6IwAAOB8jZ3wP8+Xgi+jJcxNms1mzZ89WcHCwJLlNnREAALwJI2d8D/Pl4ItI8pyMWkIAALiPC2u5MXLG+yUmJiogIEAStfvgO5w+XHPDhg2aNm2arFar7rnnHo0aNarC4zk5OXruuedUXFwsi8Wi8ePHKzIy0tlh1Zrq1BKSrqyeELWEAABwzIW13Bg54/1YTh++yKlJnsVi0dSpUzV37lyZTCYNHTpUUVFRCg0NtR/z/vvvq1+/frr//vu1b98+jRo1SmvXrnVmWG7Jmn9QloNbpNNFZ3ecLpY1/6D8mrdzbWAAAAAAPIpTh2tmZGSobdu2CgkJUWBgoOLi4pSenl7hGIPBoBMnTkiSiouL7SsT+RrL4Z2yFR+TbNazO2wWWQ7vdG1QAADAjikYADyFU3vy8vLy7AuJSJLJZFJGRkaFY5566imNGDFCixYt0qlTpzR37lxnhuS2jG26y2Ipk630pFR+WqrTQMY23V0dFgAA+B+mYADwFC4voZCWlqbBgwfr0Ucf1c6dOzVhwgStWLFCfn6+tSaMX/N2DM0EANSKC8sIsKw8AHgXp2ZSJpNJubm59u28vDyZTKYKx3zyySfq16+fJKl79+4qLS3V8ePHnRkWAAA+7cIyAiwrDwDexalJXteuXZWZmakjR46orKxMaWlpioqKqnBMy5YttWnTJknS/v37VVpaqmbNmjkzLAAAfNqFZQRYVh4AvItTh2v6+/trypQpGjlypCwWi4YMGaKwsDDNmDFDXbp0UZ8+fTRx4kRNnjxZ8+bNk8Fg0Ouvvy6DweDMsAAA8GkXlhFgWXnA+1w4LLt58+YaPXo0Q7J9hNPn5EVGRl5U927s2LH2n0NDQ5WcnOzsMAAAAACfcW5Y9jk5OTlKSUkhyfMRvrW6CQAAAOADLhyWHRAQwJBsH+Ly1TXh3jZv3qxZs2YpPz+f1dcAAAA8xIXDsoOCgvj+5kPoyUOVUlJSlJOTw+prAAC4IYvV6hXnAFCz6MlDlRITE7V7926dOXNGRqNR7du3r/GufnoLAcD7WPMPynJwi3S66OyO00U6sy1ZxnYR1IWtQUY/P63YedDh40tKy+3/d/R58d359wI8DT15qJLZbFZQUJAk2Vdfq+nki95CAPA+lsM7ZSs+Jtn+1wtks8pWfEyWwztdGxgA+ACSPLhcYmKiAgICJFGrCQC8hbFNdxkatpAC60t+RimwvgwNW8jYprurQwMAr8dwTR9lsVpl9HNuju/oOc71FlKrCQC8h1/zdgzLBAAXIcnzUdUZw38l4/clxvADAAAArsBwTQAAAADwIiR5qNKB3RkqPl4gSSoqzNdHb0/Vgd0ZLo4KAFxnw4YNio6OVt++fZWUlHTR4zk5ORo2bJgGDRqk/v37a/369S6IEgDgy0jyUKUd61bLYjk7XNNqteho9mHtWP+Zi6MCANewWCyaOnWq5syZo7S0NK1YsUL79u2rcMz777+vfv36KTU1VW+//bZeeuklF0ULAKiusnKLV5yDOXmoUo/eMSr+73GVFP2iOvXqq37DRuoRGe3Qc8vKLQr0Nzo1vto4B2rW5s2bNW/ePBUWFqq4uJjaiPAoGRkZatu2rUJCQiRJcXFxSk9PV2hoqP0Yg8GgEydOSJKKi4vtZWgAAO4v0N+oftNSHTq2rLBEkpRdWOLwcyRp1Z8HXVFs1UGShyq173ST2ne66Yqe6y1vEtSslJQU7d27175dUFCggoICpaSkkOR5KW9K7PPy8hQcHGzfNplMysioOIT9qaee0ogRI7Ro0SKdOnVKc+fOre0wAQA+juGaAGpVYmKiwsLCZDSe7YGlNqL3O5fYFxQUqKysTAUFBdq7d69SUlJcHZpTpKWlafDgwdqwYYOSkpI0YcIEWa1WV4cFAPAhJHkAapXZbNbs2bPtvSHnaiN6Wo8OHOdNib3JZFJubq59Oy8vTyaTqcIxn3zyifr16ydJ6t69u0pLS3X8+PFajROoSe46R8lSCzdPauMcgDMwXBNwkQuHsDVv3lyjR48m2YHXMZvNMpvNGj58uLKzs+2JvSfq2rWrMjMzdeTIEZlMJqWlpemtt96qcEzLli21adMmJSQkaP/+/SotLVWzZs1cFDFw9aoz/UK6sikYVzL9gpq/QOVI8gAXuXBuWk5ODvPSADfn7++vKVOmaOTIkbJYLBoyZIjCwsI0Y8YMdenSRX369NHEiRM1efJkzZs3TwaDQa+//roMBoOrQwcA+BCSPMBFEhMTdfLkSR04cEAWi0UBAQEeOXztfKyo6nm8aVGU2hIZGanIyMgK+8aOHWv/OTQ0VMnJybUdFgDUGNpzz0eSB5ez5h+UThef3ThdpDPbkmVsFyG/5t49ROLCIWxBQUEe/6WaFVU9D6udAjXPmn9QloNbpNNFZ3ecLpY1/6DXt2vwHu46RNditcrox5IijiDJg8tZDu+UbP+bcG2zylZ8TJbDO2kMgVpwYY+y0WhU+/btHepV5k4vcGmWwztlKz726w6bhXYNqAHMw3QcSR5cztimu8pLT0ilJyT/ujLUuUbGNt1dHRbgE65mURR3vdMLuJqxTXdZLGWylZ6Uyk9LdRrQrnkobmbBU5HkweX8mrdTIHc3fcbFw5h8Z4guAN/g17wdn2degmkI8FQMagVqmLvWE3IX9mFMtv/VHjpviC4AAACuHj15QA1jCFvVLhrG5CZDdDdv3qxZs2YpPz+fFSZrGL23AADULpI8ALXKXYcxpaSkKAuN1xAAACAASURBVCcnRxIrTNa0ixehYIElAACcieGaAKCzq0wGBARIkoxGo8LCwjy6bqHFanV1CHbGNt1laNhCCqwv+RmlwPoyNGzh8t5bAAC8FT15AKCzq0wGBQVVe4VJd1WdZaalK1tq2tFlpt219xYAAG9FTx7gItb8gzqzLfmiYrkAAADA1SDJA1zk4lUmLawwCQAAgKvGcE3ARSiWCwAAAGcgyQNchHlKcAcHdmdo02epKirMlyQVFebro7en6rboQWrf6SYXRwcAAK4EwzUBwIftWLdaR7MPy2q1SJKsVouOZh/WjvWfuTgyoHrcaUVZwJn4W4cj6MkDAB/Wo3eMykpPq6S4SKWnSlSnXn3Vb9hIPSKjXR0aUC3utKIs4Ez8rcMRJHkA4MPad7qJYZkAAHgZhmsC8Gq1MayFoTNwJ/w9AgDoyQPg1aozrOVKhrRIDGuBe+FvHgBAkgcAtWTz5s2aNWuW8vPz1bBhQzVr1kwPP/ywzGazq0MDAABehOGaAFBLUlJSlJOTo7KyMhUUFGjv3r1KSUlxdVgAAMDLkOQBQC1JTExUQECAJMloNCosLEyJiYkujgoAvNuB3RkqPl4g6ddaoAd2Z7g4KsC5SPIAoJaYzWYFBQVJkoKDgzV79myGagKoFQd2Z+ijt6eqqDBfkm8lOzvWrZbFcnb+KbVA4StI8gAAALzcjnWrdTT7sKxWiyTfSnZ69I5R42tbyD8gQPUbNVFQqzbUAoXXY+EVAHabN2/WvHnzVFhYqOLiYhYHAQAv0aN3jMpKT6ukuEilp0pUp1591W/YyCeSHeqBwheR5AGwS0lJ0d69e+3bBQUFKigoUEpKCkkeAHgwEh3Atzh9uOaGDRsUHR2tvn37Kikp6aLHX331VQ0cOFADBw5UdHS0brnlFmeHBKASiYmJCgsLk9FolORbi4NczcT8snKLM0OrtXMAAADv4NSePIvFoqlTp2ru3LkymUwaOnSooqKiFBoaaj/m+eeft/+8cOFC7d6925khAaiC2WyW2WzW8OHDlZ2dbV8cxBdUNjHfkTvfgf5G9ZuW6tB5ygpLJEnZhSUOP0eSVv15kMPHAgAA3+bUnryMjAy1bdtWISEhCgwMVFxcnNLT0ys9Pi0tTfHx8c4MCQAuiYn5AHB1rPkHdWZbsnS66OyO00U6sy1Z1vyDrg0M8EFO7cnLy8tTcHCwfdtkMikj49LDn7Kzs5WVlcW8HwAuwXwVALg6lsM7ZSs+9usOm1W24mOyHN4pv+btXBcY4ATW/IPS6eKzG/+7oWFsF+E2f+tuU0IhLS1N0dHR9rlAAOBtLtUgcIcbgLcwtukuQ8MWUmB9yc8oBdaXoWELGdt0d3VoTsfnu3u5mnn2jrIc3inZ/jdf/rwbGu7CqT15JpNJubm59u28vDyZTKZLHrty5UpNmTLFmeEAPs1itcro5zb3dXxSZQ2Cu9z1A4Cr4de8nc9+nvH57l6uZp69o4xtuqu89IRUekLyrytDnWvc6oaGU5O8rl27KjMzU0eOHJHJZFJaWpreeuuti47bv3+/ioqK1L27+/xiAG9j9PPTip2O3VUsKS23/9/R50hSfHcas6q4e4MAALgyfL67lx69Y1T83+MqKfrFaTUh/Zq3U6AbJ/FOTfL8/f01ZcoUjRw5UhaLRUOGDFFYWJhmzJihLl26qE+fPpLO9uLFxsbKYDA4MxwAcCl3bxAAAFeGz3f3wjz7WiiGHhkZqcjIyAr7xo4dW2F7zJgxzg4DAAAAQA2y5h+U5eCW81ZULZY1/yDDVN0AE3QA2B3YnaGP3p6qosJ8Sc6brAwAADyffUVVm/XsDpvFrRYf8WUkeQDsdqxbraPZh2W1np08fv5kZQAAgPNdtKJqvcbMRXQTTh+uCcBz9Ogdo7LS0yopLlLpqRKnTVYGAACez5dXVHV3JHkA7JioDFzehg0bNG3aNFmtVt1zzz0aNWpUhcdfffVVbdmyRZJ0+vRpFRQUaPv27a4IFQDgo0jyAABwkMVi0dSpUzV37lyZTCYNHTpUUVFRCg0NtR/z/PPP239euHChdu/e7YpQAQA+jDl5AAA4KCMjQ23btlVISIgCAwMVFxen9PT0So9PS0tTfHx8LUYIAABJHgAADsvLy1NwcLB922QyKS8v75LHZmdnKysrS2azubbCAwBAEkkeAABOkZaWpujoaBmNRleHAgDwMSR5AAA4yGQyKTc3176dl5cnk8l0yWNXrlypuLi42goNAAA7kjwAABzUtWtXZWZm6siRIyorK1NaWpqioqIuOm7//v0qKipS9+7UiwIA1D5W1wQAwEH+/v6aMmWKRo4cKYvFoiFDhigsLEwzZsxQly5d1KdPH0lne/FiY2NlMBhcHDEAwBeR5AEAUA2RkZGKjIyssG/s2LEVtseMGVObIQEAUAFJHnAJmzdv1rx581RYWKji4mI1b95co0ePZpU8AADgEQ7sztCmz1JVVJgvSSo+XqADuzPUvtNNLo4MtYE5ecAlpKSkaO/evSooKFBZWZlycnKUkpLi6rAAAAAcsmPdah3NPiyr1SJJsljKtWP9Zy6OCrWFnjzgEhITE3Xy5EkdOHBAFotFAQEBSkxMdHVYAAAADunRO0ZlpadVUlyk0lMlqt+oiXpERrs6LNQSkjzgEsxms8xms4YPH67s7GwFBQUxVBMAAHiM9p1uYmimD2O4JgAA8BkHdmfoo7enXjRPCQC8CUkeAADwGcxTAuALGK4JAAB8BvOUAPgCkjz4FIvVKqMfHdgA4KuYpwTAF5DkwacY/fy0YudBh48vKS23/9/R58V3b3dFsQFAbbmwflZRYb4+enuqboseRAIEAF6ALg0AAHzMhfPSrFaLjmYfZm4aAHgJevIAAPAxF85Lq1Ovvuo3bMTcNADwEiR5wCVcOJTp3BLbDGMC4A2YlwYA3o3hmsAlsMQ2AAAAPBU9ecAlsMQ2AAAAPBVJHnAJDGUCAACAp2K4JgAAAAB4EZI8AAAAAPAiJHkAAAAA4EVI8gAAAADAi5DkAQAAAIAXIckDAAAAAC9CkgcAAAAAXoQkDwAAAAC8CEkeAAAAAHgRkjwAAAAA8CIkeQAAAADgRUjyAAAAAMCLkOQBAAAAgBchyQMAAAAAL0KSBwAAAABehCQPAAAAALyI05O8DRs2KDo6Wn379lVSUtIlj1m5cqViY2MVFxenZ555xtkhAQAAAIDX8nfmi1ssFk2dOlVz586VyWTS0KFDFRUVpdDQUPsxmZmZSkpK0scff6zGjRuroKDAmSEBAAAAgFdzak9eRkaG2rZtq5CQEAUGBiouLk7p6ekVjklJSdEDDzygxo0bS5KuvfZaZ4YEAAAAAF7NqUleXl6egoOD7dsmk0l5eXkVjsnMzNTBgwd17733KjExURs2bHBmSAAAAADg1Zw6XNMRFotFhw4d0sKFC5Wbm6sHH3xQy5cvV6NGjVwdGgAAAAB4HKf25JlMJuXm5tq38/LyZDKZLjomKipKAQEBCgkJ0fXXX6/MzExnhgUAAAAAXsupSV7Xrl2VmZmpI0eOqKysTGlpaYqKiqpwzN13362tW7dKkgoLC5WZmamQkBBnhgUAwBVj1WgAgLtz6nBNf39/TZkyRSNHjpTFYtGQIUMUFhamGTNmqEuXLurTp49++9vf6uuvv1ZsbKyMRqMmTJigpk2bOjMsAACuCKtGAwA8gdPn5EVGRioyMrLCvrFjx9p/NhgMmjRpkiZNmuTsUAAAuCrnrxotyb5q9PlJHqtGAwBczenF0AEA8BasGg0A8AQuX10TAABvwqrRAABXoycPAAAHsWo0AMATkOQBAOAgVo0GAHgChmsCAOAgVo0GAHgCkjwAAKqBVaMBAO6O4ZoAAAAA4EVI8gAAAADAi5DkAQAAAIAXIckDAAAAAC9CkgcAAAAAXoQkDwAAAAC8CEkeAAAAAHgRkjwAAAAA8CIkeQAAAADgRUjyAAAAAMCLkOQBAAAAgBchyQMAAAAAL0KSBwAAAABehCQPAAAAALwISR4AAAAAeBGSPAAAAADwIiR5AAAAAOBFSPIAAAAAwIuQ5AEAAACAFyHJAwAAAAAvQpIHAAAAAF6EJA8AAAAAvAhJHgAAAAB4EYeSvNTUVJWWljo7FgAAAADAVXIoyZs4caJ69eqlKVOmaNeuXc6OCQAAAABwhRxK8lq1aqXi4mKlpKTovvvuU2xsrD788EMdO3bM2fEBAAAAAKrBoSQvPT1dixcv1siRI9WqVSsdOHBAb775pu666y794Q9/0LZt25wdJwAAAADAAQ4vvNK5c2eNHz9e8+fPV+/evWWz2VReXq5169Zp+PDhmjdvnhPDBAAAAAA4wqEkr7y8XKtXr9aIESPUt29frV+/XpLUvXt3TZw4Ua1atdIHH3zg1EABAAAAAJfn78hBd955p44fPy6bzaa6desqLi5ODz74oG688UZJ0pkzZzR9+nSnBgoAAAAAuDyHkrzCwkK1atVK9913n+655x41bty4wuNRUVFq3ry5UwIEAAAAADjOoSTvvffe01133SWDwXDJx2+44QbdcMMNNRoYAAAAAKD6HJqTd/r0ac2ePVsWi0XS2Tl677//vlauXOnU4AAAAAAA1eNQkjd9+nRt375dRqNRkuTv76+dO3cyDw8AAAAA3IxDSd7Ro0fVqlWrCvuCg4N19OhRpwQFAAAAALgyDiV5zZo107Zt23TmzBlJZ1fT3L59u5o1a+bU4AAAAAAA1ePQwitdu3bVF198obi4OHXu3Fk//PCDjhw5or59+zo7PgAAAABANTiU5I0ZM0YbN27U4cOHdeTIEXu9vDFjxjg7PgAAAABANTg0XLNDhw5KTk7WkCFDdNttt2no0KFKTk5WWFjYZZ+7YcMGRUdHq2/fvkpKSrro8SVLlshsNmvgwIEaOHCg/v3vf1f/KgAAqCW0awAAd+dQT54kdezYUdOmTavWi1ssFk2dOlVz586VyWTS0KFDFRUVpdDQ0ArHxcbGasqUKdV6bQAAahvtGgDAEzic5C1ZskRff/218vPzZbPZJEkGg0Hz58+v9DkZGRlq27atQkJCJElxcXFKT0+/qDEEAMAT0K4BADyBwXYuY6vCe++9p5kzZ9q3z0/yfvzxx0qft3r1am3cuNHeA5iamqqMjIwKdzeXLFmi6dOnq2nTpmrXrp0mTZqkli1bVhlPRETERSUdAACoSnZ2trZs2XJVr0G7BgBwF1W1aw715C1dulSBgYEKDg7W4cOH1atXL23ZskX9+vW76uDuuusuxcfHKzAwUMnJyXruuee0YMGCKp9ztY00AADOQrsGAHA1hxZeyc7OVt++fdW7d29J0pw5c3THHXeoYcOGVT7PZDIpNzfXvp2XlyeTyVThmKZNmyowMFCSdM899+iHH36oTvwAANQa2jUAgCdwKMnz9/dX48aNdc0110iScnNzFRgYqGXLllX5vK5duyozM1NHjhxRWVmZ0tLSFBUVVeGYo0eP2n9eu3atbrjhhupeAwAAtYJ2DQDgCRwartmsWTMVFhaqY8eOstlsio+PV0lJiRo3blz1i/v7a8qUKRo5cqQsFouGDBmisLAwzZgxQ126dFGfPn20cOFCrV27VkajUY0bN9Zrr71WIxcGAEBNo10DAHgChxZemTx5so4cOaIZM2YoOjpa//3vfyVJjzzyiJ577jmnBwkAAAAAcIxDSd75srKytHbtWgUFBSkmJsZZcQEAAAAArsBlk7wzZ86oX79+uuOOO/TSSy/VVlwAAAAAgCtw2YVXAgICdPr0aVmt1tqIx6OsWrVK0dHR6tKli2677TYNHz68Wr+nDz/8UOHh4VqyZIkTo6y+rKwshYeH6/HHH7fvc9dYa8O538f5/91yyy2uDuuqnX9dXbt2VWRkpJ555hkdOXJEkhQVFaXu3bvXakx5eXmaOXOmvvjiC6e8/tW+Z6/UzJkzFR4ertWrVzv9XFfio48+sv8tHDhwwNXhwMVo23wDbVvtcWbbRrt2abRrDi68Mnz4cM2fP18PP/wwq4T9T2FhoZ599lm1a9dOL730kv773/9q3bp1quboV3iQTp06aeTIkZLO3vyoCeXl5fL3d+ht6DSdOnXSsGHDtGXLFqWmpmrLli1aunSpJk+erDNnztRqLEePHtW7776rwYMH6+67767R166p92x1/83Ky8sVHR2t9u3bq1u3btUNu1asXLlSfn5+slqtWrVqlUaPHl0r53WHv39URNvme2jbnM9ZbRvtWuVo1xwsoZCcnKzjx4+rf//+6tWrl/r06aM+ffrU+JcwT5KVlaUzZ86oZcuW6tu3rx599FEtWLBARqNRTz/9tHr27KmuXbsqNjZWn3/+uf15H374oSIiIhQbG6s9e/a48Aour7y8XIWFhSosLNSpU6fs+yu7vnN3z+677z6NGjVKt9xyi5599lmVlZVJ+vXu2WuvvaaIiAjde++9ysnJ0Z49exQeHq5p06ZJOvuh1alTJ/3pT3+q/YuuQrNmzXTbbbfZ/9u/f78eeeQR3Xzzzbrrrrs0b948+7GJiYm6+eab9Zvf/EYJCQnavn27pLMFj8PDw/XYY49p6NChSkxMdNHV/CooKEgJCQn661//qsTERB07dkyLFi3SK6+8ookTJ0qSduzYof79+6tr164ym80aN26cJKmoqEijRo1St27dNGHCBMXExCg8PFyStGTJEoWHh+vDDz+UJD3++OMKDw9XVlaWCgoK9NBDD6l79+66+eabdc8996iwsFBDhw6VJH366ac1fne9qvdseHi44uPjJUmrV69WeHi4Zs6cKUkaNmyYwsPD9corrygiIkLr1q2z73vjjTd0++23Kz4+Xj/99JOkX+9uTpo0SX369NHf/vY3ffbZZxo3bpx27dqlU6dO6emnn1aPHj3UrVs3DRw4UHv37pUkffLJJ4qOjla3bt1077331kp9tby8PH377bfq16+fgoKCtGrVKkm//vuNGzdOAwYMUM+ePTV//nxJks1m0+uvv65bbrlFiYmJGj16tMLDw+0Fvb/88ksNGDBA3bp104ABA/T1119Lcs+/f1RE20bbRtvmOW0b7dql0a6d5VCSl5OTI6vVKqvVqvz8fGVnZ9v/81Xt27dX06ZNtX79ekVERCghIUH//ve/JZ2to/Tss8/qmWeekSRNmDBBpaWl+umnn/S3v/1NLVq00EMPPaRvvvnGlZdwWV999ZX9Q//cB4NU+fWd89133+nWW2+V2WzWsmXLlJycbH/s5MmTOnnypO69917t3LlTr776qjp06KBbbrlFy5cv15kzZ5Seni6LxaJBgwbV3sU64Pzfx6hRo/Tkk09q3759GjlypG666Sa99tprWrt2rSTp9ttv16RJk/TUU0/p2LFjev755yu81qZNm9S3b189/PDDLriSyt15552SpO+//77C/jlz5igrK0tTpkzR6NGj1bRpU0nSrFmztH79esXFxalFixY6ePCgQ+dZvny5Nm/erEceeUQTJ07UjTfeKIvFYv/y07NnT02fPl09e/assWur6j3riB9//FHjx49Xu3bt7PsOHTqkRx99VAcOHNCkSZMqHP/111/rscce01133VVh/1dffaXPPvtM/fv31+TJk3XrrbeqvLxcW7Zs0Z///Ge1bt1aTzzxhI4fP64//OEPFd5bzrB69WpZrVbFxMTod7/7nfbu3at9+/bZH9+yZYt+//vfy2Aw6K233lJZWZnWrl2ruXPnqkOHDurfv3+Fz7KDBw9qzJgxqlu3rp588kkFBgbqqaeeqlA7zl3//kHbRttG2yZ5TttGu3ZptGtnOdSfSI2fizVo0EAff/yx5s6dq40bN+qHH37Q5MmT1aJFC+3bt09paWkVhgNkZWXZ7wY89NBDuueee/Tzzz/r/fffd9UlXNZvfvMb/fGPf5R09g3zr3/9SxaLpdLrq1OnjiSpW7duGjlypA4fPqzPP/9cW7du1fDhwyVJfn5++stf/qLAwEClpqZq69atkqT77rtPzzzzjL788kt99tlnat68uXr16lXLV1y1838fderU0f333y9JmjFjhv2Yb775RhEREdq9e7eSkpJksVjsj50+fdr+c+/evSvMC3EX54Z4GAyGCvuvv/56rVu3Ths2bFCnTp3s175lyxb5+fnphRdeUGBgoJYtW1bhQ68ybdu2lXT2Q7FHjx7q16+fWrRooTvuuENvv/22Wrdurbi4uBq9tsres0FBQQ49/y9/+Ys6duxYYd+ECRPUpk0brV27Vjt27NCJEyfsj40cOVL33nuvJNnvdktSSEiI/Pz8tGvXLtWpU0dms1kdO3bU3/72N0lnG8uvvvrKfvy+ffvUuXPnK77uy1m1apUCAgLUvn17e6/GypUr1bp1a0nSkCFD9MADD2jt2rX66quvVFBQYP8se+qpp3T77bdr165dWrFihaSz74EzZ87ou+++03fffWc/z65du+y1Vd317x+0bbRttG2S57RttGuXRrt2lkNJ3uDBg50dh8c5c+aMrr/+ek2dOlWS9Pe//12zZs3SDz/8oNTUVN1+++169NFHtWjRIq1bt84+rON87j7HoWnTprr99tslnb3bI50d5lLZ9Z1rCM9dV3Wu73e/+52uvfZaLViwQLt27dIDDzwgo9FYw1d0dc7/fZwbhtCrVy+NGDHCfkzz5s21bNkyrV+/XvHx8Ro8eLCmT5+uH374ocLfgKMfwLXt3Idw586dK9z1Gj9+vHr06KFdu3bpk08+UVJSktavX1/la/n5nR0ocO7LQFFRkf2xu+66SykpKfr666+1ceNGffDBB5o7d64aNWpU05dkV9l7du/evfLz81N5ebkkqbi4+JLPv9S/WVV/65X9G3fs2FHLli3TunXrtG3bNs2bN0+vvPKK/fGJEyfahwVZrVZ7o+QMP//8s3bt2iWbzVbhi8eqVav02GOPSZK9ATs3x+D8L3cXfmE638iRI3XHHXfYt2+44QZlZmZKct+/f9C20bbRtnlS20a7djHatV85lOSlpqZW+pi7DTuoLfv27dMzzzyj2NhYtWrVStu2bZN0driHJJ06dUqHDh3St99+a39ORESEJGn+/PmyWq0evZrXpa7vnO+++05z5szRrl27JEm33nqr/TGr1aqXX35ZzZo1U25urvr27StJCgwM1NChQ/WPf/xDkvvfWGjXrp2uv/56ffvtt7rttttUr149ffPNN/brkaSSkhL9v//3/9x+fsrRo0e1ZMkSbdu2TZ9++qlatGihBx54QEuXLrUf849//EOBgYEKDQ1Vy5YtlZWVpRMnTigiIkI//vijXn75ZTVq1KjCnc5zH+Lr1q1TkyZNKtz9Wr16tX766Se1bdtWYWFh+vbbb3X06FGFhIRIknbv3q0VK1bo9ttvV7NmzWrkOit7z3bo0EGtWrVSVlaWli9frkWLFjn8mm+88Ya6deum7777Tp06dVKDBg0u+5zt27crPT1doaGhuvHGG7V+/Xrl5eWpd+/e+uc//6kVK1aoYcOGOnbsmJYuXerUlctWr14tm82mxx9/XDfddJOks/Mnvvzyywp3by8UERGh+fPna9asWdq3b599KJd0djhXQECAPv/8c7Vt21ZFRUVatWpVhV4BuC/aNto22jbPadto1y5Gu/Yrh5K8iRMnVprZ+mqS17x5c7Vv317Jycn65ZdfdO2112rMmDG64447FBcXp/T0dBkMBvXq1UsrV66UdPZOx4QJE5SUlKRFixbp1ltvtXcFewp/f/9Kr++c7t27a+vWrfYJzee69iXpmmuuUYMGDZScnKxu3bpVGM//+9//Xh988IHCwsIuGj7gbvz9/fXee+/p1Vdf1fvvvy+j0ahOnTqpQ4cOatOmjdasWWMfz92zZ0+3nqOye/duTZkyRddee63i4+P1xz/+Uc2bN69wjJ+fnxYtWqT8/Hw1adJETz/9tK677jo9+eSTOnjwoFatWqWoqChdd9119juGPXr0UHx8vNauXatrrrlGXbp0sTeG9erV05o1a+xDoWJjYxUdHa169eopPj5en3/+uZ555hl99NFHNZbkVfaevfPOOzV+/Hi9+OKLeuedd2Q2m+2TzS+nffv2+uc//6n27ds7PKy9bt262rRpkz7++GP7e+i+++7Ttddeq9dee00ffPCBpk6dqmuvvVZms/lqLvmyVq5cKYPBoIcfftj+ey4rK9OXX36pOXPmVPq8qKgoPfLII1q8eLEsFotuvvlmffXVV2rUqJHatWund999V++8846mTZumhg0bqmfPnmrcuLFPz+P2FLRttG20bZ7TttGuXYx27VeXLYYunb3wcywWi/Lz82WxWHTddddVyHTh27KystSnTx/17t3bftfyfFFRUTp+/Lh27tx50WM5OTnauHGjpkyZosmTJ2vYsGG1ETKuUnZ2tjZu3Ki2bdsqIyND06dPV3R0tP7+97+7OjSnGjZsmLZu3apNmzbVWBLqaebOnavw8HAdO3ZMr776qurUqaMvvvhCgYGBrg4NqFG0bb7HF9s22jXva9cc6sm7MJErKCjQiBEjFBsb65Sg4HsWL16s2bNnKzIy0q2Wn0XVLBaL5s2bp6ysLDVp0kQDBw60L00N77Z27Vq98847CggIUJcuXTRhwgSPbQgBZ6Ft80y0bb7J29o1h3ryLuXNN9/UmjVrtGbNmpqOCQAAAABwhRzqyXv33XcrbP/3v//V0qVLZbVanRIUAAAAAODKOJzkXbjwis1mU0xMjFOCAgAAAABcGYeSvJ49e1bYrl+/vrp06eKyqu4RERFq1aqVS84NAPBM2dnZ9oK37oZ2DQBQXVW1aw4leQsXLqzRgK5Wq1atPLoODwCg9iUkJLg6hErRrgEAqquqds3PkRfYtm2bUlNT7RXhy8vLlZqaai+6CAAAAABwDw715E2ZMkVNmza1Fz739/fX4sWLVVBQw9PRsgAAIABJREFUcFGxUAAAAACA6zjUk5edna3Q0NAK+9q1a+fWVd4BAAAAwBc51JPXqFEjff/99xX2/fDDD2rYsKFTggKA2nTmzBn98ssvKi8vd3UoTuPv768mTZooICDA1aEAAJzMZrOpqKhIp06d0hWWxHZ7BoNBDRo0UIMGDVwdiltyKMkLDw/XN998owcffFDdunXTrl27tHv3bt1xxx3Ojg8AnObUqVNatWqV9uzZowYNGsjf36GPRI9UXl6uEydOqEOHDurXr5/q1avn6pAAAE7w7bff6ptvvtHp06dVv379i8qgeYtziWzTpk3Vu3dvdejQwdUhuRWHvtE88cQT2rRpk3bs2KEdO3bIZrPJaDTqySefdHZ8AOAU5eXlWrBggUJCQjRmzBjVr1/f1SE5XUlJidavX68FCxZoxIgRXp3UAoAv2r59u77++mslJCSodevWXpvgnWO1WrV//34tXbpUgwYNumh6mS9zaE7eLbfcotmzZ8tsNqtNmzb/v707j4uq3v84/h6GxX3XwVJJE1FTc0klszC8XldyjSzTNK28WXZL8+q9/qxMrduiecsyyuuSFeGe4Vaa2uaSqVRU7guaqKCBogIz/P7wOokoDMhhZs68no+HD5kzZ+Z8DrN8+Jzvpttvv13vvPOOWrZsaXR8AGCIXbt2KSgoSF27dvWJAk+6uMZp165dFRgYqN27d7s7HABAMcrJydHGjRvVv39/1a5d2/QFniT5+fkpNDRU3bp109dff+3ucDyKy5dx77rrLt11111GxgIAJWb37t1q3LixTyTBy1ksFt1yyy3avXu3GjVq5O5wAADFJDk5WQEBAbLZbO4OpcQ1aNBAS5Ys0YULFxQUFOTucDyCSy15r732moYPH55rnbzHH39cr732mqHBAYBRMjIyVKFCBXeH4RYVKlTQ2bNn3R0GAKAY+XJe8/f3V5kyZXTu3Dl3h+IxXCrylixZolKlSslqtUq6+IssXbq0li5damhwAGCUnJwcn2vFu8RisZh2tjUA8FW+nNckctuVXCry0tPTVaZMmVzbgoKClJ6ebkhQAOAu7777rvr27auWLVsqPDxcw4cP165du3LtM3DgQIWFhSksLExNmjRR+/btNXToUC1btsylBJOZman3339fvXr10q233qo2bdooOjpasbGxyszMlCSNHTtWYWFhmjFjRq7Hbt68WWFhYUpNTZUkJSUlKSwsTG3bts3znTxw4EBNnDjxen4dAAAv9+GHHyoqKkotW7ZUy5Ytdd9992n9+vW59iGvmY9LRV7NmjW1du1aHTx4UJJ08OBBrVu3TsHBwYYGBwAlbcuWLXrggQcUGxuruXPnymq1asiQITp9+nSu/fr06aOvv/5aX3zxhd5++221aNFCzz33nEaMGOHs2n41mZmZGjp0qGbOnKk+ffro448/1qJFizR48GAtXrxY27dvd+4bFBSkWbNmORNffs6dO6eYmJiinzgAwJRsNptGjx6tJUuWaNGiRQoPD9eIESP066+/5tqPvGYuLk28cuedd2r+/Pnq2rWrKleurFOnTiknJ0f33HOP0fEBQImaNWtWrtuvvPKKbrvtNv3www+KjIx0bi9durSqV68uSQoODlazZs106623atiwYVq6dKn69u171eefO3eutm7dqgULFqhp06bO7bVr11aXLl2UkZHh3Na2bVsdO3ZMb7/9tsaPH59v3AMHDtS8efP04IMP+uSgewDA1f3lL3/Jdfvpp5/Wxx9/rB07dqhhw4bO7eQ1c3GpJe/xxx9XnTp15HA4lJKSIofDoTp16uhvf/ub0fEBgFudPXtWDofDpcHsd955pxo0aKA1a9Zcc5/ly5erXbt2uRLhJX5+fipXrlyu26NHj1ZsbKwOHTqU77G7dOmiBg0aaPr06QXGWVRJSUnq2rWrxo8fr+7du+vhhx/W+fPnNW/ePHXr1k1RUVF6+umnDTs+AOD62O12xcfHKyMjQy1atChwf/Ka9+Y1l1ryqlSpok8//VRr1qzR0aNHFRwcrIyMDA0dOlSLFy82OkavkZSUpEceeUStWrXS9u3bZbPZ9PbbbysuLk6xsbGyWq2qX7++pk2b5u5QUUx4zc1v8uTJatSokUvJUJLq16+v33777Zr3Hzx4UG3atHH5+BEREWrRooWmTZtW4Pvo2Wef1eDBgzVkyBCFhoa6fIzCOHjwoKZOnapJkybpqaee0urVqxUTE6N169YpMDBQaWlphhwX7sP3nO/hNTef3377Tf3799eFCxdUpkwZvfXWWwoLC3PpseQ178xrLrXkSVKpUqV0ww036MCBA3rhhRf04osv6pdffjEyNq908OBBDRgwQPHx8SpfvrzzjbJ06VItX75cL7zwgrtDRDHjNTevl156Sdu2bdObb77pnF24IAXNblaUmb+effZZrVq1Sj/99FO++7Vp00bt27fX66+/XuhjuKpWrVrO9fVuueUWHTlyRGFhYRo9erSWLVvm8u8J3oXvOd/Da24udevW1dKlSxUXF6f7779f//jHP/JMKnYt5DXvzGsFFnmpqamaNWuWunbtqoEDB2rZsmU6d+6cLBZLnj6+MO8bBdfGa25OU6ZMUXx8vObOnavatWu7/Li9e/fmu/9NN92kffv2FSqWZs2a6a9//ateffXVAvcdNWqUNmzYoO+//75Qx3BVYGCg82er1Sq73a6YmBg98MADSkxMVL9+/ZSdnW3IseE+fM/5Hl5zcwkMDFRISIiaNGmiUaNGqVGjRpozZ45LjyWveWdey7fIGzlypCIiIvTaa69p//79ysnJUd26dSVJkZGRevPNN0skSG9i1jdKQczcp7kgvvqam9mkSZOcBd7NN9/s8uO++uor7dq1S507d77mPj169NC3336rH3/8Mc99DodDZ86cuerjnnnmGW3btk1fffVVvjGEhYWpV69eLiXO4uBwOPT7778rPDxco0ePVnp6eq5B9jAHX/ye8+W8Jvnma+5LHA6Hc2mD/JDXvDev5Tsmb82aNbJYLLrxxhvVrVs3devWTQ0bNsw1Ew/yd/kbpVWrVs7Brq5M4uBtzNqnubB86TU3oxdeeEHLli3TjBkzVKFCBZ04cUKSVKZMGZUtW9a537lz53TixAnZ7XYdP35cGzdu1Pvvv6+OHTvmO/Pw4MGDtWHDBj388MN68skn1bp1a5UrV06//PKL/vvf/+rpp59W27Zt8zwuJCRE0dHRmjdvXoHnMHLkSGdCNmoMwyUOh0PPPvuszpw5o5ycHA0aNIj3ug/wle858tqffOU1N6PXXntNHTp0UHBwsM6ePavPPvtMW7Zs0bvvvptrP/LaRWbJay5NvJKRkaH09PRrVuK4NrO8UVyRX9eOjh07+kz3Xl96zc3oo48+knQxaV3uiSee0JNPPum8vXjxYi1evFgBAQGqWLGiGjZsqOeff149e/bMd+xCYGCgZs+erTlz5mjhwoV67bXXFBQUpLp166pPnz75TvAyYsQILVmypMBzqFmzpgYOHKj333+/wH0Lo1atWvrss8+ct4cOHVqszw/v4Svfc+S1P/nKa25GJ0+e1LPPPqsTJ06ofPnyCgsL03vvvac777wz137kNXPlNUtOPqMlx40bp1WrVjnH4ElSjRo1dPz4cUVERGjmzJklFujl+vTpw6yeHiYpKUnDhw93flBmzZqljIwMPf7449q6dau+/PJLbdy4UcuXL5e/v0vXFgBDffTRR2rVqpXLs4uZyW+//aZt27bpgQcecHcoJcqTc4cnx+aryGvwNnv37tU333yjQYMGuTsUt3jjjTf00EMPqXLlyu4OpcTklzvy/VZ66aWXNH78eH322WdasGCBfvrpJyUnJ8tisWjDhg0aPHiwy4M2zWDx4sVatWqV4cfp0qWL+vTpY/hxjGaGrh285vAVvNd9E6974ZDXXGeW1xzey9ff6wXOrlm2bFndd999WrhwoT799FM9+OCDqlChgnJycrR58+aSiNFjrFq1yuXpZotq165dJfKGLAmXunZERUWpd+/eXtm1g9fcvCwWS5GmfTaDq02HzXvdN/G6Fw55zTVmes29iS/nNSlvbvP193qh+hc0aNBA48eP15gxY7R69WotXLjQqLg8VoMGDRQTE2PY8z/66KOGPbeRzNynmdfcnMqUKeNTkyZcLi0tTWXKlMmznfe6b+J1vzryWtF562vu7Xw5r2VnZ+vcuXMqXbp0ru2+/F4vUifywMBARUVFKSoqqrjjgQfy9eZumFNoaKi2bt2q1q1b5zug3GxycnKUmJioNm3auDsUwK3IbTAbm82mzMxMHT9+XDVq1HB3OCVq9+7dqlmzpoKCgtwdiscosLsm4OvN3bg6b19DKjQ0VOfOndPq1au9cv2bosjIyNDq1at1/vx5w6egBjwduQ1X8va8ZrFYdOeddyo2NlZHjhzxia6bDodDe/bsUXx8vNq3b+/ucDwK00HBJb7c3I1r8+Y1pAICAjRo0CCtWLFC06dPV4UKFRQQEODusAyTlZWltLQ0hYaGatCgQXnONb81kIpLSRwDKAxyG67kzXlNktq0aSOr1apFixYpKytL5cqVM21vlZycHP3xxx+qWLGioqKi8ly89PW8RpFXCL7+ZvFFvOb58/Y1pMqUKaN+/fopKytLqampys7OdndIhvH391eVKlWuWcj26NHD8BhK4hgoHL7jfA+vef68Pa9JUqtWrdSyZUudPn1a586dM22LnsViUbly5a45+ZGv5zWKvELw9TeLL+I1z19gYKDzZ6vVqgsXLigmJsa5htTMmTO9Yg2pgIAA2Ww2l/ZNSkrSI488olatWmn79u2y2Wx6++23FRcXp9jYWFmtVtWvX1/Tpk0zOGrg+vEd53t4zfNnlrxmsVhUuXJll9aMI6+Zk6Fj8saNG6fbb7/9mh/2nJwcTZo0SZ06dVJUVJR+/vlnI8MBYLDL15AaPXq00tPTTTne7eDBgxowYIDi4+NVvnx5Z3eepUuXavny5XrhhRfcHSIMRG4DfAd5jbzmrQy9DNGnTx89+OCD+sc//nHV+zdu3KgDBw5ozZo12rlzp55//nktWLDAyJBQBHTtgKsurSF15swZ5eTkeOUaUq4wQ3ceFB25zRzIbXAFeY285q0MLfJat26tpKSka96/du1a9erVSxaLRc2bN1daWppPTvvq6ejagasx8xpSBTFLdx4UDbnNHMhtuBJ57SLymjm49ZVKTk5WcHCw83ZwcLCSk5NJhICbsX5U4VzenadVq1aKj49XRkaGKa/2omDkNsDzkNcKh7zm/SjHAeRxaf2oBg0aGHaMS+tTmSUZ+kJ3HgDwVuS1wiGveT+3Fnk2m03Hjh1z3j527JjLM9wBMBbrR12dL3fngWvIbYBnIq9dHXnNnNxa5EVGRmr+/Pnq3r27du7cqfLly9OdBYDb0J0HxYHcBsCTkNt8k6FF3jPPPKMtW7bo1KlTuuuuu/Tkk086Fxu+//77FRERoQ0bNqhTp04qXbq0pkyZYmQ4AJAvuvPAFeQ2AN6E3OabDC3ypk6dmu/9FotFzz33nJEhAECh0J0HBSG3AfA25Dbfw8QrAPJg/SgAgJmQ1+BrKPIA5MH6UQAAMyGvwdf4uTsAAAAAAEDxoSUPAP6H7jwAALMht/kmijwA+B+68wAAzIbc5pvorgkAAAAAJkKRBwAAAAAmQpEHAAAAACZCkQcAAAAAJsLEKwAAoNht2rRJc+bMUWpqqtLT01W+fHlVqVJFgwcPVnh4uLvDAwBToyUPAAAUu7i4OO3evVspKSnKzMxUSkqKdu/erbi4OHeHBgCmR5EHAACKXXR0tEJDQ2W1WiVJVqtVoaGhio6OdnNkAGB+dNcEAADFLjw8XOHh4Ro0aJCOHDmi4OBgzZw5091hAYBPoCUPAAAAAEyEIg8AAAAATIQiDwAAAABMhDF5AAD4GJY3AABzoyUPAAAfw/IGAGBuFHkAAPgYljcAAHOju6aHuLLrTLVq1TRixAi6zQAAip0vL29AvgXgC2jJ8xBXdp05evQo3WYAAChm5FsAvoAiz0Nc2XUmICCAbjMAABQz8i0AX0CR5yHCw8M1c+ZMBQcHS5Jq1KhB1xEAgEfKzLZ77THItwB8AWPyAABAoQT6W9V18lKX9s1MPStJOpJ61uXHSNLKf/UqUmwAAFryAAAAAMBUKPIAAAAAwEQo8gzmzeMWAAAAAHgfxuQZrDDjFqSijV0wctzCpk2bNGPGDJ08eVLly5dXlSpVNHjwYAapA4CHsjscsvpxDReA7+LvV4o8FCAuLk5Hjx6VJKWkpCglJUVxcXE+9SEBAG9i9fPTZ9v3u7Tv2QvZzv9dfYwk9WhRt0ixAUBJ4O9XijwUIDo6WomJicrKypLValW9evVYTwgAAC+zadMmzZkzR6mpqUpPT/fZ1g34Bv5+pchzyZVfjNWqVdOIESN84ksxPDxcNWrU0JEjRxQcHKyZM2e6OyQAAFBIcXFx2r17t/O2r7ZuwDfw9ysTr7jk0hdjSkqKMjMzdfToUcXFxbk7LAAA8D92h8MUxzBKdHS0QkNDZbVaJUlWq1WhoaE+17oB+Apa8lwQHR2tjIwM7du3T3a7XQEBAXwpAgCQD8fJ/bLv3yydT7u44XyasrbGylq3rfyqFf+YvsKMRZSKNh7Rm8cihoeHKzw8XIMGDfLp1g3AV1DkueDKL8YaNWoUe9eGvMkwXY6T+w1JhAAAGM1+aLty0k/8uSHHoZz0E7If2k5uQ4ny1ZkWfXm4ESjyPEbeZGgnEQIAvJa1TgvZ7ZnKuZAhZZ+X/EvJElRG1jot3B0afIyvzrR45TjMS8ONzH7eJcXTLx5Q5HmIPMkwqByJEADgtfyq1eVCJTyCr860yHAjY3n6xQPDi7yNGzdq8uTJcjgcuvfee/Xoo4/muv/IkSP65z//qdTUVFWqVEmvvvqqgoODjQ7L45AMAcA7kNdQELrJeRZfnWmxJIYb+TJPv3hgaJFnt9s1ceJEzZ49WzabTf369VNkZKTq16/v3Off//63evXqpd69e+u7777T66+/rldffdXIsAAA/8PaWYVDXoMr6CYHGMPucMjqZ+ziAK4ew9MvHhha5CUkJCgkJES1a9eWJHXv3l1r167NlQz37t2rcePGSbr4yxoxYoSRIeF/POlDAsB9WDurcMyS1/YlJui71UuVlnpSkpSWelIfTpuo2zv3Ur3Gzdwcnfejm5zv8fTxWWZRmFl0izKDruTds+heztAiLzk5OVcXFZvNpoSEhFz7NGzYUGvWrNFDDz2kzz//XGfPntWpU6dUuXJlI0PzeXxIAEh5/xj1xC4nnsQseW3b+lU6fuSQ87bDYdfxI4e0bcNqirxiQDc53+Pp47OMQtdkz+X2ZpYxY8Zo69at6tWrl7Zs2SKbzeZcqBMAYKzw8HDNnDnTWbhc6nJCgi46b8hrrTp0UY0b66hshUryDwhQ2QqVVOPGOmoV0dndoRluX2KCPpw20dmKmX4qRfsSEwp4FJC/6OhoBQQESPKtheYv9QZJSUlRZmams2sy3M/QljybzaZjx445bycnJ8tms+XZ56233pIknT17VmvWrFGFChWMDAsAgCIxS16r17iZz7bYXdmKabdn04KJ6+bp47OMQtdkz2VoS17Tpk114MABHT58WJmZmYqPj1dkZGSufVJTU+VwOCRJMTEx6tu3r5Eh5WL/33EBAHCFp+c1FOzKVsyKVat7dQtmSfwt4+oxNm3apOHDhys6Olpdu3ZVdHS0hg8frk2bNhkcIdzlyt4gdE32HIa25Pn7+2vChAkaNmyY7Ha7+vbtq9DQUE2fPl1NmjRRx44dtWXLFk2dOlUWi0W33XabnnvuOSNDyqUw49Kkoo1NY1xawRisDMBbeHpeQ8HM1orpSX/LMJET4DkMXycvIiJCERERubY99dRTzp+7dOmiLl26GB0GPJivDlYG4J3Ia8DVMZETPMW+xASln0qR5LuzB7t94hXAVwcrAwBgJkzkBE+xbf0q2e0XW60vnz3YlxjekgcUxFcHKwMAABQWax0XrFWHLkr/45TOpp1WUOmyKlu+glePvS0Kijzki+ZuAAAAz+FJ4zA9ldnG3haF95boKBE0dwMAAADehZY85IvmbsA70Z0HgNnxPQdcG0Ue8kVzt2/ZtGmT5syZo9TUVKWnp7OkhRejOw8AsyvM91xRvuMkvud8UWa2XYH+Vq8/BkWeC/YlJui71UuVlnpSkpR+KkX7EhMofmA6rHEEAAB8WaC/VV0nL3Vp38zUs5KkI6lnXX6MJK38V68ixVYYtD+7YNv6VTp+5JAcDrskyW7PZlwaTCk6OlqhoaGyWi9eXWJJCwD4U2a23RTHAGB+tOS5oFWHLsq8cF5n09N04dxZla1QiXFpMKXw8HCFh4dr0KBBLGmBYkM3YJhFYa7wS0W7yl8SV/gBmB9FngsYlwaY36ZNmzRjxgydPHmSIqSY0Q0YcL8rh56wLBLyY5Zxab6MIg9AifLUVp24uDgdPXpUEkVIcYuOjlZGRob27dsnu90uq9WqevXq0Q0YKEGXhp5ccvmySBR51+arxQ6t1t6PIg9AifLUVp3o6GglJiYqKyuLIqSY0Q0YcL8rh56wLJJrvHkSDiYO9G0UeQBKlKe26oSHh6tGjRoUIQBMiaEnvufK1ttLEwfyPvANzK4JoESFh4dr5syZCg4OliRnQUW3SPfYl5igD6dNzDNOZ19igpsjAwBcj1YduqjGjXVUtkIl+QcEqGLV6rTc+hBa8mAYX+3HDngTxukAgDnReuvbKPJgGG/uxw74CsbpAABgPhR5AODDuNILoCD0zAG8D0UeAAAArsnbp9Pfl5ig9FMpklgfEL6DiVcAAABgWtvWr5Ldni0p97hjwMxoyQOu4soFu6tVq6YRI0YU6wyQJXGMy9kdDln9jL2uUxLHAACgMFp16KL0P07pbNppxh3DZ1DkAVdx5YLdR48eLfbFukviGJez+vnps+37Xdr37IVs5/+uPkaSerSoW6TYAMBMHCf3y75/s3Q+7eKG8+lynNwvv2p8R7oDY4/hiyjygKu4csHugICAYl+suySOAc+yadMmzZgxQydPnlT58uVVpUoVDR48mDUCAZOxH9qunPQTf27Isct+aDtFHmAijpP7pfPpF2+cT1PW1lhZ67b1mM85/aqAq7hywe4aNWoU+x/iJXEMeJa4uDgdPXpUmZmZSklJ0e7duxUXF+fusAAUM2udFrKUry4FlpX8rFLpirLWaeHusIBi5zi5X1lbY/O0WvsC+6HtUo794o0ch3LST1zc5iFoyQOAEhIdHa3ExERlZWXJarWqXr16tN4CJuRXra7HXM0HjOTLrdbWOi2UfeGMdOGM5F9KlqAyHnUxhyIPAEpIeHi4atSooSNHjig4OFgzZ850d0gAABSZtU4L2e2ZyrmQIWWfl4LKeVShYyS/anUV6MHFLEUeAKBIWCAZAHwbrdaeiyIPAFAk3r5AMgAUF0+fhAO+h4lXAJia3eEwxTEAAJ7L0yfhgO+hJQ9ux9UvGIn1AQEARvP0STjgeyjy4HbXuvpFkQcAgPfIuwi871y49fRJOOB76K4Jt7PWaSGVrnhxLaHAsrKUr87VLzfZl5igD6dNVFrqSUlSWupJfThtovYlJrj0+Mxsu5HhldgxAACF55xOP+d/Xdjptgi4DS15cLuSvPpldzhk9TP22kZJHMMo29av0vEjh5y3HQ67jh85pG0bVqte42YFPr4wE3EUZRIOiYk4AMBT5ZlOn26LgNtQ5MGnFGZ8llS0MVrePD6rVYcuyrxwXmfT03Th3FkFlS6rsuUrqFVEZ3eHBgDwcEynD3gOijwATvUaN3OpxQ4AAACeyzv7lAEAAAAArooiDwAAAABMhCIPAAAAAEzE8CJv48aN6ty5szp16qSYmJg89x89elQDBw5Ur169FBUVpQ0bNhgdEgAARUZeAwB4OkMnXrHb7Zo4caJmz54tm82mfv36KTIyUvXr13fu884776hr16564IEHtGfPHj366KNat26dkWEBhsrMtivQ3+r1xwCQF3kNAOANDC3yEhISFBISotq1a0uSunfvrrVr1+ZKhhaLRWfOnJEkpaenq0aNGkaGBBiuMGvFSUVbL4614gD3IK8BALyBoUVecnKygoODnbdtNpsSEhJy7fPEE09o6NChmj9/vs6dO6fZs2cbGRIAAEVGXgMAeAO3T7wSHx+v3r17a+PGjYqJidGYMWPkcDjcHRYAoJg4Tu5X1tZY6XzaxQ3n05S1NVaOk/vdG5hByGsAAHcztMiz2Ww6duyY83ZycrJsNluufRYuXKiuXbtKklq0aKELFy7o1KlTRoYFFGhfYoI+nDZRaaknJUnpp1K0LzGhgEfBV2Vm201xDKPYD21XTvoJKed/hU6OQznpJ2Q/tN29gRUBeQ0A4A0M7a7ZtGlTHThwQIcPH5bNZlN8fLxef/31XPvUrFlT3333nfr06aO9e/fqwoULqlKlipFhAQXatn6Vjh855Lxtt2dr24bVqte4mRujgqcqzDjMoozBlLx7HKa1TgvZ7ZnKuZAhZZ+X/EvJElRG1jot3B1aoZHXAADewNAiz9/fXxMmTNCwYcNkt9vVt29fhYaGavr06WrSpIk6duyosWPHavz48ZozZ44sFotefvllWSwWI8MCCtSqQxdlXjivs+lpunDurMpWqKRWEZ3dHRbglfyq1ZVftbruDqNYkNcAAN7A0CJPkiIiIhQREZFr21NPPeX8uX79+oqNjTU6DKBQ6jVuRqudj9mXmKD0UymSpLTUk/pw2kTd3rkX7wPkQV4DAHg6t0+8AgCeYNv6VbLbsyVJDoddx48c0rYNq90cFQAAQOEZ3pIHAN6gVYcuSv/jlM6mnVZQ6bIqW74CXXQBAIBXosgDANFFFwAAmAfdNQEAAADARCgXotYeAAAfb0lEQVTyAJQoX1sYGwAAoKRR5AEoUWZaGBsAAMATMSYPQIky08LYAAAAnogiD0CJMtPC2AAAAJ6I7poAAAAAYCIUeQAAAABgIhR5AAAAAGAiFHmAm+RdSiCdZQQAAABw3SjyADfJu5SAnWUEAAAAcN2YXRNwkzxLCQSVYxkBAAAAXDeKPMBNWEoAAAAARqC7JgAAAACYCEUeAJQQx8n90vn0izfOpylrayyT7QAAgGJHkQcAJcR+aLuUY794I8ehnPQTTLYDAACKHWPyAKCEWOu0UPaFM9KFM5J/KVmCyjDZDgAAKHYUeQBQQvyq1VUgk+0AAACD0V0TAAAAAEyEIg8AAAAATIQiDwAAAABMhCIPAAAAAEyEIg8AAAAATIQiDwAAAABMhCIPAAAAAEyEIg8AAAAATIQiDwAAAABMhCIPAAAAAEyEIg8AAAAATIQiDwAAAABMhCIPAAAAAEyEIg8AAAAATIQiDwAAAABMhCIPAAAAAEzE3+gDbNy4UZMnT5bD4dC9996rRx99NNf9U6ZM0ebNmyVJ58+fV0pKir7//nujwwIAoEjIawAAT2dokWe32zVx4kTNnj1bNptN/fr1U2RkpOrXr+/c55///Kfz5w8++ECJiYlGhgQAQJGR1wAA3sDQ7poJCQkKCQlR7dq1FRgYqO7du2vt2rXX3D8+Pl49evQwMiQAAIqMvAYA8AaGFnnJyckKDg523rbZbEpOTr7qvkeOHFFSUpLCw8ONDAkAgCIjrwEAvIHHTLwSHx+vzp07y2q1ujsUAACuG3kNAOAuhhZ5NptNx44dc95OTk6WzWa76r4rVqxQ9+7djQwHAIDrQl4DAHgDQ4u8pk2b6sCBAzp8+LAyMzMVHx+vyMjIPPvt3btXaWlpatGihZHhAABwXchrAABvYOjsmv7+/powYYKGDRsmu92uvn37KjQ0VNOnT1eTJk3UsWNHSRevdnbr1k0Wi8XIcAAAuC7kNQCANzB8nbyIiAhFRETk2vbUU0/luv3kk08aHQYAAMWCvAYA8HQeM/EKAAAAAOD6UeQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiVDkAQAAAICJUOQBAAAAgIlQ5AEAAACAiRhe5G3cuFGdO3dWp06dFBMTc9V9VqxYoW7duql79+4aNWqU0SEBAFBk5DUAgKfzN/LJ7Xa7Jk6cqNmzZ8tms6lfv36KjIxU/fr1nfscOHBAMTEx+vjjj1WxYkWlpKQYGRIAAEVGXgMAeANDW/ISEhIUEhKi2rVrKzAwUN27d9fatWtz7RMXF6cBAwaoYsWKkqSqVasaGRIAAEVGXgMAeANDi7zk5GQFBwc7b9tsNiUnJ+fa58CBA9q/f7/69++v6Ohobdy40ciQAAAoMvIaAMAbGNpd0xV2u10HDx7UBx98oGPHjunBBx/U8uXLVaFCBXeHBgBAoZHXAADuZmhLns1m07Fjx5y3k5OTZbPZ8uwTGRmpgIAA1a5dWzfddJMOHDhgZFgAABQJeQ0A4A0MLfKaNm2qAwcO6PDhw8rMzFR8fLwiIyNz7fOXv/xFW7ZskSSlpqbqwIEDql27tpFhAQBQJOQ1AIA3MLS7pr+/vyZMmKBhw4bJbrerb9++Cg0N1fTp09WkSRN17NhRd955p7755ht169ZNVqtVY8aMUeXKlY0MCwCAIiGvAQC8geFj8iIiIhQREZFr21NPPeX82WKxaNy4cRo3bpzRoQAAcN3IawAAT2f4YugAAAAAgJJDkQcAAAAAJuL2JRQAAAAAwAh+OQ5V8TuvAItDFhf2/+WXXzS2o7GTZf3yyy+F2r9UqVKqVauWAgICXH4MRR4AAAAAU6rid141q1dRhUqVZbEUXOZVKhMk6++nDI2pQU3XJ+PKyclRSkqKkpKSVLduXZcfR3dNAAAAAKYUYHG4XOB5IovFoqpVq+r8+fOFehxFHgAAAABTskheW+BdUpT4KfIAAAAAwEQYkwcAAADAJ5QLCpC/Nf92rsKMmcvMtuvAibR894nqEK5e0fdr6OMX11SdNWuWMjIy9OSTT0qSli5dqvfff18Wi0VWq1VRUVEaOnSoyzFcDUUeAAAAAJ/gb/XTZ9v3F9vz9WhR8GQoAQGB+nbjevV74CFVrFQp130bNmzQ3LlzNWvWLNlsNmVmZmrp0qXXHRfdNQEAAADAIFarVV2iemnZwo/z3BcTE6MxY8bIZrNJkgIDAxUdHX3dx6TIAwAAAAADde/VTxs+X62zZ87k2r579241adKk2I9HkQcAAAAABipTtqzu7txNyxfHlcjxKPIAAAAAwGA9+92nz+M/1blz55zb6tevr59++qnYj0WRBwAAAAAGK1+hotrf3VELFy50bnvsscf06quv6sSJE5KkzMxMLViw4LqPxeyaAAAAAHxCtt3h0oyYrsrMthdq/17RD2jF0kXO2xERETp58qSGDBminJwcWSwW9e3b97rjosgDAAAA4BPOXMjK9/5KZYK06/dTxXrMBau+dP5cuUpV7dy5M9f9ffv2LZbC7nJ01wQAAAAAE6HIAwAAAAATocgDAAAAABOhyAMAAAAAE6HIAwAAAAATocgDAAAAABNhCQUAAAAAPqFUgFWlAvIvgRrUrOzy82VcyFJS6pl89+kZ2U4hdW+W3W5X7ZCbNGP6VJUuXVqNGjVSgwYNlJ2dLavVql69emnw4MHy87v+djiKPAAAAAA+oVSAv7pOXlpsz7fyX70K3CcwMEj/mfWBJOm1SRMUGxurIUOGqFSpUlq2bJkkKSUlRaNGjdKZM2c0cuTI646L7poAAAAAUAJuadpcBw8ezLO9atWqevHFF/Xhhx8qJyfnuo9DkQcAAAAABrNnZ2vblu/UoEGDq95fu3Zt2e12paSkXPex6K4JAAAAAAbJzLygkUMHSpJuaXar+vXrZ/gxKfIAAAAAwCCXj8m7eDvwqvsdPnxYVqtVVatWve5j0l0TAAAAANwoNTVVzz33nAYMGCCLxXLdz0dLHgAAAACfcD4r26UZMV2VcSGr6LGcP6+ePXs6l1Do2bOnhgwZUixxUeQBAAAA8Anns+w6n2W/5v2VygRp1++nivWYC1Z9edXtv/zyS7Ee53J01wQAAAAAE6HIAwAAAAATocgDAAAAYEo5UrEsLu5ORYmfIg8AAACAKWXl+Cnt9CmvLfRycnKUkpKiUqVKFepxTLwCAAAAwJRSHaWkE6k6efKkXFmYoHSgv47/kWFoTPbTZQq1f6lSpVSrVq1CPYYiDwAAAIApOSx+OplT5mK/TRf0aFRXz0xeamhMxbmEw7UY3l1z48aN6ty5szp16qSYmJg89y9evFjh4eHq2bOnevbsqQULFhgdEgAARUZeAwB4OkNb8ux2uyZOnKjZs2fLZrOpX79+ioyMVP369XPt161bN02YMMHIUAAAuG7kNQCANzC0JS8hIUEhISGqXbu2AgMD1b17d61du9bIQwIAYBjyGgDAG1hyDJxqZtWqVfrqq680efJkSdLSpUuVkJCQ6+rm4sWLNXXqVFWuXFl169bVuHHjVLNmzXyft23btrrxxhuNChsAYEJHjhzR5s2br+s5yGsAAE+RX15z+8Qrd999t3r06KHAwEDFxsbqH//4h+bNm5fvY643SQMAYBTyGgDA3Qztrmmz2XTs2DHn7eTkZNlstlz7VK5cWYGBgZKke++9Vz///LORIQEAUGTkNQCANzC0yGvatKkOHDigw4cPKzMzU/Hx8YqMjMy1z/Hjx50/r1u3TjfffLORIQEAUGTkNQCANzC0u6a/v78mTJigYcOGyW63q2/fvgoNDdX06dPVpEkTdezYUR988IHWrVsnq9WqihUr6qWXXjIyJAAAioy8BgDwBoZOvAIAAAAAKFmGL4YOAAAAACg5FHkAAAAAYCIUeddh5cqV6ty5s5o0aaLbb79dgwYNksPhcPnxs2bNUlhYmBYvXmxglIWXlJSksLAwPfbYY85tnhprSbj0+7j832233ebusK7b5efVtGlTRUREaNSoUTp8+LAkKTIyUi1atCjRmJKTk/Xmm2/qiy++MOT5r/czW1RvvvmmwsLCtGrVKsOPVRQffvih872wb98+d4cDNyO3+QZyW8kxMreR166OvOYB6+R5q9TUVD377LOqW7euXnjhBf3xxx9av369GOJoXo0bN9awYcMkSQEBAcXynNnZ2fL3d+/HsHHjxho4cKA2b96spUuXavPmzVq2bJnGjx+vrKysEo3l+PHjeuutt9S7d2/95S9/KdbnLq7PbGFfs+zsbHXu3Fn16tVT8+bNCxt2iVixYoX8/PzkcDi0cuVKjRgxokSO6wnvf+RGbvM95DbjGZXbyGvXRl6jJa/IkpKSlJWVpZo1a6pTp056+OGHNW/ePFmtVo0cOVKtW7dW06ZN1a1bN33++efOx82aNUtt27ZVt27dtGvXLjeeQcGys7OVmpqq1NRUnTt3zrn9Wud36erZ/fffr0cffVS33Xabnn32WWVmZkr68+rZSy+9pLZt26p///46evSodu3apbCwME2ePFnSxS+txo0b6+mnny75k85HlSpVdPvttzv/7d27V0OGDFHLli119913a86cOc59o6Oj1bJlS916663q06ePvv/+e0kXFzwOCwvTI488on79+ik6OtpNZ/OnGjVqqE+fPvr3v/+t6OhonThxQvPnz9ekSZM0duxYSdK2bdsUFRWlpk2bKjw8XM8884wkKS0tTY8++qiaN2+uMWPGqEuXLgoLC5MkLV68WGFhYZo1a5Yk6bHHHlNYWJiSkpKUkpKihx56SC1atFDLli117733KjU1Vf369ZMkLVmypNivruf3mQ0LC1OPHj0kSatWrVJYWJjefPNNSdLAgQMVFhamSZMmqW3btlq/fr1z26uvvqp27dqpR48e+vXXXyX9eXVz3Lhx6tixo1555RWtXr1azzzzjHbs2KFz585p5MiRatWqlZo3b66ePXtq9+7dkqSFCxeqc+fOat68ufr3718i66slJyfrhx9+UNeuXVWjRg2tXLlS0p+v3zPPPKN77rlHrVu31ty5cyVJOTk5evnll3XbbbcpOjpaI0aMUFhYmHNB7y+//FL33HOPmjdvrnvuuUfffPONJM98/yM3chu5jdzmPbmNvHZ15LWLKPKKqF69eqpcubI2bNigtm3bqk+fPlqwYIGki+soPfvssxo1apQkacyYMbpw4YJ+/fVXvfLKK6pevboeeughffvtt+48hQJ9/fXXzi/9S18M0rXP75KdO3eqTZs2Cg8P16effqrY2FjnfRkZGcrIyFD//v21fft2TZkyRQ0aNNBtt92m5cuXKysrS2vXrpXdblevXr1K7mRdcPnv49FHH9Xjjz+uPXv2aNiwYWrWrJleeuklrVu3TpLUrl07jRs3Tk888YROnDihf/7zn7me67vvvlOnTp00ePBgN5zJtd11112SpJ9++inX9vfff19JSUmaMGGCRowYocqVK0uSZsyYoQ0bNqh79+6qXr269u/f79Jxli9frk2bNmnIkCEaO3asGjVqJLvd7vzjp3Xr1po6dapat25dbOeW32fWFb/88otGjx6tunXrOrcdPHhQDz/8sPbt26dx48bl2v+bb77RI488orvvvjvX9q+//lqrV69WVFSUxo8frzZt2ig7O1ubN2/Wv/71L9WqVUt/+9vfdOrUKQ0fPjzXZ8sIq1atksPhUJcuXfTXv/5Vu3fv1p49e5z3b968Wffdd58sFotef/11ZWZmat26dZo9e7YaNGigqKioXN9l+/fv15NPPqlSpUrp8ccfV2BgoJ544olca8d56vsf5DZyG7lN8p7cRl67OvLaRZ7RnuiFypUrp48//lizZ8/WV199pZ9//lnjx49X9erVtWfPHsXHx+fqDpCUlOS8GvDQQw/p3nvv1e+//6533nnHXadQoFtvvVV///vfJV38wHzyySey2+3XPL+goCBJUvPmzTVs2DAdOnRIn3/+ubZs2aJBgwZJkvz8/PR///d/CgwM1NKlS7VlyxZJ0v33369Ro0bpyy+/1OrVq1WtWjW1b9++hM84f5f/PoKCgvTAAw9IkqZPn+7c59tvv1Xbtm2VmJiomJgY2e12533nz593/tyhQ4dc40I8xaUuHhaLJdf2m266SevXr9fGjRvVuHFj57lv3rxZfn5+eu655xQYGKhPP/0015fetYSEhEi6+KXYqlUrde3aVdWrV9cdd9yhadOmqVatWurevXuxntu1PrM1atRw6fH/93//p4YNG+baNmbMGNWpU0fr1q3Ttm3bdObMGed9w4YNU//+/SXJebVbkmrXri0/Pz/t2LFDQUFBCg8PV8OGDfXKK69Iupgsv/76a+f+e/bs0S233FLk8y7IypUrFRAQoHr16jlbNVasWKFatWpJkvr27asBAwZo3bp1+vrrr5WSkuL8LnviiSfUrl077dixQ5999pmki5+BrKws7dy5Uzt37nQeZ8eOHapYsaIkz33/g9xGbiO3Sd6T28hrV0deu4gir4iysrJ00003aeLEiZKk//znP5oxY4Z+/vlnLV26VO3atdPDDz+s+fPna/369c5uHZfz9DEOlStXVrt27SRdvNojXezmcq3zu5QIL51XYc7vr3/9q6pWrap58+Zpx44dGjBggKxWazGf0fW5/PdxqRtC+/btNXToUOc+1apV06effqoNGzaoR48e6t27t6ZOnaqff/4513vA1S/gknbpS/iWW27JddVr9OjRatWqlXbs2KGFCxcqJiZGGzZsyPe5/PwudhS49MdAWlqa8767775bcXFx+uabb/TVV1/pvffe0+zZs1WhQoXiPiWna31md+/eLT8/P2VnZ0uS0tPTr/r4q71m+b3Xr/UaN2zYUJ9++qnWr1+vrVu3as6cOZo0aZLz/rFjxzq7BTkcDmdSMsLvv/+uHTt2KCcnJ9cfHitXrtQjjzwiSc4EdmmMweV/3F35B9Plhg0bpjvuuMN5++abb9aBAwckee77H+Q2chu5zZtyG3ktL/LanyjyimjPnj0aNWqUunXrphtvvFFbt26VdLG7hySdO3dOBw8e1A8//OB8TNu2bSVJc+fOlcPh8OrZvK52fpfs3LlT77//vnbs2CFJatOmjfM+h8OhF198UVWqVNGxY8fUqVMnSVJgYKD69eund999V5LUu3fvEjiLoqtbt65uuukm/fDDD7r99ttVunRpffvtt87zkaSzZ8/qt99+8/jxKcePH9fixYu1detWLVmyRNWrV9eAAQO0bNky5z7vvvuuAgMDVb9+fdWsWVNJSUk6c+aM2rZtq19++UUvvviiKlSokOtK56Uv8fXr16tSpUq5rn6tWrVKv/76q0JCQhQaGqoffvhBx48fV+3atSVJiYmJ+uyzz9SuXTtVqVKlWM7zWp/ZBg0a6MYbb1RSUpKWL1+u+fPnu/ycr776qpo3b66dO3eqcePGKleuXIGP+f7777V27VrVr19fjRo10oYNG5ScnKwOHTrov//9rz777DOVL19eJ06c0LJlywyduWzVqlXKycnRY489pmbNmkm6OH7iyy+/zHX19kpt27bV3LlzNWPGDO3Zs8fZlUu62J0rICBAn3/+uUJCQpSWlqaVK1fmahWA5yK3kdvIbd6T28hreZHX/kSRV0TVqlVTvXr1FBsbq9OnT6tq1ap68skndccdd6h79+5au3atLBaL2rdvrxUrVki6eKVjzJgxiomJ0fz589WmTRtnU7C38Pf3v+b5XdKiRQtt2bLFOaD5UtO+JJUpU0blypVTbGysmjdvnqs//3333af33ntPoaGheboPeBp/f3+9/fbbmjJlit555x1ZrVY1btxYDRo0UJ06dbRmzRpnf+7WrVt79BiVxMRETZgwQVWrVlWPHj3097//XdWqVcu1j5+fn+bPn6+TJ0+qUqVKGjlypG644QY9/vjj2r9/v1auXKnIyEjdcMMNziuGrVq1Uo8ePbRu3TqVKVNGTZo0cSbD0qVLa82aNc6uUN26dVPnzp1VunRp9ejRQ59//rlGjRqlDz/8sNiKvGt9Zu+66y6NHj1azz//vN544w2Fh4c7B5sXpF69evrvf/+revXq6aWXXnLpMaVKldJ3332njz/+2PkZuv/++1W1alW99NJLeu+99zRx4kRVrVpV4eHh13PKBVqxYoUsFosGDx7s/D1nZmbqyy+/1Pvvv3/Nx0VGRmrIkCFatGiR7Ha7WrZsqa+//loVKlRQ3bp19dZbb+mNN97Q5MmTVb58ebVu3VoVK1bUkSNHDD0fXD9yG7mN3OY9uY28lhd57U+WHE/vVwGvkZSUpI4dO6pDhw7Oq5aXi4yM1KlTp7R9+/Y89x09elRfffWVJkyYoPHjx2vgwIElETKu05EjR/TVV18pJCRECQkJmjp1qjp37qz//Oc/7g7NUAMHDtSWLVv03XffFVsR6m1mz56tsLAwnThxQlOmTFFQUJC++OILBQYGujs0oFiR23yPL+Y28pr58hotefAIixYt0syZMxUREeFR088if3a7XXPmzFFSUpIqVaqknj17OqemhrmtW7dOb7zxhgICAtSkSRONGTPGaxMhYBRym3cit/kms+U1WvIAAAAAwERYJw8AAAAATIQiDwAAAABMhCIPAAAAAEyEIg8AAAAATIQiDwCA/3nzzTcVFhbGVPcAAK/GEgoAALe5tDbTlW688UatW7fOsOO++eabeuutt9SmTRt98MEHzu233nqrBg0apJCQEMOODQCA0SjyAABud/PNN+uOO+5w3q5YsaJb4rjrrrt01113ueXYAAAUF4o8AIDbNWvWTP/617/ybI+MjNSRI0c0fPhwbd68WT///LOaNm2qV155RR9//LHi4uIUFBSkv/3tbxowYIAkyeFwaNGiRfroo4904MABlS1bVk2aNNHf//53NWzYUGPHjtWSJUskSVu2bFFYWJgkae3atVqyZEmeFr7Dhw/rjTfe0LZt23T69GnVqlVLvXr10qBBgxQYGKikpCR17NhRkvT888/ro48+0sGDB9WoUSNNmTJFN998syRp5syZWrRokY4dO6agoCDdeOON6t+/v+6//37Df78AAN9CkQcAcLuEhARNnjzZebtZs2aKiopy3p41a5a6du2q5ORkbdu2Tb1791alSpUUHh6uNWvWaNKkSWrfvr1CQkI0depUvffeewoMDFS3bt10+PBhffnll/r2228VFxenO+64Q/v27dPOnTtls9nUuXNnSVK5cuXyxJWcnKx+/frp9OnTatSokdq1a6fVq1fr1VdfVWJioqZOnZpr/1deeUVdunTRmTNntGPHDr3wwguaN2+eNm3apGnTpqlcuXLq2bOnsrOztWfPHv34448UeQCAYkeRBwBwu71792rv3r3O2717985V5PXv31/jx4/XvHnzNHnyZKWlpemDDz5Qw4YNdfvttys1NVU///yzatas6WyBe/rpp/Xwww8rJydHUVFR2r17tz744ANNnjxZBw4c0M6dOxUSEnLVFsRLFi5cqNOnT6ty5cr65JNPFBQUpDvvvFN///vfFR8frzFjxuTa//nnn1fPnj21atUqPfXUU/rxxx8lSZmZmZKkqlWrqmPHjrrpppsUEhIih8NRbL9DAAAuYXZNAIDb9e7dW7/99pvz38svv5zr/ktdKitUqODcVr9+fUl/tsCdPXtWp06d0vnz5yVJoaGhkiSLxeL8+ffffy9UXJf2DwkJUVBQkCSpQYMGzvuPHj2aa/9bb71VklSpUiVJUkZGhiSpffv2Gjx4sP744w8NHz5cXbp0Udu2bfXJJ58UKh4AAFxBkQcA8HhWq1XSxYLtEn//vJ1RKleurFKlSkmS9uzZI0nKyclx/lyzZs1cz1dQS9ql/Q8dOuRsjdu9e3ee+6+M6fI4JSk7O1tjx47V5s2btXHjRk2ZMkVpaWl6+eWXZbfb840BAIDCorsmAMDtrhyTJynfbpTXEhgYqAEDBmjWrFmaNm2afv31VyUlJWnXrl3O+yTphhtukCT9+OOPmjBhgipXrqynn346z/P17dtXc+fOVWpqqvr376+GDRtqzZo1kqSuXbuqZs2aSkpKKjCu7du3a/To0WrRooWqVq2qffv2SZLKly8vPz+utwIAihdFHgDA7a4ckycVrciTpNGjR6tOnTqKjY3VmjVrVKZMGXXo0EEjR45U48aNJV0s0L744gt99913+uSTT1SpUqWrFnnBwcFasGCBpk+frm3btik+Pl61atXSI488oiFDhrgcU3BwsMLCwvTDDz/o9OnTKl26tNq0aaOnn346T6sfAADXy5KTk5Pj7iAAAAAAAMWDPiIAAAAAYCIUeQAAAABgIhR5AAAAAGAiFHkAAAAAYCIUeQAAAABgIhR5AAAAAGAiFHkAAAAAYCIUeQAAAABgIhR5AAAAAGAi/w9fJPB+Tj5mUwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1080x720 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 614
        },
        "id": "ESvB_c4lsRpC",
        "outputId": "bef9d8b3-fb7e-46c4-ef9b-ca750da172ec"
      },
      "source": [
        "from itertools import combinations\n",
        "\n",
        "multi_df_new = pd.melt(multi_df, id_vars=['Model', 'Data'], value_vars=['Sad', 'Happy', 'Fear', 'Disgust',\n",
        "                                                                'Surprise', 'Anger'],\n",
        "        var_name='Emotions', value_name='Accuracy')\n",
        "\n",
        "x = \"Data\"\n",
        "y = \"Accuracy\"\n",
        "hue = \"Emotions\"\n",
        "order = [\"NC\", \"PD\"]\n",
        "hue_order = ['Sad', 'Happy', 'Fear', 'Disgust','Surprise', 'Anger']\n",
        "box_pairs=[]\n",
        "box_pairs1 = [(('PD', emot[0]), ('PD', emot[1])) for emot in list(combinations(multi_df_new['Emotions'].unique(), 2))]\n",
        "box_pairs2 = [(('NC', emot[0]), ('NC', emot[1])) for emot in list(combinations(multi_df_new['Emotions'].unique(), 2))]\n",
        "\n",
        "\n",
        "fig = plt.figure(figsize=(15, 10))\n",
        "fig.subplots_adjust(hspace=0.08, wspace=0.1)\n",
        "ax = fig.add_subplot(2, 2, 1)\n",
        "plt.setp(ax, ylim=(0.6, 1.2))\n",
        "#palette = \"ch:start=.2,rot=-.3\"\n",
        "#palette = \"ch:s=.2,r=-.2\"\n",
        "palette = \"dark:#73D4FE_r\"\n",
        "g1 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='ML'], x=x, y=y, hue=hue, order=order,\n",
        "                 ax=ax, capsize=0.03,palette = palette, ci=68, hue_order = hue_order)\n",
        "\n",
        "#add_stat_annotation(g1, data=multi_df_new[multi_df_new['Model']=='ML'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs1,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "#add_stat_annotation(g1, data=multi_df_new[multi_df_new['Model']=='ML'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs2,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "g1.legend_.remove()\n",
        "\n",
        "ax = fig.add_subplot(2, 2, 2)\n",
        "plt.setp(ax, ylim=(0.6, 1.2))\n",
        "g2 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='1D CNN'], x=x, y=y, order=order,\n",
        "                 hue=hue, ax=ax, palette = palette, capsize=0.03, ci=68, hue_order = hue_order)\n",
        "#add_stat_annotation(g2, data=multi_df_new[multi_df_new['Model']=='1D CNN'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs1,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "#\n",
        "#add_stat_annotation(g2, data=multi_df_new[multi_df_new['Model']=='1D CNN'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs2,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "g2.legend_.remove()\n",
        "\n",
        "ax = fig.add_subplot(2, 2, 3)\n",
        "plt.setp(ax, ylim=(0.6, 1.2))\n",
        "g3 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='2D CNN'], x=x, y=y, order=order,\n",
        "                 hue=hue, ax=ax, palette = palette, capsize=0.03, ci=68, hue_order = hue_order)\n",
        "#add_stat_annotation(g3, data=multi_df_new[multi_df_new['Model']=='2D CNN'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs1,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "#add_stat_annotation(g3, data=multi_df_new[multi_df_new['Model']=='2D CNN'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs2,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "g3.legend_.remove()\n",
        "\n",
        "#ax.set_xticklabels(ax.get_xticklabels(), rotation=90)\n",
        "ax = fig.add_subplot(2, 2, 4)\n",
        "plt.setp(ax, ylim=(0.6, 1.2))\n",
        "g4 = sns.barplot(data=multi_df_new[multi_df_new['Model']=='3D CNN'], x=x, y=y, order=order,\n",
        "                 hue=hue, ax=ax, palette = palette, capsize=0.03, ci=68, hue_order = hue_order)\n",
        "#g4.set_ylim(bottom=0.55, top=1.0)\n",
        "g4.legend(loc='lower right', bbox_to_anchor=(1,0))\n",
        "#add_stat_annotation(g4, data=multi_df_new[multi_df_new['Model']=='3D CNN'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs1,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "#add_stat_annotation(g4, data=multi_df_new[multi_df_new['Model']=='3D CNN'], x=x, y=y, hue=hue,\n",
        "#                    box_pairs=box_pairs2,line_offset_to_box=0.2, line_offset=0.1, line_height=0.05,\n",
        "#                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "ax.set_ylabel('')    \n",
        "ax.set_xlabel('')\n",
        "\n",
        "props = dict(boxstyle='round', facecolor='none', edgecolor='black', alpha=0.5)\n",
        "\n",
        "# place a text box in upper left in axes coords\n",
        "g1.text(0.48, 0.96, \"ML\", transform=g1.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "g2.text(0.45, 0.96, \"1D CNN\", transform=g2.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "g3.text(0.45, 0.96, \"2D CNN\", transform=g3.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "g4.text(0.45, 0.96, \"3D CNN\", transform=g4.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "'''\n",
        "g1.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "g2.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "g3.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "g4.set_xticklabels([\"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold')\n",
        "#plt.setp(ax, ylim=(0.5, 1.0))\n",
        "'''\n",
        "fig.text(0.5, 0.08, 'Data', ha='center', fontsize = 13, weight='bold')\n",
        "fig.text(0.09, 0.5, 'Accuracy', va='center', rotation='vertical', fontsize = 13, weight='bold')\n",
        "\n",
        "plt.show()\n",
        "\n",
        "fig.savefig('multi_class.jpg', format='jpg', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAAJVCAYAAABj6TwsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3hU9bn3/8+dSUIICnIIhnLwUFBhFys2usF6USy6Ra2i1XZXqyi1pd3WTa1PN9Wfe2urPyuCT6tetbU8LSCth61Vu/k9WNm1VqXKVoJSRagVKQqBSCKRQyEJmdy/PxKnCUkmM0nWzJo179d1cc3MOsy6o4R7PrO+67vM3QUAAAAAiIaCbBcAAAAAAOg7hDwAAAAAiBBCHgAAAABECCEPAAAAACKEkAcAAAAAEULIAwAAAIAICTTkmdliM9tpZuu7WP9lM3vdzN4ws5fM7JNB1gMAQG/R2wAAYRf0mbylkmYkWf9XSZ9x94mSbpO0KOB6AADoraWitwEAQqwwyDd39xfM7Ogk619q8/J/JI0Ksh4AAHqL3gYACLswXZN3taTfZrsIAAD6EL0NAJBxgZ7JS5WZnaGWRnh6km3mSJojSQMGDPjUCSeckKHqAABRsHbt2lp3L8vU8brrbfQ1AEBvJOtrWQ95ZnaipJ9LOsfdP+hqO3dfpNbrGioqKryysjJDFQIAosDM3s3gsbrtbfQ1AEBvJOtrWR2uaWZjJD0h6Qp3/0s2awEAoC/Q2wAA2RbomTwze1jSNEnDzGybpFskFUmSu98v6WZJQyX9xMwkqcndK4KsCQCA3qC3AQDCLujZNS/tZv1XJX01yBoAAOhL9DYAQNhl/Zo8AN3btWuXtmzZooaGhj5934KCAg0ZMkTHHHOMCgv55wAA0DNNTU3avHmz6urq1NzcnO1yAlNcXKzRo0dr+PDh2S4FSIpPdUCINTQ06LHHHtOOHTs0duxY9e/fX63Dv/pEPB7Xhg0b9MQTT+j888/XhAkT+uy9AQD54Y033tCKFStUXl6u4cOHKxaLZbukQLi7Ghoa9Pzzz2vgwIG69NJLNWDAgGyXBXSKkAeE2BNPPKFBgwbpsssuU0FBcPMk7dixQw8++KAGDhyoUaO4bzMAIDXvvvuuVq5cqdmzZ+vII4/MdjkZ4e76wx/+oAcffFBf+9rX+vTLV6CvhOlm6ADa2Lt3r9577z2dc845gQY8SRoxYoQmT56sdevWBXocAEC0vPbaazr99NPzJuBJkpnpjDPO0IEDB1RdXZ3tcoBOEfKAkHrvvfd01FFHZexaubFjx+rddzN2GzEAQAS8++67Gjt2bLbLyDgzo28i1Ah5QEg1NDSof//+GTte//79+3xiFwBAtDU2NqqkpCTbZWRFSUmJGhsbs10G0ClCHgBJ4poCAECP5Gv/yNefG7mBkAfkoKuuukpmpquvvrrDuu9+97syM33uc5+TJH3ve9/TJz7xiUyXCADIYy+88IIuuOACjRw5UmampUuXdthm2rRpMjOZmYqLizVixAjNmDFDv/rVr+Tu3R6jsbFRCxcu1KRJk1RaWqohQ4Zo8uTJ+tnPfpYYmfJRv7ztttva7fvcc8/JzFRbWytJ2rJli8xMQ4cO1e7duzvUee211/bwvwSQHYQ8IEeNHj1ajz76qP72t78lljU1NWnZsmUaM2ZMFisDAOS7ffv26ROf+ITuueeepJcezJ49Wzt27NDmzZu1fPlyTZkyRV//+td10UUXKR6Pd7lfY2Ojzj77bN1+++2aPXu2XnzxRa1du1bXX3+9lixZotWrVye2LSkp0cKFC1VTU9Nt3fv379f8+fPT+2GBEOIWCkCOOvHEE7V9+3Y9+uijmj17tiRpxYoVKikp0dSpU/XBBx9kuUIAQL4699xzde6550pqOZvWldLSUpWXl0uSRo0apVNOOUWTJ0/WjBkztGzZskR/O9Tdd9+t559/Xq+88ooqKioSy4855hhdcskl2rdvX2LZGWecoW3btum2227Tvffem7TuuXPn6p577tG1116rkSNHpvrjAqHDmTwgh1199dVavHhx4vXixYs1e/ZsrhMAAOSss88+WxMnTtTjjz/e5TYPPvigzjzzzHYB7yMFBQUaOHBgu9fz58/X/fffr3feeSfpsb/whS9o4sSJuvnmm3v+AwAhQMgDcthll12myspKvf3226qurtbTTz+d9BtTAABywYQJE7R58+Yu17/99tsaP358yu937rnn6tOf/rRuuummbrddsGCBHnjgAb355pspvz8QNoQ8IIcNHjxYF110kRYvXqwHHnhA06ZN43o8AEDOc/eko1JSmZjlUHfeeacee+wxrV27Nul2n/nMZ3T22WfrxhtvTPsYQFgQ8oAc95WvfEXLli3T4sWL9ZWvfCXb5QAA0GsbNmzQscce2+X64447Ths3bkzrPU899VRdfPHFmjdvXrfbzp8/XytWrNCqVavSOgYQFoQ8IMdNnz5dxcXFqq2t1YUXXpjtcgAA6JWVK1dq/fr1uuSSS7rc5rLLLtMzzzyjysrKDuuam5u1Z8+eTvf7wQ9+oFWrVunpp59OWsPEiRM1a9aslAIhEEbMrgnkODPT66+/LndXv379Ot2mvr5e69ata7estLRUxx13XCZKBADkmX379mnTpk2SWkLXe++9p3Xr1mnIkCHtLivYv3+/qqur1dTUpB07duipp57SggULNHPmTF1++eVdvv91112nFStW6KyzztL3v/99TZ06VYMGDdJrr72mu+66Sz/4wQ80bdq0DvuNHTtWc+bM0T333NPtz3Drrbcm+iT3m0Wu4UweEAGHH354u5nEDvXOO+9o0qRJ7f5cdtllGawQAJBPKisrE/3mwIEDuuWWWzRp0qQOs1YuWbJEI0aM0LHHHqvzzz9fq1ev1v33368nn3xSsVisy/fv16+ffve73+mGG27QL37xC02ZMkUnn3yyFixYoCuvvFKnnXZal/vefPPNKizs/jzH6NGjNXfuXNXX16f+gwMhYT25cDXbKioqvLPT80CUvPrqq9q6datmzpyZkePt2bNHP//5z3X99ddn5HhAppnZWnfvON96CNDXkKsWLlyoa665RgMGDMh2KRn37LPPqrCwUFOnTs12KchTyfoaZ/IAAAAAIEIIeUBIFRQUqLm5OWPHi8fjKijgnwQAQOoKCgoUj8ezXUZW0DcRZvzNBELqiCOOUE1NTcaOV1NToyOOOCJjxwMA5L5BgwZltFeFCX0TYUbIA0JqzJgx2r17t7Zv3x74sdxda9eu1fjx4wM/FgAgOsaPH6+1a9f26Obkuayurk7vvvuuxo4dm+1SgE5xCwUgpAoKCnTeeefpoYce0tSpUzVu3DiVlpbKzPrsGPF4XO+//75eeeUV7d+/X5MmTeqz9wYARF9FRYU2bNigxx9/XKeccoqGDx+edFbMXObuqq+v1zvvvKMXXnhB06dPV0lJSbbLAjpFyANCbMKECRowYIDWrl2rVatWqaGhoU/fv6CgQEOGDNH48eM1c+ZMFRcX9+n7AwCirV+/frriiiv0yiuv6Omnn1ZdXV1GryfPtOLiYo0ZM0bnnXeexo0bl+1ygC4R8oCQO+qoo3TUUUdluwwAADpVUlKiqVOncisBIES4Jg8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAAREmjIM7PFZrbTzNZ3sf4EM1ttZg1m9p0gawEAoC/Q2wAAYRf0mbylkmYkWb9L0lxJdwVcBwAAfWWp6G0AgBALNOS5+wtqaXZdrd/p7mskHQyyDgAA+gq9DQAQdjlzTZ6ZzTGzSjOrrKmpyXY5AAD0Cn0NABCUnAl57r7I3SvcvaKsrCzb5QAA0Cv0NQBAUHIm5AEAAAAAukfIAwAAAIAIKQzyzc3sYUnTJA0zs22SbpFUJEnufr+ZlUuqlDRQUrOZXSdpgrvvCbIuAAB6it4GAAi7QEOeu1/azfpqSaOCrAEAgL5EbwMAhB3DNQEAAAAgQgh5AAAAABAhhDwAAAAAiBBCHgAAAABECCEPAAAAACKEkAcAAAAAEULIAwAAAIAIIeQBAAAAQIQQ8gAAAAAgQgh5AAAAABAhhDwAAAAAiBBCHgAAAABECCEPAAAAACKEkAcAAAAAEULIAwAAAIAIIeQBAAAAQIQQ8gAAAAAgQgh5AAAAABAhhDwAAAAAiBBCHgAAAABECCEPAAAAACKEkAcAAAAAEULIAwAAAIAIIeQBAAAAQIQQ8gAAAAAgQgh5AAAAABAhhDwAAAAAiBBCHgAAAABECCEPAAAAACKEkAcAAAAAEULIAwAAAIAIIeQBAAAAQIQQ8gAAAAAgQgh5AAAAABAhgYY8M1tsZjvNbH0X683M7jWzTWb2upmdHGQ9AAD0Fr0NABB2QZ/JWyppRpL150ga1/pnjqSfBlwPAAC9tVT0NgBAiAUa8tz9BUm7kmwyU9Iyb/E/ko4wsxFB1gQAQG/Q2wAAYZfta/JGStra5vW21mUAAOQqehsAIKuyHfJSZmZzzKzSzCpramqyXQ4AAL1CXwMABCXbIa9K0ug2r0e1LuvA3Re5e4W7V5SVlWWkOAAAeiCl3kZfAwAEJdshb7mkWa0zkU2WtNvdd2S5JgAAeoPeBgDIqsIg39zMHpY0TdIwM9sm6RZJRZLk7vdLekrSuZI2SdovaXaQ9QAA0Fv0NgBA2AUa8tz90m7Wu6RvBlkDAAB9id4GAAi7bA/XBAAAAAD0IUIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCrNdAAAAABAm8+bNU3V1tSSptrZWTU1NKiws1LBhwyRJ5eXlWrBgQTZLBJIi5AEAAABtVFdXq6qqqt2yeDzeYRkQVoQ8AAAAoI3y8vLE8+rqasXjccViscTytuuBMCLkAQAAAG20HYo5a9YsVVVVqby8XMuWLctiVUDqAp94xcxmmNlbZrbJzG7oZP1RZvZ7M3vdzJ4zs1FB1wQAQE/R1wAAYRfomTwzi0m6T9JZkrZJWmNmy919Q5vN7pK0zN0fMLPPSrpD0hVB1gUAQE/Q18KPCTOQa5L9neXvK3oq6OGap0ra5O6bJcnMHpE0U1LbZjhB0vWtz/8g6TcB1wQAQE/R10KOCTOQa/g7iyAEPVxzpKStbV5va13W1p8kfb71+UWSDjezoYe+kZnNMbNKM6usqakJpFgAALpBXwu58vJyjRw5UiNHjlQsFpMkxWKxxDImzEDYJPs7y99X9FQYJl75jqQfm9lVkl6QVCUpfuhG7r5I0iJJqqio8EwWCABAGuhrWcSEGcg1/J1FEIIOeVWSRrd5Pap1WYK7b1frN55mdpiki939w4DrAgCgJ+hrAIDQC3q45hpJ48zsGDMrlvQlScvbbmBmw8zsozpulLQ44JoAAOgp+hoAIPQCDXnu3iTpWkkrJW2U9Ki7v2lmt5rZBa2bTZP0lpn9RdKRkm4PsiYAAHqKvgYAyAWBX5Pn7k9JeuqQZTe3ef5rSb8Oug4AAPoCfQ0A8k+u3Z4lDBOvAAAAAEBo5dqtLgh5AAAAAJBE29tZVFdXKx6PKxaLJZaH7XYXhDwAAAAASCLXbnUR9OyaAAAAAIAMIuQBAAAAQIQwXBMAAABAzsm1GS8ziZAHAAAAIOfk2oyXmUTIAwAAACIuime9cm3Gy0wi5AEAAAARF8WzXrk242UmEfIAAACQN25/dVda2+9qaE48prPvTScPSes4QeOsV34h5AEAAEBS8iF9uTicD3/HWa/8QsgDAACApGgO6QPyESEPAAAAkpIP6WM4H5A7CHkAAACQxJA+9I0ozuSZawh5AAAAaeJDLNA1hv1mHyEPAAAgTXyIBbrGTJ7ZR8gDAACBiuJZLz7EAl1j2G/2EfIAAECgonjWiw+xAMKMkAcAAALFWS8AyCxCXoqiONQEAIBM4KwXAGQWIS9FURxqAgAAgGBc//hLae9Ts68+8ZjO/j+8+LS0j4VoI+SliKEmAAAAAHIBIS9FDDUBAEQNlyIAQDQR8gAAyFNcigAA0UTIk3T7q7vS2n5XQ3PiMd19bzp5SFrbAwAQFC5FyA2ccQWQLkIeAAB5iksRcgNnXAGki5AHAAAQYpxxBZAuQl6KXlt0m+rraiRJ9bt2Jh5X3zlXklQyuEyT5vxH1uoDAADRxBlXAOki5KWovq5GB2p3tFvmzfEOywAAAAAgmwh5KSoZXJZ43rD7A3k8LovF1G/Q0A7rAQBA37ty4ZK0tn+/bk/iMZ19H/i32WkdBwDChpCXIoZiAgAA5Acu00GuI+QB6LVk03sztTcAINdwmQ5yHSEPQK8xvTcAIEq4TAe5jpAHoNeSTe/N1N4AgFzDUEzkusBDnpnNkHSPpJikn7v7/EPWj5H0gKQjWre5wd2fCrouAH2H6b2RT+hrAICwCzTkmVlM0n2SzpK0TdIaM1vu7hvabPbvkh5195+a2QRJT0k6Osi6AKTm9ld3pb3ProbmxGM6+9908pC0jwVkGn0NAJALgj6Td6qkTe6+WZLM7BFJMyW1bYYuaWDr80GStgdcE4A+lmwWMmYgQ8TQ1wAgYsYePz6t7T925HAVFRVqy5Z309p301sb0y2tx4IOeSMlbW3zepukfzxkm+9J+m8z+1dJAySdGXBNAPoYs5Ahj9DXAAChV5DtAiRdKmmpu4+SdK6kX5pZh7rMbI6ZVZpZZU1NTcaLBNC1ksFl6j9shPoPG6GComJZQUwFRcXqP2wEM5AhH9HXAABZFfSZvCpJo9u8HtW6rK2rJc2QJHdfbWYlkoZJ2tl2I3dfJGmRJFVUVHhQBQNIH8MxkUfoa3ngvGu+m9b2DTtrJUnbd9amve+Kn9yZ1vZAZ65cuCSt7d+v25N4TGffB/5tdlrHQfYEfSZvjaRxZnaMmRVL+pKk5Yds856k6ZJkZuMllUjiK00AQBjR1wAAoRdoyHP3JknXSlopaaNaZht708xuNbMLWjf7X5K+ZmZ/kvSwpKvcnW80AQChQ18DAOSCwO+T13pvoKcOWXZzm+cbJH066DoAAOgL9DUAfemd5UvVuPdDSVLj3rrE48YH71bx4Ufo4xdclcXqkKsCD3lI37x581RdXS1Jqq2tVVNTkwoLCzVs2DBJUnl5ebubTwMAACST7hTxUm5MEx8FjXs/VOPuD9ovbG7uuCzPTJ5xYVrb9z+4TwWStlZtT3vfKCLkhVB1dbWqqtpfxx+PxzssAwAAQG4rPvyIxPODf9sjb47LCmIqGjCw3TogHYS8ECovL088r66uVjweVywWSyxvux7IJ5zlBgBETS4Nx0x39lipd7PPoucIeSHU9kPqrFmzVFVVpfLyci1btiyLVQHZx1luAACA7hHyAOQMznIDAAB0j5CXYdc//lJa29fsq088prPvDy8+La3jALmAs9wAAADdC/pm6AAAAACADOJMXgglu1+KJO6ZAgAAAKBLhLwQ4n4pAAAAAHqKkBdCye6Xcuh6ICry9XpVbgsBAAD6GiEvhBiKCeSPKN4WguAKAEB2EfIAIIuieFuIKAZXAAByCSEPADqRqbNRUbwtRBSDK7qWqaHWUTV5xoVpbd//4D4VSNpatT3tfQHkD0IeAHSCs1E9F8XgCgBALiHkAT3EdUeZl8nbi/TF2ajzrvluWsds2FkrSdq+szbtfVf85M60tgcAANFFyAN6iDM9mZfJ24twNqpzY48fn9b2HztyuIqKCrVly7tp77vprY1pbQ8AAFoQ8oAe4rqjzOP2IkBqbn91V9r77GpoTjz2ZH8AQHgQ8oAe4kxP5kXx9iKNW/8iP9ggSfKDjYnHhs1vSJKsqJ+KRx+XtfoAAEDuIeQByCtXLlyS9j7v1+1JPPZk/2T8YEMi5LVZ2smy3DF82BAVxlraS2FhLPH4sSOHS5Ka4k3aWcuZIgAAgkLIQ+QkmxCFyVAQNlbUL/Hcmw5K7pKZrLCow/pcURgrVFFR+/ZiZh2WAQCAYNBxETlMiIJcEsWhmE3xpsTzWKxAZiZ3Vzze3GE9kKsYag0gzAh5iJxkE6KkOhlKpmYQZPbA8KpZ83vFD+yTJDW1eax+4b8kSbH+h6nslOlZqy/MGIqJbMrU724Uh1oDiA5CHiKHCVHQF+IH9qlp/972C907LgMQKpn63Y3iUGsA0UHIA4BOxPoflnger98v92aZFShWUtphPYDwyNTvLkMxAYQZIQ8AOsFQTCA38bsLAIQ8AAAAIPK41jy/EPKAHuJeYAAAIFdwrXl+IeQBPcS9wAAAQK7gWvPeybUv9/k0ipwxecaFae/T/+A+FUjaWrW9R/snw73AAABRk+yDbNg+xCI9DMXsnVz7cj+cVQE5gEaHXDNv3jxVV1dLkmpra9XU1KTCwkINGzZMUss9JtveggRA/sm1D7JApuTal/v8xgJAnqiurlZVVVW7ZfF4vMMyAPkr2QfZsH2IBTIp177cJ+QBQJ4oLy9PPK+urlY8HlcsFkssb7seQH7KtQ+yADpHyAOAPNF2KOasWbNUVVWl8vJyLVu2LItVAQCipnHrX+QHGyRJfrAx8diw+Q1JkhX1U/Ho47JWXz4g5AEAAADoM36wIRHy2iztZBmCQshDRjDhAwAAQH6won6J5950UHKXzGSFRR3W90ZJ036Zt0x8YvLEY/+DLTd7dytQfWFpnxwr1wQe8sxshqR7JMUk/dzd5x+y/keSzmh9WSppuLsfEXRdyCwmfACCle4tQoK8vUjU0dcAILlMDcU0b1ZBa7hLLNPfA19zawDMR4GGPDOLSbpP0lmStklaY2bL3X3DR9u4+7fbbP+vkiYFWROyI5MTPiT7Viefv9EB0Hv0NQAID7eCRJCzNmHPZYn1+SroM3mnStrk7pslycwekTRT0oYutr9U0i0B14QsyOSED8m+1cnnb3QA9An6GgCEBF/cdy3oeDtS0tY2r7e1LuvAzI6SdIykZwOuCRHnVqBmmZpbv9P56E+zLK+/0QHQJ+hrAIDQC9PEK1+S9Gt3j3e20szmSJojSWPGjMlkXejGedd8N63tG3bWSpK276xNe99U8K0OgJCgrwEAsiLo0xpVkka3eT2qdVlnviTp4a7eyN0XuXuFu1eUlZX1YYkAkB9Kmvar/8F96n9wX4frVfsf3KeSpv1ZrjAn0NcAAKEX9Jm8NZLGmdkxammCX5J02aEbmdkJkgZLWh1wPQCQt5iFrE/Q15BxTBMPIF2Bnslz9yZJ10paKWmjpEfd/U0zu9XMLmiz6ZckPeLu3tn7AAB6L9n1qlyzmhr6GrLhoy9oChJzBrZ8QZNYxhc0AA4R+DV57v6UpKcOWXbzIa+/F3QdyK7GrX+RH2yQJPnBxsRjw+Y3JLXcFDNT91QB8hXf9PcN+hoyjWniAaQrTBOvIML8YEMi5LVZ2skyAADQFl/QAEgXIQ8ZYUX9Es+96aDkLpnJCos6rAcAAADQc4Q8ZARDMQEAAIDMYBA3AAAAAEQIZ/IAAMhTry26TfV1NZKk+l07E4+r75wrSSoZXKZJc/6j18d5Z/lSNe79UJLUuLcu8bjxwbslScWHH6GPX3BVr48DAGhByAMAIE/V19XoQO2Odsu8Od5hWW817v1Qjbs/aL+wubnjMgBAnyDkAQCQp0oGlyWeN+z+QB6Py2Ix9Rs0tMP63ig+/IjE84N/2yNvjssKYioaMLDDegBA7xHyAADIU30xFDMVDMUEgMxi4hUAAAAAiBDO5OWxefPmqbq6WpJUW1urpqYmFRYWatiwYZKk8vJyLViwIJslAgAAAEgTIS+PVVdXq6qqqt2yeDzeYRkAAACA3EHIy2Pl5eWJ59XV1YrH44rFYonlbdcDAAAAyA2EvDzWdijmrFmzVFVVpfLyci1btiyLVQEAAADoDSZeAQAAAIAIIeQBAAAAQIQwXDOirly4JK3t36/bk3hMd18AAAAA4cGZPAAAAACIEEIeAAAAAEQIwzXzWM2a3yt+YJ8kqanNY/UL/yVJivU/TGWnTM9afQAAAADSR8jLY/ED+9S0f2/7he4dlwEAAADIGSkN1zSzK8ysJOhikFmx/oepsPRwFZYeLiuISWayglhiWaz/YdkuEQAAAECaUj2T94Cke83sUUlL3P1/AqwJGcJQTAAAACB6Up14ZYukQZK+JulFM9tgZt8xs/LAKgMAAAAApC2lkOfux0qqkLRALYHvBEl3SnrPzJab2dTAKgQAAAAApCzlWyi4+6vufoOkz0paIcnUMtzzc5L+YGbXBVMiAAAAACBVqU68Umhml5jZ05LekXRu66qXJF2vlrN7NwRSIQAAAAAgZalOvFIlaZhazt4dkPSwpB+7+zpJMrNiSXcEUiEAAAAAIGWphrwytZyt+6mkn7t73SHr/z9J7/dhXQAAAACAHkg15M2U9H/d3Ttb6e4bJW3ss6oAAAAAAD2S6sQr/SX9P2YWkxLX6N1kZl8MrjQAAAAAQLpSDXl3SJrq7nFJcvcmSaeJ6/AAAAAAIFRSDXkfk/TXQ5ZtbV0OAAAAAAiJVENejaTPmFmRJLU+Tm1dDgAAAAAIiVRD3hpJx0naYGaPSNog6XhJLwdVGAAAAAAgfanOrnmLpBmSPi7pWP39fnnfC6YsAAAAAEBPpHQmz93Xq2WilcWSfi/pF5KmuPub3e1rZjPM7C0z22RmN3SxzRfNbIOZvWlmD6VRPwAAGUVfAwCEXapn8uTuf5L01XTevPWWC/dJOkvSNklrzGy5u29os804STdK+rS715nZ8HSOAQBAptDXAAC5IOWQZ2ZXSfonSUeqZbimJPH66VAAACAASURBVLm7T0+y26mSNrn75tb3eEQtN1bf0Gabr0m6z93rWt9wZ8rVAwCQWfQ1AEDopRTyzOzfJX2/7aLWR+9m15FqudXCR7ZJ+sdDtjmu9RgvSopJ+p67P91JDXMkzWl9uc/M3kqldmTEMEm12S4iF5lZ9xshl/G70QsB/H4c1QfvQV/LD/zu9gK9LdL43eiFTPa1VM/kzZLUoJbGNlbSSklnSHq016W11DBO0jRJoyS9YGYT3f3Dthu5+yJJi/rgeOhjZlbp7hXZrgMIG3438hZ9Lcfxuwt0jt+N3JHqLRSOkvSEpBWS5O7nSPqdpN3d7FclaXSb16Nal7W1TdJydz/o7n+V9Be1NEcAAMKGvgYACL1UQ16TpDpJ+yTJzEZJqpd0eTf7rZE0zsyOMbNiSV+StPyQbX6jlm87ZWbD1DLMZXOKdQEAkEn0NQBA6KU6XHOnpDJJ69RyPd56SYdL2pVsJ3dvMrNr1TK8MyZpsbu/aWa3Sqp09+Wt6/7JzDZIikv6N3f/oEc/DbKF4UZA5/jdiBj6Wt7gdxfoHL8bOcLcu5s7RTKz/6OWm6B/QdLbkga3rvqhu38nuPIAAAAAAOlIKeS128HsaEkXSNru7r8OoCYAAAAAQA91G/LMrEjSnyX9t7v/S0aqAgAAAAD0SLcTr7j7QUn91XLtAfKUmbmZ/e82r79jZt9r83qWma03szfM7DUzYxgv8oaZxc1sXevvwGNmVnrI8jfN7E9m9r/MLNUJrwAEjN4GdI6+lvtS/Z9yj6QLzOyEIItBqDVI+nzrTHHtmNk5kq6T9E/uPlHSZHV/ew0gSg64+0nu/glJjZK+ccjyf5B0lqRzJN2SrSIBdEBvAzpHX8txqYa8b6jlDvfrzWy7mW1u/fNOgLUhXJrUMqPStztZd6Ok77j7dkly9wZ3/z+ZLA4IkVWSxh660N13Spoj6Vozs4xXBaAz9Dage/S1HJTOzdALWv+USzq6zR/kj/skfdnMBh2y/BOS1mahHiBUzKxQLd9qvtHZenffrJah78MzWReApOhtQBfoa7kr1fvkzQ60CuQEd99jZsskzZV0INv1ACHS38zWtT5fJekX2SwGQOrobUCn6Gs5LqWQ5+4PBF0Icsbdkl6VtKTNsjclfUrSs1mpCMi+A+5+Uncbmdmxark59s7gSwKQBnob0B59LcelNFyzdXapTv8EXSDCxd13SXpU0tVtFt8haaGZlUuSmRWb2VezUR8QVmZWJul+ST/2dG9QCiBQ9DYgffS1cEt1uOZSSV39z1vWN6Ugh/xvSdd+9MLdnzKzIyU903rhrUtanK3igBD5aLhLkVomePilpB9mtyQAXaC3Ad2jr+WIbm+GLklmtkV/D3mFapl8JSbpXXc/JrDqAAAAAABpSWm4prsf7e7HtP4ZLWmkpD9J+lmy/cxssZntNLP1Xaz/spm93nqT0ZfM7JNp/wQAAGQQvQ0AEHY9ukN9630xVqr92PXOLJU0I8n6v0r6TOtNRm9Ty71qAAAIs6WitwEAQiyla/LM7OZDFg2RNEvdhER3f8HMjk6y/qU2L/9H0qhU6gEAIFvobQCAsEt14pXvqePEKybpsT6s5WpJv+3D9wMAINvobQCAjEs15L2g9iFvn6Q1kn7UF0WY2RlqaYSnJ9lmjqQ5kjRgwIBPnXDCCX1xaABAnli7dm2tu5dl6njd9Tb6GgCgN5L1tZRm1+yN1iEt/9fdP9HF+hMlPSnpHHf/SyrvWVFR4ZWVlX1WIwAg+sxsrbtX9NF7Ha0+7G30NQBAupL1tVRvhj7VzK4ws1jr68LW11N7WdgYSU9IuiLVgAcAQJjR2wAA2ZbqcM2fSap1919Kkrs3mdlXJB0paUJXO5nZw5KmSRpmZtsk3aKWmyfK3e+XdLOkoZJ+0nKfUTX11besAAAEgd4GAAi7VEPe0ZKeP2TZW5L+MdlO7n5pN+u/KumrKdYAAEDW0dsAAGGXasirk3Tot5CfkvRh35YD5Ad314cffqj6+vpslxKYgoICHXbYYRowYEC2SwEAZEB9fb12796t5ubmbJcSmOLiYg0ePFgFBT261TSQMamGvNclnWVmz0taLWmKpJMl/XdQhQFR5O56+eWX9fLLL6upqUmlpaVqHc4VOc3NzdqzZ4/Kyso0ffp0HX300dkuCQAQgJ07d2rlypXatm2bBg4cqFgslu2SAlNfX6+mpiaddNJJ+uxnP0vYQ2ilGvL+X0lnqmUa6NPVco+8uKTbAqoLiKQ//vGPeuONN/TFL35R5eXlkQ14H4nH4/rzn/+sRx99VJdeeqlGjx6d7ZIAAH2orq5Oy5Yt09SpU/XP//zPKi4uznZJgfvggw+0YsUKLV++XBdeeGG2ywE6ldLXD+7+R0mfk/SspHck/V7S+e7+UoC1AZHS1NSkF198UV/+8pc1YsSIyAc8SYrFYvqHf/gHTZ8+XS+9xD8XABA1lZWV+uQnP6lTTz01LwKeJA0dOlSXXnqp3nrrLe3evTvb5QCdSvkcs7s/7e5nuftxrY9PB1kYEDVbt27VsGHDNGjQoGyXknETJkzQpk2bFPR9OQEAmfX2229rwoQuJ1qPrKKiIo0bN06bNm3KdilAp1K9T958M1t+yH3yfmNm84MtD4iO/fv3a+DAgdkuIyv69+8vM1NjY2O2SwEA9KG//e1vedvbBg4cqP3792e7DKBTqZ7Ju0rSAXePSy33yZO0X9KVAdUFRI675/UF2mbGmTwAiKB87W0FBQX0NYRWqr+VgyTtO2TZgdblAHrhjjvu0CmnnKKBAweqrKxM559/vtavX99um2nTpsnMZGYqLi7WiBEjNGPGDP3qV79KqcE0NjZq4cKFmjRpkkpLSzVkyBBNnjxZP/vZz9TQ0CBJuuqqq2Rmuu229vMpPffcczIz1dbWSpK2bNkiM9PQoUM7XIswbdo0XXvttb35zwEAiID77rtPJ554ogYOHKiBAwdqypQpWrFiRbtt6G1AcFINee9JmmlmYyWp9fECSVuDKgzIF88995yuueYavfTSS3r22WdVWFioM888U7t27Wq33ezZs7Vjxw5t3rxZy5cv15QpU/T1r39dF110keLxeJfv39jYqLPPPlu33367Zs+erRdffFFr167V9ddfryVLlmj16tWJbUtKSrRw4ULV1NR0W/f+/fs1fz4jtgEAHY0aNUp33nmnXn31VVVWVuqzn/2sLrzwQr3++uvttqO3AcFI9RYKT0v6V0l/NrNaScPUchuFXwVVGJAvVq5c2e71L3/5Sw0aNEgvvviizj///MTy0tJSlZeXS2ppnqeccoomT56sGTNmaNmyZZo9e3an73/33Xfr+eef1yuvvKKKiorE8mOOOUaXXHKJ9u37+0n6M844Q9u2bdNtt92me++9N2ndc+fO1T333KNrr71WI0eOTPvnBgBE18yZM9u9vv322/XTn/5Uq1ev1oknnphYTm8DgpHqmbzb1HLrhAJJw1sf31HL/fMA9KG9e/equblZgwcP7nbbs88+WxMnTtTjjz/e5TYPPvigzjzzzHZN8CMFBQXtLpgvKCjQ/Pnzdf/99+udd95JeuwvfOELmjhxom6++eZu6wQA5K94PK5HHnlE+/bt02mnndbt9vQ2oPdSvU9eraQTJV0h6d/VMuHKjyStTLYfgPR961vf0kknnaQpU6aktP2ECRO0efPmLte//fbbGj9+fMrHP/fcc/XpT39aN910U7fbLliwQA888IDefPPNlN8fAJAf3njjDR122GHq16+fvvGNb+jJJ5/UxIkTU9qX3gb0Tjr3yTsg6V1Jx0n6qaQfS5oUUF1AXrr++uv1xz/+UY8//rhisVhK+7h70hur92TmrzvvvFOPPfaY1q5dm3S7z3zmMzr77LN14403pn0MAEC0HX/88Vq3bp1efvll/cu//IuuvPLKDhOLdYXeBvROtyHPzMrM7DtmtlHS82o5m1cqySX9JuD6gLzx7W9/Ww8//LCeffZZHXvssSnvt2HDhqTbH3fccdq4cWNatZx66qm6+OKLNW/evG63nT9/vlasWKFVq1aldQwAQLQVFxdr7Nix+tSnPqU77rhDJ510kn70ox+ltC+9DeidpCHPzH6tlhk075R0vFomW3mrdfVyd7842PKA/PCtb30rEfBOOOGElPdbuXKl1q9fr0suuaTLbS677DI988wzqqys7LCuublZe/bs6XS/H/zgB1q1apWefvrppDVMnDhRs2bNSqlpAgDyV3Nzc+LWBsnQ24De6+5M3uclFUnaImm+pEnuPiHoooB88s1vflNLlizRQw89pMGDB6u6ulrV1dXtZgaTWqZ1rq6u1rZt27RmzRp9//vf1+c//3nNnDlTl19+eZfvf9111+n000/XWWedpXvvvVfr1q3TX//6Vz3xxBM6/fTT9eqrr3a639ixYzVnzhzdc8893f4Mt956a2JIDgAAN9xwg1atWqUtW7bojTfe0I033qjnnntOX/7yl9ttR28DgpHqNXmHqeXG54cHWAuQl37yk59o7969mj59ukaMGJH4c9ddd7XbbsmSJRoxYoSOPfZYnX/++Vq9erXuv/9+Pfnkk0mv3+vXr59+97vf6YYbbtAvfvELTZkyRSeffLIWLFigK6+8MulMZzfffLMKC7u/08ro0aM1d+5c1dfXp/6DAwAiq7q6WpdffrmOP/54TZ8+XWvWrNFvf/tbnXPOOe22o7cBwbBkF66a2WJJX5A0QC3X4EnSdkkfk7TC3S8IvMJOVFRUeGen54EwW79+vf785z8nHX4SZXfccYe+/e1vq6SkJNulIE+Z2Vp37zjfegjQ15CrFi5cqGuuuUYDBgzIdikZ9+yzz6qwsFBTp07NdinIU8n6WtIzee7+FUkjJH1D0lq1XJP30Z0hzzOzZ/qyUAAAAABA73Q7XNPd97n7Inc/VS33yvuxpDq1BL4zAq4PiAwzU3Nzc7bLyJrupsMGAOSmfO1t+fpzIzekfJ88SXL39e4+Vy3DNS+X9FwQRQFRVFpa2uVsX1F34MABubuKi4uzXQoAoA8NGDBAe/fuzXYZWbF37968HKaK3JBWyPuIuze6+0PuPr2vCwKiavTo0aqtrc3LoLdx40Z9/OMf50weAETM2LFjtWHDhmyXkXEHDx7U22+/rbFjx2a7FKBTPQp5ANJXWFioKVOm6KGHHtL777+vZJMeRUU8HtfGjRv1+9//PulMZwCA3FRRUaF169apsrJSBw8ezHY5GVFXV6f//M//1Lhx4zRo0KBslwN0qvv5YwH0malTp6qoqEgPP/yw3F2lpaWRPbvV3Nys3bt3a+jQobr44os1ZsyYbJcEAOhjQ4YM0RVXXKGVK1fqmWee0aBBg5Le+iDX1dfXq6GhQSeddJKmT2dAG8KLkAdkkJnptNNO05QpU1RXV6cDBw5ku6TAFBQU6PDDD9dhhx2W7VIAAAE68sgjNWvWLO3fv1979uxRPB7PdkmBKS4u1tChQ1VQwGA4hBshD8gCM9OQIUOyXQYAAH2mtLRUpaWl2S4DgLgmDwAAAAAihZAHAAAAABFCyAMAAACACCHkAQAAAECEEPIAAAAAIEIIeQAAAAAQIYQ8AAAAAIgQQh4AAAAARAghDwAAAAAiJNCQZ2aLzWynma3vYv0JZrbazBrM7DtB1gIAQF+gtwEAwi7oM3lLJc1Isn6XpLmS7gq4DgAA+spS0dsAACEWaMhz9xfU0uy6Wr/T3ddIOhhkHQAA9BV6GwAg7LgmDwAAAAAiJGdCnpnNMbNKM6usqanJdjkAAPQKfQ0AEJScCXnuvsjdK9y9oqysLNvlAADQK/Q1AEBQcibkAQAAAAC6Vxjkm5vZw5KmSRpmZtsk3SKpSJLc/X4zK5dUKWmgpGYzu07SBHffE2RdAAD0FL0NABB2gYY8d7+0m/XVkkYFWQMAAH2J3gYACDuGawIAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGEPAAAAACIEEIeAAAAAEQIIQ8AAAAAIoSQBwAAAAARQsgDAAAAgAgh5AEAAABAhBDyAAAAACBCCHkAAAAAECGBhjwzW2xmO81sfRfrzczuNbNNZva6mZ0cZD0AAPQWvQ0AEHZBn8lbKmlGkvXnSBrX+meOpJ8GXA8AAL21VPQ2AECIBRry3P0FSbuSbDJT0jJv8T+SjjCzEUHWBABAb9DbAABhl+1r8kZK2trm9bbWZR2Y2RwzqzSzypqamowUBwBAD6TU2+hrAICgZDvkpczdF7l7hbtXlJWVZbscAAB6hb4GAAhKtkNelaTRbV6Pal0GAECuorcBALIq2yFvuaRZrTORTZa02913ZLkmAAB6g94GAMiqwiDf3MweljRN0jAz2ybpFklFkuTu90t6StK5kjZJ2i9pdpD1AADQW/Q2AEDYBRry3P3Sbta7pG8GWQMAAH2J3gYACLtsD9cEAAAAAPQhQh4AAAAARAghDwAAAAAihJAHAAAAABFCyAMAAACACCHkAQAAAECEEPIAAAAAIEIIeQAAAAAQIYHeDB0AAADINfPmzVN1dbUkqba2Vk1NTSosLNSwYcMkSeXl5VqwYEE2SwSSIuQBAAAAbVRXV6uqqqrdsng83mEZEFaEPAAAAKCN8vLyxPPq6mrF43HFYrHE8rbrgTAi5AEAAABttB2KOWvWLFVVVam8vFzLli3LYlVA6ph4BQAAAAAihJAHAAAAABHCcE0AAPIUMwgCQDQR8gAAyFPMIAgA0UTIAwAgTzGDIABEEyEPAIA8xQyCABBNhDwAvZbsuh6u6QHAtX8AkFmEPAC9xnU9AJLh3wgAyCxCHoBeS3ZdD9f0AODaP4TJ7a/uSmv7XQ3Nicd09r3p5CFpHQfoS4Q8AL3GdT0AkuHfCADILG6GDgAAAAARQsgDAAAAgAgh5AEAAABAhHBNXggx1TTCIt2L0yUuUAfCgN9d9BS3xAGigZAXQkw1DQAAsoHPIEA0EPJCiKmmAQBANnBLHCAaCHkpyuQQSqaaBgCgZ7jkoXf4DAJEAyEvRQxfAACgo+sffymt7Wv21Sce0933hxef1u029GsAIOSljCGUQNdeW3Sb6utqJEn1u3YmHlffOVclg8s0ac5/ZLM8AHkkiv2as5OZl6yvSaK3IfQIeSli+ALQtfq6Gh2o3dFumTfHOywDgKBFsV9zdjLz6Gs4VK592ULIA9BrJYPLEs8bdn8gj8dlsZj6DRrabh0AREUmP/BF8exk2CXra4euR37ItS9bCHkAeo0hKwDyTSY/8EXx7GTYZbKvcW/C3JBrX7YEHvLMbIakeyTFJP3c3ecfsv4oSYsllUnaJelyd98WdF1Ab+XaaXsAfYO+Bin3PvCNPX582vt87MjhKioq1JYt76a1/6a3NqZ9rHyWa2eI8lWufdkSaMgzs5ik+ySdJWmbpDVmttzdN7TZ7C5Jy9z9ATP7rKQ7JF0RZF3ZlKlZyFKZgQy9wz/KQP6JWl9jcomey7UPfAivKN6bkC/Csy/oM3mnStrk7pslycwekTRTUttmOEHS9a3P/yDpNwHXBPSJXPsWF0CfiFRfY3IJIPui+IVBpr4IJ0x2LeiQN1LS1javt0n6x0O2+ZOkz6tl6MtFkg43s6Hu/kHAtQG9EsV/lAF0K1J9LVOTS7yzfKka934oSWrcW5d43Pjg3ZKk4sOP0McvuKpPjgUg+zL1RTijqroWholXviPpx2Z2laQXJFVJih+6kZnNkTRHksaMGZPJ+gAASEfO9LVMDcVs3PuhGncfknGbmzsuAxAJmfoinFFVXQs65FVJGt3m9ajWZQnuvl0t33jKzA6TdLG7f3joG7n7IkmLJKmiosKDKhgAgCToaz1QfPgRiecH/7ZH3hyXFcRUNGBgh/UAkCpGVXUt6JC3RtI4MztGLU3wS5Iua7uBmQ2TtMvdmyXdqJYZyTLq9ld3pbX9robmxGO6+wIAclpO9LWw6auhmFcuXJLW9u/X7Uk8prPvA/82O63jALmA69fyS6Ahz92bzOxaSSvVMtX0Ynd/08xulVTp7sslTZN0h5m5Woa1fDPImgDkLhoUso2+BiBXcf1afgn8mjx3f0rSU4csu7nN819L+nXQdQDIfTQohAF9DVE2fNgQFcZaPh4WFsYSjx87cria4k3aWcsIplzF9Wv5JQwTrwBASmhQAKJm8owL09q+/8F9KpC0tWp72vumojBWqKKi9h8PzazDMuQerl/LL/zGhhBTTQOdo0EBQLCa4k2J57FYgcxM7q54vLndOgDhRsgLIaaaBgAA2cBwTCAaCHkhxFTT2Tf2+PFpbf+xI4erqKhQW7a8m9a+m97amG5pAICAnXfNd9PavmFnrSRp+87atPcFgCAQ8kIoikMxmRURAAAAYRTFL/cJecgIZkUEAAAAMoOQl6LXFt2m+roaSVL9rp2Jx9V3zpUklQwu06Q5/5G1+sKOWREBAEA+uf7xl9Lep2ZffeIxnf1/ePFpaR8L0UbIS1F9XY0O1O5ot8yb4x2WoXPMiggAAABkBiEvRSWDyxLPG3Z/II/HZbGY+g0a2mE9AOQzrsFFNtWs+b3iB/ZJkpraPFa/8F+SpFj/w1R2yvSs1QcgNWG7h2SuIeSliKGYuSPZB0w+XCJsohiIuAYX2RQ/sE9N+/e2X+jecRmAtPVk9lhmn80OQh4ihw+YyCVR/PvKNbjIplj/wxLP4/X75d4sswLFSko7rM8VJU37Zd4sSTJ54rH/wZYzlW4Fqi8szVp9AMKHkIfISfYBkw+XCJsoBiKuwUU2RXEopnmzClrDXWKZ/h74mlsDIAB8hJCHyMnUB8zhw4aoMNbyK1RYGEs8fuzI4ZKkpniTdtbu6tNjInoIRAC641aQCHLWJuy5LLEeANoi5AE9VBgrVFFR+18hM+uwDKlJd6ppppkGkC8YigkgXXwaBXqoKd6UeB6LFcjM5O6Kx5s7rAcAREvj1r/IDzZIkvxgY+KxYfMbkiQr6qfi0cdlrT4A+Y2QB/QQQzEBIH/5wYZEyGuztJNlQPCuXLgkre3fr9uTeEx3X+QGQh4AAECarKhf4rk3HZTcJTNZYVGH9QCQaYQ8AOhEFO9fB/z/7d15fJXVnfjxz0kCJiibAipLR1xwSRRQAZVqXYpKXUAH64IyLuOCooNW6g6Ouz9trZSl1Y5orVYdlJ3RihtOa20Q7AiISBEFVESUNRByk/P7I2lMSAQCubnJzef9evG69znPec79Psj15JtznnNUe5yKKak+M8lrxGrrh9iabmy5o5tiThvzYI0+R9oZ6bh/nSRJ2jENbVV1k7xGzB9ipe+XjvvXSZKkHdPQVlWvn1GpTvhDrPT93L9OkqT6LTtRQNhiD8lAJKdoPVC6h2RtbUHS0FZVN8lrxPwhVmpcfM5Qkuqff0x+ks3rVgOwed235a8fPvMrmjZvxX5nXpzC6Oq3EEvIKEvuysv4LuErKUsAa0N9moq5PUzyJKmRcIq2JNU/m9etZvOaVZULS0qqlqmKGDLKE7lQIdmLhPLzjZVJniQ1Ek7RlqT6p2nzVuXvizasJZYUEzIyabJri0rnGpLNSxeW7xkZizaXvxYu/gAo3WKkNlaora2pmOnIJE+SGonamKK9/4EH1+gz2+/ZjiZNsliy5NMaX7voow9rVF+SGqJ0nI4ZiwrLk7wKpdWUKVlM8iRJkiTVmtBkl/L3MVEEMUIIhKwmVc4rOUzy1GAcdWr/Gl+TU7SeDGDp8s936HpJkiTVTG1MxdTOMclTnairudmSJEmq33bN3oWzjz+KvXZvSShbJGVLxYmiOounJFE32x98+OGOPYaQnZ1Nx44dadKkyXZfY5KnOuHcbNWGrS0zDdS7paZPu/qmGtUv/OprAD7/6usaXzttzIM1qi9JUqqcffxR5B6wH7vkNCOE6pO8ROGmOosnsbluPuvgg2v2bDpAjJFVq1axbNkyOnfuvN3XmeSpTjg3W7XBZaYlSWr49tq95VYTPH0nhMAee+zBypUra3SdSV6a+reHxtWo/opv15a/1vTa7eFUTNWGrS0zveV5SZL0nZX5r1G8cT0AiQqvX86cBEBmzm607XFSncQSCCZ4NbAjf1cmeZIajPo0FVOSpIakeON6EgXrKhfGWLVMacEkT5IkSUpzmTm7lb8v3lRAjCWEkEFmdrMq5+vaC6uq29R8xzc6P3u3b7ZZ5zdjxzBt6hQyMzIJAe4YPpzDDjtsm9ctX76cIUOGMGHChB2Ory6Y5EmSJElprq6mYjYE78+Zw1tvvsGLEybStOkurFzxBUVFdbeaZ10wyZMkSZLUaKxc+RWtW7emadPShf9at24NwG/GjuWtt95iU2Eh3bp2ZfiIEYQQmD9vHsOHDwfg6GOOSVncNZGR7A8IIZwaQvgohLAohHBzNed/EEJ4I4QwJ4TwfyGEnyQ7JkmSdpT9miQ1bMf0/iFffPEFfU/pw113jmBWfj4A519wAX987jkmTJhAYWEhb731FgB33HEHN99yC+NffDGVYddIUkfyQgiZwGigD7AMyA8hTI4xzq9Q7XbghRjj2BDCIcB0YJ9kxiWp8dqR1WOTvfpsumnXZneyMku7l6yszPLX9nu2AyBRnOCrr7f9vER9ZL8mSQ3frrvuyviXJvLerFm8++5fGTZsGEOHDqXZrrsybtw4Nm3cyNq1a9lvv/04/PDDWbduHUceeSQAZ5x+Ov/7yLIdKgAAGjFJREFUv/+b4jvYtmRP1+wJLIoxLgYIITwH9AMqdoYRaFH2viXweZJjkiQlUVZmFk2aVO5eQghVyhoo+zVJSgOZmZn07NWLnr16sd++nRn/3//NwoULee7559lrr70YM2YMhZs3pzrMHZbs6ZodgKUVjpeVlVV0J3BhCGEZpb/tvLa6hkIIV4QQZoUQZtV0M0BJUt1JFCcoKir9U1JSQoyRkpKS8rJEcSLVIe4M+zVJauA+WbyYJUuWlB9/tGAB++yzDwCtWrWioKCAV199FYAWLVrQvHlzZs+eDcC0adPqOtwdUh9+rXo+8GSM8RchhKOBp0MIeTHGkoqVYoyPAY8BHHnkkTEFcaad+rQppqSdc9Sp/WtUP6doPRnA0uWf1/jabWmoUzFrkf2aJNXAT/coqFKWKNyUtM8rKCjg3nvuYu3atWRlZtGpU0eGjxhB8+bNOfuss2jTpg15ubnl9e+++26GDx9OCKHBLLyS7CRvOdCpwnHHsrKKLgNOBYgxvhNCyAbaAF8lObZGz00xJanG7NckqYHLzcvj2edeKD9ObC5NKK+97jquve66KvUPyc2ttOjKDTfckPwgd1Kyp2vmAweEEDqHEJoC5wGTt6jzGXASQAjhYCAbcN5KHcjM2Y2sZs3JatackJEJIRAyMsvLUrkp5s7IThSQU7SenKL1BEp/OR6I5BStJztR9TdFklQD9muSpHovqSN5McZECGEI8AqQCTwRY5wXQrgLmBVjnAz8DHg8hHA9pQ+rXxxjdNpKHUjXqZghlpBB5X9CgdJEr6TybClJqhH7NUlSQ5D0Z/JijNMpffC8YtnwCu/nA72THYcajxgyypO5UCHZiwRiSPrWkJLSnP2aJKm+qw8Lr0i1alNWs1SHIG23zUsXEosKAYhFm8tfCxd/AEBosgtNO3VJWXySJKnhMcmTpBSKRYXlSV6F0mrKJEmSto9JniSlUGiyS/n7mCiCGEsXQcpqUuW8JEnS9jDJk6QUciqmJKmxGzvrs1pt7/JD222zzhHdu/LenL+XH0+aOJF58+Zx62231WosqeIqFJIkSZKURhzJkyRJkqQyb775Jo899hiJoiJatmzJAw88wB5t2jBmzBiWLV3KZ599xurVq7n4kksYMGAA+fn5jB41il133ZXPli6lZ48e3Hb77UyaNImFCxdy0003AfD4448zf/58HnnkkaTfg0meJEmSpEalcNMmzup3RulBjKxZs4bjjz8egMO7d+eZZ54hhMCLL77IuHHjuHHYMAAWLlzIH555ho0bN/LTc87huOOOA2Du3LlMnDiRvdu3Z/BVVzFjxgxOOeUUHn/sMW644QaaNGnCuHHj+O1vf1sn92eSJ0mSJKlR2SU7mwmTpgCQ2Lyp/Jk8gC9XrGDYsGGsXLmSokSCDh06lF93wgknkJ2dTXZ2Nj169mTuBx/QvEUL8vLy6NipEwB9f/IT5syZw8knn0zPXr2YOXMm+3buTFFREYceemid3J9JniRJkiSVeeD++7lo0CBOOOEE8vPzGTtmzHcnQ6hUN5Qdhy3Ly17PPvtsfvf443Tu3JlLLrkkmWFX4sIrklSNlfmv8eXMSXw5cxKJjesBSGxcX162Mv+1FEcoSZKSYf369ezZrnSFzsmTJlU69+Ybb1BYWMjq1auZlZ9Pbl4eUDpdc9myZZSUlPDKyy/T/fDDATjssMP4csUKpk+fzvnnn19n9+BIniRVo3jjehIF6yoXxli1TJIk7ZTBR/6gSlmicFMKIik1ePBgfvazn9GiRQt69urF8uXLy88d0KULl116KatXr+aKK6+kXbt2fPrpp+Tm5nL/ffeVL7xy0kknlV9zyskn89FHH9G6des6uweTPEmqRmbObuXvizcVEGMJIWSQmd2synlJktSwVNwjD6Bf//70698fgBNOPJETTjyx2uu6dOnCfffdV6V8t912Y9To0dVeM2fOHC686KKdjLhmTPIkqRpte5y07UqSJEnfY+3atQy84AK6dOnCUUcdVaefbZInSZIkSdtw9dVXV1veo0cPevToUaW8RYsWTJk6NdlhVcskT5IaiexEASGWABCI5a85RaULy8SQwaasZimLT5Ik1Q6TPElqJEIsIaMsuSsv47uEr6QsAZQkSQ2bSZ4kNRIxZJQncqFCshfLdvOJwV11JElKByZ5ktRIOBVTkqTGwSRPkiRJUsrc88Krtdrezf2O3WadvIMP5IAuXUoPYuRXjz5Khw4dajWOVDLJkyRJktSo7JKdzYRJUwBIbN75jdcTiQRZWfUntao/kUiSJElSisyfN4+HHnqIgoICWrduzd333EPbtm0ZP348L44fT1FREZ1+8APuu+8+cnJyuP2229hll134cMECunfrxrCf/zzVt1DOJE+SJElSo1K4aRNn9TsDgA7t2/PQww9z//338+jIkey+++68/PLL/HrkSO66+25+/OMfM2DAAAB+PXIkE156iQsGDgRgxYoVPP3002RmZqbsXqpjkidJkiSpUdlyuubHH3/MokWLuPKKKwAoLi6mbdu2ACz6+GN+PWoU69aupWDjRnofc0x5O31OPrneJXhgkidJkiSpkYsxst9++/GHZ56pcu72O+7g0Ucf5cADD2TSxInk5+eXn8vJyanLMLebmyJJkiRJatQ6d+7Mt99+y9/ffx+AoqIiFi1aBEDBhg20adOGoqIipk2blsowt5sjeZIkSZJS5vaf9qlSlijc+RUva6JJkyb84pe/5IH772f9+vUUFxcz8MIL2X///blmyBAGDhzI7q1bc+ihh7Jhw4Y6jW1HmORJkiRJalTem/P3KmUHHXQQTz71VJXyc889l3PPPbdK+T333puU2GqD0zUlSZIkKY2Y5EmSJElSGjHJkyRJkqQ0YpInSZIkSWnEJE+SJEmS0ohJniRJkiSlEbdQkCRJkpQyQx/8da229/DQy7dZJ+/gAzmgSxcSiQSZGRmcceaZXHTRRWRkZDBv3jymTJ7MzbfcUqtxbWnJkiX85S9/4YILLqj1tpOe5IUQTgUeBTKB38UYH9ji/CPACWWHzYB2McZWyY5LkqQdYb8mSQ3fLtnZTJg0BYAVXyzn5ptuYv369VxzzTXk5uaSm5ub9BiWLFnCs88+m5QkL6nTNUMImcBooC9wCHB+COGQinVijNfHGLvFGLsBvwZeSmZMkiTtKPs1SUo/e+yxByNGjOC5P/6RGCP5+fkMueYaAGbl53POgAGcM2AAPz3nHDZs2EBJSQn33HMPZ55xBldcfjlXDx7Mn/70JwBOPeUUvv32WwDmzZvHpZdcUt5Ot27d6NatG927d2fdunXcfPPNvP3223Tr1o1HHnmkVu8p2SN5PYFFMcbFACGE54B+wPzvqX8+MCLJMUmStKPs1yQpDXXs1ImSkhK+WbWqUvmTTz3FrbfdRvfu3SkoKKBp06bMmDGDz5cvZ+KkSXzzzTf0O/NM+p911lbbf/Kppxg9ejS9e/dm/fr1ZGdn88ADD/Dwww8zderUWr+fZC+80gFYWuF4WVlZFSGEfwE6A68nOSZJknaU/ZokNSLdu3XjoYce4plnnmHt2rVkZWUxZ84cTj75ZDIyMmjTpg09e/bcrnZuuOEGRo4cyerVq8nKSu5YW31aXfM8YHyMsbi6kyGEK0IIs0IIs1auXFnHoUmSVGP2a5LUQCxbupSMjAx232OPSuWX/fu/c+edd1K4aRP/NmgQnyxevNV2MjMzKSkpAaCwsLBSO7/73e/YuHEjvXv3ZsGCBbV/ExUkO8lbDnSqcNyxrKw65wF//L6GYoyPxRiPjDEe2bZt21oMUZKk7Wa/Jklp5ptvvuHuu+/mvPPPJ4RQ6dzSpUvp0qULl152Gbl5eXzyySd079aNGTNmUFJSwqqvvyY/P7+8fvv27Zk/v3QG/4xXX63UzqGHHspNN91Ejx49WLBgAc2bN2fdunVJuadkP5OXDxwQQuhMaSd4HlBl+ZgQwkFAa+CdJMcjSdLOsF+TpFr2q5uurVKWKNyU1M8s3LSJs/qdUb6FwulnnMGgQYOq1PvD00/zt/x8MkJgv/3354fHHktWVhbvvvsu/fv1Y6+99uLggw+m+W67AXDV4MHcOWIEo0eN4sgePSq18/Nhw8jIyCA3N5e+ffuSkZFBZmYmXbt25eKLL+b666+vtftLapIXY0yEEIYAr1C61PQTMcZ5IYS7gFkxxsllVc8DnosxxmTGI0nSzrBfk6T0MPfDj8rfJzZXTih79OhBj7IE7ZZbb632+p/deCPNmjVj9erVXHD++RzQpQsARxxxBFOqWUjllltv5dC8vCrlr7+enMe2k75PXoxxOjB9i7LhWxzfmew4JEmqDfZrkqQh11zDunXrKCoq4sorr6RNmzapDqmSpCd5kiRJkpROnhg3LtUhbFV9Wl1TkiRJkrSTTPIkSZIkKY2Y5EmSJElSGjHJkyRJkqQ04sIrkiRJklLmoiuH1Gp740Y+vM06vxk7hmlTp5CZkUkIcMfw4Rx22GG1GgfA1YMH88CDD9KiRYtab3trTPIkSZIkNRrvz5nDW2++wYsTJtK06S6sXPEFRUVF23VtIpEgK2vbKVSMkRgjY8aO3dlwd4jTNSVJkiQ1GitXfkXr1q1p2nQXAFq3bk27du049ZRT+PbbbwGYN28el15yCQBjxozh1ltuYdBFF3HrrbcyaeJErrv2Wi695BJOP+00xpYlcsuXL+eMM87g1ltv5eyzzuLLL78sb7OgoIDTTjuNrl27kpeXx/PPPw/Ae++9x49+9COOOOIITjnlFL744otauUdH8iRJkiQ1Gsf0/iFjRo+i7yl9OProYzi5z485skePrV7zj3/8g6d+/3uys7OZNHEic+fO5aUJE8jOzub888/nuOOOo1WrVnz26afcc889dO3atdL1f/7zn2nfvj3Tpk0DYM2aNRQVFXHttdcyadIk2rZty/PPP89tt93GE088sdP3aJInSZIkqdHYddddGf/SRN6bNYt33/0rw4YNY+jQoVu95vgTTiA7O7v8+Kijj6ZVq1YA/Pikk5g9ezYnnngie7dvXyXBAzjggAMY+eij3HTTTZx++ukce+yxzJ07l7lz59KnTx8AiouL2XvvvWvlHk3yJEmSJDUqmZmZ9OzVi569erHfvp2ZPHkymZmZlJSUAFBYWFipfk5OTqXjEEK1x1vW+6d99tmH2bNnM336dG6//XZOOukkzjrrLHJzc3nnnXdq67bK+UyeJEmSpEbjk8WLWbJkSfnxRwsW0H7vvWnfvj3z588HYMarr261jb++8w5r1qxh06ZNvP7663Tv3n2r9b/66iuaNWvGhRdeyLBhw5g9ezYHHnggK1euLE/yioqKmDdv3s7dXBlH8iRJkiSlzNO/HVWlLFG4KWmfV1BQwL333MXatWvJysyiU6eODB8xgsWLF3PniBGMHjVqm8/o5eXlccP117NixQpOO/10cnNzWb58+ffW//jjj7l+6FAyMjJo0qQJY8eOpWnTpowfP57rrruONWvWkEgkGDp0KLm5uTt9jyZ5kiRJkhqN3Lw8nn3uhfLjxObShPKII45gytSpVepfffXVVcr23HNPHh05slJZhw4dmDBhQqWyl195BYDevXtz1ZVXVmmnW7duzJw5s+Y3sQ1O15QkSZKkNOJIniRJkiRtp379+9Ovf/9Uh7FVjuRJkiRJUhoxyZMkSZKkNGKSJ0mSJElpxCRPkiRJktKIC69IkiRJSpm+J/ep1famTJmyXfVmzHiV6665mkmTJtF5331rNYZUcyRPkiRJUqMzfepUDj/iSKb/z/8k9XMSiURS26+OI3mSJEmSGpUNGzYw+733ePL3TzP4qiu45ppryM/PZ+yYMbRq3ZpFH3/MIYccwv0PPEAIgbdnzuShhx4iJyeH7t27s2zZMkaNHk1BQQEP3H8/ixYtIpFIMHjwYE448UQmTZzIjNdeo6CggJLiYsY9+WSd3p9JniRJkqRG5fXXZvDDY49ln86dadWyJfPnzQNgwYIFvDRhAu3atWPQRRcxZ84ccnNzueuuuxj35JN07NiRn//85+XtPP744/Ts2ZO77r6btWvXMvCCC+h11FEAfDh/Pi++9BItW7as8/tzuqYkSZKkRmX6tKn85LTTADi1b9/yKZt5eXnstddeZGRkcOBBB/H58uV88skndOzYkY4dOwLQt2/f8nbe+ctf+K8nnuCcAQO47NJLKSws5MsvvwTg6KOPTkmCB47kSZIkSWpEVq9ezbt//SsLFy4khEBxIkEIgeOOO46mTZuW18vMyKC4uHirbcUY+eUvf0nnzp0rlX/wf/9HTk5OUuLfHo7kSZIkSWo0/vTKy5zRrx+vvfEWM15/k1dnzKBDhw7Mfu+9auvvs88+LFu2jOXLlwPwyssvl587pndv/vjss8QYAfjwww+TfwPbwZE8SZIkSSnzP396tUpZonBT0j5v+tSpXHb5FZXKftynDy88/zydOnWqUj87O5vbbr+dwVddRU5ODnl5eeXnrrzySv7fgw/yr2efTYyRDh06MGr06KTFvr1M8iRJkiQ1Gk8+/YcqZQMHDmTgwIGVym697bby9z179mTylCnEGLn33nvJzc0FShPA4SNGVGmvX//+9Ovfv5Yj334meZIkSZK0FS+OH8/kyZMpKirioIMOYsA556Q6pK0yyZMkSZKkrbho0CAuGjQo1WFsNxdekSRJklRnIrF8oRJt2478XZnkSZIkSaozX36zhsKNBSZ62yHGyKpVq8jOzq7RdU7XlCRJklRnXnrzrwDstXtLAqHaOsWJojqLpySRqJPPycrM3KHrsrOzyzdi3+7P2qFPkiRJkqQdsGFTIU+//NZW66xa/FEdRQNff1I3n7Xoo7rbQy/p0zVDCKeGED4KISwKIdz8PXV+GkKYH0KYF0J4NtkxSZK0o+zXJEn1XVJH8kIImcBooA+wDMgPIUyOMc6vUOcA4Bagd4zx2xBCu2TGJEnSjrJfkyQ1BMkeyesJLIoxLo4xbgaeA/ptUedyYHSM8VuAGONXSY5JkqQdZb8mSar3kv1MXgdgaYXjZUCvLep0AQgh/BnIBO6MMb68ZUMhhCuAK8oO14cQ6m6irralDfB1qoNoiEKo/mFjpQ2/GzshCd+Pf6mFNuzXGge/uzvBvi2t+d3YCXXZr9WHhVeygAOA44GOwMwQwqExxtUVK8UYHwMeq/vwtC0hhFkxxiNTHYdU3/jdaLTs1xo4v7tS9fxuNBzJnq65HOhU4bhjWVlFy4DJMcaiGOMnwEJKO0dJkuob+zVJUr2X7CQvHzgghNA5hNAUOA+YvEWdiZT+tpMQQhtKp7ksTnJckiTtCPs1SVK9l9QkL8aYAIYArwAfAi/EGOeFEO4KIZxZVu0VYFUIYT7wBjAsxrgqmXGp1jndSKqe3400Y7/WaPjdlarnd6OBCDHGVMcgSZIkSaolSd8MXZIkSZJUd0zyJEmSJCmNmORpu4QQYgjhFxWObwwh3FnheFAIYW4I4YMQwpwQwo0pCVRKgRBCcQjh/bLvwH+HEJptUT4vhPD3EMLPQgj+f1eqJ+zbpOrZrzV8/kfR9ioEzi5bKa6SEEJfYChwcozxUOAoYE0dxyel0sYYY7cYYx6wGbhqi/JcoA/QFxiRqiAlVWHfJlXPfq2BM8nT9kpQuqLS9dWcuwW4Mcb4OUCMsTDG+HhdBifVI28D+29ZGGP8CrgCGBJCCHUelaTq2LdJ22a/1gCZ5KkmRgMDQwgttyjPA95LQTxSvRJCyKL0t5ofVHc+xrgYyATa1WVckrbKvk36HvZrDVdWqgNQwxFjXBtC+D1wHbAx1fFI9UhOCOH9svdvA/+VymAkbT/7Nqla9msNnEmeaupXwGxgXIWyecARwOspiUhKvY0xxm7bqhRC2BcoBr5KfkiSasC+TarMfq2Bc7qmaiTG+A3wAnBZheL7gYdCCHsBhBCahhD+PRXxSfVVCKEt8BtgVIwxpjoeSd+xb5Nqzn6tfnMkTzviF8CQfx7EGKeHEPYEZpQ9eBuBJ1IVnFSP/HO6SxNKF3h4GvhlakOS9D3s26Rts19rIIKJtyRJkiSlD6drSpIkSVIaMcmTJEmSpDRikidJkiRJacQkT5IkSZLSiEmeJEmSJKURkzxJkiRJSiMmeZIkSZKURkzyJEkNWgjhzRBCLPuTCCF8G0KYHUL4zxBCyxq0c/w/20lmvJIkJVtWqgOQJKmWfAi8DnQG+gDdgUEhhB/GGJenNDJJkuqQI3mSpHTxtxjjkBjjaUBXYC2wDzAyhNA6hPBaCOGLEEJhCGFDCCE/hHA2QAjhYuCNfzZUYWTw4hBClxDC2yGElSGEohDCuhDCzBDCcSm4R0mStsmRPElS2okxfhhCeAy4ETgDaAa0BWYAa4D9gVOAP4YQugLzgReBfy1r4tGy1/nAHkBTYBqwgdIE8lhgYgjhgBjjqjq5KUmStpMjeZKkdPVJ2WsTIAIDgPeA9cACYCOlydvxMca/AaP+eWGMcWjZn7/FGN8BBgNzKU3y3i+r1hroURc3IklSTTiSJ0lKV/uWvRYBxwHPAqGaenturZEQwn8Av/qe01u9VpKkVHAkT5KUdkIIBwOXlx1OBc6lNMH7M9AKyKF02iZ8l/glKlxfsX+8uOz1BWBXYO+KH1WbcUuSVBtM8iRJ6aJnCGFUCGEq8HegBbAEuA74vKzOoZSOyv2Z0uf0Kvq0wvsXQgi/CiHsVeHaHwEjKX2uT5KkesskT5KULg6m9Nm53pQ+P/efQLcY4zJgBKUjelmUbq/wGN8lbwDEGJcCdwIrKV2A5T+ANpQmiTOBlsDxwO1JvxNJknZCiNE9XyVJkiQpXTiSJ0mSJElpxCRPkiRJktKISZ4kSZIkpRGTPEmSJElKIyZ5kiRJkpRGTPIkSZIkKY2Y5EmSJElSGjHJkyRJkqQ0YpInSZIkSWnk/wOhWQH6oInqKAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1080x720 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 431
        },
        "id": "sdFLmOsXlGhf",
        "outputId": "46c8c8fd-72cd-432a-c917-04f132c77d66"
      },
      "source": [
        "2multi_df_new = pd.melt(multi_df, id_vars=['Model', 'Data'], value_vars=['Sad', 'Happy', 'Fear', 'Disgust',\n",
        "                                                                'Surprise', 'Anger'],\n",
        "        var_name='Emotions', value_name='Accuracy')\n",
        "x = \"Emotions\"\n",
        "y = \"Accuracy\"\n",
        "hue = \"Model\"\n",
        "\n",
        "g = sns.catplot(data=multi_df_new, x=x, y=y, hue = hue, palette = \"Blues\",\n",
        "                 col=\"Data\", kind=\"bar\", height=4, aspect=2)\n",
        "g.set(ylim=(0.4, None))\n",
        "g.set_titles(\"{col_name} {col_var}\")\n",
        "g.set_axis_lables(\"Emotions\", \"Accuracy\")\n",
        "#ax.legend(loc='upper left', bbox_to_anchor=(0, 1))\n",
        "ax.set_xlabel('Emotions',fontsize = 15, )\n",
        "#ax.set_ylabel('Sensitivity',fontsize = 15 )\n",
        "#ax.set_xticklabels([\"ML\", \"1D CNN\", \"2D CNN\", \"3D CNN\"], rotation=0, fontsize = 13);\n",
        "'''\n",
        "add_stat_annotation(ax, data=arousal_df_new, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"ML\", \"PD\"), (\"ML\", \"NC\")),\n",
        "                                 ((\"1D CNN\", \"PD\"), (\"1D CNN\", \"NC\")),\n",
        "                                 ((\"2D CNN\", \"PD\"), (\"2D CNN\", \"NC\")),\n",
        "                               ((\"3D CNN\", \"PD\"), (\"3D CNN\", \"NC\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "                    '''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-107-72c78c87cec9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mylim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_titles\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{col_name} {col_var}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_axis_lables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Emotions\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Accuracy\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;31m#ax.legend(loc='upper left', bbox_to_anchor=(0, 1))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_xlabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Emotions'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfontsize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m15\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'FacetGrid' object has no attribute 'set_axis_lables'"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABNUAAAEmCAYAAABF6KYCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXRUVbr38V8loQiEwYBSREUUCEEIRgUvoEIkEMAMRBIGu5VBQL1XEZwaQftCCw5Ig92AitAoAaTFKAiSiKChBdtrQLw2QUCJIBBsE5GAQOZUzvsHr3UtM1aoSk3fz1qsw6m9a5+nMpzaeWoPJsMwDAEAAAAAAACotwB3BwAAAAAAAAB4G5JqAAAAAAAAgINIqgEAAAAAAAAOIqkGAAAAAAAAOIikGgAAAAAAAOAgkmoAAAAAAACAg4LcHQAA1CUiIkL33HOPZsyYIUl67bXXVFRUpIceekiStHHjRq1YsUImk0mBgYFKTEzUpEmT7NpYsmSJ0tLS1KZNGxUXF6tr1656+OGH1aVLl1qvvWHDBt1yyy2yWCyueXEAAMBnXHvtteratausVqs6deqkF154Qc2aNbM9XlFRocDAQN1xxx2aMGGCAgLsxzicOHFCcXFx6tSpk0pLSxUSEqLf//73Sk5OrvW6Bw8e1I8//qjo6GhXvjwAwG8wUg2AxzObzdq2bZsKCgqqlO3YsUOrVq3Sa6+9ps2bNystLU0tW7astp0JEyZo06ZN2rZtm+Li4jR+/Phq2/y1d999Vz/++KNTXgcAAPBtwcHB2rRpk9LT09WkSROtW7fO7vGMjAytXLlSO3fu1EsvvVRtG1dddZU2btyoLVu26C9/+YtWrVql9evX13rdgwcPaseOHU5/PQCA2pFUA+DxgoKCNGbMGK1atapK2fLlyzV9+nTbSDKz2azRo0fX2WZcXJxuueUWbd68WZL00ksvKSUlRQkJCfrv//5vGYahDz74QF999ZUef/xxJSUlqaSkpNp6AAAAv9W7d28dO3asyuNt27bV3LlztXbt2jr7ER06dNCMGTO0Zs0aSVJ2drbGjBmjO+64Q3feeaeOHDmisrIyLV68WO+//76SkpL0/vvvV1sPAOB8JNUAeIW77rpLmzdv1rlz5+wez8nJUWRkZIPa7N69u62Teffdd2v9+vVKT09XSUmJ/vGPf2jYsGGKjIzUggULtGnTJgUHB1dbDwAA4NcqKiq0c+dOde3atdryDh06yGq16tSpU3W21aNHD1t/pVOnTlq7dq02btyoqVOn6i9/+YvMZrOmTp2quLg4bdq0yTZ99Lf1AADOx5pqALxCixYtlJSUpNWrVys4ONjp7e/atUsrVqxQSUmJzpw5o/DwcMXExDS4HgAA8D8lJSVKSkqSdGGk2siRIy+6zV+PZjt37pyeeOIJHTt2TCaTSeXl5dU+p771AAAXh6QaAK8xfvx4JScn2y3W26VLF3311Vfq16+fw+0dOHBAkZGRKi0t1dNPP63169crLCxMS5YsUWlpaZX69a0HAAD80y9rp9UlNzdXgYGBatu2bZ11Dxw4oM6dO0uSFi1apD59+ujll1/WiRMnNG7cuGqfU996AICLw/RPAF7jkksu0bBhw/TOO+/YHrv//vv15z//WSdPnpQklZWV6e23366zra1bt+rTTz9VQkKCLTEWGhqqwsJCbd261VYvJCREhYWFklRrPQAAgPooKCjQ7Nmzddddd8lkMtVa98SJE5o/f77uvvtuSRdGoP2yjuy7775rq/fr/kpt9QAAzsVINQBeZeLEiVq7dq3tPDo6Wj/99JPuueceGYYhk8mklJSUap+bmpqq9957T8XFxQoPD9eqVavUpk0bSdKoUaOUkJCgSy+9VD179rQ9Z8SIEZo9e7aCg4P11ltv1VgPAACgJr9MC62oqFBgYKCSkpJ0zz33VFv3+PHjuuOOO1RaWqqQkBCNHTvWNkp/8uTJmjFjhpYuXaro6Gjbc/r06aPly5crKSlJ999/f431AADOZTLYug4AAAAAAABwCNM/AQAAAAAAAAeRVAMAAAAAAAAcRFINAAAAAAAAcBBJNQAAAAAAAMBBXpdUmzRpkrtDAAAAqBN9FgAAAN/mdUm106dPuzsEAACAOtFnAQAA8G1el1QDAAAAAAAA3I2kGgAAAAAAAOAgkmoAAAAAAACAg0iqAQAAAAAAAA5yWVJt5syZ6tevnxISEqotNwxDzzzzjGJjY5WYmKj9+/e7KhQAAAAAAADAqVyWVEtOTtaKFStqLN+5c6eOHj2qbdu2ae7cufrTn/7kqlAAAAAAAAAAp3JZUu2mm25S69atayzPzMzUHXfcIZPJpOuvv15nz57Vjz/+6KpwAAAAAAAAAKdx25pq+fn5at++ve28ffv2ys/Pd1c4AAAAAAAAQL2xUQEAAAAAAADgILcl1SwWi/Ly8mzneXl5slgs7goHAAAAAAAAqDe3JdViYmK0ceNGGYahf/3rX2rZsqXatWvnrnAAAAAAAACAegtyVcOPPvqodu/erdOnT2vAgAF66KGHVFFRIUn63e9+p+joaO3YsUOxsbFq1qyZnnvuOVeFAgAAAAAAADiVy5JqL774Yq3lJpNJs2fPdtXlAQAAAAAAAJdhowIAAAAAAADAQSTVAAAAAAAAAAeRVAMAAAAAAAAcRFINAAAAjS4rK0uPPvqosrKy3B0KAABAg7hsowIAAACgJqmpqcrJyVFRUZH69u3r7nAAAAAcxkg1AAAANLqioiK7IwAAgLchqeZjmEoBAAAAAADgekz/9DFMpQAAAAAAAHA9Rqr5GKZSAAAAAAAAuB5JNaCBmGoLAAAAAID/IqkGNFBqaqr27t2r1NRUd4cCAECj4UMlAACAC1hTDWggptoCAPwR67cCAABcwEg1AAAA1BsfKgEAAFxAUg0AAAAAAABwEEk1+B3WggEAAAAAABeLNdXgd1gLBgAAAAAAXCxGqnkIRk81HtaCAQD/wnvsBY39dSgtr/CodgAAAJyNkWoegtFTQM2ysrKUlpam0aNH8/sBwGG8x17Q2F+Hpk2C1H3y8hrLL8v/WUGSjuX/XGu9Ayvuc0F0AAB4Jv728S4k1TwEo6eAmvEHMYCLwXvsBXwd/Bd/oAGA9+BvH+/C9E8AHo8/BBsP0+QA4OJ52r00NTVVe/fuVWpqqrtDAZzK037XAGfgbx/vwkg1AIANn4w1HkaOwFOVVVhlDgp0dxhezdPupfyBBl/lab9rAPwPSTUvU1peoaZNLv7b5qx2APgW/vBqPPwhUDcSj+5hDgrUXS9vq7H85M8X7g95PxfVWm/tg0OcHpu34F4KNA5+1wC4G1kVL8OivwDgG/hDoG4kHgEA8Ax80AVUj6QaAADwSCQeAQDwDHzQBVSPjQqAGpSWV3hUOwAAAADgDnzQBVSPkWpADfx5qi3DuwEAcD/ej70P3zMA8C8k1QBUwfBuAADcj/dj78P3DAD8C9M/4TWysrL06KOPKisry92h+DyGdwMA4H68H3sfvmcA4F9cmlTbuXOnhg4dqtjYWC1fXnV63Pfff6/x48crMTFRY8eOVV5enivDgZdLTU3V3r17lZqa6u5QAAAAAABejoEbuFguS6pZrVbNmTNHK1asUEZGhtLT0/Xtt9/a1XnhhRd0xx13aPPmzXrggQe0cOFCV4UDH8AnfwCA6jhzQxg2lwEAwH8wcAMXy2VrqmVnZ6tjx47q0KGDJCk+Pl6ZmZnq0qWLrc7hw4c1c+ZMSVLfvn314IMPuioc+JGyCqvMQYHuDgMA0EictbGM5J2bywAAfAsbXjQeBm7gYrksqZafn6/27dvbzi0Wi7Kzs+3qdOvWTdu2bdP48eP14YcfqrCwUKdPn1ZoaKirwoIfMAcF6q6Xt9VYfvLnCzfMvJ+Laq239sEhTo8NgPejowsAAFyJDS8A7+HWjQqmT5+uzz//XHfccYd2794ti8WiwEBGGAGAo1gPovEwTaBu/Dz6NlOQ2e4IAI7gPaJujJ7yPvxc+y+XjVSzWCx2Gw/k5+fLYrFUqfPSSy9JkgoLC7Vt2za1atXKVSG5FVMSgZqVlleoaZOLvx05qx1vxCeajYeObt34efRtLa7tr6Kc3Woe/h/uDgWAF+I9Ar6In2v/5bK/Pnv27KmjR48qNzdXFotFGRkZVTYiKCgo0CWXXKKAgAAtX75cKSkprgrHZeo7DYgpiUDNnLUekjeuheSsqYQkeuBJ+Hn0bcFhnRUc1tndYQCSmJLvjXiPgC/i59p/uSypFhQUpFmzZmny5MmyWq1KSUlReHi4Fi1apMjISA0aNEi7d+/Wiy++KJPJpN69e2v27NmuCsdlyEgDuBjcQwAAaDjeRxsPCUwAqMql86Sio6MVHR1t99i0adNs/x82bJiGDRvmyhBcjow0gIvBPQQAgIbjfbTxkMAEgKrculEBAAC+qrS8wqPaATyNEdDE7gjAs5HABICq/HNFbwAAXMyf1woE6uNcaKRCfv5Gha0j3B0KAABAgzBSzcfwqS+A2vjz6Cm2Ovc8ZRVWd4cANyptfrkKwgaqtPnl7g4F4D0CANAgjFTzMXzqC6A2/jx6irVgPA87YwPwFLxHAGgspeUVatrk4lMxzmoHF4fvgI8pbX45n/gCQDVYCwYAUBPeIwA0Fn/+kNsXMf0TaCCm2gIAAE/gz1P7AdSMac2A6zFSDWggptrWLSsrS2lpaRo9ejRTKQAAcBFGPQCojiPTmpmSCDQMP+1AAzHVtm6sTwLAE/jLyOKKykoFBTAJAYBvcGZyxl8TPY5MayY5DzSM/91ZADQa1icB4An8ZWRxUECAXv/kQI3lZ4vLbMfa6k3s393psQGAo+pK8kgkegC4H0k1+B1TkNnuCADwbYwsBgAAgCswRwB+p8W1/WW+9Cq1uLa/u0MBAAAAAABeipFq8DvBYZ0VHNbZ3WEAAAA/V1ZhlTko0N1hAACABiKpBgAAALiBOShQd728rcbykz9fWJM07+eiWuutfXCI02ND9Vg8HwDwa355F8/KylJaWppGjx7NjoQAAAAA6sVZOyRKLJ4PAL7AL5NqqampysnJUVFREUk1AF6BKUIAAAAA4Fn8MqlWVFRkd/QE7EgJoDZMEQIAAAAAz8Lunx6CHSkBAAAAAAC8h1+OVPNE7EgJAADgHSoqKxUUwGfTAAD4O5JqAAAAgAOCAgL0+icHaiw/W1xmO9ZWb2L/7k6PDQAANB4+YgPg8YyAJnZHAAAAAADczSeTapWG4VHtALg450IjVRrcTudCI90dCuA0JIsBAAC8Q1mF1d0hwEP55PTPAJNJ+4+frrG8rKLSdqytXo+rQp0eGwDHlTa/XKXNL3d3GIBTnQuNVMjP36iwdYS7QwEAAEAtzEGBuuvlbTWWn/y5SJKU93NRrfUkae2DQ5waG9zLJ5NqAAA0VFmFVeagQJdfh2Rx3UxBZrsjAAAA4ElIqgEA8CvO+iSSTyEvXotr+6soZ7eah/+Hu0MBAAAAqvDJNdUA1K6istKj2gGA6gSHdVabAb9TcFhnd4fiVllZWXr00UeVlZXl7lAAoE6sPQXAnzBSDfBDQQEBev2TAzWWny0usx1rq3d3vwhS8wDgYqmpqcrJyVFRUZH69u3r7nAAoFaM+AbgT/wyqdY0uJndEUDD0GnyPexICXieoqIiuyMAAAA8g1+OMYkfMUZduvVQ/Igx7g4FADzKudBIlQa307nQSHeHAgAAAAAezS9HqvWI6qUeUb3cHQYAeBx2pAQAAMBvMZsBqJ5LR6rt3LlTQ4cOVWxsrJYvX16l/N///rfGjh2rO+64Q4mJidqxY4crwwEAAPAolYbh7hAAAKgTsxmA6rlspJrVatWcOXO0cuVKWSwWjRw5UjExMerSpYutztKlS3X77bfr97//vb799lvdd9992r59u6tCAgAA8CgBJpP2Hz9da52yikrbsba6Pa4KdWpsgC8pq7DKHBTo7jAAr8VsBqB6LkuqZWdnq2PHjurQoYMkKT4+XpmZmXZJNZPJpPPnz0uSzp07p3bt2rkqHAAAAMAvMW2LzZUAf1VRWamgAO9aSp57tndxWVItPz9f7du3t51bLBZlZ2fb1ZkyZYomTZqkN954Q8XFxVq5cqWrwgEAAAD80rnQSIX8/I0KW0e4OxQAaFRBAQF6/ZMDNZafLS6zHWurN7F/d6fHVhPu2d7FrSnbjIwMjRgxQjt37tTy5cs1ffp0VVZWujMkOFlWVpYeffRRZWVluTsUwC0quKcBANystPnlKggb6JVTt3gfBWpWVmF1dwhwAW++Z/sjl41Us1gsysvLs53n5+fLYrHY1XnnnXe0YsUKSdINN9yg0tJSnT59Wm3btnVVWGhkqampysnJUVFRkfr27evucIBG542fjgHwLE2Dm9kd4TpZWVlKS0vT6NGj6bd4CN5HgZo5a1qzxNRmoKFcNlKtZ8+eOnr0qHJzc1VWVqaMjAzFxMTY1QkLC9Nnn30mSTp8+LBKS0vVpk0bV4UENygqKrI7AgAAx8SPGKMu3XoofsQYd4fi81JTU7V3716lpqa6OxQAAOAFXDZSLSgoSLNmzdLkyZNltVqVkpKi8PBwLVq0SJGRkRo0aJBmzJihP/7xj0pNTZXJZNK8efNkMplcFRIA+CxTkNnuCMB39IjqpR5Rvdwdhl/gw0AAjcEbF88HUD2XJdUkKTo6WtHR0XaPTZs2zfb/Ll26aN26da4MAQD8Qotr+6soZ7eah/+Hu0OBH+CPAaBx8IEJ4JuY1gz4Dpcm1QAAjSM4rLOCwzq7Owz4Cf4YABoHH5gAAODZSKoBAAAAHogPTNCYvHEUshHQxO4IAI2NpBoAAAAA+DlvHIV8LjRSIT9/o8LWEY12TQCOiYiIUGJiohYsWCBJqqio0K233qqoqCgtW7as3u3ExMTonXfeqXVzy/rUcTbv+igCHqfSMDyqHQAAAAD/JysrS48++qiysrLcHYrTlTa/XAVhA1Xa/HJ3hwKgBs2bN1dOTo5KSkokSZ9++qksFoubo3IeRqrhogSYTNp//HSN5WUVlbZjbfV6XBXq9NgAAI7JyspSWlqaRo8erb59+7o7HADwOp44HTE1NVU5OTkqKiri3g6fQZ/Fu0RHR+vjjz/WsGHDlJGRofj4eH3xxReSpDNnzujJJ59Ubm6umjVrpjlz5qhbt246ffq0HnvsMeXn5+v666+X8auBOJs2bdKaNWtUXl6uqKgozZ49W4GBgW55bYxUA+Ay7FoGeJfU1FTt3btXqamp7g4FgI/z1dFT50IjVRrcTudCI90dik1RUZHdEfAF9Fm8S1xcnN5//32Vlpbqm2++UVRUlK1syZIl6t69uzZv3qxHHnlETzzxhCTp5Zdf1o033qiMjAzFxsbq3//+tyTp8OHD2rJli958801t2rRJAQEB2rx5s1tel8RINQAuxK5lgHfhDy8AjcVXR0+VNr+cqYhAI6DP4l26deumEydOKD09XdHR0XZlX3zxhZYsWSJJ6tevn86cOaPz58/r888/10svvSRJuu2229S6dWtJ0meffaavvvpKI0eOlCSVlJSobdu2jfhq7JFUA+Ay7FoGAPAklYahAJPJ3WFA/EEMAP4mJiZG8+fP1+rVq3XmzJkGt2MYhkaMGKHHHnvMidE1XJ3TP7dv367KysrGiAUAgCp8dYoQgMb3y1qwNf377VqwNf0DAACOGTlypB588EFFRNjv1tu7d2+99957kqRdu3YpNDRULVq00E033WSb1rljxw79/PPPki6MZtu6datOnTol6cKabN9//30jvhJ7dSbV3n//fQ0ZMkTz58/X4cOHGyMm+KEKErcAalDfNTO4jwAAAACeqX379ho3blyVx6dMmaL9+/crMTFRCxcu1Lx58yRJDz74oPbs2aP4+Hh9+OGHuvzyC1Pru3TpoocfflgTJ05UYmKiJk6cqJMnTzbqa/m1Oqd/LliwQOfPn1d6erpmzpwpk8mk5ORkxcfHq0WLFo0RI7xY0+BmdseaBAUE6PVPDtRa52xxme1YW92J/bs7GCUAT1bfKUJ13Ue4h8DfNWkabHcEAABwtS+//LLKY3369FGfPn0kSZdccoleeeWVKnVCQ0P1+uuvV9tmXFyc4uLiqjy+ffv2i4zWcfXa/bNFixYaOnSo4uLidPLkSX344YdKTk7WmjVrXB0fvFz8iDHq0q2H4keMcXcoAAD4tesGxslydbiuG1i1EwrAe1UahrtDAAC/VedItczMTG3YsEHHjx9XUlKS3n77bbVt21bFxcWKj4/X2LFjGyNOt6iorFRQQL3yjqhBj6he6hHVy91hAADg967oGqkruka6OwwATvbLWoE1+e1agTXpcVWo02MDGoqNZeAt6kyqbdu2TRMmTNBNN91k93izZs307LPPuiwwT8CURAAAAAAAGhfJYniLOpNqU6ZMUbt27WznJSUl+umnn3TllVeqX79+Lg0OAABPYwoy2x0BAFUxygSAP6F/6L/qTKpNmzZN69ats50HBARo2rRpWr9+vUsDAwDAE7W4tr+Kcnarefh/uDsUAPBYjDIB4E/oH/qvOpNqVqtVZvP/ZVvNZrPKy8tdGhQAAJ4qOKyzgsM6uzsMAC5Q313LAQD4NfqH/qvOVfjbtGmjzMxM2/lHH32k0FA+MQIAwNuwQxxQO3YtBwCg8UVEROjxxx+3nVdUVKhv3766//77JUkbNmzQnDlz3BVereocqfb000/r8ccf19y5c2UYhsLCwvTCCy80RmwAAMCJmI4F1I5dy+GLGIEJwBHOXhOzPu01b95cOTk5KikpUXBwsD799FNZLBanxeBKdSbVrrrqKqWlpamwsFCSFBIS4vKgAAAAAAAXL37EGGV+8J4GDRvu7lAAeIG6PoR1VH0/jI2OjtbHH3+sYcOGKSMjQ/Hx8friiy+cFoer1JlUk6SPP/5YOTk5Ki0ttT02ZcoUlwUFAAAAALh4jMBsPFlZWUpLS9Po0aPVt29fd4fj0xiB6Xvi4uL0yiuvaODAgfrmm2+UkpLiG0m1WbNmqaSkRLt27dKoUaO0detW9ezZszFiAwAAAAD4EFOQ2e7oS1JTU5WTk6OioiKSai7GCEzf061bN504cULp6emKjo52dzj1VudGBV9++aXmz5+vVq1aacqUKVq3bp2OHj3aCKEBAAAA8EWMMvFfLa7tL/OlV6nFtf3dHYrTFRUV2R3hOj2iemnqE08zCtPHxMTEaP78+YqPj3d3KPVW50i1pk2bSpKaNWum/Px8hYaG6uTJky4PDADgH5y9GCoAwPMxysR/BYd1VnBYZ3eHAcADjRw5Uq1atVJERIR27drl7nDqpc6k2sCBA3X27FlNmjRJycnJMplMGjVqVGPEBgDwA+xICQD+h3W+AAC/1b59e40bN67asnfffVcfffSR7TwtLU3t27dvrNBqVGtSrbKyUv369VOrVq00dOhQDRw4UKWlpWrZsmVjxQcAHomFaAEAAAD4mkrDcOqH1fWZlfLll19WeaxPnz7q06ePJCk5OVnJyclOi8mZal1TLSAgQHPmzLGdm81mEmoAoAsL0e7du1epqanuDgUA4GGaNA22O8Lz+fLi+QDgCGcvy+Lry7zUuVFBv379tHXrVhmG0RjxAIBXYCFa+CIWDgec47qBcbJcHa7rBsa5OxTUky8vng8AcJ0611Rbt26dVq5cqaCgIJnNZhmGIZPJpP/93/9tjPgAAEAjYeFwwDmu6BqpK7pGujsMOIDF8wEADVFnUq26ua31tXPnTj377LOqrKzUqFGjdN9999mVP/fcc7YdHUpKSnTq1Cnt2bOnwdcDAAANx8LhAADUjB3LAfxWnUm1zz//vNrHb7rpplqfZ7VaNWfOHK1cuVIWi0UjR45UTEyMunTpYqvz5JNP2v6/Zs0aHThwoL5xAwAAAADQaNixHMBv1ZlUe+2112z/Ly0tVXZ2tnr06KHVq1fX+rzs7Gx17NhRHTp0kCTFx8crMzPTLqn2axkZGXrooYcciR0AAAAAAFSDDTgA16tzo4JXX33V9m/lypVKT09X69at62w4Pz9f7du3t51bLBbl5+dXW/f777/XiRMn1LdvXwdCBwAAAAAA1WEDDniLmTNnql+/fkpISLB7fMaMGYqJidHw4cM1dOhQTZ8+XXl5edW2UV5ergULFmjIkCEaMWKExowZox07dkiSYmJi7AZxffDBB5oxY4YkacOGDerWrZu+/vprW3lCQoJOnDhRr9jrTKr9Vvv27XX48GFHn1arjIwMDR06VIGBgU5tFwAaopLdjgEAAODlgsM6q82A37EJBxxSUVnZ6O0lJydrxYoV1ZZNnz5d7733nj744AN1795d48ePV1lZWZV6ixYt0smTJ5Wenq53331XL7/8sgoLC23l+/fv17ffflvtNdq3b69XX321nq/IXp3TP+fOnSvT/1+MsbKyUgcPHlT37t3rbNhisdhlEPPz82WxWKqt+/7772vWrFn1jRkAXKqu9TIk1swAAAAA4HuCAgL0+ifOW+9+Yv+680c33XRTnSPDTCaTJkyYoA8//FA7d+7U4MGDbWXFxcV6++23lZmZKbP5wnTnSy+9VHFxcbY699xzj5YuXaqFCxdWafu2227Tnj17dOTIEXXq1Km+L01SPZJqkZH/tx14YGCg4uPj1atX3TuD9ezZU0ePHlVubq4sFosyMjKqDf7w4cM6e/asbrjhBocCBwAAAAAAgP/o3r27jhw5YvfYsWPHFBYWphYtWtT4vNtvv11///vfdezYsSplAQEBmjx5spYtW6YXXnjBoXjqTKoNHTpUTZs2tU3NtFqtKi4uVrNmzWpvOChIs2bN0uTJk2W1WpWSkqLw8HAtWrRIkZGRGjRokKQLo9Ti4uJso+EAAAAAAACA3zIauFRPQECAJk2apGXLlmnAgAFVyhMSErR06VLl5uY61G6dSbUJEyZo5cqVCgkJkSSVlJRo0qRJWrduXZ2NR0dHKzo62u6xadOm2Z2z4ycAoDZNg5vZHQEAAAD4p4MHD6pfv352j3Xs2FE//PCDzp8/X+totaSkJC1fvlxdu3atUhYUFKSJEyfqb3/7m0Px1LlRQWlpqS2hJnWfNEMAACAASURBVEkhISEqLi526CIAADRU/Igx6tKth+JHjHF3KAAAADXig0DAdQzD0OrVq3Xy5En172+/o22zZs2UkpKiZ5991raJQUFBgbZs2WJXr0mTJho/frxSU1OrvcaIESP02WefqaCgoN5x1ZlUa9asmfbv3287/+qrrxQcHFzvCwAAcDF6RPXS1CeeVo+outfzBAAAcBc+CAQa5tFHH9Wdd96p7777TgMGDNDbb79tK5s/f76GDx+uoUOHat++fVq9erVtM4Jfe/jhhxUaGqr4+HglJCTo/vvvr3bU2qhRo1RRUVFtHGazWWPHjtWpU6fqHXud0z+ffPJJTZs2Te3atZNhGPrpp5/0l7/8pd4XAADUrEnTYLsjAAAAvFOPqF58CAivV1FZWa8dOx1pLyig9vFcL774YrWPz5s3r97XMZvNmj59uqZPn16lbPv27Xb1/vnPf9rOk5OTlZycbDsfN26cxo0bV+/r1plUu+6667RlyxZ99913kqRrrrlGTZo0qfcFADSOrKwspaWlafTo0erbt6+7w/F5zhref93AOB38n+269uYYZ4QFAAAAAA1WVwLM3e15mjpf3dq1a1VcXKyuXbuqa9euKioq0tq1axsjNgAOSE1N1d69e2ucHw7nctbw/iu6RmrwhKm6omukkyIDAAAAADSGOpNqaWlpatWqle28devWdvNbAXiGoqIiuyNci3W+AAAAAMC/1ZlUq6yslGEYtnOr1ary8nKXBgUAAAAAAAB4sjrXVLv11lv18MMP684775QkrVu3TgMGDHB5YAAAAAAAAICnqjOp9oc//EFvvfWW3nzzTUlSRESEfvrpJ5cHBgAAAAAAAHiqOqd/BgQEKCoqSldccYX27dunrKwsde7cuTFiAwAAAAAAgI/64YcfNHbsWMXFxSk+Pl6rVq2ylc2YMUMxMTEaPny4hg4dqunTpysvL6/adsrLy7VgwQINGTJEI0aM0JgxY7Rjxw5JUkxMjB566CFb3Q8++EAzZsyQJG3YsEHdunXT119/bStPSEjQiRMn6hV/jSPVvvvuO2VkZCg9PV2hoaGKi4uTJK1Zs6ZeDQMAAAAAAMB7lFVYZQ4KbLT2AgMDNWPGDPXo0UPnz59XSkqKbrnlFnXp0kWSNH36dA0bNkyGYWjVqlUaP368Nm/eLLPZbNfOokWLdPLkSaWnp8tsNuunn37S7t27beX79+/Xt99+a2v319q3b69XX31Vf/3rXx1+fTUm1W6//Xb17t1by5YtU8eOHSVJqampDl8AgPdp0jTY7ggAAAAA8H3moEDd9fI2p7W39sEhtZa3a9dO7dq1kyS1aNFCnTp1Un5+fpXkl8lk0oQJE/Thhx9q586dGjx4sK2suLhYb7/9tjIzM23JtksvvdQ2OEyS7rnnHi1dulQLFy6sEsNtt92mPXv26MiRI+rUqZNDr6/G6Z8vvfSSLrvsMo0bN05//OMf9dlnn9ntAgqgcVU24u/fdQPjZLk6XNcNjKu7MgAAAAAAF+nEiRM6ePCgoqKiaqzTvXt3HTlyxO6xY8eOKSwsTC1atKjxebfffrsOHDigY8eOVSkLCAjQ5MmTtWzZModjrnGk2uDBgzV48GAVFRUpMzNTq1atUkFBgWbPnq3Y2FjdeuutDl8MQMMFmEzaf/x0jeVlFZW2Y231JKnHVaG1ll/RNVJXdI10PEgAAAAAABxUWFioqVOn6sknn6w1OdbQwV4BAQGaNGmSli1bpgEDBlQpT0hI0NKlS5Wbm+tYu3VVaN68uRITE/Xqq69qx44d6t69u/72t785dBEAAAAAcDaWrAAA71deXq6pU6cqMTFRQ4bUPl304MGDVTbP7Nixo3744QedP3++1ucmJSVpz5491W52EBQUpIkTJzqc76ozqfZrrVu31pgxY+x2YwAAAAAAd2DJCgDwboZh6KmnnlKnTp10zz331Fpv9erVOnnypPr3729X1qxZM6WkpOjZZ59VWVmZJKmgoEBbtmyxq9ekSRONHz++xv0CRowYoc8++0wFBQX1jt+hpBoAAAAAeIorukZq8ISpLFsBAF7qiy++0KZNm5SVlaWkpCQlJSVpx44dtvL58+dr+PDhGjp0qPbt26fVq1dX2flTkh5++GGFhoYqPj5eCQkJuv/++6udRjpq1ChVVFRUG4vZbNbYsWN16tSpesdf45pqAAAADcF0LAAAAO9UVmGtc8dOR9szBwXWWN67d29988031ZbNmzev3tcxm82aPn26pk+fXqVs+/btdvX++c9/2s6Tk5OVnJxsOx83bpzGjRtX7+syUg0AADgV07EAAAC8U20JME9oz9MwUg0AADgVOwgDAABvwOh6XCxGqgEAAAAAAL/D6HpcLEaqAT6iaXAzuyMAAAAAoGaMrsfFYqQa4CPiR4xRl249FD9ijLtDAQAAAADA5zFSDfARPaJ6qUdUL3eHAQAAAACAX2CkGgAAAACgVizoDsAVSktLNXLkSA0fPlzx8fFavHixrWzs2LEaOnSoEhMTNWzYMM2ZM0dnz56ttp3CwkLNmjVLgwcPVnJyssaOHau9e/dKkiIiIjRv3jxb3ddee01LliyRJC1ZskRRUVE6deqUrfyGG26od/wk1QAAAAAAtWJBd8A/lJZXNGp7ZrNZq1at0nvvvaeNGzfqk08+0b/+9S9b+YIFC7R582a99957MpvNeuCBB6pt549//KNat26tbdu2acOGDXruued0+vRp2zW2bdumgoKCap8bGhqq119/vUGvj+mfAAAAAIBasaA74B+aNglS98nLndbegRX31VpuMpkUEhIiSaqoqFBFRYVMJlOVemazWX/4wx8UGxurr7/+Wt26dbOVHT9+XHv37tWCBQsUEHBh7FiHDh3UoUMHSVJQUJDGjBmjVatW6ZFHHqnSdkpKit59913de++9uuSSSxx6fYxUAwD4BaatAAAAAJ7HarUqKSlJN998s26++WZFRUVVWy8wMFDdunXTkSNH7B7PycnRtddeq8DAwBqvcdddd2nz5s06d+5clbLmzZsrOTlZq1evdjh2kmoAAL/AtBUAAADA8wQGBmrTpk3asWOHsrOzdejQoRrrGobRoGu0aNFCSUlJNSbOxo0bp40bN+r8+fMOtevSpNrOnTs1dOhQxcbGavny6ocPvv/++4qLi1N8fLwee+wxV4YDAPBjV3SN1OAJU5m6AgAAPBqj6+GvWrVqpT59+uiTTz6pttxqterQoUPq1KmT3ePh4eH6+uuvZbVaa21//PjxWr9+vYqLi6u9dkJCgv7+9787FLPLkmpWq1Vz5szRihUrlJGRofT0dH377bd2dY4eParly5frzTffVEZGhp588klXhQMAAAAAgMdjdD38SUFBgW1Hz5KSEv3P//xPlaSZJJWXl2vhwoUKCwuzW09Nkq666ipFRkZq8eLFtpFsJ06c0Mcff2xX75JLLtGwYcP0zjvvVBvLhAkTtG7dOlVU1H+zBpdtVJCdna2OHTvaFoaLj49XZmamunTpYquTlpamu+66S61bt5YktW3b1lXhAAAAAADg8dgUAv7kxx9/1IwZM2S1WmUYhoYNG6aBAwfayh9//HGZzWaVlZXp5ptv1iuvvFJtO88++6zmzZun2NhYBQcHKzQ0VH/4wx+q1Js4caLWrl1bbRtt2rRRbGysUlNT6x2/y5Jq+fn5at++ve3cYrEoOzvbrs7Ro0clSXfeeacqKys1ZcoUDRgwwFUhAQAAAAAAoAal5RV17tjpaHtNm9SceurWrZs2btxYbdmaNWvqfZ0WLVromWeeqbbsyy+/tP3/0ksv1d69e23nDz30kF3dmTNnaubMmfW+rsuSavVhtVp17NgxrVmzRnl5ebr77ru1efNmtWrVyp1hAQAAAAAA+J3aEmCe0J6ncdmaahaLRXl5ebbz/Px8WSyWKnViYmLUpEkTdejQQVdffbVt9BoAAAAAAADgqVyWVOvZs6eOHj2q3NxclZWVKSMjQzExMXZ1Bg8erN27d0u6sDjd0aNHbWuwAQAAAAAAAJ7KZePwgoKCNGvWLE2ePFlWq1UpKSkKDw/XokWLFBkZqUGDBql///769NNPFRcXp8DAQE2fPl2hoaGuCgkAAAAAAABwCpdObo2OjlZ0dLTdY9OmTbP932QyObwIHAAAAAAAAOBuLpv+CQAAAAAAAPgqkmoAAAAAAACAg0iqAQAAAAAAAA4iqQYAAAAAAAA4iKQavEaTpsF2RwAAAAAAAHchqQavcd3AOFmuDtd1A+PcHQoAAAAAAPBzQe4OwNsxeqrxXNE1Uld0jXR3GAAAAAAAAIxUu1iMngIAAAAAAPA/jFS7SIyeAgAAAAAA8D+MVAMAAAAAAAAcRFINAAAAAAAAcBBJNQAAAAAAAMBBJNUAAAAAAAAAB5FUAwAAAAAAABxEUg0AAAAAAABwEEk1AAAAAAAAwEEk1QAAAAAAAAAHkVQDAAAAAAAAHERSDQAAAAAAAHAQSTUAAAAAAADAQSTVAAAAAAAAAAeRVAMAAAAAAAAcRFINAAAAAAAAcBBJNQAAAAAAAMBBJNUAAAAAAAAAB5FUAwAAAAAAABxEUg0AAAAAAABwEEk1AAAAAAAAwEEk1QAAAAAAAAAHuTSptnPnTg0dOlSxsbFavnx5lfINGzaob9++SkpKUlJSkt5++21XhgMAAAAAAAA4RZCrGrZarZozZ45Wrlwpi8WikSNHKiYmRl26dLGrFxcXp1mzZrkqDAAAAAAAAMDpXDZSLTs7Wx07dlSHDh1kNpsVHx+vzMxMV10OAAAAAAAAaDQuS6rl5+erffv2tnOLxaL8/Pwq9bZt26bExERNnTpVP/zwg6vCAQAAAAAAAJzGrRsVDBw4UNu3b9fmzZt1880364knnnBnOAAAAAAAAEC9uCypZrFYlJeXZzvPz8+XxWKxqxMaGiqz2SxJGjVqlPbv3++qcAAAAAAAAACncVlSrWfPnjp69Khyc3NVVlamjIwMxcTE2NX58ccfbf/fvn27Onfu7KpwAAAAAAAAAKdx2e6fQUFBmjVrliZPniyr1aqUlBSFh4dr0aJFioyM1KBBg7RmzRpt375dgYGBat26tZ5//nlXhQMAAAAAAAA4jcuSapIUHR2t6Ohou8emTZtm+/9jjz2mxx57zJUhAAAAAAAAAE7n1o0KAAAAAAAAAG9EUg0AAAAAAABwEEk1AAAAAAAAwEEk1QAAAAAAAAAHkVQDAAAAAAAAHERSDQAAAAAAAHAQSTUAAAAAAADAQSTVAAAAAAAAAAeRVAMAAAAAAAAcRFINAAAAAAAAcBBJNQAAAAAAAMBBJNUAAAAAAAAAB5FUAwAAAAAAABxEUg0AAAAAAABwEEk1AAAAAAAAwEEk1QAAAAAAAAAHkVQDAAAAAAAAHERSDQAAAAAAAHAQSTUAAAAAAADAQSTVAAAAAAAAAAeRVAMAAAAAAAAcRFINAAAAAAAAcBBJNQAAAAAAAMBBJNUAAAAAAAAAB5FUAwAAAAAAABxEUg0AAAAAAABwEEk1AAAAAAAAwEEk1QAAAAAAAAAHkVQDAAAAAAAAHOTSpNrOnTs1dOhQxcbGavny5TXW27p1qyIiIrRv3z5XhgMAAAAAAAA4hcuSalarVXPmzNGKFSuUkZGh9PR0ffvtt1XqnT9/XqtXr1ZUVJSrQgEAAAAAAACcymVJtezsbHXs2FEdOnSQ2WxWfHy8MjMzq9RbtGiR7r33XjVt2tRVoQAAAAAAAABOZTIMw3BFwx988IE++eQTPfvss5KkjRs3Kjs7W7NmzbLV2b9/v1599VUtWbJEY8eO1fTp09WzZ89a2+3Tp4+uuOIKV4QMAABQp9DQUL322mt11qPPAgAA3Km+fRY0XJC7LlxZWal58+bp+eefd+h5u3btclFEAAAAzkOfBQAAwLe5bPqnxWJRXl6e7Tw/P18Wi8V2XlhYqEOHDmncuHGKiYnRv/71L/3Xf/0XmxUAAAAAAADA47lspFrPnj119OhR5ebmymKxKCMjQwsXLrSVt2zZ0u4T3PpO/wQAAAAAAADczWVJtaCgIM2aNUuTJ0+W1WpVSkqKwsPDtWjRIkVGRmrQoEGuujQAAAAAAADgUi7bqAAAAAAAAADwVS5bUw0AAAAAAADwVSTVAAAAAAAAAAeRVHOSpUuXKj4+XomJiUpKStLevXvr9bwTJ04oISHBxdF5nxtuuMHufMOGDZozZ46bovE/1157rZKSkmz/Tpw44e6QvNYvX8v4+HgNHz5cr7/+uiorKyVJ+/bt0zPPPOPyGE6cOKHNmze7/Dru1NB7sKPuvfdenT171iVte7uPPvpIEREROnz4sLtDQR3oszgXfRb3os/iPPRZGgd9Fs9AvwXO4rKNCvzJl19+qY8//ljvvvuuzGazCgoKVF5e7u6wgAYLDg7Wpk2bnNZeRUWFgoL883bz66/lqVOn9Nhjj+n8+fOaOnWqevbs2Sg7Hn///fdKT09XYmKiy6/lDhdzD67vz6ZhGDIMQ3/7298uNlyflZ6erl69eikjI0NTp0512XX8+X7iDPRZ4GvoszgPfRbXo8/iOei3wFn47jrByZMnFRoaKrPZLElq06aNJOmll17SP/7xD5WWluqGG27QnDlzZDKZ9NVXX+nJJ5+UJN1yyy1ui9tbbd++XUuXLlV5ebkuueQSLViwQJdeeqmWLFmi48eP6/jx4zp9+rQmT56s0aNHa9euXVq8eLFCQkJ07Ngx9enTR3/605+0YcMGffPNN3rqqackSWlpafr2229t3xvY++qrrzRv3jwVFRUpNDRUzz//vNq1a6e0tDS99dZbKi8vV8eOHTV//nw1a9ZMM2bMkNls1sGDB3XjjTdq5syZ7n4Jbte2bVvNnTtXI0eO1EMPPaTdu3fr9ddf17Jly7R79249++yzkiSTyaQ33nhDzZs315w5c5SVlaWwsDAFBQUpJSVFw4YNU0xMjN555x21adNG+/bt0/z587VmzZpq21m4cKEOHz6spKQkjRgxQhMmTHDjV8H5aroH1/Q1+uVekZubq8svv1y33nqrPvzwQ50/f175+fkaPny4pkyZohMnTmjSpEmKiorS/v37tXz5co0dO1bvvPOOgoOD9fDDDysvL0+VlZV64IEHFBcXV+Pvia8rLCzUF198odWrV+s///M/NXXqVO3atUsvvfSSQkNDdejQIfXo0UMLFiyQyWTSjh079Pzzz6t58+a68cYblZubq2XLlqmoqEhz585VTk6OKioqNGXKFA0ePFgbNmzQtm3bVFRUpMrKSr3xxhvufsleiz5L46LP4h70WS4efRbXoM/iGei3wKkMXLTz588bw4cPN4YMGWLMnj3b2LVrl2EYhnH69Glbnccff9zIzMw0DMMwEhISjN27dxuGYRjz5s0z4uPjGz9oD9etWzdj+PDhtn/R0dHG008/bRiGYZw5c8aorKw0DMMw0tLSjOeff94wDMNYvHixkZiYaBQXFxunTp0yBgwYYOTl5RlZWVlGZGSkcfz4caOiosKYMGGCsWXLFuP8+fPGoEGDjLKyMsMwDGPMmDHG119/7Z4X7GF+/fV/4IEHjLKyMmPMmDHGqVOnDMMwjIyMDGPGjBmGYRhGQUGB7XkvvviisXr1asMwDOOJJ54w7rvvPqOioqLxX4AHuf7666s81qtXL+PkyZNGVlaWcd999xmGYRj333+/sWfPHsMwLtxTysvLjS1bthiTJ082rFar8eOPPxq9e/c2tmzZYhiGYQwcOND2/cjOzjbuvvvuGtv59XV8UU334Jq+RosXLzZGjBhhFBcXG4ZhGOvXrzduueUWo6CgwCguLjbi4+ON7OxsIzc314iIiDC+/PJL27V+afODDz4wnnrqKdvjZ8+erfX3xNdt2rTJmDlzpmEYF+6l+/btM7Kysowbb7zR+OGHHwyr1WqMHj3a+Pzzz42SkhJjwIABxvHjxw3DMIxHHnnE9vO5cOFCY+PGjYZhGMbPP/9sDBkyxCgsLDTWr19v9O/f3+59FQ1Dn8X56LO4F30W56HP4nr0WTwD/RY4EyPVnCAkJEQbNmzQnj17tGvXLj3yyCN67LHHFBISohUrVqikpERnzpxReHi4evfurXPnzummm26SJCUlJemTTz5x8yvwPL8dyr9hwwZ99dVXkqS8vDw98sgjOnnypMrKynTllVfa6g0aNEjBwcEKDg5Wnz59tG/fPrVs2VLXXXedOnToIEmKj4/XF198oWHDhqlv3776+OOP1alTJ5WXlysiIqJxX6iH+u3X/9ChQzp06JDuueceSVJlZaUuu+wySVJOTo7++te/6ty5cyosLNStt95qe96wYcMUGBjYuMF7qRtvvFHz5s1TYmKihgwZopCQENvPaUBAgC677DL16dOnQe34upruwbWJiYlRcHCw7fzmm29WaGioJCk2NlZffPGFBg8erMsvv1zXX399led37dpVL7zwgv785z9r4MCB6t27d62/J74uIyND48aNkyTFxcUpIyNDt912m6677jq1b99ektStWzd9//33CgkJUYcOHezuyWlpaZKkf/7zn9q+fbtef/11SVJpaal++OEHSRdGSV1yySWN/dJ8Dn0W56PP4l70WRoffZaGo8/iGei3wJlIqjlJYGCg+vTpoz59+qhr165666239M0332j9+vUKCwvTkiVLVFpa6u4wfcIzzzyjCRMmaNCgQbZhur8wmUzVPue3j/9yPmrUKL366qvq1KmTkpOTXRe0lzMMQ+Hh4XrrrbeqlM2YMUOvvPKKunXrpg0bNmj37t22smbNmjVmmF4hNzdXgYGBatu2rd3CqPfdd5+io6O1Y8cO/e53v9OKFStqbScwMFCGYUiS3b3F0XZ8xW/vwRs3bqzxayRV/dms6R7RvHnzaq93zTXXaMOGDdqxY4f++te/qm/fvoqNja3x98SXnTlzRllZWTp06JBMJpOsVqtMJpOio6Nt01ukC98jq9VaZ3uLFy9Wp06d7B7bu3cv9xMnos/SeOizND76LM5Dn8U16LO4F/0WOBu7fzrBkSNHdPToUdv5wYMHdc0110iSQkNDVVhYqK1bt0qSWrVqpZYtW2rPnj2S5PO727jCuXPnZLFYJEkbN260K8vMzFRpaalOnz6t3bt32xZUzc7OVm5uriorK7Vlyxb16tVLkhQVFaW8vDylp6ezo1ktrrnmGhUUFOjLL7+UJJWXlysnJ0fShTUJLrvsMpWXl/PzXIeCggLNnj1bd911V5UO0fHjxxUREaH77rtPPXv21Hfffacbb7xR27ZtU2VlpX766Se7zv8VV1xhGwmxbdu2WtsJCQlRYWFh47xIN6juHnz55ZfX+DWqzqeffqozZ86opKREH330kW688cZa6+fn56tZs2ZKSkrSpEmTdODAgVp/T3zZ1q1blZSUpH/84x/avn27duzYoSuvvNL2Pvdb11xzjXJzc2079L3//vu2sltvvVVvvPGG7Q+LAwcOuP4F+Bn6LI2LPkvjo8/iHPRZXIM+i/vRb4GzMVLNCYqKivTMM8/o7NmzCgwMVMeOHTVnzhy1bNlSCQkJuvTSS+12y3n++ef15JNPymQysehvA0yZMkXTpk1T69at1adPH7ut0yMiIjRu3DidPn1aDzzwgCwWi44ePaqePXtq7ty5tkV/Y2Njbc+5/fbbdfDgQbVu3dodL8crmM1mLV68WM8884zOnTsnq9Wq8ePHKzw8XNOmTdOoUaPUpk0bRUVF+XRHqCFKSkqUlJSkiooKBQYGKikpyTbU/tdWrVqlXbt2yWQyKTw8XAMGDFBQUJA+++wzxcXFKSwsTN27d1fLli0lXfg9eOqpp7Ro0SK7KRbVtWMymRQQEKDhw4crOTnZ5xb9rekefOTIkWq/RtW57rrr9NBDD9kW/e3Zs6fdveW3Dh06pPnz5ysgIEBBQUH605/+VOvviS9LT0/Xvffea/fYkCFD9Oabb+qqq66qUj84OFizZ8/W5MmT1bx5c0VGRtrKHnjgAT333HMaPny4KisrdeWVV2rZsmUufw3+hD5L46LP0vjoszQcfRbXo8/ifvRb4Gwm45e0KuDllixZoubNm2vSpEl2j+/atcu2W1F17r//fk2YMEH9+vVrjDABhxQWFiokJESnT5/WqFGj9Oabb/rVmheN4Zf1j2bNmuXuUPzGLz/XhmHo6aef1tVXX+1zfzgBtaHPAl9En8X16LO4x/9r715C4d3jOI5/ZobJxAL9KUubYSyUGgsLCVmYwRORZsPCwiULhchKSC6RGikWFjYWioaFsmQhRRZKUpJILFwal+Q2Z3FqOuffOTpz/s4xY96vmsUzze/2rD59n+c3P3ILPsKbaohafr9fNTU1ysjIIJwibDU1Ncnv9+vl5UUtLS2EU3wLCwsLWlpa0svLixwOh2pra796SkBYI7MgEpBZ8F2RW/AR3lQDAAAAAAAAQsRBBQAAAAAAAECIKKoBAAAAAAAAIaKoBgAAAAAAAISIohqAsORwOGQYRvAzMzPzy32enZ1pZWUleL23t6eBgYFf7hcAAEQ3cgsARCcOKgAQlnJycrS7u/upfW5tbWl2dlbT09Of2i8AAIhu5BYAiE4xXz0BAAhFUVGR3G631tfXZbFY1N/fr/HxcZ2cnKihoUEej0eBQEAjIyPa2NiQyWRSc3OzXC6XxsbGdHR0JMMwVFlZKYfDEQyrt7e36unp0enpqWw2m/r6+pSZmSmv16vz83OdnZ3p/Pxc9fX1qqur0+Pjo9ra2nRxcaH393e1tLTI5XJ99e0BAABhhNwCAN8bRTUAYenp6UmGYQSvGxsbg+EvLS1NPp9Pg4OD6u7u1vz8vJ6fn1VWViaPx6O1tTUdHBzI5/Pp5uZG1dXVcjqdam9v/9MT362trWD/Xq9XWVlZmpqa0ubmprq6uuTz+SRJx8fHmpub0/39yZJQJQAAAehJREFUvUpLS+XxeLSxsaHU1NTg9o67u7v/69YAAIAwQ24BgOhEUQ1AWIqLiwuGw58VFxdLkux2ux4fH5WQkCBJslqt8vv92tnZkdvtlsVi0Y8fP5Sbm6u9vb3g7/7Kzs6OvF6vJCkvL0+3t7e6v7+XJBUUFMhqtSo5OVnJycm6urqS3W7X8PCwRkdHVVhYKKfT+ZnLBwAAEYTcAgDRiYMKAESc2NhYSZLZbJbVag1+bzab9fr6+unj/XEMi8Wi19dXpaena3FxUXa7XRMTE5qcnPz0cQEAQOQjtwDA90VRDcC343Q6tbq6qre3N11fX2t7e1vZ2dmKj4/Xw8PD37ZZXl6W9Pv2iqSkpA+fEF9eXspms8kwDDU0NGh/f/8/WQsAAPjeyC0AELnY/gkgLP383yT5+fnq6Oj4R21LSkq0u7srwzBkMpnU2dmplJQUJSYmymw2q6KiQlVVVXI4HME2ra2t6unpUXl5uWw2m4aGhj4c4/DwUCMjIzKbzYqJiVFvb++/WicAAIh85BYAiE6mQCAQ+OpJAAAAAAAAAJGE7Z8AAAAAAABAiCiqAQAAAAAAACGiqAYAAAAAAACEiKIaAAAAAAAAECKKagAAAAAAAECIKKoBAAAAAAAAIaKoBgAAAAAAAIToN0+3uRyscIw2AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1227.12x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "6mKo6TigwA5Q",
        "outputId": "0396257e-0af9-4e73-8372-058cc3180226"
      },
      "source": [
        "#Plot comparing sensitivity of NC and PD.\n",
        "labels = ['Sad', 'Happy', 'Fear', 'Disgust', 'Surprise', 'Anger']\n",
        "x_pos = np.arange(len(labels))\n",
        "\n",
        "multi_pd_m = [sad_3pd[0], happy_3pd[0], fear_3pd[0], disgust_3pd[0], surprise_3pd[0], anger_3pd[0]]\n",
        "multi_pd_s = [sad_3pd[1], happy_3pd[1], fear_3pd[1], disgust_3pd[1], surprise_3pd[1], anger_3pd[1]]\n",
        "\n",
        "multi_nc_m = [sad_3nc[0], happy_3nc[0], fear_3nc[0], disgust_3nc[0], surprise_3nc[0], anger_3nc[0]]\n",
        "multi_nc_s = [sad_3nc[1], happy_3nc[1], fear_3nc[1], disgust_3nc[1], surprise_3nc[1], anger_3nc[1]]\n",
        "\n",
        "#ax = fig.add_axes([0,0,1,1])\n",
        "barWidth = 0.25\n",
        "m1 = plt.bar(x_pos - barWidth/2, multi_pd_m, yerr=multi_pd_s, align='center', alpha=0.5,\n",
        "             width = barWidth)\n",
        "m2 = plt.bar(x_pos + barWidth/2, multi_nc_m, yerr=multi_nc_s, align='center', alpha=0.5,\n",
        "            width = barWidth)\n",
        "#plt.ylabel('Mean brands recalled',fontsize=14,weight='bold')\n",
        "#plt.xlabel('Methods',fontsize=14,weight='bold')\n",
        "plt.ylim(0,1.05)\n",
        "plt.ylim(bottom = 0.6)\n",
        "plt.xticks(ticks=[0,1,2,3,4,5], labels=labels, weight='bold')\n",
        "plt.yticks(weight='bold')\n",
        "plt.legend((m1[0], m2[0]), ('PD', 'NC'))\n",
        "plt.title('MultiClass Classification')\n",
        "\n",
        "#plt.savefig('U2_IFbrrec.png',bbox_inches='tight')\n",
        "#files.download('U2_IFbrrec.png')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'MultiClass Classification')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcHklEQVR4nO3de5hcVZ3u8e9rEtJBQtAkgCGGBAXkEkGIjg4y0yLo4eJlGMIIDAwhEuDAEcJNGW8w+ohANMBgDgbCcDGAcvHMEYhDFCIwChouQsABIYSxQcgFgiQkIQm/+WOtIpWmuru6U92dXnk/z1PP7tp71d5r16791qq1L62IwMzMyvGO3q6AmZk1loPdzKwwDnYzs8I42M3MCuNgNzMrjIPdzKwwDnZrCEkh6f3tTH9cUnOd8xqd59e/YRXcAJKaJbV04/wvl/T1qucnSXpJ0jJJQ/Nwh25Ybt3bxPoWB/smTtICSW9IGtZq/MM5XEd3YZ5XS/p29biI2C0i5lSV2UnSTZIWS3pV0qOSTpfUr4urskEkfUTSHZKWSnpZ0m8lTeiJZUfEiRHxrVyPAcD3gU9FxBYRsSQP52/IMurZJlYOB7sBPAscUXkiaSyweXctTNL7gAeAPwFjI2IIMB4YBwzuruW2U5+PAXcBvwLeDwwFTgIO7Om6ANsATcDjvbBsK0VE+LEJP4AFwNeA31WNmwJ8FQhgdB43B/hiVZljgfuqngcpFCcBq4E3gGXAz6qWs3/++0fA7e3UaXSeX//8fALwB+A1YD5wQlXZYcBtwFLgZeBe4B152peB5/PrngQ+2cby7gN+0E59moGWqudfAZ7J830C+Luqae8nfUG8CiwGfpzHC5gKLAT+AjwG7J6nXQ18G9gJWJ7XfRlwV/V7m/8eBHwPeC4v4z5gUJ52E/BiHn8PsFseX882GQhcDLyQHxcDA6vXHzgj1//PwITe/uz60fbDLXYDuB/YUtIuuSvkC6Tw7bSImA7MBC6M1IXwmRrF9gdu7sRsFwKHAFuSQn6qpL3ytDNIoTOc1Nr9ZyAk7QycAnw4IgYDnyYF2XokbQ58rJP1eQbYFxgCnAf8SNJ78rRvAXcC7wJGAv+ax38K+BtSeA8BDgeWVM80Ip4CdstPt4qI/WosewqwN/DXwLuBs4E387RZwI7A1sBDpO1Q7zb5KvBRYE9gD+AjpC/8im1zvbcDJgI/kPSumu+O9ToHu1VcBxwDHEBqHT/fjcsaSmr11SUibo+IZyL5FSk4982TVwPvAbaPiNURcW+kZuZaUit0V0kDImJBRDxTY/bvIu0HnanPTRHxQkS8GRE/Bv5ICsJKfbYHRkTEyoi4r2r8YOADgCLiDxFR9zIBJL0DOA44NSKej4i1EfHriFiV63VVRLyWn58L7CFpSJ2zPwr4l4hYGBGLSF9YR1dNX52nr46IO0gt/507U3/rOQ52q7gOOJLUxXJtNy9rCSmM6yLpQEn354OaS4GDSF0wABcBTwN3Spov6SsAEfE0cBop4BZKulHSiBqzf4XU4u1MfY6R9Eg+0LoU2L2qPmeTul1+m886OS7X5y7gMuAHuT7TJW1Z7zKzYaT+97d9QUnqJ+m7kp6R9BfW/ToZ1rpsG0aQuncqnsvjKpZExJqq568DW9RbcetZDnYDICKeIx1EPQi4tUaR5ax/QHXb9mbXweJ+Afx9PfWSNBC4hdQFsU1EbAXcQQpPcgv1jIjYAfgscLqkT+Zp10fEx0kt6AAueFtFI14HftOJ+mwPXEHq5hma6zOvqj4vRsTxETECOAGYVjkNNCIujYi9gV1JXTJn1bPMKouBlcD7akw7EvgcqZtrCOk4BZV60fE2eYH0PlWMyuOsD3KwW7WJwH4RsbzGtEeAQyVtnoNqYjvzeQlo77zrbwJ/LekiSdsCSHq/pB9J2qpV2c1IXSqLgDWSDiT1V5Nfd0h+rUgHDdcCb0raWdJ++YthJbCCdX3RrZ0NHCvpLElD83z3kHRjjbLvJIXkolxuAqnFXqnPeEkj89NXctk3JX1Y0l/l0xmX5zq1VZ+aIuJN4Crg+5JG5Fb6x/I6DgZWkX4NbQ58p9XLO9omNwBfkzQ8n/r6Dbp4nMV6n4Pd3pL7see2MXkq6ayKl4BryAfm2jCD1Le9VNL/q7Uc0gHL0cDjkl4ltcrnks40qS77GvAl4CekoDwS+P9VRXYk/QJYRmp5T4uIu0lfBt8ltXJfJB1QPKeN9f41sF9+zJf0MjCd9MugddknSGel/Ca/F2OB/6wq8mHgAUnLcj1PjXQO+paklv4rpG6OJaRupM46k3RGze9IZwFdQNqPr83zfZ50ps79rV7X7jYhnZUzF3g0z/+hPM76IKXjTGZmVgq32M3MCuNgNzMrjIPdzKwwDnYzs8L0+m1Rhw0bFqNHj+7tapiZ9SkPPvjg4ogYXmtarwf76NGjmTu3rTPszMysFknPtTXNXTFmZoVxsJuZFcbBbmZWmF7vYzcz6y6rV6+mpaWFlStX9nZVuqypqYmRI0cyYMCAul/jYDezYrW0tDB48GBGjx5Nuk9c3xIRLFmyhJaWFsaMGVP369wVY2bFWrlyJUOHDu2ToQ4giaFDh3b6F4eD3cyK1ldDvaIr9Xewm5kVxn3sZrbJmDr7qYbOb/IBO7U7vV+/fowdO5Y1a9awyy67cM0117D55pu/NX716tX079+fY445hsmTJ/OOdzSmre0Wu5lZNxk0aBCPPPII8+bNY7PNNuPyyy9fb/zjjz/O7NmzmTVrFuedd17DlutgNzPrAfvuuy9PP/3028ZvvfXWTJ8+ncsuu4xG/eMjB7uZWTdbs2YNs2bNYuzYsTWn77DDDqxdu5aFCxc2ZHnuYzcz6yYrVqxgzz33BFKLfeLE9v4HfOM42M3MukmlL70j8+fPp1+/fmy99dYNWa67YszMetGiRYs48cQTOeWUUxp2zr1b7Ga2yejo9MSeUumiqZzuePTRR3P66ac3bP4OdjOzbrJs2bKa49euXduty3VXjJlZYRzsZmaF6TDYJV0q6SVJIem2dsrtI+lRSaskPSRpr8ZW1czM6lFvi/3G9iZKagJuAQYDk4FtgJsl9duw6pmZWWd1GOwR8SVgagfFDiSF+bSImAbMAMYAzRtaQTMz65xG9bFX/rXH83nYkoc7NGj+ZmZWp+463bHds+wlTQImAYwaNarhC29ubgZgzpw5DZ+3mfVhd5/f2Pl94px2J0vi9NNP53vf+x4AU6ZMYdmyZZx77rkAXHvttVx44YVIon///hx11FGceeaZG1ytLrfYJQ2UNDA/fTYPR+bhdnk4v9ZrI2J6RIyLiHHDhw/vahXMzDZqAwcO5NZbb2Xx4sVvmzZr1iwuvvhi7rzzTh577DHuv/9+hgwZ0pDl1nNWzMHAP+Sn75X0RUk7Ak8CSyp1BBYCJ0k6CZgILADmNKSWZmZ9UP/+/Zk0aRJTp779MOX555/PlClTGDFiBJC+BI4//viGLLeeFvtZwHfz3x8ErgD2qS4QESuB8cAy4BJSyI+PiO69vMrMbCN38sknM3PmTF599dX1xs+bN4+99967W5bZYR97RDS3MenqVuXuAWrfbNjMbBO15ZZbcswxx3DppZcyaNCgHlmmrzw1M+tmp512GjNmzGD58uVvjdttt9148MEHu2V5DnYzs2727ne/m8MPP5wZM2a8Ne6cc87hrLPO4sUXXwTgjTfe4Morr2zI8nx3RzPbdHRwemJ3OuOMM7jsssveen7QQQfx0ksvsf/++xMRSOK4445ryLIc7GZm3aT6tr3bbLMNr7/++nrTJ0yYwIQJExq+XHfFVGlubn7r4iYzs77KwW5mVhgHu5kVLSJ6uwobpCv1d7CbWbGamppYsmRJnw33iGDJkiU0NTV16nU+eGpmxRo5ciQtLS0sWrSot6vSZU1NTYwcObLjglUc7JuIvnLHy75ST+sbBgwYwJgxYzouWBh3xZiZFcbBbmZWGAe7mVlhHOxmZoVxsPdBvkK2b+rqdvP27n19bRs42M3MCuNgNzMrjIPdzKwwDnYzs8I42K0Ife3glll3crCbmRXGwW5mVhgHu5lZYRzsZmaFcbCbWa8p/aB3b62fg93MrDAOdjOzwjjYzcwK06f/Nd7U2U/VHN/yyop2p08+YKduq5PZpsr/1nDj4Ra7mVlhHOxmZoVxsJuZFcbBbmZWmD598LTL7j6/9vil/93+9E+c0z31MTNrILfYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MytMXcEuaR9Jj0paJekhSXvVKDNA0sWSXpL0qqRpkgY0vspmZtaeDoNdUhNwCzAYmAxsA9wsqV+rol8CTgX+HbgKOCmPMzOzHlRPi/1AUphPi4hpwAxgDNDcqtzf5uF5wFfy3//UgDqamVkn1HPl6Zg8fD4PW/JwB+CXVeUW5eEngaWtXrseSZOASQCjRo2qt65WD19Va7bJ68rBU7Ux/gLgBeAaUtfNCmBlrYIRMT0ixkXEuOHDh3ehCmZm1pZ6gv3ZPByZh9vl4XxJTZUDpBHxFLAj8FFgN+AN4IkG1tXMzOpQT1fMLGAhcJKk14CJwIL8WAHcDhwiaU/gM6SumiOAIcCUxlfZzMza02GLPSJWAuOBZcAlpJAfD6ytUfw44IfATsD/joifNa6qZmZWj7pu2xsR9wBja0xSVZlHaONgqZmZ9RxfeWpmVhgHu5lZYRzsZmaFcbCbmRVm0/yfp33E1NlP1Rzf8sqKNqdP9hY167Ku7HMAkw/Yqdvq1BVusZuZFcbBbmZWGAe7mVlh3CNrZrahNrK7qrrFbmZWGAe7mVlhHOxmZoVxsJuZFabIg6cnT7muS6+bM/X4BtfEGm4jO0jVUCWvm/Uot9jNzArjYDczK4yD3cysMA52M7PCFHnw1DZ+Xb6Lnj+xZh1yi93MrDAOdjOzwjjYzcwK42A3MyuMD0WZWffzVbU9yi12M7PCONjNzArjYDczK4yD3cysMD54uonwLYl7jq+qtd7mFruZWWEc7GZmhXGwm5kVxr16ZtYpXTmG4OMHPcstdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwKU1ewS9pH0qOSVkl6SNJeNcoMlHSlpEWSVkh6WNJ+ja+ymZm1p8Ngl9QE3AIMBiYD2wA3S+rXqugxwETgEeDrwB7AFQ2trZmZdaieFvuBpDCfFhHTgBnAGKC5jXnNA34BrAKWNqaaZmZWr3quBxuTh8/nYUse7gD8sqrcNcCngdPy42Xg2FozlDQJmAQwatSoTlXY4OQp1/V2Fcw2KV3d53rrrqpdOXiqNsZ/FDgYmAl8AegHXC3pbeUjYnpEjIuIccOHD+9CFczMrC31BPuzeTgyD7fLw/mSmiQNyM/HA5sBl0fEj4G5wF7AsEZV1szMOlZPV8wsYCFwkqTXSAdIF+THCuB24BBgfi5/tqQ9gI8BS4DFja2ymZm1p8MWe0SsJLXGlwGXkEJ+PLC2VdEfkA6s/hVwEfBfwPiIiEZW2MzM2lfXzTQj4h5gbI1JqiqzEvhig+plZmZd5CtPzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwKU9eVp2Ybu966ParZxsgtdjOzwjjYzcwK42A3MyuM+9jNNnI+fmCd5Ra7mVlhHOxmZoVxsJuZFcbBbmZWGAe7mVlhHOxmZoVxsJuZFcbBbmZWGAe7mVlhfOWpmfUaX1XbPdxiNzMrjIPdzKwwDnYzs8I42M3MCuODp7ZROXnKdb1dBbM+zy12M7PCONjNzArjYDczK4yD3cysMA52M7PCONjNzArjYDczK4yD3cysMA52M7PC+MpTM2sIXzW88airxS5pH0mPSlol6SFJe9Uoc7WkaPVY0PAam5lZuzoMdklNwC3AYGAysA1ws6R+rYr+X+CI/Dgvj3uocVU1M7N61NNiP5AU5tMiYhowAxgDNFcXiogHIuLGiLgRGJZHX97AupqZWR3q6WMfk4fP52FLHu4A/LJ1YUmbA/8IPA3MrjVDSZOASQCjRo3qRHXN+i73QVtP6cpZMepg+heAIcD0iIhaBSJiekSMi4hxw4cP70IVzMysLfUE+7N5ODIPt8vD+ZKaJA1oVf5EYBXwbw2on5mZdVI9XTGzgIXASZJeAyYCC/JjBXA7cAiApA8BHwZmRsTibqivmZl1oMMWe0SsBMYDy4BLSCE/Hlhbo/gJeeiDpmZmvaSuC5Qi4h5gbI1JalXuRFJXjJmZ9RLfUsDMrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MyuMg93MrDAOdjOzwjjYzcwK42A3MytMXcEuaR9Jj0paJekhSXu1UW53SXdJWiFpiaQLG1tdMzPrSP+OCkhqAm4BVgCTga8CN0vaMSLWVpUbBPwcGAR8A3gdGNYdlTYzs7Z1GOzAgcA2wNkRMU3StsDXgWbgl1XljgC2A44HZkbEigbX1czM6qCIaL+AdDrwPeCoiLhe0iTgh8CkiLiiqtwU4AzgD8AuwGLg5Ij4SY15TgIm5ac7A082YF06YxipfiUqed2g7PXzuvVdvbF+20fE8FoT6mmxt6Y2xg/Mwz8DXwMuBa6WNCsiXqsuGBHTgeldWHZDSJobEeN6a/ndqeR1g7LXz+vWd21s61fPwdNn83BkHm6Xh/MlNUka0KrcTyLiVuBeUn/7iIbU1MzM6lJPi30WsBA4SdJrwERgQX6sAG4HDgFuBL4DTJD0JvBJ4HngmYbX2szM2tRhiz0iVgLjgWXAJaSQHw+sbVXuBeBIYHgu9zTw2YhY0+A6N0KvdQP1gJLXDcpeP69b37VRrV+HB0/NzKxv8ZWnZmaFcbCbmRWmuGCXNF7Sk/n2BwvzLQ7qXk9JZ0oKScd2YzXbWvbovOzbNob69ISqda5+LO3tetWjVd1XSvqTpJmSxuTpCyQt6+E6jZB0rqTPd/H1G7T/dFWuc0g6rLuX1U4dTq7anjv3Vj0aoahglzQM+BHwBnAicEFlUq9Vyur1MOnq5SOA4xoxQ0lduU6jKx4mfd7uIp1A8GtJWwP/B/inHqpDxQjgm0Cng71R+09n3/dc/mbStr+/M69tsH8A3sx/H95TC+2Wz2lEFPMAPgIE6RTMrVpNuwl4BVgJPAH8XdW0M0lXjT0BXJPncWwv1H90XvZ/kK5kG0a6704Ax7a1DlWvuy+v+1LgOmBgnr6AdFbT94ElwK+BUcDu+XUX53LDgDXAjb28zkNIVy/PBv4CPAdMrnrN/Xn868CDwL55fHOe1x3Ab4G5PVT326rGTc/j/qXyvufx+wCP5m23CLghj98qb7PlwLXAf6XdMsjbPIAz8/Pb8vPRwNakW3osy+/FA6Qz0qLVo+7PMe3vPwHMy38flp+fm5/Pyc8vIe1Hn68adwHwEjAP2COXPzdPuwqYD0ytGncYsDkp6F/N78sjwG75tceRrlRfnj/HezVoW44gnel3A+k07cq6VrbBDcDvSfvfqXmaSFflL82fyZ/mss15+iH5Ncvz8ICe+pwW1WIn3c5gMXAQsETSXElfzNN+B5wNnJOfX5svsNoDuAh4EbgY2L+H61zLp0g7/yLgvKrxNdehavpHSTvUXcA/AidUTXsnsAVwOfAxUpjPI11IdlS+0OzzQD9SwPS06nW+Hfh3YFfgQlJofV/SZ3LZ2cDppDDYlhQQ1fYn7WRTu73WbzcrD1tfhXg2sANwMin0K5eff4P0eb2BdNV2vV0ARwH7kYLlDFL49SPdpA/gHlIL+FedqHt7+089PgR8mfVvEbIjMAX4APBvrcp/ihT8t7Ua/2ng74GZpF89c4ABkpqBGaQvzG8DQ4GftdoHumo8qQfjJuBWYDdJu1ZN/wTrvrS/K2kz4DOkz+Fjua4HVApL2ol1N0/8FrAK+Kmk91TNs/s+pz3VMuupB7AT6V42z7Gu1XIwqSW+ivVbMx8ATs1/T8yv/xa932K/P2/0/fO6BOnCsLbWofK6e/N83pef35qfLyC1RjbLz/8EvJz/PiKXPZQUSi8C/Xp5nffh7S3PIN2mYgtS8K9pNW0Q61pCt/Zw3atb7IeyrtW7gHUt9il5G9xCCt8P5PEP5/GVX1fPU1+L/RDW/Ur7LrBfLjMuj7+6gfvPgdTXYv9g1Xwq496Xn9+bn2/Jutb5l6rKV8YdBuyR35OHSb8yP0tqHV/Uxudig1vtwH+S9q1dSY2iIDWqKtvgO7ncz/Pz95ICOYD987SZ+Xkz6Qu8Vl0P7YnPaVEt9tzq/GNEnBAR25NaRgB7AceQWjH/i7TTAVR/06vVsDctjohfRMQvgD/mcQPonnW4hXTR2amkFuD1UXU75h5Uvc6Vg6f/QWoFVR7TSTvdQcCPSS27B3PZgVXzeqFHalzbp/PwwVbjv0zaqf9I+pKeK2mrqulRY16V7VDpg32rfETcRvqF9nPg48AvJe3fxnzq0s7+szup7/lt9Wil1vve3mey5naKiN8DHyT9itmZ9OttYlWRM1j3mfg0625n0iWS3kv6FbsZ8DipGxPW72d/OQ8rF1z2q65yO7O/kPU/ww9UTeu2z2lPHVzqKbsB10u6kdTi+Js8/nd5+E7ST8N9ql4zJw9Py0f/J/RAPTdErXWo+Kiks0gfUli3bpB+Zl4maRHpvj8/BYiINyTNYF33zjXdUelOepIUgB8n9SO/zrqfrRVbknb+sT1eu7cbkc9a+lvSwdIXgR+QvogrziG1CB8n/WIaQ1qHu4E9SdvmFda/t9KCPDxY0hJSkAOQzx7Zg3SF9+Okz8MIUp81wIckHQHMjoh67zrY1v7zWK7LGElHAqfUOT+ACyX9Jtf94Yj4i9R+u0PSvsDnSOv1MOmLfASpMXMm6VfmUuA9wNER8YFO1KeW8aQvnvNJfd6QvkgOIW2jttwNnAZ8I3fbfK5q2mzSQehDSdtoK9IXxfgNrGt9euIna089SP2tt5K+CVeRdqBvkr5dbyAFxH2k+9oEsGd+XeXg6WPA9fR+V0z1T/szWdcVU3Mdql53D+nD/yrp7IbWB0+nkA6e/gYYVbWM7Umtw99vDOucx+9CarG/Smot/YLUhzuYtNOsAH6W/w7SjtOc/76sh+seVZ+3HwGjq9/3/Pc/A/9N2tn/DHw9j38X6w54X0sK1KV5mvLn8TVSN9n9rOuKOYh0AH0F6YDejcDm+XUz8/gAPr6h+0+edhjp+Md84Epqd8UMq5pXZdx3aPvg6WFV5d8aB+xNCvTled1/Dmydyx1LOhawIr9XVzVgOz5A+kUyvGrc4bk+LbTdHVY5ePoKaZ+sdNNUcuXgvB6v5/f0RtKJAc108+fUtxQogKTRpJ+jt0fEITWmLyDtdFvUmDaK1LXzQ1Kf5792a2VtPZK2J73/T5POSvkOcEtE9Nr53I0gaQ7pF8zwqP8XQ5+T/1/Fo6QvxUtIXzjvi4hVvVmv0rpirPOOIx3MuwO4ooOy1nj9Sf9ycgzpl8l1pD5k6xs+SzpD5w1gLnBWb4c6+CZgZmbFKeqsGDMzc7CbmRXHwW5mVhgHu5lZYRzsZmaF+R/MIUD7+MIDkAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SBZoInt-knRr"
      },
      "source": [
        "## PDNC"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tv2pcZookpPp"
      },
      "source": [
        "def get_metric(dataPath, metric):\n",
        "  matContent = sio.loadmat(dataPath)\n",
        "  metric = matContent[metric]\n",
        "  return metric\n",
        "\n",
        "data = 'Full'\n",
        "pdnc_Full_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_Full_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/Spectral/CNN_1D_results_PDNC_Spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_Full_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(data)+'.mat', 'conf_mat_list')\n",
        "\n",
        "data = 'E1'\n",
        "pdnc_E1_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E1_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/Spectral/CNN_1D_results_PDNC_Spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E1_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(data)+'.mat', 'conf_mat_list')\n",
        "\n",
        "data = 'E2'\n",
        "pdnc_E2_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E2_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/IIR/CNN_1D_results_PDNC_IIR_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E2_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(data)+'.mat', 'conf_mat_list')\n",
        "\n",
        "data = 'E3'\n",
        "pdnc_E3_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E3_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/Spectral/CNN_1D_results_PDNC_Spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E3_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(data)+'.mat', 'conf_mat_list')\n",
        "\n",
        "data = 'E4'\n",
        "pdnc_E4_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E4_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/Spectral/CNN_1D_results_PDNC_Spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E4_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(data)+'.mat', 'conf_mat_list')\n",
        "\n",
        "data = 'E5'\n",
        "pdnc_E5_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E5_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/IIR/CNN_1D_results_PDNC_IIR_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E5_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(data)+'.mat', 'conf_mat_list')\n",
        "\n",
        "data = 'E6'\n",
        "pdnc_E6_ml = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/MLResults/Spectral/ML_PDNC_spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E6_1d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/1D_CNN_Results/Spectral/CNN_1D_results_PDNC_Spectral_'+str(data)+'.mat', 'conf_mat_list')\n",
        "pdnc_E6_2d = get_metric('/content/drive/MyDrive/EEG_Analysis_PD_NC/Results/CNN_2D_our/CNN_2D_results_PDNC_'+str(data)+'.mat', 'conf_mat_list')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQchgYKLswuF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e9bf5598-7564-4932-ab0c-55721a4f3df5"
      },
      "source": [
        "data1 = {'Sensitivity': [i[1][1] for i in pdnc_Full_ml], 'Specificity': [i[0][0] for i in pdnc_Full_ml],\n",
        "          'Data': len([i[1][1] for i in pdnc_Full_ml])*['Full'], 'Model': len([i[1][1] for i in pdnc_Full_ml])*['ML']}\n",
        "df1 = pd.DataFrame(data1)\n",
        "data2 = {'Sensitivity': [i[1][1] for i in pdnc_Full_1d], 'Specificity': [i[0][0] for i in pdnc_Full_1d],\n",
        "          'Data': len([i[1][1] for i in pdnc_Full_1d])*['Full'], 'Model': len([i[1][1] for i in pdnc_Full_1d])*['1D CNN']}\n",
        "df2 = pd.DataFrame(data2)\n",
        "data3 = {'Sensitivity': [i[1][1] for i in pdnc_Full_2d], 'Specificity': [i[0][0] for i in pdnc_Full_2d],\n",
        "          'Data': len([i[1][1] for i in pdnc_Full_2d])*['Full'], 'Model': len([i[1][1] for i in pdnc_Full_2d])*['2D CNN']}\n",
        "df3 = pd.DataFrame(data3)\n",
        "\n",
        "data4 = {'Sensitivity': [i[1][1] for i in pdnc_E1_ml], 'Specificity': [i[0][0] for i in pdnc_E1_ml],\n",
        "          'Data': len([i[1][1] for i in pdnc_E1_ml])*['Sad'], 'Model': len([i[1][1] for i in pdnc_E1_ml])*['ML']}\n",
        "df4 = pd.DataFrame(data4)\n",
        "data5 = {'Sensitivity': [i[1][1] for i in pdnc_E1_1d], 'Specificity': [i[0][0] for i in pdnc_E1_1d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E1_1d])*['Sad'], 'Model': len([i[1][1] for i in pdnc_E1_1d])*['1D CNN']}\n",
        "df5 = pd.DataFrame(data5)\n",
        "data6 = {'Sensitivity': [i[1][1] for i in pdnc_E1_2d], 'Specificity': [i[0][0] for i in pdnc_E1_2d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E1_2d])*['Sad'], 'Model': len([i[1][1] for i in pdnc_E1_2d])*['2D CNN']}\n",
        "df6 = pd.DataFrame(data6)\n",
        "\n",
        "data7 = {'Sensitivity': [i[1][1] for i in pdnc_E2_ml], 'Specificity': [i[0][0] for i in pdnc_E2_ml],\n",
        "          'Data': len([i[1][1] for i in pdnc_E2_ml])*['Happy'], 'Model': len([i[1][1] for i in pdnc_E2_ml])*['ML']}\n",
        "df7 = pd.DataFrame(data7)\n",
        "data8 = {'Sensitivity': [i[1][1] for i in pdnc_E2_1d], 'Specificity': [i[0][0] for i in pdnc_E2_1d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E2_1d])*['Happy'], 'Model': len([i[1][1] for i in pdnc_E2_1d])*['1D CNN']}\n",
        "df8 = pd.DataFrame(data8)\n",
        "data9 = {'Sensitivity': [i[1][1] for i in pdnc_E2_2d], 'Specificity': [i[0][0] for i in pdnc_E2_2d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E2_2d])*['Happy'], 'Model': len([i[1][1] for i in pdnc_E2_2d])*['2D CNN']}\n",
        "df9 = pd.DataFrame(data9)\n",
        "\n",
        "data10 = {'Sensitivity': [i[1][1] for i in pdnc_E3_ml], 'Specificity': [i[0][0] for i in pdnc_E3_ml],\n",
        "          'Data': len([i[1][1] for i in pdnc_E3_ml])*['Fear'], 'Model': len([i[1][1] for i in pdnc_E3_ml])*['ML']}\n",
        "df10 = pd.DataFrame(data10)\n",
        "data11 = {'Sensitivity': [i[1][1] for i in pdnc_E3_1d], 'Specificity': [i[0][0] for i in pdnc_E3_1d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E3_1d])*['Fear'], 'Model': len([i[1][1] for i in pdnc_E3_1d])*['1D CNN']}\n",
        "df11 = pd.DataFrame(data11)\n",
        "data12 = {'Sensitivity': [i[1][1] for i in pdnc_E3_2d], 'Specificity': [i[0][0] for i in pdnc_E3_2d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E3_2d])*['Fear'], 'Model': len([i[1][1] for i in pdnc_E3_2d])*['2D CNN']}\n",
        "df12 = pd.DataFrame(data12)\n",
        "\n",
        "data13 = {'Sensitivity': [i[1][1] for i in pdnc_E4_ml], 'Specificity': [i[0][0] for i in pdnc_E4_ml],\n",
        "          'Data': len([i[1][1] for i in pdnc_E4_ml])*['Disgust'], 'Model': len([i[1][1] for i in pdnc_E4_ml])*['ML']}\n",
        "df13 = pd.DataFrame(data13)\n",
        "data14 = {'Sensitivity': [i[1][1] for i in pdnc_E4_1d], 'Specificity': [i[0][0] for i in pdnc_E4_1d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E4_1d])*['Disgust'], 'Model': len([i[1][1] for i in pdnc_E4_1d])*['1D CNN']}\n",
        "df14 = pd.DataFrame(data14)\n",
        "data15 = {'Sensitivity': [i[1][1] for i in pdnc_E4_2d], 'Specificity': [i[0][0] for i in pdnc_E4_2d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E4_2d])*['Disgust'], 'Model': len([i[1][1] for i in pdnc_E4_2d])*['2D CNN']}\n",
        "df15 = pd.DataFrame(data15)\n",
        "\n",
        "data16 = {'Sensitivity': [i[1][1] for i in pdnc_E5_ml], 'Specificity': [i[0][0] for i in pdnc_E5_ml],\n",
        "          'Data': len([i[1][1] for i in pdnc_E5_ml])*['Surprise'], 'Model': len([i[1][1] for i in pdnc_E5_ml])*['ML']}\n",
        "df16 = pd.DataFrame(data16)\n",
        "data17 = {'Sensitivity': [i[1][1] for i in pdnc_E5_1d], 'Specificity': [i[0][0] for i in pdnc_E5_1d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E5_1d])*['Surprise'], 'Model': len([i[1][1] for i in pdnc_E5_1d])*['1D CNN']}\n",
        "df17 = pd.DataFrame(data17)\n",
        "data18 = {'Sensitivity': [i[1][1] for i in pdnc_E5_2d], 'Specificity': [i[0][0] for i in pdnc_E5_2d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E5_2d])*['Surprise'], 'Model': len([i[1][1] for i in pdnc_E5_2d])*['2D CNN']}\n",
        "df18 = pd.DataFrame(data18)\n",
        "\n",
        "data19 = {'Sensitivity': [i[1][1] for i in pdnc_E6_ml], 'Specificity': [i[0][0] for i in pdnc_E6_ml],\n",
        "          'Data': len([i[1][1] for i in pdnc_E6_ml])*['Anger'], 'Model': len([i[1][1] for i in pdnc_E6_ml])*['ML']}\n",
        "df19 = pd.DataFrame(data19)\n",
        "data20 = {'Sensitivity': [i[1][1] for i in pdnc_E6_1d], 'Specificity': [i[0][0] for i in pdnc_E6_1d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E6_1d])*['Anger'], 'Model': len([i[1][1] for i in pdnc_E6_1d])*['1D CNN']}\n",
        "df20 = pd.DataFrame(data20)\n",
        "data21 = {'Sensitivity': [i[1][1] for i in pdnc_E6_2d], 'Specificity': [i[0][0] for i in pdnc_E6_2d],\n",
        "          'Data': len([i[1][1] for i in pdnc_E6_2d])*['Anger'], 'Model': len([i[1][1] for i in pdnc_E6_2d])*['2D CNN']}\n",
        "df21 = pd.DataFrame(data21)\n",
        "\n",
        "\n",
        "pdnc_df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8, df9, df10,\n",
        "                        df11, df12, df13, df14, df15, df16, df17, df18, df19,\n",
        "                        df20, df21], axis=0).reset_index()\n",
        "print(pdnc_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "     index  Sensitivity  Specificity   Data   Model\n",
            "0        0     0.971545     0.967655   Full      ML\n",
            "1        1     0.975610     0.963612   Full      ML\n",
            "2        2     0.976996     0.971660   Full      ML\n",
            "3        3     0.967524     0.970310   Full      ML\n",
            "4        4     0.964817     0.962213   Full      ML\n",
            "..     ...          ...          ...    ...     ...\n",
            "205      5     0.984252     0.992063  Anger  2D CNN\n",
            "206      6     0.976378     0.992063  Anger  2D CNN\n",
            "207      7     1.000000     1.000000  Anger  2D CNN\n",
            "208      8     0.984127     0.992063  Anger  2D CNN\n",
            "209      9     1.000000     1.000000  Anger  2D CNN\n",
            "\n",
            "[210 rows x 5 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aH_F7V5xla6i",
        "outputId": "dda46f41-8376-4186-9f94-8b666ee38f4d"
      },
      "source": [
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='ML'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "temp = pdnc_df_new[pdnc_df_new['Data']=='Full']\n",
        "print(\"Sensitivity for full data\")\n",
        "print(temp[temp['Test']=='Sensitivity'].mean())\n",
        "print(temp[temp['Test']=='Sensitivity'].std())\n",
        "\n",
        "print(\"Specificity for full data\")\n",
        "print(temp[temp['Test']=='Specificity'].mean())\n",
        "print(temp[temp['Test']=='Specificity'].std())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity for full data\n",
            "Value    0.971847\n",
            "dtype: float64\n",
            "Value    0.007541\n",
            "dtype: float64\n",
            "Specificity for full data\n",
            "Value    0.963032\n",
            "dtype: float64\n",
            "Value    0.006308\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ctMmIfnj0Lak",
        "outputId": "4b70ae9f-4315-4633-df0d-55545f10c449"
      },
      "source": [
        "#print mean and std sensitivity or specificity for each data\n",
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='ML'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "print(pdnc_df_new)\n",
        "\n",
        "df_dummy_ml = pdnc_df_new.groupby(['Data','Test']).mean().round(2)\n",
        "print(df_dummy_ml)\n",
        "df_dummy_ml['SE'] = pdnc_df_new.groupby(['Data','Test']).sem().round(3)\n",
        "print(df_dummy_ml)\n",
        "pdnc_df_new.groupby(['Data','Test']).mean().round(3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Data         Test     Value\n",
            "0     Full  Sensitivity  0.971545\n",
            "1     Full  Sensitivity  0.975610\n",
            "2     Full  Sensitivity  0.976996\n",
            "3     Full  Sensitivity  0.967524\n",
            "4     Full  Sensitivity  0.964817\n",
            "..     ...          ...       ...\n",
            "135  Anger  Specificity  0.952381\n",
            "136  Anger  Specificity  0.968254\n",
            "137  Anger  Specificity  0.976190\n",
            "138  Anger  Specificity  0.944444\n",
            "139  Anger  Specificity  0.976000\n",
            "\n",
            "[140 rows x 3 columns]\n",
            "                      Value\n",
            "Data     Test              \n",
            "Anger    Sensitivity   0.97\n",
            "         Specificity   0.96\n",
            "Disgust  Sensitivity   0.96\n",
            "         Specificity   0.96\n",
            "Fear     Sensitivity   0.98\n",
            "         Specificity   0.95\n",
            "Full     Sensitivity   0.97\n",
            "         Specificity   0.96\n",
            "Happy    Sensitivity   0.95\n",
            "         Specificity   0.98\n",
            "Sad      Sensitivity   0.98\n",
            "         Specificity   0.97\n",
            "Surprise Sensitivity   0.97\n",
            "         Specificity   0.94\n",
            "                      Value     SE\n",
            "Data     Test                     \n",
            "Anger    Sensitivity   0.97  0.006\n",
            "         Specificity   0.96  0.006\n",
            "Disgust  Sensitivity   0.96  0.012\n",
            "         Specificity   0.96  0.006\n",
            "Fear     Sensitivity   0.98  0.004\n",
            "         Specificity   0.95  0.005\n",
            "Full     Sensitivity   0.97  0.002\n",
            "         Specificity   0.96  0.002\n",
            "Happy    Sensitivity   0.95  0.007\n",
            "         Specificity   0.98  0.003\n",
            "Sad      Sensitivity   0.98  0.004\n",
            "         Specificity   0.97  0.005\n",
            "Surprise Sensitivity   0.97  0.006\n",
            "         Specificity   0.94  0.008\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>Value</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Data</th>\n",
              "      <th>Test</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Anger</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.966</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.959</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Disgust</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.964</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.960</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Fear</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.952</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Full</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.972</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.963</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Happy</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.952</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.982</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Sad</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.971</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Surprise</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.937</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      Value\n",
              "Data     Test              \n",
              "Anger    Sensitivity  0.966\n",
              "         Specificity  0.959\n",
              "Disgust  Sensitivity  0.964\n",
              "         Specificity  0.960\n",
              "Fear     Sensitivity  0.977\n",
              "         Specificity  0.952\n",
              "Full     Sensitivity  0.972\n",
              "         Specificity  0.963\n",
              "Happy    Sensitivity  0.952\n",
              "         Specificity  0.982\n",
              "Sad      Sensitivity  0.977\n",
              "         Specificity  0.971\n",
              "Surprise Sensitivity  0.968\n",
              "         Specificity  0.937"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 829
        },
        "id": "j0ZYw3lmF22I",
        "outputId": "a70707c8-896f-49a8-bb5b-a35af9157874"
      },
      "source": [
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='ML'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "print(pdnc_df_new)\n",
        "\n",
        "\n",
        "x = \"Data\"\n",
        "y = \"Value\"\n",
        "hue = \"Test\"\n",
        "\n",
        "ax = sns.barplot(data=pdnc_df_new, x=x, y=y, hue = hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.9)\n",
        "ax.legend(loc='upper right', bbox_to_anchor=(1,1))\n",
        "ax.set_xlabel('Data',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Sensitivity/Specificity',fontsize = 13, weight='bold')\n",
        "#ax.set_xticklabels([\"Full\", \"Sad\", \"Happy\", \"Fear\", \"Disgust\", \"Surprise\", \"Anger\"], rotation=0, fontsize = 10, weight='bold');\n",
        "#ax.set_yticklabels([\"0.5\", \"0.6\", \"0.7\", \"0.8\", \"0.9\", \"1.0\"], size= 10, weight='bold')\n",
        "#ax.set_yticklabels(ax.get_yticklabels(), size= 10, weight='bold')\n",
        "add_stat_annotation(ax, data=pdnc_df_new, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"Full\", \"Sensitivity\"), (\"Full\", \"Specificity\")),\n",
        "                                ((\"Sad\", \"Sensitivity\"), (\"Sad\", \"Specificity\")),\n",
        "                                 ((\"Happy\", \"Sensitivity\"), (\"Happy\", \"Specificity\")),\n",
        "                               ((\"Fear\", \"Sensitivity\"), (\"Fear\", \"Specificity\")),\n",
        "                               ((\"Disgust\", \"Sensitivity\"), (\"Disgust\", \"Specificity\")),\n",
        "                               ((\"Surprise\", \"Sensitivity\"), (\"Surprise\", \"Specificity\")),\n",
        "                               ((\"Anger\", \"Sensitivity\"), (\"Anger\", \"Specificity\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "\n",
        "props = dict(boxstyle='round', facecolor='none', edgecolor='black', alpha=0.5)\n",
        "\n",
        "# place a text box in upper left in axes coords\n",
        "ax.text(0.45, 0.95, \"ML\", transform=ax.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "\n",
        "plt.savefig('sens_and_speci_pdnc_ml.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Data         Test     Value\n",
            "0     Full  Sensitivity  0.971545\n",
            "1     Full  Sensitivity  0.975610\n",
            "2     Full  Sensitivity  0.976996\n",
            "3     Full  Sensitivity  0.967524\n",
            "4     Full  Sensitivity  0.964817\n",
            "..     ...          ...       ...\n",
            "135  Anger  Specificity  0.952381\n",
            "136  Anger  Specificity  0.968254\n",
            "137  Anger  Specificity  0.976190\n",
            "138  Anger  Specificity  0.944444\n",
            "139  Anger  Specificity  0.976000\n",
            "\n",
            "[140 rows x 3 columns]\n",
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "Sad_Sensitivity v.s. Sad_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=8.543e-01\n",
            "Full_Sensitivity v.s. Full_Specificity: t-test independent samples with Bonferroni correction, P_val=7.683e-02 stat=2.835e+00\n",
            "Happy_Sensitivity v.s. Happy_Specificity: t-test independent samples with Bonferroni correction, P_val=4.699e-03 stat=-4.101e+00\n",
            "Fear_Sensitivity v.s. Fear_Specificity: t-test independent samples with Bonferroni correction, P_val=1.511e-02 stat=3.576e+00\n",
            "Disgust_Sensitivity v.s. Disgust_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=3.373e-01\n",
            "Surprise_Sensitivity v.s. Surprise_Specificity: t-test independent samples with Bonferroni correction, P_val=4.934e-02 stat=3.040e+00\n",
            "Anger_Sensitivity v.s. Anger_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=9.211e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEJCAYAAABlmAtYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU5fn/8feHAKKCsgRFQAguKIiAErC2ClErCl93bRG0iFr5WXdbL2urX6WoLVqtilqRVkpxp+4Fql83qlZcgiKKKCJFDUJlERQrSML9++M5E4YwycwkM8lMcr+uay7m7HdOhtxzznme+5GZ4ZxzztWkWUMH4JxzLvd5snDOOZeUJwvnnHNJebJwzjmXlCcL55xzSTVv6ACyobCw0IqKiho6DOecyytz585dZWYdEy1rlMmiqKiI0tLShg7DOefyiqRPqlvmt6Gcc84l5cnCOedcUp4snHPOJdUon1k45/Lfpk2bKCsrY8OGDQ0dSqPTqlUrunbtSosWLVLexpOFc0mYGWVlZSxbtozy8vKM7rugoIBOnTrRvXt3mjXzC/14ZWVltGnThqKiIiQ1dDiNhpmxevVqysrK6NGjR8rbebJwrgarV6/m/vvvp6CggKKiIlq2bJnRP1ybNm1i/vz5rF+/npEjR9K5c+eM7TvfbdiwwRNFFkiiQ4cOrFy5Mq3tPFk4V43y8nKmTZvGkCFDOPDAA7N6rA8++ID777+f888/nx122CGrx8onniiyozbntV6ueyVNkfSFpPeqWS5JEyUtljRf0oFVlu8kqUzSHfURr3MAH3/8Me3atct6ogDYd999KSoqYuHChVk/lnO1UV9XFlOBO4Bp1SwfBuwdvQ4C7or+jbkWeCmL8Tm3jU8++YQ999yz3o631157sXTpUgYMGFBvx8wnM97+d0b3d8wBqd2vv/7663nggQcoKCigWbNm3H333Rx00EHJN0zB97//fV599VWWLl3Kq6++yqhRowAoLS1l2rRpTJw4sdptJ02axA477MDo0aOZOnUqQ4cOzeptzHpJFmb2kqSiGlY5HphmYSSm1yS1lbSbmS2XNADYFXgaKM5+tM4F3333He3atau347Vq1YqNGzfW2/FccnPmzGHGjBm89dZbbLfddqxatYrvvvsuY/t/9dVXAVi6dCkPPPBAZbIoLi6muLjmP3fnnntu5fupU6fSp0+frCaLXGl+0QX4LG66DOgiqRlwM3BZsh1IGiupVFJpug9unHMukeXLl1NYWMh2220HQGFhIZ07d2bu3LkMGTKEAQMGcNRRR7F8+XIASkpK+OUvf8mgQYPo2bMnL7/8MgALFixg0KBB9O/fn759+/LRRx8B0Lp1awCuuOIKXn75Zfr3788tt9zC7NmzOeaYY9i8eTNFRUWsXbu2Mqa9996b//znP4wbN46bbrqJRx55hNLSUk477TT69+/PzJkzOeGEEyrXf/bZZznxxBPrfC5yJVlU5zxglpmVJVvRzCabWbGZFXfsmLAOlnMZMWbMGCRx9tlnb7Psl7/8JZI45phjABg3bhx9+vSp7xBdhgwdOpTPPvuMnj17ct555/HPf/6TTZs2ceGFF/LII48wd+5czjrrLK688srKbcrLy3njjTe49dZb+c1vfgOEW0YXX3wx8+bNo7S0lK5du251nAkTJnDooYcyb948Lr300sr5zZo14/jjj+fxxx8H4PXXX6d79+7suuuuleuccsopFBcXc//99zNv3jyGDx/OBx98UNna6S9/+QtnnXVWnc9FriSLZcDucdNdo3kHAxdIWgrcBIyWNKH+w3Nua7vvvjvTp0/nm2++qZwXaz3VrVu3BozMZVLr1q2ZO3cukydPpmPHjowYMYK7776b9957jyOPPJL+/ftz3XXXUVa25fvsSSedBMCAAQNYunQpAAcffDC//e1vueGGG/jkk0/YfvvtU45hxIgRPPzwwwA89NBDjBgxosb1JfGTn/yE++67j7Vr1zJnzhyGDRuW5k++rVxpOvsUISk8RHiwvc7MlgOnxVaQNAYoNrMrGiZE57bo27cvn3/+OdOnT+fMM88EYObMmbRq1YrBgwezevXqBo7QZUpBQQElJSWUlJSw//77c+edd7LffvsxZ86chOvHblkVFBRUduIcNWoUBx10EDNnzmT48OHcfffdHH744Skd/+CDD2bx4sWsXLmSJ554gquuuirpNmeeeSbHHnssrVq14kc/+hHNm9f9T319NZ19EJgD7BM1gT1b0rmSYk9oZgFLgMXAnwi3n5zLaWeffTZTpkypnJ4yZQpnnnmm9w1oRD788MPK5wsA8+bNo1evXqxcubIyWWzatIkFCxbUuJ8lS5awxx57cNFFF3H88cczf/78rZa3adOGr7/+OuG2kjjxxBP5+c9/Tq9evejQocM261TdvnPnznTu3Jnrrruu8stMXdVXa6iRSZYbcH6SdaYSmuA6lxNGjRrFZZddxkcffUSbNm14+umnuf3227n66qsbOrRGKdWmrpm0fv16LrzwQtauXUvz5s3Za6+9mDx5MmPHjuWiiy5i3bp1lJeXc8kll7DffvtVu5/p06dz77330qJFCzp16sSvf/3rrZb37duXgoIC+vXrx5gxYzjggAO2Wj5ixAgGDhzI1KlTE+5/zJgxnHvuuWy//fbMmTOH7bffntNOO42VK1fSq1evOp8HyJ3bUM7lnXbt2nHiiScyZcoU2rZtS0lJiT+vaGQGDBhQ2bw1XmFhIS+9tG3Xr9mzZ2+1TuyZxRVXXMEVV2x7B339+vUAtGjRghdeeGGrZSUlJZXvi4uLCd+ptxg3blzl+5NPPpmTTz55q+WvvPIK55xzTsKfqzY8WThXB2eddRZnnHEGrVu3Zvz48Q0djnNASHI77rgjN998c8b26cnCuTo44ogjaNmyJatWrdqqbbtzDWnu3LkZ36cnC+fqQBLz58/HzCpbwVS1YcMG5s2bt9W8HXbYgZ49e9ZHiM5lhCcL5+qoTZs2NS7/+OOPt3lgOWDAAEpLS7MZlnMZ5cnCuTRV1yIl0fJx48Zt9SDSuXyVKz24nXPO5TC/snCuGs2aNaOioqLejldRUeFDq9Zg2PVPZHR//7gytQYJ2SxRPnz4cB544AHatm3LxIkTueuuuzjwwAMZMWIE77//fsLmtjHVlTfPFk8WzlWjXbt2aQ89WRcrV66s15LoLrlslyifNWtW5fs//vGPPPfcc5VFBo877rgat62uvHm2+NcY56qx7777snDhwsqOU9m0ceNG5s+fn7Heti4zqitRXlRUxOWXX87+++/PoEGDWLx4MRAS/sknn8zAgQMZOHAg//rXv4DQ+e7MM89k//33p2/fvjz66KMAFBUVsWrVKs4991yWLFnCsGHDuOWWW5g6dSoXXHABAP/5z3848cQT6devH/369atMEtWVNx88ePBWre8OOeQQ3nnnnTqfC7+ycK4a7dq1Y9CgQUydOpVDDjmEoqKiapvH1tamTZv49NNPefXVV9lzzz3p0qVLRvfv6mbo0KGMHz+enj178sMf/pARI0YwZMgQAHbeeWfeffddpk2bxiWXXMKMGTO4+OKLufTSSznkkEP49NNPOeqoo1i4cCHXXntt5foAX3755VbHmTRpEk8//TQvvvgihYWFWzWSuOiiixgyZAiPP/44FRUV23x5mTBhAjfddBMzZswAoH379kydOpVbb72VRYsWsWHDBvr161fnc+HJwrkalJSUsMsuuzB//nyef/55Nm3alNH9N2/enE6dOjFo0CD69evnRQhzTKxE+csvv8yLL77IiBEjmDAhjJIwcuTIyn9jY1A899xzvP/++5Xbf/XVV6xfv57nnnuOhx56qHJ+OrcbX3jhBaZNCyNSFxQUsPPOO9e4/o9+9COuvfZafv/73zNlyhTGjBmT8rFq4snCuSR69+5N7969GzoM10Cqlij/61//CrBVYo+937x5M6+99hqtWrVqkFghdPg88sgjefLJJ5k+fXrGenP7MwvnnKtGohLl3bt3B6gckOjhhx/m4IMPBsJtq9tvv32r9QGOPPJI7rzzzsr5VW9D1eSII47grrvuAkKLuXXr1m21PFF585/+9KdcdNFFDBw4MGONJlK+spB0D3CPmW1bgtE557Is1aaumVRdifIZM2bw5Zdf0rdvX7bbbjsefPBBACZOnMj5559P3759KS8vZ/DgwUyaNImrrrqK888/nz59+lBQUMA111xTOaJeMrfddhtjx47lnnvuoaCggLvuuqsyOcG25c0vvfRSBgwYwE477ZSxsSwAVLXsbbUrSpsBIwxQNAW418w+z1gkGVRcXGxeSsG5/LZw4cKcbR1WVFREaWkphYWFDR1KQp9//jklJSV88MEH1fbdSXR+Jc01s+JE66dzG2oqsBbYG/gt8ImkmZJOltQijf0455zLkmnTpnHQQQdx/fXXZ7STZ8q3oczsLEnNgR8CpwAnAEdHrzWSbgd+a2blGYvOOedyUGxQo1w0evRoRo8enfH9ppV2okTwIvBPYBGg6NUBuAb4c6YDdM41XaneJnfpqc15TTlZSOov6Q5gOeGW1PeAlYRbUgcSEkhqT2yccy6JVq1asXr1ak8YGWZmrF69Ou3mven0s3iL8IBbwBvAHcB0M/sOQNLfgZJEG0qaAhwDfGFmfRIsF3AbMBz4LzDGzN6S1B+4C9gJqACuN7OH04jZOZenunbtSllZWb3W52oqWrVqVVmDKlXpJIuNwHTgDjN7M8HyPwOPVbPtVEJymVbN8mGEB+d7AwcREsRBhMQx2sw+ktQZmCvpGTNbm0bczrk81KJFC3r06NHQYbhIOsnif4HZZlbZJlXSLkChmb1vZl8DXyfa0MxeklRUw76PB6ZZuN58TVJbSbuZ2aK4fXwu6QugI6FVlnPOuXqSzgPuG4CqNXB/BbybgTi6AJ/FTZdF8ypJGgS0BD7OwPGcc86lIZ1kkajCWXo3vWpJ0m7AvcCZZra5mnXGSiqVVOr3OJ1zLrOSJgtJFZIqCA+3L45NR/NOIrSIqqtlwO5x012jeUjaCZgJXGlmr1W3AzObbGbFZlbcsWPHDITknHMuJpUri1hfivj38a87q9kuHU8BoxV8D1hnZssltQQeJzzPeCQDx3HOOVcLqSSLw4DDCYnhkWj6MOBQoJuZXZtsB5IeBOYA+0gqk3S2pHMlnRutMgtYQqg79SfgvGj+j4HBwBhJ86JX/9R/PNeYxNrbjxs3rsZp51zmpVNI8AxgQXxrqFzlhQQbp/vuu4/ly5ezevVq2rdvT+fOnbeZPv300xs6TOfyVq0LCUoaLSm2oQG9o3lbvTIdsMu+pUuX0qtXL8455xz2228/hg4dyrfffsvEiRPp3bs3ffv25dRTT23oMLdy+umn07VrV37/+9/TrVu3hNOu7vLxs+Gyr8Yri6gs+S1m9ou4EuVVmZnl1Ih7fmWR3NKlS9lrr70oLS2lf//+/PjHP+a4447j8ssv59///jfbbbcda9eupW3btg0daqUHHniAZcuWVV5JdO3adZvpUaOqtu526crHz4bLjJquLJL9kf8U+DLuvd8UbkR69OhB//7hEdCAAQNYunQpffv25bTTTuOEE07ghBPqf7CZmowcORJJjBs3jssvvxwz22baZUa+fTZc9tV4G8rMiszsurj3PRK96idUl2nbbbdd5fuCggLKy8uZOXMm559/Pm+99RYDBw6kvDx3Ks7HxjmOPdCubtrVXb59Nlz2pVN1tr2kblHRP6Jmrt0ktc9eeK4+bd68mc8++4zDDjuMG264gXXr1rF+/fqGDsvlAP9suHSeNcwC2prZvhAeVEj6B/AVcHCNW7q8UFFRwemnn866deswMy666CK/L+0A/2y49JrOfgU8bGbnxM2bDJxqZjtlKb5a8QfczjmXvro84I5XAXSqMq8z4Dcuc9xjjz3G008/nZF9HX300Zx0UvbHuMrHmPORn2eXqnQKCS4ChksaL+loSeMJ41B8lJ3QXKY8/fTTLFq0KPmKSSxatChjf1iSyceY85GfZ5eqdK4s7iQMYnRllfl3ZCwalzU9e/Zk8uTJddrH2LFjMxRNavIx5nzk59mlIuUrCzObBkwAviXUifoWuMHM7s1SbM41Kl7LKvv8HGdPWj2vzezXkq4hjFa30sw2ZScs5xqf+++/n+XLl7NhwwZuvPFGr2WVBX6OsydZbajBkvaIez+Y0Ex2L+DguHkuh+2zzz7ss88+ObOf+jxWfcacTC7Wsmps5zkXz3Ei+Vh/K1ltqArgVq8N5VzdJapt5bWsMitfznGu1t+qS9PZ+IGPqPK+pnnOuSoS1bZymZVP5zjf6m8lqw3VzMx+Hvc+4at+QnUuv3ktq+zLp3Ocb/W3kj2z+ELSb6L3L0j6Wf2E5ZxzTUuu199KdhuqMHoBlADzshqNc841UblefyvZA+4VwE7AG4SxsMsI42THMzM7ImsR1oI/4HbOufTV5QH3w8CFhERhQNfoFS93nyA5V4+8zlL25eM5zseYE6kxWZjZxZJmA/sB44HXgGfqIS7n8k6szlLPnj3rtJ9YrSZPFtvKx3OcjzEnkrR/hJk9DjwuaS/gGTN7MN2DSJoCHAN8YWZ9EiwXcBswHPgvMMbM3oqWnQFcFa16nZn9Nd3jO1dfvM5S9uXjOc7HmKtKuTOdmY2pw3GmEgoOTqtm+TBg7+h1EHAXcFA0Ct81QDHhdtdcSU+Z2ZfV7Mc551wWJGs6WyHp5rj3iV5JGwKb2UvAmhpWOR6YZsFrQFtJuwFHAc+a2ZooQTwLHJ3qD+eccy4z0unBnc3eLV2Az+Kmy6J51c3fhqSxwFiAbt26ZSfKGixdupRhw4ZxyCGH8Oqrr9KlSxeefPJJ/vSnPzFp0iSaN29O7969eeihh+o9Nlc/MlUbKRdqLOWqfDzH+RhzIsmazg4Byszs4+h9Qmb2z6QHkoqAGdU8s5gBTDCzV6Lp54FfEvp2tDKz66L5/wt8a2Y31XSshmg6m6u1XpxzLlW1bjobnwRSSQh1sAzYPW66azRvGSFhxM+fncU46iTfar0451yqUq7rJGmCpKckFUTTzSU9IWlCBuJ4Chit4HvAOjNbTmimO1RSO0ntgKHkcNPdfKv14pxzqUqnCOAYwi2gCgAzKyc0cz0j2YaSHgTmAPtIKpN0tqRzJZ0brTILWELoHf4n4LzoGGuAa4E3o9f4aF5eyPVaL845l6p0xqHYGaj6l+7baH6NzGxkkuUGnF/NsinAlBRjzCm5XuvFOedSVeMD7q1WlD4EOgDfM7PFUSe9OcAaM8up5hteG8o559JXl9pQ8Z4m1In6QNIqQjVaAffVPcT80lhqvTjnXKrSeWZxLfBxtM0u0b9LgOuyEFdOi9V6qatFixZlLOk451w2pVPuY5WkvsBJQHfgE+AxM/s2W8HlssZQ68U551KVzm0ogA1EParN7P7Mh+Occy4XpdPPoi3wOvAi8FdJe0a1ocZlKzjnnHO5IZ0ri/GE6q8ARCVASgkVY8dlOK6c1lhqvTjnXKrSaTq7BFgOvA38zMwKJP0FONbMCmveun5501nnnEtfTU1n02kNtQvwPrCpyvyWtQ3MOedcfkgnWfwbOIzQMQ9J3yeMQ/HvLMTlnHMuh6STLKYBewCnRdMvE0p93JvpoJxzzuWWdJLFrcB0tgyIJOBRwtjZzjnnGrF0OuVtAk6VdBnQDfjUzMqyFplzzrmckVanPEmdgVGEZPGJpAfMbFlWInPOOZczUk4Wkg4ljDuxQ9zsayQNN7OXMh6Zc865nJHOlcWtwI7Af4CPgL2BXYE/ENdZz7lccfnll/Pee+9RXl5O8+bN6dOnDzfeeGNDh+VcXkonWfQCSoEfmNkmSS2BV4D9shKZc3W0YsUKNm7cCISBqFasWNHAETmXv9JpDTUfeC960I2ZfQcsIPTodi7ndOrUiYKCAiCMid6pU6cGjsi5/JVOspgOjJB0iaQjJV0KnAI8KGlw7JWdMJ1L34033liZIDp16uS3oJyrg3RuQ90EGHBzlfkT495bmvt0zjmXB9L9w67aLpd0NKEDXwHwZzObUGV5d2AK0BFYA5we68ch6UbgfwhXQs8CF1uqFRBdozbs+idqXP7dmm8AWLbmmxrX/ceVJ9QpDn+Y7hq7lG9DmVmzVF6JtpVUANxJKGfeGxgpqXeV1W4CpplZX0I59N9F234f+AHQF+gDDASGpPlzOpdVsYfpFRUVbNy40R+mu0anvm4ZDQIWm9kSAEkPEYoQvh+3Tm/g59H7F4HY10ADWhGq2wpoQWi+61Lg33jrR6dOnVixYgUVFRX+MN01SkmvLCTtFT283j6a/p6kFyWVSvqNpFSuTroQDccaKYvmxXuHML43wIlAG0kdzGwOIXksj17PmNnCBHGOjWIqXblyZQohNQ3+jbd++MN019il8of+NuDvQIWkVsBjwGDgQOAq4FcZiuUyYIiktwm3mZZFx9yL0MejKyHBHB71Jt+KmU02s2IzK+7YsWOGQsp/Tbn56Ka3n4ANX4WJDV+FaedcraSSLPYDXon6VRwGdAL+S2gFtYFQKyqZZcDucdNdo3mVzOxzMzvJzA4ArozmrSVcZbxmZuvNbD3wD+DgFI7paNrfeG3D12Cbo4nNYdo5VyupJItC4JPofewb/V/M7BLgcUJRwWTeBPaW1CPq+X0q8FT8CpIK425p/YrQMgrgU8IVR3NJLQhXHdvchnKuKrVqA82ag5pBs+Zh2jlXK6k84N4MtIvef4/wwPnlaPrbVA5iZuWSLgCeITSdnWJmCySNB0rN7CmgBPidJANeAs6PNn8EOBx4Nzr202b291SO65q2FgfUrTmsc26LVJLFR8BJkp4gfKs34J/Rsj2BL1I5kJnNIlStjZ93ddz7RwiJoep2FcD/S+UYTdWMt2se2fabjeWV/9a07jEH9MhoXI2Nn2fXlKWSLG4DpgLHRdOPmdkXkgqB7wNPZik2l0O8CW7j479Tl46kycLMpklaDRxJeCh9R7Rod+B2qlwtuMbJK7g2Pv47delImiwk/QiYaWYz4+eb2ds0wYqzTfXbmHc6a3z8d+rSkcptqAeATZKeJfSx+LuZrcluWLmrqX4bu/HGGxk9ejTLli1rck1wG6t8+51W/aJWWFiYF3E3Fqk0ne0EXBitOwlYIel5SedJqtoLu9HLt05uj07+A1+tWQXAV2tW8ejkPzRwRI2Tn+fsq1qNYNmyZU3my1ouSJoszGy1md1jZscSKsL+BFhJKPT3iaTXJF0uqWuWY80J+dbJ7as1q9m8uQKAzZsr+GrN6gaOqHHy85x9Vb+odenSJee/rDUmaRUSjHpQPww8HHWuG0roYX0Zodjf+IxH2AAaUxPJndp3YP26L9lcUUGzggJ2at+hoUNqlPw8Z1/V22bTpk1r6JCSakzPOFNOFpIONbNYZ7zYsKozgBlRz2v/35GDTh778+QrRXJlbIh8lM55dk1HY3rGmc6wqv+U9KGkX1V9VmFmm83MS70651ycfHvGWZN0ksV/gb2B64ClkmZJOiWq19Rk+INM51yq8u0ZZ03SSRaFwCnA3wjVZo8mPL9YLmmipD2zEF/O8QeZzrmmKJ1hVTeY2WNmdirQA5hJGLmuPXAB8L6kU7MTZu7YqX0HmrdoSbNmBTRv0dIfZDrnmoS0WkNJKgZ+SigxHqv3/ClhCNRRwATgoUwGmGv8QaZz2dOYWiI2Num0hnob6Eu4mgB4gVAn6ikz2yzpC+DazIfonHOuoaVzZdEPWA/cC9yRYBzs54Em9bC7KUk0RKmPF+Fc05FOsriYMELe+kQLzex14PWMRNXIXX755axYsYJVq1blTWcdH6LUJZKPn+VMayq3ztJJFrcCRcAvYjMkXQqMNLNBGY6rUVuxYgXLlm0ZgjwfOuuoVRts4/qQMNTMhyh1QH5+lvNRLvQETydZiC3PK2L2AAZkLpymIdbuOp/KQ/stp/yU7W+9+fhZzke50BM8lfEspsRNDo2bbgYcQ+is59IQ+0aQT+WhnUvEP8v1IxfGHknlymIMYdxtA3oBvassfz7DMTUamai11BTrLDnntpYLY4+kkiz+Gv17BrAQeCOaLgc+Ae7OQlzOObeVqqV2/jLhSnZq38H7PtWTVMbgPhNAUg/gb2Z2Z20OJOlo4DagAPizmU2osrw7MIUwZsYa4HQzK4uWdQP+TBj324DhZra0NnHkgk1vPxFaE3lTVOdSVrXUztpV/2ngiJJLVEsuX5Nbyg+4zayktgeRVADcCRwJlAFvSnrKzN6PW+0mYJqZ/VXS4YTBlX4SLZsGXG9mz0pqDWyubSy5wDZ8Dd+ujZvhTVGdS6bqmCGtd26X8+V2GlMtuRqThaQK4FYz+0X0PhEzs2RJZxCw2MyWRPt9CDgeiE8WvYFYyn2RUEIESb2B5mb2bHSwhP088olatcEAvCmqcynLx2/kjWlQrGR/5OOby1ZtNpuOLsBncdNlwEFV1nkHOIlwq+pEoI2kDkBPYK2kxwgFDJ8DrjCzrZKXpLHAWIBu3brVIdTs89tNLt/4wFi1k48JrjrJksVhhD/ssffZdBlwh6QxwEvAMqCCEOOhwAGEooUPE1po3RO/sZlNBiYDFBcXW5ZjdVmUCx2QnHNbqzFZmNk/4ya/NrO3anmcZYSH0zFdo3nxx/qccGVB9FziZDNbK6kMmBd3C+sJ4HtUSRau8ciFDkgNxRNl05XrV2/pDH70pqQ3JZ0jacc0j/MmsLekHpJaEkqcPxW/gqTCaCxvgF8RWkbFtm0rqWM0fThbP+twjUxjGooyXbFEWVFRwcaNG5tUonS5LZ1kUUEo7TEJ+FzSJEkplfows3LCAEnPEPpqTDezBZLGSzouWq0E+FDSImBX4Ppo2wrCLarnJb1LeHbypzTidnmmMQ1Fma6mnChdbkunNtRuwEhCc9aBhIfJ50h628yKk21sZrOAWVXmXR33/hHgkWq2fZYwloZzjVou9NR1LpF0hlVdbWZ3mNlBhLIfLxC+5R+QreCcc7lt09tP8N2ce7fpYOoan3SHVd0DOJ1wdbFHNLu6/hfOuUbOO5g2HekMq/oKcHBsklAX6h68VZJzTeaSW7EAABK4SURBVJZ3MG060rmy+D6heODfCf0Z/s/MvD+Dc3kqE3WLvINp/ciFYY3TSRa/JgyrmvvVu5xzSTWmukWNXS4Ma5xOIcEJyddyLjWZGsEtH+VK56vGVLeoscuFYY3rq5Cgcy7HNKa6RY1dLtzuq69Cgs455/JYLhUSdM45l6PSKSTYHXjfzEpjMyTtAhRmKTbnnHM5Ip3aUFOBUVXm/Qp4N2PROOecy0npJItEumYkCucckLg9vXO5IGmykFQRtYQy4OLYdDTvJGBltoN0rqnIhfb0ziWSypVF1RZRVV93Zic055oetWoDzZqDmkGz5l46w+WMVPpHHEZICi8QSojHkkM58ImZlVW3oXO1kYkyFPkqF9rTO5dI0mQRaxEl6UxgQXxrKOeywctQOJd7kvXgHs2W5rIG9JbUu+p6ZjYtS/G5JsjLULhs8nHOayfZlcVU4BagNHqfqMqsAZ4sXMY0lVtOrmHExjkHqKio8HHOU5QsWXwKfBn33kuSO+fyWqdOnVixYgUVFRU+znkakvXgLkr03jnn8pWPc147teqUJ2k7SaMkHZXGNkdL+lDSYklXJFjeXdLzkuZLmi2pa5XlO0kqk3RHbWJ2zjlXeyknC0lPSFotqRnwN+BeYJakX6SwbQGhye0woDcwMsGD8puAaWbWFxgP/K7K8muBl1KN1znnXOakc2XRF3gV2B4YDiwGNgA/TWHbQcBiM1tiZt8BDwHHV1mnN6EvB8CL8cslDQB2Bf4vjXidc85lSDrJYjfCQ+59CJ30RhH+6O+ewrZdgM/ipsuiefHeIZQPATgRaCOpQ3QlczNwWU0HkDRWUqmk0pUrvQKJc85lUjrJYgOhTPkhhFZRC4EKYHOGYrkMGCLpbWAIsCza/3nArGQ9xc1sspkVm1lxx44dMxSSc845SGMMbuA9wjOHYcB7ZvZfSUWEP+rJLGPrK5CuVbczs8+JriwktQZONrO1kg4GDpV0HtAaaClpvZlt85DcOedcdqSTLK4AHgBaAr+UtB0hAfwjhW3fBPaW1IOQJE6lytgYkgqBNWa2mTBOxhQAMzstbp0xQLEnCuecq18pJwsz+xfhNlS8XiluWy7pAuAZoACYYmYLJI0HSs3sKaAE+J0kI7R6Oj/V2JxzLt6w62seB+S7Nd8AsGzNNzWu+48rvbBjTDpXFkjqDhwIbFU3OZXaUGY2C5hVZd7Vce8fIVS1rWkfUwllR5xzztWjlJOFpHMIfSUKEiz22lDOOdeIpXNlcXW0/lfAWrxOlHPONRnpJIt2wLPA0WbmicI555qQdPpZ/AMwTxTOOdf0pHNl8QVwkqT/A14HNsUWmNn4TAfmnHMud6STLH5GeE7xQ+CIKss8WTjnXCOWTrJ4CX+o7ZxzTVI6nfJKshiHc865HJbW4EeSdpQ0WtKvowGQuknaIVvBOeecyw3pDH7UBVgA/IUwEFF74GPgmuyE5pxzmbfp7Sdgw1dhYsNXYdollc6Vxe+AbsBGADNbThgMKeWhVZ1zrqHZhq/BopEVbHOYdkmlkywOI4xUd3fcvEVAUSYDcs65bFKrNtCsOagZNGsepl1S6bSG2hlYXmXejngLKedcHmlxgFeSrY10riw+AoYCPQAkjSKMk704C3E555zLIekkiz8SxuE+Npq+F2gFTMp0UM4553JLOsliCnAj8C0gwpjcN5vZPdkIzDnnXO5Ip1OeAVdIuho4hVCqfE62AnPOOZc7kl5ZRB3wnpfULpo1iXAL6kngA0kDshmgc865hpfKbaiTgI5m9qWkvYExhNtQAjoQBkVyzjnXiKWSLLoD86P3R0b/vk7owT0PGJiFuJxzzuWQVJLFTsC66P0gQr+K6Wa2lvDMon0qB5J0tKQPJS2WdEWC5d2j213zJc2W1DWa31/SHEkLomUjUvrJnHPOZUwqyeIL4DBJ/dlyZRF7sL0LWxJJtSQVAHcCw4DewEhJvausdhMwzcz6EsbH+F00/7/AaDPbDzgauFVS2xTids45lyGpJIsXgH2BuUAnQvJ4PVo2AFiSwj4GAYvNbImZfQc8ROjQF693dCyAF2PLzWyRmX0Uvf88On7HFI7pnHMuQ1JJFr8mJAoBXwM/NTOTVEKoC/VSCvvoAnwWN10WzYv3DuFhOsCJQBtJHeJXkDQIaEmodkuVZWMllUoqXblyZQohOeecS1XSfhZmtgwYGN36+drMKqJFrwBtCJ3zMuEy4A5JYwgJaBkQOxaSdiM02T3DLFYycqs4JwOTAYqLi71elXPOZVA6nfLWVpkuB8pT3HwZsHvcdNdoXvz+Pie6spDUGjg5dkxJOwEzgSvN7LVUY3bOOZcZaY2UVwdvAntL6iGpJXAq8FT8CpIKJcXi+RWhvAjR+o8THn4/Uk/xOueci1MvySK6CrkAeAZYSGh6u0DSeEnHRauVAB9KWgTsClwfzf8xMBgYI2le9OpfH3E755wL0hnPok7MbBYwq8q8q+PePwJsc+VgZvcB92U9QOecc9Wqr9tQzjnn8pgnC+ecc0l5snDOOZeUJwvnnHNJebJwzjmXlCcL55xzSXmycM45l5QnC+ecc0l5snDOOZeUJwvnnHNJebJwzjmXlCcL55xzSXmycM45l5QnC+ecc0l5snDOOZeUJwvnnHNJebJwzjmXlCcL55xzSXmycM45l5QnC+ecc0l5snDOOZdUvSULSUdL+lDSYklXJFjeXdLzkuZLmi2pa9yyMyR9FL3OqK+YnXPOBfWSLCQVAHcCw4DewEhJvausdhMwzcz6AuOB30XbtgeuAQ4CBgHXSGpXH3E755wL6uvKYhCw2MyWmNl3wEPA8VXW6Q28EL1/MW75UcCzZrbGzL4EngWOroeYnXPORZrX03G6AJ/FTZcRrhTivQOcBNwGnAi0kdShmm27VD2ApLHA2GhyvaQPMxN6nRQCq+qyA12VoUhSU+d4wWNOgcecffkWL+RGzN2rW1BfySIVlwF3SBoDvAQsAypS3djMJgOTsxNa7UgqNbPiho4jVfkWL3jM9SXfYs63eCH3Y66vZLEM2D1uums0r5KZfU64skBSa+BkM1sraRlQUmXb2dkM1jnn3Nbq65nFm8DeknpIagmcCjwVv4KkQkmxeH4FTInePwMMldQuerA9NJrnnHOuntRLsjCzcuACwh/5hcB0M1sgabyk46LVSoAPJS0CdgWuj7ZdA1xLSDhvAuOjefkgp26LpSDf4gWPub7kW8z5Fi/keMwys4aOwTnnXI7zHtzOOeeS8mThnHMuKU8WdSCpQtK8uFdRDeuOkXRH9H6cpMvqK84qcVwpaUFUVmWepKr9XarbrkjSe1mKaX2V6cpzlU/S+Tw0QEwLJL0j6RexhiSSiiVNrIcYiiSNSmG9Wn02axHPLElts7HvuGOcIMkk7ZvN49SnXOpnkY++NbP+DR1EqiQdDBwDHGhmGyUVAi0bOKzGJKOfB0nNo8YhdVEZk6RdgAeAnYBrzKwUKK3j/lNRBIyKjp1QXT6bqZ4nSSI8px2eUtR1MxJ4Jfr3mmwdJEOfkZT4lUWGSVoafdBj39xmN3BI8XYDVpnZRgAzW2Vmn0u6WtKbkt6TNDn6T4WkAdG30XeA8xsiYEnHSnpd0tuSnpO0azR/nKR7Jc2JCkyeE80vkfSSpJlR4cpJkppJOkvSrXH7PUfSLfUQ/wBJ/5Q0V9IzknaLO/6b0fl9VNIO0fypUcyvAzdmMhYz+4JQ5eACBSWSZkTHHRJ3RfS2pDbRefujpA8kPRt9Iz8lWj/h5zzRfoAJwKHRvEurCa+6z2Z1x4n9/v8F3BtdjT6pUIT0I0nXROsVRZ+DacB7wO6xfUraMfqcvBN99kfU9DtLlUI/sUOAswndBGKfy9mSHonO5/1x/8+GR/PmSpoY9zvZUdIUSW9E5/L4aP4YSU9JegF4Pp3Y6sTM/FXLF6GH+bzo9Xg0bylQGL0vBmZH78cAd0TvxwGXNUC8raNYFwF/BIZE89vHrXMvcGz0fj4wOHr/e+C9ejiP84BP485VO7a02vspcHPcOXwH2J5QJuEzoDOhCfYGYA+ggFBL7JToZ/8YaBFt/yqwfzY/D0CL6Dgdo+UjgCnR+w5x210HXBi9nwrMAAoyFNP6BPPWEpqnlwAzonl/B34Q9zlpHp23WYQvlZ2AL4FTknzOE+2n8ji1+GxWd5xxwFxg+7j/X8uBDtFn4r1o/SJgM/C9uGMtjT4zJwN/ipu/c02/szTO+WnAPXGfswHROVhH6FTcDJhDSCitCJ/dHtH6D8b9Tn4LnB69bxudmx2jn7WMuP+39fHy21B1k1e3ocxsvaQBwKHAYcDDCuXiv5Z0ObAD0B5YIOlloK2ZvRRtfi+hanA2bHUeFUq+xMoedI3i3I1wW+Lfcds9aWbfAt9KepFQsHIt8IaZLYn29SBwiJk9En0TO0bSQkLSeDfLP0cfoA/wbPQlsoDwBw2gj6TrCH8EWrN1R9O/mVnKpW4y5F/AHyTdDzxmZmWSDoli2QysiM5xbfaTdKMaPps1eSr6/cc8a2arASQ9Rvhj/ATwiZm9lmD7d4GbJd1A+AP9cpLfWapGEmrcQSiaOpLwBeANMyuL4ptHSGTrgSVmFvtcP8iWGndDgeO05flmK6Bb3M9ar/3NPFlkXjlbbu+1ashAEon+CM0GZkt6F/h/QF+g2Mw+kzSO3Ir7duAPZvaUpBLCN8qYqp2ELMn8PwO/Bj4A/pLZMBMSsMDMDk6wbCpwgpm9EyXHkrhl32QtIGkPwhXQF0Cv2HwzmyBpJjAc+Jeko5LsKuHnvBb7qZTgs3lGdceJVD1P1f3eE55PM1sk6cAo1uskPU+4Iqzud5aUwpAKhwP7SzJCsjFgJrAxbtUKkv/9FaHs0VZFURUe/GftM1Idf2aReUsJl50QLnNzhqR9JO0dN6s/EPsgrorutZ4CYGZrgbXRt0sIl9YNYWe21BGrOvDV8ZJaKVQnLiH08AcYpFBaphnhNsIrAGb2OqFG2SjCN7hs+xDoqPDwFkktJO0XLWsDLJfUgno6t5I6ApMIt/isyrI9zexdM7uBcB73JVwlnBw9u4jdtopZSoLPeTX7+Zrw89YUW6LP5ifVHacaR0pqL2l74IQo/pqO2Rn4r5ndR7jNeiA1/85ScQpwr5l1N7MiM9udcDV8aDXrfwjsoS0t50bELXsGuDDu2cYBacSRcX5lkXm/Ae6RdC25V/CwNXC7QrPBcmAx4ZJ3LeEe7wq2/MEFOBOYEn1D+r96jjVmHPA3SV8SxjvpEbdsPmHsk0LgWgsPRHsSfoY7gL2i5Y/HbTMd6G9hbJSsMrPvFB4IT5S0M+H/263AAuB/gdeBldG/Nf4xrYPto1seLQi/83uBPyRY7xJJhxHu7y8A/gFsAo4A3ifcV3+LcN8dqv+cJ9rPZqBCoaHEVDNL1LCgus9mr2qOk8gbwKOEW5f3mVmpam6+vD/we0mbo5/1Z0l+Z6kYCdxQZd6jwM8Iz8y2YmbfSjoPeFrSN2z9/+/a6Njzoy8+/ya0GGsQXu7D5aXodtl6M7upyvwSQuOBhP+popYmt5hZ/bUiyWOSWkfPEzoQ/hj/wMxWNHRcVcWec5nZBQ0dS7rizrEII4p+VE1CbVB+G8o1CZLaKhSp/NYTRVpmRFcmLxOu3nIuUTQC50TneAHhtuvdDRxPQn5l4ZxzLim/snDOOZeUJwvnnHNJebJwzjmXlCcL55xzSXmycC5NUUE4i17lkr6U9Jak30Rt81PdT0lsP9mM17lM8E55ztXeQrZ0FDwSOAAYLekQM1tW45bO5Rm/snCu9t4wswvM7H+AfsBXhOJwEyW1k/S8pOWSNkr6RqEk+UlQ2YmssjBf3JXKGEk9Jb0saaWkTZK+Vii7PrgBfkbnAL+ycC4jzGyhpMnAZcCxhAq+HYHnCCUy9gKOAh6U1I9QQuNRttQ7ilUpfZ9QZrslofjcN4REdCjwhKS9Y5VVnatPfmXhXObEyky3IFQaPYUw5sJ6QqXbbwlJoMTM3iDUrwLAzC6JXm+Y2RxCLaH3CMliXrRaO2BgffwgzlXlVxbOZc4e0b+bgMGEYUQTDeawa007kXQxoYBcIjVu61y2+JWFcxkgqRdwTjQ5g1BqWoQy2W0Jo7fFKrbGEkh53Pbx/xfHRP9OJ4yMFj+sZ/KRhJzLAk8WztXeIEl3RJVs3wF2Ioy/cBHwebTO/oSrhH8RnmPE+yTu/XRJt0rqFLftEGAi4bmHcw3Kk4VztdeL8GzhB4TnC78hjJVRBlxDuMJoTmhWO5ktSQAAM/uMMF7HSsKD7osJY3NcBLxEqEBaAlyV9Z/EuSS86qxzzrmk/MrCOedcUp4snHPOJeXJwjnnXFKeLJxzziXlycI551xSniycc84l5cnCOedcUp4snHPOJfX/AX3Qfn5Q3IEIAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "T_FpcxSa56j1",
        "outputId": "31452080-4cee-4a7c-82e5-0e19f7888baa"
      },
      "source": [
        "#print mean and std sensitivity or specificity for each data\n",
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='1D CNN'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "print(pdnc_df_new)\n",
        "\n",
        "df_dummy = pdnc_df_new.groupby(['Data','Test']).mean().round(2)\n",
        "print(df_dummy)\n",
        "df_dummy['SE'] = pdnc_df_new.groupby(['Data','Test']).sem().round(3)\n",
        "print(df_dummy)\n",
        "\n",
        "#pdnc_df_new.groupby(['Data','Test']).mean().round(2)\n",
        "pdnc_df_new.groupby(['Data','Test']).sem().round(3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Data         Test     Value\n",
            "0     Full  Sensitivity  0.966125\n",
            "1     Full  Sensitivity  0.933604\n",
            "2     Full  Sensitivity  0.968877\n",
            "3     Full  Sensitivity  0.986468\n",
            "4     Full  Sensitivity  0.994587\n",
            "..     ...          ...       ...\n",
            "135  Anger  Specificity  0.992063\n",
            "136  Anger  Specificity  0.825397\n",
            "137  Anger  Specificity  1.000000\n",
            "138  Anger  Specificity  0.976000\n",
            "139  Anger  Specificity  1.000000\n",
            "\n",
            "[140 rows x 3 columns]\n",
            "                      Value\n",
            "Data     Test              \n",
            "Anger    Sensitivity   0.90\n",
            "         Specificity   0.93\n",
            "Disgust  Sensitivity   0.98\n",
            "         Specificity   0.97\n",
            "Fear     Sensitivity   0.96\n",
            "         Specificity   0.96\n",
            "Full     Sensitivity   0.98\n",
            "         Specificity   0.99\n",
            "Happy    Sensitivity   0.92\n",
            "         Specificity   0.98\n",
            "Sad      Sensitivity   0.95\n",
            "         Specificity   0.98\n",
            "Surprise Sensitivity   0.95\n",
            "         Specificity   0.93\n",
            "                      Value     SE\n",
            "Data     Test                     \n",
            "Anger    Sensitivity   0.90  0.047\n",
            "         Specificity   0.93  0.037\n",
            "Disgust  Sensitivity   0.98  0.008\n",
            "         Specificity   0.97  0.014\n",
            "Fear     Sensitivity   0.96  0.016\n",
            "         Specificity   0.96  0.008\n",
            "Full     Sensitivity   0.98  0.007\n",
            "         Specificity   0.99  0.001\n",
            "Happy    Sensitivity   0.92  0.059\n",
            "         Specificity   0.98  0.012\n",
            "Sad      Sensitivity   0.95  0.016\n",
            "         Specificity   0.98  0.008\n",
            "Surprise Sensitivity   0.95  0.029\n",
            "         Specificity   0.93  0.060\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>Value</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Data</th>\n",
              "      <th>Test</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Anger</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.047</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.037</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Disgust</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.014</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Fear</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.016</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Full</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.007</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.001</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Happy</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.059</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.012</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Sad</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.016</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Surprise</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.029</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.060</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      Value\n",
              "Data     Test              \n",
              "Anger    Sensitivity  0.047\n",
              "         Specificity  0.037\n",
              "Disgust  Sensitivity  0.008\n",
              "         Specificity  0.014\n",
              "Fear     Sensitivity  0.016\n",
              "         Specificity  0.008\n",
              "Full     Sensitivity  0.007\n",
              "         Specificity  0.001\n",
              "Happy    Sensitivity  0.059\n",
              "         Specificity  0.012\n",
              "Sad      Sensitivity  0.016\n",
              "         Specificity  0.008\n",
              "Surprise Sensitivity  0.029\n",
              "         Specificity  0.060"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_RVefigpour5",
        "outputId": "8d33acdd-f580-447e-c848-1689d26a5ba3"
      },
      "source": [
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='1D CNN'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "temp = pdnc_df_new[pdnc_df_new['Data']=='Full']\n",
        "print(\"Sensitivity for full data\")\n",
        "print(temp[temp['Test']=='Sensitivity'].mean())\n",
        "print(temp[temp['Test']=='Sensitivity'].std())\n",
        "\n",
        "print(\"Specificity for full data\")\n",
        "print(temp[temp['Test']=='Specificity'].mean())\n",
        "print(temp[temp['Test']=='Specificity'].std())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity for full data\n",
            "Value    0.979283\n",
            "dtype: float64\n",
            "Value    0.021028\n",
            "dtype: float64\n",
            "Specificity for full data\n",
            "Value    0.993661\n",
            "dtype: float64\n",
            "Value    0.004493\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 799
        },
        "id": "0SicVKeJG5KD",
        "outputId": "b3ce7e82-0a89-44ed-ca44-854c9066e847"
      },
      "source": [
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='1D CNN'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "print(pdnc_df_new)\n",
        "\n",
        "x = \"Data\"\n",
        "y = \"Value\"\n",
        "hue = \"Test\"\n",
        "\n",
        "ax = sns.barplot(data=pdnc_df_new, x=x, y=y, hue = hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.80, top = 1.10)\n",
        "ax.legend(loc='upper right', bbox_to_anchor=(1,1))\n",
        "ax.set_xlabel('Data',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Sensitivity/Specificity',fontsize = 13, weight='bold')\n",
        "add_stat_annotation(ax, data=pdnc_df_new, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"Full\", \"Sensitivity\"), (\"Full\", \"Specificity\")),\n",
        "                                ((\"Sad\", \"Sensitivity\"), (\"Sad\", \"Specificity\")),\n",
        "                                 ((\"Happy\", \"Sensitivity\"), (\"Happy\", \"Specificity\")),\n",
        "                               ((\"Fear\", \"Sensitivity\"), (\"Fear\", \"Specificity\")),\n",
        "                               ((\"Disgust\", \"Sensitivity\"), (\"Disgust\", \"Specificity\")),\n",
        "                               ((\"Surprise\", \"Sensitivity\"), (\"Surprise\", \"Specificity\")),\n",
        "                               ((\"Anger\", \"Sensitivity\"), (\"Anger\", \"Specificity\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "\n",
        "props = dict(boxstyle='round', facecolor='none', edgecolor='black', alpha=0.5)\n",
        "\n",
        "# place a text box in upper left in axes coords\n",
        "ax.text(0.40, 0.95, \"1D CNN\", transform=ax.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "\n",
        "plt.savefig('sens_and_speci_pdnc_1d.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Data         Test     Value\n",
            "0     Full  Sensitivity  0.966125\n",
            "1     Full  Sensitivity  0.933604\n",
            "2     Full  Sensitivity  0.968877\n",
            "3     Full  Sensitivity  0.986468\n",
            "4     Full  Sensitivity  0.994587\n",
            "..     ...          ...       ...\n",
            "135  Anger  Specificity  0.992063\n",
            "136  Anger  Specificity  0.825397\n",
            "137  Anger  Specificity  1.000000\n",
            "138  Anger  Specificity  0.976000\n",
            "139  Anger  Specificity  1.000000\n",
            "\n",
            "[140 rows x 3 columns]\n",
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "Sad_Sensitivity v.s. Sad_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-1.357e+00\n",
            "Full_Sensitivity v.s. Full_Specificity: t-test independent samples with Bonferroni correction, P_val=3.408e-01 stat=-2.115e+00\n",
            "Happy_Sensitivity v.s. Happy_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-9.185e-01\n",
            "Fear_Sensitivity v.s. Fear_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-4.539e-01\n",
            "Disgust_Sensitivity v.s. Disgust_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=7.150e-01\n",
            "Surprise_Sensitivity v.s. Surprise_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=2.819e-01\n",
            "Anger_Sensitivity v.s. Anger_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-5.074e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAENCAYAAAD+CUlOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5yUdd3/8debBcUTKodEMQUTCxXEOKh5wkpFMk/g7TGFUrI8W7dZ9suzoZUpaSL3rRKap9SMG03TFDUlBRQVPCKigqIgIpJy2N3P74/r2mVYZ3dmYGZ3hn0/H4957Mx1/MzszHzm+/1e1+dSRGBmZtaUNi0dgJmZlT8nCzMzy8nJwszMcnKyMDOznJwszMwsJycLMzPLqVmShaSbJH0oaUYj878mabKk5ZJ+2mDeYEmvSZol6bzmiNfMzFbXXC2LccDgJuYvAs4Afps5UVIVcB1wELAjcIykHUsUo5mZNaJZkkVEPEGSEBqb/2FETAFWNpg1EJgVEbMjYgVwB3Bo6SI1M7Ns2rZ0ADl0A97NeDwX2C3bgpJGAiMBNtpoo35f+9rXSh+dmdk6ZNq0aQsjoku2eeWeLPIWEWOBsQD9+/ePqVOntnBEZmaVRdLbjc0r96Oh5gFfzni8dTrNzMyaUbkniylAT0k9JK0HHA1MaOGYzMxanWbphpJ0OzAI6CxpLnAB0A4gIsZI6gpMBToAtZLOAnaMiCWSTgMeAqqAmyJiZnPEbGZmqzRLsoiIY3LMn0/SxZRt3gPAA6WIy8zM8lPu3VBmZlYGnCzMzCwnJwszM8vJycLMzHJaZ07KM7N128Tn3yrq9g7etUdey1122WXcdtttVFVV0aZNG2644QZ22y1rIYmCfeMb3+Dpp59mzpw5PP300xx77LEATJ06lfHjxzN69OhG1x0zZgwbbrghJ5xwAuPGjeOAAw5gq622Kkpc2ThZmJk1YvLkyUycOJHnnnuO9ddfn4ULF7JixYqibf/pp58GYM6cOdx22231yaJ///7079+/yXVPOeWU+vvjxo1j5513LmmycDeUmVkj3n//fTp37sz6668PQOfOndlqq62YNm0a++67L/369ePAAw/k/fffB2DQoEH87Gc/Y+DAgeywww48+eSTAMycOZOBAwfSt29f+vTpwxtvvAHAxhtvDMB5553Hk08+Sd++ffn973/PpEmTOPjgg6mtraV79+4sXry4PqaePXvywQcfcOGFF/Lb3/6Wu+++m6lTp3LcccfRt29f7r//fg477LD65R9++GEOP/zwtX4tnCysVXniiSc45JBD6NatG5IYN27cF5YZNGgQkpDEeuutx5ZbbsngwYO59dZbiYic+1ixYgW/+c1v2HXXXdlwww3p2LEju+++OzfccAPLly8HYPjw4UjikksuWW3dSZMmIYmFCxcCyS9OSXTq1IlPPvnkC3Gedtppa/hKWD4OOOAA3n33XXbYYQd+/OMf8/jjj7Ny5UpOP/107r77bqZNm8b3v/99zj///Pp1qqurefbZZ7n66qu56KKLgKTL6Mwzz2T69OlMnTqVrbde/bSyUaNGsffeezN9+nTOPvvs+ult2rTh0EMP5a9//SsAzzzzDNtuuy1bbLFF/TLDhg2jf//+/PnPf2b69OkMGTKEV199lQULFgBw88038/3vf3+tXwsnC2tVli5dys4778w111zDBhts0OhyI0aM4P3332f27NlMmDCBPfbYgx/+8Iccfvjh1NTUNLreihUrOPDAA7nssssYMWIETz31FNOmTeOcc87h5ptvZvLkyfXLtm/fnt/85jf1H+qmfPbZZ4waNaqwJ2trbeONN2batGmMHTuWLl26cNRRR3HDDTcwY8YM9t9/f/r27cull17K3Llz69c54ogjAOjXrx9z5swBYI899uDyyy/niiuu4O23327yvdfQUUcdxZ133gnAHXfcwVFHHdXk8pL43ve+x6233srixYuZPHkyBx10UIHP/Is8ZmGtypAhQxgyZAiQ/LpvzIYbbkjXrl0B2HrrrRkwYAC77747gwcPZvz48YwYMSLreldffTWPP/44zz777Gp9zj169GDYsGEsXbq0ftp+++3H3LlzueSSS5ocyAQ444wzuOaaazjttNPo1q1bvk/XiqCqqopBgwYxaNAgevfuzXXXXcdOO+20WuLPVNdlVVVVRXV1NQDHHnssu+22G/fffz9Dhgzhhhtu4Jvf/GZe+99jjz2YNWsWCxYs4L777uOXv/xlznVGjBjBd7/7Xdq3b8+RRx5J27Zr/1XvloVZng488EB69+7NPffc0+gyf/7zn/n2t7+ddXCyTZs2dOjQYbXHo0aNYsyYMbz55ptN7vvII4+kd+/e/OpXv1rzJ2AFe+211+rHFwCmT59Or169WLBgQX2yWLlyJTNnNl2ybvbs2Wy33XacccYZHHroobz44ourzd9kk0349NNPs64ricMPP5xzzjmHXr160alTpy8s03D9rbbaiq222opLL7200R82hXLLwqwAO+644xc+6JneeOMNBg0alPf2hgwZwp577sn555/PHXfc0eSyV155Jd/61rc455xz2GmnnfLex7oi30Ndi2np0qWcfvrpLF68mLZt27L99tszduxYRo4cyRlnnMEnn3xCdXU1Z511VpP/k7vuuotbbrmFdu3a0bVrV37xi1+sNr9Pnz5UVVWxyy67MHz4cHbdddfV5h911FEMGDAg6xgbJK3kU045hQ022IDJkyezwQYbcNxxx7FgwQJ69eq11q8DOFmYFSQikNTk/EJdccUV7LHHHvz3f/93k8vtu+++HHjggfz85z9nwgRX6m8O/fr1qz+8NVPnzp154oknvjB90qRJqy1TN2Zx3nnncd55531h+bpuyXbt2vHoo4+uNi/zR0f//v2/8N668MIL6+8PHTqUoUOHrjb/X//6FyeffHLW57Um3A1lVoCXX36Z7bbbrtH5O+ywA6+88kpB2xw4cCBDhw7l3HPPzbnsqFGjuP/+++sPyTTLpl+/frz44oscf/zxRdumk4VZnh566CFmzJjBsGHDGl3m2GOP5ZFHHiHbZX1ra2tZsmRJ1vUuv/xynnzySR588MEmY+jduzcnnHBCXonFWq9p06bxxBNP1A+2F4OThbUqS5cuZfr06UyfPp3a2lreeecdpk+fzjvvvLPacp999hnz589n7ty5TJkyhYsuuogjjjiCQw89tMlfa2eddRZ77bUX+++/P6NHj2b69Om89dZb3Hvvvey1114899xzWdfbfvvtGTlyJNdcc03O53DxxRczffp0nnnmmcKevNnaiIh17tavX78wy+axxx4L4Au3E088sX6Zfffdt356u3btYosttogDDzwwxo8fH7W1tTn3sWzZshg1alT06dMn2rdvH5tttlnstttuMWbMmFi+fHlERJx44onxne98Z7X1Pvjgg9h4440DiAULFkRExFtvvRVATJkyZbVlzz333ADi1FNPXctXxGwVYGo08r2qWIMBuXLXv3//yNYNYGZmjZM0LSKyFqVyN5SZmeXkQ2fNrCIcdNl9Rd3e388/LPdClLZE+ZAhQ7jtttvYbLPNGD16NNdffz1f//rXOeqoo3j55ZezHm5bp7Hy5qXiZGFm1ohSlyh/4IEH6u//8Y9/5JFHHqkvMnjIIYc0uW5j5c1Lxd1QZmaNaKxEeffu3Tn33HPp3bs3AwcOZNasWQAsWLCAoUOHMmDAAAYMGMBTTz0FJEfhjRgxgt69e9OnT5/6kjHdu3dn4cKFnHLKKcyePZuDDjqI3//+94wbN66+ovAHH3zA4Ycfzi677MIuu+xSnyQaK2++zz77MH369PrnsNdee/HCCy+s9WvhZGFm1ohsJcrrbLrpprz00kucdtppnHXWWQCceeaZnH322UyZMoV77rmHk046CYBLLrmkfvkXX3zxC0UEx4wZw1ZbbcVjjz22WolySIpI7rvvvrzwwgs899xzXygr0rC8+Q9+8IP6siCvv/46y5YtY5dddlnr18LJwsysEdlKlNd9ER9zzDH1f+uKCj7yyCOcdtpp9O3bl0MOOYQlS5awdOlSHnnkEU499dT67W6++eZ5x/Doo4/yox/9CEgq2W666aZNLn/kkUcyceJEVq5cyU033dRkdeVCeMzCzKwJDUuU/+lPfwJYrUZY3f3a2lr+/e9/0759+xaJFZLy+vvvvz9/+9vfuOuuu5g2bVpRtuuWhZlZI7KVKN92220B6i9IdOedd7LHHnsASbfVH/7wh9WWB9h///257rrr6qd//PHHecfwrW99i+uvvx6AmpqaL1wxMVt585NOOokzzjiDAQMGFNSKaUreLQtJNwI3RsQXSzCamZVYvoe6FlNjJconTpzIxx9/TJ8+fVh//fW5/fbbARg9ejSnnnoqffr0obq6mn322YcxY8bwy1/+klNPPZWdd96ZqqoqLrjggvor6uVyzTXXMHLkSG688Uaqqqq4/vrr65MTfLG8+dlnn02/fv3o0KFD0a5lAeR/BrekWpISCLOAm4BbIuK9PNe9CTgY+DAids4yX8A1wBDgM2B4RDyXzqsBXkoXfScimj6eDJ/BbWal1b17d6ZOnUrnzp1bOpSs3nvvPQYNGsSrr75Kmzb5dyAV6wzuccBioCdwOfC2pPslDZXULo91Bzcx/6B0uz2BkcD1GfM+j4i+6S1nojAza83Gjx/PbrvtxmWXXVZQosiloNpQktoC3waGAYcBdZ1hi4A/AJdHRHUj63YHJjbSsrgBmBQRt6ePXwMGRcT7kpZGxMZ5B4lbFmZma6JotaHSRPAY8DjwOqD01gm4APjfNYyxG/BuxuO56TSA9pKmSvq3pObvtDQzs/yThaS+kq4F3ifpVtodWEDSJfV1kgSS34hNYbZNM92xwNWSvtJIfCPTpDJ1wYIFJQjDzKz1KuQ8i+dIBrgFPAtcC9wVESsAJP0fMGgN45gHfDnj8dbpNCKi7u9sSZOAXYE3G24gIsYCYyHphlrDOMzMLItCuqGWA7cCu0XE7hFxa12iSP0v0GMN45gAnKDE7sAn6XjF5pLWB5DUGdgTeHkN92FmZmuokJbF/yMZhK4fOZb0JaBzRLwcEZ8Cn2ZbUdLtJK2OzpLmkoxvtAOIiDHAAySHzc4iOXS27uDgXsAN6WG7bYBREeFkYWbWzApJFleQnAuReZjRz4EzgKqmVoyIY3LMD+DULNOfBnoXEKOZmZVAId1QyjJt62IFYmZm5StnspBUk55FHcCZdY/TaUeQHBFlZmbrsHy6oepaFEH21sV1WaaZmdk6JJ9ksR9JkngUuJtVyaEaeDsi5pYoNjMzKxM5k0VEPA4gaQQwM/NoKDMzax2aHLOQdIKkujohAeyYTlvtVvowy9+cOXPo1asXJ598MjvttBMHHHAAn3/+OaNHj2bHHXekT58+HH300S0dZr1Kixccc3OptJgrLV6ozJiJiEZvQC3wu4z7NVlu1U1toyVu/fr1i+b21ltvRVVVVTz//PMREXHkkUfGLbfcEltuuWUsW7YsIiI+/vjjZo+rMZUWb4Rjbi6VFnOlxRtRvjEDU6OR79VcR0O9A3yccT/b7d3sq7Y+PXr0oG/fvgD069ePOXPm0KdPH4477jhuvfVW2rYtr6vYVlq84JibS6XFXGnxQuXF3GSyiIjuEXFpxv0e2W7NE2r5W3/99evvV1VVUV1dzf3338+pp57Kc889x4ABA6iuzlrBvUVUWrzgmJtLpcVcafFC5cVcSNXZjpK2Sa9qR1rHaRtJHUsXXmWrra3l3XffZb/99uOKK67gk08+YenSpS0dVqMqLV5wzM2l0mKutHih/GMupJ3zALBZRHwNkhIdkv4OLAH2aHLNVqqmpobjjz+eTz75hIjgjDPOYLPNNmvpsBpVafGCY24ulRZzpcUL5R9zIdfgXgLcGREnZ0wbCxwdER1KFN8a8ZXyzMwK19SV8gppWdQAXRtM24rk5LxW5d577+XBBx8syrYGDx7MEUeU4ppRq1RavOCYHXN2lRYvVGbM2RRSSPB1YIikiyUNlnQxcBDwRmlCK18PPvggr7/++lpv5/XXXy/am6gplRYvOGbHnF2lxQuVGXM2hbQsriO5nOr5DaZfW7RoKsgOO+zA2LFj12obI0eOLFI0uVVavOCYm0ulxVxp8UJlxtxQ3i2LiBgPjAI+J6kV9TlwRUTcUqLYzMysTBR01kdE/ELSBUAXYEFErCxNWGZmVk6aTBaS9gHmRsTs9H6m7dNTLoiIJ0oUn5mZlYFcLYvHgKuBnwCTSIoJNhR5bGed8tWvfrWsttNc+2mueIu5L8fcPPvye7n0+2rOmLNp8jwLSbXA1RFxTno/q4go5KiqkvN5FmZmhVvj8ywyk0C5JQQzM2s+ua5n8aGki9L7j0r6UfOEZWZm5SRXa6FzegMYBPQsaTRmZlaWcg1MfwiMkLRT+niYpL4NlomI+FbxQzMzs3KRK1ncCZwO7ENy1NPW6S1TfpUIzcysYuUa4D5T0iRgJ+Bi4N/AQ80Ql5mZlZGc50dExF+Bv0raHngoIm4vfVhmZlZO8j6ZLiKGlzAOMzMrY7kOna2R9LuM+9luOa9nIemm9DDcGY3Ml6TRkmZJelHS1zPmnSjpjfR2YqFP0MzM1l6uQ2eV3jLvZ7vlMg4Y3MT8g0gOy+0JjASuh+S638AFwG7AQOACSZvnsT8zMyuiXN1Q+wFzM+6vkYh4QlL3JhY5FBgfSe2Rf0vaTNKWJOd2PBwRiwAkPUySdDxuYmbWjHIdDfV4tvsl0A14N+Px3HRaY9O/QNJIklYJ22yzTWmiNDNrpfKu9yRplKQJkqrSx20l3SdpVOnCy19EjI2I/hHRv0uXLi0djpnZOqWQ0uLDgccjogYgIqolfQacCJy3lnHMA76c8XjrdNo8kq6ozOmT1nJfLe7cc89l/vz5LFy4kOrqatq2bcvOO+/MlVde2dKhmZllVUgl2U2BpQ2mfZ5OX1sTgBPSo6J2Bz6JiPdJTgA8QNLm6cD2AawDJwXOnz+fefPmsXz5cmpqali+fDnz589v6bDMzBpVSMviHeBQSb+OiFnpSXqHsPqYQlaSbidpIXSWNJfkCKd2ABExBngAGALMAj4DRqTzFkm6BJiSburiusHuSnDQZfdlnb7ig4WgNhCrLhEy74OFWZf/+/mHlSw+M7N8FZIsHiSpE/WqpIUk1WgF3JprxYg4Jsf8AE5tZN5NwE0FxFn+2m0A1cu/OM2sGZ177rnMmDGj1XWFttbnvbYK6Ya6BHgzXedL6d/ZwKUliGudpvabwAabQZu2SQujTdtkmlkzmj9/fqvsCm2tz3ttFVLuY6GkPsARwLbA28C9EfF5qYJbV7Xb1V1L1vK6du3K/Pnzqampoaqqiq5du7Z0SM2itT7vtVVINxTAMtIxioj4c/HDsXJViU33Soy5OV155ZWccMIJzJs3j65du7aa16a1Pu+1lXeykLQZ8A+gHxCS/g28DlwSEReWJrzy01q/gOqa7gA1NTUV0XSvxJgrjQ8Dbz0KGbO4GOhPWg8qIt4EppLUdWo1Wmt/Z9euXamqqgKomKZ7JcZcaXwYeOtRSLI4GJgM/DFj2svAV4oaUZlrrV9AV155Zf1zrZSmeyXGXGm6du1Kt27dWuVnorUpZMziS8A/gZUNpq9XvHDKn/s7rZJMfP6tJuf/Z3l1/d+mlj141x5Zp9e9//2ZWPcVkizeIqk8+zSApG+QVItt+t1oVqDWOi5kVs4KSRbjgSuAup8YT6Z/Ly9qRNbqeWDasmn4I6Jz585uyTSjQpLF1SRHQv1XxrS7gWuKGlEZKFbTvdI0Vp6kzopF/wFg3qL/NLns2pYo8XHwlk3DHxHz5s1r4Yhal7wHuCNiZUQcDWwD7AVsExH/FRENxzDM1ooHppvHPWOvYsmihQAsWbSQe8Ze1cIRNa3hwSXdunXzD4lmVNBJeZK2Ao4lSRhvS7otIpzercWUS2uoEi1Z9BG1tTUA1NbWsGTRRy0cUdMaHlwyfvz4lg6pVSnkpLy9SarDbpgx+QJJQyLiiaJHZtYKNefgfoeOnVj6ycfU1tTQpqqKDh07lWQ/tm4odMxiI+AD4A2gJ7AFcBXJyXqtQram+9CR57RwVLauaM7Bfb9vrRCFJIteJGds7xkRKyWtB/wL2KkkkZWpSmu6F8vK5++DZUuSB8uWsPL5+1wQMUOxWgTlOrjv7j4r5AzuF4EZdQPaEbECmAk8X4rAylWHjp1o22492rSpom279VpN0z2WfbrqYk1Rmzy2esUqA+PBfStXhbQs7gIukfQiSZLYGRgGnCdpn7qF1vXxi0pruhfrF6/ab0IsX5okDLXx9TcaKNcWga0byuFE1UKSxW+BAH7XYProjPtR4DatxIrVB+4up6a5DIyVUjmcqFpINxSkFWebuBW6PSux1lr4ELKPs5hVonL4HBdypTwnggrUmn/xlus4iweLrVDl8Dl2ArB1ltpv4uucmxVJzpaFpO2BrYApEfG5pN2BXwObAPcDF0XU/XwzKx8eZzErnny6oa4hqQXVRVJ74F6Sk/EE7AqsAC4rWYRmZtbi8umG2gn4V3pexX5AV+AzkqOglpHUijKzVmjl8/exYvItPpCgFcgnWXQG3k7v753+vTkizgL+SlJU0MyKoNKO4Ipln8Lni8vyQAIrrny6oWqBzdP7u5OcS1F34aPPSxGUWTlqjuuclOsRXI1R+00IAJ+wuc7LJ1m8ARwh6T5gX5Jk8Xg67yvAhyWKzazVqbQz5X0QQeuR7wD3OOCQ9PG9EfGhpM7AN4C/5bMjSYPTbVUB/xsRoxrM3xa4CegCLAKOj4i56bwa4KV00Xci4hCsXmu9st+6yF++Vq5yJouIGC/pI2B/YB5wbTrry8AfSK5x0SRJVcB16TbmAlMkTYiIlzMW+y0wPiL+JOmbJIfnfi+d93lE9M3zOZmZWZHlc57FkcD9EXF/5vSIeJ78K84OBGZFxOx0m3cAhwKZyWJHoK5K32NAeY/smZm1Ivl0Q90GrJT0MMk5Fv8XEYsK3E834N2Mx3OB3Ros8wJwBElX1eHAJpI6RcRHQHtJU4FqYFREfCGRSBoJjATYZhsfoGVWiYrVpXrwrj2KGpfld+hsV+D0dNkxwHxJ/5T0Y0ndihjLT4F9JT1PMpA+D6hJ520bEf1Jzum4WtJXGq4cEWMjon9E9O/SpUsRwzIzs3zGLD4CbgRulLQx8B2SX/6/Bkanv/jvBW6rG5DOYh7JGEedrdNpmft5j6RlQbqfoRGxOJ03L/07W9IkkjPH38zzOZqZtYhyuA5FsRRUSDAilkbEnRFxNMlRS4eRXAjpp8D3m1h1CtBTUo/0cqxHAxMyF5DUWVJdPD8nOTIKSZtLWr9uGWBPVh/rMDMrS8W6gmI5yDtZSNo783FErIiIiRHxA5KuqusbWzciqoHTgIeAV4C7ImKmpIsl1R0GOwh4TdLrJLWn6upN9QKmSnqBZOB7VIOjqMzMylI5XIeiWAppWTwu6TVJP284VhERtRGxoKmVI+KBiNghIr4SEZel034VERPS+3dHRM90mZMiYnk6/emI6B0Ru6R/byzwOZqV3D1jr2LJooUALFm0kHvGXtXCEVk5WJeuqV5IsvgM6AlcCsyR9ICkYZLalSY0s8qxZNFH1NYmx2PU1tawZNFHLRyRWXEVkiw6A8OAv5BUmx0M3Am8L2l0tiOUzFqLDh070bbderRpU0XbduvRoWOnlg7JrKgKuazqMpKjnu5NB5pvJjkyqiPJeMQPJZ0YEXeUJFJbI9m6R4aOPCfHWlYov6a2rivoaChJ/SWNAWYBQ9LJ75Bc2+ITYFRj61rLcPeImRVD3i2L9GS5PiRXyAN4lKRO1ISIqJX0IXBJ8UO0tdGhYyeWfvIxtTU1tKmqcveIma2RvJMFsAuwFLgFuDYiXmkw/5+AB7vLjLtHzKwYCkkWZ5JcIW9ptpkR8QzwTFGiMjOzslJIsrga6A78pG6CpLOBYyJiYJHjMjNrVQ66rOlC2ysW/QeAeYv+0+Syfz+/NNdEKSRZiFXjFXW2A/oVLxwzs7VX7l+8lSif61nclPHwgIzHbYCDSU7WMzOzdVg+LYvhJNfdDpI6TTs2mP/PIsdkZmZlJp9k8af074kkRQCfTR9XA28DN5QgLjMzKyP5XM9iBICkHsBfIuK6kkdlZmZlpZByH4NKGIeZWZMalq65edT5dOjYyecSNZMmy31IqpH0u4z72W7VzROqmbVmDUvXLF74gcvXNKNcLYvMw2UbHjZrtsYmPv9Wk/P/s7y6/m+uZa11aFi6ZuNNN3f5mmaUK1nsB8zNuG9m1iLc3dSymkwWEfF4xsNPI+K5EsdjZmZlqJAS5VMkTZF0sqSNShaRmZmVnUKSRQ1JaY8xwHuSxkhyqQ8zs1agkGSxJXAGMBXYBBgJPCtpaikCMzOz8pF3soiIjyLi2ojYjaTsx6MkR0jtWqrgzMysPBRSdRZJ2wHHA98jqTgLSfeUmZmtwwq5rOq/gD3qHpLUhboxvZmZ2TqskJbFN0iKB/4fMBb4R0RESaIyM7OyUkiy+AXJZVU/KFUwZmZWngopJDiqlIGYmVn5arZCgpIGS3pN0ixJ52WZv62kf0p6UdIkSVtnzDtR0hvp7cRCn6SZma2dXIfONiwk2Nit6Y1IVcB1wEEkV9o7RlLDK+79FhgfEX2Ai4Ffp+t2BC4AdgMGAhdI2jznMzMzs6JprkKCA4FZETEbQNIdwKHAyxnL7AjUVQp7DKi7ivqBwMMRsShd92FgMHD7WsRjZlYUzVFBeeXz98GyJcmDZUtY+fx9tNv1sDXa1poqpJDgtsDLEVF/xrakLwGd89hPN+DdjMdzSVoKmV4AjgCuAQ4HNpHUqZF1uzXcgaSRJGeVs8022+QRkplZZYhln0LUpg9qk8fNrJByH+OAYxtM+znwUpFi+Smwr6TngX2BeRRwwl9EjI2I/hHRv0uXLkUKycys5an9JtCmLagNtGmbPG5mBZ3BncXWuRcBki/+LzdYb17mAhHxHknLAkkbA0MjYrGkecCgButOWsN4zcwqTnN3OWWTs2VRd9QTEMCZmUdCkXy5L++pXxgAABEbSURBVMhjP1OAnpJ6SFoPOBqY0GA/nSXVxfNz4Kb0/kPAAZI2Twe2D0inmZlZM8mnGyrXEVHX5dpARFQDp5F8yb8C3BURMyVdLOmQdLFBwGuSXge2AC5L110EXEKScKYAF9cNdpuZWfPIpxtqP5Kk8ChwN6uSQzXwdkTMbWzFTBHxAPBAg2m/yrh/d7r9bOvexKqWhpmZNbOcyaLuiChJI4CZmUdDmZlZ69BkspB0AqsOlw1gxywn0xER40sUn5mZlYFcLYtxwO9Jro43jiRhNBSAk4WZ2TosV7J4B/g4475LkpuZtUK5zuDunu2+mZm1LoWcwV1P0vqSjpV0YLEDMjOz8pN3spB0n6SP0hPn/gLcAjwg6Scli87MzMpCIS2LPsDTwAbAEGAWsAw4qQRxmZlZGSkkWWxJMsj9VZKT9I4F7mD1mk9mZrYOKiRZLCMpU74XyVFRr5BUha0tQVxmZlZGCqk6O4PkSncHATMi4jNJ3WlQPdbMzNY9hbQsziO58NCHwM8krU/SBfX3UgRmZmblI++WRUQ8RdINlalXccMxM7NyVNDFjyRtC3wdWO0yTa4NZWaVohyuZ12J8k4Wkk4mKU9elWW2k4WZVYRyuJ51JSqkZfGrdPklwGJcJ8rMKpDab0IsX5okDLVpketZV6JCksXmwMPA4IhwojCziuQupzVTyNFQfwfCicLMrPUppGXxIXCEpH8AzwAr62ZExMXFDszMzMpHIcniRyTjFN8GvtVgnpOFmdk6rJBk8QQe1DYzy9s9Y69iyaKFACxZtJB7xl7F0JHntHBUa6aQk/IGlTAOs3rr0gfMWrcliz6itrYGgNraGpYs+qiFI1pzBV38SNJGkk6Q9Iv0AkjbSNqwVMFZ67QufcCsdevQsRNt261HmzZVtG23Hh06dmrpkNZYISfldQOeYlVJ8puBN4GrgJ8VPzRrrTp07MTSTz6mtqaGNlVVFf0Bs9ZtXWoRFzJm8WtgG5JS5etHxPuSngYOxMnCimhd+oCZrSsK6YbaD/gHcEPGtNeB7sUMyMzMyk8hyWJT4P0G0zbCR0iZma3zCkkWbwAHAD0AJB0LHEpyLe6cJA2W9JqkWZLOyzJ/G0mPSXpe0ouShqTTu0v6XNL09DamgJjNzKwIChmz+CPwP8B308e3pH9zfnlLqiKpWLs/yQWUpkiaEBEvZyz2S+CuiLhe0o7AA6zq4nozIvoWEKuZmRVRIS2Lm4Argc8BkQx0/y4ibsxj3YHArIiYHRErgDtIWiWZAuiQ3t8UeK+A2MzMrITyThaROA/oCBwPHAVckefq3YB3Mx7PTadluhA4XtJcklbF6RnzeqTdU49L2jvfmM3MrDhyJov0BLx/Sto8nTSGpAvqb8CrkvoVKZZjgHERsTUwBLhFUhuSQfVtImJX4BzgNkkdGq4saaSkqZKmLliwoEghmZkZ5NeyOALoEhEfS+oJDCfphhLQieSiSLnMY9XJfABbp9My/QC4CyAiJgPtgc4RsTwiPkqnTyM5EXCHhjuIiLER0T8i+nfp0iWPkMzMLF/5JIttgRfT+/unf58h6Y6aDgzIYxtTgJ6SekhaDzgamNBgmXdIq9lK6kWSLBZI6pIOkCNpO6AnMDuPfZqZWZHkkyw6AJ+k9weSDETfFRGLgckkSaNJEVENnAY8BLySrj9T0sWSDkkX+wlwsqQXgNuB4emFlvYBXpQ0HbgbOCUiFuX9DM3MbK3lc+jsh8B+kvqyqmUxOf37JVYlkiZFxAMkA9eZ036Vcf9lYM8s690D3JPPPszMrDTyaVk8CnwNmAZ0JUkez6Tz+uEuITOzdV4+yeIXJIlCwKfASRERkgaRnDT3RMmiMzOzspCzGyoi5gEDJG0GfBoRNemsfwGbkJycZ2Zm67BCrpS3uMHjaqC66BGZmVnZKehKeWZm1jo5WZiZWU5OFmZmlpOThZmZ5eRkYWZmOTlZmJlZTk4WZmaWk5OFmZnl5GRhZmY5OVmYmVlOThZmZpaTk4WZmeXkZGFmZjk5WZiZWU5OFmZmlpOThZmZ5eRkYWZmOTlZmJlZTk4WZmaWk5OFmZnl5GRhZmY5OVmYmVlOThZmZpaTk4WZmeXUbMlC0mBJr0maJem8LPO3kfSYpOclvShpSMa8n6frvSbpwOaK2czMEm2bYyeSqoDrgP2BucAUSRMi4uWMxX4J3BUR10vaEXgA6J7ePxrYCdgKeETSDhFR0xyxm5lZ87UsBgKzImJ2RKwA7gAObbBMAB3S+5sC76X3DwXuiIjlEfEWMCvdnpmZNRNFROl3Ig0DBkfESenj7wG7RcRpGctsCfwD2BzYCPh2REyTdC3w74i4NV3uRuDvEXF3g32MBEamD78KvFbip5WPzsDClg6iAJUWLzjm5lJpMVdavFAeMW8bEV2yzWiWbqg8HQOMi4jfSdoDuEXSzvmuHBFjgbEli24NSJoaEf1bOo58VVq84JibS6XFXGnxQvnH3FzJYh7w5YzHW6fTMv0AGAwQEZMltSfJtPmsa2ZmJdRcYxZTgJ6Sekhaj2TAekKDZd4BvgUgqRfQHliQLne0pPUl9QB6As82U9xmZkYztSwiolrSacBDQBVwU0TMlHQxMDUiJgA/Af5H0tkkg93DIxlQmSnpLuBloBo4tYKOhCqrbrE8VFq84JibS6XFXGnxQpnH3CwD3GZmVtl8BreZmeXkZGFmZjk5WawFSTWSpmfcujex7PD0nBEkXSjpp80VZ4M4zpc0My2pMl3Sbnmu113SjBLFtLTB4/rXqpIU8n5ogZhmSnpB0k8ktUnn9Zc0uhli6C7p2DyWW6P35hrE84CkzUqx7Yx9HCYpJH2tlPtpTuV0nkUl+jwi+rZ0EPlKz185GPh6RCyX1BlYr4XDWpcU9f0gqW1EVK/lZupjkvQl4DaSSgkXRMRUYOpabj8f3YFj031ntTbvzXxfJ0kiGacdkmvZIjgG+Ff694JS7aRI75G8uGVRZJLmpG/0ul9uk1o4pExbAgsjYjlARCyMiPck/UrSFEkzJI1NP1RI6pf+Gn0BOLUlApb0XUnPpAUmH5G0RTr9Qkm3SJos6Q1JJ6fTB0l6QtL9aeHJMZLaSPq+pKsztnuypN83Q/z9JD0uaZqkh9JKBXX7n5K+vvdI2jCdPi6N+RngymLGEhEfklQ5OE2JQZImpvvdN6NF9LykTdLX7Y+SXpX0cPqLfFi6fNb3ebbtAKOAvdNpZzcSXmPvzcb2U/f/f4rkBN7hkv4maVL6frggXa57+j4YD8wAvly3TUkbpe+TF9L3/lFN/c/yJWljYC+Sc8eOTqcNSmO7O309/5zxORuSTpsmaXTG/2QjSTdJejZ9LQ9Npw+XNEHSo8A/C4ltrUSEb2t4A2qA6entr+m0OUDn9H5/YFJ6fzhwbXr/QuCnLRDvxmmsrwN/BPZNp3fMWOYW4Lvp/ReBfdL7vwFmNMPrOJ3knJu612pzVh21dxLwu4zX8AVgA5KTN98lKTQ5CFgGbEdymPbDwLD0ub8JtEvXfxroXcr3A9Au3U+XdP5RJIeNA3TKWO9S4PT0/jhgIlBVpJiWZpm2GNgifa0mptP+D9gz433SNn3dHiD5UdkV+BgYluN9nm079ftZg/dmY/u5EJgGbJDx+Xof6JS+J2aky3cHaoHdM/Y1J33PDAX+J2P6pk39zwp4zY8Dbsx4n/VLX4NPSE4qbgNMJkko7Uneuz3S5W/P+J9cDhyf3t8sfW02Sp/rXDI+t81xczfU2qmobqiIWCqpH7A3sB9wp5Jy8Z9KOhfYEOhIcm7Lk8BmEfFEuvotwEElCm2111HScJIPOiQfrjvTX3frAW9lrPe3iPgc+FzSYyQFJhcDz0bE7HRbtwN7RcTd6S+xgyW9QpI0Xirx89gZ2Bl4OP0RWUXyhQaws6RLSb4ENiY5B6nOX6L5zyV6CrhK0p+BeyNirqS90lhqgfnpa7wm28m5UhPvzaZMSP//dR6OiI8AJN1L8mV8H/B2RPw7y/ovAb+TdAXJF/STOf5n+ToGuCa9f0f6eCLJ+3JuGt90kkS2FJgdSZFUSJJFXY27A4BDtGp8sz2wTcZzXVRgXGvFyaL4qlnVvde+JQPJJv0SmgRMkvQS8EOgD9A/It6VdCHlFfcfgKsiYoKkQSS/KOs0PEkockz/X+AXwKvAzcUNMysBMyNijyzzxgGHRcQLaXIclDHvPyULSNqOpAX0IdCrbnpEjJJ0PzAEeEq5rxuT9X2+Btupl+W9eWJj+0k1fJ0a+79nfT0j4nVJX09jvVTSP0lahI39z3KS1BH4JtBbUpAkmwDuB5ZnLFpD7u9fAUMjYrWiqEoG/kv2HmmMxyyKbw5JsxOSZm7ZkPRVST0zJvVlVXXehWlf6zCAiFgMLE5/XULStG4Jm7KqFtiJDeYdKqm9pE4kX7ZT0ukDlZSWaUPSjfAvgIh4hqTO2LEkv+BK7TWgi5LBWyS1k7RTOm8T4H1J7Wim11ZSF2AMSRdfNJj3lYh4KSKuIHkdv0bSShiajl3UdVvVmUOW93kj2/mU5Pk2FVu29+bbje2nEftL6ihpA+CwNP6m9rkV8FkkFa1/A3ydpv9n+RgG3BIR20ZE94j4MklreO9Gln8N2E6rjpw7KmPeQ8DpGWMbuxYQR9G5ZVF8FwE3SrqE5FdSOdkY+IOSwwarSa4NMpKk62YGMJ9VX7gAI4Cb0l9I/2jmWOtcCPxF0sfAo0CPjHkvAo+R9D9fEsmA6A4kz+FaYPt0/l8z1rkL6BsRH5c68IhYoWRAeLSkTUk+b1cDM4H/BzxDUv/sGXJ8ma6FDdIuj3Yk//NbgKuyLHeWpP1I+vdnAn8HVpLUa3uZpF/9OZJ+d2j8fZ5tO7VAjZIDJcZFRLYDCxp7b/ZqZD/ZPAvcQ9J1eWtETFXThy/3Bn4jqTZ9rj/K8T/LxzHAFQ2m3QP8iGTMbDUR8bmkHwMPSvoPq3/+Lkn3/WL6w+ctkiPGWoTLfVhFSrvLlkbEbxtMH0Ry8EDWD1V6pMnvI6L5jiKpYJI2TscTOpF8Ge8ZEfNbOq6G6sa5IuMaOZUi4zUWyRVF32gkobYod0NZqyBpM0mvkwxCO1Hkb2LaMnmSpPVWdoliHXBy+hrPJOl2vaGF48nKLQszM8vJLQszM8vJycLMzHJysjAzs5ycLMzMLCcnC7MCpQXhIr1VS/pY0nOSLkqPzc93O4PqtlPKeM2KwSflma25V1h1ouD+wK7ACZL2ioh5Ta5pVmHcsjBbc89GxGkR8R1gF2AJSXG40ZI2l/RPSe9LWi7pP0pKkh8B9SeR1Rfmy2ipDJe0g6QnJS2QtFLSp0rKru/TAs/RDHDLwqwoIuIVSWOBnwLfJang2wV4hKRExvbAgcDtknYhKaFxD6vqHdVVKX2ZpMz2eiTF5/5Dkoj2Bu6T1LOusqpZc3LLwqx46spMtyOpNDqM5JoLS0kq3X5OkgQGRcSzJPWrAIiIs9LbsxExmaSW0AySZDE9XWxzYEBzPBGzhtyyMCue7dK/K4F9SC4jmu1iDls0tRFJZ5IUkMumyXXNSsUtC7MikNQLODl9OJGk1LRIymRvRnL1trqKrXUJpDpj/czP4vD0710kV0bLvKxn7isJmZWAk4XZmhso6dq0ku0LQAeS6y+cAbyXLtObpJXwFMk4Rqa3M+7fJelqSV0z1t0XGE0y7mHWopwszNZcL5KxhT1JxhcuIrlWxlzgApIWRluSw2rHsioJABAR75Jcr2MByUD3mSTX5jgDeIKkAukg4JclfyZmObjqrJmZ5eSWhZmZ5eRkYWZmOTlZmJlZTk4WZmaWk5OFmZnl5GRhZmY5OVmYmVlOThZmZpbT/wemJcNnP9T55AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "tmF2P6CK7qXk",
        "outputId": "72296108-4190-4395-e2c6-5d8af10c6716"
      },
      "source": [
        "#print mean and std sensitivity or specificity for each data\n",
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='2D CNN'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "print(pdnc_df_new)\n",
        "\n",
        "df_dummy = pdnc_df_new.groupby(['Data','Test']).mean().round(2)\n",
        "print(df_dummy)\n",
        "df_dummy['SE'] = pdnc_df_new.groupby(['Data','Test']).sem().round(3)\n",
        "print(df_dummy)\n",
        "\n",
        "#pdnc_df_new.groupby(['Data','Test']).mean().round(2)\n",
        "pdnc_df_new.groupby(['Data','Test']).sem().round(3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Data         Test     Value\n",
            "0     Full  Sensitivity  0.970230\n",
            "1     Full  Sensitivity  0.985115\n",
            "2     Full  Sensitivity  0.994587\n",
            "3     Full  Sensitivity  0.994587\n",
            "4     Full  Sensitivity  0.990528\n",
            "..     ...          ...       ...\n",
            "135  Anger  Specificity  0.992063\n",
            "136  Anger  Specificity  0.992063\n",
            "137  Anger  Specificity  1.000000\n",
            "138  Anger  Specificity  0.992063\n",
            "139  Anger  Specificity  1.000000\n",
            "\n",
            "[140 rows x 3 columns]\n",
            "                      Value\n",
            "Data     Test              \n",
            "Anger    Sensitivity   0.99\n",
            "         Specificity   0.98\n",
            "Disgust  Sensitivity   0.98\n",
            "         Specificity   0.98\n",
            "Fear     Sensitivity   0.99\n",
            "         Specificity   0.98\n",
            "Full     Sensitivity   0.99\n",
            "         Specificity   0.99\n",
            "Happy    Sensitivity   0.99\n",
            "         Specificity   0.97\n",
            "Sad      Sensitivity   0.99\n",
            "         Specificity   0.99\n",
            "Surprise Sensitivity   0.97\n",
            "         Specificity   0.98\n",
            "                      Value     SE\n",
            "Data     Test                     \n",
            "Anger    Sensitivity   0.99  0.003\n",
            "         Specificity   0.98  0.012\n",
            "Disgust  Sensitivity   0.98  0.010\n",
            "         Specificity   0.98  0.006\n",
            "Fear     Sensitivity   0.99  0.004\n",
            "         Specificity   0.98  0.006\n",
            "Full     Sensitivity   0.99  0.003\n",
            "         Specificity   0.99  0.004\n",
            "Happy    Sensitivity   0.99  0.003\n",
            "         Specificity   0.97  0.012\n",
            "Sad      Sensitivity   0.99  0.006\n",
            "         Specificity   0.99  0.004\n",
            "Surprise Sensitivity   0.97  0.016\n",
            "         Specificity   0.98  0.004\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>Value</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Data</th>\n",
              "      <th>Test</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Anger</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.012</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Disgust</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Fear</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.004</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Full</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.004</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Happy</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.012</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Sad</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.004</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Surprise</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <td>0.016</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Specificity</th>\n",
              "      <td>0.004</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      Value\n",
              "Data     Test              \n",
              "Anger    Sensitivity  0.003\n",
              "         Specificity  0.012\n",
              "Disgust  Sensitivity  0.010\n",
              "         Specificity  0.006\n",
              "Fear     Sensitivity  0.004\n",
              "         Specificity  0.006\n",
              "Full     Sensitivity  0.003\n",
              "         Specificity  0.004\n",
              "Happy    Sensitivity  0.003\n",
              "         Specificity  0.012\n",
              "Sad      Sensitivity  0.006\n",
              "         Specificity  0.004\n",
              "Surprise Sensitivity  0.016\n",
              "         Specificity  0.004"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XSRcXbw3oylD",
        "outputId": "4b5eb97e-7c82-4d89-800c-4a213a476169"
      },
      "source": [
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='2D CNN'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "temp = pdnc_df_new[pdnc_df_new['Data']=='Full']\n",
        "print(\"Sensitivity for full data\")\n",
        "print(temp[temp['Test']=='Sensitivity'].mean())\n",
        "print(temp[temp['Test']=='Sensitivity'].std())\n",
        "\n",
        "print(\"Specificity for full data\")\n",
        "print(temp[temp['Test']=='Specificity'].mean())\n",
        "print(temp[temp['Test']=='Specificity'].std())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sensitivity for full data\n",
            "Value    0.990526\n",
            "dtype: float64\n",
            "Value    0.007916\n",
            "dtype: float64\n",
            "Specificity for full data\n",
            "Value    0.987719\n",
            "dtype: float64\n",
            "Value    0.012873\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 829
        },
        "id": "eCmsLWnQHKeE",
        "outputId": "68719d19-7233-416a-8bda-13c78256be9f"
      },
      "source": [
        "pdnc_df_new = pd.melt(pdnc_df[pdnc_df['Model']=='2D CNN'], id_vars=['Data'], value_vars=['Sensitivity', 'Specificity'],\n",
        "        var_name='Test', value_name='Value')\n",
        "print(pdnc_df_new)\n",
        "\n",
        "x = \"Data\"\n",
        "y = \"Value\"\n",
        "hue = \"Test\"\n",
        "\n",
        "ax = sns.barplot(data=pdnc_df_new, x=x, y=y, hue = hue, palette = \"Blues\", capsize=.05, ci=68)\n",
        "ax.set_ylim(bottom=0.80)\n",
        "ax.legend(loc='lower right', bbox_to_anchor=(1,0))\n",
        "ax.set_xlabel('Data',fontsize = 13, weight='bold')\n",
        "ax.set_ylabel('Sensitivity/Specificity',fontsize = 13, weight='bold')\n",
        "add_stat_annotation(ax, data=pdnc_df_new, x=x, y=y, hue=hue,\n",
        "                    box_pairs=[((\"Full\", \"Sensitivity\"), (\"Full\", \"Specificity\")),\n",
        "                                ((\"Sad\", \"Sensitivity\"), (\"Sad\", \"Specificity\")),\n",
        "                                 ((\"Happy\", \"Sensitivity\"), (\"Happy\", \"Specificity\")),\n",
        "                               ((\"Fear\", \"Sensitivity\"), (\"Fear\", \"Specificity\")),\n",
        "                               ((\"Disgust\", \"Sensitivity\"), (\"Disgust\", \"Specificity\")),\n",
        "                               ((\"Surprise\", \"Sensitivity\"), (\"Surprise\", \"Specificity\")),\n",
        "                               ((\"Anger\", \"Sensitivity\"), (\"Anger\", \"Specificity\"))\n",
        "                                ],\n",
        "                    test='t-test_ind', text_format='star', loc='inside', verbose=2)\n",
        "\n",
        "\n",
        "props = dict(boxstyle='round', facecolor='none', edgecolor='black', alpha=0.5)\n",
        "\n",
        "# place a text box in upper left in axes coords\n",
        "ax.text(0.4, 0.98, \"2D CNN\", transform=ax.transAxes, fontsize=14,\n",
        "        verticalalignment='top', bbox=props)\n",
        "\n",
        "plt.savefig('sens_and_speci_pdnc_2d.eps', format='eps', bbox_inches='tight')\n",
        "#FILE.download('sensitivity_valence.eps')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Data         Test     Value\n",
            "0     Full  Sensitivity  0.970230\n",
            "1     Full  Sensitivity  0.985115\n",
            "2     Full  Sensitivity  0.994587\n",
            "3     Full  Sensitivity  0.994587\n",
            "4     Full  Sensitivity  0.990528\n",
            "..     ...          ...       ...\n",
            "135  Anger  Specificity  0.992063\n",
            "136  Anger  Specificity  0.992063\n",
            "137  Anger  Specificity  1.000000\n",
            "138  Anger  Specificity  0.992063\n",
            "139  Anger  Specificity  1.000000\n",
            "\n",
            "[140 rows x 3 columns]\n",
            "p-value annotation legend:\n",
            "ns: 5.00e-02 < p <= 1.00e+00\n",
            "*: 1.00e-02 < p <= 5.00e-02\n",
            "**: 1.00e-03 < p <= 1.00e-02\n",
            "***: 1.00e-04 < p <= 1.00e-03\n",
            "****: p <= 1.00e-04\n",
            "\n",
            "Sad_Sensitivity v.s. Sad_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-1.382e-01\n",
            "Full_Sensitivity v.s. Full_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=5.873e-01\n",
            "Happy_Sensitivity v.s. Happy_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=1.391e+00\n",
            "Fear_Sensitivity v.s. Fear_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=2.395e-01\n",
            "Disgust_Sensitivity v.s. Disgust_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-4.976e-01\n",
            "Surprise_Sensitivity v.s. Surprise_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=-6.843e-01\n",
            "Anger_Sensitivity v.s. Anger_Specificity: t-test independent samples with Bonferroni correction, P_val=1.000e+00 stat=9.009e-01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEJCAYAAABlmAtYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU1f3/8dd7l7IIojRFRQVjAwUhFBsCNkAioogNDUKimFgwNmJivmIX7BIriYBgLCgWAiqiiFhQioICKiqigvATpCiRuvv5/XHvLrPr7M4MOzM7M3yej8c89s6tn5mduZ8595x7jswM55xzriJ5VR2Ac865zOfJwjnnXEyeLJxzzsXkycI551xMniycc87FVK2qA0iFhg0bWtOmTas6DJcBNm3axC+//MKmTZsoKiqq6nBSKj8/n4KCAmrXrk21ajn51XYpNmfOnFVm1ijaspz8RDVt2pTZs2dXdRiuin322WdMnDiRLl26cNBBB1G7dm0kVXVYKVFUVMTq1av5+OOPmTdvHv3796d+/fpVHZbLMpK+KW9ZTiYL58yMV155hTPPPJN99tmnqsNJufz8fBo1asTxxx9PtWrVmD59OqeeempVh+VyiNdZuJy0fPlyqlevzt57713VoaRdmzZt+Oyzz/Abbl0yebJwOWndunU0aNAgZy87VaRu3boUFRWxefPmqg7F5RBPFi4nmRn5+flVHUaVyc/Pz/kKfZdeniycc87F5MnC7VBuv/122rdvT926dWnUqBE9e/Zk/vz5pdbp0qULkpBEjRo12GOPPejevTtPPPFEXPUAmzdv5s4776RNmzbstNNO1K9fnyOOOIJHH32UTZs2AdC/f38kcfPNN5fadtq0aUhi1apVACxZsgRJNGjQgHXr1v0qzksvvbQyb4dzcUtLspA0UtIPkuaXs1yShkv6UtLHkn4bsaxQ0tzwMSEd8brcNW3aNC6++GLee+89pk6dSrVq1TjhhBNYvXp1qfUGDBjA8uXLWbx4MRMmTODII4/koosu4rTTTqOwsLDc/W/evJlu3bpx6623MmDAAN59913mzJnDlVdeyahRo5gxY0bJugUFBdx5552sXLkyZty//PILQ4cO3f4X7lwlpavp7GjgAWBMOctPAg4IH4cDD4d/ATaYWetUB+h2DJMnTy71fOzYseyyyy68++679OzZs2T+TjvtROPGjQFo0qQJ7du354gjjqB79+6MGTOGAQMGRN3/fffdx1tvvcXMmTNp165dyfxmzZrRp08f1q9fXzLv2GOPZenSpdx8880MHz68wrgHDRrE/fffz6WXXspee+2V8Ot2rrLSUrIws+nA6gpW6QWMscD7wK6S9khHbG7H9vPPP1NUVES9evVirtutWzdatmzJ+PHjy13nP//5DyeccEKpRFEsLy+PunXrlno+dOhQHnnkEb766qsKj33GGWfQsmVLrr/++phxOpcKmVJnsRfwXcTzpeE8gAJJsyW9L6ncu4wkDQzXmx1Psd45gMsvv5zWrVtz5JFHxrV+ixYtWLx4cbnLv/jiC5o3bx738Xv06MHRRx/NddddF3PdO+64g8cff5wFCxbEvX/nkiVTkkVF9jWzdkBf4D5Jv4m2kpmNMLN2ZtauUaOoXZs4V8qVV17JO++8w/jx4+NuZmtmFd67sT03wg0bNoxnn32WOXPmVLhe586d6datG3/7298SPoZzlZUpyWIZEHmrbZNwHmZW/HcxMA1ok+7gXO654ooreOqpp5g6dSr77bdf3NstXLiwwvUPPPBAPv3004Ri6dChA6effjqDBw+Oue7QoUOZNGkSb7/9dkLHcK6yMiVZTAD6ha2ijgDWmdlySfUk1QSQ1BA4GlhYlYG67Hf55ZeXJIqDDz447u0mT57M/Pnz6dOnT7nr9O3bl9dffz1qR5ZFRUX89NNPUbe77bbbePvtt3n11VcrjKFly5b069cvrsTiXDKlq+nsU8AM4CBJSyX9UdKfJP0pXOVlYDHwJfAv4OJwfnNgtqR5wJvAUDPzZOG22yWXXMKoUaN48sknqVevHitWrGDFihWlWilB0FR1xYoVLF26lFmzZnHjjTfSu3dvevXqxXnnnVfu/v/yl7/QsWNHTjzxRIYPH87cuXP5+uuvef755+nYsSMffvhh1O32339/Bg4cyP333x/zNdx0003MnTuXDz74ILEX71wlpKXprJmdE2O5AZdEmf8e0DJVcbkdz0MPPQTA8ccfX2r+kCFDuOGGG0qejxo1ilGjRlG9enXq169P69ateeSRRzjvvPMqrLOoWbMmU6ZM4b777uOxxx7jr3/9KwUFBRx00EEMGDCAo446qtxtr7/+eh5//PGYr2Hvvfdm0KBB3HHHHTHXdS5ZlIs9U7Zr1858PIsd28KFC5k/fz5nnnlmVYdSJYYNG8agQYOoVatWVYfisoikOWGDol/JlDoL55IuF38IxWtHfu0uNTxZuJxUs2ZNNmzYUNVhVImtW7eyZcsWatSoUdWhuBziycLlpL322ovly5ezcePGqg4l7RYvXsxee+21Q3fR7pLPk4XLSQUFBbRo0YKXXnqJLVu2VHU4abN27VomT54ctbsR5yrDx+B2Oevkk0/mxRdf5O6776Zp06bUrl07Z0fOKyoqYvXq1axYsYLOnTvTqlWrqg7J5RhPFi5n5efnc/rpp7N+/XqWLFnCxo0bc7biNy8vj4MPPphmzZpRvXr1qg7H5SBPFi7n1alTh0MPPbSqw3Auq3mdhXPOuZg8WTjnnIvJk4VzzrmYPFkkyZIlS2jevDkXXnghhxxyCF27dmXDhg0MHz6cFi1a0KpVK84+++yqDrNEtsULHnO6ZFvM2RYvZGfMmFnOPdq2bWvp9vXXX1t+fr599NFHZmZ2xhln2NixY22PPfawjRs3mpnZmjVr0h5XebItXjOPOV2yLeZsi9csc2MGZls551UvWSRRs2bNaN26NQBt27ZlyZIltGrVinPPPZcnnniCatUyq/FZtsULHnO6ZFvM2RYvZF/MniySqGbNmiXT+fn5bN26lUmTJnHJJZfw4Ycf0r59e7Zu3VqFEZaWbfGCx5wu2RZztsUL2Rdz3MlC0mOSyu+M3/1KUVER3333HcceeyzDhg1j3bp1vxpkJ5NkW7zgMadLtsWcbfFC5secSDlnANBf0pfASGCsmX2fmrByQ2FhIeeddx7r1q3DzBg0aBC77rprVYdVrmyLFzzmdMm2mLMtXsj8mOMe/EjSSKAXUA8woAh4jSBxTDCzjOmtzQc/cs65xFU0+FHcJQsz+4OkasAJQB/gVKB7+Fgt6Z/AbWaWORfZUuT555/n1VdfTcq+unfvTu/evZOyr/JkW7zgMXvM0WVbvJCdMUeTUAV3mAjeBN4CFgEKHw2AIcC/kx1gJnr11VdZtGhRpfezaNGipH2IKpJt8YLH7DFHl23xQnbGHE3cJQtJrYELgL7ALgRJYiXwL+A54B6gN9A/6VFmoAMPPJARI0ZUah8DBw5MUjSxZVu84DGnS7bFnG3xQnbGXFYiFdwfEtRVCJgJPACMM7PNAJL+C3RJdoDOOeeqXiLJYhMwDnjAzGZFWf5v4PmkROWccy6jJJIs/g+YZmYlzYwk7QY0NLOFZvYz8HOyA8xEBx10UEbtJ13HSVe8yTyWx5yeY/lnOfXHSmfM0STSdLYQuN/MroyYdy8wyMwyamR4bzrrnHOJq6jpbCKtoaINXtxk+0JyzjmXTWImC0mFYanCgMuLn4fzehO0iHLOOZfD4qmzKC5RGNFLFw8mLxznnHOZKJ5kcSxBkphKcD9FcXLYCnxjZktTFJtzzrkMETNZmNlbAJIGAAsiW0M555zbMVSYLCT1AxaGCcKAFpJalF3PzMakKD7nnHMZIFbJYjRwLzA7nI7WztYATxbOOZfDYrWG+hZYEzEd7fFdrINIGinpB0nzy1kuScMlfSnpY0m/jVh2vqQvwsf5cbwm55xzSVZhycLMmkab3g6jCfqSKq8EchJwQPg4HHgYOFxSfYLebNsRlGDmSJpgZmvK2Y9zzrkUSGRY1fqS9pGk8LnC5/VjbWtm04HVFazSCxhjgfeBXSXtAXQDppjZ6jBBTCEYP8M551waJXIH98vAaxb2DxL+fQWYlIQ49qL05ayl4bzy5v+KpIGSZkuavXKl3yfonHPJlEiyaAG8XWbeu8AhyQtn+5nZCDNrZ2btGjVqVNXhOOdcTkkkWRQCjcvM25Pg5rzKWgbsHfG8STivvPnOOefSKJFksQjoIekmSd0l3URQMf1FEuKYAPQL60GOANaZ2XJgMtBVUj1J9YCu4TznnHNplMh4Fg8StGq6rsz8B2JtKOkpglH0GkpaStDCqTqAmT1CUB/SA/gS+AUYEC5bLelmoHiwpZvMrKKKcueccykQd7IwszGSDgYGATsRnNSHm9nYOLY9J8ZyAy4pZ9lIYGS8caba4MGDmT9/Plu3bqVatWoceuih3HHHHVUdVrmyLV7nXGZKpGSBmf1d0hCgEbDSzLakJqzMtWLFCjZt2gRAYWEhK1asSHgfgwcPZsWKFaxatSrlJ/FkxAvpTTqe4FyuyKXPcqy+oToBS81scTgdaf/wlovi+yh2CI0bN2bFihUUFhaSn59P48Zl6/xjW7FiBcuWbaunr8xJPJZkxAvJSzqZdiznUikbf6yVJ1bJ4k3gPuAqYBrl9w2VUAkl00386Otyl3U658989c11rF31/9i5XkM6nfPnctc/uU2zqPOLT9jJOInHcscdd9CvXz+WLVtG48aNt/sDlqykk2nHSpZkfZkz4aTgkicbf6yVJ9ZJXpQe8Cja4EfR5uWs8SPu4afVqwD4afUqxo+4h9MHXhljq9KKv/zJOImnS7KSTrFUJ+R0S9aXORNOCi4x6fgsZ8IPqFh9Q+VFm96R/bT6R4qKCgEoKirkp9U/lrvuSbe+WOG+Nq/+HwDLVv+v3HVfue7U7Yw0ccmIFyofczIScrol68ucCSeFRKSz/i0bJeuznOwfa9sjVp3FD8DDZjZE0lTgWTN7OD2hZaa69Ruwft0aigoLycvPp279BlUdUoW/bAD+t2lryd9Y62aCRBJypkjWlzkTTgqJSGf9WzbKxs9yeWJdhmoYPiC4T2JuSqPJApn+CzdVtnz0Imz8KXiy8Se2fPQi1dukptSTiQkZkpeUM+XSWTLqR9JZ/5aNMvWzvD1iJYsfgAGSivt/6iOpdZl1zMyOT35ouWvLRy9iG39O28k3GWzjz2BF4ZOi4HmKJCshZ2plcaZc7ktG/Ug21r+lUy79uIxVD/EMUAB0Imj11ISghFH24RJgG3+GDWvTcvKNds10e6hgZ8irBsqDvGrB8wxXfDIsLCxk06ZNKb08kqz3OZ0aN25Mfn4+wA5VIhg8eDA9evSga9eu9OjRg8GDB1d1SFkhVgX35ZKmEfQsexPwPt43U6WpYOegDfKm9UHCUF7KTr7JumaayaWe8qSzsjgbr01nW/1I2ZJiw4YNtytub3G2fWLeH2FmLwAvSNofmGxmT6U+rNyWzhNvLl0zTVQ6T4a5/j5nwqWzsif5yIr1RGRbi7NMkUjfUP1TGIdLkVy6ZprJkvU+p7MhQbaJdpIv70RfYXLb5SgKa3wFG9ZSWGNnPtnlqIxoup7pYjWdLQTuM7OrwulozMxy6g5u56pKOhsSZJuyJcUxY8ZUdUhJlQmlt4rEquCOvINbFTycc0mQjQ0Jsk200puLLVaJ4FiCca+Lp51zKZRtl5y8GfiOI1ZrqLeiTTvnskOq7+4vaQZeMiPzT74q2BlLQ0vEXBN3XYOkoUAL4DQzK5RUDXgO+MzMrk1VgM65zJXOZuDJksmlnkyWSMV0f+AtMysEMLOtkn4Bzgc8WbgqkemVgrnOT7w7jkR6kt0FWF9m3oZwvnPOuRyWSLL4FugV3pxH+PcU4LtUBOaccy5zJJIsXgXqA59JWgF8Fj5/ORWBOeecyxyJ1FncDPQAfgPsFs77Crgl2UE551IvEweZyrWxWZIlE+7sT6S7j1WSWgG9gX2Bb4DnzWxDqoJzzqVONnZ+uKPKhHtDEu2mYyNhHYWZ/Sf54Tjn0iXXOz/MJZlwb0gi91nsCrwGtAVM0vvAIuBmM7shNeE551Klqi85ufhlQhPlRCq4bwLaEfYHZWZfAbOBk1IRmHOV5X0AOZc8iSSLk4EZwEMR8xYSVHg7l3Ey4Tqvc7kikWSxG0Fy2FJmfo3kheNc8ngPrs4lTyIV3F8T9Dz7HoCko4Be4XznMk4mXOd1LlckUrIYA+wHnBs+f5ugq4+xyQ7KOedcZkkkWdwHjKP0oEfjgftTEJdzzrkMkshNeVuAsyVdDewDfGtmS2Ns5pxzLgckUrJA0p5A3/BxjqS9Eti2u6TPJX0p6VddmkvaV9Ibkj6WNE1Sk4hlhZLmho8JicTsnMsNZbsnGTX0OsaPuKeKo9pxJHJT3jEEnQbuFDF7iKQeZjY9xrb5wIPAiQTDtM6SNMHMFkasdhcwxswel3QccDvw+3DZBjNrHW+szrncU7Z7krWr/l8VR7RjSbTOojbwA/Bu+HcnIJ7U3gH40swWm9lm4GmCllSRWgBTw+k3oyx3zu3A6tZvQLXqNcjLy6da9Rrs2nB376IkjRJpOtuc4I7to81si6QawDvAIXFsuxelx71YChxeZp15BJ0U3g+cBuwsqYGZ/QgUSJoNbAWGmpnfiuvcDsa7J6laiSSLj4GFYUU3ZrZZ0gJgc5JiuRp4QFJ/YDqwDCgMl+1rZssk7QdMlfRJ2N1ICUkDgYEA++yzT5JCcs45B4kli3HAzZI+BhYAhwJ9gGsldSpeqZz6i2XA3hHPm4TzSpjZ9wQlCyTVAU43s7XhsmXh38WSpgFtCMbSiNx+BDACoF27dpbA63LOORdDIsniLsCAu8vMHx4xbeXscxZwgKRmBEnibIIWVSUkNQRWm1kR8DdgZDi/HvCLmW0K1zkauCOBuJ1zzlVSQk1nKX1DXrRH1P2Z2VbgUmAy8CkwzswWSLpJ0inhal2AzyUtAnYHbg3nNwdmS5pHUPE9tEwrKueccymWyE15iSaWstu/TJnxus3s+ojp54Dnomz3HtCyMsd2zjlXOZVKAM4553YMMZOFpP0ldZJUK3x+hKQ3Jc2WdKMkTzjOOZfj4rkMdT/QEWgkqQB4nqBOQQStkjazrX7BOedcDoqnVHAI8E545/WxQGPgF4JWUBsp06rJOedc7omnZNEQ+CacPib8O8rM/hI2ZfVuOZxzLsfFU7IoAuqF00cQ3Evxdvh8QyqCcs45l1niSRZfAL0lvQh0JkgWb4XLfkPQoaBzzrkcFk+yuB+oDpxCUKn9gpn9EF6COoqgc0HnnHM5LGadhZmNkfQjwVgUy4AHwkV7A/+kzI12zjnnck/MZCHpDGCSmU2KnG9mHwEfpSow55xzmSOey1BPAqskvSTpfEn1Ux2Uc865zBJPsmgMXBau+wiwIhwr++JExuB2zjmXvWImCzP70cweM7OeQCOCcbFXEoyR/Y2k9yUNltQkxbE655yrIgn162Rm683sGTM7myBxnEowENLVwB9SEJ9zzrkMEHcX5ZKOMbPim/EIu/+YCEwMOxP0kdOdcy5HJVKyeEvS55L+VrauwsyKzGxlkmNzzjmXIRJJFr8ABwC3AEskvSypj6TqqQnNOedcpkgkWTQE+gDPEvQ22x14Blguabik36QgPueccxkg7mRhZhvN7PmwcrsZMImg+4/6BONrL5R0dmrCdM45V5USag0lqZ2kR4AvgR7h7G8JxrZYBwxNbnjOOecyQSKtoT4CWhGUJgCmEvQTNcHMiiT9ANyc/BCdc85VtbiTBXAYsB4YCzxgZp+WWf4GQe+0zjnnckwiyeJyghHy1kdbaGYfAB8kJSrnnHMZJZE6i/uAGyNnSLpC0szkhuSccy7TJJIsxLb6imL7AW2TF45zzrlMFM94FiMjnnaNeJ4HnExws55zzrkcFk+dRX+CcbcNaA60KLP8jSTH5JxzLsPEkyweD/+eD3wKFNdRbAW+AR5NQVzOOecySDxjcA8AkNQMeNbMHkx5VM455zJK3E1nzaxLCuNwzjmXwSpsDSWpUNLdEdPRHlvTE6pzzrmqEqtkEdlctmyzWeecczuIWMniWGBpxLRzzrkdUIXJwszeinj6s5l9uL0HktQduB/IB/5tZkPLLN8XGEkwtvdq4DwzWxouOx/4R7jqLWb2OM4559ImkTu4Z0maJelCSbUTOYikfOBB4CSC+zTOkVT2fo27gDFm1gq4Cbg93LY+MAQ4HOgADJFUL5HjO+ecq5xEkkUhQdcejwDfS3pEUrxdfXQAvjSzxWa2GXga6FVmnRYE3Z4DvBmxvBswxcxWm9kaYArBKH3OOefSJJFksQcwCJgN7AwMBGZKmh3HtnsB30U8XxrOizQP6B1OnwbsLKlBnNsiaaCk2ZJmr1y5Mo6QnHPOxSuRYVV/NLMHzOxwgm4/phK0kGqTpFiuBjqHgyx1BpYRlGbijW+EmbUzs3aNGjVKUkjOOecgsfEskLQfcB7we4IeZyG+E/oyYO+I503CeSXM7HvCkoWkOsDpZrZW0jKgS5ltpyUSt3POucqJu2Qh6R3gC4LK5t8QjL19PbBvHJvPAg6Q1ExSDeBsYEKZ/TeUVBzP3whaRgFMJujttl5Ysd01nOeccy5NEilZHEXQeeB/gRHAa2Zm8WxoZlslXUpwks8HRprZAkk3AbPNbAJB6eF2SQZMBy4Jt10t6WaChANwk5mtTiBu55xzlZRIsvg7wbCq/297DmRmLwMvl5l3fcT0c8Bz5Ww7km0lDeecc2mWSEeCQ2Ov5ZxzLhd5R4LOOedi8o4EnXPOxeQdCTrnnIspkY4E9wUWmlnJHduSdgMapig255xzGSKR7j5GA33LzPsb8EnSonHOOZeREkkW0TRJShTOOecyWsxkUdzqCTDg8siWUATdc3ivfc45l+Piuc+iuBWUEb1F1IPJC8c551wmiidZHEuQJKYS3GFdnBy2At8Uj2bnnHMud8VMFsUtoiQNABZEtoZyzjm3Y6gwWUjqx7bmsga0iDIcKmY2JkXxOeecywCxShajgXsJRscbTZAwyjLAk4VzzuWwWMniW2BNxHRcXZI755zLLbHu4G4abdo559yOZbtuypNUU1JfSd2SHZBzzrnMk8iwqi9K+jEc+vRZYCzwsqSrUhadc865jJBIyaIV8B5QC+gBfAlsBC5IQVzOOecySCLJYg+CSu6DCG7S6ws8Deydgricc85lkESSxUaCbso7ErSK+hQoBIpSEJdzzrkMEvcY3MB84KTwMd/MfpHUFFiWgricc85lkERKFtcSjJr3A/BXSTUJLkG9korAnHPOZY64SxZm9i7BZahIzZMbjnPOuUyUyGUoJO0L/BbYOXK+9w3lnHO5Le5kIelCgu7J86Ms9mThnHM5LJGSxfXh+j8Ba/F+opxzboeRSLKoB0wBupuZJwrnnNuBJNIa6hXAPFE459yOJ5GSxQ9Ab0mvAR8AW4oXmNlNyQ7MOedc5kgkWfyZoJ7iBOD4Mss8WTjnXA5LJFlMxyu1nXNuh5TITXldUhiHc865DJbQ4EeSakvqJ+nv4QBI+0jaKc5tu0v6XNKXkq6NsnwfSW9K+kjSx5J6hPObStogaW74eCSRmJ1zzlVeIjfl7QW8y7YuyUcBXwH3AH+NsW0+wQ19JxL0LzVL0gQzWxix2j+AcWb2sKQWwMtA03DZV2bWOt5YnXPOJVciJYvbgX2ATQBmtpxgMKR4hlbtAHxpZovNbDPBOBi9yqxjQN1wehfg+wRic845l0KJJItjgdeARyPmLWLbr/+K7AV8F/F8aTgv0g3AeZKWEpQqLotY1iy8PPWWpGOiHUDSQEmzJc1euXJlHCE555yLVyLJYhdgeZl5tUleC6lzgNFm1oRg2Nax4Xjfy4F9zKwNcCXwpKS6ZTc2sxFm1s7M2jVq1ChJITnnnIPEksUXQFegGYCkvgSXkr6MY9tllB5+tQm/HjTpj8A4ADObARQADc1sk5n9GM6fQ1BPcmACcTvnnKukRJLFQwTjcPcMn48lOKHH0zppFnCApGaSagBnAxPKrPMt4c1+kpqH+14pqVFYQY6k/YADgMUJxO2cc66SErkpbyTBifpSYCdgA/CgmT0Wa0Mz2yrpUmAyQRfnI81sgaSbgNlmNgG4CviXpCsILm31NzOT1Am4SdIWgvG+/2RmqxOI2zmXhfKsiPp5G6muIpTgttcev3fsleLw6aefxlxnz7ytSTlWOmMuKCigSZMmVK9ePe79JnJTngHXSroe6EPQVfmMBLZ/maDiOnLe9RHTC4Gjo2w3Hhgf73Gcc7mhft5G9mhUn7q71kNKLF38sO6XpMRw4B71Yq6z9pdNSTlWumI2M3788UeWLl1Ks2bN4t5vzMtQ4Q14b0gqjuARgktQLwGfSWob99Gccy5O1VW0XYnCVUwSDRo0YOPGjQltF0+dRW+gkZmtkXQA0B9Q+GhAMCiSc84llcATRYpsz/saT7LYF/g4nD4x/PsBUB+YC7RP+KjOOeeySjx1FnWBdeF0B4LK53FmtlbSDIImr845l1LvfJ7cTh0ObLxrXOvdeuutPPnkk+Tn55OXl8ejjz7K4YcfnpQYuh7XhdemTuPbb5YwZep0evbuA8An8z7ipWef4R+3DC1326fGjKJWrVqcesbZPP/Mkxzd+Vh2b7xHUuKKJp5k8QNwrKTWbCtZFFds78a2ROKcczllxowZTJw4kQ8//JCaNWuyatUqNm/enLT9vzZ1GgDffvMNE198riRZtDysDS0Pa1Phtuf0G1Ay/cK4pzjg4OYpTRbxXIaaChwMzAEaEySPD8JlbfF7HpxzOWr58uU0bNiQmjVrAtCwYUP23HNP5syZQ+fOnWnbti2nn3IyK5YHnVuc3P1EhvzjOo7v1JF2hx3Ke+++A8CnCxdyfKeOHHNEB47u0I6vvgzuZW6yWwMAbrz+/5j9wQx6ndCJ0SMe4oP33uGifmdTVFTEcR0O46d1236Tdz26HatW/sA/7xrKYw//k1cnvsT8eXO5+pKL6HVCJyZNmsSpp55asv6UKV6sYCgAABK4SURBVFM47bTTKv1exJMs/k6QKAT8DFwQ3v/QhaBfqOmVjsI55zJQ165d+e677zjwwAO5+OKLeeutt9iyZQuXXXYZzz33HHPmzOHcfudzy41DSrYp3LqVN6a/w23D7uKO224FYNRj/+Kiiy/h7fdn8uY777HnXqW7xhty0820O/xIXnp9Ov0HXlwyPy8vj+O6ncSUVyYCMO/D2ey51940bLRbyTrdT+7FoYe15q4HH+Wl16fTo0cPPvvsM4r7yBs1ahR/+MMfKv1exEwWZrbMzNoTVGg3MLNJ4aJ3gJ0JkolzzuWcOnXqMGfOHEaMGEGjRo0466yzePTRR5k/fz4nnngirVu35q5hQ/l+2bbei07uFXSo3bpNG7799hsA2nc4nHvvuoP77r6L7779llq1asUdQ49TTuPlCS8AMOml5+nR69QK15fE73//e5544gnWrl3LjBkzOOmkkxJ96b+SyE15a8s83wok59ZF55zLUPn5+XTp0oUuXbrQsmVLHnzwQQ455BBmzAiqbsvelFd8ySo/P5+tW4NT5BlnnU279u157dVXObN3L+4d/gCduhwb1/HbtOvAt0u+ZvWPq3j91Zf58+VXx9xmwIAB9OzZk4KCAs444wyqVUuks47oEhopzznndiSff/45X3zxRcnzuXPn0rx5c1auXFmSLLZs2cKnCxeWtwsAlny9mKbN9uOiiy+hx+96smD+/FLL6+y8M/9bvz7qtpI4ofvvuP2Gf/CbAw6kXv36v1qndu06pbbfc8892XPPPbnlllsYMGDAr9bfHpVPN845lwYdD9oz7nWT1XXG+vXrueyyy1i7di3VqlVj//33Z8SIEQwcOJBBgwaxbt06Nm/Zwp8uuZTmLVqUu58Xxo9n3NNPUq1adXbffXeuvGZwqeWHHNqSvPx8TjnhGHqfeQ7ND21VanmPXqfR56TjGXrfg1H3f9pZ5zDkr1dRUFDAh7NnUqtWLc4991xWrlxJ8+bNK/9G4MnCOefK1bZtW957771fzW/YsCHTpwdteyIvQ018dUrJdIOGDfn400UAXHH1NVxx9TW/2s/SH34EoHr16ox59qVSyw4/qmPJdMvD2vD596X7T73s6mtLprv97hS6/e4UgJL6kHfeeYcLL7wwjlcZH08WzjmXY9q2bUvt2rW5++67k7ZPTxbOOZdj5syZk/R9egW3c865mDxZOOeci8mThXPOuZg8WTjnnIvJK7idc1nhnHtfSer+7v9D57jWS2UX5Wec1ot/j3qcXXbdlTH/fpSnxoykRcvD6HHKaXy16HMGXvaXcrc9u2c3nv7vZJZ+9y0fzZpZ0mNtqniycM65cqS6i/JnX9h2b8WTj49k9DPP03jPoJPB47tV3J/T0/+dDMCy774t1b15qvhlKOecK0d5XZQ3bdqUwYMH07JlS47v1JHFX30FwKqVK+nX92yOO+ZojjvmaN6fEdzQt379ei656EKOat+Wozu0Y8KLQceArZofyI+rVnHFoEtZ+u0SLjzvTEaPeIjnn3mSm/4+ONznD1zyh99zygnHcMoJx/DhrGCEiDb77w3A3bfdWKp7806dOjF37tyS19CxY0fmzZtX6ffCk4VzzpUjWhflxXbZZRc++eQTLrzoT/xtcNC537XXXMWfL72MqW+/y+NPPsXll/wZgDuH3k7durvw3qw5vDtzNp06dyl1nHuHP8Buuzfm8WcnlOqiHOCWf1xL+yOPYsLrb/PC5GkccFDp7juu+vuQUt2b//GPf2T06NEALFq0iI0bN3LYYYdV+r3wZOGcc+WI1kV58Yn4nHPOAeD0M89i1szg1/5bb77J4Cuv4JgjOtD3jD78/NPPrF+/nrfenMoFF/2pZL+71qsXdwzvv/s2ffsF41Hk5+ezc926Fa5/xhlnMHHiRLZs2cLIkSPp379/Aq+4fF5n4ZxzFSjbRfnjjz8OBL3BFiueLrIipkybTkFBQZXECrDTTjtx4okn8tJLLzFu3Lik3c3tJQvnnCtHtC7K9913XwCeeeYZAJ5/7lnadwhaRx173PGMePihkvU/CesKuhx3PP9+9JGS+WvXrIk7hiM7duLJMSMBKCws5Oeffiq1vHadOr/q3vyCCy5g0KBBtG/fnnoJlGIq4iUL51xWeOqK+Ed7S3UX5RMnTmTNmjW0atWKatVr8O/RYwAYdtc9XHPlXzi6QzsKC7dy5NEduXf4A1z912u55orLObLdb8nPz+evf7+OnjFGvCt23c2383/XXMH4p54gLy+fG4beRZt2HUqWH9T8kFLdm9825Dratm1L3bp1kzaWBXiycM65cpXXRTnANddcw7Bhw0p1Ud6gYUNGjnniV+vWqVOHh//12K/mF3dhDjB15rYWS73P6kvvs/oC0LDRbjw8+j+/2vajL78Dondv/v3331NUVETXrl0renkJ8ctQzjmXQ8aMGcPhhx/OrbfeSl5e8k7xXrJwzrkELVmypKpDKFe/fv3o169f0vfrJQvnXEYywMyqOoyctD3vqycL51xG2mJ5/LR2jSeMJDMzfvzxx4Sb9/plKOdcRlpdVAArV7Nq1SoUe/VSft6QnP6bCtfuFHOdDZu3JuVY6Yy5oKCAJk2aJLRfTxbOuYxUpDxW2U7B9agEPfhG5ftCAnjlutjNWyd+9HVSjpXOmLdH2i5DSeou6XNJX0q6NsryfSS9KekjSR9L6hGx7G/hdp9L6paumJ1zzgXSUrKQlA88CJwILAVmSZpgZgsjVvsHMM7MHpbUAngZaBpOnw0cAuwJvC7pQDMrTEfszjnn0ley6AB8aWaLzWwz8DTQq8w6BhT3kLUL8H043Qt42sw2mdnXwJfh/pxzzqWJ0tHSQFIfoLuZXRA+/z1wuJldGrHOHsBrQD2gNnCCmc2R9ADwvpk9Ea73GPCKmT1X5hgDgYHh04OAz1P8suLREFhV1UEkINviBY85XbIt5myLFzIj5n3NrFG0BZlUwX0OMNrM7pZ0JDBW0qHxbmxmI4ARKYtuO0iabWbtqjqOeGVbvOAxp0u2xZxt8ULmx5yuZLEM2DvieZNwXqQ/At0BzGyGpAKCTBvPts4551IoXXUWs4ADJDWTVIOgwnpCmXW+BY4HkNQcKABWhuudLammpGbAAcDMNMXtnHOONJUszGyrpEuByUA+MNLMFki6CZhtZhOAq4B/SbqCoLK7vwUVKgskjQMWAluBS7KoJVRGXRaLQ7bFCx5zumRbzNkWL2R4zGmp4HbOOZfdvG8o55xzMXmycM45F5Mni0qQVChpbsSjaQXr9g/vGUHSDZKuTlecZeK4TtKCsEuVuZIOj3O7ppLmpyim9WWel7xX2SSRz0MVxLRA0jxJV0nKC5e1kzQ8DTE0ldQ3jvW267O5HfG8LGnXVOw74hinSjJJB6fyOOmUSfdZZKMNZta6qoOIV3j/ysnAb81sk6SGQI0qDiuXJPXzIKmamVW2S9OSmCTtBjxJ0FPCEDObDcyu5P7j0RToGx47qsp8NuN9nySJoJ62R6x1k+Ac4J3w75BUHSRJn5G4eMkiySQtCT/oxb/cplVxSJH2AFaZ2SYAM1tlZt9Lul7SLEnzJY0Iv1RIahv+Gp0HXFIVAUvqKemDsIPJ1yXtHs6/QdJYSTMkfSHpwnB+F0nTJU0KO558RFKepD9Iui9ivxdKujcN8beV9JakOZImhz0VFB9/Vvj+jpe0Uzh/dBjzB8AdyYzFzH4g6OXgUgW6SJoYHrdzRInoI0k7h+/bQ5I+kzQl/EXeJ1w/6uc82n6AocAx4bwrygmvvM9meccp/v+/S3ADb39JL0maFn4ehoTrNQ0/B2OA+cDexfuUVDv8nMwLP/tnVfQ/i5ekOkBHgnvHzg7ndQljey58P/8T8T3rEc6bI2l4xP+ktqSRkmaG72WvcH5/SRMkTQXeSCS2SjEzf2znAygE5oaPF8J5S4CG4XQ7YFo43R94IJy+Abi6CuKtE8a6CHgI6BzOrx+xzligZzj9MdApnL4TmJ+G93EuwT03xe9VPba12rsAuDviPZwH1CK4efM7go4muwAbgf0ImmlPAfqEr/0roHq4/XtAy1R+HoDq4XEahcvPImg2DtAgYrtbgMvC6dHARCA/STGtjzJvLbB7+F5NDOf9Fzg64nNSLXzfXib4UdkYWAP0ifE5j7afkuNsx2ezvOPcAMwBakV8v5YDDcLPxPxw/aZAEXBExLGWhJ+Z04F/RczfpaL/WQLv+bnAYxGfs7bhe7CO4KbiPGAGQUIpIPjsNgvXfyrif3IbcF44vWv43tQOX+tSIr636Xj4ZajKyarLUGa2XlJb4BjgWOAZBd3F/yxpMLATUJ/g3pa3gV3NbHq4+VjgpBSFVup9lNSf4IsOwZfrmfDXXQ0gcvCAl8xsA7BB0psEHUyuBWaa2eJwX08BHc3sufCX2MmSPiVIGp+k+HUcChwKTAl/ROYTnNAADpV0C8FJoA7BPUjFnrX030v0LnCPpP8Az5vZUkkdw1iKgBXhe7w9+4m5UQWfzYpMCP//xaaY2Y8Akp4nOBm/CHxjZu9H2f4T4G5JwwhO0G/H+J/F6xzg/nD66fD5RILP5dIwvrkEiWw9sNiCTlIhSBbFfdx1BU7RtvrNAmCfiNe6OsG4KsWTRfJtZdvlvcTGLUyD8CQ0DZgm6RPgIqAV0M7MvpN0A5kV9z+Be8xsgqQuBL8oi5W9SchizP838HfgM2BUcsOMSsACMzsyyrLRwKlmNi9Mjl0ilv0vZQFJ+xGUgH4AmhfPN7OhkiYBPYB3FXvcmKif8+3YT4kon83zyztOqOz7VN7/Per7aWaLJP02jPUWSW8QlAjL+5/FJKk+cBzQUpIRJBsDJgGbIlYtJPb5V8DpZlaqU1QFFf8p+4yUx+sskm8JQbETgmJuxpB0kKQDIma1ZlvvvKvCa619AMxsLbA2/HUJQdG6KuzCtr7Azi+zrJekAkkNCE62s8L5HRR0LZNHcBnhHQAz+4Cgn7G+BL/gUu1zoJGCylskVZd0SLhsZ2C5pOqk6b2V1Ah4hOASn5VZ9hsz+8TMhhG8jwcTlBJOD+suii9bFVtClM95Ofv5meD1VhRbtM/mN+UdpxwnSqovqRZwahh/RcfcE/jFgh6t7wR+S8X/s3j0Acaa2b5m1tTM9iYoDR9TzvqfA/tpW8u5syKWTQYui6jbaJNAHEnnJYvkuxF4TNLNBL+SMkkd4J8Kmg1uJRgbZCDBpZv5wAq2nXABBgAjw19Ir6U51mI3AM9KWgNMBZpFLPsYeJPg+vPNFlSIHkjwGh4A9g+XvxCxzTigtZmtSXXgZrZZQYXwcEm7EHzf7gMWAP8HfEDQ/9kHxDiZVkKt8JJHdYL/+Vjgnijr/UXSsQTX9xcArwBbCPprW0hwXf1DguvuUP7nPNp+ioBCBQ0lRptZtIYF5X02m5dznGhmAuMJLl0+YWazVXHz5ZbAnZKKwtf65xj/s3icAwwrM2888GeCOrNSzGyDpIuBVyX9j9Lfv5vDY38c/vD5mqDFWJXw7j5cVgovl603s7vKzO9C0Hgg6pcqbGlyr5mlrxVJFpNUJ6xPaEBwMj7azFZUdVxlFddzWcQYOdki4j0WwYiiX5STUKuUX4ZyOwRJu0paRFAJ7YkifhPDksnbBKW3jEsUOeDC8D1eQHDZ9dEqjicqL1k455yLyUsWzjnnYvJk4ZxzLiZPFs4552LyZOGccy4mTxbOJSjsEM7Cx1ZJayR9KOnGsG1+vPvpUryfVMbrXDL4TXnObb9P2Xaj4IlAG6CfpI5mtqzCLZ3LMl6ycG77zTSzS83sd8BhwE8EncMNl1RP0huSlkvaJOl/Crok7w0lN5GVdMwXUVLpL+lASW9LWilpi6SfFXS73qkKXqNzgJcsnEsKM/tU0gjgaqAnQQ++jYDXCbrI2B/oBjwl6TCCLjTGs62/o+JeShcSdLNdg6Dzuf8RJKJjgBclHVDcs6pz6eQlC+eSp7ib6eoEPY32IRhzYT1BT7cbCJJAFzObSdB/FQBm9pfwMdPMZhD0JTSfIFnMDVerB7RPxwtxriwvWTiXPPuFf7cAnQiGEY02mMPuFe1E0uUEHchFU+G2zqWKlyycSwJJzYELw6cTCbqaFkE32bsSjN5W3GNrcQLZGrF95Hexf/h3HMHIaJHDesYeSci5FPBk4dz26yDpgbAn23lAXYLxFwYB34frtCQoJbxLUI8R6ZuI6XGS7pPUOGLbzsBwgnoP56qUJwvntl9zgrqFownqF24kGCtjKTCEoIRRjaBZ7Qi2JQEAzOw7gvE6VhJUdF9OMDbHIGA6QQ+kXYB/pPyVOBeD9zrrnHMuJi9ZOOeci8mThXPOuZg8WTjnnIvJk4VzzrmYPFk455yLyZOFc865mDxZOOeci8mThXPOuZj+P1eoHTplLrMDAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}