notebooks/thesis-experiments3-plots.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b31d7309",
   "metadata": {},
   "source": [
    "Produce plots for the third experimental section (models trained on real data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b64a768c",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import seaborn as sns\n",
    "import glob\n",
    "import wandb\n",
    "\n",
    "sns.set()\n",
    "\n",
    "# matplotlib.use(\"pgf\")\n",
    "matplotlib.rcParams.update(\n",
    "    {\n",
    "        \"pgf.texsystem\": \"pdflatex\",\n",
    "        \"font.family\": \"serif\",\n",
    "        \"font.serif\": \"Palatino\",\n",
    "        \"text.usetex\": True,\n",
    "        \"pgf.rcfonts\": False,\n",
    "    }\n",
    ")\n",
    "\n",
    "plot_dir = \"/Users/vaclav/prog/thesis/figures/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "160e3c9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\n",
    "    # default params\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0725-ddspae-cnn-1\", \"display_name\": \"DDSP-CNN\"},\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0804-ddspae-cnn-3\", \"display_name\": \"DDSP-CNN\"},\n",
    "    # ch=16\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0809-ddspae-cnn-5\", \"display_name\": \"DDSP-CNN-Tiny\"},\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0809-ddspae-cnn-4\", \"display_name\": \"DDSP-CNN-Tiny\"},\n",
    "    #     # inverted bottleneck, ch=32\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0726-ddspae-cnn\", \"display_name\": \"DDSP-CNN-IB\"},\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0809-ddspae-cnn\", \"display_name\": \"DDSP-CNN-IB\"},\n",
    "]\n",
    "\n",
    "models2 = []\n",
    "for m in models:\n",
    "    m = m.copy()\n",
    "    m[\"quantization\"] = False\n",
    "    models2.append(m)\n",
    "    m = m.copy()\n",
    "    m[\"quantization\"] = True\n",
    "    models2.append(m)\n",
    "\n",
    "models = models2\n",
    "\n",
    "eval_dir = \"/Users/vaclav/prog/thesis/data/eval_data/\"\n",
    "\n",
    "api = wandb.Api()\n",
    "\n",
    "for model in models:\n",
    "    model_name = model[\"name\"]\n",
    "\n",
    "    suffix = \"rtq\" if model[\"quantization\"] else \"rt\"\n",
    "    model_eval_name = f\"{model_name}-{suffix}\"\n",
    "\n",
    "    artifact = api.artifact(\n",
    "        f\"neural-audio-synthesis-thesis/nas-evaluation/eval-{model_eval_name}:latest\"\n",
    "    )\n",
    "    csv_dir = artifact.checkout()\n",
    "\n",
    "    run = artifact.logged_by()\n",
    "\n",
    "    model[\"csv_path\"] = os.path.join(csv_dir, f\"eval-{model_eval_name}.csv\")\n",
    "\n",
    "    model[\"loss\"] = run.summary[\"losses/total_loss\"]\n",
    "    \n",
    "    df = pd.read_csv(model[\"csv_path\"], index_col=0)\n",
    "    model[\"decoder_inference_time\"] = df[\"Autoencoder.decoder\"].mean()\n",
    "    model[\"decoder_inference_times\"] = df[\"Autoencoder.decoder\"]\n",
    "    \n",
    "    std = model[\"decoder_inference_times\"].std()\n",
    "    print(f\"standard deviation: {std:.05f}, relative {std/model['decoder_inference_time']:.3f}\")\n",
    "\n",
    "    print(artifact.created_at, model_eval_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95e977f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_times = []\n",
    "\n",
    "for model in models:\n",
    "    cur = pd.DataFrame(\n",
    "        model[\"decoder_inference_times\"].copy() * 1000\n",
    "    )\n",
    "    cur.columns = [\"inference_time_ms\"]\n",
    "    cur[\"display_name\"] = model[\"display_name\"]\n",
    "    cur[\"quantization\"] = model[\"quantization\"]\n",
    "    cur[\"dataset\"] = model[\"dataset\"]\n",
    "    df_times.append(cur)\n",
    "\n",
    "df_times = pd.concat(df_times)\n",
    "df_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83f5fb3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(\n",
    "    [\n",
    "        [\n",
    "            m[\"display_name\"],\n",
    "            m[\"quantization\"],\n",
    "            m[\"dataset\"],\n",
    "            m[\"loss\"],\n",
    "            m[\"decoder_inference_time\"] * 1000,\n",
    "        ]\n",
    "        for m in models\n",
    "    ],\n",
    "    columns=[\"Model\", \"Quantization\", \"Dataset\", \"Loss\", \"Decoder inference time (ms)\"],\n",
    ")\n",
    "df_agg = df.pivot(index=[\"Model\", \"Dataset\"], columns=\"Quantization\")\n",
    "# df.columns = df.columns.get_level_values(1)\n",
    "# df.columns.name = None\n",
    "# df = df.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56f63ac0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_agg[\"Loss\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9126b13",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_latex_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73a7dacc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_latex_table = df_agg[\"Loss\"].copy()\n",
    "df_latex_table.columns = [\"Unquantized\", \"Quantized\"]\n",
    "# df_latex_table.columns.name = \"Quantization\"\n",
    "# df_latex_table.index.name = \"Runtime\"\n",
    "df_latex_table.columns.name = None\n",
    "df_latex_table.index.name = None\n",
    "\n",
    "styler = (\n",
    "    df_latex_table.style\n",
    "        .format_index(escape=\"latex\")\n",
    "        .format(precision=2, na_rep=\"---\")\n",
    ")\n",
    "\n",
    "print(styler.to_latex(hrules=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfe8d66f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_agg[\"Decoder inference time (ms)\"].mean(axis=\"columns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e19394f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee754473",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdf_metadata = None\n",
    "if pdf_metadata is None:\n",
    "    pdf_metadata = {}\n",
    "\n",
    "ci = 95\n",
    "g = sns.catplot(\n",
    "    data=df_times,\n",
    "    kind=\"bar\",\n",
    "    y=\"display_name\",\n",
    "    x=\"inference_time_ms\",\n",
    "    hue=\"quantization\",\n",
    "    ci=ci,\n",
    "    palette=\"dark\",\n",
    "    alpha=0.6,\n",
    "    height=3,\n",
    "    aspect=3 / 2,\n",
    ")\n",
    "plt.xlabel(\"Inference time (ms)\")\n",
    "plt.ylabel(\"Model\")\n",
    "\n",
    "# plt.title()\n",
    "\n",
    "pdf_metadata = {\"models\": [x[\"name\"] for x in models]}\n",
    "pdf_metadata[\"ci\"] = ci\n",
    "\n",
    "g.fig.savefig(\n",
    "    os.path.join(plot_dir, f\"inference-time-optimized.pdf\"),\n",
    "    bbox_inches=\"tight\",\n",
    "    # Metadata doc: https://matplotlib.org/stable/api/backend_pdf_api.html#matplotlib.backends.backend_pdf.PdfFile\n",
    "    metadata={\"Title\": \" \".join([f\"{k}={v}\" for k, v in pdf_metadata.items()])},\n",
    "    backend=\"pgf\",\n",
    ")\n",
    "\n",
    "# print(make_table(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "606f1def",
   "metadata": {},
   "outputs": [],
   "source": [
    "models[2][\"decoder_inference_times\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b79be8f0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}