evaluation_geometric_icml2022.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geometric distribution example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load generated data from file:\n",
    "import pickle\n",
    "\n",
    "from evaluation_icml2022 import *\n",
    "\n",
    "experiments = {}\n",
    "num_chains = 10\n",
    "configs = [\n",
    "    (L, alpha, K) for L in [5, 2] for alpha in [1.0, 0.5, 0.1] for K in [0]\n",
    "]\n",
    "for L, alpha, K in configs:\n",
    "    key = toconfigstr(L, alpha, K)\n",
    "    experiments[key] = []\n",
    "    for i in range(num_chains):\n",
    "        filename = f\"lookahead_samples/geometric_{i}__count1000_eps0.1_L{L}_alpha{alpha}_K{K}.pickle\"\n",
    "        with open(filename, \"rb\") as f:\n",
    "            experiments[key].append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L=5\n",
      "npdhmc: 355.91s    0.0356s per sample (after thinning)\n",
      "L=5, α=0.5\n",
      "npdhmc-persistent: 362.65s    0.0363s per sample (after thinning)\n",
      "L=5, α=0.1\n",
      "npdhmc-persistent: 366.57s    0.0367s per sample (after thinning)\n",
      "L=2\n",
      "npdhmc: 152.71s    0.0153s per sample (after thinning)\n",
      "L=2, α=0.5\n",
      "npdhmc-persistent: 150.25s    0.0150s per sample (after thinning)\n",
      "L=2, α=0.1\n",
      "npdhmc-persistent: 149.60s    0.0150s per sample (after thinning)\n"
     ]
    }
   ],
   "source": [
    "values = {}\n",
    "chains = {}\n",
    "for config, runs in experiments.items():\n",
    "    print(f\"{config}\")\n",
    "    thinned_runs = thin_runs(runs)\n",
    "    chains.update(collect_chains(thinned_runs, config=config))\n",
    "    values.update(collect_values(thinned_runs, config=config))\n",
    "    print_running_time(runs, thinned_runs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total variation distance from the ground truth (mean over all chains):\n",
      "NP-DHMC (L=5):  0.0524 +- 0.0069 (std)\n",
      "NP-DHMC pers. (L=5, α=0.5):  0.0464 +- 0.0074 (std)\n",
      "NP-DHMC pers. (L=5, α=0.1):  0.0461 +- 0.0083 (std)\n",
      "NP-DHMC (L=2):  0.0768 +- 0.0181 (std)\n",
      "NP-DHMC pers. (L=2, α=0.5):  0.0570 +- 0.0115 (std)\n",
      "NP-DHMC pers. (L=2, α=0.1):  0.0534 +- 0.0058 (std)\n"
     ]
    }
   ],
   "source": [
    "def total_variational_distance(samples, p = 0.2):\n",
    "    samples = sorted(samples)\n",
    "    N = len(samples)\n",
    "    mx = int(samples[-1])\n",
    "    freq = [0 for _ in range(mx + 1)]\n",
    "    for sample in samples:\n",
    "        freq[int(sample)] += 1\n",
    "    dist = 0\n",
    "    for x in range(1,mx + 1):\n",
    "        dist += abs(freq[x] / N - (1 - p)**(x-1)*p)\n",
    "    dist += (1 - p)**mx # unsampled tail\n",
    "    return dist / 2\n",
    "import torch\n",
    "print(\"Total variation distance from the ground truth (mean over all chains):\")\n",
    "for method in chains.keys():\n",
    "    tvds = torch.tensor([total_variational_distance(chain) for chain in chains[method]])\n",
    "    std, mean = torch.std_mean(tvds)\n",
    "    print(f\"{legend_str(method)}:  {mean:.4f} +- {std:.4f} (std)\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}