Evaluation_examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "09d37d22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "82bc296d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rylee/python/pna/env/lib/python3.7/site-packages/dgl/base.py:45: DGLWarning: Recommend creating graphs by `dgl.graph(data)` instead of `dgl.DGLGraph(data)`.\n",
      "  return warnings.warn(message, category=category, stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "# Generate graphs for demonstration purposes\n",
    "import utils.graph_generators as gen\n",
    "import torch\n",
    "import dgl\n",
    "\n",
    "grids = gen.make_grid_graphs()\n",
    "lobsters = gen.make_lobster_graphs()\n",
    "\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "grids = [dgl.DGLGraph(g).to(device) for g in grids] # Convert graphs to DGL from NetworkX\n",
    "lobsters = [dgl.DGLGraph(g).to(device) for g in lobsters] # Convert graphs to DGL from NetworkX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4e01c2ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "orthogonal\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'activations_time': 0.02939605712890625,\n",
       " 'fid': 5024.989816615811,\n",
       " 'fid_time': 0.04246258735656738,\n",
       " 'kid': 575293.3,\n",
       " 'kid_time': 0.03537917137145996,\n",
       " 'precision': 0.0,\n",
       " 'recall': 0.0,\n",
       " 'f1_pr': 1e-05,\n",
       " 'precision_time': 0.3994009494781494,\n",
       " 'recall_time': 0.3994009494781494,\n",
       " 'f1_pr_time': 0.3994009494781494,\n",
       " 'density': 0.0,\n",
       " 'coverage': 0.0,\n",
       " 'f1_dc': 1e-05,\n",
       " 'density_time': 0.23938417434692383,\n",
       " 'coverage_time': 0.23938417434692383,\n",
       " 'f1_dc_time': 0.23938417434692383,\n",
       " 'mmd_rbf': 1.3218982964754105,\n",
       " 'mmd_rbf_time': 0.3472146987915039,\n",
       " 'mmd_linear': 4667.4316,\n",
       " 'mmd_linear_time': 0.029571056365966797}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute all GNN-based metrics at once\n",
    "from evaluation.evaluator import Evaluator\n",
    "evaluator = Evaluator(device=device)\n",
    "evaluator.evaluate_all(generated_dataset=grids, reference_dataset=lobsters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4018835c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "orthogonal\n",
      "result: {'mmd_rbf': 1.3096226453781128}, time to compute: 0.321s\n"
     ]
    }
   ],
   "source": [
    "# Alternatively, compute a single GNN-based metric. See evaluation/gin_evaluation.py for other metrics.\n",
    "from evaluation.gin_evaluation import load_feature_extractor, MMDEvaluation\n",
    "\n",
    "# Can tweak GIN hyperparameters, however defaults are set to our recommendations\n",
    "gin = load_feature_extractor(device=device)\n",
    "# Can tweak hyperparameters of MMD RBF, however defaults are set to our recommendations\n",
    "mmd_eval = MMDEvaluation(gin)\n",
    "result, time = mmd_eval.evaluate(generated_dataset=grids, reference_dataset=lobsters)\n",
    "print('result: {}, time to compute: {:.3f}s'.format(result, time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f9541366",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "orthogonal\n",
      "result: {'mmd_rbf': 0.9764921963214874}, time to compute: 0.323s\n"
     ]
    }
   ],
   "source": [
    "# Evaluation incorporating custom node/edge features.\n",
    "# If no node features are provided, the default is degree features\n",
    "# expressed as an integer.\n",
    "\n",
    "node_feat_loc = 'feat'\n",
    "edge_feat_loc = 'feat'\n",
    "node_feat_dim = 10\n",
    "edge_feat_dim = 5\n",
    "\n",
    "# Create random node/edge features\n",
    "for g in grids:\n",
    "    num_nodes = g.number_of_nodes()\n",
    "    g.ndata[node_feat_loc] = torch.randn(num_nodes, node_feat_dim).to(g.device)\n",
    "    num_edges = g.number_of_edges()\n",
    "    g.edata[edge_feat_loc] = torch.randn(num_edges, edge_feat_dim).to(g.device)\n",
    "    \n",
    "for g in lobsters:\n",
    "    num_nodes = g.number_of_nodes()\n",
    "    g.ndata[node_feat_loc] = torch.randn(num_nodes, node_feat_dim).to(g.device)\n",
    "    num_edges = g.number_of_edges()\n",
    "    g.edata[edge_feat_loc] = torch.randn(num_edges, edge_feat_dim).to(g.device)\n",
    "\n",
    "# Load GIN\n",
    "gin = load_feature_extractor(\n",
    "    device=device, input_dim=node_feat_dim, edge_feat_dim=edge_feat_dim,\n",
    "    node_feat_loc=node_feat_loc, edge_feat_loc=edge_feat_loc)\n",
    "\n",
    "mmd_eval = MMDEvaluation(gin)\n",
    "result, time = mmd_eval.evaluate(generated_dataset=lobsters, reference_dataset=grids)\n",
    "print('result: {}, time to compute: {:.3f}s'.format(result, time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e4143dbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MMD RBF for grids with itself: 0\n",
      "MMD RBF after altering node/edge feat. distributions: 0.03274035453796387\n"
     ]
    }
   ],
   "source": [
    "# To quickly demonstrate sensitivity to node/edge features, let's create\n",
    "# a copy of the grid graphs and slightly alter the node/edge feat distributions\n",
    "import copy\n",
    "\n",
    "result, time = mmd_eval.evaluate(generated_dataset=grids, reference_dataset=grids)\n",
    "print('MMD RBF for grids with itself: {}'.format(result['mmd_rbf']))\n",
    "\n",
    "grids_copy = [copy.deepcopy(g) for g in grids]\n",
    "for g in grids_copy:\n",
    "    g.ndata[node_feat_loc] *= 1.15\n",
    "    g.edata[edge_feat_loc] *= 1.15\n",
    "    \n",
    "result, time = mmd_eval.evaluate(reference_dataset=grids, generated_dataset=grids_copy)\n",
    "print('MMD RBF after altering node/edge feat. distributions: {}'.format(result['mmd_rbf']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b331942a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pna",
   "language": "python",
   "name": "pna"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}