DOC add benchmark notebooks to website

.circleci/config.yml

@@ -27,7 +27,9 @@ jobs:
             - "venv"
   test-module:
     parameters:
-      module:
+      benchmarks:
+        type: string
+      docs:
         type: string
     docker:
       - image: cimg/python:3.8
@@ -49,7 +51,8 @@ jobs:
           name: run code linting
           command: |
             . venv/bin/activate
-            black --check --diff ./<< parameters.module >>
+            black --check --diff ./<< parameters.benchmarks >>
+            black --check --diff ./<< parameters.docs >>
 workflows:
   build-test:
     jobs:
@@ -58,6 +61,7 @@ workflows:
           version: "3.8"
       - test-module:
           name: "benchmarks"
-          module: "benchmarks"
+          benchmarks: "benchmarks"
+          docs: "docs"
           requires:
             - "v3.8"

.gitattributes

@@ -0,0 +1 @@
+docs/** -linguist-documentation

CITATION.cff

@@ -67,6 +67,6 @@ keywords:
   - "deep networks"
 license: MIT
 message: "If you use the benchmark code of DF/DN, please cite it using these metadata."
-repository-code: "https://github.com/neurodata/rf-dn-paper"
+repository-code: "https://github.com/neurodata/df-dn-paper"
 title: "When are Deep Networks really better than Decision Forests at small sample sizes, and how?"
 ...

README.md

@@ -0,0 +1,16 @@
+# When are Deep Networks really better than Decision Forests at small sample sizes, and how?
+
+[![arXiv](https://img.shields.io/badge/arXiv-2108.13637-red.svg?style=flat)](https://arxiv.org/abs/2108.13637)
+[![CircleCI](https://circleci.com/gh/neurodata/df-dn-paper/tree/main.svg?style=shield)](https://circleci.com/gh/neurodata/df-dn-paper/tree/main)
+[![Netlify](https://img.shields.io/netlify/e77b134b-1e9b-4ae9-b378-822615333dbd)](https://app.netlify.com/sites/dfdn/deploys)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![License](https://img.shields.io/badge/License-MIT-blue)](https://opensource.org/licenses/MIT)
+
+**DF/DN:** conceptual & empirical comparisons between **D**ecision **F**orests & **D**eep **N**etworks.
+
+**This is preliminary work. More details will be available.**
+
+- **Documentation:** https://dfdn.neurodata.io/
+- **Abstract:** https://dfdn.neurodata.io/#abstract
+- **Replication Guide:** https://dfdn.neurodata.io/#replicate
+- **Benchmark Figures:** https://dfdn.neurodata.io/#benchmarks

benchmarks/tabular/cc18_figures.ipynb → docs/cc18_figures.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# RF/DN Figures for CC18 Data Set Suite"
+    "# DF/DN on CC18"
    ]
   },
   {
@@ -83,27 +83,29 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "directory = \"../benchmarks/tabular/\"\n",
+    "\n",
     "# Load model performances across range of sample sizes\n",
-    "full_dataset_rf_scores_kappa = load_result(\"results/cc18_rf_kappa.txt\")\n",
-    "full_dataset_dn_scores_kappa = load_result(\"results/cc18_dn_kappa.txt\")\n",
-    "full_dataset_rf_scores_ece = load_result(\"results/cc18_rf_ece.txt\")\n",
-    "full_dataset_dn_scores_ece = load_result(\"results/cc18_dn_ece.txt\")\n",
+    "full_dataset_rf_scores_kappa = load_result(directory + \"results/cc18_rf_kappa.txt\")\n",
+    "full_dataset_dn_scores_kappa = load_result(directory + \"results/cc18_dn_kappa.txt\")\n",
+    "full_dataset_rf_scores_ece = load_result(directory + \"results/cc18_rf_ece.txt\")\n",
+    "full_dataset_dn_scores_ece = load_result(directory + \"results/cc18_dn_ece.txt\")\n",
     "\n",
     "# Load model wall times\n",
-    "rf_times_train = load_result(\"results/cc18_rf_times_train.txt\")\n",
-    "rf_times_test = load_result(\"results/cc18_rf_times_test.txt\")\n",
-    "dn_times_train = load_result(\"results/cc18_dn_times_train.txt\")\n",
-    "dn_times_test = load_result(\"results/cc18_dn_times_test.txt\")\n",
+    "rf_times_train = load_result(directory + \"results/cc18_rf_times_train.txt\")\n",
+    "rf_times_test = load_result(directory + \"results/cc18_rf_times_test.txt\")\n",
+    "dn_times_train = load_result(directory + \"results/cc18_dn_times_train.txt\")\n",
+    "dn_times_test = load_result(directory + \"results/cc18_dn_times_test.txt\")\n",
     "\n",
     "# Load sample sizes per dataset\n",
-    "full_dataset_all_sample_sizes = load_result(\"metrics/cc18_sample_sizes.txt\")"
+    "full_dataset_all_sample_sizes = load_result(directory + \"metrics/cc18_sample_sizes.txt\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### RF/DN Comparison Figure\n",
+    "## DF/DN Comparison Figure\n",
     "\n",
     "- Panel 1 visualizes the change in kappa scores between random forest and deep networks for each dataset. \n",
     "\n",
@@ -133,8 +135,6 @@
    "source": [
     "# Set color palette\n",
     "sns.set(color_codes=True, style=\"white\", context=\"talk\", font_scale=1.5)\n",
-    "PALETTE = sns.color_palette(\"Set1\")\n",
-    "sns.set_palette(PALETTE[1:5] + PALETTE[6:], n_colors=9)\n",
     "fig = plt.figure(figsize=(19, 6))\n",
     "\n",
     "# Subplot 1\n",
@@ -384,14 +384,14 @@
     "ax.legend(fontsize=20, loc=\"lower right\", frameon=False)\n",
     "plt.tight_layout()\n",
     "plt.xticks([1e1, 1e2, 1e3, 1e4])\n",
-    "plt.savefig(\"../../paper/figures/cc18.pdf\", transparent=True, bbox_inches=\"tight\")"
+    "plt.savefig(\"../paper/figures/cc18.pdf\", transparent=True, bbox_inches=\"tight\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### RF/DN Scatter Plot\n",
+    "## DF/DN Scatter Plot\n",
     "\n",
     "- Column 1 visualizes the relationship between kappa and features for both models. \n",
     "\n",

benchmarks/vision/cifar_figures.ipynb → docs/cifar_figures.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# RF/DN Multiclass Comparisons on CIFAR10/100"
+    "# DF/DN on CIFAR"
    ]
   },
   {
@@ -26,9 +26,7 @@
    "outputs": [],
    "source": [
     "# Define color palette\n",
-    "sns.set(color_codes=True, style=\"white\", context=\"talk\", font_scale=1.5)\n",
-    "PALETTE = sns.color_palette(\"Set1\")\n",
-    "sns.set_palette(PALETTE[1:5] + PALETTE[6:], n_colors=9)"
+    "sns.set(color_codes=True, style=\"white\", context=\"talk\", font_scale=1.5)"
    ]
   },
   {
@@ -282,7 +280,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## RF & DN with Unbounded Time & Cost"
+    "## DF/DN with Unbounded Time & Cost"
    ]
   },
   {
@@ -304,27 +302,28 @@
     }
    ],
    "source": [
+    "directory = \"../benchmarks/vision/\"\n",
     "names = [\"naive_rf\", \"cnn32\", \"cnn32_2l\", \"cnn32_5l\", \"resnet18\"]\n",
     "\n",
     "# Load 3-classes results\n",
-    "acc_3, time_3 = load_results(\"3_class/\")\n",
+    "acc_3, time_3 = load_results(directory + \"3_class/\")\n",
     "\n",
     "# Load 8-classes results\n",
-    "acc_8, time_8 = load_results(\"8_class/\")\n",
+    "acc_8, time_8 = load_results(directory + \"8_class/\")\n",
     "\n",
     "# Load 90-classes results\n",
-    "acc_90, time_90 = load_results(\"90_class/\")\n",
+    "acc_90, time_90 = load_results(directory + \"90_class/\")\n",
     "\n",
     "accs = [acc_3, acc_8, acc_90, time_3, time_8, time_90]\n",
     "plot_six()\n",
-    "plt.savefig(\"../../paper/figures/cifar.pdf\", transparent=True, bbox_inches=\"tight\")"
+    "plt.savefig(\"../paper/figures/cifar.pdf\", transparent=True, bbox_inches=\"tight\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## RF & DN with Fixed Training Time"
+    "## DF/DN with Fixed Training Time"
    ]
   },
   {
@@ -345,24 +344,24 @@
    ],
    "source": [
     "# Load 3-classes results\n",
-    "acc_3, time_3 = load_results_st(\"3_class/\")\n",
+    "acc_3, time_3 = load_results_st(directory + \"3_class/\")\n",
     "\n",
     "# Load 8-classes results\n",
-    "acc_8, time_8 = load_results_st(\"8_class/\")\n",
+    "acc_8, time_8 = load_results_st(directory + \"8_class/\")\n",
     "\n",
     "# Load 90-classes results\n",
-    "acc_90, time_90 = load_results_st(\"90_class/\")\n",
+    "acc_90, time_90 = load_results_st(directory + \"90_class/\")\n",
     "\n",
     "accs = [acc_3, acc_8, acc_90, time_3, time_8, time_90]\n",
     "plot_six()\n",
-    "plt.savefig(\"../../paper/figures/cifar_st.pdf\", transparent=True, bbox_inches=\"tight\")"
+    "plt.savefig(\"../paper/figures/cifar_st.pdf\", transparent=True, bbox_inches=\"tight\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## RF & DN with Fixed Training Cost"
+    "## DF/DN with Fixed Training Cost"
    ]
   },
   {
@@ -383,17 +382,17 @@
    ],
    "source": [
     "# Load 3-classes results\n",
-    "acc_3, time_3 = load_results_sc(\"3_class/\")\n",
+    "acc_3, time_3 = load_results_sc(directory + \"3_class/\")\n",
     "\n",
     "# Load 8-classes results\n",
-    "acc_8, time_8 = load_results_sc(\"8_class/\")\n",
+    "acc_8, time_8 = load_results_sc(directory + \"8_class/\")\n",
     "\n",
     "# Load 90-classes results\n",
-    "acc_90, time_90 = load_results_sc(\"90_class/\")\n",
+    "acc_90, time_90 = load_results_sc(directory + \"90_class/\")\n",
     "\n",
     "accs = [acc_3, acc_8, acc_90, time_3, time_8, time_90]\n",
     "plot_six()\n",
-    "plt.savefig(\"../../paper/figures/cifar_sc.pdf\", transparent=True, bbox_inches=\"tight\")"
+    "plt.savefig(\"../paper/figures/cifar_sc.pdf\", transparent=True, bbox_inches=\"tight\")"
    ]
   }
  ],

docs/conf.py

@@ -27,7 +27,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = []
+extensions = ["nbsphinx"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -77,11 +77,11 @@
         ),
         (
             '<i class="fa fa-exclamation-circle fa-fw"></i> Issue Tracker',
-            "https://github.com/neurodata/rf-dn-paper/issues",
+            "https://github.com/neurodata/df-dn-paper/issues",
         ),
         (
             '<i class="fa fa-github fa-fw"></i> Source Code',
-            "https://github.com/neurodata/rf-dn-paper",
+            "https://github.com/neurodata/df-dn-paper",
         ),
     ],
     # Enable the "Edit in GitHub link within the header of each page.
@@ -91,7 +91,7 @@
     # /{{ github_repo }}/blob/{{ github_version }}{{ conf_py_path }}{{ pagename }}
     # {{ suffix }}
     "github_user": "neurodata",
-    "github_repo": "rf-dn-paper",
+    "github_repo": "df-dn-paper",
     "github_version": "main/",
     "conf_py_path": "docs/",
 }