Initial draft with benchmarking

Author: Andy Chambers

env/profiling/hitchhiker/bench.clj

@@ -2,8 +2,10 @@
   (:require [clojure.pprint :as pp]
             [clojure.string :as str]
             [clojure.tools.cli :refer [parse-opts]]
+            [clojure.java.jdbc :as jdbc]
             [excel-templates.build :as excel]
             [hitchhiker.redis :as redis]
+            [hitchhiker.sqlite :as sqlite]
             [hitchhiker.tree.core :as core]
             [hitchhiker.tree.messaging :as msg])
   (:import [java.io File FileWriter]))
@@ -129,7 +131,7 @@
     :validate [#(#{"fractal" "b-tree" "sorted-set"} %) "Data structure must be fractal, b-tree, or sorted set"]]
    [nil "--backend testing" "Runs the benchmark with the specified backend"
     :default "testing"
-    :validate [#(#{"redis" "testing"} %) "Backend must be redis or testing"]]
+    :validate [#(#{"redis" "sqlite" "testing"} %) "Backend must be redis, sqlite or testing"]]
    ["-d" "--delete-pattern PATTERN" "Specifies how the operations will be reordered on delete"
     :default "forward"
     :validate [#(#{"forward" "reverse" "shuffle" "zero"} %) "Incorrect delete pattern"]
@@ -194,74 +196,77 @@
 
 (defn -main
   [& [root & args]]
-  (let [outputs (atom [])]
-    (doseq [args (or (->> args
-                          (partition-by #(= % "--"))
-                          (map-indexed vector)
-                          (filter (comp even? first))
-                          (map second)
-                          (seq))
-                     [[]])] ; always do one iteration
-      (let [{:keys [options arguments errors summary]} (parse-opts args options)
-            tree-to-test (atom {})
-            results (atom [])]
-        (cond
-          (or (= "-h" root)
-              (= "--help" root)
-              (nil? root)
-              (:help options)) (exit 0 (usage summary))
-          (not= (count arguments) 0) (exit 1 (usage summary))
-          errors (exit 1 (error-msg errors)))
-        (let [backend (case (:backend options)
-                        "testing" (core/->TestingBackend)
-                        "redis" (do (redis/start-expiry-thread!)
-                                    (redis/->RedisBackend)))
-              delete-xform (case (:delete-pattern options)
-                             "forward" identity
-                             "reverse" reverse
-                             "shuffle" shuffle
-                             "zero" #(repeat (count %) 0.0))
-              [tree-name structure]
-              (case (:data-structure options)
-                "b-tree" ["b-tree" (core-b-tree (:tree-width options) backend)]
-                "fractal" ["fractal" (msg-b-tree (:tree-width options) backend)]
-                "sorted-set" ["sorted-set" (sorted-set-repr)])
-              flush-freq (:flush-freq options)
-              codename (str tree-name
-                            "__flush_"
-                            flush-freq
-                            "__b_"
-                            (:tree-width options)
-                            "__"
-                            (:backend options)
-                            "__n_"
-                            (:num-operations options)
-                            "__del_"
-                            (:delete-pattern options))]
-          (doseq [ds (generate-test-datasets)
-                  :let [codename (str codename
-                                      "_"
-                                      (:name ds))
-                        out (create-output-dir
-                              root
-                              codename)
-                        _ (println "Doing" codename)
-                        bench-res (benchmark (:num-operations options) ds flush-freq structure out delete-xform)]]
-            (swap! results conj
-                   {:tree tree-name
-                    :ds (:name ds)
-                    :freq flush-freq
-                    :n (:num-operations options)
-                    :b (:tree-width options)
-                    :delete-pattern (:delete-pattern options)
-                    :results bench-res}))
-          ;(println "results")
-          ;(clojure.pprint/pprint @results)
-          (swap! outputs conj (template-one-sheet @results)))))
-    (excel/render-to-file
-      "template_benchmark.xlsx"
-      (.getPath (File. root "analysis.xlsx"))
-      {"SingleDS"
-       (map-indexed (fn [i s]
-                      (assoc s :sheet-name (str "Trial " (inc i))))
-                    @outputs)})))
+  (jdbc/with-db-connection [db (sqlite/db-spec ":memory:")]
+    (let [outputs (atom [])]
+      (doseq [args (or (->> args
+                            (partition-by #(= % "--"))
+                            (map-indexed vector)
+                            (filter (comp even? first))
+                            (map second)
+                            (seq))
+                       [[]])] ; always do one iteration
+        (let [{:keys [options arguments errors summary]} (parse-opts args options)
+              tree-to-test (atom {})
+              results (atom [])]
+          (cond
+            (or (= "-h" root)
+                (= "--help" root)
+                (nil? root)
+                (:help options)) (exit 0 (usage summary))
+            (not= (count arguments) 0) (exit 1 (usage summary))
+            errors (exit 1 (error-msg errors)))
+          (let [backend (case (:backend options)
+                          "testing" (core/->TestingBackend)
+                          "redis" (do (redis/start-expiry-thread!)
+                                      (redis/->RedisBackend))
+                          "sqlite" (do (sqlite/ensure-schema db)
+                                       (sqlite/->SQLiteBackend db)))
+                delete-xform (case (:delete-pattern options)
+                               "forward" identity
+                               "reverse" reverse
+                               "shuffle" shuffle
+                               "zero" #(repeat (count %) 0.0))
+                [tree-name structure]
+                (case (:data-structure options)
+                  "b-tree" ["b-tree" (core-b-tree (:tree-width options) backend)]
+                  "fractal" ["fractal" (msg-b-tree (:tree-width options) backend)]
+                  "sorted-set" ["sorted-set" (sorted-set-repr)])
+                flush-freq (:flush-freq options)
+                codename (str tree-name
+                              "__flush_"
+                              flush-freq
+                              "__b_"
+                              (:tree-width options)
+                              "__"
+                              (:backend options)
+                              "__n_"
+                              (:num-operations options)
+                              "__del_"
+                              (:delete-pattern options))]
+            (doseq [ds (generate-test-datasets)
+                    :let [codename (str codename
+                                        "_"
+                                        (:name ds))
+                          out (create-output-dir
+                               root
+                               codename)
+                          _ (println "Doing" codename)
+                          bench-res (benchmark (:num-operations options) ds flush-freq structure out delete-xform)]]
+              (swap! results conj
+                     {:tree tree-name
+                      :ds (:name ds)
+                      :freq flush-freq
+                      :n (:num-operations options)
+                      :b (:tree-width options)
+                      :delete-pattern (:delete-pattern options)
+                      :results bench-res}))
+                                        ;(println "results")
+                                        ;(clojure.pprint/pprint @results)
+            (swap! outputs conj (template-one-sheet @results)))))
+      (excel/render-to-file
+       "template_benchmark.xlsx"
+       (.getPath (File. root "analysis.xlsx"))
+       {"SingleDS"
+        (map-indexed (fn [i s]
+                       (assoc s :sheet-name (str "Trial " (inc i))))
+                     @outputs)}))))

project.clj

@@ -6,7 +6,10 @@
   :dependencies [[org.clojure/clojure "1.7.0"]
                  [org.clojure/core.memoize "0.5.8"]
                  [com.taoensso/carmine "2.12.2"]
-                 [org.clojure/core.rrb-vector "0.0.11"]]
+                 [org.clojure/core.rrb-vector "0.0.11"]
+
+                 [org.clojure/java.jdbc "0.6.2-alpha2"]
+                 [org.xerial/sqlite-jdbc "3.7.2"]]
   :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]}
   :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"]
   :profiles {:test