From b04cd54a51fd5641a8439bee83b6db177d81659f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cam=20Sa=C3=BCl?= Date: Mon, 17 Oct 2016 13:23:12 -0700 Subject: [PATCH 1/2] Shave another second off of Metabase launch time :racing_car: --- src/metabase/driver.clj | 6 ++---- src/metabase/events.clj | 6 ++---- src/metabase/models/hydrate.clj | 6 ++---- src/metabase/task.clj | 8 +++----- src/metabase/util.clj | 30 ++++++++++++++++++++++++------ 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj index b1c403b12f5ca..798d445e6a228 100644 --- a/src/metabase/driver.clj +++ b/src/metabase/driver.clj @@ -1,8 +1,6 @@ (ns metabase.driver - (:require [clojure.java.classpath :as classpath] - [clojure.math.numeric-tower :as math] + (:require [clojure.math.numeric-tower :as math] [clojure.tools.logging :as log] - [clojure.tools.namespace.find :as ns-find] [medley.core :as m] (metabase [config :as config] [db :as db]) @@ -272,7 +270,7 @@ "Search Classpath for namespaces that start with `metabase.driver.`, then `require` them and look for the `driver-init` function which provides a uniform way for Driver initialization to be done." [] - (doseq [ns-symb (ns-find/find-namespaces (classpath/classpath)) + (doseq [ns-symb @u/metabase-namespace-symbols :when (re-matches #"^metabase\.driver\.[a-z0-9_]+$" (name ns-symb))] (require ns-symb))) diff --git a/src/metabase/events.clj b/src/metabase/events.clj index d5e7b0e32d94a..c31deef10c9ae 100644 --- a/src/metabase/events.clj +++ b/src/metabase/events.clj @@ -9,10 +9,8 @@ once when the application goes through normal startup procedures. Inside this function you can do any work needed and add your events subscribers to the bus as usual via `start-event-listener`." (:require [clojure.core.async :as async] - [clojure.java.classpath :as classpath] [clojure.string :as s] [clojure.tools.logging :as log] - [clojure.tools.namespace.find :as ns-find] (metabase [config :as config] [util :as u]))) @@ -27,8 +25,8 @@ "Search Classpath for namespaces that start with `metabase.events.`, and call their `events-init` function if it exists." [] (when-not config/is-test? - (doseq [ns-symb (ns-find/find-namespaces (classpath/classpath)) - :when (re-find #"^metabase\.events\." (name ns-symb))] + (doseq [ns-symb @u/metabase-namespace-symbols + :when (.startsWith (name ns-symb) "metabase.events.")] (require ns-symb) ;; look for `events-init` function in the namespace and call it if it exists (when-let [init-fn (ns-resolve ns-symb 'events-init)] diff --git a/src/metabase/models/hydrate.clj b/src/metabase/models/hydrate.clj index e9a2f28a8d722..d142c1f7b46be 100644 --- a/src/metabase/models/hydrate.clj +++ b/src/metabase/models/hydrate.clj @@ -1,8 +1,6 @@ (ns metabase.models.hydrate "Functions for deserializing and hydrating fields in objects fetched from the DB." - (:require [clojure.java.classpath :as classpath] - [clojure.tools.namespace.find :as ns-find] - [medley.core :as m] + (:require [medley.core :as m] [metabase.db :as db] [metabase.models.interface :as i] [metabase.util :as u])) @@ -150,7 +148,7 @@ e.g. `:user -> User`. This is built pulling the `hydration-keys` set from all of our entities." - (delay (for [ns-symb (ns-find/find-namespaces (classpath/classpath)) ; Seems to work fine without this but better safe than sorry IMO + (delay (for [ns-symb @u/metabase-namespace-symbols :when (re-matches #"^metabase\.models\.[a-z0-9]+$" (name ns-symb))] (require ns-symb)) (into {} (for [ns (all-ns) diff --git a/src/metabase/task.clj b/src/metabase/task.clj index 10e76440173fb..220ef3b6c8393 100644 --- a/src/metabase/task.clj +++ b/src/metabase/task.clj @@ -7,9 +7,7 @@ `task-init` function which accepts zero arguments. This function is dynamically resolved and called exactly once when the application goes through normal startup procedures. Inside this function you can do any work needed and add your task to the scheduler as usual via `schedule-task!`." - (:require [clojure.java.classpath :as classpath] - [clojure.tools.logging :as log] - [clojure.tools.namespace.find :as ns-find] + (:require [clojure.tools.logging :as log] [clojurewerkz.quartzite.scheduler :as qs] [metabase.util :as u])) @@ -20,8 +18,8 @@ (defn- find-and-load-tasks! "Search Classpath for namespaces that start with `metabase.tasks.`, then `require` them so initialization can happen." [] - (doseq [ns-symb (ns-find/find-namespaces (classpath/classpath)) - :when (re-find #"^metabase\.task\." (name ns-symb))] + (doseq [ns-symb @u/metabase-namespace-symbols + :when (.startsWith (name ns-symb) "metabase.task.")] (log/info "Loading tasks namespace:" (u/format-color 'blue ns-symb) "📆") (require ns-symb) ;; look for `task-init` function in the namespace and call it if it exists diff --git a/src/metabase/util.clj b/src/metabase/util.clj index 3382db1d03d2d..58aaa7d5918ac 100644 --- a/src/metabase/util.clj +++ b/src/metabase/util.clj @@ -1,11 +1,13 @@ (ns metabase.util "Common utility functions useful throughout the codebase." (:require [clojure.data :as data] - [clojure.java.jdbc :as jdbc] + (clojure.java [classpath :as classpath] + [jdbc :as jdbc]) [clojure.math.numeric-tower :as math] (clojure [pprint :refer [pprint]] [string :as s]) [clojure.tools.logging :as log] + [clojure.tools.namespace.find :as ns-find] (clj-time [core :as t] [coerce :as coerce] [format :as time]) @@ -715,9 +717,25 @@ :else (throw (Exception. (str "Not something with an ID: " object-or-id))))) (defmacro profile - "Like `clojure.core/time`, but lets you specify a message that gets printed with the total time, and formats the time nicely using `format-nanoseconds`." + "Like `clojure.core/time`, but lets you specify a MESSAGE that gets printed with the total time, + and formats the time nicely using `format-nanoseconds`." {:style/indent 1} - [message & body] - `(let [start-time# (System/nanoTime)] - (prog1 (do ~@body) - (println (format-color '~'green "%s took %s" ~message (format-nanoseconds (- (System/nanoTime) start-time#))))))) + ([form] + `(profile ~(str form) ~form)) + ([message & body] + `(let [start-time# (System/nanoTime)] + (prog1 (do ~@body) + (println (format-color '~'green "%s took %s" ~message (format-nanoseconds (- (System/nanoTime) start-time#)))))))) + +(def metabase-namespace-symbols + "Delay to a vector of symbols of all Metabase namespaces, excluding test namespaces. + This is intended for use by various routines that load related namespaces, such as task and events initialization. + Using `ns-find/find-namespaces` is fairly slow, and can take as much as half a second to iterate over the thousand or so + namespaces that are part of the Metabase project; use this instead for a massive performance increase." + ;; Actually we can go ahead and start doing this in the background once the app launches while other stuff is loading, so use a future here + ;; This would be faster when running the *JAR* if we just did it at compile-time and made it ^:const, but that would inhibit the "plugin system" + ;; from loading "plugin" namespaces at launch if they're on the classpath + (future (vec (for [ns-symb (ns-find/find-namespaces (classpath/classpath)) + :when (and (.startsWith (name ns-symb) "metabase.") + (not (.contains (name ns-symb) "test")))] + ns-symb)))) From c019b785817e2b858109c52b07cf87c0e71125b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cam=20Sa=C3=BCl?= Date: Mon, 17 Oct 2016 13:44:55 -0700 Subject: [PATCH 2/2] Update old hydrate documentation :page_facing_up: --- src/metabase/models/hydrate.clj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/metabase/models/hydrate.clj b/src/metabase/models/hydrate.clj index d142c1f7b46be..e7b4d8825db97 100644 --- a/src/metabase/models/hydrate.clj +++ b/src/metabase/models/hydrate.clj @@ -362,22 +362,21 @@ You can hydrate several keys at one time: - (hydrate {:a (delay 1) :b (delay 2)} :a :b) - -> {:a 1 :b 2} + (hydrate {...} :a :b) + -> {:a 1, :b 2} ** Nested Hydration ** You can do recursive hydration by listing keys inside a vector: - (hydrate {:a (delay {:b (delay 1)})} [:a :b]) + (hydrate {...} [:a :b]) -> {:a {:b 1}} The first key in a vector will be hydrated normally, and any subsequent keys will be hydrated *inside* the corresponding values for that key. - (hydrate {:a (delay {:b (delay {:c (delay 1)}) - :e (delay 2)})} + (hydrate {...} [:a [:b :c] :e]) -> {:a {:b {:c 1} :e 2}}" [results k & ks]