Merge pull request metabase#3608 from metabase/faster-metabase-launch…

…-time

Shave another second off of Metabase launch time 🏎️

src/metabase/driver.clj

@@ -1,8 +1,6 @@
 (ns metabase.driver
-  (:require [clojure.java.classpath :as classpath]
-            [clojure.math.numeric-tower :as math]
+  (:require [clojure.math.numeric-tower :as math]
             [clojure.tools.logging :as log]
-            [clojure.tools.namespace.find :as ns-find]
             [medley.core :as m]
             (metabase [config :as config]
                       [db :as db])
@@ -272,7 +270,7 @@
   "Search Classpath for namespaces that start with `metabase.driver.`, then `require` them and look for the `driver-init`
    function which provides a uniform way for Driver initialization to be done."
   []
-  (doseq [ns-symb (ns-find/find-namespaces (classpath/classpath))
+  (doseq [ns-symb @u/metabase-namespace-symbols
           :when   (re-matches #"^metabase\.driver\.[a-z0-9_]+$" (name ns-symb))]
     (require ns-symb)))
 

src/metabase/events.clj

@@ -9,10 +9,8 @@
    once when the application goes through normal startup procedures.  Inside this function you can do any work
    needed and add your events subscribers to the bus as usual via `start-event-listener`."
   (:require [clojure.core.async :as async]
-            [clojure.java.classpath :as classpath]
             [clojure.string :as s]
             [clojure.tools.logging :as log]
-            [clojure.tools.namespace.find :as ns-find]
             (metabase [config :as config]
                       [util :as u])))
 
@@ -27,8 +25,8 @@
   "Search Classpath for namespaces that start with `metabase.events.`, and call their `events-init` function if it exists."
   []
   (when-not config/is-test?
-    (doseq [ns-symb (ns-find/find-namespaces (classpath/classpath))
-            :when (re-find #"^metabase\.events\." (name ns-symb))]
+    (doseq [ns-symb @u/metabase-namespace-symbols
+            :when   (.startsWith (name ns-symb) "metabase.events.")]
       (require ns-symb)
       ;; look for `events-init` function in the namespace and call it if it exists
       (when-let [init-fn (ns-resolve ns-symb 'events-init)]

src/metabase/models/hydrate.clj

@@ -1,8 +1,6 @@
 (ns metabase.models.hydrate
   "Functions for deserializing and hydrating fields in objects fetched from the DB."
-  (:require [clojure.java.classpath :as classpath]
-            [clojure.tools.namespace.find :as ns-find]
-            [medley.core :as m]
+  (:require [medley.core :as m]
             [metabase.db :as db]
             [metabase.models.interface :as i]
             [metabase.util :as u]))
@@ -150,7 +148,7 @@
    e.g. `:user -> User`.
 
    This is built pulling the `hydration-keys` set from all of our entities."
-  (delay (for [ns-symb (ns-find/find-namespaces (classpath/classpath))               ; Seems to work fine without this but better safe than sorry IMO
+  (delay (for [ns-symb @u/metabase-namespace-symbols
                :when   (re-matches #"^metabase\.models\.[a-z0-9]+$" (name ns-symb))]
            (require ns-symb))
          (into {} (for [ns       (all-ns)
@@ -364,22 +362,21 @@
 
   You can hydrate several keys at one time:
 
-    (hydrate {:a (delay 1) :b (delay 2)} :a :b)
-      -> {:a 1 :b 2}
+    (hydrate {...} :a :b)
+      -> {:a 1, :b 2}
 
 
   ** Nested Hydration **
 
   You can do recursive hydration by listing keys inside a vector:
 
-    (hydrate {:a (delay {:b (delay 1)})} [:a :b])
+    (hydrate {...} [:a :b])
       -> {:a {:b 1}}
 
   The first key in a vector will be hydrated normally, and any subsequent keys
   will be hydrated *inside* the corresponding values for that key.
 
-    (hydrate {:a (delay {:b (delay {:c (delay 1)})
-                         :e (delay 2)})}
+    (hydrate {...}
              [:a [:b :c] :e])
       -> {:a {:b {:c 1} :e 2}}"
   [results k & ks]

src/metabase/task.clj

@@ -7,9 +7,7 @@
    `task-init` function which accepts zero arguments.  This function is dynamically resolved and called
    exactly once when the application goes through normal startup procedures.  Inside this function you
    can do any work needed and add your task to the scheduler as usual via `schedule-task!`."
-  (:require [clojure.java.classpath :as classpath]
-            [clojure.tools.logging :as log]
-            [clojure.tools.namespace.find :as ns-find]
+  (:require [clojure.tools.logging :as log]
             [clojurewerkz.quartzite.scheduler :as qs]
             [metabase.util :as u]))
 
@@ -20,8 +18,8 @@
 (defn- find-and-load-tasks!
   "Search Classpath for namespaces that start with `metabase.tasks.`, then `require` them so initialization can happen."
   []
-  (doseq [ns-symb (ns-find/find-namespaces (classpath/classpath))
-          :when   (re-find #"^metabase\.task\." (name ns-symb))]
+  (doseq [ns-symb @u/metabase-namespace-symbols
+          :when   (.startsWith (name ns-symb) "metabase.task.")]
     (log/info "Loading tasks namespace:" (u/format-color 'blue ns-symb) "📆")
     (require ns-symb)
     ;; look for `task-init` function in the namespace and call it if it exists

src/metabase/util.clj

@@ -1,11 +1,13 @@
 (ns metabase.util
   "Common utility functions useful throughout the codebase."
   (:require [clojure.data :as data]
-            [clojure.java.jdbc :as jdbc]
+            (clojure.java [classpath :as classpath]
+                          [jdbc :as jdbc])
             [clojure.math.numeric-tower :as math]
             (clojure [pprint :refer [pprint]]
                      [string :as s])
             [clojure.tools.logging :as log]
+            [clojure.tools.namespace.find :as ns-find]
             (clj-time [core :as t]
                       [coerce :as coerce]
                       [format :as time])
@@ -715,9 +717,25 @@
     :else                   (throw (Exception. (str "Not something with an ID: " object-or-id)))))
 
 (defmacro profile
-  "Like `clojure.core/time`, but lets you specify a message that gets printed with the total time, and formats the time nicely using `format-nanoseconds`."
+  "Like `clojure.core/time`, but lets you specify a MESSAGE that gets printed with the total time,
+   and formats the time nicely using `format-nanoseconds`."
   {:style/indent 1}
-  [message & body]
-  `(let [start-time# (System/nanoTime)]
-     (prog1 (do ~@body)
-       (println (format-color '~'green "%s took %s" ~message (format-nanoseconds (- (System/nanoTime) start-time#)))))))
+  ([form]
+   `(profile ~(str form) ~form))
+  ([message & body]
+   `(let [start-time# (System/nanoTime)]
+      (prog1 (do ~@body)
+        (println (format-color '~'green "%s took %s" ~message (format-nanoseconds (- (System/nanoTime) start-time#))))))))
+
+(def metabase-namespace-symbols
+  "Delay to a vector of symbols of all Metabase namespaces, excluding test namespaces.
+   This is intended for use by various routines that load related namespaces, such as task and events initialization.
+   Using `ns-find/find-namespaces` is fairly slow, and can take as much as half a second to iterate over the thousand or so
+   namespaces that are part of the Metabase project; use this instead for a massive performance increase."
+  ;; Actually we can go ahead and start doing this in the background once the app launches while other stuff is loading, so use a future here
+  ;; This would be faster when running the *JAR* if we just did it at compile-time and made it ^:const, but that would inhibit the "plugin system"
+  ;; from loading "plugin" namespaces at launch if they're on the classpath
+  (future (vec (for [ns-symb (ns-find/find-namespaces (classpath/classpath))
+                     :when   (and (.startsWith (name ns-symb) "metabase.")
+                                  (not (.contains (name ns-symb) "test")))]
+                 ns-symb))))