more UI work

- rewrote the REPL thing entirely to be mostly server side
- introduced a simple sync-db mechanism

src/main/shadow/cljs/devtools/server.clj

@@ -28,6 +28,7 @@
  [shadow.cljs.devtools.server.build-history :as build-history]
  [shadow.cljs.devtools.server.system-bus :as system-bus]
  [shadow.cljs.devtools.server.remote-ext :as remote-ext]
+ [shadow.cljs.devtools.server.sync-db :as sync-db]
  [shadow.remote.relay.local :as relay]
  [shadow.remote.runtime.clj.local :as clj-runtime]
  [shadow.remote.runtime.obj-support :as obj-support]
@@ -437,7 +438,7 @@
  (let [app
  (merge
  {:dev-http
- {:depends-on [:system-bus :config :ssl-context :out]
+ {:depends-on [:system-bus :config :ssl-context :out :sync-db]
  :start dev-http/start
  :stop dev-http/stop}
 
@@ -461,7 +462,7 @@
  :stop fs-watch/stop}
 
  :config-watch
- {:depends-on [:system-bus]
+ {:depends-on [:system-bus :sync-db]
  :start config-watch/start
  :stop config-watch/stop}
 
@@ -476,12 +477,12 @@
  :stop reload-classpath/stop}
 
  :build-history
- {:depends-on [:system-bus]
+ {:depends-on [:system-bus :sync-db]
  :start build-history/start
  :stop build-history/stop}
 
  :supervisor
- {:depends-on [:config :system-bus :build-executor :relay :clj-runtime :clj-runtime-obj-support :cache-root :http :classpath :npm :babel]
+ {:depends-on [:config :system-bus :build-executor :relay :clj-runtime :clj-runtime-obj-support :cache-root :http :classpath :npm :babel :sync-db]
  :start super/start
  :stop super/stop}
 
@@ -516,10 +517,15 @@
  :stop explore-support/stop}
 
  :clj-runtime-shadow-ext
- {:depends-on [:clj-runtime :supervisor :system-bus]
+ {:depends-on [:clj-runtime :sync-db :supervisor :system-bus]
  :start remote-ext/start
  :stop remote-ext/stop}
 
+ :sync-db
+ {:depends-on []
+ :start sync-db/start
+ :stop sync-db/stop}
+
  :out
  {:depends-on [:config]
  :start (fn [config]

src/main/shadow/cljs/devtools/server/build_history.clj

@@ -2,6 +2,7 @@
  (:require
  [clojure.core.async :as async :refer (go alt!)]
  [shadow.cljs :as-alias m]
+ [shadow.cljs.devtools.server.sync-db :as sync-db]
  [shadow.cljs.devtools.server.system-bus :as sys-bus]
  [shadow.build.log :as build-log]))
 
@@ -31,7 +32,7 @@
  (into []))]
 
  (assoc state
- :info info
+ ;; :info info
  :status :completed
  :resources (count sources)
  :compiled (count compiled)
@@ -73,26 +74,29 @@
  ;; mostly REPL related things
  state))
 
-(defn build-status-loop [{:keys [system-bus state-ref sub-chan]}]
+(defn build-status-loop [{:keys [sync-db state-ref sub-chan]}]
  ;; FIXME: figure out which update frequency makes sense for the UI
  ;; 10fps is probably more than enough?
  (let [flush-delay 100
  flush-fn
  (fn [needs-flush]
- (doseq [build-id needs-flush]
- (let [state
- (get @state-ref build-id)
+ (sync-db/update! sync-db update ::m/build
+ (fn [table]
+ (reduce
+ (fn [table build-id]
+ (let [state
+ (get @state-ref build-id)
 
- flush-state
- (-> state
- (cond->
- (not (contains? #{:failed :completed} (:status state)))
- (dissoc :log)))
+  flush-state
+  (-> state
+  (cond->
+  (not (contains? #{:failed :completed} (:status state)))
+  (dissoc :log)))]
 
- msg {:build-id build-id
- :build-status flush-state}]
+ (assoc-in table [build-id ::m/build-status] flush-state)))
 
- (sys-bus/publish! system-bus ::m/build-status-update msg))))]
+ table
+ needs-flush))))]
 
  (go (loop [needs-flush #{}
  timeout (async/timeout flush-delay)]
@@ -112,7 +116,7 @@
  (do (swap! state-ref update build-id update-build-status msg)
  (recur (conj needs-flush build-id) timeout)))))))))
 
-(defn start [sys-bus]
+(defn start [sys-bus sync-db]
  (let [sub-chan
  (-> (async/sliding-buffer 100)
  (async/chan))
@@ -123,6 +127,7 @@
  svc
  {:system-bus sys-bus
  :sub-chan sub-chan
+ :sync-db sync-db
  :state-ref state-ref}]
 
  (build-status-loop svc)

src/main/shadow/cljs/devtools/server/config_watch.clj

@@ -3,6 +3,7 @@
  [shadow.cljs.devtools.server.env :as env]
  [clojure.java.io :as io]
  [clojure.core.async :as async :refer (thread <!!)]
+ [shadow.cljs.devtools.server.sync-db :as sync-db]
  [shadow.cljs.devtools.server.system-bus :as sys-bus]
  [shadow.cljs :as-alias m]
  [shadow.jvm-log :as log]))
@@ -17,10 +18,37 @@
  0
  (.lastModified file))))
 
-(defn watch-loop [stop-ref system-bus]
+(defn sync-builds [sync-db builds]
+ (sync-db/update! sync-db update ::m/build
+ (fn [current]
+ (reduce-kv
+ (fn [table build-id build-config]
+ (let [prev-entry (get current build-id)]
+ (cond
+ ;; new build config added
+ (not prev-entry)
+ (assoc table build-id {::m/build-id build-id
+ ::m/build-config-raw build-config})
+
+ ;; diffs a bit faster if we keep the old entry
+ (= (::m/build-config-raw prev-entry) build-config)
+ (assoc table build-id prev-entry)
+
+ ;; preserve other fields while updating config
+ :else
+ (assoc table build-id (assoc prev-entry ::m/build-config-raw build-config))
+ )))
+ ;; starting with empty instead of updating table
+ ;; so that deleted configs are actually deleted
+ {}
+ builds))))
+
+(defn watch-loop [stop-ref system-bus sync-db]
  (let [{:keys [dependencies] :as config}
  (config/load-cljs-edn)]
 
+ (sync-builds sync-db (:builds config))
+
  (loop [ts (get-last-modified)
  config config]
 
@@ -42,17 +70,19 @@
  (log/debug ::update-build {:build-id build-id})
  (sys-bus/publish! system-bus [::m/config-watch build-id] {:config new}))
 
+ (sync-builds sync-db (:builds new-config))
+
  (reset! env/dependencies-modified-ref (not= dependencies (:dependencies new-config)))
 
  (recur ts-test new-config)
  )))))))
 
-(defn start [system-bus]
+(defn start [system-bus sync-db]
  (let [stop-ref
  (volatile! false)
 
  thread-ref
- (thread (watch-loop stop-ref system-bus))]
+ (thread (watch-loop stop-ref system-bus sync-db))]
 
  {::service true
  :system-bus system-bus

src/main/shadow/cljs/devtools/server/dev_http.clj

@@ -4,6 +4,7 @@
  [clojure.java.io :as io]
  [clojure.core.async :as async :refer (>!! <!!)]
  [clojure.spec.alpha :as s]
+ [shadow.cljs.devtools.server.sync-db :as sync-db]
  [shadow.jvm-log :as log]
  [shadow.cljs.devtools.config :as config]
  [shadow.cljs.devtools.server.system-bus :as sys-bus]
@@ -407,7 +408,20 @@
  (reset! file-watch-ref false)))
  (dissoc state :server :configs))
 
-(defn start [sys-bus config ssl-context out]
+(defn sync-servers [sync-db servers]
+ (sync-db/update! sync-db update ::m/http-server
+ (fn [table]
+ (reduce-kv
+ (fn [table idx {:keys [http-url https-url config]}]
+ (assoc table idx {::m/http-server-id idx
+ ::m/http-url http-url
+ ::m/http-config config
+ ::m/https-url https-url}))
+ ;; FIXME: don't fully reset table in case these ever receive updates elsewhere?
+ {}
+ servers))))
+
+(defn start [sys-bus config ssl-context out sync-db]
  (let [sub-chan
  (-> (async/sliding-buffer 1)
  (async/chan))
@@ -439,9 +453,14 @@
  (swap! state-ref stop-servers)
  (swap! state-ref assoc
  :servers (start-servers sys-bus new-configs ssl-context out)
- :configs new-configs))
+ :configs new-configs)
+
+ (sync-servers sync-db (:servers @state-ref)))
+
  (recur new-configs)))))]
 
+ (sync-servers sync-db (:servers @state-ref))
+
  (swap! state-ref assoc :loop-chan loop-chan)
 
  state-ref