Make no-cache on side-effecting var invalidate dependents (#158)

Fixes #157.

src/nextjournal/clerk.clj

@@ -129,10 +129,10 @@
     viewers
     (update :nextjournal/viewers eval)))
 
-(defn read+eval-cached [results-last-run ->hash doc-visibility codeblock]
-  (let [{:keys [ns-effect? form var]} codeblock
-        no-cache?      (or ns-effect?
-                           (hashing/no-cache? form))
+(defn read+eval-cached [{:as _doc doc-visibility :visibility :keys [blob->result ->analysis-info ->hash]} codeblock]
+  (let [{:keys [form var]} codeblock
+        {:keys [ns-effect? no-cache?]} (->analysis-info (if var var form))
+        no-cache?      (or ns-effect? no-cache?)
         hash           (when-not no-cache? (or (get ->hash (if var var form))
                                                (hashing/hash-codeblock ->hash codeblock)))
         digest-file    (when hash (->cache-file (str "@" hash)))
@@ -151,17 +151,17 @@
                           :else :no-digest-file)
            :hash hash :cas-hash cas-hash :form form :var var :ns-effect? ns-effect?)
     (fs/create-dirs config/cache-dir)
-    (cond-> (or (when-let [result-last-run (and (not no-cache?) (get-in results-last-run [hash :nextjournal/value]))]
-                  (wrapped-with-metadata result-last-run visibility hash))
+    (cond-> (or (when-let [blob->result (and (not no-cache?) (get-in blob->result [hash :nextjournal/value]))]
+                  (wrapped-with-metadata blob->result visibility hash))
                 (when cached-result?
                   (lookup-cached-result var hash cas-hash visibility))
                 (eval+cache! form hash digest-file var no-cache? visibility))
       (seq opts-from-form-meta)
       (merge opts-from-form-meta))))
 
+
 #_(eval-file "notebooks/test123.clj")
 #_(eval-file "notebooks/how_clerk_works.clj")
-#_(read+eval-cached {} {} #{:show} "(subs (slurp \"/usr/share/dict/words\") 0 1000)")
 
 (defn clear-cache! []
   (swap! webserver/!doc dissoc :blob->result)
@@ -177,7 +177,7 @@
   (let [{:as evaluated-doc :keys [blob-ids]}
         (reduce (fn [{:as acc :keys [blob->result]} {:as cell :keys [type]}]
                   (let [{:as result :nextjournal/keys [blob-id]} (when (= :code type)
-                                                                   (read+eval-cached blob->result ->hash visibility cell))]
+                                                                   (read+eval-cached analyzed-doc cell))]
                     (cond-> (update acc :blocks conj (cond-> cell result (assoc :result result)))
                       blob-id (update :blob-ids conj blob-id)
                       blob-id (assoc-in [:blob->result blob-id] result))))

src/nextjournal/clerk/hashing.clj

@@ -40,11 +40,15 @@
                     ana/analyze
                     (ana.passes.ef/emit-form #{:hygenic :qualified-symbols})))
 
+(defn ns? [form]
+  (and (seq? form) (= 'ns (first form))))
+
 (defn no-cache? [form]
   (let [var-or-form    (if-let [vn (var-name form)] vn form)
         no-cache-meta? (comp boolean :nextjournal.clerk/no-cache meta)]
     (or (no-cache-meta? var-or-form)
-        (no-cache-meta? *ns*)
+        (when-not (ns? form)
+          (no-cache-meta? *ns*))
         (and (seq? form) (= `deref (first form))))))
 
 #_(no-cache? '(rand-int 10))
@@ -95,7 +99,8 @@
           deps (cond-> (var-dependencies analyzed-form) var (disj var))]
       (cond-> {:form form
                ;; TODO: drop var downstream so hash stays stable under change
-               :ns-effect? (some? (some #{'clojure.core/require 'clojure.core/in-ns} deps))}
+               :ns-effect? (some? (some #{'clojure.core/require 'clojure.core/in-ns} deps))
+               :no-cache? (no-cache? form)}
         var (assoc :var var)
         (seq deps) (assoc :deps deps)))))
 
@@ -116,9 +121,6 @@
 (defn remove-leading-semicolons [s]
   (str/replace s #"^[;]+" ""))
 
-(defn ns? [form]
-  (and (list? form) (= 'ns (first form))))
-
 (defn ->visibility [form]
   (when-let [visibility (-> form meta :nextjournal.clerk/visibility)]
     (let [visibility-set (cond-> visibility (not (set? visibility)) hash-set)]
@@ -303,13 +305,15 @@
                                  (assoc-in [:->analysis-info (if var var form)] (cond-> analyzed
                                                                                   (:file doc) (assoc :file (:file doc)))))
                        state (cond-> state
-                               doc? (update-in [:blocks i] merge (dissoc analyzed :deps)))]
+                               doc? (update-in [:blocks i] merge (dissoc analyzed :deps :no-cache? :ns-effect?)))]
                    (when ns-effect?
                      (eval form))
                    (if (seq deps)
-                     (reduce (partial analyze-deps var form)
-                             state
-                             deps)
+                     (-> (reduce (partial analyze-deps var form)
+                                 state
+                                 deps)
+                         (update-in [:->analysis-info (if var var form) :no-cache?]
+                                    (fn [no-cache?] (or no-cache? (boolean (some (fn [dep] (get-in state [:->analysis-info dep :no-cache?])) deps))))))
                      state)))))
            (cond-> state
              doc? (merge doc))
@@ -319,6 +323,8 @@
 #_(let [doc (parse-clojure-string {:doc? true} "(ns foo) (def a 41) (def b (inc a))")]
     (analyze-doc doc))
 
+*ns*
+
 (defn analyze-file
   ([file] (analyze-file {:graph (dep/graph)} file))
   ([state file] (analyze-doc state (parse-file {} file))))
@@ -394,9 +400,6 @@
 #_(find-location 'io.methvin.watcher.hashing.FileHasher/DEFAULT_FILE_HASHER)
 #_(find-location 'String)
 
-
-(symbol->jar 'java.net.http.HttpClient)
-
 (def hash-jar
   (memoize (fn [f]
              {:jar f :hash (sha1-base58 (io/input-stream f))})))
@@ -442,10 +445,7 @@
           {}
           (dep/topo-sort graph)))
 
-#_(hash "notebooks/hello.clj")
-#_(hash "notebooks/elements.clj")
-#_(clojure.data/diff (hash "notebooks/how_clerk_works.clj")
-                     (hash "notebooks/how_clerk_works.clj"))
+#_(hash (build-graph (parse-clojure-string (slurp "notebooks/hello.clj"))))
 
 (defn exceeds-bounded-count-limit? [x]
   (reduce (fn [_ xs]

test/nextjournal/clerk/hashing_test.clj

@@ -175,6 +175,16 @@
               (analyze-string "^:nextjournal.clerk/no-cache (ns example-notebook)
 #{3 1 2}"))))
 
+(deftest no-cache-dep
+  (is (match? [{:no-cache? true} {:no-cache? true} {:no-cache? true}]
+              (->> "(def ^:nextjournal.clerk/no-cache my-uuid
+  (java.util.UUID/randomUUID))
+(str my-uuid)
+my-uuid"
+                   analyze-string
+                   h/analyze-doc
+                   :->analysis-info
+                   vals))))
 
 (deftest circular-dependency
   (is (match? {:graph {:dependencies {'(ns circular) any?
@@ -187,3 +197,7 @@
 (declare a)
 (def b (str a \" boom\"))
 (def a (str \"boom \" b))"))))
+
+
+
+

test/nextjournal/clerk_test.clj

@@ -66,29 +66,35 @@
         (is (= 99 @(find-var 'my-test-ns/some-var))))))
 
   (testing "random expression gets cached"
-    (is (= (clerk/eval-string "(ns my-random-test-ns) (java.util.UUID/randomUUID)")
-           (clerk/eval-string "(ns my-random-test-ns) (java.util.UUID/randomUUID)"))))
+    (is (= (clerk/eval-string "(ns my-random-test-ns-1) (rand-int 1000000)")
+           (clerk/eval-string "(ns my-random-test-ns-1) (rand-int 1000000)"))))
 
   (testing "random expression that cannot be serialized in nippy gets cached in memory"
-    (let [{:as result :keys [blob->result]} (clerk/eval-string "(ns my-random-test-ns) {inc (java.util.UUID/randomUUID)}")]
+    (let [{:as result :keys [blob->result]} (clerk/eval-string "(ns my-random-test-ns-2) {inc (java.util.UUID/randomUUID)}")]
       (is (= result
-             (clerk/eval-string blob->result "(ns my-random-test-ns) {inc (java.util.UUID/randomUUID)}")))))
+             (clerk/eval-string blob->result "(ns my-random-test-ns-2) {inc (java.util.UUID/randomUUID)}")))))
 
   (testing "random expression doesn't get cached with no-cache"
-    (is (not= (clerk/eval-string "(ns ^:nextjournal.clerk/no-cache my-random-test-ns) (java.util.UUID/randomUUID)")
-              (clerk/eval-string "(ns ^:nextjournal.clerk/no-cache my-random-test-ns) (java.util.UUID/randomUUID)"))))
+    (is (not= (clerk/eval-string "(ns ^:nextjournal.clerk/no-cache my-random-test-ns-3) (java.util.UUID/randomUUID)")
+              (clerk/eval-string "(ns ^:nextjournal.clerk/no-cache my-random-test-ns-3) (java.util.UUID/randomUUID)"))))
+
+  (testing "random dependent expression doesn't get cached with no-cache"
+    (let [code "(ns my-random-test-ns-4) (def ^:nextjournal.clerk/no-cache my-uuid (java.util.UUID/randomUUID)) (str my-uuid)"
+          eval+get-last-block-val (fn [] (-> code clerk/eval-string :blocks peek :result :nextjournal/value))]
+      (is (not= (eval+get-last-block-val)
+                (eval+get-last-block-val)))))
 
   (testing "random expression that cannot be frozen with nippy gets cached via in-memory cache"
-    (let [code "(ns my-random-test-ns) {:my-fn inc :my-uuid (java.util.UUID/randomUUID)}"
+    (let [code "(ns my-random-test-ns-5) {:my-fn inc :my-uuid (java.util.UUID/randomUUID)}"
           result  (clerk/eval-string code)
           result' (clerk/eval-string (:blob->result result) code)
           extract-my-uuid #(-> % :blocks last :result :nextjournal/value :my-uuid)]
       (is (= (extract-my-uuid result)
              (extract-my-uuid result')))))
 
   (testing "old values are cleared from in-memory cache"
-    (let [{:keys [blob->result]} (clerk/eval-string "(ns my-random-test-ns) ^:nextjournal.clerk/no-cache {inc (java.util.UUID/randomUUID)}")]
-      (is (= 2 (count (:blob->result (clerk/eval-string blob->result "(ns my-random-test-ns) {inc (java.util.UUID/randomUUID)}")))))))
+    (let [{:keys [blob->result]} (clerk/eval-string "(ns my-random-test-ns-6) ^:nextjournal.clerk/no-cache {inc (java.util.UUID/randomUUID)}")]
+      (is (= 2 (count (:blob->result (clerk/eval-string blob->result "(ns my-random-test-ns-6) {inc (java.util.UUID/randomUUID)}")))))))
 
   (testing "defonce returns correct result on subsequent evals (when defonce would eval to nil)"
     (clerk/eval-string "(ns ^:nextjournal.clerk/no-cache my-defonce-test-ns) (defonce state (atom {}))")