Skip to content

Commit

Permalink
Sketch fix for #289
Browse files Browse the repository at this point in the history
  • Loading branch information
@mk
mk committed Nov 21, 2022
1 parent 6c0a44a commit d47f8c1
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 92 deletions.
2 changes: 1 addition & 1 deletion resources/viewer-js-hash
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2RLfgCuCanXq34BD48eoVxVa4nmH
3qpLvPwx18MsC4zPMtYMe6inft8f
113 changes: 56 additions & 57 deletions src/nextjournal/clerk/analyzer.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,6 @@
(defn- circular-dependency-error? [e]
(-> e ex-data :reason #{::dep/circular-dependency}))

(defn ->key [{:as _analyzed :keys [vars deps form]}])

(defn- analyze-circular-dependency [state vars form dep {:keys [node dependency]}]
(let [rec-form (concat '(do) [form (get-in state [:->analysis-info dependency :form])])
rec-var (symbol (str/join "+" (sort (conj vars dep))))
Expand All @@ -200,12 +198,18 @@
(dep/depend dep rec-var)))
(assoc-in [:->analysis-info rec-var :form] rec-form))))

(defn ->ana-keys [{:as _analyzed :keys [form vars]}]
(if (seq vars) vars [form]))

(defn- analyze-deps [{:as analyzed :keys [form vars]} state dep]
(defn ->key [{:as codeblock :keys [var id]}]
(if var var id))

(defn ->ana-keys [{:as analyzed :keys [form vars id]}]
(if (seq vars) vars [(->key analyzed)]))

#_(->> (nextjournal.clerk.eval/eval-string "(rand-int 100) (rand-int 100) (rand-int 100)") :blocks (mapv #(-> % :result :nextjournal/value)))

(defn- analyze-deps [{:as analyzed :keys [form vars id]} state dep]
(try (reduce (fn [state var]
(update state :graph #(dep/depend % (if var var form) dep)))
(update state :graph #(dep/depend % (->key analyzed) dep)))
state
(->ana-keys analyzed))
(catch Exception e
Expand All @@ -220,24 +224,19 @@
(reduce (fn [state k]
(assoc-in state [:->analysis-info k :no-cache?] no-cache-deps?)) state (->ana-keys analyzed)))))

(defn add-block-id [{:as state :keys [id->count]} {:as block :keys [var form type doc]}]
(let [id (if var
(defn get-block-id [!id->count {:as block :keys [var form type doc]}]
(let [id->count @!id->count
id (if var
var
(let [hash-fn #(-> % nippy/fast-freeze digest/sha1 multihash/base58)]
(symbol (str *ns*)
(case type
:code (str "anon-expr-" (hash-fn (cond-> form (instance? clojure.lang.IObj form) (with-meta {}))))
:markdown (str "markdown-" (hash-fn doc))))))
unique-id (if (id->count id)
(symbol (str *ns*) (str (name id) "#" (inc (id->count id))))
id)]
(-> state
(update :blocks conj (assoc block :id unique-id))
(update :id->count update id (fnil inc 0)))))

(defn add-block-ids [{:as analyzed-doc :keys [blocks]}]
(-> (reduce add-block-id (assoc analyzed-doc :blocks [] :id->count {} ) blocks)
(dissoc :id->count)))
:markdown (str "markdown-" (hash-fn doc))))))]
(swap! !id->count update id (fnil inc 0))
(if (id->count id)
(symbol (str *ns*) (str (name id) "#" (inc (id->count id))))
id)))

(defn ^:private internal-proxy-name?
"Returns true if `sym` represents a var name interned by `clojure.core/proxy`."
Expand All @@ -259,38 +258,43 @@
(analyze-doc {:doc? true :graph (dep/graph)} doc))
([{:as state :keys [doc?]} doc]
(binding [*ns* *ns*]
(cond-> (reduce (fn [state i]
(let [{:keys [type text loc]} (get-in state [:blocks i])]
(if (not= type :code)
state
(let [form (read-string text)
form+loc (cond-> form
(instance? clojure.lang.IObj form)
(vary-meta merge (cond-> loc
(:file doc) (assoc :clojure.core/eval-file (str (:file doc))))))
{:as analyzed :keys [vars deps ns-effect?]} (cond-> (analyze form+loc)
(:file doc) (assoc :file (:file doc)))
_ (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*`
(eval form))
state (cond-> (reduce (fn [state ana-key]
(assoc-in state [:->analysis-info ana-key] analyzed))
(dissoc state :doc?)
(->ana-keys analyzed))
doc? (update-in [:blocks i] merge (dissoc analyzed :deps :no-cache? :ns-effect?))
(and doc? (not (contains? state :ns))) (merge (parser/->doc-settings form) {:ns *ns*}))]
(when (:ns? state)
(throw-if-dep-is-missing doc state analyzed))
(if (seq deps)
(-> (reduce (partial analyze-deps analyzed) state deps)
(make-deps-inherit-no-cache analyzed))
state)))))
(cond-> state
doc? (merge doc))
(-> doc :blocks count range))
doc? (-> add-block-ids
parser/add-block-visibility
parser/add-open-graph-metadata
parser/add-auto-expand-results)))))
(let [!id->count (atom {})]
(cond-> (reduce (fn [state i]
(let [{:as block :keys [type text loc]} (get-in state [:blocks i])]
(if (not= type :code)
state
(let [form (read-string text)
form+loc (cond-> form
(instance? clojure.lang.IObj form)
(vary-meta merge (cond-> loc
(:file doc) (assoc :clojure.core/eval-file (str (:file doc))))))
{:as analyzed :keys [vars deps ns-effect?]} (cond-> (analyze form+loc)
(:file doc) (assoc :file (:file doc)))
_ (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*`
(eval form))
block-id (get-block-id !id->count (merge analyzed block))
analyzed (assoc analyzed :id block-id)
state (cond-> (reduce (fn [state ana-key]
(assoc-in state [:->analysis-info ana-key] analyzed))
(dissoc state :doc?)
(->ana-keys analyzed))
doc? (update-in [:blocks i] merge (dissoc analyzed :deps :no-cache? :ns-effect?))
(and doc? (not (contains? state :ns))) (merge (parser/->doc-settings form) {:ns *ns*}))]
(when (:ns? state)
(throw-if-dep-is-missing doc state analyzed))
(if (seq deps)
(-> (reduce (partial analyze-deps analyzed) state deps)
(make-deps-inherit-no-cache analyzed))
state)))))
(cond-> state
doc? (merge doc))
(-> doc :blocks count range))
doc? (-> parser/add-block-visibility
parser/add-open-graph-metadata
parser/add-auto-expand-results))))))

#_(let [parsed (nextjournal.clerk.parser/parse-clojure-string "clojure.core/dec")]
(build-graph (analyze-doc parsed)))

(defn analyze-file
([file] (analyze-file {:graph (dep/graph)} file))
Expand Down Expand Up @@ -398,7 +402,7 @@
#_(dep/immediate-dependencies (:graph (build-graph "src/nextjournal/clerk/analyzer.clj")) #'nextjournal.clerk.analyzer/long-thing)
#_(dep/transitive-dependencies (:graph (build-graph "src/nextjournal/clerk/analyzer.clj")) #'nextjournal.clerk.analyzer/long-thing)

(defn hash-codeblock [->hash {:as codeblock :keys [hash form deps vars]}]
(defn hash-codeblock [->hash {:as codeblock :keys [hash form id deps vars]}]
(let [->hash' (if (and (not (ifn? ->hash)) (seq deps))
(binding [*out* *err*]
(println "->hash must be `ifn?`" {:->hash ->hash :codeblock codeblock})
Expand All @@ -409,8 +413,6 @@
(pr-str (set/union (conj hashed-deps (if form form hash))
vars))))))

#_(nextjournal.clerk/build-static-app! {:paths nextjournal.clerk/clerk-docs})

(defn hash
([{:as analyzed-doc :keys [graph]}] (hash analyzed-doc (dep/topo-sort graph)))
([{:as analyzed-doc :keys [->analysis-info graph]} deps]
Expand All @@ -422,9 +424,6 @@
->hash)))
deps)))

(binding [*ns* (create-ns 'my-foo)]
(analyze-doc (parser/parse-clojure-string "(ns my-foo) `bar")))

#_(hash (build-graph (parser/parse-clojure-string "^{:nextjournal.clerk/hash-fn (fn [x] \"abc\")}(def contents (slurp \"notebooks/hello.clj\"))")))
#_(hash (build-graph (parser/parse-clojure-string (slurp "notebooks/hello.clj"))))

Expand Down
7 changes: 4 additions & 3 deletions src/nextjournal/clerk/eval.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@
(defn ^:private eval+cache! [{:keys [form var ns-effect? no-cache? freezable?] :as form-info} hash digest-file]
(try
(let [{:keys [result]} (time-ms (binding [config/*in-clerk* true]
(assert form "form must be set")
(eval form)))
result (if (and (nil? result) var (= 'defonce (first form)))
(find-var var)
Expand Down Expand Up @@ -139,10 +140,10 @@
(update :nextjournal/viewers eval)))

(defn read+eval-cached [{:as _doc :keys [blob->result ->analysis-info ->hash]} codeblock]
(let [{:keys [form vars var deref-deps]} codeblock
{:as form-info :keys [ns-effect? no-cache? freezable?]} (->analysis-info (if (seq vars) (first vars) form))
(let [{:keys [form vars var id deref-deps]} codeblock
{:as form-info :keys [ns-effect? no-cache? freezable?]} (->analysis-info (if (seq vars) (first vars) (analyzer/->key codeblock)))
no-cache? (or ns-effect? no-cache?)
hash (when-not no-cache? (or (get ->hash (if var var form))
hash (when-not no-cache? (or (get ->hash (analyzer/->key codeblock))
(analyzer/hash-codeblock ->hash codeblock)))
digest-file (when hash (->cache-file (str "@" hash)))
cas-hash (when (and digest-file (fs/exists? digest-file)) (slurp digest-file))
Expand Down
56 changes: 25 additions & 31 deletions test/nextjournal/clerk/analyzer_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,38 +172,33 @@
ana/analyze-doc))

(deftest analyze-doc
(is (match? {:graph {:dependencies {'(ns example-notebook) set?}
:dependents map?}
:blocks [{:type :code
:text "^:nextjournal.clerk/no-cache (ns example-notebook)"
:form '(ns example-notebook)}
{:type :code
:text "#{3 1 2}"
:form #{1 2 3}}]
:->analysis-info {'(ns example-notebook) {:form '(ns example-notebook),
:deps set?}
#{1 3 2} {:form '#{1 3 2}}}}
(analyze-string "^:nextjournal.clerk/no-cache (ns example-notebook)
#{3 1 2}")))

(testing "preserves *ns*"
(with-ns-binding 'nextjournal.clerk.analyzer-test
(is (= (find-ns 'nextjournal.clerk.analyzer-test)
(do (analyze-string ";; boo\n\n (ns example-notebook)") *ns*)))))
(is (match? #{{:form '(ns example-notebook),
:deps set?}
{:form '#{1 3 2}}}
(-> "^:nextjournal.clerk/no-cache (ns example-notebook)
#{3 1 2}"
analyze-string :->analysis-info vals set))))

(testing "preserves *ns*"
(with-ns-binding 'nextjournal.clerk.analyzer-test
(is (= (find-ns 'nextjournal.clerk.analyzer-test)
(do (analyze-string ";; boo\n\n (ns example-notebook)") *ns*)))))

(testing "should fail when var is only present at runtime but not in file"
(let [ns (create-ns 'missing-var)]
(intern (create-ns 'missing-var) 'foo :bar)
(is (thrown? Exception (analyze-string "(ns missing-var) foo")))))
(testing "should fail when var is only present at runtime but not in file"
(let [ns (create-ns 'missing-var)]
(intern (create-ns 'missing-var) 'foo :bar)
(is (thrown? Exception (analyze-string "(ns missing-var) foo")))))

(testing "should not fail on var present at runtime if there's no ns form"
(let [ns (create-ns 'existing-var)]
(intern (create-ns 'existing-var) 'foo :bar)
(analyze-string "(in-ns 'existing-var) foo")))
(testing "should not fail on var present at runtime if there's no ns form"
(let [ns (create-ns 'existing-var)]
(intern (create-ns 'existing-var) 'foo :bar)
(analyze-string "(in-ns 'existing-var) foo")))

(testing "defmulti has no deref deps"
(is (empty? (-> "(defmulti foo :bar)" analyze-string :blocks first :deref-deps)))))
(testing "defmulti has no deref deps"
(is (empty? (-> "(defmulti foo :bar)" analyze-string :blocks first :deref-deps))))

(testing "can analyze plain var reference (issue #289)"
(ana/build-graph (analyze-string "clojure.core/inc")))

(deftest add-block-ids
(testing "assigns block ids"
Expand All @@ -230,9 +225,8 @@ my-uuid"
(is (analyze-string "(ns proxy-example-notebook) (proxy [clojure.lang.ISeq][] (seq [] '(this is a test seq)))")))

(deftest circular-dependency
(is (match? {:graph {:dependencies {'(ns circular) any?
'circular/b #{'clojure.core/str 'circular/a+circular/b}
'circular/a #{'clojure.core/declare 'clojure.core/str 'circular/a+circular/b}}}
(is (match? {:graph {:dependencies {'circular/b #{'clojure.core/str 'circular/a+circular/b}
'circular/a #{#_'clojure.core/declare 'clojure.core/str 'circular/a+circular/b}}}
:->analysis-info {'circular/a any?
'circular/b any?
'circular/a+circular/b {:form '(do (def a (str "boom " b)) (def b (str a " boom")))}}}
Expand Down

0 comments on commit d47f8c1

Please sign in to comment.