From 4d49931dfccee1a695206648673919a9a8906e9c Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Tue, 30 May 2023 17:17:22 +0200 Subject: [PATCH 01/23] Highlight non-clojure languages --- notebooks/markdown_fences.md | 31 ++++++++++++++++++++++---- src/nextjournal/clerk/render/code.cljs | 31 ++++++++++++++++++-------- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/notebooks/markdown_fences.md b/notebooks/markdown_fences.md index 486beb917..938872537 100644 --- a/notebooks/markdown_fences.md +++ b/notebooks/markdown_fences.md @@ -1,27 +1,50 @@ # 🀺 Markdown Fences +## Handling Clojure blocks ``` '(evaluated :and "highlighted") ``` +```clj +'(evaluated :and "highlighted") +``` + ```clojure '(evaluated :and "highlighted") ``` ```clojure {:nextjournal.clerk/code-listing true} -'(1 2 "not evaluated" :but-still-highlighted) +(1 2 "not evaluated" :but-still-highlighted) ``` -```clojure {:nextjournal.clerk/code-listing true} -'(1 2 "not evaluated" :but-still-highlighted) +```{:nextjournal.clerk/code-listing true} +(1 2 "not evaluated" :but-still-highlighted) ``` +## πŸ³οΈβ€πŸŒˆ Polyglot Highlighting + +```edn +(1 2 "not evaluated" :but-still-highlighted) +``` + +javascript + ```js () => { if (true) { return 'not evaluated' } else { - return 'what' + return 123 } } ``` + +python + +```python +class Foo(object): + def __init__(self): + pass + def do_this(self): + return 1 +``` diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index 82c7242a9..987f59105 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -1,5 +1,5 @@ (ns nextjournal.clerk.render.code - (:require ["@codemirror/language" :refer [HighlightStyle syntaxHighlighting]] + (:require ["@codemirror/language" :refer [HighlightStyle syntaxHighlighting LanguageDescription]] ["@codemirror/state" :refer [EditorState RangeSetBuilder Text]] ["@codemirror/view" :refer [EditorView Decoration]] ["@lezer/highlight" :refer [tags highlightTree]] @@ -7,7 +7,8 @@ [applied-science.js-interop :as j] [clojure.string :as str] [nextjournal.clerk.render.hooks :as hooks] - [nextjournal.clojure-mode :as clojure-mode])) + [nextjournal.clojure-mode :as clojure-mode] + [shadow.esm])) (def highlight-style (.define HighlightStyle @@ -87,10 +88,24 @@ (< pos to) (concat [(.sliceString text pos to)])))))))) -(defn lang->deco-range [lang code] - (let [builder (RangeSetBuilder.)] - (when lang - (highlightTree (.. lang -parser (parse code)) highlight-style +(defn matching-language-parser [language] + (if (= "clojure" language) + (js/Promise.resolve (.-parser clojureLanguage)) + (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0") + (then (fn [mod] + (when-some [langs (.-languages mod)] + (when-some [^js matching (or (.matchFilename LanguageDescription langs language) + (.matchLanguageName LanguageDescription langs language))] + (.load matching))))) + (then (fn [lang] (when lang + (js/console.log :found language lang) + (.. lang -language -parser))))))) + +(defn lang->deco-range [language code] + (let [^js builder (RangeSetBuilder.) + ^js parser (hooks/use-promise (matching-language-parser language))] + (when parser + (highlightTree (.parse parser code) highlight-style (fn [from to style] (.add builder from to (.mark Decoration (j/obj :class style)))))) (.finish builder))) @@ -100,9 +115,7 @@ [:div.cm-editor [:cm-scroller (into [:div.cm-content.whitespace-pre] - (map (partial style-line - ;; TODO: use-promise hook resolving to language data according to @codemirror/language-data - (lang->deco-range (when (= "clojure" language) clojureLanguage) code) text)) + (map (partial style-line (lang->deco-range language code) text)) (range 1 (inc (.-lines text))))]])) ;; editable code viewer From 16f4206808e34e16f1c174e2f037f1be8d376fc5 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Tue, 30 May 2023 17:26:41 +0200 Subject: [PATCH 02/23] Match language by filename --- notebooks/markdown_fences.md | 13 ++++++++++++- src/nextjournal/clerk/render/code.cljs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/notebooks/markdown_fences.md b/notebooks/markdown_fences.md index 938872537..816a12235 100644 --- a/notebooks/markdown_fences.md +++ b/notebooks/markdown_fences.md @@ -41,10 +41,21 @@ javascript python -```python +```py class Foo(object): def __init__(self): pass def do_this(self): return 1 ``` + +C++ + +```c++ +#include + +int main() { + std::cout << "Hello, world!" << std::endl; + return 0; +} +``` diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index 987f59105..8842779f4 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -94,8 +94,8 @@ (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0") (then (fn [mod] (when-some [langs (.-languages mod)] - (when-some [^js matching (or (.matchFilename LanguageDescription langs language) - (.matchLanguageName LanguageDescription langs language))] + (when-some [^js matching (or (.matchLanguageName LanguageDescription langs language) + (.matchFilename LanguageDescription langs (str "code." language)))] (.load matching))))) (then (fn [lang] (when lang (js/console.log :found language lang) From 5ca7896b3005d38942a90b526e7e13b6b755123a Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Tue, 30 May 2023 17:32:40 +0200 Subject: [PATCH 03/23] Use our parser when possible --- notebooks/markdown_fences.md | 6 ++++-- src/nextjournal/clerk/render/code.cljs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/notebooks/markdown_fences.md b/notebooks/markdown_fences.md index 816a12235..0593276da 100644 --- a/notebooks/markdown_fences.md +++ b/notebooks/markdown_fences.md @@ -23,11 +23,13 @@ ## πŸ³οΈβ€πŸŒˆ Polyglot Highlighting +EDN + ```edn (1 2 "not evaluated" :but-still-highlighted) ``` -javascript +Javascript ```js () => { @@ -39,7 +41,7 @@ javascript } ``` -python +Python ```py class Foo(object): diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index 8842779f4..b915d986a 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -89,7 +89,7 @@ (concat [(.sliceString text pos to)])))))))) (defn matching-language-parser [language] - (if (= "clojure" language) + (if (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) (js/Promise.resolve (.-parser clojureLanguage)) (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0") (then (fn [mod] From 961e06eafcecdbb6d20d42f25d9890a477875237 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Tue, 30 May 2023 18:26:33 +0200 Subject: [PATCH 04/23] Normalize language from info string --- notebooks/markdown_fences.md | 8 +++--- src/nextjournal/clerk/parser.cljc | 45 ++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/notebooks/markdown_fences.md b/notebooks/markdown_fences.md index 0593276da..0db548f0a 100644 --- a/notebooks/markdown_fences.md +++ b/notebooks/markdown_fences.md @@ -6,17 +6,19 @@ ``` ```clj -'(evaluated :and "highlighted") +'(evaluated :and "highlighted" :language clj) ``` ```clojure -'(evaluated :and "highlighted") +'(evaluated :and "highlighted" :language clojure) ``` +Use `{:nextjournal.clerk/code-listing true}` in the fence info to signal that a block should not be evaluated. + ```clojure {:nextjournal.clerk/code-listing true} (1 2 "not evaluated" :but-still-highlighted) ``` - +if no language is specified we assume it's a clojure cell ```{:nextjournal.clerk/code-listing true} (1 2 "not evaluated" :but-still-highlighted) ``` diff --git a/src/nextjournal/clerk/parser.cljc b/src/nextjournal/clerk/parser.cljc index 9e7f6f45d..3766d6026 100644 --- a/src/nextjournal/clerk/parser.cljc +++ b/src/nextjournal/clerk/parser.cljc @@ -357,24 +357,45 @@ :nodes (rest nodes) ::md-slice [])) -(defn fenced-clojure-code-block? [{:as block :keys [type info language]}] - (and (code? block) - (or (empty? language) - (re-matches #"clj(c?)|clojure" language)) - (not (:nextjournal.clerk/code-listing (let [parsed (p/parse-string-all (subs info (count language)))] - (when (n/sexpr-able? parsed) - (n/sexpr parsed))))))) +(defn parse-code-info-string + "https://spec.commonmark.org/0.30/#code-fence" + [{:as node :keys [info]}] + (if-not (code? node) + {:eval? false} + (try + ;; TODO: really? + (let [sexprs (n/child-sexprs (p/parse-string-all info)) + language (some #(and (symbol? %) %) sexprs) + metadata (some #(and (map? %) %) sexprs)] + {:language (if language (name language) "clojure") + :metadata metadata + :eval? (and (or (not language) (#{'clojure 'clojurescript 'clj 'cljs 'cljc} language)) + (not (:nextjournal.clerk/code-listing metadata)))}) + (catch #?(:clj Throwable :cljs :default) cause + (throw (ex-info "We allow info strings of the following shape: +```language {map with clerk metadata} +code +```" node cause)))))) + +#_ (parse-code-info-string {:type :heading}) +#_ (parse-code-info-string {:type :code :info "clojure"}) +#_ (parse-code-info-string {:type :code :info " clojure"}) +#_ (parse-code-info-string {:type :code :info "clojure {"}) +#_ (parse-code-info-string {:type :code :info "clojure &@foo !!!"}) +#_ (parse-code-info-string {:type :code :info "clojure {:nextjournal.clerk/code-listing true}"}) +#_ (ex-message *e) (defn parse-markdown-string [{:as opts :keys [doc?]} s] (let [{:as ctx :keys [content]} (parse-markdown (markdown-context) s)] (loop [{:as state :keys [nodes] ::keys [md-slice]} {:blocks [] ::md-slice [] :nodes content :md-context ctx}] (if-some [node (first nodes)] (recur - (if (fenced-clojure-code-block? node) - (-> state - (update :blocks #(cond-> % (seq md-slice) (conj {:type :markdown :doc {:type :doc :content md-slice}}))) - (parse-markdown-cell opts)) - (-> state (update :nodes rest) (cond-> doc? (update ::md-slice conj node))))) + (let [{:keys [language eval?]} (parse-code-info-string node)] + (if eval? + (-> state + (update :blocks #(cond-> % (seq md-slice) (conj {:type :markdown :doc {:type :doc :content md-slice}}))) + (parse-markdown-cell opts)) + (-> state (update :nodes rest) (cond-> doc? (update ::md-slice conj (cond-> node language (assoc :language language)))))))) (-> state (update :blocks #(cond-> % (seq md-slice) (conj {:type :markdown :doc {:type :doc :content md-slice}}))) From 8e56a5b4cc437e8963239f81827d6d1d7f46f96a Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Tue, 30 May 2023 18:28:15 +0200 Subject: [PATCH 05/23] Docs --- src/nextjournal/clerk/builder.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nextjournal/clerk/builder.clj b/src/nextjournal/clerk/builder.clj index c64bb3cc6..d7365483b 100644 --- a/src/nextjournal/clerk/builder.clj +++ b/src/nextjournal/clerk/builder.clj @@ -20,6 +20,7 @@ (into ["CHANGELOG.md" "README.md" "notebooks/markdown.md" + "notebooks/markdown_fences.md" "notebooks/onwards.md"] (map #(str "notebooks/" % ".clj")) ["cards" From 19728844a03c2dfbeedf10159ae3bd8bf4f8fa3d Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Tue, 30 May 2023 18:36:59 +0200 Subject: [PATCH 06/23] Defend --- src/nextjournal/clerk/parser.cljc | 1 + src/nextjournal/clerk/render/code.cljs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nextjournal/clerk/parser.cljc b/src/nextjournal/clerk/parser.cljc index 3766d6026..7999adc2e 100644 --- a/src/nextjournal/clerk/parser.cljc +++ b/src/nextjournal/clerk/parser.cljc @@ -383,6 +383,7 @@ code #_ (parse-code-info-string {:type :code :info "clojure {"}) #_ (parse-code-info-string {:type :code :info "clojure &@foo !!!"}) #_ (parse-code-info-string {:type :code :info "clojure {:nextjournal.clerk/code-listing true}"}) +#_ (parse-code-info-string {:type :code :info "{:nextjournal.clerk/code-listing true} c++"}) #_ (ex-message *e) (defn parse-markdown-string [{:as opts :keys [doc?]} s] diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index b915d986a..a9de3db8c 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -99,7 +99,8 @@ (.load matching))))) (then (fn [lang] (when lang (js/console.log :found language lang) - (.. lang -language -parser))))))) + (.. lang -language -parser)))) + (catch (fn [err] (js/console.warn err)))))) (defn lang->deco-range [language code] (let [^js builder (RangeSetBuilder.) From b33f7db477fcc54e04967050efe0146886116588 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Wed, 31 May 2023 09:50:34 +0200 Subject: [PATCH 07/23] Do not eval cljs server-side --- src/nextjournal/clerk/parser.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nextjournal/clerk/parser.cljc b/src/nextjournal/clerk/parser.cljc index 7999adc2e..7198903db 100644 --- a/src/nextjournal/clerk/parser.cljc +++ b/src/nextjournal/clerk/parser.cljc @@ -369,7 +369,7 @@ metadata (some #(and (map? %) %) sexprs)] {:language (if language (name language) "clojure") :metadata metadata - :eval? (and (or (not language) (#{'clojure 'clojurescript 'clj 'cljs 'cljc} language)) + :eval? (and (or (not language) (#{'clojure 'clj 'cljc} language)) (not (:nextjournal.clerk/code-listing metadata)))}) (catch #?(:clj Throwable :cljs :default) cause (throw (ex-info "We allow info strings of the following shape: From 4e2fa72101da2fcdfeb3cb7397620e5d40c3b25c Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Wed, 31 May 2023 10:18:20 +0200 Subject: [PATCH 08/23] Handle missing language --- notebooks/cherry.clj | 9 ++++----- src/nextjournal/clerk/render/code.cljs | 14 ++++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/notebooks/cherry.clj b/notebooks/cherry.clj index f8960d90e..7d458bd59 100644 --- a/notebooks/cherry.clj +++ b/notebooks/cherry.clj @@ -70,11 +70,10 @@ [:div [:div.flex [:div.viewer-code.flex-auto.w-80.mb-2 [nextjournal.clerk.render.code/editor !input]] - [:button.flex-none.bg-slate-100.mb-2.pl-2.pr-2 - {:on-click click-handler} - "Compile!"]] - [:div.bg-slate-50 - [nextjournal.clerk.render/render-code @!compiled]] + [:button.flex-none.rounded-md.border.border-slate-200.bg-slate-100.mb-2.pl-2.pr-2.font-sans + {:on-click click-handler} "Compile!"]] + [:div.bg-slate-100.p-2.border.border-slate-200 + [nextjournal.clerk.render/render-code @!compiled {:language "js"}]] [nextjournal.clerk.render/inspect (try (js/eval @!compiled) (catch :default e e))]])))} diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index a9de3db8c..bdfdea6d8 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -89,18 +89,20 @@ (concat [(.sliceString text pos to)])))))))) (defn matching-language-parser [language] - (if (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) + (cond + (not language) + (js/Promise.resolve nil) + (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) (js/Promise.resolve (.-parser clojureLanguage)) + :else (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0") - (then (fn [mod] + (then (fn [^js mod] (when-some [langs (.-languages mod)] (when-some [^js matching (or (.matchLanguageName LanguageDescription langs language) (.matchFilename LanguageDescription langs (str "code." language)))] (.load matching))))) - (then (fn [lang] (when lang - (js/console.log :found language lang) - (.. lang -language -parser)))) - (catch (fn [err] (js/console.warn err)))))) + (then (fn [^js lang-support] (when lang-support (.. lang-support -language -parser)))) + (catch (fn [err] (js/console.warn (str "Cannot load language parser for: " language) err)))))) (defn lang->deco-range [language code] (let [^js builder (RangeSetBuilder.) From 18385627f9dcfbef51027584d0535cbf0c610e35 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Wed, 31 May 2023 10:32:23 +0200 Subject: [PATCH 09/23] Fix highlighting for folded code viewer --- src/nextjournal/clerk/render.cljs | 6 +++--- src/nextjournal/clerk/viewer.cljc | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/nextjournal/clerk/render.cljs b/src/nextjournal/clerk/render.cljs index e8bb70bb8..a3cd938fd 100644 --- a/src/nextjournal/clerk/render.cljs +++ b/src/nextjournal/clerk/render.cljs @@ -917,9 +917,9 @@ (defn render-code-block [code-string {:as opts :keys [id]}] [:div.viewer.code-viewer.w-full.max-w-wide {:data-block-id id} - [code/render-code code-string opts]]) + [code/render-code code-string (assoc opts :language "clojure")]]) -(defn render-folded-code-block [code-string {:keys [id]}] +(defn render-folded-code-block [code-string {:as opts :keys [id]}] (let [!hidden? (hooks/use-state true)] (if @!hidden? [:div.relative.pl-12.font-sans.text-slate-400.cursor-pointer.flex.overflow-y-hidden.group @@ -952,7 +952,7 @@ {:class "text-[10px]"} "evaluated in 0.2s"]] [:div.code-viewer.mb-2.relative.code-viewer.w-full.max-w-wide {:data-block-id id :style {:margin-top 0}} - [render-code code-string]]]))) + [render-code code-string (assoc opts :language "clojure")]]]))) (defn url-for [{:as src :keys [blob-id]}] diff --git a/src/nextjournal/clerk/viewer.cljc b/src/nextjournal/clerk/viewer.cljc index 5512c55dc..af52b25f4 100644 --- a/src/nextjournal/clerk/viewer.cljc +++ b/src/nextjournal/clerk/viewer.cljc @@ -598,8 +598,7 @@ code? (conj (with-viewer (if fold? `folded-code-block-viewer `code-block-viewer) {:nextjournal/opts (assoc (select-keys cell [:loc]) - :id (processed-block-id (str id "-code")) - :language "clojure")} + :id (processed-block-id (str id "-code")))} (dissoc cell :result))) (or result? eval?) (conj (cond-> (ensure-wrapped (-> cell (assoc ::doc doc) (set/rename-keys {:result ::result}))) From 65bcc9bd98f82d43acdb9b0344c638d797177c4a Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Wed, 31 May 2023 10:38:32 +0200 Subject: [PATCH 10/23] Allow to pass language to code-viewer, default to clj --- notebooks/viewers/code.clj | 16 ++++++++++++++++ src/nextjournal/clerk/viewer.cljc | 6 +++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/notebooks/viewers/code.clj b/notebooks/viewers/code.clj index ee0f5e7cb..e3648b7c2 100644 --- a/notebooks/viewers/code.clj +++ b/notebooks/viewers/code.clj @@ -36,6 +36,22 @@ with quite some whitespace ") +;; Code in some other language, say Rust: +(clerk/code {::clerk/opts {:language "rust"}} + "fn calculate_factorial(n: u32, result: &mut u32) { + if n == 0 { + *result = 1; + } else { + *result = n * calculate_factorial(n - 1, result); + } +} +fn main() { + let number = 5; + let mut result = 0; + calculate_factorial(number, &mut result); + println!(\"The factorial of {} is: {}\", number, result); +}") + ;; Editable code viewer (clerk/with-viewer '(fn [code-str _] [:div.viewer-code [nextjournal.clerk.render.code/editor (reagent.core/atom code-str)]]) diff --git a/src/nextjournal/clerk/viewer.cljc b/src/nextjournal/clerk/viewer.cljc index af52b25f4..fba8bdac1 100644 --- a/src/nextjournal/clerk/viewer.cljc +++ b/src/nextjournal/clerk/viewer.cljc @@ -913,7 +913,11 @@ (with-md-viewer)))}) (def code-viewer - {:name `code-viewer :render-fn 'nextjournal.clerk.render/render-code :transform-fn (comp mark-presented (update-val (fn [v] (if (string? v) v (str/trim (with-out-str (pprint/pprint v)))))))}) + {:name `code-viewer + :render-fn 'nextjournal.clerk.render/render-code + :transform-fn (comp mark-presented + #(update-in % [:nextjournal/opts :language] (fn [lang] (or lang "clojure"))) + (update-val (fn [v] (if (string? v) v (str/trim (with-out-str (pprint/pprint v)))))))}) (def reagent-viewer {:name `reagent-viewer :render-fn 'nextjournal.clerk.render/render-reagent :transform-fn mark-presented}) From df5c45889e01d2067e635df409186ddca3b6a1ce Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Wed, 31 May 2023 10:58:26 +0200 Subject: [PATCH 11/23] More examples --- notebooks/viewers/markdown.clj | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/notebooks/viewers/markdown.clj b/notebooks/viewers/markdown.clj index cf3133953..db7d10bbc 100644 --- a/notebooks/viewers/markdown.clj +++ b/notebooks/viewers/markdown.clj @@ -33,8 +33,8 @@ It's [Markdown](https://daringfireball.net/projects/markdown/), like you know it ;; > β€” Special Forms ;; ## Code Listings - -;; ``` +;; Clojure +;; ```clj ;; {:name :code, ;; :render-fn 'nextjournal.clerk.render/render-code, ;; :transform-fn @@ -45,7 +45,19 @@ It's [Markdown](https://daringfireball.net/projects/markdown/), like you know it ;; [v] ;; (if (string? v) v (str/trim (with-out-str (pprint/pprint v)))))))} ;; ``` - +;; APL +;; ```apl +;; numbers ← 1 2 3 4 5 +;; sum ← 0 +;; n ← β‰’numbers ⍝ Get the number of elements in the array +;; +;; :For i :In ⍳n +;; sum ← sum + numbers[i] +;; :End +;; +;; sum +;; ``` +;; ;; ## Soft vs. Hard Line Breaks ;; This one β‡₯ ;; ⇀ is a [soft break](https://spec.commonmark.org/0.30/#soft-line-breaks) and is rendered as a space. From 5787474992c85687ab55f4e9e4d10a93cd3b6607 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Wed, 31 May 2023 11:08:08 +0200 Subject: [PATCH 12/23] Parse fail example --- src/nextjournal/clerk/parser.cljc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nextjournal/clerk/parser.cljc b/src/nextjournal/clerk/parser.cljc index 7198903db..b7f80b72d 100644 --- a/src/nextjournal/clerk/parser.cljc +++ b/src/nextjournal/clerk/parser.cljc @@ -381,6 +381,8 @@ code #_ (parse-code-info-string {:type :code :info "clojure"}) #_ (parse-code-info-string {:type :code :info " clojure"}) #_ (parse-code-info-string {:type :code :info "clojure {"}) +#_ (parse-code-info-string {:type :code :info "{r, eval=FALSE}"}) ;; RMarkdown/Knitr +#_ (parse-code-info-string {:type :code :info "{r, eval=FALSE, include=TRUE}"}) ;; RMarkdown/Knitr (failing) #_ (parse-code-info-string {:type :code :info "clojure &@foo !!!"}) #_ (parse-code-info-string {:type :code :info "clojure {:nextjournal.clerk/code-listing true}"}) #_ (parse-code-info-string {:type :code :info "{:nextjournal.clerk/code-listing true} c++"}) From eb4b66fb25fce19feb53fa3ddae91dda88eb34b0 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 12:12:09 +0200 Subject: [PATCH 13/23] Move language clojure/SSR switch at component level --- src/nextjournal/clerk/render/code.cljs | 38 ++++++++++++++++++-------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index bdfdea6d8..b2af36b1e 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -104,22 +104,36 @@ (then (fn [^js lang-support] (when lang-support (.. lang-support -language -parser)))) (catch (fn [err] (js/console.warn (str "Cannot load language parser for: " language) err)))))) -(defn lang->deco-range [language code] +(defn add-style-ranges! [range-builder syntax-tree] + (highlightTree syntax-tree highlight-style + (fn [from to style] + (.add range-builder from to (.mark Decoration (j/obj :class style)))))) + +(defn clojure-style-rangeset [code] + (.finish (doto (RangeSetBuilder.) + (add-style-ranges! (.. ^js clojureLanguage -parser (parse code)))))) + +(defn syntax-highlight [{:keys [code style-rangeset]}] + (let [text (.of Text (.split code "\n"))] + (into [:div.cm-content.whitespace-pre] + (map (partial style-line style-rangeset text)) + (range 1 (inc (.-lines text)))))) + +(defn highlight-clojure [{:keys [code]}] + [syntax-highlight {:code code :style-rangeset (clojure-style-rangeset code)}]) + +(defn highlight-language [{:keys [code language]}] (let [^js builder (RangeSetBuilder.) ^js parser (hooks/use-promise (matching-language-parser language))] - (when parser - (highlightTree (.parse parser code) highlight-style - (fn [from to style] - (.add builder from to (.mark Decoration (j/obj :class style)))))) - (.finish builder))) + (when parser (add-style-ranges! builder (.parse parser code))) + [syntax-highlight {:code code :style-rangeset (.finish builder)}])) (defn render-code [^String code {:keys [language]}] - (let [text (.of Text (.split code "\n"))] - [:div.cm-editor - [:cm-scroller - (into [:div.cm-content.whitespace-pre] - (map (partial style-line (lang->deco-range language code) text)) - (range 1 (inc (.-lines text))))]])) + [:div.cm-editor + [:cm-scroller + (if (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) + [highlight-clojure {:code code}] + [highlight-language {:code code :language language}])]]) ;; editable code viewer (def theme From 613f4d994ba784ff7d0ab3a564507a3fa336875e Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 12:26:37 +0200 Subject: [PATCH 14/23] Naming --- src/nextjournal/clerk/render/code.cljs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index b2af36b1e..7cf8dce96 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -122,7 +122,7 @@ (defn highlight-clojure [{:keys [code]}] [syntax-highlight {:code code :style-rangeset (clojure-style-rangeset code)}]) -(defn highlight-language [{:keys [code language]}] +(defn highlight-imported-language [{:keys [code language]}] (let [^js builder (RangeSetBuilder.) ^js parser (hooks/use-promise (matching-language-parser language))] (when parser (add-style-ranges! builder (.parse parser code))) @@ -133,7 +133,7 @@ [:cm-scroller (if (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) [highlight-clojure {:code code}] - [highlight-language {:code code :language language}])]]) + [highlight-imported-language {:code code :language language}])]]) ;; editable code viewer (def theme From a68a03949c43e05b7a43b6ed7d3cf377cf9d5bf3 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 12:28:22 +0200 Subject: [PATCH 15/23] Naming --- src/nextjournal/clerk/render/code.cljs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index 7cf8dce96..270ae59da 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -88,7 +88,7 @@ (< pos to) (concat [(.sliceString text pos to)])))))))) -(defn matching-language-parser [language] +(defn import-matching-language-parser [language] (cond (not language) (js/Promise.resolve nil) @@ -124,7 +124,7 @@ (defn highlight-imported-language [{:keys [code language]}] (let [^js builder (RangeSetBuilder.) - ^js parser (hooks/use-promise (matching-language-parser language))] + ^js parser (hooks/use-promise (import-matching-language-parser language))] (when parser (add-style-ranges! builder (.parse parser code))) [syntax-highlight {:code code :style-rangeset (.finish builder)}])) From 7eaf1d7e5f0cd306ac8101ec1dd89ac4798e82b0 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 12:30:54 +0200 Subject: [PATCH 16/23] Cleanup --- src/nextjournal/clerk/render/code.cljs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index 270ae59da..040d41048 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -89,12 +89,8 @@ (concat [(.sliceString text pos to)])))))))) (defn import-matching-language-parser [language] - (cond - (not language) + (if (not language) (js/Promise.resolve nil) - (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) - (js/Promise.resolve (.-parser clojureLanguage)) - :else (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0") (then (fn [^js mod] (when-some [langs (.-languages mod)] From 59a589823f4d13ea0eab75b546654d0facd14c80 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 12:46:18 +0200 Subject: [PATCH 17/23] Lift control --- src/nextjournal/clerk/render/code.cljs | 31 +++++++++++++------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/nextjournal/clerk/render/code.cljs b/src/nextjournal/clerk/render/code.cljs index 040d41048..097e0bfe1 100644 --- a/src/nextjournal/clerk/render/code.cljs +++ b/src/nextjournal/clerk/render/code.cljs @@ -1,6 +1,6 @@ (ns nextjournal.clerk.render.code (:require ["@codemirror/language" :refer [HighlightStyle syntaxHighlighting LanguageDescription]] - ["@codemirror/state" :refer [EditorState RangeSetBuilder Text]] + ["@codemirror/state" :refer [EditorState RangeSet RangeSetBuilder Text]] ["@codemirror/view" :refer [EditorView Decoration]] ["@lezer/highlight" :refer [tags highlightTree]] ["@nextjournal/lang-clojure" :refer [clojureLanguage]] @@ -89,16 +89,14 @@ (concat [(.sliceString text pos to)])))))))) (defn import-matching-language-parser [language] - (if (not language) - (js/Promise.resolve nil) - (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0") - (then (fn [^js mod] - (when-some [langs (.-languages mod)] - (when-some [^js matching (or (.matchLanguageName LanguageDescription langs language) - (.matchFilename LanguageDescription langs (str "code." language)))] - (.load matching))))) - (then (fn [^js lang-support] (when lang-support (.. lang-support -language -parser)))) - (catch (fn [err] (js/console.warn (str "Cannot load language parser for: " language) err)))))) + (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0") + (then (fn [^js mod] + (when-some [langs (.-languages mod)] + (when-some [^js matching (or (.matchLanguageName LanguageDescription langs language) + (.matchFilename LanguageDescription langs (str "code." language)))] + (.load matching))))) + (then (fn [^js lang-support] (when lang-support (.. lang-support -language -parser)))) + (catch (fn [err] (js/console.warn (str "Cannot load language parser for: " language) err))))) (defn add-style-ranges! [range-builder syntax-tree] (highlightTree syntax-tree highlight-style @@ -115,9 +113,6 @@ (map (partial style-line style-rangeset text)) (range 1 (inc (.-lines text)))))) -(defn highlight-clojure [{:keys [code]}] - [syntax-highlight {:code code :style-rangeset (clojure-style-rangeset code)}]) - (defn highlight-imported-language [{:keys [code language]}] (let [^js builder (RangeSetBuilder.) ^js parser (hooks/use-promise (import-matching-language-parser language))] @@ -127,8 +122,12 @@ (defn render-code [^String code {:keys [language]}] [:div.cm-editor [:cm-scroller - (if (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) - [highlight-clojure {:code code}] + (cond + (not language) + [syntax-highlight {:code code :style-rangeset (.-empty RangeSet)}] + (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language) + [syntax-highlight {:code code :style-rangeset (clojure-style-rangeset code)}] + :else [highlight-imported-language {:code code :language language}])]]) ;; editable code viewer From 030af3bf55daaebe5302a86f0e06580f94dc338e Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 14:26:14 +0200 Subject: [PATCH 18/23] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddc5f9ff3..9a940780b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ Changes can be: * πŸ’« Support non-evaluated clojure code listings in markdown documents by specifying `{:nextjournal.clerk/code-listing true}` after the language ([#482](https://github.com/nextjournal/clerk/issues/482)). +* πŸ³οΈβ€πŸŒˆ Syntax highlighting for code listings in all [languages supported by codemirror](https://github.com/codemirror/language-data) ([#500](https://github.com/nextjournal/clerk/issues/500)). + * 🐜 Turn off analyzer pass for validation of `:type` tags, fixes [#488](https://github.com/nextjournal/clerk/issues/488) @craig-latacora * 🐜 Strip `:type` metadata from forms before printing them to hash, fixes [#489](https://github.com/nextjournal/clerk/issues/489) @craig-latacora From a0176a5087aaab502fd68a1c39b3766707cc4f50 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 14:54:21 +0200 Subject: [PATCH 19/23] Assign languages in book --- book.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/book.clj b/book.clj index 307957789..961f52cdc 100644 --- a/book.clj +++ b/book.clj @@ -95,7 +95,7 @@ ;; In Emacs, add the following to your config: -;; ```elisp +;; ```el ;; (defun clerk-show () ;; (interactive) ;; (when-let @@ -121,7 +121,7 @@ ;; With [neovim](https://neovim.io/) + [conjure](https://github.com/Olical/conjure/) one can use the following vimscript function to save the file and show it with Clerk: -;; ``` +;; ```vimscript ;; function! ClerkShow() ;; exe "w" ;; exe "ConjureEval (nextjournal.clerk/show! \"" . expand("%:p") . "\")" From dbfbab567e595e62f29658912141f7c72e3d593b Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 15:01:15 +0200 Subject: [PATCH 20/23] Default to clojure highlighting in indented code blocks --- notebooks/markdown_fences.md | 5 +++-- src/nextjournal/clerk/viewer.cljc | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/notebooks/markdown_fences.md b/notebooks/markdown_fences.md index 06a4f431c..bb9ae4bcc 100644 --- a/notebooks/markdown_fences.md +++ b/notebooks/markdown_fences.md @@ -64,9 +64,10 @@ int main() { } ``` -## [Indented Code Blocks](https://spec.commonmark.org/0.30/#indented-code-blocks) should not be highlighted +## Indented Code Blocks +[Indented code blocks](https://spec.commonmark.org/0.30/#indented-code-blocks) default to clojure highlighting (no (off) :fence) - (not "highlighted") + (but "highlighted") fin. diff --git a/src/nextjournal/clerk/viewer.cljc b/src/nextjournal/clerk/viewer.cljc index fba8bdac1..052ee43d2 100644 --- a/src/nextjournal/clerk/viewer.cljc +++ b/src/nextjournal/clerk/viewer.cljc @@ -707,7 +707,7 @@ :transform-fn (update-val #(with-viewer `html-viewer [:div.code-viewer.code-listing (with-viewer `code-viewer - {:nextjournal/opts (select-keys % [:language])} + {:nextjournal/opts {:language (:language % "clojure")}} (str/trim-newline (md.transform/->text %)))]))} ;; marks From 469c7ee5ed2576bac9c98c064e5aadb9ad0395b6 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Thu, 1 Jun 2023 15:19:53 +0200 Subject: [PATCH 21/23] Document language option for the code viewer --- book.clj | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/book.clj b/book.clj index 961f52cdc..17f4c86c2 100644 --- a/book.clj +++ b/book.clj @@ -251,7 +251,7 @@ ;; ### 🎼 Code -;; The code viewer uses +;; By default the code viewer uses ;; [clojure-mode](https://nextjournal.github.io/clojure-mode/) for ;; syntax highlighting. (clerk/code (macroexpand '(when test @@ -262,6 +262,21 @@ (clerk/code "(defn my-fn\n \"This is a Doc String\"\n [args]\n 42)") +;; You might specify different langauges via `::clerk/opts` like so +(clerk/code {::clerk/opts {:language "python"}} " +class Foo(object): + def __init__(self): + pass + def do_this(self): + return 1") + +(clerk/code {::clerk/opts {:language "c++"}} " +#include +int main() { + std::cout << \" Hello, world! \" << std::endl + return 0 +}") + ;; ### 🏞 Images ;; Clerk now has built-in support for the From 0b8b316179d249f1953b1fed7225fc7aaa003317 Mon Sep 17 00:00:00 2001 From: Martin Kavalar Date: Fri, 2 Jun 2023 10:36:25 +0200 Subject: [PATCH 22/23] Tweak prose --- book.clj | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/book.clj b/book.clj index 17f4c86c2..fa2082f34 100644 --- a/book.clj +++ b/book.clj @@ -262,7 +262,7 @@ (clerk/code "(defn my-fn\n \"This is a Doc String\"\n [args]\n 42)") -;; You might specify different langauges via `::clerk/opts` like so +;; You can specify the language for syntax highlighting via `::clerk/opts`. (clerk/code {::clerk/opts {:language "python"}} " class Foo(object): def __init__(self): @@ -270,12 +270,15 @@ class Foo(object): def do_this(self): return 1") -(clerk/code {::clerk/opts {:language "c++"}} " +;; Or use a code fence with a language in a markdown. + +(clerk/md "```c++ #include int main() { std::cout << \" Hello, world! \" << std::endl return 0 -}") +} +```") ;; ### 🏞 Images From 19f5a877d813cd946e69ba356f374a89e66b87d1 Mon Sep 17 00:00:00 2001 From: Andrea Amantini Date: Fri, 2 Jun 2023 17:28:27 +0200 Subject: [PATCH 23/23] Simplify --- notebooks/markdown_fences.md | 4 --- src/nextjournal/clerk/parser.cljc | 56 ++++++++++--------------------- 2 files changed, 18 insertions(+), 42 deletions(-) diff --git a/notebooks/markdown_fences.md b/notebooks/markdown_fences.md index bb9ae4bcc..a8335059c 100644 --- a/notebooks/markdown_fences.md +++ b/notebooks/markdown_fences.md @@ -18,10 +18,6 @@ Use `{:nextjournal.clerk/code-listing true}` in the fence info to signal that a ```clojure {:nextjournal.clerk/code-listing true} (1 2 "not evaluated" :but-still-highlighted) ``` -if no language is specified we assume it's a clojure cell -```{:nextjournal.clerk/code-listing true} -(1 2 "not evaluated" :but-still-highlighted) -``` ## πŸ³οΈβ€πŸŒˆ Polyglot Highlighting diff --git a/src/nextjournal/clerk/parser.cljc b/src/nextjournal/clerk/parser.cljc index 8bab7c653..1699df4f2 100644 --- a/src/nextjournal/clerk/parser.cljc +++ b/src/nextjournal/clerk/parser.cljc @@ -357,50 +357,30 @@ :nodes (rest nodes) ::md-slice [])) -(defn parse-code-info-string - "https://spec.commonmark.org/0.30/#code-fence" - [{:as node :keys [info]}] - (if-not (and (code? node) info) - {:eval? false} - (try - (let [sexprs (n/child-sexprs (p/parse-string-all info)) - language (some #(and (symbol? %) %) sexprs) - metadata (some #(and (map? %) %) sexprs)] - {:language (if language (name language) "clojure") - :metadata metadata - :eval? (and (or (not language) (#{'clojure 'clj 'cljc} language)) - (not (:nextjournal.clerk/code-listing metadata)))}) - (catch #?(:clj Throwable :cljs :default) cause - (throw (ex-info "We allow info strings of the following shape: -```language {map with clerk metadata} -code -```" node cause)))))) - -#_ (parse-code-info-string {:type :heading}) -#_ (parse-code-info-string {:type :code :info "clojure"}) -#_ (parse-code-info-string {:type :code :info " clojure"}) -#_ (parse-code-info-string {:type :code :info "clojure {"}) -#_ (parse-code-info-string {:type :code :info "{r, eval=FALSE}"}) ;; RMarkdown/Knitr -#_ (parse-code-info-string {:type :code :info "{r, eval=FALSE, include=TRUE}"}) ;; RMarkdown/Knitr (failing) -#_ (parse-code-info-string {:type :code :info "clojure &@foo !!!"}) -#_ (parse-code-info-string {:type :code :info "clojure {:nextjournal.clerk/code-listing true}"}) -#_ (parse-code-info-string {:type :code :info "{:nextjournal.clerk/code-listing true} c++"}) -#_ (ex-message *e) +(defn runnable-code-block? [{:as block :keys [info language]}] + (and (code? block) + info + (or (empty? language) + (re-matches #"clj(c?)|clojure" language)) + (not (:nextjournal.clerk/code-listing + (when-some [parsed (when (and (seq language) (str/starts-with? info language)) + (p/parse-string-all (subs info (count language))))] + (when (n/sexpr-able? parsed) + (n/sexpr parsed))))))) + +#_(runnable-code-block? {:type :code :language "clojure" :info "clojure"}) +#_(runnable-code-block? {:type :code :language "clojure" :info "clojure {:nextjournal.clerk/code-listing true}"}) (defn parse-markdown-string [{:as opts :keys [doc?]} s] (let [{:as ctx :keys [content]} (parse-markdown (markdown-context) s)] (loop [{:as state :keys [nodes] ::keys [md-slice]} {:blocks [] ::md-slice [] :nodes content :md-context ctx}] (if-some [node (first nodes)] (recur - (let [{:keys [language eval? _metadata]} (parse-code-info-string node)] - (if eval? - (-> state - (update :blocks #(cond-> % (seq md-slice) (conj {:type :markdown :doc {:type :doc :content md-slice}}))) - (parse-markdown-cell opts)) - (-> state - (update :nodes rest) - (cond-> doc? (update ::md-slice conj - (cond-> node language (assoc :language language)))))))) + (if (runnable-code-block? node) + (-> state + (update :blocks #(cond-> % (seq md-slice) (conj {:type :markdown :doc {:type :doc :content md-slice}}))) + (parse-markdown-cell opts)) + (-> state (update :nodes rest) (cond-> doc? (update ::md-slice conj node))))) (-> state (update :blocks #(cond-> % (seq md-slice) (conj {:type :markdown :doc {:type :doc :content md-slice}})))