Write hash in pre-commit hook for predicable viewer.js location, build JS in CI

.github/workflows/main.yml

@@ -4,6 +4,7 @@ on: push
 
 jobs:
   test:
+    needs: [build-and-upload-viewer-resources]
     runs-on: ${{matrix.sys.os}}
 
     strategy:
@@ -45,9 +46,9 @@ jobs:
       - name: 🧪 Run tests
         run: clojure -X:test
 
-
   static-build:
     runs-on: ubuntu-latest
+    needs: [build-and-upload-viewer-resources]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -63,6 +64,11 @@ jobs:
         with:
           cli: '1.10.3.943'
 
+      - name: Setup Babashka
+        uses: turtlequeue/setup-babashka@v1.3.0
+        with:
+          babashka-version: 0.7.6
+
       - name: 🗝 maven cache
         uses: actions/cache@v2
         with:
@@ -79,9 +85,6 @@ jobs:
           path: .clerk
           key: ${{ runner.os }}-clerk-cache
 
-      - name: 🏗 Build Clerk Static App with default Notebooks
-        run: clojure -X:demo nextjournal.clerk/build-static-app! :git/sha '"${{ github.sha }}"' :git/url '"https://github.com/nextjournal/clerk"'
-
       - name: 🔐 Google Auth
         uses: google-github-actions/auth@v0
         with:
@@ -90,6 +93,9 @@ jobs:
       - name: 🔧 Setup Google Cloud SDK
         uses: google-github-actions/setup-gcloud@v0.3.0
 
+      - name: 🏗 Build Clerk Static App with default Notebooks
+        run: clojure -J-Dclojure.main.report=stderr -X:demo nextjournal.clerk/build-static-app! :git/sha '"${{ github.sha }}"' :git/url '"https://github.com/nextjournal/clerk"' :browse false
+
       - name: 📠 Copy static build to bucket under SHA
         run: |
           gsutil cp -r public/build gs://nextjournal-snapshots/clerk/build/${{ github.sha }}
@@ -102,3 +108,46 @@ jobs:
           state: 'success'
           sha: ${{github.event.pull_request.head.sha || github.sha}}
           target_url: https://snapshots.nextjournal.com/clerk/build/${{ github.sha }}
+
+  build-and-upload-viewer-resources:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch_depth: 0
+
+      - name: 🔧 Install java
+        uses: actions/setup-java@v1
+        with:
+          java-version: '11.0.7'
+
+      - name: 🔧 Install clojure
+        uses: DeLaGuardo/setup-clojure@master
+        with:
+          cli: '1.10.3.943'
+
+      - name: Setup Babashka
+        uses: turtlequeue/setup-babashka@v1.3.0
+        with:
+          babashka-version: 0.7.6
+
+      - name: 🗝 maven cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.m2
+            ~/.gitlibs
+          key: ${{ runner.os }}-maven-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: 🔐 Google Auth
+        uses: google-github-actions/auth@v0
+        with:
+          credentials_json: ${{ secrets.GCLOUD_SERVICE_KEY }}
+
+      - name: 🔧 Setup Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0.3.0
+
+      - name: Build and upload viewer resources
+        run: bb build+upload-viewer-resources

.gitignore

@@ -3,7 +3,7 @@
 .clerk/
 .cpcache/
 .lsp/
-/node_modules/
+node_modules/
 /public/js/
 /.shadow-cljs/
 /target/
@@ -13,3 +13,6 @@ public/girouette.css
 public/images/
 yarn.lock
 *~
+.work
+/build
+/public/build

bb.edn

@@ -1,4 +1,9 @@
-{:tasks {:requires ([clojure.edn :as edn]
+{:min-bb-version "0.7.6"
+ :paths ["bb"]
+ :deps {io.github.nextjournal/dejavu
+        {:git/sha "3c4cf5e4dda3648e0d4c575a3433d19c511dd3e5"}}
+ :tasks {:requires ([clojure.edn :as edn]
+                    [clojure.edn :as edn]
                     [clojure.string :as str]
                     [babashka.deps :as deps]
                     [babashka.fs :as fs]
@@ -18,13 +23,22 @@
                      :task (clojure "-M:test:sci:demo:dev watch browser")
                      :depends [yarn-install]}
 
-         release:js {:doc "Builds & uploads the js release and updates the sha reference."
-                     :task (do (clojure "-M:sci:demo:dev release browser")
-                               (clojure "-T:build upload-to-cas+rewrite-sha :resource viewer.js"))
-                     :depends [yarn-install]}
+         build:js {:doc "Builds JS"
+                   :depends [yarn-install]
+                   :task (clojure "-M:sci:demo:dev release browser")}
+
+         write-hash {:doc "Write viewer resource hash to resources. Intended to be used in a pre-commit hook."
+                     :task viewer-resources-hashing/write-viewer-resource-hash}
+
+         build+upload-viewer-resources {:doc "Refreshes assets stored on CDN (google storage)"
+                                        :task viewer-resources-hashing/build+upload-viewer-resources}
+
+         release:js {:doc "Uploads the js release and updates the sha reference."
+                     :task (clojure "-T:build upload-to-cas :resource viewer.js")
+                     :depends [build:js]}
 
          build:static-app {:doc "Builds a static app with default notebooks"
-                           :depends [release:js release:css]
+                           :depends [release:js]
                            :task (clojure "-X:demo nextjournal.clerk/build-static-app!")}
 
          release:jar {:doc "Builds the jar"
@@ -45,7 +59,9 @@
                       (shell "git tag" tag)
                       (shell "git push origin" tag))}
 
-         -dev {:depends [watch:cljs copy-viewer-css]}
+         install-git-hooks {:doc "Install git hooks"
+                            :task (fs/copy "dev/pre-commit" ".git/hooks" {:replace-existing true})}
+
+         -dev {:depends [install-git-hooks watch:cljs copy-viewer-css]}
          dev {:doc "Start app in dev mode, compiles cljs and css"
-              :task (run '-dev {:parallel true})}}
- :min-bb-version "0.5.1"}
+              :task (run '-dev {:parallel true})}}}

bb/viewer_resources_hashing.clj

@@ -0,0 +1,71 @@
+(ns viewer-resources-hashing
+  (:require [babashka.classpath :as cp]
+            [babashka.fs :as fs]
+            [babashka.tasks :as tasks :refer [shell]]
+            [clojure.string :as str]
+            [nextjournal.dejavu :as djv]))
+
+(def output-dirs ["resources/public/ui"
+                  "resources/public/build"])
+
+;; Example link in bucket:
+;; "https://storage.googleapis.com/nextjournal-cas-eu/data/8VwKauX6JACEP3K6ahNmP5p1w7rWdhKzeGXCDrHMnJiVrUxHVxcm3Xj84K2r3fcAKWxMQKzqoFe92osgFEHCuKCtZC"
+
+(def gs-bucket "gs://nextjournal-cas-eu/data")
+(def base-url "https://storage.googleapis.com/nextjournal-cas-eu/data")
+
+(defn sha512s []
+  (let [files (map str (mapcat #(fs/glob % "**.{js,css}") output-dirs))
+        sha512 (map (comp djv/sha512 slurp) files)]
+    (zipmap files sha512)))
+
+(defn resource [f]
+  (fs/file "resources/public" (str/replace f #"^/" "")))
+
+(defn classpath-dirs []
+  (tasks/run 'yarn-install);
+  (->> (shell {:out :string} "yarn --silent shadow-cljs classpath")
+       :out
+       str/trim
+       str/split-lines
+       first
+       cp/split-classpath
+       (remove (comp not fs/exists?))
+       (take-while fs/directory?)
+       (remove #(str/includes? % "test"))))
+
+(defn file-set []
+  (reduce into []
+          [["deps.edn"
+            "shadow-cljs.edn"
+            "yarn.lock"]
+           (djv/cljs-files (classpath-dirs))]))
+
+(def viewer-js-hash-file "resources/viewer-js-hash")
+
+(defn write-viewer-resource-hash
+  []
+  (let [front-end-hash (str (djv/file-set-hash (file-set)))]
+    (spit viewer-js-hash-file front-end-hash)))
+
+(def gs-url-prefix "https://storage.googleapis.com/nextjournal-cas-eu/data")
+
+(defn lookup-url [lookup-hash]
+  (str gs-bucket "/lookup/" lookup-hash))
+
+(defn cas-link [hash]
+  (str gs-url-prefix "/" hash))
+
+(defn build+upload-viewer-resources []
+  (let [front-end-hash (str/trim (slurp viewer-js-hash-file))
+        manifest (str (fs/create-temp-file))
+        res (djv/gs-copy (str (lookup-url front-end-hash)) manifest false)]
+    (when (= res ::djv/not-found)
+      (tasks/run 'build:js)
+      (let [content-hash (djv/sha512 (slurp "build/viewer.js"))
+            viewer-js-http-link (str (cas-link content-hash) "?cache=false")]
+        (spit manifest {"/js/viewer.js" viewer-js-http-link})
+        (println "Manifest:" (slurp manifest))
+        (println "Coping manifest to" (lookup-url front-end-hash))
+        (djv/gs-copy manifest (lookup-url front-end-hash))
+        (djv/gs-copy "build/viewer.js" (str gs-bucket "/" content-hash))))))

build.clj

@@ -41,36 +41,17 @@
 
 (defn asserting [val msg] (assert (seq val) msg) val)
 
-(def clerk-config-path "src/nextjournal/clerk/config.clj")
-
-(defn replace-resource [resource url]
-  (-> (io/file clerk-config-path)
-      z/of-file
-      (z/find-token z/next #(and (= 'def (z/sexpr %))
-                                 (= 'default-resource-manifest (-> % z/right z/sexpr))))
-      (z/find-value z/next resource)
-      z/right
-      (z/edit (fn [old-url]
-                (println "Replacing resource: " resource "\n"
-                         (if (= old-url url)
-                           (str "url " old-url " is up-to-date.")
-                           (str "updated from " old-url " to " url ".")))
-                url))
-      z/root-string))
-
 (defn upload! [opts file] (:url (cas/upload! opts file)))
 
-(defn update-resource! [opts resource _ file]
-  (spit clerk-config-path
-        (replace-resource resource
-                          (upload! opts file))))
+(defn update-resource! [opts _resource _ file]
+  (upload! opts file))
 
 (defn get-gsutil [] (str/trim (:out (process/sh ["which" "gsutil"]))))
 
 (def resource->path
   '{viewer.js "/js/viewer.js"})
 
-(defn upload-to-cas+rewrite-sha [{:keys [resource]}]
+(defn upload-to-cas [{:keys [resource]}]
   (if-let [target (resource->path resource)]
     (update-resource! {:exec-path (str/trim (asserting (get-gsutil) "Can't find gsutil executable."))
                        :target-path "gs://nextjournal-cas-eu/data/"} target :uploading (str "build/" resource))

deps.edn

@@ -2,7 +2,7 @@
  :deps {org.clojure/clojure {:mvn/version "1.10.3"}
         org.clojure/java.classpath {:mvn/version "1.0.0"}
         org.clojure/tools.analyzer.jvm {:mvn/version "1.1.0"}
-        babashka/fs {:mvn/version "0.1.2"}
+        babashka/fs {:mvn/version "0.1.3"}
         borkdude/edamame {:mvn/version "0.0.11"}
         weavejester/dependency {:mvn/version "0.2.1"}
 
@@ -22,7 +22,7 @@
 
  :aliases {:nextjournal/clerk {:exec-fn nextjournal.clerk/build-static-app!}
            :sci {:extra-deps  {applied-science/js-interop  {:mvn/version "0.3.3"}
-                               org.babashka/sci {:mvn/version "0.3.1"}
+                               org.babashka/sci {:mvn/version "0.3.2"}
                                reagent/reagent {:mvn/version "1.1.0"}
                                io.github.babashka/sci-configs {:git/sha "fd6a3332f358e56b96b5a791f75f97573e018580"}
                                io.github.nextjournal/viewers {:git/sha "63bda9bb89dafa6181c71e21dd363b2bbdae7b13"}

dev/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+bb write-hash
+git add resources/viewer-js-hash

resources/viewer-js-hash

@@ -0,0 +1 @@
+yh4majqZ75XpcUpjEFqRovMNjpk

src/nextjournal/clerk/config.clj

@@ -1,5 +1,7 @@
 (ns nextjournal.clerk.config
-  (:require [clojure.string :as str]))
+  (:require [clojure.edn :as edn]
+            [clojure.java.io :as io]
+            [clojure.string :as str]))
 
 (def cache-dir
   (or (System/getProperty "clerk.cache_dir")
@@ -9,8 +11,9 @@
   (when-let [prop (System/getProperty "clerk.disable_cache")]
     (not= "false" prop)))
 
-(def default-resource-manifest
-  {"/js/viewer.js" "https://storage.googleapis.com/nextjournal-cas-eu/data/8Vt3Eko1RGonTrRXaLTP6Ly5XugGnYiRapACBxSoZ2RKwFztsi2NYXvTb5vyCTFya2RDgK9vXwVvKBnQ8dWopcmvA5"})
+(def gs-url-prefix "https://storage.googleapis.com/nextjournal-cas-eu/data")
+(def lookup-hash (str/trim (slurp (io/resource "viewer-js-hash"))))
+(def lookup-url (str gs-url-prefix "/lookup/" lookup-hash))
 
 (def resource-manifest-from-props
   (when-let [prop (System/getProperty "clerk.resource_manifest")]
@@ -19,7 +22,9 @@
 
 (defonce !resource->url
   (atom (or resource-manifest-from-props
-            default-resource-manifest)))
+            ;; assume that CI will have published a CAS-link under this lookup,
+            ;; prior to hitting this code-path
+            (edn/read-string (slurp lookup-url)))))
 
 #_(swap! !resource->url assoc "/css/viewer.css" "https://storage.googleapis.com/nextjournal-cas-eu/data/8VvAV62HzsvhcsXEkHP33uj4cV9UvdDz7DU9qLeVRCfEP9kWLFAzaMKL77trdx898DzcVyDVejdfxvxj5XB84UpWvQ")
 #_(swap! !resource->url dissoc "/css/viewer.css")