diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3f8bdf74..16a57b142 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,6 +4,7 @@ on: push jobs: test: + needs: [build-and-upload-viewer-resources] runs-on: ${{matrix.sys.os}} strategy: @@ -45,9 +46,9 @@ jobs: - name: ๐Ÿงช Run tests run: clojure -X:test - static-build: runs-on: ubuntu-latest + needs: [build-and-upload-viewer-resources] steps: - uses: actions/checkout@v2 with: @@ -63,6 +64,11 @@ jobs: with: cli: '1.10.3.943' + - name: Setup Babashka + uses: turtlequeue/setup-babashka@v1.3.0 + with: + babashka-version: 0.7.6 + - name: ๐Ÿ— maven cache uses: actions/cache@v2 with: @@ -79,9 +85,6 @@ jobs: path: .clerk key: ${{ runner.os }}-clerk-cache - - name: ๐Ÿ— Build Clerk Static App with default Notebooks - run: clojure -X:demo nextjournal.clerk/build-static-app! :git/sha '"${{ github.sha }}"' :git/url '"https://github.com/nextjournal/clerk"' - - name: ๐Ÿ” Google Auth uses: google-github-actions/auth@v0 with: @@ -90,6 +93,9 @@ jobs: - name: ๐Ÿ”ง Setup Google Cloud SDK uses: google-github-actions/setup-gcloud@v0.3.0 + - name: ๐Ÿ— Build Clerk Static App with default Notebooks + run: clojure -J-Dclojure.main.report=stderr -X:demo nextjournal.clerk/build-static-app! :git/sha '"${{ github.sha }}"' :git/url '"https://github.com/nextjournal/clerk"' :browse false + - name: ๐Ÿ“  Copy static build to bucket under SHA run: | gsutil cp -r public/build gs://nextjournal-snapshots/clerk/build/${{ github.sha }} @@ -102,3 +108,46 @@ jobs: state: 'success' sha: ${{github.event.pull_request.head.sha || github.sha}} target_url: https://snapshots.nextjournal.com/clerk/build/${{ github.sha }} + + build-and-upload-viewer-resources: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch_depth: 0 + + - name: ๐Ÿ”ง Install java + uses: actions/setup-java@v1 + with: + java-version: '11.0.7' + + - name: ๐Ÿ”ง Install clojure + uses: DeLaGuardo/setup-clojure@master + with: + cli: '1.10.3.943' + + - name: Setup Babashka + uses: turtlequeue/setup-babashka@v1.3.0 + with: + babashka-version: 0.7.6 + + - name: ๐Ÿ— maven cache + uses: actions/cache@v2 + with: + path: | + ~/.m2 + ~/.gitlibs + key: ${{ runner.os }}-maven-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: ๐Ÿ” Google Auth + uses: google-github-actions/auth@v0 + with: + credentials_json: ${{ secrets.GCLOUD_SERVICE_KEY }} + + - name: ๐Ÿ”ง Setup Google Cloud SDK + uses: google-github-actions/setup-gcloud@v0.3.0 + + - name: Build and upload viewer resources + run: bb build+upload-viewer-resources diff --git a/.gitignore b/.gitignore index 361e80bf1..a5db64f57 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ .clerk/ .cpcache/ .lsp/ -/node_modules/ +node_modules/ /public/js/ /.shadow-cljs/ /target/ @@ -13,3 +13,6 @@ public/girouette.css public/images/ yarn.lock *~ +.work +/build +/public/build diff --git a/bb.edn b/bb.edn index 7f2ac1bfe..32f85178f 100644 --- a/bb.edn +++ b/bb.edn @@ -1,4 +1,9 @@ -{:tasks {:requires ([clojure.edn :as edn] +{:min-bb-version "0.7.6" + :paths ["bb"] + :deps {io.github.nextjournal/dejavu + {:git/sha "3c4cf5e4dda3648e0d4c575a3433d19c511dd3e5"}} + :tasks {:requires ([clojure.edn :as edn] + [clojure.edn :as edn] [clojure.string :as str] [babashka.deps :as deps] [babashka.fs :as fs] @@ -18,13 +23,22 @@ :task (clojure "-M:test:sci:demo:dev watch browser") :depends [yarn-install]} - release:js {:doc "Builds & uploads the js release and updates the sha reference." - :task (do (clojure "-M:sci:demo:dev release browser") - (clojure "-T:build upload-to-cas+rewrite-sha :resource viewer.js")) - :depends [yarn-install]} + build:js {:doc "Builds JS" + :depends [yarn-install] + :task (clojure "-M:sci:demo:dev release browser")} + + write-hash {:doc "Write viewer resource hash to resources. Intended to be used in a pre-commit hook." + :task viewer-resources-hashing/write-viewer-resource-hash} + + build+upload-viewer-resources {:doc "Refreshes assets stored on CDN (google storage)" + :task viewer-resources-hashing/build+upload-viewer-resources} + + release:js {:doc "Uploads the js release and updates the sha reference." + :task (clojure "-T:build upload-to-cas :resource viewer.js") + :depends [build:js]} build:static-app {:doc "Builds a static app with default notebooks" - :depends [release:js release:css] + :depends [release:js] :task (clojure "-X:demo nextjournal.clerk/build-static-app!")} release:jar {:doc "Builds the jar" @@ -45,7 +59,9 @@ (shell "git tag" tag) (shell "git push origin" tag))} - -dev {:depends [watch:cljs copy-viewer-css]} + install-git-hooks {:doc "Install git hooks" + :task (fs/copy "dev/pre-commit" ".git/hooks" {:replace-existing true})} + + -dev {:depends [install-git-hooks watch:cljs copy-viewer-css]} dev {:doc "Start app in dev mode, compiles cljs and css" - :task (run '-dev {:parallel true})}} - :min-bb-version "0.5.1"} + :task (run '-dev {:parallel true})}}} diff --git a/bb/viewer_resources_hashing.clj b/bb/viewer_resources_hashing.clj new file mode 100644 index 000000000..a88a81662 --- /dev/null +++ b/bb/viewer_resources_hashing.clj @@ -0,0 +1,71 @@ +(ns viewer-resources-hashing + (:require [babashka.classpath :as cp] + [babashka.fs :as fs] + [babashka.tasks :as tasks :refer [shell]] + [clojure.string :as str] + [nextjournal.dejavu :as djv])) + +(def output-dirs ["resources/public/ui" + "resources/public/build"]) + +;; Example link in bucket: +;; "https://storage.googleapis.com/nextjournal-cas-eu/data/8VwKauX6JACEP3K6ahNmP5p1w7rWdhKzeGXCDrHMnJiVrUxHVxcm3Xj84K2r3fcAKWxMQKzqoFe92osgFEHCuKCtZC" + +(def gs-bucket "gs://nextjournal-cas-eu/data") +(def base-url "https://storage.googleapis.com/nextjournal-cas-eu/data") + +(defn sha512s [] + (let [files (map str (mapcat #(fs/glob % "**.{js,css}") output-dirs)) + sha512 (map (comp djv/sha512 slurp) files)] + (zipmap files sha512))) + +(defn resource [f] + (fs/file "resources/public" (str/replace f #"^/" ""))) + +(defn classpath-dirs [] + (tasks/run 'yarn-install); + (->> (shell {:out :string} "yarn --silent shadow-cljs classpath") + :out + str/trim + str/split-lines + first + cp/split-classpath + (remove (comp not fs/exists?)) + (take-while fs/directory?) + (remove #(str/includes? % "test")))) + +(defn file-set [] + (reduce into [] + [["deps.edn" + "shadow-cljs.edn" + "yarn.lock"] + (djv/cljs-files (classpath-dirs))])) + +(def viewer-js-hash-file "resources/viewer-js-hash") + +(defn write-viewer-resource-hash + [] + (let [front-end-hash (str (djv/file-set-hash (file-set)))] + (spit viewer-js-hash-file front-end-hash))) + +(def gs-url-prefix "https://storage.googleapis.com/nextjournal-cas-eu/data") + +(defn lookup-url [lookup-hash] + (str gs-bucket "/lookup/" lookup-hash)) + +(defn cas-link [hash] + (str gs-url-prefix "/" hash)) + +(defn build+upload-viewer-resources [] + (let [front-end-hash (str/trim (slurp viewer-js-hash-file)) + manifest (str (fs/create-temp-file)) + res (djv/gs-copy (str (lookup-url front-end-hash)) manifest false)] + (when (= res ::djv/not-found) + (tasks/run 'build:js) + (let [content-hash (djv/sha512 (slurp "build/viewer.js")) + viewer-js-http-link (str (cas-link content-hash) "?cache=false")] + (spit manifest {"/js/viewer.js" viewer-js-http-link}) + (println "Manifest:" (slurp manifest)) + (println "Coping manifest to" (lookup-url front-end-hash)) + (djv/gs-copy manifest (lookup-url front-end-hash)) + (djv/gs-copy "build/viewer.js" (str gs-bucket "/" content-hash)))))) diff --git a/build.clj b/build.clj index 84296b630..a06de1b84 100644 --- a/build.clj +++ b/build.clj @@ -41,36 +41,17 @@ (defn asserting [val msg] (assert (seq val) msg) val) -(def clerk-config-path "src/nextjournal/clerk/config.clj") - -(defn replace-resource [resource url] - (-> (io/file clerk-config-path) - z/of-file - (z/find-token z/next #(and (= 'def (z/sexpr %)) - (= 'default-resource-manifest (-> % z/right z/sexpr)))) - (z/find-value z/next resource) - z/right - (z/edit (fn [old-url] - (println "Replacing resource: " resource "\n" - (if (= old-url url) - (str "url " old-url " is up-to-date.") - (str "updated from " old-url " to " url "."))) - url)) - z/root-string)) - (defn upload! [opts file] (:url (cas/upload! opts file))) -(defn update-resource! [opts resource _ file] - (spit clerk-config-path - (replace-resource resource - (upload! opts file)))) +(defn update-resource! [opts _resource _ file] + (upload! opts file)) (defn get-gsutil [] (str/trim (:out (process/sh ["which" "gsutil"])))) (def resource->path '{viewer.js "/js/viewer.js"}) -(defn upload-to-cas+rewrite-sha [{:keys [resource]}] +(defn upload-to-cas [{:keys [resource]}] (if-let [target (resource->path resource)] (update-resource! {:exec-path (str/trim (asserting (get-gsutil) "Can't find gsutil executable.")) :target-path "gs://nextjournal-cas-eu/data/"} target :uploading (str "build/" resource)) diff --git a/deps.edn b/deps.edn index cd98e1680..4bebc2ce1 100644 --- a/deps.edn +++ b/deps.edn @@ -2,7 +2,7 @@ :deps {org.clojure/clojure {:mvn/version "1.10.3"} org.clojure/java.classpath {:mvn/version "1.0.0"} org.clojure/tools.analyzer.jvm {:mvn/version "1.1.0"} - babashka/fs {:mvn/version "0.1.2"} + babashka/fs {:mvn/version "0.1.3"} borkdude/edamame {:mvn/version "0.0.11"} weavejester/dependency {:mvn/version "0.2.1"} @@ -22,7 +22,7 @@ :aliases {:nextjournal/clerk {:exec-fn nextjournal.clerk/build-static-app!} :sci {:extra-deps {applied-science/js-interop {:mvn/version "0.3.3"} - org.babashka/sci {:mvn/version "0.3.1"} + org.babashka/sci {:mvn/version "0.3.2"} reagent/reagent {:mvn/version "1.1.0"} io.github.babashka/sci-configs {:git/sha "fd6a3332f358e56b96b5a791f75f97573e018580"} io.github.nextjournal/viewers {:git/sha "63bda9bb89dafa6181c71e21dd363b2bbdae7b13"} diff --git a/dev/pre-commit b/dev/pre-commit new file mode 100755 index 000000000..6cc290d14 --- /dev/null +++ b/dev/pre-commit @@ -0,0 +1,4 @@ +#!/bin/sh + +bb write-hash +git add resources/viewer-js-hash diff --git a/resources/viewer-js-hash b/resources/viewer-js-hash new file mode 100644 index 000000000..10039aae7 --- /dev/null +++ b/resources/viewer-js-hash @@ -0,0 +1 @@ +yh4majqZ75XpcUpjEFqRovMNjpk \ No newline at end of file diff --git a/src/nextjournal/clerk/config.clj b/src/nextjournal/clerk/config.clj index abff6a125..fe6744eb9 100644 --- a/src/nextjournal/clerk/config.clj +++ b/src/nextjournal/clerk/config.clj @@ -1,5 +1,7 @@ (ns nextjournal.clerk.config - (:require [clojure.string :as str])) + (:require [clojure.edn :as edn] + [clojure.java.io :as io] + [clojure.string :as str])) (def cache-dir (or (System/getProperty "clerk.cache_dir") @@ -9,8 +11,9 @@ (when-let [prop (System/getProperty "clerk.disable_cache")] (not= "false" prop))) -(def default-resource-manifest - {"/js/viewer.js" "https://storage.googleapis.com/nextjournal-cas-eu/data/8Vt3Eko1RGonTrRXaLTP6Ly5XugGnYiRapACBxSoZ2RKwFztsi2NYXvTb5vyCTFya2RDgK9vXwVvKBnQ8dWopcmvA5"}) +(def gs-url-prefix "https://storage.googleapis.com/nextjournal-cas-eu/data") +(def lookup-hash (str/trim (slurp (io/resource "viewer-js-hash")))) +(def lookup-url (str gs-url-prefix "/lookup/" lookup-hash)) (def resource-manifest-from-props (when-let [prop (System/getProperty "clerk.resource_manifest")] @@ -19,7 +22,9 @@ (defonce !resource->url (atom (or resource-manifest-from-props - default-resource-manifest))) + ;; assume that CI will have published a CAS-link under this lookup, + ;; prior to hitting this code-path + (edn/read-string (slurp lookup-url))))) #_(swap! !resource->url assoc "/css/viewer.css" "https://storage.googleapis.com/nextjournal-cas-eu/data/8VvAV62HzsvhcsXEkHP33uj4cV9UvdDz7DU9qLeVRCfEP9kWLFAzaMKL77trdx898DzcVyDVejdfxvxj5XB84UpWvQ") #_(swap! !resource->url dissoc "/css/viewer.css")