diff --git a/etc/bootstrap.sh b/etc/bootstrap.sh new file mode 100644 index 0000000..89254b7 --- /dev/null +++ b/etc/bootstrap.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# A helper script for developing Katamari - lets kat re-uberjar itself, backs the current jar up, +# tries to reboot into the new jar, and if that fails puts the old jar back. + +jar=$(./kat -j compile me.arrdem/katamari+uberjar | jq -r '.["me.arrdem/katamari+uberjar"]["paths"][0]') +mv .kat.d/bootstrap.jar kat-backup.jar +cp $jar .kat.d/bootstrap.jar +./kat restart-server || ( + cp kat-backup.jar .kat.d/bootstrap.jar + ./kat start-server +) diff --git a/example/src/main/java/demo/Demo.java b/example/src/main/java/demo/Demo.java index 19272c0..eced452 100644 --- a/example/src/main/java/demo/Demo.java +++ b/example/src/main/java/demo/Demo.java @@ -2,7 +2,7 @@ public class Demo { public static int main(String[] args) { - System.out.println(String.format("Got %d args!", args.length())); + System.out.println(String.format("Got %d args!", args.length)); return 0; } } diff --git a/kat b/kat index 99657bb..a10d06d 100755 --- a/kat +++ b/kat @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + ## About # # A shim for starting the Katamari server, and sending it requests @@ -24,7 +26,7 @@ function _get_conf() { export KAT="${0}" # Find java executable -if [ -z "${JAVA_CMD}" ]; then +if [ -z "${JAVA_CMD+x}" ]; then set +e export JAVA_CMD=$(type -p java) set -e @@ -39,38 +41,38 @@ if [ -z "${JAVA_CMD}" ]; then fi # The kat install location -if [ -z "${KAT_BIN_ROOT}" ] ; then +if [ -z "${KAT_BIN_ROOT+x}" ] ; then export KAT_BIN_ROOT=$(dirname $(realpath "${0}")) fi # The kat repo root -if [ -z "${KAT_REPO_ROOT}" ]; then +if [ -z "${KAT_REPO_ROOT+x}" ]; then # FIXME (reid.mckenzie 2018-10-16): # Decouple Kat from git's root? - export KAT_REPO_ROOT=$(realpath $(git rev-parse --show-toplevel)) + export KAT_REPO_ROOT=$(realpath $(git rev-parse --show-toplevel)) fi -if [ -z "${KAT_CONFIG}" ]; then +if [ -z "${KAT_CONFIG+x}" ]; then KAT_CONFIG="${KAT_REPO_ROOT}/kat.conf" fi # Where do the cache, lockfiles and other state live -if [ -z "${KAT_SERVER_WORK_DIR}" ]; then +if [ -z "${KAT_SERVER_WORK_DIR+x}" ]; then KAT_SERVER_WORK_DIR="${KAT_REPO_ROOT}/"$(_get_conf "${KAT_CONFIG}" server_work_dir .kat.d) fi ## Bootstrapping -if [ ! -f "${KAT_CONFIG}" ]; then - if [ ! -d "${KAT_SERVER_WORK_DIR}" ]; then - mkdir "${KAT_SERVER_WORK_DIR}" - fi +if [ ! -d "${KAT_SERVER_WORK_DIR}" ]; then + mkdir "${KAT_SERVER_WORK_DIR}" +fi - bootstrap_jar="${KAT_SERVER_WORK_DIR}/bootstrap.jar" - if [ ! -f "${bootstrap_jar}" ]; then - curl -L https://github.com/arrdem/katamari/releases/download/v0.0.4/katamari-0.0.4.jar \ - --output "${bootstrap_jar}" - fi +bootstrap_jar="${KAT_SERVER_WORK_DIR}/bootstrap.jar" +if [ ! -f "${bootstrap_jar}" ]; then + curl -L https://github.com/arrdem/katamari/releases/download/v0.0.4/katamari-0.0.4.jar \ + --output "${bootstrap_jar}" +fi +if [ ! -f "${KAT_CONFIG}" ]; then cat < "${KAT_CONFIG}" # Katamari's config file @@ -93,12 +95,15 @@ server_ns=katamari.server.web-server # How do I get away from having to code this? Bootstrapping without a dist is HARD server_classpath=$(realpath "${bootstrap_jar}") -# The log to record build history and any errors -server_log_file=kat.log - # Where to put cached build products and analysis data # This cache lives at the repo root server_work_dir=.kat.d +# The log to record build history and any errors, lives under server_work_dir +server_log_file=kat.log +# server build cache, lives under server_work_dir +server_build_cache=buildcache +# 30 day product TTL +server_build_cache_ttl=2592000000 # (class)paths to (load) after application boot but before the server(s) start server_extensions=[ @@ -111,6 +116,7 @@ server_extensions=[ katamari.server.extensions.core-handlers katamari.server.extensions.fuzzy-not-found katamari.server.extensions.roll-handlers + katamari.roll.extensions.defaults katamari.roll.extensions.jvm katamari.roll.extensions.clj katamari.roll.extensions.jar @@ -129,8 +135,10 @@ deps_defaults_data={} target_dir=target EOF +fi - [ ! -f kat-deps-defaults.edn ] && cat < kat-deps-defaults.edn +if [ ! -f kat-deps-defaults.edn ]; then + cat < kat-deps-defaults.edn ;; An implicit defaults file which is passed to deps.edn as more deps data. ;; ;; This file should be used to configure any Maven or other repositories, to provide global @@ -144,58 +152,62 @@ EOF { "central" {:url "https://repo1.maven.org/maven2/"} "clojars" {:url "https://repo.clojars.org/"} - } } +} EOF +fi - [ ! -f kat-deps-resolve.edn ] && cat < kat-deps-resolve.edn +if [ ! -f kat-deps-resolve.edn ]; then + cat < kat-deps-resolve.edn ;; Deps data which getsapplied as an always-on resolve alias. ;; ;; This file should be used for version pinning and overrides where required. {:default-deps {org.clojure/clojure {:mvn/version "1.9.0"} - } } +} EOF +fi - [ ! -d target ] && mdir target +if [ ! -d target ]; then + mkdir target fi # Katamari's server port -if [ -z "${KAT_SERVER_HTTP_PORT}" ]; then +if [ -z "${KAT_SERVER_HTTP_PORT+x}" ]; then KAT_SERVER_HTTP_PORT=$(_get_conf "${KAT_CONFIG}" server_http_port 3636) fi # Katamari's server port -if [ -z "${KAT_SERVER_ADDR}" ]; then +if [ -z "${KAT_SERVER_ADDR+x}" ]; then KAT_SERVER_ADDR=$(_get_conf "${KAT_CONFIG}" server_addr localhost) fi # Katamari's classpath -if [ -z "${KAT_SERVER_CP}" ]; then +if [ -z "${KAT_SERVER_CP+x}" ]; then KAT_SERVER_CP=$(_get_conf "${KAT_CONFIG}" server_classpath) # FIXME (arrdem 2018-09-26): # How to get away from setting this? Can it be made stand-alone? - if [ -z "${KAT_SERVER_CP}" ]; then + if [ -z "${KAT_SERVER_CP+x}" ]; then >&2 echo "Couldn't find kat's server classpath. Please set 'server_classpath'." exit 1 fi fi # How long to wait for the server to come up -if [ -z "${KAT_SERVER_START_SEC}" ]; then +if [ -z "${KAT_SERVER_START_SEC+x}" ]; then KAT_SERVER_START_SEC=$(_get_conf "${KAT_CONFIG}" server_start_sec 15) fi # What namespace to boot -if [ -z "${KAT_SERVER_NS}" ]; then +if [ -z "${KAT_SERVER_NS+x}" ]; then KAT_SERVER_NS=$(_get_conf "${KAT_CONFIG}" server_ns 'katamari.server.web-server') fi # Where do the logs go -if [ -z "${KAT_SERVER_LOG_FILE}" ]; then +if [ -z "${KAT_SERVER_LOG_FILE+x}" ]; then KAT_SERVER_LOG_FILE=$(_get_conf "${KAT_CONFIG}" server_log_file kat.log) fi @@ -228,7 +240,7 @@ EOF CWD=${KAT_SERVER_WORK_DIR} "${JAVA_CMD}" -cp "${KAT_SERVER_CP}" \ clojure.main -m "${KAT_SERVER_NS}" "${KAT_CONFIG}" \ - 2>&1 >> "${KAT_SERVER_LOG_FILE}" & + >> "${KAT_SERVER_WORK_DIR}/${KAT_SERVER_LOG_FILE}" & KAT_SERVER_PID="$!" disown "${KAT_SERVER_PID}" @@ -355,6 +367,8 @@ EOF ## Core behavior while true; do + [ -z "${1+x}" ] && break + case "${1}" in -r|--raw) KAT_INTENT=raw diff --git a/kat-deps-resolve.edn b/kat-deps-resolve.edn index 29098ec..e4a7772 100644 --- a/kat-deps-resolve.edn +++ b/kat-deps-resolve.edn @@ -8,7 +8,10 @@ org.clojure/data.xml {:mvn/version "0.2.0-alpha5"} org.clojure/tools.gitlibs {:mvn/version "0.2.64"} org.clojure/tools.cli {:mvn/version "0.3.5"} + + ;; The logging swamp org.clojure/tools.logging {:mvn/version "0.5.0-alpha"} + ch.qos.logback/logback-classic {:mvn/version "RELEASE"} ;; Maven ^:group org.apache.maven.resolver {:mvn/version "1.1.1"} @@ -27,6 +30,7 @@ ;; Dev tools instaparse/instaparse {:mvn/version"1.4.9"} io.replikativ/hasch {:mvn/version "0.3.5"} + com.taoensso/nippy {:mvn/version "2.14.0"} pandect/pandect {:mvn/version "0.6.1"} clj-fuzzy {:mvn/version "0.4.1"} me.raynes/fs {:mvn/version "1.4.6"} diff --git a/kat.conf b/kat.conf index a5ba2b0..302f6b9 100644 --- a/kat.conf +++ b/kat.conf @@ -18,18 +18,18 @@ server_start_sec=15 server_ns=katamari.server.web-server # A classpath string to use when booting the server -# Used when bootstrapping Kat -# -# FIXME (arrdem 2018-09-29): -# How do I get away from having to code this? Bootstrapping without a dist is HARD -server_classpath=.kat.d/bootstrap.jar - -# The log to record build history and any errors -server_log_file=kat.log +# If you want to inject plugins or deps - throw those here. +server_classpath=katamari/resources:katamari/src:.kat.d/bootstrap.jar # Where to put cached build products and analysis data # This cache lives at the repo root server_work_dir=.kat.d +# The log to record build history and any errors, lives under server_work_dir +server_log_file=kat.log +# server build cache, lives under server_work_dir +server_build_cache=buildcache +# 30 day product TTL +server_build_cache_ttl=2592000000 # paths to (load) after application boot server_extensions=[ @@ -42,6 +42,7 @@ server_extensions=[ katamari.server.extensions.core-handlers katamari.server.extensions.fuzzy-not-found katamari.server.extensions.roll-handlers + katamari.roll.extensions.defaults katamari.roll.extensions.jvm katamari.roll.extensions.clj katamari.roll.extensions.jar diff --git a/katamari/Rollfile b/katamari/Rollfile index 7f7dd58..2926e9c 100644 --- a/katamari/Rollfile +++ b/katamari/Rollfile @@ -10,8 +10,12 @@ {org.clojure/clojure nil org.clojure/tools.deps.alpha nil org.clojure/clojure-tools nil + + ;; The logging swamp org.clojure/tools.logging nil + ch.qos.logback/logback-classic nil + ;; Ring ring/ring nil ring/ring-jetty-adapter nil ring/ring-json nil @@ -23,6 +27,7 @@ instaparse/instaparse nil clj-fuzzy nil io.replikativ/hasch nil + com.taoensso/nippy nil ;; Embedded development nrepl/nrepl nil diff --git a/katamari/dev/user.clj b/katamari/dev/user.clj index 2a472d9..eaa6d2c 100644 --- a/katamari/dev/user.clj +++ b/katamari/dev/user.clj @@ -1,8 +1,9 @@ (ns user - (:require [katamari.roll.reader])) + (:require [me.raynes.fs :as fs] + [katamari.roll.reader])) (def +root+ - "/Users/reid.mckenzie/Documents/dat/git/arrdem/katamari") + (str (System/getenv "HOME") "/doc/dat/git/arrdem/katamari")) (def +conf+ (merge (katamari.conf/load (str +root+ "/kat.conf") @@ -11,3 +12,9 @@ (def +graph+ (katamari.roll.reader/compute-buildgraph +conf+)) + +(def +cache+ + (katamari.roll.cache/->buildcache + (fs/file (:repo-root +conf+) + (:server-work-dir +conf+) + (:server-build-cache +conf+)))) diff --git a/katamari/resources/logback.xml b/katamari/resources/logback.xml new file mode 100644 index 0000000..314cf77 --- /dev/null +++ b/katamari/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/katamari/src/katamari/roll/cache.clj b/katamari/src/katamari/roll/cache.clj new file mode 100644 index 0000000..038ef3d --- /dev/null +++ b/katamari/src/katamari/roll/cache.clj @@ -0,0 +1,119 @@ +(ns katamari.roll.cache + "Provides a pseudo-tempdir cache tree. + + Used to provide a somewhat content addressable store of build products by + their build rule ID. + + " + {:authors ["Reid 'arrdem' McKenzie "]} + (:require [me.raynes.fs :as fs] + [clojure.edn :as edn] + [clojure.tools.logging :as log])) + +;; FIXME (arrdem 2018-10-28): +;; +;; Rethink this API so that buildcaches are hierarchical, and remote fetch is +;; achievable. It should be possible to build a shared say team / org +;; buildcache which nobody ever really misses in. + +(defn ->buildcache + "Given a root path, returns a `::buildcache` usable with this API." + [root] + (let [^java.io.File root (fs/file root)] + (.mkdirs root) + {:type ::buildcache + :root root})) + +(defn- key-prefix + "Used to emulate Git's two level content tree addressing. + + This reduces strain on the host directory system, as keys are grouped + by (small!) prefixes, reducing total directory size of the cache root and of + the buckets below the root." + [key] + (.substring ^String key 0 2)) + +(defn ^java.io.File get-key* + "Implementation detail. + + Used to return the root for a cache key." + [{:keys [^java.io.File root]} key] + (fs/file root (key-prefix key) key)) + +(defn get-product + "Given a cache key, attempts to return the cache entry dir for that key. + + Returns `nil` if there is no such entry." + [buildcache key] + (let [product-root (get-key* buildcache key)] + (when (and (.exists product-root) (.isDirectory product-root)) + ;; Mark the dir so we can track when it was last used. This lets us use + ;; dir modified timestamps to order build products in least frequently + ;; used order. + (.setLastModified product-root (System/currentTimeMillis)) + (edn/read (java.io.PushbackReader. + (java.io.BufferedReader. + (java.io.FileReader. + (fs/file product-root "product.edn")))))))) + +(defn get-workdir + "Given a cache key, creates and returns the product directory for the specified key." + [buildcache key] + (let [target-dir (fs/file (get-key* buildcache key) "target")] + (.mkdirs target-dir) + target-dir)) + +(defn put-product + "Given a cache key and a product as a serializable datastructure, dumps that + product into the given cache key. + + Future calls to `#'get-product` shall return an equivalent structure if at all + possible." + [buildcache key product] + (let [product-root (get-key* buildcache key)] + (binding [*out* (java.io.FileWriter. (fs/file product-root "product.edn"))] + (prn product)))) + +;;; Cache enumeration + +;; FIXME (arrdem 2018-10-28): +;; There are DEFINITELY concurrent modification bugs here + +(defn seq-products + "Given a buildcache, returns a `[k, v]` seq of all keys in the cache and their + roots." + [{:keys [^java.io.File root] :as buildcache}] + (for [^java.io.File child (.listFiles root) + :when (.isDirectory child) + ^java.io.File key (.listFiles child) + :when (.isDirectory key)] + [(.getName key) key])) + +;;; Cache cleaning + +(defn filter-cache-by-ttl + "Given a TTL in milliseconds, select cache products which haven't been accessed + in at least that long, sorting by least recently accessed. + + Returns a `[k, v]` seq of all keys in the cache, same as `#'seq-products`." + [buildcache ttl] + (let [now (System/currentTimeMillis)] + (->> (seq-products buildcache) + (filter (fn [[k ^java.io.File f]] + (< (.lastModified f) (- now ttl)))) + (sort-by (fn [[_ ^java.io.File %]] (.lastModified %)))))) + +(def +thirty-days-ms+ (* 1 1000 60 60 24 30)) +(def +seven-days-ms+ (* 1 1000 60 60 24 30)) + +;; FIXME (arrdem 2018-10-28): +;; There are DEFINITELY concurrent modification bugs to be stomped here. + +(defn clean-products + "Given a `[k, v]` seq of products say from `#'filter-cache-by-ttl`, recursively + delete the selected cache keys and all files under them." + [products] + (doseq [[_ ^java.io.File f] products + child (reverse (file-seq f))] + (printf "Deleting file %s\n" child) + (.delete child))) diff --git a/katamari/src/katamari/roll/core.clj b/katamari/src/katamari/roll/core.clj index 4ad035a..1c3b801 100644 --- a/katamari/src/katamari/roll/core.clj +++ b/katamari/src/katamari/roll/core.clj @@ -2,7 +2,10 @@ "The API by which to execute rolling." {:authors ["Reid 'arrdem' McKenzie "]} (:require [clojure.spec.alpha :as s] + [clojure.tools.logging :as log] + [me.raynes.fs :as fs] [katamari.diff :as diff] + [katamari.roll.cache :as cache] [katamari.roll.specs :as rs] [katamari.roll.extensions :refer :all])) @@ -164,8 +167,9 @@ (s/fdef roll :args (s/cat :conf any? + :cache any? :graph ::rs/buildgraph - :target (s/? ::rs/target)) + :targets (s/? (s/coll-of ::rs/target))) :ret (s/map-of ::rs/target any?)) (defn roll @@ -175,21 +179,30 @@ supplied only that target and its dependencies. Return a mapping from built targets to their products." - [config {:keys [targets] :as buildgraph} & [targets?]] + [config cache {:keys [targets] :as buildgraph} & [targets?]] (let [[config targets plan] (plan config buildgraph targets?)] (reduce (fn [products target] - (let [rule (get targets target) - inputs (into {} - (map (fn [[key targets]] - [key (mapv #(get products %) targets)])) - (rule-inputs config buildgraph target rule))] + (let [rule (get targets target) + inputs (into {} + (map (fn [[key targets]] + [key (mapv #(get products %) targets)])) + (rule-inputs config buildgraph target rule)) + id (rule-id config buildgraph target rule products inputs) + _ (log/infof "Building %s@%s\n" target id) + product (if-let [cached-product (cache/get-product cache id)] + (do (log/debugf "Hit the product cache!\n") + cached-product) + ;; Fill the cache + (do (log/infof "Missed the cache, filling\n") + (let [dir (cache/get-workdir cache id) + _ (log/infof "Trying to build in workdir %s" dir) + product (fs/with-cwd dir + (-> (rule-build config buildgraph + target rule + products inputs) + (assoc :id id)))] + (cache/put-product cache id product) + product)))] (assoc products - target (rule-build config buildgraph - target rule - products inputs)))) - ;; FIXME (arrdem 2018-10-20): - ;; Lol no product caching - {} - ;; FIXME (arrdem 2018-10-20): - ;; Lol sequential execution - (apply concat plan)))) + target product))) + {} (apply concat plan)))) diff --git a/katamari/src/katamari/roll/extensions.clj b/katamari/src/katamari/roll/extensions.clj index 7815981..72ccd75 100644 --- a/katamari/src/katamari/roll/extensions.clj +++ b/katamari/src/katamari/roll/extensions.clj @@ -8,9 +8,11 @@ ;;; Manifests -(def ^{:doc "Multi-spec. Given a rule, being a list `(manifest & kvs)`, - dispatch on the manifest to methods returning an `s/keys*` form for the - remaining kvs. These kvs together with the manifest will define a build rule." +(def ^{:doc "Multi-spec. + + Given a rule, being a list `(manifest & kvs)`, dispatch on the manifest to + methods returning an `s/keys*` form for the remaining kvs. These kvs together + with the manifest will define a build rule." :arglists '([rule-expr])} parse-manifest @#'rs/parse-manifest) @@ -70,10 +72,6 @@ in the filesystem or on the path."} (fn [_conf _graph manifest] manifest)) -(defmethod manifest-prep :default [config buildgraph manifest] - #_(printf "No configured prep.manifest for %s\n" manifest) - [config buildgraph]) - (s/fdef rule-prep :args (s/cat :conf any? :graph :katamari.roll.specs/buildgraph @@ -92,11 +90,6 @@ By default, tasks require no preparation."} rule-prep #'dispatch) -(defmethod rule-prep :default [config buildgraph target rule] - #_(printf "No configured prep.rule for manifest %s (target %s)\n" - (rule-manifest rule) target) - [config buildgraph]) - ;;; The dependency tree (s/fdef rule-inputs) @@ -115,31 +108,32 @@ without having to take separate steps to recover information about those deps."} rule-inputs #'dispatch) -(defmethod rule-inputs :default [config buildgraph target rule] - (throw (ex-info "No `rule-inputs` implementation for manifest!" - {:target target - :rule rule - :manifest (rule-manifest rule)}))) - (defmulti - ^{:arglusts '([config buildgraph target rule products inputs]) + ^{:arglists '([config buildgraph target rule products inputs]) :doc "Compute and return a content hash string for this build target. The returned string is required to match `#\"[a-z0-9]{32,}\"`, must be deterministic, cheap to compute and factor in the `rule-id` of its inputs as well as the content of any files on the path. +Any change to your rule's inputs which would cause a different output to be +produced MUST cause the key returned by this function to change. + Invoked once per rule in the build graph, in topological order."} rule-id #'dispatch) -(defmethod rule-id :default [config buildgraph target rule products inputs] - ) - (defmulti ^{:arglists '([config buildgraph target rule products inputs]) - :doc "Apply the rule to its inputs, producing a build product."} + :doc "Apply the rule to its inputs, producing a build product. + +Rule builds will execute with `#'raynes.fs/*cwd*` bound to a writable output +directory unique to this build rule by it's self-reported `rule-id`. Rules MUST +produce any file products which will be referred to in the build product into +this directory. + +Rules are strongly discouraged from using say other tempdirs."} rule-build #'dispatch) diff --git a/katamari/src/katamari/roll/extensions/defaults.clj b/katamari/src/katamari/roll/extensions/defaults.clj new file mode 100644 index 0000000..acc390f --- /dev/null +++ b/katamari/src/katamari/roll/extensions/defaults.clj @@ -0,0 +1,31 @@ +(ns katamari.roll.extensions.defaults + "Sane-ish default implementations of the roll extensions API." + {:authors ["Reid 'arrdem' McKenzie "]} + (:require [katamari.roll.extensions :as ext] + [me.raynes.fs :as fs] + [hasch.core :as hasch] + [pandect.algo.sha256 :refer [sha256-file]])) + +(defmethod ext/manifest-prep :default [config buildgraph manifest] + #_(printf "No configured prep.manifest for %s\n" manifest) + [config buildgraph]) + +(defmethod ext/rule-prep :default [config buildgraph target rule] + #_(printf "No configured prep.rule for manifest %s (target %s)\n" + (rule-manifest rule) target) + [config buildgraph]) + +(defmethod ext/rule-inputs :default [config buildgraph target rule] + (throw (ex-info "No `rule-inputs` implementation for manifest!" + {:target target + :rule rule + :manifest (ext/rule-manifest rule)}))) + +(defmethod ext/rule-id :default [config buildgraph target rule products inputs] + (str (hasch/uuid [target rule inputs + (into (sorted-set) + (comp (map fs/file) + (mapcat file-seq) + (remove #(.isDirectory ^java.io.File %)) + (map sha256-file)) + (:paths rule))]))) diff --git a/katamari/src/katamari/roll/extensions/jar.clj b/katamari/src/katamari/roll/extensions/jar.clj index 4427cb4..9da4edf 100644 --- a/katamari/src/katamari/roll/extensions/jar.clj +++ b/katamari/src/katamari/roll/extensions/jar.clj @@ -56,8 +56,7 @@ (defmethod ext/rule-build 'jar [config buildgraph target rule products {:keys [targets] :as inputs}] - (let [target-dir (fs/file (:repo-root config) - (:target-dir config)) + (let [target-dir fs/*cwd* jar-name (:jar-name rule (str (name target) ".jar")) jar-file (fs/file target-dir jar-name) canonical-path (.getCanonicalPath jar-file)] @@ -104,8 +103,7 @@ (rejvm/make-classpath config products {:deps (:deps rule)}) - target-dir (fs/file (:repo-root config) - (:target-dir config)) + target-dir fs/*cwd* jar-name (:jar-name rule (str (name target) ".jar")) jar-file (fs/file target-dir jar-name) canonical-path (.getCanonicalPath jar-file)] diff --git a/katamari/src/katamari/roll/extensions/jvm.clj b/katamari/src/katamari/roll/extensions/jvm.clj index fe29bdd..37e0046 100644 --- a/katamari/src/katamari/roll/extensions/jvm.clj +++ b/katamari/src/katamari/roll/extensions/jvm.clj @@ -114,31 +114,33 @@ products inputs] - (fs/with-cwd (:repo-root config) - (let [source-files (->> (map (partial fs/file) paths) - (mapcat file-seq) - (filter #(.isFile %)) - (map #(.getCanonicalPath %))) - dest-dir (->> (into-array FileAttribute []) - (Files/createTempDirectory "javac") - (.toFile) - (.getCanonicalPath))] - - ;; FIXME (arrdem 2018-10-21): - ;; Capture the exit results nicely - (when source-files - (let [cp (make-classpath config products - {:deps (:deps target)}) - cmd (cond-> ["javac"] - (:classpath cp) (into ["-cp" (:classpath cp)]) - source-version (into ["-source" source-version]) - target-version (into ["-target" target-version]) - true (-> (into ["-d" dest-dir]) - (into source-files)))] - (apply sh/sh cmd))) - - {:type ::product - :from target - :mvn/manifest :roll - :deps (:deps rule {}) - :paths [dest-dir]}))) + (let [source-files (->> (map (partial fs/file) paths) + (mapcat file-seq) + (filter #(do (prn %) + (.isFile %))) + (map #(.getCanonicalPath %))) + dest-dir (.getCanonicalPath fs/*cwd*)] + (when source-files + (let [cp (make-classpath config products + {:deps (:deps target)}) + cmd (cond-> ["javac"] + (:classpath cp) (into ["-cp" (:classpath cp)]) + source-version (into ["-source" source-version]) + target-version (into ["-target" target-version]) + true (-> (into ["-d" dest-dir]) + (into source-files) + (into [:dir fs/*cwd*]))) + res (apply sh/sh cmd)] + + (when (zero? (:exit res)) + (throw (ex-info "Failed to javac" + (assoc res + :target target + :rule rule + :command cmd)))))) + + {:type ::product + :from target + :mvn/manifest :roll + :deps (:deps rule {}) + :paths [dest-dir]})) diff --git a/katamari/src/katamari/server/extensions/fuzzy_not_found.clj b/katamari/src/katamari/server/extensions/fuzzy_not_found.clj index 060e4e4..57722cb 100644 --- a/katamari/src/katamari/server/extensions/fuzzy_not_found.clj +++ b/katamari/src/katamari/server/extensions/fuzzy_not_found.clj @@ -4,7 +4,7 @@ (:require [clojure.string :as str] [katamari.server.extensions :refer [defwrapper]] [ring.util.response :as resp] - [clj-fuzzy.metrics :as fuz])) + [clj-fuzzy.jaro-winkler :refer [jaro]])) (defwrapper wrap-not-found [handler config stack request] @@ -16,7 +16,7 @@ word (.replaceAll (first request) "-" "") candidates (->> meta (map :kat/task-name) - (sort-by #(fuz/jaro word (.replaceAll % "-" "")) + (sort-by #(jaro word (.replaceAll % "-" "")) #(> %1 %2)) (take 3))] (-> {:intent :msg diff --git a/katamari/src/katamari/server/extensions/roll_handlers.clj b/katamari/src/katamari/server/extensions/roll_handlers.clj index 065b3c2..f995148 100644 --- a/katamari/src/katamari/server/extensions/roll_handlers.clj +++ b/katamari/src/katamari/server/extensions/roll_handlers.clj @@ -18,6 +18,7 @@ ;; Katamari [katamari.roll.core :as roll] [katamari.roll.reader :refer [compute-buildgraph refresh-buildgraph-for-changes]] + [katamari.roll.cache :as cache] [katamari.deps.extensions.roll :as der] [katamari.server.extensions :refer [defhandler defwrapper]] @@ -40,13 +41,19 @@ #(or (and % (refresh-buildgraph-for-changes config %)) (compute-buildgraph config))) (:repo-root config))] - (handler (assoc config :buildgraph graph) stack request))) + (handler (assoc config + :buildgraph graph + :buildcache (cache/->buildcache + (fs/file (:repo-root config) + (:server-work-dir config) + (:server-build-cache config)))) + stack request))) (defhandler compile "Compile specified target(s), producing any outputs. Usage: - ./compile target + ./kat compile target1 target2... Causes the specified targets to be compiled. @@ -57,7 +64,7 @@ Produces a map from target identifiers to build products." (case (first request) "compile" (if-let [targets (map symbol (rest request))] - (-> (roll/roll config (:buildgraph config) targets) + (-> (roll/roll config (:buildcache config) (:buildgraph config) targets) (assoc :intent :json) resp/response (resp/status 200)) @@ -76,3 +83,28 @@ Produces a map from target identifiers to build products." :targets (-> config :buildgraph :targets keys)} resp/response (resp/status 200))) + +(defhandler clean-cache + "Flush the cache on a TTL basis. + +Usage: + ./kat clean-cache [ttl-ms] + +Removes cache entries older than the specified number of milliseconds, +defaulting to the server's configured cache TTL." + [handler config stack request] + (case (first request) + "clean-cache" + (if-let [ttl (or (some-> request second Long/parseLong) + (some-> config :server-build-cache-ttl Long/parseLong))] + (let [products (cache/filter-cache-by-ttl (:buildcache config) ttl)] + (cache/clean-products products) + (-> {:intent :json + :deleted-keys (map first products)} + resp/response + (resp/status 200))) + + (-> {:intent :msg + :msg "No TTL provided!"} + resp/response + (resp/status 400))))) diff --git a/katamari/src/katamari/server/web_server.clj b/katamari/src/katamari/server/web_server.clj index e06b0e3..e55c269 100644 --- a/katamari/src/katamari/server/web_server.clj +++ b/katamari/src/katamari/server/web_server.clj @@ -32,8 +32,8 @@ (defn json-response "Helper for returning JSON coded responses. - JSON codes the given object in a single shot, returning a 200 response by default unless the user - specifies an optional status code." + JSON codes the given object in a single shot, returning a 200 response by + default unless the user specifies an optional status code." ([obj] (json-response obj 200)) ([obj code] @@ -92,9 +92,18 @@ (let [cfg (conf/load config-file key-fn)] (log/info "Loaded config" cfg) (doseq [path (:server-extensions cfg)] - (try (require (symbol path)) - (log/infof "Loaded extension %s" path) - (catch Exception e - (log/error e "Failed to load extension!")))) + (let [start (System/currentTimeMillis) + libs (clojure.core/loaded-libs) + error? (try (require (symbol path)) + (catch Exception e e)) + libs* (clojure.core/loaded-libs) + end (System/currentTimeMillis)] + (if error? + (log/fatalf error? "Failed to load extension %s" path) + (do (log/infof "Loaded extension %s in %d ms" path (- end start)) + (doseq [lib libs* + :when (not (contains? libs lib)) + :when (not= path (name lib))] + (log/infof " Also loaded %s" lib)))))) (start-web-server! cfg) (start-nrepl-server! cfg)))