Add Bazel path mapping support

WORKSPACE

@@ -51,7 +51,8 @@ maven_install(
         "org.clojure:data.json:2.4.0",
         "org.clojure:java.classpath:1.0.0",
         "org.clojure:tools.namespace:1.1.0",
-        "org.clojure:tools.deps.alpha:0.14.1212"
+        "org.clojure:tools.deps.alpha:0.14.1212",
+        "org.clojure:tools.cli:1.2.245"
     ],
     maven_install_json = "@//:frozen_deps_install.json",
     fail_if_repin_required = True,

rules.bzl

@@ -10,7 +10,7 @@ clojure_library = rule(
         "data": attr.label_list(default = [], allow_files = True),
         "resources": attr.label_list(default=[], allow_files=True),
         "aot": attr.string_list(default = [], doc = "namespaces to be compiled"),
-        "resource_strip_prefix": attr.string(default = ""),
+        "resource_strip_prefix": attr.string(),
         "compiledeps": attr.label_list(default = []),
         "javacopts": attr.string_list(default = [], allow_empty = True, doc = "Optional javac compiler options"),
         "jvm_flags": attr.string_list(default=[], doc = "Optional jvm_flags to pass to the worker binary"),

rules/jar.bzl

@@ -10,68 +10,6 @@ def distinct(lst):
         d[i] = True
     return d.keys()
 
-def paths(resources, resource_strip_prefix):
-    """Return a list of path tuples (target, source) where:
-        target - is a path in the archive (with given prefix stripped off)
-        source - is an absolute path of the resource file
-
-    Tuple ordering is aligned with zipper format ie zip_path=file
-
-    Args:
-        resources: list of file objects
-        resource_strip_prefix: string to strip from resource path
-    """
-    return [(_target_path(resource, resource_strip_prefix), resource.path) for resource in resources]
-
-def _strip_prefix(path, prefix):
-    return path[len(prefix):] if path.startswith(prefix) else path
-
-def _target_path(resource, resource_strip_prefix):
-    path = _target_path_by_strip_prefix(resource, resource_strip_prefix) if resource_strip_prefix else _target_path_by_default_prefixes(resource)
-    return _strip_prefix(path, "/")
-
-def _target_path_by_strip_prefix(resource, resource_strip_prefix):
-    # Start from absolute resource path and then strip roots so we get to correct short path
-    # resource.short_path sometimes give weird results ie '../' prefix
-    path = resource.path
-    if resource_strip_prefix != resource.owner.workspace_root:
-        path = _strip_prefix(path, resource.owner.workspace_root + "/")
-    path = _strip_prefix(path, resource.root.path + "/")
-
-    # proto_library translates strip_import_prefix to proto_source_root which includes root so we have to strip it
-    prefix = _strip_prefix(resource_strip_prefix, resource.root.path + "/")
-    if not path.startswith(prefix):
-        fail("Resource file %s is not under the specified prefix %s to strip" % (path, prefix))
-    return path[len(prefix):]
-
-def _target_path_by_default_prefixes(resource):
-    path = resource.path
-
-    #  Here we are looking to find out the offset of this resource inside
-    #  any resources folder. We want to return the root to the resources folder
-    #  and then the sub path inside it
-    dir_1, dir_2, rel_path = path.partition("resources")
-    if rel_path:
-        return rel_path
-
-    #  The same as the above but just looking for java
-    (dir_1, dir_2, rel_path) = path.partition("java")
-    if rel_path:
-        return rel_path
-
-    # Both short_path and path have quirks we wish to avoid, in short_path there are times where
-    # it is prefixed by `../` instead of `external/`. And in .path it will instead return the entire
-    # bazel-out/... path, which is also wanting to be avoided. So instead, we return the short-path if
-    # path starts with bazel-out and the entire path if it does not.
-    return resource.short_path if path.startswith("bazel-out") else path
-
-def restore_prefix(src, stripped):
-    """opposite of _target_path. Given a source and stripped file, return the prefix """
-    if src.path.endswith(stripped):
-        return src.path[:len(src.path)-len(stripped)]
-    else:
-        fail("Resource file %s is not under the specified prefix %s to strip" % (src, stripped))
-
 def argsfile_name(label):
     return str(label).replace("@","_").replace("/","_") + "_args"
 
@@ -122,55 +60,51 @@ def clojure_jar_impl(ctx):
 
     input_files = ctx.files.srcs + ctx.files.resources
 
-    if len(input_files):
-        src_dir = restore_prefix(input_files[0], _target_path(input_files[0], ctx.attr.resource_strip_prefix))
-    else:
-        src_dir = None
-
-    compile_classpath = compile_info.transitive_runtime_jars.to_list() + ctx.files.compiledeps + [classes_dir]
-    compile_classpath = [f.path for f in compile_classpath]
-    compile_classpath = compile_classpath + [p for p in [src_dir] if p]
+    compile_classpath = depset(
+        ctx.files.compiledeps + [classes_dir],
+        transitive = [compile_info.transitive_runtime_jars],
+    )
 
     native_libs = []
-    for f in runfiles.files.to_list():
-        ## Bazel on mac sticks weird looking directories in runfiles, like _solib_darwin/_U_S_Snative_C_Ulibsodium___Unative_Slibsodium_Slib. filter them out
-        if (f.path.endswith(".dylib") or f.path.endswith(".so")) and (f.path.rfind("solib_darwin") == -1):
-            native_libs.append(f)
 
     aot_nses = distinct(aot_nses)
 
     javaopts_str = " ".join(ctx.attr.javacopts)
 
-    compile_args = {"classes-dir": classes_dir.path,
-                    "output-jar": output_jar.path,
-                    "src-dir": src_dir,
-                    "srcs": [_target_path(s, ctx.attr.resource_strip_prefix) for s in ctx.files.srcs],
-                    "resources": [_target_path(s, ctx.attr.resource_strip_prefix) for s in ctx.files.resources],
-                    "aot-nses": aot_nses,
-                    "classpath": compile_classpath}
+    compile_args = ctx.actions.args()
+    compile_args.use_param_file("@%s", use_always = True)
+
+    compile_args.add_all([classes_dir], before_each = "--classes-dir", expand_directories = False)
+    compile_args.add_all([output_jar], before_each = "--output-jar", expand_directories = False)
 
-    args_file = ctx.actions.declare_file(argsfile_name(ctx.label))
-    ctx.actions.write(
-        output = args_file,
-        content = json.encode(compile_args))
+    if ctx.attr.resource_strip_prefix != "":
+        compile_args.add("--resource-strip-prefix")
+        compile_args.add(ctx.attr.resource_strip_prefix)
 
-    inputs = ctx.files.srcs + ctx.files.resources + compile_info.transitive_runtime_jars.to_list() + native_libs + [args_file] + worker_classpath_depset.to_list()
+    compile_args.add_all(ctx.files.srcs, before_each="--src")
+    compile_args.add_all(ctx.files.resources, before_each="--resource")
+    compile_args.add_all(aot_nses, before_each="--aot-nses")
+    compile_args.add("--classpath")
+    compile_args.add_joined(compile_classpath, join_with=":")
 
-    worker_classpath_str = ":".join([d.path for d in worker_classpath_depset.to_list()])
+    inputs = depset(
+        ctx.files.srcs + ctx.files.resources +  native_libs,
+        transitive = [compile_info.transitive_runtime_jars, worker_classpath_depset],
+    )
 
     ctx.actions.run(
         executable= ctx.executable._clojureworker_binary,
         arguments=
         ["--jvm_flags=" + f for f in ctx.attr.jvm_flags] +
-        ["-m", "rules-clojure.worker",
-         "@%s" % args_file.path],
+        ["-m", "rules-clojure.worker"] + [compile_args],
         outputs = [output_jar, classes_dir],
         inputs = inputs,
         mnemonic = "ClojureCompile",
         progress_message = "Compiling %s" % ctx.label,
         execution_requirements={"supports-workers": "1",
                                 "supports-multiplex-workers": "1",
-                                "requires-worker-protocol": "json"})
+                                "requires-worker-protocol": "json",
+                                "supports-path-mapping": "1"})
 
     return [
         default_info,

src/rules_clojure/BUILD

@@ -24,7 +24,8 @@ java_library(
                   "@rules_clojure_maven_deps//:org_clojure_core_cache",
                   "@rules_clojure_maven_deps//:org_clojure_data_json",
                   "@rules_clojure_maven_deps//:org_clojure_java_classpath",
-                  "@rules_clojure_maven_deps//:org_clojure_tools_namespace"])
+                  "@rules_clojure_maven_deps//:org_clojure_tools_namespace",
+                  "@rules_clojure_maven_deps//:org_clojure_tools_cli"])
 
 java_binary(
     name="bootstrap-bin",
@@ -62,6 +63,8 @@ java_import(name="libcompile",
             jars=["libcompile.jar"],
             data=[":bootstrap-compiler"])
 
+
+
 java_binary(name="worker",
             main_class="clojure.main",
             jvm_flags=["-Dclojure.main.report=stderr",
@@ -90,6 +93,7 @@ clojure_library(
           "@rules_clojure_maven_deps//:org_clojure_data_json",
           "libfs"],
     aot=["clojure.java.classpath",
+         "clojure.tools.namespace.parse",
          "clojure.tools.deps.alpha.extensions",
          "clojure.tools.deps.alpha.util.session",
          "clojure.tools.deps.alpha.util.io",

src/rules_clojure/worker.clj

@@ -3,17 +3,20 @@
             [clojure.data.json :as json]
             [clojure.java.io :as io]
             [clojure.spec.alpha :as s]
+            [clojure.string :as str]
+            [clojure.tools.cli :refer [parse-opts]]
             [rules-clojure.fs :as fs]
             [rules-clojure.jar :as jar]
             [rules-clojure.util :as util]
             [rules-clojure.persistent-classloader :as pcl])
   (:import java.nio.charset.StandardCharsets
-           java.util.concurrent.TimeUnit))
+           java.util.concurrent.TimeUnit
+           [java.util.logging ConsoleHandler FileHandler Logger Level SimpleFormatter]
+           java.lang.ProcessHandle))
 
 (s/def ::classes-dir string?) ;; path to the *compile-files* dir
 (s/def ::output-jar string?) ;; path where the output jar should be written
-(s/def ::srcs (s/coll-of string?)) ;; seq of paths to src files to put on classpath while compiling. Relative to src-dir
-(s/def ::src-dir (s/nilable string?)) ;; path to root of source tree, relative to execroot
+(s/def ::srcs (s/coll-of string?)) ;; seq of paths to src files to put on classpath while compiling.
 (s/def ::resources (s/coll-of string?)) ;; seq of paths to include in the jar
 (s/def ::aot-nses (s/coll-of string?)) ;; seq of namespaces to AOT
 (s/def ::classpath (s/coll-of string?)) ;; seq of jars to put on compile classpath
@@ -23,8 +26,7 @@
                                           ::classpath
                                           ::aot-nses]
                                  :opt-un [::resources
-                                          ::srcs
-                                          ::src-dir]))
+                                          ::srcs]))
 
 (s/def ::arguments (s/cat :c ::compile-req))
 
@@ -46,6 +48,24 @@
 (defn all-classpath-jars [classpath]
   (set classpath))
 
+;; bazel requires us to write to stdout, and doesn't reliably report
+;; stderr, so log to a temp file to guarantee we find everything.
+
+(defn pid []
+  (-> (ProcessHandle/current) .pid))
+
+(defn configure-logging! []
+  (let [handler (FileHandler. (format "/tmp/rules-clojure-worker-%s.log" (pid)))
+        formatter (SimpleFormatter.)
+        logger (Logger/getLogger (str *ns*))]
+    (.setFormatter handler formatter)
+    (.addHandler logger handler)
+    (.addHandler logger (ConsoleHandler.))
+    (.setLevel logger Level/INFO)))
+
+(defn log [& args]
+  (Logger/.log (Logger/getLogger (str *ns*)) Level/INFO (apply str args)))
+
 (defn process-request
   [{:keys [classloader-strategy
            input-map] :as req}]
@@ -103,10 +123,9 @@
         real-out *out*]
     (let [exit (binding [*out* out-printer]
                  (try
-                   (let [compile-req (json/read-str (first arguments) :key-fn keyword)]
-                     (process-request (assoc compile-req
-                                             :classloader-strategy classloader-strategy
-                                             :input-map (input-map inputs))))
+                   (process-request (assoc work-req
+                                           :classloader-strategy classloader-strategy
+                                           :input-map (input-map inputs)))
                    0
                    (catch Throwable t
                      (println t) ;; print to bazel str
@@ -117,42 +136,83 @@
                  :output (str baos)}
                 (when requestId
                   {:requestId requestId}))]
+      (util/print-err "persistent done:" resp)
       (.write real-out (json/write-str resp))
       (.write real-out "\n")
       (.flush real-out))))
 
+;; [--classes-dir bazel-out/darwin_arm64-fastbuild/bin/external/deps/.ns_metosin_reitit_core_reitit_exception.classes --output-jar bazel-out/darwin_arm64-fastbuild/bin/external/deps/ns_metosin_reitit_core_reitit_exception.jar --resource-strip-prefix '' --aot-ns reitit.exception --classpath external/deps/repository/metosin/reitit-core/0.6.0/reitit-core-0.6.0.jar:external/deps/repository/meta-merge/meta-merge/1.0.0/meta-merge-1.0.0.jar:external/deps/repository/org/clojure/clojure/1.12.2/clojure-1.12.2.jar:external/deps/repository/org/clojure/core.specs.alpha/0.4.74/core.specs.alpha-0.4.74.jar:external/deps/repository/org/clojure/spec.alpha/0.5.238/spec.alpha-0.5.238.jar:bazel-out/darwin_arm64-fastbuild/bin/external/rules_clojure/src/rules_clojure/libcompile.jar]
+
+(defn parse-classpath [classpath-str]
+  (str/split classpath-str #":"))
+
+(def cli-options
+  ;; An option with an argument
+  [[nil "--classes-dir dir" "output directory where classfiles will be written"]
+   [nil "--output-jar jar" "output jar name"]
+   [nil "--resource-strip-prefix path" ]
+   [nil "--aot-nses ns" "names of namespaces to AOT. May be repeated"
+    :default []
+    :update-fn conj
+    :multi true]
+   [nil "--classpath cp" "classpath to use while compiling, separated by :"
+    :parse-fn parse-classpath]])
+
+(defn parse-arguments [^String args]
+  {:post [(do (log "worker parse-req" args "=>" %) true)]}
+  (-> args
+      (parse-opts cli-options)
+      :options))
+
 (defn process-persistent []
   (let [executor (java.util.concurrent.Executors/newWorkStealingPool)
         classloader-strategy (pcl/caching-threadsafe)]
     (loop []
-      (if-let [line (read-line)]
-        (let [work-req (json/read-str line :key-fn keyword)]
-          (util/print-err "got req" work-req)
-          (let [out *out*
-                err *err*]
-            (.submit executor ^Runnable (fn []
-                                          (binding [*out* out
-                                                    *err* err]
-                                            (process-persistent-1 (assoc work-req
-                                                                         :classloader-strategy classloader-strategy)))))
-            (recur)))
-        (do
-          (util/print-err "no request, exiting")
-          (.shutdown executor)
-          (util/print-err "awating task completion")
-          (util/print-err "finished cleanly?" (.awaitTermination executor 60 TimeUnit/SECONDS))
-          :exit)))))
+      (log "blocking on read-line")
+      (let [line (read-line)]
+        (if (and line (seq line))
+          (let [_ (log "persistent: line" line)
+                work-req (json/read-str line :key-fn keyword)
+                arguments (parse-arguments (:arguments work-req))
+                prefix (:resource-strip-prefix arguments)
+                _ (log "persistent: prefix:" prefix)
+                arguments (if (seq prefix)
+                            (update arguments :classpath (fn [classpath] (distinct (conj classpath prefix))))
+                            arguments)
+                work-req (-> work-req
+                             (dissoc :arguments)
+                             (merge arguments))]
+            (log "persistent: req" work-req)
+            (let [out *out*
+                  err *err*]
+              (.submit executor ^Runnable (fn []
+                                            (binding [*out* out
+                                                      *err* err]
+                                              (process-persistent-1 (assoc work-req
+                                                                           :classloader-strategy classloader-strategy)))))
+              (recur)))
+          (do
+            (log "no request, exiting")
+            (.shutdown executor)
+            (log "awating task completion")
+            (log "finished cleanly?" (.awaitTermination executor 60 TimeUnit/SECONDS))
+            :exit))))))
 
 (defn set-uncaught-exception-handler! []
   (Thread/setDefaultUncaughtExceptionHandler
    (reify Thread$UncaughtExceptionHandler
      (uncaughtException [_ _ ex]
-       (util/print-err ex "uncaught exception")))))
+       (log ex "uncaught exception")))))
 
 (defn -main [& args]
   (set-uncaught-exception-handler!)
+  (configure-logging!)
   (let [persistent? (some (fn [a] (= "--persistent_worker" a)) args)
         f (if persistent?
             (fn [_args] (process-persistent))
             process-ephemeral)]
-    (f args)))
+    (try
+      (f args)
+      (catch Exception e
+        (log e)
+        (throw e)))))