From b205a2e6706b840e9534b7a2ecf90619cc944ac2 Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Wed, 18 Jul 2018 12:52:28 +0200 Subject: [PATCH 01/10] add deps.edn --- codox/deps.edn | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 codox/deps.edn diff --git a/codox/deps.edn b/codox/deps.edn new file mode 100644 index 0000000..400b197 --- /dev/null +++ b/codox/deps.edn @@ -0,0 +1,8 @@ +{:paths ["src" "resources"] + :deps {org.clojure/clojure {:mvn/version "1.7.0"} + org.clojure/tools.namespace {:mvn/version "0.2.11"} + org.clojure/clojurescript {:mvn/version "1.7.189"} + hiccup {:mvn/version "1.0.5"} + enlive {:mvn/version "1.1.6"} + org.pegdown/pegdown {:mvn/version "1.6.0"} + org.ow2.asm/asm-all {:mvn/version "5.0.3"}}} From 3079986bd9500c62775f795ec92b30a7aeb742c9 Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Wed, 15 Aug 2018 21:42:20 +0200 Subject: [PATCH 02/10] trust cljs analyzer to read macros correctly --- codox/src/codox/main.clj | 23 ++--------------------- codox/src/codox/reader/clojurescript.clj | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/codox/src/codox/main.clj b/codox/src/codox/main.clj index 1a11ab3..ed9ae6b 100644 --- a/codox/src/codox/main.clj +++ b/codox/src/codox/main.clj @@ -4,6 +4,7 @@ (:require [clojure.string :as str] [clojure.java.shell :as shell] [codox.reader.clojure :as clj] + [codox.reader.clojurescript :as cljs] [codox.reader.plaintext :as text])) (defn- writer [{:keys [writer]}] @@ -19,29 +20,9 @@ (throw (Exception. (str "Could not resolve codox writer " writer-sym)))))) -(defn- macro? [var] - (= (:type var) :macro)) - -(defn- read-macro-namespaces [paths read-opts] - (->> (clj/read-namespaces paths read-opts) - (map (fn [ns] (update-in ns [:publics] #(filter macro? %)))) - (remove (comp empty? :publics)))) - -(defn- merge-namespaces [namespaces] - (for [[name namespaces] (group-by :name namespaces)] - (assoc (first namespaces) :publics (mapcat :publics namespaces)))) - -(defn- cljs-read-namespaces [paths read-opts] - ;; require is here to allow Clojure 1.3 and 1.4 when not using ClojureScript - (require 'codox.reader.clojurescript) - (let [reader (find-var 'codox.reader.clojurescript/read-namespaces)] - (merge-namespaces - (concat (reader paths read-opts) - (read-macro-namespaces paths read-opts))))) - (def ^:private namespace-readers {:clojure clj/read-namespaces - :clojurescript cljs-read-namespaces}) + :clojurescript cljs/read-namespaces}) (defn- var-symbol [namespace var] (symbol (name (:name namespace)) (name (:name var)))) diff --git a/codox/src/codox/reader/clojurescript.clj b/codox/src/codox/reader/clojurescript.clj index dee1892..254d8e7 100644 --- a/codox/src/codox/reader/clojurescript.clj +++ b/codox/src/codox/reader/clojurescript.clj @@ -50,15 +50,16 @@ :else :var)) (defn- read-var [file vars var] - (-> var - (select-keys [:name :line :arglists :doc :dynamic :added :deprecated :doc/format]) - (update-some :name (comp symbol name)) - (update-some :arglists remove-quote) - (update-some :doc correct-indent) - (assoc-some :file (.getPath file) - :type (var-type var) - :members (map (partial read-var file vars) - (protocol-methods var vars))))) + (let [vt (var-type var)] + (-> var + (select-keys [:name :line :arglists :doc :dynamic :added :deprecated :doc/format]) + (update-some :name (comp symbol name)) + (update-some :arglists remove-quote) + (update-some :doc correct-indent) + (assoc-some :file (if (= vt :macro) (:file var) (.getPath file)) + :type vt + :members (map (partial read-var file vars) + (protocol-methods var vars)))))) (defn- read-publics [state namespace file] (let [vars (vals (ana/ns-publics state namespace))] From 25b01ae5c471bd4d1ef4290d7724d6878157f31d Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Tue, 28 Aug 2018 18:20:32 +0200 Subject: [PATCH 03/10] check for seq? instead of list? when removing 'quote from arglists --- codox/src/codox/reader/clojurescript.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codox/src/codox/reader/clojurescript.clj b/codox/src/codox/reader/clojurescript.clj index 254d8e7..4599070 100644 --- a/codox/src/codox/reader/clojurescript.clj +++ b/codox/src/codox/reader/clojurescript.clj @@ -15,7 +15,7 @@ (-> file .getName cljs-filename?))) (defn- remove-quote [x] - (if (and (list? x) (= (first x) 'quote)) + (if (and (seq? x) (= (first x) 'quote)) (second x) x)) From 3f2ea84ae5e27e6f527b38bbc8d8fcecc24fe47f Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Thu, 18 Oct 2018 11:18:49 +0200 Subject: [PATCH 04/10] dont find namespaces twice https://github.com/cljdoc/cljdoc/issues/155 --- codox/src/codox/reader/clojure.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codox/src/codox/reader/clojure.clj b/codox/src/codox/reader/clojure.clj index f37346c..119f2e8 100644 --- a/codox/src/codox/reader/clojure.clj +++ b/codox/src/codox/reader/clojure.clj @@ -116,8 +116,8 @@ (defn- find-namespaces [file] (cond - (.isDirectory file) (ns/find-namespaces-in-dir file) - (jar-file? file) (ns/find-namespaces-in-jarfile (JarFile. file)))) + (.isDirectory file) (set (ns/find-namespaces-in-dir file)) + (jar-file? file) (set (ns/find-namespaces-in-jarfile (JarFile. file))))) (defn read-namespaces "Read Clojure namespaces from a set of source directories (defaults From b1b380bb67e57795c5b0ff43fdfdc6edcefd0934 Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Fri, 9 Nov 2018 17:36:58 +0100 Subject: [PATCH 05/10] delete all writing related dependencies --- codox/deps.edn | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/codox/deps.edn b/codox/deps.edn index 400b197..7bc1466 100644 --- a/codox/deps.edn +++ b/codox/deps.edn @@ -1,8 +1,4 @@ -{:paths ["src" "resources"] - :deps {org.clojure/clojure {:mvn/version "1.7.0"} +{:paths ["src" "resources" "test-sources"] + :deps {org.clojure/clojure {:mvn/version "1.9.0"} org.clojure/tools.namespace {:mvn/version "0.2.11"} - org.clojure/clojurescript {:mvn/version "1.7.189"} - hiccup {:mvn/version "1.0.5"} - enlive {:mvn/version "1.1.6"} - org.pegdown/pegdown {:mvn/version "1.6.0"} - org.ow2.asm/asm-all {:mvn/version "5.0.3"}}} + org.clojure/clojurescript {:mvn/version "1.10.339"}}} From c3a860789197e2c6832f48e2fccce823b4ac8dbb Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Fri, 9 Nov 2018 17:45:04 +0100 Subject: [PATCH 06/10] add some simple -main for testing purposes --- codox/src/codox/main.clj | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/codox/src/codox/main.clj b/codox/src/codox/main.clj index ed9ae6b..9ae226d 100644 --- a/codox/src/codox/main.clj +++ b/codox/src/codox/main.clj @@ -2,6 +2,7 @@ "Main namespace for generating documentation" (:use [codox.utils :only (add-source-paths)]) (:require [clojure.string :as str] + [clojure.pprint] [clojure.java.shell :as shell] [codox.reader.clojure :as clj] [codox.reader.clojurescript :as cljs] @@ -106,3 +107,17 @@ (write-fn (assoc options :namespaces namespaces :documents documents))))) + +(defn -main + [lang path] + (println "Analyzing lang:" lang) + (println "Analyzing path:" path) + (assert (#{"clojure" "clojurescript"} lang)) + (->> (generate-docs {:writer 'clojure.core/identity + :source-paths [path] + :language (keyword lang)}) + :namespaces + ;; Walk/realize entire structure, otherwise "Excluding ->Xyz" + ;; messages will be mixed with the pretty printed output + (clojure.walk/prewalk identity) + clojure.pprint/pprint)) From 214e119bc7a328e48bfef4757d4bf352805f7c9a Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Fri, 9 Nov 2018 17:48:12 +0100 Subject: [PATCH 07/10] add fork notes and 1.10.439 repro notes to readme --- README.md | 9 +++++++++ codox/test-sources/codox_test/macro.cljc | 18 ++++++++++++++++++ codox/test-sources/codox_test/multimethod.cljc | 9 +++++++++ codox/test-sources/codox_test/record.cljc | 3 +++ codox/test-sources/codox_test/spec.cljc | 4 ++++ codox/test-sources/codox_test/spec_two.cljs | 4 ++++ 6 files changed, 47 insertions(+) create mode 100644 codox/test-sources/codox_test/macro.cljc create mode 100644 codox/test-sources/codox_test/multimethod.cljc create mode 100644 codox/test-sources/codox_test/record.cljc create mode 100644 codox/test-sources/codox_test/spec.cljc create mode 100644 codox/test-sources/codox_test/spec_two.cljs diff --git a/README.md b/README.md index 6a98b63..4a988d6 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ +### Notes about this fork + +- Focused on extracting structured data (EDN) from jars. +- Any writing-related (HTML etc.) dependencies have been removed, code is kept to keep diff to mainline manageable. +- Various tweaks have been done to align the results of Clojure and ClojureScript analysis. + + +--- + # Codox A tool for generating API documentation from Clojure or ClojureScript diff --git a/codox/test-sources/codox_test/macro.cljc b/codox/test-sources/codox_test/macro.cljc new file mode 100644 index 0000000..0023108 --- /dev/null +++ b/codox/test-sources/codox_test/macro.cljc @@ -0,0 +1,18 @@ +(ns codox-test.macro) + +(defmacro test + [a b] + `(+ ~a ~b)) + +(defmacro test2 + [a & xs] + `(reduce + ~a ~(vec xs))) + +;; https://github.com/jarohen/nomad/blob/5520c332c5c6d5eef4dcb0930c735900d63dea2a/src/nomad/config.clj#L98-L111 + +(defn with-config-override* [{:keys [switches secret-keys override-switches] :as opts-override} f] + nil) + +(doto (defmacro with-config-override [opts & body] + `(with-config-override* ~opts (fn [] ~@body))) + (alter-meta! assoc :arglists '([{:keys [switches secret-keys override-switches] :as opts-override} & body]))) diff --git a/codox/test-sources/codox_test/multimethod.cljc b/codox/test-sources/codox_test/multimethod.cljc new file mode 100644 index 0000000..00b9d53 --- /dev/null +++ b/codox/test-sources/codox_test/multimethod.cljc @@ -0,0 +1,9 @@ +(ns codox-test.multimethod) + +(defmulti start (fn [k opts] k)) + +(defmethod start :car [_ opts] + (println "Starting car..." opts)) + +(defmethod start :helicopter [_ opts] + (println "Starting helicopter..." opts)) diff --git a/codox/test-sources/codox_test/record.cljc b/codox/test-sources/codox_test/record.cljc new file mode 100644 index 0000000..480302d --- /dev/null +++ b/codox/test-sources/codox_test/record.cljc @@ -0,0 +1,3 @@ +(ns codox-test.record) + +(defrecord CodoxSystem [components]) diff --git a/codox/test-sources/codox_test/spec.cljc b/codox/test-sources/codox_test/spec.cljc new file mode 100644 index 0000000..c228c6e --- /dev/null +++ b/codox/test-sources/codox_test/spec.cljc @@ -0,0 +1,4 @@ +(ns codox-test.spec + (:require [clojure.spec.test.alpha :as st])) + +(st/with-instrument-disabled (+ 1 2 3)) diff --git a/codox/test-sources/codox_test/spec_two.cljs b/codox/test-sources/codox_test/spec_two.cljs new file mode 100644 index 0000000..a985d42 --- /dev/null +++ b/codox/test-sources/codox_test/spec_two.cljs @@ -0,0 +1,4 @@ +(ns codox-test.spec-two + (:require [clojure.spec.test.alpha :as st])) + +(st/with-instrument-disabled (+ 1 2 3)) From 9a018ae30a001649ef7b2a2e870f59dc5c52a88c Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Fri, 9 Nov 2018 19:01:42 +0100 Subject: [PATCH 08/10] set *analyze-deps* to true so macros are loaded --- codox/src/codox/reader/clojurescript.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codox/src/codox/reader/clojurescript.clj b/codox/src/codox/reader/clojurescript.clj index 4599070..aa44b11 100644 --- a/codox/src/codox/reader/clojurescript.clj +++ b/codox/src/codox/reader/clojurescript.clj @@ -72,7 +72,7 @@ (defn- analyze-file [file] (let [state (ana/empty-state)] - (binding [an/*analyze-deps* false] + (binding [an/*analyze-deps* true] (ana/no-warn (ana/analyze-file state file {}))) state)) From e0cd26910704c416611fc81f43f890a26861c221 Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Mon, 12 Nov 2018 16:11:41 +0100 Subject: [PATCH 09/10] fix analysis issues with cljs 1.10.439 More details in this Gist: https://gist.github.com/martinklepsch/9f885feb061ec3f03f365e22d0d9bf5b Fixes https://github.com/cljdoc/cljdoc/issues/201 --- codox/src/codox/reader/clojurescript.clj | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/codox/src/codox/reader/clojurescript.clj b/codox/src/codox/reader/clojurescript.clj index aa44b11..1db1296 100644 --- a/codox/src/codox/reader/clojurescript.clj +++ b/codox/src/codox/reader/clojurescript.clj @@ -4,6 +4,8 @@ (:require [clojure.java.io :as io] [cljs.analyzer :as an] [cljs.analyzer.api :as ana] + [cljs.closure] + [cljs.env] [clojure.string :as str])) (defn- cljs-filename? [filename] @@ -71,10 +73,11 @@ (sort-by (comp str/lower-case :name))))) (defn- analyze-file [file] - (let [state (ana/empty-state)] - (binding [an/*analyze-deps* true] - (ana/no-warn - (ana/analyze-file state file {}))) + (let [opts (cljs.closure/add-implicit-options {}) + state (cljs.env/default-compiler-env opts)] + (ana/no-warn + (cljs.closure/validate-opts opts) + (ana/analyze-file state file opts)) state)) (defn- read-file [path file exception-handler] From b29b16ae686e3400689bbdc93df57200b709443d Mon Sep 17 00:00:00 2001 From: Martin Klepsch Date: Fri, 1 Mar 2019 12:21:14 +0100 Subject: [PATCH 10/10] add some docs to -main --- codox/src/codox/main.clj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/codox/src/codox/main.clj b/codox/src/codox/main.clj index 9ae226d..777da7f 100644 --- a/codox/src/codox/main.clj +++ b/codox/src/codox/main.clj @@ -109,6 +109,18 @@ :documents documents))))) (defn -main + "The main entry point for reading API information from files in a directory. + + To analyze a project (debugging etc.) follow these steps: + + 1. unzip the project's jar into a directory + 2. add the project's coordinates to the local `deps.edn` file + 3. add the jar contents directory to `:paths` in `deps.edn` + + You can then call this main function as follows: + + clj -m codox.main clojurescript jar-contents-dir/ + clj -m codox.main clojure jar-contents-dir/" [lang path] (println "Analyzing lang:" lang) (println "Analyzing path:" path)